17_FRONT_Process_training_tears_using_cubes_match.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-20T05:59:13.548655Z",
     "start_time": "2019-10-20T05:59:13.538480Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# RUN Main import block and TODO list\n",
    "\n",
    "# TODO: see how uri calculated the ridges\n",
    "\n",
    "# TODO: Perform Histogram equalization - start with it\n",
    "# TODO: \n",
    "# take integral from the Highest peak+-0.005 divide by integral of the entire graph \n",
    "# This will be the peakness measure for the PSD ==> The desired ridge index\n",
    "# TODO:\n",
    "# take integral from the Highest peak+-0.005 divide by integral of the entire graph - it's the peakness measure for the PSD\n",
    "# must select a peak above a min threshold in order to ignore noisy frequency\n",
    "# must ignore peaks above a certain threshold in order to detect meaningful frequency\n",
    "# run the PSD in moving windows every 200 px (deduced from the below PSD pointing to a freq of 1/0.02=50-> times 4= 200px)\n",
    "# and medianf the result of the windows\n",
    "# TODO:\n",
    "# Another alternative: (with Yariv)\n",
    "# Run PSD column by column - get the phase, freq, peakness and reconstruct an artificial ridge slice\n",
    "# from this - reconstruct a \"clean\" artificial ridge image\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.image as img\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import urllib.request\n",
    "import os\n",
    "import shutil\n",
    "import glob\n",
    "from scipy import ndimage\n",
    "from scipy import signal\n",
    "#import cv2\n",
    "\n",
    "from PIL import Image, ImageDraw, ImageFont\n",
    "\n",
    "import mahotas as mh\n",
    "from mahotas import polygon\n",
    "# import pymorph as pm\n",
    "\n",
    "import networkx as nx\n",
    "\n",
    "from scipy import ndimage as nd\n",
    "import skimage.transform as transform\n",
    "import skimage.morphology as mp\n",
    "import skimage.io as sio\n",
    "import scipy.misc as sm\n",
    "from skimage.filters import threshold_otsu, threshold_adaptive\n",
    "from skimage.feature import hessian_matrix, hessian_matrix_eigvals\n",
    "from skimage import exposure\n",
    "from skimage import data, img_as_float\n",
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "\n",
    "from bisect import bisect_left\n",
    "import math\n",
    "import warnings\n",
    "import csv\n",
    "import tensorflow as tf\n",
    "from time import gmtime, strftime\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-20T05:59:13.704053Z",
     "start_time": "2019-10-20T05:59:13.616687Z"
    },
    "code_folding": [
     0,
     28,
     31,
     49,
     57,
     71,
     77,
     83,
     98,
     107,
     122,
     140,
     157,
     175,
     276,
     286,
     296,
     306,
     318,
     323,
     328,
     335,
     357,
     364,
     373,
     410
    ]
   },
   "outputs": [],
   "source": [
    "# RUN Utility functions\n",
    "\n",
    "# One time init\n",
    "# with open('results.csv', 'w') as csvfile:\n",
    "#     csvout = csv.writer(csvfile)\n",
    "#     csvout.writerow([\"File\", \"Model\", \"Gap\", \"Slice_size\", \"Count\", \"Precision\", \"Recall\", \"F-score\", \"True Count\", \"Error Rate\"])\n",
    "\n",
    "#BASIC CROP FRAME\n",
    "X_START = 1000\n",
    "X_END = 6000\n",
    "Y_START = 800\n",
    "Y_END = 4300\n",
    "BG_2_OBJ_RATIO = 0.91\n",
    "CUBE_SIZE = 250\n",
    "EDGE_GAP = 50\n",
    "ROOT_FOLDER = \"/media/1KGB_ILAN/papyrus/\"\n",
    "fragments_folder = \"files/fragments_train_front/\"\n",
    "# ROOT_FOLDER = \"/Users/il239838/Downloads/private/Thesis/Papyrus/PX303/files/\"\n",
    "LEARNING_RATE = 0.001\n",
    "BATCHES = 2000\n",
    "BATCH_SIZE = 60\n",
    "BREAK_VAL = 1000\n",
    "\n",
    "# Simple crop by x/y ranges\n",
    "def crop(image, ymin, ymax, xmin, xmax):\n",
    "    return image[ymin:ymax, xmin:xmax]\n",
    "\n",
    "# returns a logical matrix of values beyond a threshld\n",
    "def thresholded(image, val): \n",
    "    return np.logical_and(*[image[...] > val  for t in enumerate([0, 0])])\n",
    "\n",
    "def find_min_max_without_orphand_pixels(nonzero_dimension, crop_filter=20):\n",
    "    sorted = np.sort(nonzero_dimension)\n",
    "    prev=-1\n",
    "    min_val = sorted[0]\n",
    "    for i, x in enumerate(sorted[:100]):\n",
    "        if prev >= 0 and x - prev > crop_filter:\n",
    "            min_val = x\n",
    "        prev = x\n",
    "    prev=-1\n",
    "    max_val = sorted[-1]\n",
    "    for i, x in enumerate(sorted[-100:]):\n",
    "        if prev >= 0 and x - prev > crop_filter:\n",
    "            max_val = prev\n",
    "            break\n",
    "        prev = x\n",
    "    \n",
    "    return min_val, max_val\n",
    "\n",
    "def calc_min_max_coordinates(image, crop_val=50):\n",
    "    temp = thresholded(image, crop_val)\n",
    "    temp = temp * 1\n",
    "    temp = np.nonzero(temp)\n",
    "    ymin, ymax = find_min_max_without_orphand_pixels(temp[0])\n",
    "    xmin,xmax = find_min_max_without_orphand_pixels(temp[1])\n",
    "    return ymin, ymax, xmin, xmax\n",
    "\n",
    "def calc_min_max_coordinates_dynamic(image, cutoff=1):\n",
    "    temp = exposure.equalize_adapthist(image, clip_limit=0.03)\n",
    "    flat = np.sort(np.matrix.getA1(temp))\n",
    "    sum_all = np.sum(flat)\n",
    "    index = np.argmin(flat.cumsum() < (sum_all * cutoff))\n",
    "\n",
    "    temp = thresholded(temp, flat[index])\n",
    "    temp = temp * 1\n",
    "    temp = np.nonzero(temp)\n",
    "    ymin, ymax = find_min_max_without_orphand_pixels(temp[0])\n",
    "    xmin,xmax = find_min_max_without_orphand_pixels(temp[1])\n",
    "    return ymin, ymax, xmin, xmax\n",
    "\n",
    "# initial static crop and a seondary dynamic crop based on signal2noise ratio\n",
    "def crop_full_scan(image, x_start, x_end, y_start, y_end):\n",
    "    temp = crop(image, y_start, y_end, x_start, x_end)\n",
    "    ymin, ymax, xmin, xmax = calc_min_max_coordinates_dynamic(temp, cutoff=BG_2_OBJ_RATIO)\n",
    "    temp = crop(image, y_start+ymin, y_start+ymax, x_start+xmin, x_start+xmax)\n",
    "    return temp\n",
    "\n",
    "def crop_thresholded(image):\n",
    "    temp = crop(image, 0, image.shape[0]-1, 0, image.shape[1]-1)\n",
    "    ymin, ymax, xmin, xmax = calc_min_max_coordinates(temp)\n",
    "    temp = crop(image, ymin, ymax, xmin, xmax)\n",
    "    return temp\n",
    "\n",
    "def read_and_crop(image_name, x_start=X_START, x_end=X_END, y_start=Y_START, y_end=Y_END):\n",
    "    #if \"il239838\" in os.getcwd():\n",
    "    #    image = img.imread(ROOT_FOLDER + image_name)\n",
    "    #else:\n",
    "    #    f = urllib.request.urlopen(\"https://dl.dropboxusercontent.com/s/31b96942qdcn73k/\" + image_name)\n",
    "    #    image = img.imread(f, format='jpeg')\n",
    "    image = img.imread(ROOT_FOLDER + image_name)\n",
    "    \n",
    "    # Smart-crop the image to get rid of all the noise and redundant area\n",
    "    # return crop_full_scan(image)\n",
    "    cropped = crop_full_scan(image, x_start, x_end, y_start, y_end)\n",
    "    return exposure.equalize_adapthist(cropped, clip_limit=0.03)\n",
    "\n",
    "\n",
    "# TODO: fix performance!!! http://scikit-image.org/docs/dev/user_guide/tutorial_parallelization.html\n",
    "def combine_3_images_to_RGB(red, green, blue):\n",
    "    new_image = np.empty((blue.shape[0],blue.shape[1],3))\n",
    "    for x in range(0, blue.shape[0]):\n",
    "        for y in range(0, blue.shape[1]):\n",
    "            new_image[x,y,0] = red[x,y]\n",
    "            new_image[x,y,1] = green[x,y]\n",
    "            new_image[x,y,2] = blue[x,y]\n",
    "    return new_image\n",
    "\n",
    "def slice_image_left_edge(original, width=200, rotate=0):\n",
    "    rot = ndimage.rotate(original, rotate)\n",
    "    # Slice the left slice of the so-called \"blue\" image\n",
    "    left_edge_orig = crop(rot, 1, 1400, 1, width)\n",
    "    left_edge_orig = crop_thresholded(left_edge_orig)\n",
    "\n",
    "    # Copy to a new array so we don't thrash the origin\n",
    "    left_edge = np.empty_like (left_edge_orig)\n",
    "    np.copyto(left_edge, left_edge_orig)\n",
    "\n",
    "    # Zero down low level \"noise\" values\n",
    "    low_values_indices = left_edge < 30  # Where values are low\n",
    "    left_edge[low_values_indices] = 0  # All low values set to 0\n",
    "    return left_edge\n",
    "\n",
    "def get_best_angle_rotation(original, crop=True, width=200):\n",
    "    min_var = 99999999999\n",
    "    best_angle = -10\n",
    "    for x in range(-5,5):\n",
    "        if crop:            \n",
    "            rot_edge = slice_image_left_edge(original, width, x)\n",
    "        else:\n",
    "            rot_edge = ndimage.rotate(original, x)\n",
    "        left_var = np.var(rot_edge, axis=1)\n",
    "        # left_var = np.apply_along_axis(lambda v: np.var(v[np.nonzero(v)]), 1, rot_edge)\n",
    "        var_sum = np.sum(left_var)\n",
    "        if (var_sum < min_var):\n",
    "            min_var = var_sum\n",
    "            best_angle = x\n",
    "    print (\"best_angle=\"+str(best_angle))\n",
    "    return best_angle\n",
    "\n",
    "#     import pdb; pdb.set_trace()\n",
    "def calc_neighbors(slice_map, col, row):\n",
    "    # import pdb; pdb.set_trace()\n",
    "    if ((col-1, row) in slice_map and slice_map[(col-1, row)] != None):\n",
    "        slice_map[(col, row)][\"left\"] = slice_map[(col-1, row)]\n",
    "        slice_map[(col-1, row)][\"right\"] = slice_map[(col, row)]\n",
    "    if ((col+1, row) in slice_map and slice_map[(col+1, row)] != None):\n",
    "        slice_map[(col, row)][\"right\"] = slice_map[(col+1, row)]\n",
    "        slice_map[(col+1, row)][\"left\"] = slice_map[(col, row)]\n",
    "    if ((col, row-1) in slice_map and slice_map[(col, row-1)] != None):\n",
    "        slice_map[(col, row)][\"top\"] = slice_map[(col, row-1)]\n",
    "        slice_map[(col, row-1)][\"bottom\"] = slice_map[(col, row)]\n",
    "    if ((col, row+1) in slice_map and slice_map[(col, row+1)] != None):\n",
    "        slice_map[(col, row)][\"bottom\"] = slice_map[(col, row+1)]\n",
    "        slice_map[(col, row+1)][\"top\"] = slice_map[(col, row)]\n",
    "    \n",
    "\n",
    "\n",
    "def VAL_create_cube(name, raw, x, y):\n",
    "    cube = {}\n",
    "    cube[\"cube\"] = raw\n",
    "    cube[\"file\"] = name\n",
    "    #     if name.find('P') == 0:\n",
    "    #         cube[\"index\"] = int(name[name.find('P')+1:name.find('P')+4]) * 1000 + int(name[name.find('Fg')+2:name.find('Fg')+5])\n",
    "    #     else:\n",
    "    # print(\"Found a ZERO index cube with the name:\"+name)\n",
    "    cube[\"index\"] = 0\n",
    "    cube[\"top_row\"] = x\n",
    "    cube[\"left_col\"] = y\n",
    "    cube[\"right_col\"] = y + CUBE_SIZE\n",
    "    return cube\n",
    "    \n",
    "\n",
    "ZERO_CUBE = VAL_create_cube(\"ZERO\", np.zeros((CUBE_SIZE, CUBE_SIZE), dtype=np.int), -1, -2)\n",
    "\n",
    "# slice an image to cubes with 250X250 pixel size\n",
    "def VAL_slice_TEAR_to_static_slices(name, cropped_original):\n",
    "    structure = {}\n",
    "    # cropped_original = cropped_original / 256 # divide by 256 to \"normalize\" between 0 and 1\n",
    "\n",
    "    x, y = cropped_original[\"cut\"].shape\n",
    "    # print (x,y)\n",
    "    n = 0\n",
    "    # see n offset to see offset in pixels on the x axis == rows. every n equals CUBE_SIZE\n",
    "    while ((n + 1) * CUBE_SIZE < x):\n",
    "        # mark the piece as narrow so the first would be counted as lastt too\n",
    "        narrow = True if ((CUBE_SIZE + (4 * EDGE_GAP)) > y) else False\n",
    "        # cut a cube of 250X250 at the FIRST column\n",
    "        start_row_px = int(np.round(n * CUBE_SIZE, -1))\n",
    "        end_row_px = int(np.round((n + 1) * CUBE_SIZE, -1))\n",
    "\n",
    "        # crop a cube on the starting edge of the current row. \n",
    "        # start with EDGE_GAP and move inwards untill we have value\n",
    "        fg_value = 0\n",
    "        iteration = 1\n",
    "        while fg_value <= 0.2 and iteration <= 5:\n",
    "            cube = (crop(cropped_original[\"cut\"], start_row_px, end_row_px, \\\n",
    "                         EDGE_GAP * iteration, CUBE_SIZE + (EDGE_GAP * iteration)))\n",
    "            iteration += 1\n",
    "            fg_value = np.median(cube)\n",
    "            \n",
    "        # keep only cubes for which half of the pixels have some \"color\"\n",
    "        if np.median(cube) > 0.2: # aligned with the normalization 0.2 correlates to 50\n",
    "            # keep the cube\n",
    "            new_cube = VAL_create_cube(name, cube, start_row_px, EDGE_GAP)\n",
    "            new_cube[\"col\"] = 0 # marks that the cube is on the first col of the piece\n",
    "            new_cube[\"row\"] = n\n",
    "            new_cube[\"last\"] = narrow # marks that the cube is on the last col of the piece\n",
    "            new_cube[\"orig\"] = cropped_original\n",
    "            new_cube[\"col_px_left\"] = cropped_original[\"col_px\"] + EDGE_GAP\n",
    "            new_cube[\"col_px_right\"] = cropped_original[\"col_px\"] + CUBE_SIZE + EDGE_GAP\n",
    "            new_cube[\"row_px_top\"] = cropped_original[\"row_px\"] + start_row_px\n",
    "            new_cube[\"row_px_bottom\"] = cropped_original[\"row_px\"] + end_row_px\n",
    "            structure[(0, n)] = new_cube\n",
    "\n",
    "        # crop a cube on the starting edge of the current row. \n",
    "        # start with EDGE_GAP and move inwards untill we have value\n",
    "        fg_value = 0\n",
    "        iteration = 1\n",
    "        while fg_value <= 0.2 and iteration <= 5:\n",
    "            cube = (crop(cropped_original[\"cut\"], start_row_px, end_row_px, \\\n",
    "                         y - CUBE_SIZE - (EDGE_GAP * iteration), y - (EDGE_GAP * iteration)))\n",
    "            iteration += 1\n",
    "            fg_value = np.median(cube)\n",
    "            \n",
    "        # keep only cubes for which half of the pixels have some \"color\"\n",
    "        # aligned with the normalization 0.2 correlates to 50\n",
    "        if np.median(cube) > 0.2:\n",
    "            # keep the cube\n",
    "            new_cube = VAL_create_cube(name, cube, start_row_px, y - CUBE_SIZE - EDGE_GAP)\n",
    "            new_cube[\"col\"] = 1 # marks that the cube is on the last col of the piece\n",
    "            new_cube[\"row\"] = n\n",
    "            new_cube[\"last\"] = not narrow # like col - marks that the cube is on the last col of the piece\n",
    "            new_cube[\"orig\"] = cropped_original\n",
    "            new_cube[\"col_px_left\"] = cropped_original[\"col_px\"] + y - CUBE_SIZE - EDGE_GAP\n",
    "            new_cube[\"col_px_right\"] = cropped_original[\"col_px\"] + y - EDGE_GAP\n",
    "            new_cube[\"row_px_top\"] = cropped_original[\"row_px\"] + start_row_px\n",
    "            new_cube[\"row_px_bottom\"] = cropped_original[\"row_px\"] + end_row_px\n",
    "            structure[(1, n)] = new_cube\n",
    "\n",
    "    #         m = 0\n",
    "    #         # every 250 pixels on the y axis == cols\n",
    "    #         while ((m + 1) * CUBE_SIZE < y):            \n",
    "    #             if ((m == 0) or ((m + 2) * CUBE_SIZE >= y)): # Only keep the left and right edges of the piece for matching!!\n",
    "    #                 # cut a cube of 250X250\n",
    "    #                 cube = crop(cropped_original[\"cut\"], n * CUBE_SIZE, (n + 1) * CUBE_SIZE, m * CUBE_SIZE, (m + 1) * CUBE_SIZE)\n",
    "    #                 # keep only cubes for which half of the pixels have some \"color\"\n",
    "    #                 # print(np.median(cube))\n",
    "    #                 if np.median(cube) > 0.2: # aligned with the normalization 0.2 correlates to 50\n",
    "    #                     # keep the cube\n",
    "    #                     new_cube = VAL_create_cube(name, cube, n * CUBE_SIZE, m * CUBE_SIZE)\n",
    "    #                     new_cube[\"col\"] = m\n",
    "    #                     new_cube[\"row\"] = n\n",
    "    #                     new_cube[\"orig\"] = cropped_original\n",
    "    #                     new_cube[\"col_px_left\"] = cropped_original[\"col_px\"] + m * CUBE_SIZE\n",
    "    #                     new_cube[\"col_px_right\"] = cropped_original[\"col_px\"] + (m + 1) * CUBE_SIZE\n",
    "    #                     new_cube[\"row_px_top\"] = cropped_original[\"row_px\"] + n * CUBE_SIZE\n",
    "    #                     new_cube[\"row_px_bottom\"] = cropped_original[\"row_px\"] + (n + 1) * CUBE_SIZE\n",
    "    #                     if ((m + 2) * CUBE_SIZE >= y):\n",
    "    #                         new_cube[\"last\"] = True\n",
    "    #                     else:\n",
    "    #                         new_cube[\"last\"] = False\n",
    "    #                     structure[(m, n)] = new_cube\n",
    "    #             m += 1\n",
    "        n += 0.2 # currently set to jump in 50 px offset\n",
    "        n = np.round(n, 1)\n",
    "        \n",
    "    # this loop has to be performed only after we've established all the None cubes\n",
    "    # import pdb; pdb.set_trace()\n",
    "    for cube in structure.values():\n",
    "        # set the reference to neighbor cubes\n",
    "        if cube != None:\n",
    "            calc_neighbors(structure, cube[\"col\"], cube[\"row\"])\n",
    "            \n",
    "    # return the data structure with all the cubes and the counters of the rows and columns\n",
    "    return structure.values()\n",
    "\n",
    "def pad_above(original, above, amount):\n",
    "    res = np.insert(original[\"cube\"], np.zeros(amount), above[\"cube\"][-amount:], axis=0)\n",
    "    res = np.delete(res, np.arange(CUBE_SIZE,CUBE_SIZE+amount), axis=0)\n",
    "    cube = VAL_create_cube(original[\"file\"], res, original[\"top_row\"] - amount, original[\"left_col\"])\n",
    "    cube[\"col_px_left\"] = original[\"col_px_left\"]\n",
    "    cube[\"col_px_right\"] = original[\"col_px_right\"]\n",
    "    cube[\"row_px_top\"] = original[\"row_px_top\"] - amount\n",
    "    cube[\"row_px_bottom\"] = original[\"row_px_bottom\"] - amount\n",
    "    return cube\n",
    "\n",
    "def pad_below(original, below, amount):\n",
    "    res = np.insert(original[\"cube\"], np.full(amount, CUBE_SIZE), below[\"cube\"][:amount], axis=0)\n",
    "    res = np.delete(res, np.arange(0, amount), axis=0)\n",
    "    cube =  VAL_create_cube(original[\"file\"], res, original[\"top_row\"] + amount, original[\"left_col\"])\n",
    "    cube[\"col_px_left\"] = original[\"col_px_left\"]\n",
    "    cube[\"col_px_right\"] = original[\"col_px_right\"]\n",
    "    cube[\"row_px_top\"] = original[\"row_px_top\"] + amount\n",
    "    cube[\"row_px_bottom\"] = original[\"row_px_bottom\"] + amount\n",
    "    return cube\n",
    "    \n",
    "def pad_left(original, left, amount):\n",
    "    res = np.insert(original[\"cube\"], np.zeros(amount, dtype=int), left[\"cube\"][:,-amount:], axis=1)\n",
    "    res = np.delete(res, np.arange(CUBE_SIZE, CUBE_SIZE+amount), axis=1)\n",
    "    cube = VAL_create_cube(original[\"file\"], res, original[\"top_row\"], original[\"left_col\"] - amount)\n",
    "    cube[\"col_px_left\"] = original[\"col_px_left\"] - amount\n",
    "    cube[\"col_px_right\"] = original[\"col_px_right\"] - amount\n",
    "    cube[\"row_px_top\"] = original[\"row_px_top\"]\n",
    "    cube[\"row_px_bottom\"] = original[\"row_px_bottom\"]\n",
    "    return cube\n",
    "\n",
    "def pad_right(original, right, amount):\n",
    "    res = np.insert(original[\"cube\"], [CUBE_SIZE], right[\"cube\"][:,:amount], axis=1)\n",
    "    res = np.delete(res, np.arange(0, amount), axis=1)\n",
    "    cube = VAL_create_cube(original[\"file\"], res, original[\"top_row\"], original[\"left_col\"] + amount)\n",
    "    cube[\"col_px_left\"] = original[\"col_px_left\"] + amount\n",
    "    cube[\"col_px_right\"] = original[\"col_px_right\"] + amount\n",
    "    cube[\"row_px_top\"] = original[\"row_px_top\"]\n",
    "    cube[\"row_px_bottom\"] = original[\"row_px_bottom\"]\n",
    "    return cube\n",
    "    \n",
    "\n",
    "# \"Shave\" the right edge of the cube with <gap> pixels and pad with zeros on the left\n",
    "def shave_right(original, amount):\n",
    "    return pad_left(original, ZERO_CUBE, amount)\n",
    "    \n",
    "\n",
    "# \"Shave\" the left edge of the cube with <gap> pixels and pad with zeros on the right    \n",
    "def shave_left(original, amount):\n",
    "    return pad_right(original, ZERO_CUBE, amount)\n",
    "    \n",
    "\n",
    "# concatenate cubes \n",
    "def concatenate_cubes(left, right, slice_size):\n",
    "    con = np.concatenate((left[\"cube\"][:,-slice_size:], right[\"cube\"][:,:slice_size]), axis=1)\n",
    "    x_delta = right[\"top_row\"] - left[\"top_row\"]\n",
    "    y_delta = right[\"left_col\"] - left[\"right_col\"] \n",
    "    return con, x_delta, y_delta\n",
    "\n",
    "# concatenate cubes \n",
    "def VAL_concatenate_cubes(left, right, slice_size):\n",
    "    right_img = right[\"cube\"]\n",
    "    # next block is not relevant for training ...\n",
    "    #     # if the left cube is matched to another left cube (or right cube to another right cube) then rotate the right\n",
    "    #     # cube by 180 so we try to match it upside down, covering the option that the cube was pictured rotated\n",
    "    #     if ((left[\"col\"] == 0 and right[\"col\"] == 0) or (left[\"col\"] != 0 and right[\"col\"] != 0)):\n",
    "    #         right_img = np.rot90(right[\"cube\"], 2);\n",
    "\n",
    "    con = np.concatenate((left[\"cube\"][:,-slice_size:], right_img[:,:slice_size]), axis=1)\n",
    "\n",
    "    # next block calculates distance based on the distance between left's right-top corner and right's left-top corner    \n",
    "    #     x_delta = right[\"top_row\"] - left[\"top_row\"]\n",
    "    #     y_delta = right[\"left_col\"] - left[\"right_col\"] \n",
    "\n",
    "    # next block calculates the distance between the centers of cubes, accounting for test set's possibility of reverse slices (left instead of right and vice versa)\n",
    "    x_delta = right[\"row_px_top\"] - left[\"row_px_top\"] # equivalent to distance between vertical centers\n",
    "    y_delta = (right[\"col_px_left\"] + (slice_size / 2)) - (left[\"col_px_right\"] - (slice_size / 2)) # measuring the distance between horizontal centers of the slices\n",
    "\n",
    "    return con, x_delta, y_delta, left[\"file\"], right[\"file\"]\n",
    "    \n",
    "\n",
    "# concatenate cubes while artificially creating a gap between them. Pad the other end of the cube with zeros\n",
    "def concatenate_cubes_zero_pad_gaps(left_orig, right_orig, gap):\n",
    "    left = left_orig if gap == 0 else shave_right(left_orig, gap)\n",
    "    right = right_orig if gap == 0 else shave_left(right_orig, gap)\n",
    "    return concatenate_cubes(left, right)    \n",
    "\n",
    "# concatenate cubes while artificially creating a gap between them. Pad the other end of the cobe with the nearby\n",
    "# continuation of the cubes\n",
    "def concatenate_cubes_with_gap(left_orig, right_orig, gap, left_pad, right_pad, slice_size):\n",
    "    # import pdb; pdb.set_trace()\n",
    "    left = left_orig if gap == 0 else pad_left(left_orig, left_pad, gap)\n",
    "    right = right_orig if gap == 0 else pad_right(right_orig, right_pad, gap)\n",
    "    return concatenate_cubes(left, right, slice_size)        \n",
    "\n",
    "# convert the data structure of cubes into a train set of 2 arrays of images and labels\n",
    "# each image is a concatanation of 2 images from the original cubes set, covering all combinations of images\n",
    "# effectively creating Nx(N-1) images\n",
    "def VAL_build_train_set_for_euclidean_distance(cubes, slice_size, folder):\n",
    "    # clean folder before starting\n",
    "    for root, dirs, files in os.walk(folder):\n",
    "        for f in files:\n",
    "            os.unlink(os.path.join(root, f))\n",
    "\n",
    "    #import pdb; pdb.set_trace()\n",
    "    warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "    train_imgs = []\n",
    "    train_x_delta = []\n",
    "    train_y_delta = []\n",
    "    train_left_obj = []\n",
    "    train_right_obj = []\n",
    "    # iterate over all cubes   \n",
    "    for curr in cubes:\n",
    "        # iterate over the others (effectively n^2)\n",
    "        for adj in cubes:\n",
    "            if (adj[\"file\"] != curr[\"file\"]): # no need to test against self CURRENTLY checking from directions!!!\n",
    "                #import pdb; pdb.set_trace()\n",
    "                # append the adjacent image to the current image\n",
    "                conc, x_delta, y_delta, x_file, y_file = VAL_concatenate_cubes(curr, adj, slice_size)\n",
    "                output = folder+x_file+\"_\"+str(curr[\"top_row\"])+\"_\"+str(curr[\"left_col\"])+\"---\"+y_file+\"_\"+str(adj[\"top_row\"])+\"_\"+str(adj[\"left_col\"])\n",
    "                np.save(output, conc)\n",
    "                train_imgs.append(output)\n",
    "                train_x_delta.append(x_delta)\n",
    "                train_y_delta.append(y_delta)\n",
    "                train_left_obj.append(curr)\n",
    "                train_right_obj.append(adj)\n",
    "\n",
    "    warnings.filterwarnings(\"default\")\n",
    "    return train_imgs, train_x_delta, train_y_delta, train_left_obj, train_right_obj\n",
    "\n",
    "\n",
    "# convert the data structure of cubes into a train set of 2 arrays of images and labels\n",
    "# each image is a concatanation of 2 images from the original cubes set, covering all combinations of images\n",
    "# effectively creating Nx(N-1) images\n",
    "def ORIG_build_train_set(cubes, gap):\n",
    "    # import pdb; pdb.set_trace()\n",
    "    warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "    train_imgs = []\n",
    "    train_lbls = []\n",
    "    train_x_delta = []\n",
    "    train_y_delta = []\n",
    "    # iterate over the rows and cols, essentially going over the grid of sliced cubes\n",
    "    for row in range(0, rows):\n",
    "        for col in range(0, cols):\n",
    "            # if this cube exists (could have been removed previously due to lack of data)\n",
    "            if (cubes[(col, row)] != None):\n",
    "                # for each \"current\" image in the iteration\n",
    "                curr = cubes[(col, row)]\n",
    "                # iterate over all the cubes to find all the \"other\" (adjacent) cubes\n",
    "                for adj_row in range(0, rows):\n",
    "                    for adj_col in range(0, cols):\n",
    "                        if (adj_row != row or adj_col != col):\n",
    "                            if (cubes[(adj_col, adj_row)] != None):\n",
    "                                adj = cubes[(adj_col, adj_row)]\n",
    "                                # append the adjacent image to the current image\n",
    "                                # pass the filling cubes on the right and left to pad against the gap\n",
    "                                if (gap == 0 or (\"left\" in curr.keys() and \"right\" in adj.keys())):\n",
    "                                    if (gap == 0):\n",
    "                                        conc, x_delta, y_delta = concatenate_cubes(curr, adj, slice_size)\n",
    "                                    else:\n",
    "                                        conc, x_delta, y_delta = concatenate_cubes_with_gap(curr, adj, gap, curr[\"left\"], adj[\"right\"], slice_size)\n",
    "                                    train_imgs.append(conc)\n",
    "                                    train_x_delta.append(x_delta)\n",
    "                                    train_y_delta.append(y_delta)\n",
    "                                    # if the adj image is on the same row and on the right of the curr image - it will be marked as match    \n",
    "                                    if (adj_row == row and adj_col == (col + 1)):\n",
    "                                        # mark the image as matched\n",
    "                                        train_lbls.append([0,1])\n",
    "                                        # need to enrich the set with a few more tru positive samples - so we offset \n",
    "                                        # the matched images up ad down a few times and create more matches\n",
    "                                        if (\"top\" in curr.keys() and \"top\"in adj.keys()):\n",
    "                                            for i in range(5, 101, 5):\n",
    "                                                curr1 = pad_above(curr, curr[\"top\"],i)\n",
    "                                                adj1 = pad_above(adj, adj[\"top\"],i)\n",
    "                                                if (gap == 0 or (\"left\" in curr.keys() and \"right\" in adj.keys() and \"top\" in curr[\"left\"].keys() and \"top\"in curr[\"right\"].keys())):\n",
    "                                                    if (gap == 0):\n",
    "                                                        conc, x_delta, y_delta = concatenate_cubes(curr1, adj1, slice_size)\n",
    "                                                    else:\n",
    "                                                        curr1Left = pad_above(curr[\"left\"], curr[\"left\"][\"top\"], i) # FIXIT?\n",
    "                                                        adj1Right = pad_above(adj[\"right\"], curr[\"right\"][\"top\"], i) # FIXIT?\n",
    "                                                        conc, x_delta, y_delta = concatenate_cubes_with_gap(curr1, adj1, gap, curr1Left, adj1Right, slice_size)\n",
    "                                                    train_imgs.append(conc)\n",
    "                                                    train_x_delta.append(x_delta)\n",
    "                                                    train_y_delta.append(y_delta)\n",
    "                                                    # mark the image as matched\n",
    "                                                    train_lbls.append([0,1])\n",
    "                                        if (\"bottom\" in curr.keys() and \"bottom\"in adj.keys()):\n",
    "                                            for i in range(5, 101, 5):\n",
    "                                                curr1 = pad_below(curr, curr[\"bottom\"],i)\n",
    "                                                adj1 = pad_below(adj, adj[\"bottom\"],i)\n",
    "                                                if (gap == 0 or (\"left\" in curr.keys() and \"right\" in adj.keys() and \"bottom\" in curr[\"left\"].keys() and \"bottom\"in curr[\"right\"].keys())):\n",
    "                                                    if (gap == 0):\n",
    "                                                        conc, x_delta, y_delta = concatenate_cubes(curr1, adj1, slice_size)\n",
    "                                                    else:\n",
    "                                                        curr1Left = pad_below(curr[\"left\"], curr[\"left\"][\"bottom\"], i) # FIXIT?\n",
    "                                                        adj1Right = pad_below(adj[\"right\"], curr[\"right\"][\"bottom\"], i) # FIXIT?\n",
    "                                                        conc, x_delta, y_delta = concatenate_cubes_with_gap(curr1, adj1, gap, curr1Left, adj1Right, slice_size)\n",
    "                                                    train_imgs.append(conc)\n",
    "                                                    train_x_delta.append(x_delta)\n",
    "                                                    train_y_delta.append(y_delta)\n",
    "                                                    # mark the image as matched\n",
    "                                                    train_lbls.append([0,1])\n",
    "                                        if (\"left\" in curr.keys()): # enough to check only the curr as the left of the adj is the curr\n",
    "                                            for i in range(5, 101, 5):\n",
    "                                                curr1 = pad_left(curr, curr[\"left\"],i)\n",
    "                                                adj1 = pad_left(adj, adj[\"left\"],i) # essentially the curr\n",
    "                                                if (gap == 0 or (\"left\" in curr.keys() and \"right\" in adj.keys())):\n",
    "                                                    if (gap == 0):\n",
    "                                                        conc, x_delta, y_delta = concatenate_cubes(curr1, adj1, slice_size)\n",
    "                                                    else:\n",
    "                                                        curr1Left = pad_left(curr[\"left\"], ZERO_CUBE, i) # FIXIT? + assuming the gap will not be more than 150\n",
    "                                                        adj1Right = pad_left(adj[\"right\"], ZERO_CUBE, i) # FIXIT? + assuming the gap will not be more than 150\n",
    "                                                        conc, x_delta, y_delta = concatenate_cubes_with_gap(curr1, adj1, gap, curr1Left, adj1Right, slice_size)\n",
    "                                                    train_imgs.append(conc)\n",
    "                                                    train_x_delta.append(x_delta)\n",
    "                                                    train_y_delta.append(y_delta)\n",
    "                                                    # mark the image as matched\n",
    "                                                    train_lbls.append([0,1])\n",
    "                                        if (\"right\" in adj.keys()): # enough to check only the adj as the right of the curr is the adj\n",
    "                                            for i in range(5, 101, 5):\n",
    "                                                curr1 = pad_right(curr, curr[\"right\"],i) # essentially the adj\n",
    "                                                adj1 = pad_right(adj, adj[\"right\"],i)\n",
    "                                                if (gap == 0 or (\"left\" in curr.keys() and \"right\" in adj.keys())):\n",
    "                                                    if (gap == 0):\n",
    "                                                        conc, x_delta, y_delta = concatenate_cubes(curr1, adj1, slice_size)\n",
    "                                                    else:\n",
    "                                                        curr1Left = pad_right(curr[\"left\"], ZERO_CUBE, i) # FIXIT? + assuming the gap will not be more than 150\n",
    "                                                        adj1Right = pad_right(adj[\"right\"], ZERO_CUBE, i) # FIXIT? + assuming the gap will not be more than 150\n",
    "                                                        conc, x_delta, y_delta = concatenate_cubes_with_gap(curr1, adj1, gap, curr1Left, adj1Right, slice_size)\n",
    "                                                    train_imgs.append(conc)\n",
    "                                                    train_x_delta.append(x_delta)\n",
    "                                                    train_y_delta.append(y_delta)\n",
    "                                                    # mark the image as matched\n",
    "                                                    train_lbls.append([0,1])\n",
    "                                    else:\n",
    "                                        # mark the image as not matched\n",
    "                                        train_lbls.append([1,0])\n",
    "                                \n",
    "    warnings.filterwarnings(\"default\")\n",
    "    return train_imgs, train_lbls, train_x_delta, train_y_delta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-20T05:59:13.759547Z",
     "start_time": "2019-10-20T05:59:13.706352Z"
    },
    "code_folding": [
     0,
     2,
     7,
     86,
     248,
     253,
     278,
     297
    ],
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# RUN Utility functions 2\n",
    "SAVE_PNG=False\n",
    "def save_img(path, img):\n",
    "    np.save(path, img)  \n",
    "    if SAVE_PNG:\n",
    "        plt.imsave(path+\".png\", img, cmap=plt.cm.gray)\n",
    "                    \n",
    "def VAL_add_tolerance_matches(slice_size, folder, train_imgs, train_lbls, train_x_delta, \n",
    "                              train_y_delta, is_enriched, curr, adj, tolerance_factor=0):\n",
    "    # need to enhance the set with a few more true positive samples\n",
    "    # allowing some up and down tolerance\n",
    "    if (\"top\" in curr.keys()):\n",
    "        for i in range(0, tolerance_factor * 10, 10):\n",
    "            if i == 0:\n",
    "                continue\n",
    "            curr1 = pad_above(curr, curr[\"top\"],i)\n",
    "            adj1 = adj\n",
    "            conc, x_delta, y_delta, x_file, y_file = VAL_concatenate_cubes(curr1, adj1, slice_size)\n",
    "            output = folder+\"1_1=\"+x_file+\"_\"+str(curr1[\"top_row\"])+\"_\"+str(curr1[\"left_col\"])+\"---\"+y_file+\"_\"+str(adj1[\"top_row\"])+\"_\"+str(adj1[\"left_col\"])\n",
    "            # print(\">>> MATCH >>>\"+output)\n",
    "            save_img(output, conc)\n",
    "            # print(\">>> >>> >>> SAVED\")\n",
    "            train_imgs.append(output)\n",
    "            train_x_delta.append(x_delta)\n",
    "            train_y_delta.append(y_delta)\n",
    "            # mark the image as matched\n",
    "            train_lbls.append([0,1])\n",
    "            is_enriched.append(True)\n",
    "    if (\"top\" in adj.keys()):\n",
    "        for i in range(0, tolerance_factor * 10, 10):\n",
    "            if i == 0:\n",
    "                continue\n",
    "            curr1 = curr\n",
    "            adj1 = pad_above(adj, adj[\"top\"],i)\n",
    "            conc, x_delta, y_delta, x_file, y_file = VAL_concatenate_cubes(curr1, adj1, slice_size)\n",
    "            output = folder+\"1_1=\"+x_file+\"_\"+str(curr1[\"top_row\"])+\"_\"+str(curr1[\"left_col\"])+\"---\"+y_file+\"_\"+str(adj1[\"top_row\"])+\"_\"+str(adj1[\"left_col\"])\n",
    "            # print(\">>> MATCH >>>\"+output)\n",
    "            save_img(output, conc)\n",
    "            # print(\">>> >>> >>> SAVED\")\n",
    "            train_imgs.append(output)\n",
    "            train_x_delta.append(x_delta)\n",
    "            train_y_delta.append(y_delta)\n",
    "            # mark the image as matched\n",
    "            train_lbls.append([0,1])\n",
    "            is_enriched.append(True)\n",
    "    if (\"bottom\" in curr.keys()):\n",
    "        for i in range(0, tolerance_factor * 10, 10):\n",
    "            if i == 0:\n",
    "                continue\n",
    "            curr1 = pad_below(curr, curr[\"bottom\"],i)\n",
    "            adj1 = adj\n",
    "            conc, x_delta, y_delta, x_file, y_file = VAL_concatenate_cubes(curr1, adj1, slice_size)\n",
    "            output = folder+\"1_1=\"+x_file+\"_\"+str(curr1[\"top_row\"])+\"_\"+str(curr1[\"left_col\"])+\"---\"+y_file+\"_\"+str(adj1[\"top_row\"])+\"_\"+str(adj1[\"left_col\"])\n",
    "            # print(\">>> MATCH >>>\"+output)\n",
    "            save_img(output, conc)\n",
    "            # print(\">>> >>> >>> SAVED\")\n",
    "            train_imgs.append(output)\n",
    "            train_x_delta.append(x_delta)\n",
    "            train_y_delta.append(y_delta)\n",
    "            # mark the image as matched\n",
    "            train_lbls.append([0,1])\n",
    "            is_enriched.append(True)\n",
    "    if (\"bottom\"in adj.keys()):\n",
    "        for i in range(0, tolerance_factor * 10, 10):\n",
    "            if i == 0:\n",
    "                continue\n",
    "            curr1 = curr\n",
    "            adj1 = pad_below(adj, adj[\"bottom\"],i)\n",
    "            conc, x_delta, y_delta, x_file, y_file = VAL_concatenate_cubes(curr1, adj1, slice_size)\n",
    "            output = folder+\"1_1=\"+x_file+\"_\"+str(curr1[\"top_row\"])+\"_\"+str(curr1[\"left_col\"])+\"---\"+y_file+\"_\"+str(adj1[\"top_row\"])+\"_\"+str(adj1[\"left_col\"])\n",
    "            # print(\">>> MATCH >>>\"+output)\n",
    "            save_img(output, conc)\n",
    "            # print(\">>> >>> >>> SAVED\")\n",
    "            train_imgs.append(output)\n",
    "            train_x_delta.append(x_delta)\n",
    "            train_y_delta.append(y_delta)\n",
    "            # mark the image as matched\n",
    "            train_lbls.append([0,1])\n",
    "            is_enriched.append(True)\n",
    "\n",
    "                            \n",
    "# IMPORTANT: enrich_factor determines how many \"duplications\" of TRUE values will we have in the train set\n",
    "# This allows for a more balanced train set however, it reduces the strictness of the matches \n",
    "# i.e. (not sure why) when we have multiple nearby \"duplicates\" matches we get much more matches in the validation\n",
    "# PARAMS: enrich_factor=1 means no enrich/duplicate, 20 means duplicate by 20, every 10 pixels\n",
    "# PARAMS: tolerance_factor=0 means only match against exact horizon, each notch equals additional 10 pixels tolerance\n",
    "def NEW_build_train_set_for_binary_labeling(cubes, adjs, slice_size, folder, enrich_factor=1, tolerance_factor=0): \n",
    "    # enrich_factor is split by 2 because it is dual-sided and 1 means actually no enrichment - i.e. 0.5\n",
    "    enrich_factor = enrich_factor / 2 \n",
    "    # clean folder before starting\n",
    "\n",
    "    for root, dirs, files in os.walk(folder):\n",
    "        for f in files:\n",
    "            os.unlink(os.path.join(root, f))\n",
    "\n",
    "    warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "    train_imgs = []\n",
    "    train_lbls = []\n",
    "    train_x_delta = []\n",
    "    train_y_delta = []\n",
    "    is_enriched = []\n",
    "    discard_c = 0\n",
    "    skipped_c = 0\n",
    "    curr_idx = 0\n",
    "\n",
    "    # import pdb; pdb.set_trace()\n",
    "    # iterate over the cubes\n",
    "    for curr in cubes:\n",
    "        curr_idx = curr_idx + 1\n",
    "        if curr_idx % 100 == 0:\n",
    "            print(\"*** CUBES=\"+str(curr_idx))\n",
    "            print(\" MATCHED=\"+str(sum(x[1] == 1 for x in train_lbls)))\n",
    "            print(\" NOT MATCHED=\"+str(sum(x[0] == 1 for x in train_lbls)))\n",
    "            print(\" DISCARDED=\"+str(discard_c))\n",
    "            print(\" SKIPPED=\"+str(skipped_c))\n",
    "            \n",
    "        # iterate over the others (effectively n^2)\n",
    "        for adj in adjs:\n",
    "            # Initial filter: what CAN be matched against what?\n",
    "            # 1 - not of the same fragment (file==fragment)\n",
    "            # 2 - they ARE of the same tear - don't want to confuse the learning with false data coming from different tears\n",
    "            # 3 - no need to test against self and avoid checking from both directions\n",
    "            if  adj[\"file\"] != curr[\"file\"] and \\\n",
    "                adj[\"tear\"] == curr[\"tear\"] and \\\n",
    "                curr[\"piece_col\"] < adj[\"piece_col\"]: \n",
    "                # last condition above - actually ignores pieces of the same col but different rows\n",
    "                # the assumption is that they are either \"not-match\" and then will tilt the balance further to not-match\n",
    "                # or they are \"somewhat-matching\" but in a way that might confuse the algorithm\n",
    "                # import pdb; pdb.set_trace()\n",
    "                \n",
    "                # print(\">>> >>>\"+str(curr[\"cube\"].shape)+\" <<< <<<\"+str(adj[\"cube\"].shape))\n",
    "                # append the adjacent image to the current image\n",
    "                conc, x_delta, y_delta, x_file, y_file = VAL_concatenate_cubes(curr, adj, slice_size)\n",
    "\n",
    "                train_x_delta.append(x_delta)\n",
    "                train_y_delta.append(y_delta)\n",
    "                                \n",
    "                # Condition for marking as match:\n",
    "                # 1 - the adj piece is on the same row as the curr\n",
    "                # 2 - the adj piece is just to the right of the curr\n",
    "                # 3 - the curr cube is on the right edge of the piece\n",
    "                # 4 - the adj cube is on the left edge of the piece\n",
    "                # 5 - the cubes are in the same horizon\n",
    "                if  curr[\"piece_row\"] == adj[\"piece_row\"] and \\\n",
    "                    curr[\"piece_col\"] + 1 == adj[\"piece_col\"] and \\\n",
    "                    (curr[\"col\"] != 0 or curr[\"last\"]) and \\\n",
    "                    (adj[\"col\"] == 0 or not adj[\"last\"]) and \\\n",
    "                    np.abs(x_delta) < 50: \n",
    "\n",
    "                    #print(\"TTT:\", curr[\"col\"], curr[\"last\"], adj[\"col\"], adj[\"last\"], x_delta)\n",
    "                    #print(\"TTT:\", curr[\"left_col\"], curr[\"top_row\"], adj[\"left_col\"], adj[\"top_row\"], x_delta)\n",
    "                    \n",
    "                    # tag the image as matched (2nd digit) and not-enriched (1st digit)\n",
    "                    output = folder+\"0_1=\"+x_file+\"_\"+str(curr[\"top_row\"])+\"_\"+str(curr[\"left_col\"])+\"---\"+y_file+\"_\"+str(adj[\"top_row\"])+\"_\"+str(adj[\"left_col\"])\n",
    "                    # print(\">>> MATCH >>>\"+output)\n",
    "                    save_img(output, conc)\n",
    "                    # print(\">>> >>> >>> SAVED\")\n",
    "                    train_imgs.append(output)\n",
    "                    train_lbls.append([0,1])\n",
    "                    is_enriched.append(False)\n",
    "\n",
    "                    # TOLERANCE\n",
    "                    VAL_add_tolerance_matches(slice_size, folder, train_imgs, train_lbls, train_x_delta, \n",
    "                                              train_y_delta, is_enriched, curr, adj, tolerance_factor)\n",
    "                    \n",
    "                    # ENRICH/DUPLICATE\n",
    "                    # need to enrich the set with a few more true positive samples - so we offset \n",
    "                    # the matched images up and down a few times and create more matches\n",
    "                    if (\"top\" in curr.keys() and \"top\" in adj.keys()):\n",
    "                        for i in range(0, 121, int(120/enrich_factor)):\n",
    "                            if i == 0:\n",
    "                                continue\n",
    "                            curr1 = pad_above(curr, curr[\"top\"],i)\n",
    "                            adj1 = pad_above(adj, adj[\"top\"],i)\n",
    "                            conc, x_delta, y_delta, x_file, y_file = VAL_concatenate_cubes(curr1, adj1, slice_size)\n",
    "                            # tag the image as matched (2nd digit) and enriched (1st digit)\n",
    "                            output = folder+\"1_1=\"+x_file+\"_\"+str(curr1[\"top_row\"])+\"_\"+str(curr1[\"left_col\"])+\"---\"+y_file+\"_\"+str(adj1[\"top_row\"])+\"_\"+str(adj1[\"left_col\"])\n",
    "                            # print(\">>> MATCH >>>\"+output)\n",
    "                            save_img(output, conc)\n",
    "                            # print(\">>> >>> >>> SAVED\")\n",
    "                            train_imgs.append(output)\n",
    "                            train_x_delta.append(x_delta)\n",
    "                            train_y_delta.append(y_delta)\n",
    "                            # mark the image as matched\n",
    "                            train_lbls.append([0,1])\n",
    "                            is_enriched.append(True)\n",
    "\n",
    "                            # TOLERANCE\n",
    "                            VAL_add_tolerance_matches(slice_size, folder, train_imgs, train_lbls, train_x_delta, \n",
    "                                                      train_y_delta, is_enriched, curr1, adj1, tolerance_factor)\n",
    "                            \n",
    "                    if (\"bottom\" in curr.keys() and \"bottom\"in adj.keys()):\n",
    "                        for i in range(0, 121, int(120/enrich_factor)):\n",
    "                            if i == 0:\n",
    "                                continue\n",
    "                            curr1 = pad_below(curr, curr[\"bottom\"],i)\n",
    "                            adj1 = pad_below(adj, adj[\"bottom\"],i)\n",
    "                            conc, x_delta, y_delta, x_file, y_file = VAL_concatenate_cubes(curr1, adj1, slice_size)\n",
    "                            # tag the image as matched (2nd digit) and enriched (1st digit)\n",
    "                            output = folder+\"1_1=\"+x_file+\"_\"+str(curr1[\"top_row\"])+\"_\"+str(curr1[\"left_col\"])+\"---\"+y_file+\"_\"+str(adj1[\"top_row\"])+\"_\"+str(adj1[\"left_col\"])\n",
    "                            # print(\">>> MATCH >>>\"+output)\n",
    "                            save_img(output, conc)\n",
    "                            # print(\">>> >>> >>> SAVED\")\n",
    "                            train_imgs.append(output)\n",
    "                            train_x_delta.append(x_delta)\n",
    "                            train_y_delta.append(y_delta)\n",
    "                            # mark the image as matched\n",
    "                            train_lbls.append([0,1])\n",
    "                            is_enriched.append(True)\n",
    "\n",
    "                            # TOLERANCE\n",
    "                            VAL_add_tolerance_matches(slice_size, folder, train_imgs, train_lbls, train_x_delta, \n",
    "                                                      train_y_delta, is_enriched, curr1, adj1, tolerance_factor)\n",
    " \n",
    "                # adding a condition for marking as not-matched - we mark only the \"key\" cubes which are every 250px\n",
    "                # and not overlap - hence we reduce the ratio in favour of not matched which is enormous\n",
    "                \n",
    "                # altering between the next 2 lines allows to control the number/ratio of non-match:match\n",
    "                # elif int(curr[\"row\"]) == curr[\"row\"] and int(adj[\"row\"]) == adj[\"row\"]: # this condition will match curr key cubes with adj key cubes only\n",
    "                elif int(adj[\"row\"]) == adj[\"row\"]: # this condition will allow curr cubes which are not just key\n",
    "                    #print(\"NNN:\", curr[\"col\"], curr[\"last\"], adj[\"col\"], adj[\"last\"], x_delta)\n",
    "                    #print(\"NNN:\", curr[\"left_col\"], curr[\"top_row\"], adj[\"left_col\"], adj[\"top_row\"], x_delta)\n",
    "                    # tag the image as not-matched (2nd digit) and not-enriched (1st digit)\n",
    "                    output = folder+\"0_0=\"+x_file+\"_\"+str(curr[\"top_row\"])+\"_\"+str(curr[\"left_col\"])+\"---\"+y_file+\"_\"+str(adj[\"top_row\"])+\"_\"+str(adj[\"left_col\"])\n",
    "                    # print(\"<<< nonmatch <<<\"+output)\n",
    "                    save_img(output, conc)\n",
    "                    # print(\"<<< <<< <<< SAVED\")\n",
    "                    train_imgs.append(output)\n",
    "                    train_lbls.append([1,0]) # not matched\n",
    "                    is_enriched.append(False)\n",
    "                \n",
    "                # discard not matched which are not \"key\" cubes (every 250px)\n",
    "                else:\n",
    "                    #if x_delta == 0:\n",
    "                    #    print(\"DDD:\", curr[\"col\"], curr[\"last\"], adj[\"col\"], adj[\"last\"], x_delta)\n",
    "                    #    print(\"DDD:\", curr[\"left_col\"], curr[\"top_row\"], adj[\"left_col\"], adj[\"top_row\"], x_delta)\n",
    "                    discard_c += 1\n",
    "            else:\n",
    "                skipped_c += 1\n",
    "    print(\"*** MATCHED=\"+str(sum(x[1] == 1 for x in train_lbls)))\n",
    "    print(\"*** NOT MATCHED=\"+str(sum(x[0] == 1 for x in train_lbls)))\n",
    "    print(\"*** DISCARDED=\"+str(discard_c))\n",
    "    print(\"*** SKIPPED=\"+str(skipped_c))\n",
    "    warnings.filterwarnings(\"default\")\n",
    "    return train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched\n",
    "\n",
    "\n",
    "def PARALLEL_build_train_set_for_binary_labeling(args): \n",
    "    print('args:', args)\n",
    "    return NEW_build_train_set_for_binary_labeling(args[0], args[1], args[2], args[3], args[4], args[5])\n",
    "\n",
    "\n",
    "def frame_to_n_by_m(orig, start_vector, end_vector, is_col):\n",
    "    max_val = np.amax(end_vector)\n",
    "    min_val = np.amin(start_vector)\n",
    "    width = max_val - min_val\n",
    "    if width < CUBE_SIZE:\n",
    "        width = CUBE_SIZE\n",
    "    if (is_col):\n",
    "        result = np.zeros((start_vector.size, width))\n",
    "    else:\n",
    "        result = np.zeros((width, start_vector.size))\n",
    "    \n",
    "    for i in range(0, start_vector.size):\n",
    "        if (is_col):\n",
    "            row_vec = orig[i, start_vector[i]:end_vector[i]]\n",
    "        else:\n",
    "            row_vec = orig[start_vector[i]:end_vector[i],i]\n",
    "        temp = np.lib.pad(row_vec, (start_vector[i]-min_val, max_val-end_vector[i]), 'constant', constant_values=(0.09, 0.09))\n",
    "        if (is_col):\n",
    "            if (result[i].size != width):\n",
    "                import pdb; pdb.set_trace()\n",
    "            result[i] = temp[0:width]\n",
    "        else:\n",
    "            result[:,i] = temp[0:width]\n",
    "    return min_val, result\n",
    "\n",
    "def rough_tear_line(orig, start_vector, cut_mean, is_col, chew_factor):\n",
    "    end_vector = np.empty(start_vector.size).astype(int)\n",
    "    if (is_col and np.absolute(cut_mean-orig.shape[1]) < 10):\n",
    "        end_vector.fill(orig.shape[1])\n",
    "    elif (not is_col and np.absolute(cut_mean-orig.shape[0]) < 10):\n",
    "        end_vector.fill(orig.shape[0])\n",
    "    else:\n",
    "        deviation_vector = np.random.normal(0, chew_factor, start_vector.size).astype(int)\n",
    "        end_vector[0] = cut_mean + deviation_vector[0]\n",
    "        for i in range(1, end_vector.size):\n",
    "            end_vector[i] = end_vector[i - 1] + deviation_vector[i]\n",
    "\n",
    "        cut_max = start_vector + (CUBE_SIZE + EDGE_GAP)\n",
    "        max_and_end = np.append([end_vector], [cut_max], axis=0)\n",
    "        end_vector = np.amax(max_and_end, axis=0)\n",
    "            \n",
    "    start_px, cut_piece = frame_to_n_by_m(orig, start_vector, end_vector, is_col)    \n",
    "    return start_px, cut_piece, end_vector\n",
    "\n",
    "def rough_tear_image(image, cols, rows):\n",
    "    pieces = []\n",
    "    col_width = int(image.shape[1] / cols)\n",
    "    row_height = int(image.shape[0] / rows)\n",
    "    # print(col_width, row_height)\n",
    "    next_col_start_vec = np.zeros((image.shape[0],), dtype=int)\n",
    "    for col_idx in range(0, cols):\n",
    "    #         import pdb; pdb.set_trace()\n",
    "        start_col_px, cut_column, next_col_start_vec =  rough_tear_line(image, next_col_start_vec, col_width * (col_idx + 1), True, 3)\n",
    "        next_row_start_vec = np.zeros((cut_column.shape[1],), dtype=int)\n",
    "        for row_idx in range(0, rows):\n",
    "            start_row_px, cut_piece, next_row_start_vec = rough_tear_line(cut_column, next_row_start_vec, row_height * (row_idx + 1), False, 1)\n",
    "\n",
    "            ymin, ymax, xmin, xmax = calc_min_max_coordinates_dynamic(cut_piece, cutoff=BG_2_OBJ_RATIO)\n",
    "            temp = crop(cut_piece, ymin, ymax, xmin, xmax)\n",
    "            \n",
    "            #import pdb; pdb.set_trace()\n",
    "            piece = {}\n",
    "            piece[\"orig\"] = cut_piece\n",
    "            piece[\"cut\"] = temp\n",
    "            piece[\"col\"] = col_idx\n",
    "            piece[\"row\"] = row_idx\n",
    "            piece[\"col_px\"] = start_col_px + xmin\n",
    "            piece[\"row_px\"] = start_row_px + ymin\n",
    "            pieces.append(piece)\n",
    "            \n",
    "    return pieces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-20T05:59:13.769015Z",
     "start_time": "2019-10-20T05:59:13.761070Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# RUN Define model util functions\n",
    "\n",
    "# initialize a shaped matrix of weights with random values\n",
    "def weight_variable(shape):\n",
    "  initial = tf.truncated_normal(shape, stddev=0.1)\n",
    "  return tf.Variable(initial)\n",
    "\n",
    "# initialize a shaped matrix of bias with random values\n",
    "def bias_variable(shape):\n",
    "  initial = tf.constant(0.1, shape=shape)\n",
    "  return tf.Variable(initial)\n",
    "\n",
    "def conv2d(x, W):\n",
    "  return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')\n",
    "\n",
    "def max_pool_2x2(x):\n",
    "  return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],\n",
    "                        strides=[1, 2, 2, 1], padding='SAME')\n",
    "\n",
    "def max_pool_1x2(x):\n",
    "  return tf.nn.max_pool(x, ksize=[1, 1, 2, 1],\n",
    "                        strides=[1, 1, 2, 1], padding='SAME')\n",
    "\n",
    "def max_pool_2x1(x):\n",
    "  return tf.nn.max_pool(x, ksize=[1, 2, 1, 1],\n",
    "                        strides=[1, 2, 1, 1], padding='SAME')\n",
    "\n",
    "def max_pool_1x1(x):\n",
    "  return tf.nn.max_pool(x, ksize=[1, 1, 1, 1],\n",
    "                        strides=[1, 1, 1, 1], padding='SAME')\n",
    "\n",
    "def max_pool_5x5(x):\n",
    "  return tf.nn.max_pool(x, ksize=[1, 5, 5, 1],\n",
    "                        strides=[1, 5, 5, 1], padding='SAME')\n",
    "\n",
    "def max_pool_5x2(x):\n",
    "  return tf.nn.max_pool(x, ksize=[1, 5, 2, 1],\n",
    "                        strides=[1, 5, 2, 1], padding='SAME')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-20T05:59:13.789202Z",
     "start_time": "2019-10-20T05:59:13.771456Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# RUN Image utility functions (external source)\n",
    "def branchedPoints(skel):\n",
    "    branch1=np.array([[2, 1, 2], [1, 1, 1], [2, 2, 2]])\n",
    "    branch2=np.array([[1, 2, 1], [2, 1, 2], [1, 2, 1]])\n",
    "    branch3=np.array([[1, 2, 1], [2, 1, 2], [1, 2, 2]])\n",
    "    branch4=np.array([[2, 1, 2], [1, 1, 2], [2, 1, 2]])\n",
    "    branch5=np.array([[1, 2, 2], [2, 1, 2], [1, 2, 1]])\n",
    "    branch6=np.array([[2, 2, 2], [1, 1, 1], [2, 1, 2]])\n",
    "    branch7=np.array([[2, 2, 1], [2, 1, 2], [1, 2, 1]])\n",
    "    branch8=np.array([[2, 1, 2], [2, 1, 1], [2, 1, 2]])\n",
    "    branch9=np.array([[1, 2, 1], [2, 1, 2], [2, 2, 1]])\n",
    "    br1=mh.morph.hitmiss(skel,branch1)\n",
    "    br2=mh.morph.hitmiss(skel,branch2)\n",
    "    br3=mh.morph.hitmiss(skel,branch3)\n",
    "    br4=mh.morph.hitmiss(skel,branch4)\n",
    "    br5=mh.morph.hitmiss(skel,branch5)\n",
    "    br6=mh.morph.hitmiss(skel,branch6)\n",
    "    br7=mh.morph.hitmiss(skel,branch7)\n",
    "    br8=mh.morph.hitmiss(skel,branch8)\n",
    "    br9=mh.morph.hitmiss(skel,branch9)\n",
    "    return br1+br2+br3+br4+br5+br6+br7+br8+br9\n",
    "\n",
    "def endPoints(skel):\n",
    "    endpoint1=np.array([[0, 0, 0],\n",
    "                        [0, 1, 0],\n",
    "                        [2, 1, 2]])\n",
    "    \n",
    "    endpoint2=np.array([[0, 0, 0],\n",
    "                        [0, 1, 2],\n",
    "                        [0, 2, 1]])\n",
    "    \n",
    "    endpoint3=np.array([[0, 0, 2],\n",
    "                        [0, 1, 1],\n",
    "                        [0, 0, 2]])\n",
    "    \n",
    "    endpoint4=np.array([[0, 2, 1],\n",
    "                        [0, 1, 2],\n",
    "                        [0, 0, 0]])\n",
    "    \n",
    "    endpoint5=np.array([[2, 1, 2],\n",
    "                        [0, 1, 0],\n",
    "                        [0, 0, 0]])\n",
    "    \n",
    "    endpoint6=np.array([[1, 2, 0],\n",
    "                        [2, 1, 0],\n",
    "                        [0, 0, 0]])\n",
    "    \n",
    "    endpoint7=np.array([[2, 0, 0],\n",
    "                        [1, 1, 0],\n",
    "                        [2, 0, 0]])\n",
    "    \n",
    "    endpoint8=np.array([[0, 0, 0],\n",
    "                        [2, 1, 0],\n",
    "                        [1, 2, 0]])\n",
    "    \n",
    "    ep1=mh.morph.hitmiss(skel,endpoint1)\n",
    "    ep2=mh.morph.hitmiss(skel,endpoint2)\n",
    "    ep3=mh.morph.hitmiss(skel,endpoint3)\n",
    "    ep4=mh.morph.hitmiss(skel,endpoint4)\n",
    "    ep5=mh.morph.hitmiss(skel,endpoint5)\n",
    "    ep6=mh.morph.hitmiss(skel,endpoint6)\n",
    "    ep7=mh.morph.hitmiss(skel,endpoint7)\n",
    "    ep8=mh.morph.hitmiss(skel,endpoint8)\n",
    "    ep = ep1+ep2+ep3+ep4+ep5+ep6+ep7+ep8\n",
    "    return ep\n",
    "\n",
    "def pruning(skeleton, size):\n",
    "    '''remove iteratively end points \"size\" \n",
    "       times from the skeleton\n",
    "    '''\n",
    "    for i in range(0, size):\n",
    "        endpoints = endPoints(skeleton)\n",
    "        endpoints = np.logical_not(endpoints)\n",
    "        skeleton = np.logical_and(skeleton,endpoints)\n",
    "    return skeleton\n",
    "\n",
    "def plot_comparison(original, filtered, filter_name):\n",
    "\n",
    "    fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(8, 4), sharex=True, sharey=True)\n",
    "    ax1.imshow(original, cmap=plt.cm.gray)\n",
    "    ax1.set_title('original')\n",
    "    ax1.axis('off')\n",
    "    ax1.set_adjustable('box-forced')\n",
    "    ax2.imshow(filtered, cmap=plt.cm.gray)\n",
    "    ax2.set_title(filter_name)\n",
    "    ax2.axis('off')\n",
    "    ax2.set_adjustable('box-forced')\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-20T05:59:13.803965Z",
     "start_time": "2019-10-20T05:59:13.790944Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# RUN model_tf_deep - Define the model - 250, 125, 62, 25\n",
    "def model_tf_deep(input_width, forced_bias=0): \n",
    "    global accuracy, correct_prediction, train_step, x, y_, y_conv, keep_prob, probability, probabilities #, W_fc, b_fc, cost, y_conv_temp\n",
    "    \n",
    "    # foundation of the model - the input layer of the image 250 x input_width*2\n",
    "    x = tf.placeholder(tf.float32, [None, 250, input_width*2], \"001\")\n",
    "    x_image = tf.reshape(x, [-1,250,input_width*2,1], \"0011\") # 1 is the number of color channels\n",
    "\n",
    "    # the target digits of the model\n",
    "    y_ = tf.placeholder(tf.float32, [None, 2], \"002\") # 1\n",
    "\n",
    "    # zero convolutional layer: one input image and 32 output filters of 5x5\n",
    "    W_conv0 = weight_variable([5, 5, 1, 32])\n",
    "    b_conv0 = bias_variable([32])\n",
    "    h_conv0 = tf.nn.relu(conv2d(x_image, W_conv0) + b_conv0, \"0020\")\n",
    "    h_pool0 = max_pool_1x1(h_conv0) # size is maintained\n",
    "\n",
    "    # first convolutional layer: one input image and 32 output filters of 5x5\n",
    "    W_conv1 = weight_variable([5, 5, 32, 32])\n",
    "#     W_conv1 = weight_variable([5, 5, 1, 32])\n",
    "    b_conv1 = bias_variable([32])\n",
    "    h_conv1 = tf.nn.relu(conv2d(h_pool0, W_conv1) + b_conv1, \"0021\")\n",
    "#     h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1, \"0021\")\n",
    "    if (input_width == 250):\n",
    "        h_pool1 = max_pool_2x2(h_conv1) # size is reduced to 125x250\n",
    "    elif (input_width == 125):\n",
    "        h_pool1 = max_pool_2x1(h_conv1) # size is reduced to 125x250\n",
    "    elif (input_width == 62):\n",
    "        h_pool1 = max_pool_2x1(h_conv1) # size is reduced to 125x125\n",
    "    elif (input_width == 25):\n",
    "        h_pool1 = max_pool_2x1(h_conv1) # size is reduced to 125x50\n",
    "    else:\n",
    "        print(\"ERROR - unsupported slice width\")\n",
    "        return\n",
    "\n",
    "    # second convolutional layer: 32 input (filtered) images and 32 output filters of 5x5\n",
    "    W_conv2 = weight_variable([5, 5, 32, 32])\n",
    "    b_conv2 = bias_variable([32])\n",
    "    h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2, \"0022\")\n",
    "    if (input_width == 62):\n",
    "        h_pool2 = max_pool_1x1(h_conv2) # size is reduced to 125x125\n",
    "    elif (input_width == 25):\n",
    "        h_pool2 = max_pool_1x1(h_conv2) # size is reduced to 125x50\n",
    "    else:\n",
    "        h_pool2 = max_pool_1x2(h_conv2) # size is reduced to 125x125\n",
    "\n",
    "\n",
    "    # third convolutional layer: 32 input (filtered) images and 32 output filters of 5x5\n",
    "    W_conv3 = weight_variable([5, 5, 32, 32])\n",
    "    b_conv3 = bias_variable([32])\n",
    "    h_conv3 = tf.nn.relu(conv2d(h_pool2, W_conv3) + b_conv3, \"0023\")\n",
    "    if (input_width == 25):\n",
    "        h_pool3 = max_pool_5x2(h_conv3) # size is reduced to 25x25\n",
    "    else:\n",
    "        h_pool3 = max_pool_5x5(h_conv3) # size is reduced to 25x25\n",
    "\n",
    "\n",
    "    h_pool3_flat = tf.reshape(h_pool3, [-1, 25*25*32]) # shape as an array \n",
    "\n",
    "    # fourth layer - fully connected with input 25*25*128 and output 1024\n",
    "    W_fc1 = weight_variable([25*25*32, 1024])\n",
    "    b_fc1 = bias_variable([1024])\n",
    "    h_fc1 = tf.nn.relu(tf.matmul(h_pool3_flat, W_fc1) + b_fc1, \"0024\")\n",
    "\n",
    "    # a drop layer with probability \n",
    "    keep_prob = tf.placeholder(tf.float32, name=\"003\")\n",
    "    h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob, name=\"0031\")\n",
    "\n",
    "    #     # final layer - reduce to one \"class\" for the linear regression\n",
    "    #     W_fc = weight_variable([1024, 1]) \n",
    "    #     b_fc = bias_variable([1])         \n",
    "    #     y_conv_temp = tf.matmul(h_fc1_drop, W_fc, name=\"0032\") + b_fc\n",
    "    #     y_conv = tf.minimum(y_conv_temp, tf.constant(BREAK_VAL, tf.float32))\n",
    "\n",
    "    #     #     # minimize loss function\n",
    "    #     #     cross_entropy = tf.reduce_mean(\n",
    "    #     #       tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))\n",
    "    #     # cost = tf.reduce_sum(tf.pow(y_conv - y_, 2))/(2*BATCHES*BATCH_SIZE) # Mean squared error\n",
    "    #     cost = tf.reduce_mean(tf.square(y_conv_temp - y_), name=\"0033\") # Mean squared error\n",
    "\n",
    "    #     #     # define train step and rate\n",
    "    #     #     train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n",
    "    #     train_step = tf.train.AdamOptimizer(LEARNING_RATE).minimize(cost) # Gradient descent\n",
    "\n",
    "    #     # evaluate the prediction and the accuracy on the train test - needed only for printing during the training\n",
    "    #     correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))\n",
    "    #     accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    \n",
    "    # final layer - softmax reduction 2 outputs\n",
    "    W_fc2 = weight_variable([1024, 2])\n",
    "    b_fc2 = bias_variable([2])\n",
    "    c_fc2 = tf.constant([0, forced_bias], dtype=tf.float32)\n",
    "    y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2 + c_fc2\n",
    "\n",
    "    # minimize loss function\n",
    "    cross_entropy = tf.reduce_mean(\n",
    "      tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))\n",
    "      # tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_, logits=y_conv))\n",
    "\n",
    "    probability = tf.nn.softmax(y_conv,1)\n",
    "    \n",
    "    probabilities=tf.reduce_sum(probability,1)\n",
    "    \n",
    "    # define train step and rate\n",
    "    train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n",
    "\n",
    "    # evaluate the prediction and the accuracy on the train test - needed only for printing during the training\n",
    "    correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-20T05:59:13.818228Z",
     "start_time": "2019-10-20T05:59:13.805507Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# RUN model_tf_orig - Define the model - 250, 125, 62, 25\n",
    "def model_tf_orig(input_width): \n",
    "    global accuracy, correct_prediction, train_step, x, y_, y_conv, keep_prob #, W_fc, b_fc, cost, y_conv_temp\n",
    "    \n",
    "    # foundation of the model - the input layer of the image 250 x input_width*2\n",
    "    x = tf.placeholder(tf.float32, [None, 250, input_width*2], \"001\")\n",
    "    x_image = tf.reshape(x, [-1,250,input_width*2,1], \"0011\") # 1 is the number of color channels\n",
    "\n",
    "    # the target digits of the model\n",
    "    y_ = tf.placeholder(tf.float32, [None, 2], \"002\") # 1\n",
    "\n",
    "    # zero convolutional layer: one input image and 32 output filters of 5x5\n",
    "#     W_conv0 = weight_variable([5, 5, 1, 32])\n",
    "#     b_conv0 = bias_variable([32])\n",
    "#     h_conv0 = tf.nn.relu(conv2d(x_image, W_conv0) + b_conv0, \"0020\")\n",
    "#     h_pool0 = max_pool_1x1(h_conv0) # size is maintained\n",
    "\n",
    "    # first convolutional layer: one input image and 32 output filters of 5x5\n",
    "#     W_conv1 = weight_variable([5, 5, 32, 32])\n",
    "    W_conv1 = weight_variable([5, 5, 1, 32])\n",
    "    b_conv1 = bias_variable([32])\n",
    "#     h_conv1 = tf.nn.relu(conv2d(h_pool0, W_conv1) + b_conv1, \"0021\")\n",
    "    h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1, \"0021\")\n",
    "    if (input_width == 250):\n",
    "        h_pool1 = max_pool_2x2(h_conv1) # size is reduced to 125x250\n",
    "    elif (input_width == 125):\n",
    "        h_pool1 = max_pool_2x1(h_conv1) # size is reduced to 125x250\n",
    "    elif (input_width == 62):\n",
    "        h_pool1 = max_pool_2x1(h_conv1) # size is reduced to 125x125\n",
    "    elif (input_width == 25):\n",
    "        h_pool1 = max_pool_2x1(h_conv1) # size is reduced to 125x50\n",
    "    else:\n",
    "        print(\"ERROR - unsupported slice width\")\n",
    "        return\n",
    "\n",
    "    # second convolutional layer: 32 input (filtered) images and 32 output filters of 5x5\n",
    "    W_conv2 = weight_variable([5, 5, 32, 32])\n",
    "    b_conv2 = bias_variable([32])\n",
    "    h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2, \"0022\")\n",
    "    if (input_width == 62):\n",
    "        h_pool2 = max_pool_1x1(h_conv2) # size is reduced to 125x125\n",
    "    elif (input_width == 25):\n",
    "        h_pool2 = max_pool_1x1(h_conv2) # size is reduced to 125x50\n",
    "    else:\n",
    "        h_pool2 = max_pool_1x2(h_conv2) # size is reduced to 125x125\n",
    "\n",
    "\n",
    "    # third convolutional layer: 32 input (filtered) images and 32 output filters of 5x5\n",
    "    W_conv3 = weight_variable([5, 5, 32, 32])\n",
    "    b_conv3 = bias_variable([32])\n",
    "    h_conv3 = tf.nn.relu(conv2d(h_pool2, W_conv3) + b_conv3, \"0023\")\n",
    "    if (input_width == 25):\n",
    "        h_pool3 = max_pool_5x2(h_conv3) # size is reduced to 25x25\n",
    "    else:\n",
    "        h_pool3 = max_pool_5x5(h_conv3) # size is reduced to 25x25\n",
    "\n",
    "\n",
    "    h_pool3_flat = tf.reshape(h_pool3, [-1, 25*25*32]) # shape as an array \n",
    "\n",
    "    # fourth layer - fully connected with input 25*25*128 and output 1024\n",
    "    W_fc1 = weight_variable([25*25*32, 1024])\n",
    "    b_fc1 = bias_variable([1024])\n",
    "    h_fc1 = tf.nn.relu(tf.matmul(h_pool3_flat, W_fc1) + b_fc1, \"0024\")\n",
    "\n",
    "    # a drop layer with probability \n",
    "    keep_prob = tf.placeholder(tf.float32, name=\"003\")\n",
    "    h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob, name=\"0031\")\n",
    "\n",
    "    #     # final layer - reduce to one \"class\" for the linear regression\n",
    "    #     W_fc = weight_variable([1024, 1]) \n",
    "    #     b_fc = bias_variable([1])         \n",
    "    #     y_conv_temp = tf.matmul(h_fc1_drop, W_fc, name=\"0032\") + b_fc\n",
    "    #     y_conv = tf.minimum(y_conv_temp, tf.constant(BREAK_VAL, tf.float32))\n",
    "\n",
    "    #     #     # minimize loss function\n",
    "    #     #     cross_entropy = tf.reduce_mean(\n",
    "    #     #       tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))\n",
    "    #     # cost = tf.reduce_sum(tf.pow(y_conv - y_, 2))/(2*BATCHES*BATCH_SIZE) # Mean squared error\n",
    "    #     cost = tf.reduce_mean(tf.square(y_conv_temp - y_), name=\"0033\") # Mean squared error\n",
    "\n",
    "    #     #     # define train step and rate\n",
    "    #     #     train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n",
    "    #     train_step = tf.train.AdamOptimizer(LEARNING_RATE).minimize(cost) # Gradient descent\n",
    "\n",
    "    #     # evaluate the prediction and the accuracy on the train test - needed only for printing during the training\n",
    "    #     correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))\n",
    "    #     accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    \n",
    "    # final layer - softmax reduction 2 outputs\n",
    "    W_fc2 = weight_variable([1024, 2])\n",
    "    b_fc2 = bias_variable([2])\n",
    "    y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2\n",
    "\n",
    "    # minimize loss function\n",
    "    cross_entropy = tf.reduce_mean(\n",
    "      tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))\n",
    "\n",
    "    # define train step and rate\n",
    "    train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n",
    "\n",
    "    # evaluate the prediction and the accuracy on the train test - needed only for printing during the training\n",
    "    correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-20T05:59:13.853580Z",
     "start_time": "2019-10-20T05:59:13.837627Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# RUN model_tf_wide - Define the model - 250, 125, 62, 25\n",
    "def model_tf_wide(input_width): \n",
    "    global accuracy, correct_prediction, train_step, x, y_, y_conv, keep_prob #, W_fc, b_fc, cost, y_conv_temp\n",
    "    \n",
    "    # foundation of the model - the input layer of the image 250 x input_width*2\n",
    "    x = tf.placeholder(tf.float32, [None, 250, input_width*2], \"001\")\n",
    "    x_image = tf.reshape(x, [-1,250,input_width*2,1], \"0011\") # 1 is the number of color channels\n",
    "\n",
    "    # the target digits of the model\n",
    "    y_ = tf.placeholder(tf.float32, [None, 2], \"002\") # 1\n",
    "\n",
    "    # first convolutional layer: one input image and 32 output filters of 5x5\n",
    "    W_conv1 = weight_variable([5, 5, 1, 64])\n",
    "    b_conv1 = bias_variable([64])\n",
    "    h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1, \"0021\")\n",
    "    if (input_width == 250):\n",
    "        h_pool1 = max_pool_2x2(h_conv1) # size is reduced to 125x250\n",
    "    elif (input_width == 125):\n",
    "        h_pool1 = max_pool_2x1(h_conv1) # size is reduced to 125x250\n",
    "    elif (input_width == 62):\n",
    "        h_pool1 = max_pool_2x1(h_conv1) # size is reduced to 125x125\n",
    "    elif (input_width == 25):\n",
    "        h_pool1 = max_pool_2x1(h_conv1) # size is reduced to 125x50\n",
    "    else:\n",
    "        print(\"ERROR - unsupported slice width\")\n",
    "        return\n",
    "\n",
    "    # second convolutional layer: 32 input (filtered) images and 32 output filters of 5x5\n",
    "    W_conv2 = weight_variable([5, 5, 64, 64])\n",
    "    b_conv2 = bias_variable([64])\n",
    "    h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2, \"0022\")\n",
    "    if (input_width == 62):\n",
    "        h_pool2 = max_pool_1x1(h_conv2) # size is reduced to 125x125\n",
    "    elif (input_width == 25):\n",
    "        h_pool2 = max_pool_1x1(h_conv2) # size is reduced to 125x50\n",
    "    else:\n",
    "        h_pool2 = max_pool_1x2(h_conv2) # size is reduced to 125x125\n",
    "\n",
    "\n",
    "    # third convolutional layer: 32 input (filtered) images and 32 output filters of 5x5\n",
    "    W_conv3 = weight_variable([5, 5, 64, 64])\n",
    "    b_conv3 = bias_variable([64])\n",
    "    h_conv3 = tf.nn.relu(conv2d(h_pool2, W_conv3) + b_conv3, \"0023\")\n",
    "    if (input_width == 25):\n",
    "        h_pool3 = max_pool_5x2(h_conv3) # size is reduced to 25x25\n",
    "    else:\n",
    "        h_pool3 = max_pool_5x5(h_conv3) # size is reduced to 25x25\n",
    "\n",
    "\n",
    "    h_pool3_flat = tf.reshape(h_pool3, [-1, 25*25*64]) # shape as an array \n",
    "\n",
    "    # fourth layer - fully connected with input 25*25*128 and output 1024\n",
    "    W_fc1 = weight_variable([25*25*64, 2048])\n",
    "    b_fc1 = bias_variable([2048])\n",
    "    h_fc1 = tf.nn.relu(tf.matmul(h_pool3_flat, W_fc1) + b_fc1, \"0024\")\n",
    "\n",
    "    # a drop layer with probability \n",
    "    keep_prob = tf.placeholder(tf.float32, name=\"003\")\n",
    "    h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob, name=\"0031\")\n",
    "\n",
    "    #     # final layer - reduce to one \"class\" for the linear regression\n",
    "    #     W_fc = weight_variable([1024, 1]) \n",
    "    #     b_fc = bias_variable([1])         \n",
    "    #     y_conv_temp = tf.matmul(h_fc1_drop, W_fc, name=\"0032\") + b_fc\n",
    "    #     y_conv = tf.minimum(y_conv_temp, tf.constant(BREAK_VAL, tf.float32))\n",
    "\n",
    "    #     #     # minimize loss function\n",
    "    #     #     cross_entropy = tf.reduce_mean(\n",
    "    #     #       tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))\n",
    "    #     # cost = tf.reduce_sum(tf.pow(y_conv - y_, 2))/(2*BATCHES*BATCH_SIZE) # Mean squared error\n",
    "    #     cost = tf.reduce_mean(tf.square(y_conv_temp - y_), name=\"0033\") # Mean squared error\n",
    "\n",
    "    #     #     # define train step and rate\n",
    "    #     #     train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n",
    "    #     train_step = tf.train.AdamOptimizer(LEARNING_RATE).minimize(cost) # Gradient descent\n",
    "\n",
    "    #     # evaluate the prediction and the accuracy on the train test - needed only for printing during the training\n",
    "    #     correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))\n",
    "    #     accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    \n",
    "    # final layer - softmax reduction 2 outputs\n",
    "    W_fc2 = weight_variable([2048, 2])\n",
    "    b_fc2 = bias_variable([2])\n",
    "    y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2\n",
    "\n",
    "    # minimize loss function\n",
    "    cross_entropy = tf.reduce_mean(\n",
    "      tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))\n",
    "\n",
    "    # define train step and rate\n",
    "    train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n",
    "\n",
    "    # evaluate the prediction and the accuracy on the train test - needed only for printing during the training\n",
    "    correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-20T05:59:13.871561Z",
     "start_time": "2019-10-20T05:59:13.860390Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# RUN train\n",
    "def train(train_imgs, train_lbls, output_model, input_model=\"\"):\n",
    "    print(\"#####################################################################\")\n",
    "    print(\"TRAINING:\")\n",
    "    print(\"MODEL:\"+output_model)\n",
    "    print(\"#####################################################################\")\n",
    "\n",
    "    from random import randrange\n",
    "    \n",
    "    # TRAIN Prepare the session\n",
    "\n",
    "    # create a saver object\n",
    "    saver = tf.train.Saver()\n",
    "\n",
    "    # start session and initialize variables\n",
    "    sess = tf.InteractiveSession()\n",
    "    if input_model != \"\":\n",
    "        # Restore variables from disk.\n",
    "        saver.restore(sess, input_model)\n",
    "        print(\"Model restored.\")\n",
    "    else:\n",
    "        sess.run(tf.initialize_all_variables())\n",
    "\n",
    "    # TRAIN Train the model\n",
    "    x_batch = []\n",
    "    y_batch = []\n",
    "    # run the train batches\n",
    "    for i in range(BATCHES):\n",
    "        x_batch = []\n",
    "        y_batch = []\n",
    "        for _ in range(BATCH_SIZE):\n",
    "            random_index = randrange(0,len(train_imgs))\n",
    "            image = np.load(train_imgs[random_index]+\".npy\")\n",
    "            # print(train_imgs[random_index])\n",
    "            x_batch.append(image)\n",
    "            y_batch.append(train_lbls[random_index])\n",
    "\n",
    "        # train\n",
    "        # print(\"step %d\"%(i))\n",
    "        train_step.run(feed_dict={x: x_batch, y_: y_batch, keep_prob: 0.5})\n",
    "\n",
    "        # print the accuracy thus far\n",
    "        if (i+1)%50 == 0:\n",
    "            train_accuracy = accuracy.eval(feed_dict={\n",
    "                x:x_batch, y_: y_batch, keep_prob: 1.0})\n",
    "            print(\"step %d, training accuracy %.2f\"%(i, train_accuracy))\n",
    "\n",
    "\n",
    "    print(\"Optimization Finished!\")\n",
    "    train_accuracy = accuracy.eval(feed_dict={\n",
    "        x:x_batch, y_: y_batch, keep_prob: 1.0})\n",
    "    print(\"step %d, training accuracy %g\"%(i, train_accuracy))\n",
    "\n",
    "    # Save the variables to disk.\n",
    "    save_path = saver.save(sess, output_model)\n",
    "    print(\"Model saved in file: %s\" % save_path)\n",
    "\n",
    "    # Close the Session when we're done. If un-commented - need to run next bock of restore...\n",
    "    sess.close()   \n",
    "    print(\"#####################################################################\")\n",
    "    print(\"TRAINING ENDED\")\n",
    "    print(\"#####################################################################\")\n",
    "    print(\" \")\n",
    "    print(\" \")    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-20T05:59:13.878266Z",
     "start_time": "2019-10-20T05:59:13.873827Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# RUN pre_process - OLD?\n",
    "def pre_process(folder):\n",
    "    print(\"#####################################################################\")\n",
    "    print(\"PRE_PROCESS:\"+folder)\n",
    "    print(\"#####################################################################\")\n",
    "    result = []\n",
    "    \n",
    "    for root, dirs, files in os.walk(folder):\n",
    "        for file_ in files:\n",
    "            # Read the image\n",
    "            # image = img.imread(os.path.join(root, file_))\n",
    "            image = np.load(os.path.join(root, file_))\n",
    "            # import pdb; pdb.set_trace()\n",
    "            cubes = VAL_slice_to_static_slices(file_, image)\n",
    "            print(\"File: %s >>> cubes: %d\"%(file_, len(cubes)))\n",
    "            result.extend(cubes)\n",
    "    \n",
    "    return result\n",
    "    print(\"#####################################################################\")\n",
    "    print(\"PRE_PROCESS ENDED\")\n",
    "    print(\"#####################################################################\")\n",
    "    print(\" \")\n",
    "    print(\" \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-20T05:59:13.888879Z",
     "start_time": "2019-10-20T05:59:13.879782Z"
    },
    "code_folding": [
     2
    ]
   },
   "outputs": [],
   "source": [
    "# RUN pre_process_training - crop image, then tear it randomly to various tears, then per tear create cubes out of the edges, return cube set\n",
    "    # cols and rows are transposed from FRONT\n",
    "def pre_process_training(img_name, x_start=X_START, x_end=X_END, y_start=Y_START, y_end=Y_END, max_cols=4, max_rows=8):\n",
    "    print(\"#####################################################################\")\n",
    "    print(\"PRE_PROCESS:\"+img_name)\n",
    "    print(\"#####################################################################\")\n",
    "    short_name = img_name[:img_name.rfind('-D')]\n",
    "    image = read_and_crop(img_name, x_start, x_end, y_start, y_end)\n",
    "    \n",
    "    # need to rotate for FRONT\n",
    "    image = np.rot90(image)\n",
    "    result = []\n",
    "    \n",
    "    for root, dirs, files in os.walk(ROOT_FOLDER+fragments_folder):\n",
    "        for f in files:\n",
    "            os.unlink(os.path.join(root, f))\n",
    "\n",
    "    # cols and rows are transposed from FRONT\n",
    "    for col_cut in range(2, max_cols): # 9 3...10\n",
    "        for row_cut in range(3, max_rows): # 6 2...5\n",
    "            print(\"PRE_PROCESS:::\"+\"TEAR_\"+str(col_cut)+\"X\"+str(row_cut))\n",
    "            pieces = rough_tear_image(image, col_cut, row_cut)\n",
    "            \n",
    "            for piece in pieces:\n",
    "                # print(\"PRE_PROCESS:::\"+\"PIECE_\"+str(piece[\"col\"])+\"X\"+str(piece[\"row\"]))\n",
    "                fragment_name = short_name + \"_TEAR_\"+str(col_cut)+\"X\"+str(row_cut)+\"_PIECE_\"+str(piece[\"col\"])+\"X\"+str(piece[\"row\"])\n",
    "                fragment_file_name = short_name + \"_\"+str(col_cut)+\"X\"+str(row_cut)+\"_\"+str(piece[\"col\"])+\"X\"+str(piece[\"row\"])\n",
    "                # import pdb; pdb.set_trace()\n",
    "                plt.imsave(os.path.join(ROOT_FOLDER+fragments_folder,fragment_file_name+\".jpg\"), piece[\"cut\"], cmap=plt.cm.gray)\n",
    "                cubes = VAL_slice_TEAR_to_static_slices(fragment_name, piece)\n",
    "                for cube in cubes:\n",
    "                    cube[\"tear\"] = str(col_cut)+\"X\"+str(row_cut)\n",
    "                    cube[\"piece_col\"] = piece[\"col\"]\n",
    "                    cube[\"piece_row\"] = piece[\"row\"]\n",
    "                # print(\"File: %s >>> cubes: %d\"%(file_, len(cubes)))\n",
    "                result.extend(cubes)\n",
    "    \n",
    "    return result\n",
    "    print(\"#####################################################################\")\n",
    "    print(\"PRE_PROCESS ENDED\")\n",
    "    print(\"#####################################################################\")\n",
    "    print(\" \")\n",
    "    print(\" \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-20T05:59:13.894221Z",
     "start_time": "2019-10-20T05:59:13.891290Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "def validate1(cubes, model, slice_size, folder, curr_cube):    \n",
    "    # VALIDATE prepare the data sets\n",
    "    test_imgs, test_x_delta, test_y_delta, test_x_file, test_y_file = VAL_build_train_set(cubes, slice_size, folder, curr_cube)\n",
    "    print(\"loaded %d images\"%(len(test_imgs)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-20T05:59:13.909562Z",
     "start_time": "2019-10-20T05:59:13.896635Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "def validate2(folder, model, slice_size):\n",
    "    test_imgs = []\n",
    "    test_x_file = []\n",
    "    test_y_file = []\n",
    "    the_root = \"\"\n",
    "    for root, dirs, files in os.walk(folder):\n",
    "        the_root = root\n",
    "        for file_ in files:\n",
    "            test_imgs.append( os.path.join(root, file_) )\n",
    "            test_x_file.append(file_[:file_.rfind('---P')])\n",
    "            test_y_file.append(file_[file_.rfind('---P')+3:])\n",
    "            \n",
    "    print(len(test_imgs))\n",
    "    \n",
    "    # VALIDATE Prepare a test session \n",
    "\n",
    "    # Add ops to save and restore all the variables.\n",
    "    saver = tf.train.Saver()\n",
    "\n",
    "    # start session and initialize variables\n",
    "    sess = tf.InteractiveSession()\n",
    "\n",
    "    # Restore variables from disk.\n",
    "    saver.restore(sess, model)\n",
    "    print(\"Model restored.\")\n",
    "\n",
    "    # VALIDATE Validate the model\n",
    "\n",
    "    # import pdb; pdb.set_trace()\n",
    "    v1t = []\n",
    "    count = 0\n",
    "    length = len(test_imgs)\n",
    "    batch = 100\n",
    "    x_batch = []\n",
    "    # change the ranges in the loop below - first number is the start point (multiplied by batch size)\n",
    "    # second number is the end point (multiplied by batch size)\n",
    "    # third number is the jump from batch to batch\n",
    "    # use the length about to set the batch length\n",
    "    for start in range(0, length, batch):\n",
    "        for i in range(start, start+batch):\n",
    "            if (i < length):\n",
    "                image = np.load(test_imgs[i])\n",
    "                x_batch.append(image)\n",
    "                count += 1\n",
    "\n",
    "        # print(\"Validating start at #%d end at %d\"%(start*batch,(start+length)*batch))\n",
    "        my_prediction=tf.argmax(y_conv,1)\n",
    "        v1 = my_prediction.eval(feed_dict={x:x_batch, keep_prob: 1.0})\n",
    "        v1t = np.concatenate((v1t, v1), axis=0)\n",
    "        x_batch = []\n",
    "        print(\">>> step %d\"%(start+batch))\n",
    "\n",
    "\n",
    "    match_indexes = np.nonzero(v1t)[0]\n",
    "    A = np.array(test_x_file)\n",
    "    B = np.array(test_y_file)\n",
    "    C = np.array(test_imgs)\n",
    "    match_x_files = A[match_indexes]\n",
    "    match_y_files = B[match_indexes]\n",
    "    match_images = C[match_indexes]\n",
    "    \n",
    "    for matched_img in match_images:\n",
    "        load_img = np.load(matched_img)\n",
    "        plt.imsave(os.path.join(\"/Volumes/250GB/matched/\",matched_img[matched_img.rfind('/')+1:]+\".png\"), load_img, cmap=plt.cm.gray)\n",
    "    \n",
    "    for root, dirs, files in os.walk(folder):\n",
    "        for file_ in files:\n",
    "            os.remove( os.path.join(root, file_) ) # delete it from the FS\n",
    "                \n",
    "    with open('matches.csv', 'a') as csvfile:\n",
    "        csvout = csv.writer(csvfile)\n",
    "        for match_index in match_indexes:\n",
    "            print(\"MATCH %s === %s\"%(test_x_file[match_index], test_y_file[match_index]))\n",
    "            # print(\"MATCH %s === %s\"%(A[match_index], B[match_index]))\n",
    "            # csvout.writerow([A[match_index], B[match_index]])\n",
    "            csvout.writerow([test_x_file[match_index], test_y_file[match_index]])\n",
    "            # plt.imsave(\"match_\"+match_index+\".jpg\", C[match_index])\n",
    "\n",
    "    # Close the Session when we're done.\n",
    "    sess.close()   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-20T05:59:13.924379Z",
     "start_time": "2019-10-20T05:59:13.910902Z"
    },
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "def validate2_for_cross_validation(test_imgs, test_lbls, is_enriched, model, max_samples=0):\n",
    "    print(len(test_imgs))\n",
    "    \n",
    "    # VALIDATE Prepare a test session \n",
    "\n",
    "    # Add ops to save and restore all the variables.\n",
    "    saver = tf.train.Saver()\n",
    "\n",
    "    # start session and initialize variables\n",
    "    sess = tf.InteractiveSession()\n",
    "\n",
    "    # Restore variables from disk.\n",
    "    saver.restore(sess, model)\n",
    "    print(\"Model restored.\")\n",
    "\n",
    "    # VALIDATE Validate the model\n",
    "\n",
    "    count = 0\n",
    "    se = 0\n",
    "    st = 0\n",
    "    v1t = []\n",
    "    v2t = []\n",
    "    v1tt = []\n",
    "    v2tt = []\n",
    "    length = len(test_imgs)\n",
    "    if max_samples != 0:\n",
    "        length = max_samples\n",
    "    batch = 100\n",
    "    x_batch = []\n",
    "    y_batch = []\n",
    "    # change the ranges in the loop below - first number is the start point (multiplied by batch size)\n",
    "    # second number is the end point (multiplied by batch size)\n",
    "    # third number is the jump from batch to batch\n",
    "    # use the length about to set the batch length\n",
    "    for start in range(0, length, batch):\n",
    "        for i in range(start, start+batch):\n",
    "            if (i < length):\n",
    "                image = np.load(test_imgs[i]+\".npy\")\n",
    "                x_batch.append(image)\n",
    "                y_batch.append(train_lbls[i])                \n",
    "                \n",
    "        # print the accuracy thus far\n",
    "    #         train_accuracy = accuracy.eval(feed_dict={\n",
    "    #             x:x_batch, y_: y_batch, keep_prob: 1.0})\n",
    "    #         print(\"step %d, training accuracy %g\"%(i, train_accuracy))\n",
    "\n",
    "        # print(\"Validating start at #%d end at %d\"%(start*batch,(start+length)*batch))\n",
    "    #         my_prediction=tf.argmax(y_conv,1)\n",
    "    #         v1 = my_prediction.eval(feed_dict={x:x_batch, keep_prob: 1.0})\n",
    "    #         v1t = np.concatenate((v1t, v1), axis=0)\n",
    "\n",
    "        ######## printing the predictions and their normalized values\n",
    "        # print(\"y_conv=\"+str(y_conv.eval(feed_dict={x:x_batch, y_: y_batch, keep_prob: 1.0})))\n",
    "        # print(\"probability=\"+str(probability.eval(feed_dict={x:x_batch, y_: y_batch, keep_prob: 1.0})))\n",
    "        # print(\"probabilities=\"+str(probabilities.eval(feed_dict={x:x_batch, y_: y_batch, keep_prob: 1.0})))\n",
    "        \n",
    "        my_prediction=tf.argmax(y_conv,1)\n",
    "        my_target=tf.argmax(y_,1)\n",
    "        v1 = my_prediction.eval(feed_dict={x:x_batch, y_: y_batch, keep_prob: 1.0})\n",
    "        v2 = my_target.eval(feed_dict={x:x_batch, y_: y_batch, keep_prob: 1.0})\n",
    "        v1t = np.concatenate((v1t, v1), axis=0)\n",
    "        v2t = np.concatenate((v2t, v2), axis=0)\n",
    "\n",
    "        c1 = np.sum(np.absolute(np.subtract(v2, v1)))\n",
    "        c2 = np.sum(np.absolute(v2))\n",
    "        se += c1\n",
    "        st += c2\n",
    "\n",
    "        x_batch = []\n",
    "        y_batch = []\n",
    "        print(\">>> step %d\"%(start+batch))\n",
    "        \n",
    "        count += ((i+1) - start)\n",
    "        precision, recall, f_score, support = precision_recall_fscore_support(v2t, v1t, average='binary')\n",
    "        print(\"step %d-%d, precision %f, recall %f, f_score %f\"%(start, i, precision, recall, f_score))\n",
    "        # print(\"Accumulated total true = %d\"%(st));\n",
    "        # print(\"Accumulated total error rate = %f\"%(se/count));\n",
    "        # v1tt = np.concatenate((v1tt, v1t), axis=0)\n",
    "        # v2tt = np.concatenate((v2tt, v2t), axis=0)\n",
    "        print(\"=== total %d match %d\"%(count, len(np.nonzero(v1t)[0])))\n",
    "\n",
    "    precision, recall, f_score, support = precision_recall_fscore_support(v2t, v1t, average='binary')\n",
    "    print(\"TOTAL %d, precision %f, recall %f, f_score %f\"%(count, precision, recall, f_score))\n",
    "    print(\"TOTAL true = %d\"%(st));\n",
    "    print(\"TOTAL error rate = %f\"%(se/count));\n",
    "    \n",
    "    match_indexes = np.nonzero(v1t)[0]\n",
    "    C = np.array(test_imgs)\n",
    "    match_images = C[match_indexes]\n",
    "    \n",
    "    # for matched_img in match_images:\n",
    "    #    load_img = np.load(matched_img+\".npy\")\n",
    "    #    plt.imsave(os.path.join(ROOT_FOLDER+\"synt_matched/\",matched_img[matched_img.rfind('/')+1:]+\".png\"), load_img, cmap=plt.cm.gray)\n",
    "\n",
    "    with open(strftime(\"%Y%m%d_%H%M%S\", gmtime())+'_synt_all.csv', 'a') as csvfile:\n",
    "        csvout = csv.writer(csvfile)\n",
    "        for idx, test_img in enumerate(test_imgs):\n",
    "            # print(\"MATCH %s === %s\"%(test_imgs[match_index], train_lbls[match_index]))\n",
    "            match_class = 0\n",
    "            if idx in match_indexes:\n",
    "                match_class = 1\n",
    "            csvout.writerow([test_img, train_lbls[idx], match_class, is_enriched[idx]])\n",
    "            # plt.imsave(\"match_\"+match_index+\".jpg\", C[match_index])\n",
    "\n",
    "    with open(strftime(\"%Y%m%d_%H%M%S\", gmtime())+'_synt_matches.csv', 'a') as csvfile:\n",
    "        csvout = csv.writer(csvfile)\n",
    "        for match_index in match_indexes:\n",
    "            # print(\"MATCH %s === %s\"%(test_imgs[match_index], train_lbls[match_index]))\n",
    "            csvout.writerow([test_imgs[match_index], train_lbls[match_index], is_enriched[match_index]])\n",
    "            # plt.imsave(\"match_\"+match_index+\".jpg\", C[match_index])\n",
    "\n",
    "    # Close the Session when we're done.\n",
    "    sess.close()   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-20T05:59:13.930448Z",
     "start_time": "2019-10-20T05:59:13.925711Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "def iter_validate(cubes, model, slice_size, folder):\n",
    "    print(\"#####################################################################\")\n",
    "    print(\"VALIDATING\")\n",
    "    print(\"#####################################################################\")\n",
    "    cubes_len = len(cubes)\n",
    "    batch_size = 100\n",
    "    count = 0\n",
    "    # iterate over the cubes\n",
    "    for curr in cubes:\n",
    "        count += 1\n",
    "        if count < batch_size: ### TEMP LIMITATION\n",
    "            print(\"CUBE:%s\"%(curr[\"file\"]+\"_\"+str(curr[\"top_row\"])+\"_\"+str(curr[\"left_col\"])))\n",
    "            validate1(cubes, model, slice_size, folder, curr)\n",
    "            validate2(folder, model, slice_size)\n",
    "        \n",
    "    print(\"#####################################################################\")\n",
    "    print(\"VALIDATION ENDED\")\n",
    "    print(\"#####################################################################\")\n",
    "    print(\" \")\n",
    "    print(\" \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-20T05:59:13.935644Z",
     "start_time": "2019-10-20T05:59:13.931851Z"
    },
    "code_folding": [
     0
    ],
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def run_all(folder, model, slice_size):\n",
    "    model_tf(slice_size)\n",
    "    cubes_set = pre_process(folder)\n",
    "    validate(cubes_set, model, slice_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-20T05:59:13.939703Z",
     "start_time": "2019-10-20T05:59:13.937332Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# HELPER block\n",
    "# image = read_and_crop(\"PX303/FG001/PX303-Fg001-V-C01-R01-D05032015-T112602-ML924__012.jpg\")\n",
    "## image = read_and_crop(\"PX303/FG004/PX303-Fg004-V-C01-R01-D08032015-T110900-ML924__012.jpg\", 100, -1, 400, -1)\n",
    "# image = read_and_crop(\"PX303/FG004/PX303-Fg004-V-C01-R02-D08032015-T105147-ML924__012.jpg\")\n",
    "# image = read_and_crop(\"PX303/FG004/PX303-Fg004-V-C02-R01-D08032015-T110025-ML924__012.jpg\")\n",
    "# image = read_and_crop(\"PX303/FG004/PX303-Fg004-V-C02-R02-D08032015-T105553-ML924__012.jpg\")\n",
    "# image = read_and_crop(\"PX303/FG006/PX303-Fg006-V-C01-R01-D08032015-T120605-ML924__012.jpg\")\n",
    "# image = read_and_crop(\"PX303/FG006/PX303-Fg006-V-C01-R02-D08032015-T115230-ML924__012.jpg\")\n",
    "# image = read_and_crop(\"PX303/FG006/PX303-Fg006-V-C02-R01-D08032015-T120158-ML924__012.jpg\")\n",
    "##image = read_and_crop(\"PX303/FG006/PX303-Fg006-V-C02-R02-D08032015-T115704-ML924__012.jpg\", 0, 6200, 0, 4400)\n",
    "##plt.imshow(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-20T05:59:13.945862Z",
     "start_time": "2019-10-20T05:59:13.941216Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "def load_train_from_disk(path):\n",
    "    train_imgs = []\n",
    "    train_lbls = []\n",
    "    is_enriched = []\n",
    "    for root, dirs, files in os.walk(path):\n",
    "        for file_ in files:\n",
    "            file_name = os.path.join(root, file_)\n",
    "            file_name = file_name[:file_name.rfind(\".\")]\n",
    "            train_imgs.append(file_name)\n",
    "    #        train_lbls.append([1,0] if file_.startswith(\"0=\") else [0,1])\n",
    "    # return train_imgs, train_lbls\n",
    "            enriched = file_[0] == '1'\n",
    "            is_enriched.append(enriched)\n",
    "            label = [0,1] if file_[2] == '1' else [1,0] \n",
    "            train_lbls.append(label)\n",
    "    return train_imgs, train_lbls, is_enriched"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-18T12:23:16.497307Z",
     "start_time": "2019-10-18T12:23:16.493573Z"
    }
   },
   "outputs": [],
   "source": [
    "##########################################\n",
    "########################################## RUN ALL ABOVE !!! Then select the relevant code below\n",
    "##########################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-30T04:59:30.940130Z",
     "start_time": "2019-09-30T04:33:29.673860Z"
    },
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#####################################################################\n",
      "PRE_PROCESS:PX303-Fg001-R-C01-R01-D05032015-T112011-ML638__006.jpg\n",
      "#####################################################################\n",
      "PRE_PROCESS:::TEAR_2X3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/util/dtype.py:141: UserWarning: Possible precision loss when converting from float64 to uint16\n",
      "  .format(dtypeobj_in, dtypeobj_out))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PRE_PROCESS:::TEAR_2X4\n",
      "PRE_PROCESS:::TEAR_2X5\n",
      "PRE_PROCESS:::TEAR_2X6\n",
      "PRE_PROCESS:::TEAR_2X7\n",
      "PRE_PROCESS:::TEAR_3X3\n",
      "PRE_PROCESS:::TEAR_3X4\n",
      "PRE_PROCESS:::TEAR_3X5\n",
      "PRE_PROCESS:::TEAR_3X6\n",
      "PRE_PROCESS:::TEAR_3X7\n",
      "*** CUBES=100\n",
      " MATCHED=3189\n",
      " NOT MATCHED=3345\n",
      " DISCARDED=12390\n",
      " SKIPPED=334917\n",
      "*** CUBES=200\n",
      " MATCHED=5172\n",
      " NOT MATCHED=5507\n",
      " DISCARDED=20405\n",
      " SKIPPED=678977\n",
      "*** CUBES=300\n",
      " MATCHED=5172\n",
      " NOT MATCHED=5507\n",
      " DISCARDED=20405\n",
      " SKIPPED=1033277\n",
      "*** CUBES=400\n",
      " MATCHED=6973\n",
      " NOT MATCHED=7699\n",
      " DISCARDED=28938\n",
      " SKIPPED=1376785\n",
      "*** CUBES=500\n",
      " MATCHED=9162\n",
      " NOT MATCHED=9916\n",
      " DISCARDED=37578\n",
      " SKIPPED=1720151\n",
      "*** CUBES=600\n",
      " MATCHED=9162\n",
      " NOT MATCHED=9916\n",
      " DISCARDED=37578\n",
      " SKIPPED=2074451\n",
      "*** CUBES=700\n",
      " MATCHED=11374\n",
      " NOT MATCHED=12410\n",
      " DISCARDED=46417\n",
      " SKIPPED=2417330\n",
      "*** CUBES=800\n",
      " MATCHED=13046\n",
      " NOT MATCHED=14565\n",
      " DISCARDED=54068\n",
      " SKIPPED=2761760\n",
      "*** CUBES=900\n",
      " MATCHED=13046\n",
      " NOT MATCHED=14565\n",
      " DISCARDED=54068\n",
      " SKIPPED=3116060\n",
      "*** CUBES=1000\n",
      " MATCHED=15033\n",
      " NOT MATCHED=17482\n",
      " DISCARDED=63164\n",
      " SKIPPED=3458256\n",
      "*** CUBES=1100\n",
      " MATCHED=16076\n",
      " NOT MATCHED=19055\n",
      " DISCARDED=68072\n",
      " SKIPPED=3806028\n",
      "*** CUBES=1200\n",
      " MATCHED=16121\n",
      " NOT MATCHED=19471\n",
      " DISCARDED=69573\n",
      " SKIPPED=4158408\n",
      "*** CUBES=1300\n",
      " MATCHED=17730\n",
      " NOT MATCHED=22054\n",
      " DISCARDED=78905\n",
      " SKIPPED=4500708\n",
      "*** CUBES=1400\n",
      " MATCHED=18066\n",
      " NOT MATCHED=22441\n",
      " DISCARDED=80300\n",
      " SKIPPED=4853208\n",
      "*** CUBES=1500\n",
      " MATCHED=20054\n",
      " NOT MATCHED=26785\n",
      " DISCARDED=96363\n",
      " SKIPPED=5187033\n",
      "*** CUBES=1600\n",
      " MATCHED=22898\n",
      " NOT MATCHED=33044\n",
      " DISCARDED=119486\n",
      " SKIPPED=5511861\n",
      "*** CUBES=1700\n",
      " MATCHED=25301\n",
      " NOT MATCHED=36226\n",
      " DISCARDED=130823\n",
      " SKIPPED=5851561\n",
      "*** CUBES=1800\n",
      " MATCHED=26951\n",
      " NOT MATCHED=38038\n",
      " DISCARDED=137279\n",
      " SKIPPED=6197539\n",
      "*** CUBES=1900\n",
      " MATCHED=26951\n",
      " NOT MATCHED=38038\n",
      " DISCARDED=137279\n",
      " SKIPPED=6551839\n",
      "*** CUBES=2000\n",
      " MATCHED=29168\n",
      " NOT MATCHED=44327\n",
      " DISCARDED=159815\n",
      " SKIPPED=6877233\n",
      "*** CUBES=2100\n",
      " MATCHED=31172\n",
      " NOT MATCHED=48975\n",
      " DISCARDED=176576\n",
      " SKIPPED=7210046\n",
      "*** CUBES=2200\n",
      " MATCHED=33834\n",
      " NOT MATCHED=51855\n",
      " DISCARDED=187102\n",
      " SKIPPED=7550846\n",
      "*** CUBES=2300\n",
      " MATCHED=34335\n",
      " NOT MATCHED=52200\n",
      " DISCARDED=188362\n",
      " SKIPPED=7903526\n",
      "*** CUBES=2400\n",
      " MATCHED=35362\n",
      " NOT MATCHED=55477\n",
      " DISCARDED=199829\n",
      " SKIPPED=8243039\n",
      "*** CUBES=2500\n",
      " MATCHED=37470\n",
      " NOT MATCHED=61133\n",
      " DISCARDED=219539\n",
      " SKIPPED=8571887\n",
      "*** CUBES=2600\n",
      " MATCHED=39633\n",
      " NOT MATCHED=64015\n",
      " DISCARDED=229170\n",
      " SKIPPED=8913587\n",
      "*** CUBES=2700\n",
      " MATCHED=40507\n",
      " NOT MATCHED=65081\n",
      " DISCARDED=232732\n",
      " SKIPPED=9263225\n",
      "*** CUBES=2800\n",
      " MATCHED=41274\n",
      " NOT MATCHED=67515\n",
      " DISCARDED=240179\n",
      " SKIPPED=9607609\n",
      "*** CUBES=2900\n",
      " MATCHED=43789\n",
      " NOT MATCHED=74015\n",
      " DISCARDED=260072\n",
      " SKIPPED=9935401\n",
      "*** CUBES=3000\n",
      " MATCHED=45379\n",
      " NOT MATCHED=76997\n",
      " DISCARDED=269218\n",
      " SKIPPED=10277501\n",
      "*** CUBES=3100\n",
      " MATCHED=46844\n",
      " NOT MATCHED=78303\n",
      " DISCARDED=273213\n",
      " SKIPPED=10626433\n",
      "*** CUBES=3200\n",
      " MATCHED=47358\n",
      " NOT MATCHED=80066\n",
      " DISCARDED=279820\n",
      " SKIPPED=10972335\n",
      "*** CUBES=3300\n",
      " MATCHED=49034\n",
      " NOT MATCHED=85165\n",
      " DISCARDED=298927\n",
      " SKIPPED=11302337\n",
      "*** CUBES=3400\n",
      " MATCHED=50362\n",
      " NOT MATCHED=87549\n",
      " DISCARDED=307769\n",
      " SKIPPED=11645337\n",
      "*** CUBES=3500\n",
      " MATCHED=51016\n",
      " NOT MATCHED=88287\n",
      " DISCARDED=310498\n",
      " SKIPPED=11996134\n",
      "*** MATCHED=51016\n",
      "*** NOT MATCHED=88287\n",
      "*** DISCARDED=310498\n",
      "*** SKIPPED=12152026\n",
      "WARNING:tensorflow:From <ipython-input-6-6d6c00010ca0>:97: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
      "\n",
      "#####################################################################\n",
      "TRAINING:\n",
      "MODEL:/media/1KGB_ILAN/papyrus/models/front_tear_model1.ckpt\n",
      "#####################################################################\n",
      "WARNING:tensorflow:From /home/ilan/venv35/lib/python3.5/site-packages/tensorflow/python/util/tf_should_use.py:189: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n",
      "Instructions for updating:\n",
      "Use `tf.global_variables_initializer` instead.\n",
      "step 49, training accuracy 0.65\n",
      "step 99, training accuracy 0.68\n",
      "step 149, training accuracy 0.73\n",
      "step 199, training accuracy 0.68\n",
      "step 249, training accuracy 0.67\n",
      "step 299, training accuracy 0.80\n",
      "step 349, training accuracy 0.78\n",
      "step 399, training accuracy 0.82\n",
      "step 449, training accuracy 0.78\n",
      "step 499, training accuracy 0.77\n",
      "step 549, training accuracy 0.78\n",
      "step 599, training accuracy 0.77\n",
      "step 649, training accuracy 0.78\n",
      "step 699, training accuracy 0.80\n",
      "step 749, training accuracy 0.78\n",
      "step 799, training accuracy 0.80\n",
      "step 849, training accuracy 0.85\n",
      "step 899, training accuracy 0.85\n",
      "step 949, training accuracy 0.82\n",
      "step 999, training accuracy 0.80\n",
      "step 1049, training accuracy 0.82\n",
      "step 1099, training accuracy 0.80\n",
      "step 1149, training accuracy 0.90\n",
      "step 1199, training accuracy 0.87\n",
      "step 1249, training accuracy 0.87\n",
      "step 1299, training accuracy 0.92\n",
      "step 1349, training accuracy 0.90\n",
      "step 1399, training accuracy 0.85\n",
      "step 1449, training accuracy 0.85\n",
      "step 1499, training accuracy 0.97\n",
      "step 1549, training accuracy 0.95\n",
      "step 1599, training accuracy 0.92\n",
      "step 1649, training accuracy 0.97\n",
      "step 1699, training accuracy 0.97\n",
      "step 1749, training accuracy 0.90\n",
      "step 1799, training accuracy 0.90\n",
      "step 1849, training accuracy 0.95\n",
      "step 1899, training accuracy 0.92\n",
      "step 1949, training accuracy 0.93\n",
      "step 1999, training accuracy 0.90\n",
      "Optimization Finished!\n",
      "step 1999, training accuracy 0.9\n",
      "Model saved in file: /media/1KGB_ILAN/papyrus/models/front_tear_model1.ckpt\n",
      "#####################################################################\n",
      "TRAINING ENDED\n",
      "#####################################################################\n",
      " \n",
      " \n"
     ]
    }
   ],
   "source": [
    "# RUN1 - take 1st large pieces and train on it\n",
    "cubes_set = pre_process_training(\"PX303-Fg001-R-C01-R01-D05032015-T112011-ML638__006.jpg\")\n",
    "train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "    NEW_build_train_set_for_binary_labeling(cubes_set, cubes_set, CUBE_SIZE, ROOT_FOLDER + \"train_concats3/\", 12, 7)\n",
    "tf.reset_default_graph()\n",
    "model_tf_deep(250)\n",
    "train(train_imgs, train_lbls, ROOT_FOLDER + \"models/front_tear_model1.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T14:21:01.997116Z",
     "start_time": "2019-04-03T14:16:57.383608Z"
    },
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# RE-RUN1 - take 1st large pieces and train on it\n",
    "train_imgs, train_lbls, is_enriched = \\\n",
    "    load_train_from_disk(ROOT_FOLDER + \"train_concats3/\")\n",
    "tf.reset_default_graph()\n",
    "model_tf_deep(250)\n",
    "train(train_imgs, train_lbls, ROOT_FOLDER + \"model_binary/tear_model1.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-01T06:55:34.462694Z",
     "start_time": "2019-04-01T06:55:34.270110Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# TEMP DEBUG\n",
    "len_cubes = len(cubes_set)\n",
    "print(len_cubes)\n",
    "# bat_cubes = 1000\n",
    "# BATCHES = 500 # smaller train batches\n",
    "# for start in range(0, len_cubes, bat_cubes):\n",
    "#     end = ((start+bat_cubes) if start+bat_cubes < len_cubes else (len_cubes - 1))\n",
    "#     print(\"start:%s end:%s\", start, end)\n",
    "train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "    NEW_build_train_set_for_binary_labeling(cubes_set[1:2], cubes_set, CUBE_SIZE, \n",
    "                                            ROOT_FOLDER + \"train_concats2/\", 1, 5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-30T05:35:05.952850Z",
     "start_time": "2019-09-30T04:59:30.942012Z"
    },
    "code_folding": [],
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#####################################################################\n",
      "PRE_PROCESS:PX303-Fg004-R-C01-R01-D05032015-T115135-ML638__006.jpg\n",
      "#####################################################################\n",
      "PRE_PROCESS:::TEAR_2X3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/util/dtype.py:141: UserWarning: Possible precision loss when converting from float64 to uint16\n",
      "  .format(dtypeobj_in, dtypeobj_out))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PRE_PROCESS:::TEAR_2X4\n",
      "PRE_PROCESS:::TEAR_2X5\n",
      "PRE_PROCESS:::TEAR_2X6\n",
      "PRE_PROCESS:::TEAR_2X7\n",
      "PRE_PROCESS:::TEAR_3X3\n",
      "PRE_PROCESS:::TEAR_3X4\n",
      "PRE_PROCESS:::TEAR_3X5\n",
      "PRE_PROCESS:::TEAR_3X6\n",
      "PRE_PROCESS:::TEAR_3X7\n",
      "*** CUBES=100\n",
      " MATCHED=2093\n",
      " NOT MATCHED=4737\n",
      " DISCARDED=17962\n",
      " SKIPPED=492921\n",
      "*** CUBES=200\n",
      " MATCHED=5180\n",
      " NOT MATCHED=9517\n",
      " DISCARDED=36083\n",
      " SKIPPED=990821\n",
      "*** CUBES=300\n",
      " MATCHED=5446\n",
      " NOT MATCHED=9996\n",
      " DISCARDED=37896\n",
      " SKIPPED=1509421\n",
      "*** CUBES=400\n",
      " MATCHED=5446\n",
      " NOT MATCHED=9996\n",
      " DISCARDED=37896\n",
      " SKIPPED=2030321\n",
      "*** CUBES=500\n",
      " MATCHED=6397\n",
      " NOT MATCHED=12751\n",
      " DISCARDED=47948\n",
      " SKIPPED=2538381\n",
      "*** CUBES=600\n",
      " MATCHED=9472\n",
      " NOT MATCHED=17329\n",
      " DISCARDED=64671\n",
      " SKIPPED=3037881\n",
      "*** CUBES=700\n",
      " MATCHED=10875\n",
      " NOT MATCHED=19481\n",
      " DISCARDED=72530\n",
      " SKIPPED=3548723\n",
      "*** CUBES=800\n",
      " MATCHED=10875\n",
      " NOT MATCHED=19481\n",
      " DISCARDED=72530\n",
      " SKIPPED=4069623\n",
      "*** CUBES=900\n",
      " MATCHED=11597\n",
      " NOT MATCHED=21348\n",
      " DISCARDED=78944\n",
      " SKIPPED=4582216\n",
      "*** CUBES=1000\n",
      " MATCHED=13923\n",
      " NOT MATCHED=26130\n",
      " DISCARDED=95374\n",
      " SKIPPED=5081816\n",
      "*** CUBES=1100\n",
      " MATCHED=15620\n",
      " NOT MATCHED=29044\n",
      " DISCARDED=105394\n",
      " SKIPPED=5589723\n",
      "*** CUBES=1200\n",
      " MATCHED=15620\n",
      " NOT MATCHED=29044\n",
      " DISCARDED=105394\n",
      " SKIPPED=6110623\n",
      "*** CUBES=1300\n",
      " MATCHED=16019\n",
      " NOT MATCHED=30263\n",
      " DISCARDED=109412\n",
      " SKIPPED=6626271\n",
      "*** CUBES=1400\n",
      " MATCHED=18077\n",
      " NOT MATCHED=34942\n",
      " DISCARDED=124849\n",
      " SKIPPED=7126971\n",
      "*** CUBES=1500\n",
      " MATCHED=19823\n",
      " NOT MATCHED=38123\n",
      " DISCARDED=135338\n",
      " SKIPPED=7634135\n",
      "*** CUBES=1600\n",
      " MATCHED=19823\n",
      " NOT MATCHED=38123\n",
      " DISCARDED=135338\n",
      " SKIPPED=8155035\n",
      "*** CUBES=1700\n",
      " MATCHED=20130\n",
      " NOT MATCHED=39440\n",
      " DISCARDED=139948\n",
      " SKIPPED=8669995\n",
      "*** CUBES=1800\n",
      " MATCHED=22282\n",
      " NOT MATCHED=43821\n",
      " DISCARDED=155273\n",
      " SKIPPED=9171095\n",
      "*** CUBES=1900\n",
      " MATCHED=23596\n",
      " NOT MATCHED=46449\n",
      " DISCARDED=164471\n",
      " SKIPPED=9680115\n",
      "*** CUBES=2000\n",
      " MATCHED=23596\n",
      " NOT MATCHED=46449\n",
      " DISCARDED=164471\n",
      " SKIPPED=10201015\n",
      "*** CUBES=2100\n",
      " MATCHED=24487\n",
      " NOT MATCHED=50476\n",
      " DISCARDED=179611\n",
      " SKIPPED=10702721\n",
      "*** CUBES=2200\n",
      " MATCHED=27366\n",
      " NOT MATCHED=60057\n",
      " DISCARDED=215641\n",
      " SKIPPED=11177921\n",
      "*** CUBES=2300\n",
      " MATCHED=30023\n",
      " NOT MATCHED=68488\n",
      " DISCARDED=247307\n",
      " SKIPPED=11658641\n",
      "*** CUBES=2400\n",
      " MATCHED=32351\n",
      " NOT MATCHED=73273\n",
      " DISCARDED=265150\n",
      " SKIPPED=12156841\n",
      "*** CUBES=2500\n",
      " MATCHED=35602\n",
      " NOT MATCHED=78051\n",
      " DISCARDED=282971\n",
      " SKIPPED=12655041\n",
      "*** CUBES=2600\n",
      " MATCHED=35720\n",
      " NOT MATCHED=78338\n",
      " DISCARDED=284042\n",
      " SKIPPED=13174579\n",
      "*** CUBES=2700\n",
      " MATCHED=35720\n",
      " NOT MATCHED=78338\n",
      " DISCARDED=284042\n",
      " SKIPPED=13695479\n",
      "*** CUBES=2800\n",
      " MATCHED=37012\n",
      " NOT MATCHED=84424\n",
      " DISCARDED=306990\n",
      " SKIPPED=14187301\n",
      "*** CUBES=2900\n",
      " MATCHED=40030\n",
      " NOT MATCHED=93506\n",
      " DISCARDED=341212\n",
      " SKIPPED=14664801\n",
      "*** CUBES=3000\n",
      " MATCHED=42136\n",
      " NOT MATCHED=100107\n",
      " DISCARDED=366275\n",
      " SKIPPED=15153965\n",
      "*** CUBES=3100\n",
      " MATCHED=44823\n",
      " NOT MATCHED=104591\n",
      " DISCARDED=383502\n",
      " SKIPPED=15653065\n",
      "*** CUBES=3200\n",
      " MATCHED=46658\n",
      " NOT MATCHED=107368\n",
      " DISCARDED=394182\n",
      " SKIPPED=16160449\n",
      "*** CUBES=3300\n",
      " MATCHED=46658\n",
      " NOT MATCHED=107368\n",
      " DISCARDED=394182\n",
      " SKIPPED=16681349\n",
      "*** CUBES=3400\n",
      " MATCHED=46958\n",
      " NOT MATCHED=109345\n",
      " DISCARDED=400693\n",
      " SKIPPED=17193749\n",
      "*** CUBES=3500\n",
      " MATCHED=49779\n",
      " NOT MATCHED=119225\n",
      " DISCARDED=433216\n",
      " SKIPPED=17672149\n",
      "*** CUBES=3600\n",
      " MATCHED=52304\n",
      " NOT MATCHED=128204\n",
      " DISCARDED=462796\n",
      " SKIPPED=18154401\n",
      "*** CUBES=3700\n",
      " MATCHED=54417\n",
      " NOT MATCHED=133087\n",
      " DISCARDED=478934\n",
      " SKIPPED=18654201\n",
      "*** CUBES=3800\n",
      " MATCHED=57114\n",
      " NOT MATCHED=137771\n",
      " DISCARDED=494413\n",
      " SKIPPED=19154845\n",
      "*** CUBES=3900\n",
      " MATCHED=57114\n",
      " NOT MATCHED=137771\n",
      " DISCARDED=494413\n",
      " SKIPPED=19675745\n",
      "*** CUBES=4000\n",
      " MATCHED=57114\n",
      " NOT MATCHED=137771\n",
      " DISCARDED=494413\n",
      " SKIPPED=20196645\n",
      "*** CUBES=4100\n",
      " MATCHED=59315\n",
      " NOT MATCHED=146402\n",
      " DISCARDED=523550\n",
      " SKIPPED=20679694\n",
      "*** CUBES=4200\n",
      " MATCHED=61809\n",
      " NOT MATCHED=155680\n",
      " DISCARDED=554878\n",
      " SKIPPED=21159894\n",
      "*** CUBES=4300\n",
      " MATCHED=63269\n",
      " NOT MATCHED=160454\n",
      " DISCARDED=571378\n",
      " SKIPPED=21659458\n",
      "*** CUBES=4400\n",
      " MATCHED=65899\n",
      " NOT MATCHED=164934\n",
      " DISCARDED=586900\n",
      " SKIPPED=22160258\n",
      "*** CUBES=4500\n",
      " MATCHED=66345\n",
      " NOT MATCHED=165471\n",
      " DISCARDED=588761\n",
      " SKIPPED=22678746\n",
      "*** CUBES=4600\n",
      " MATCHED=66345\n",
      " NOT MATCHED=165471\n",
      " DISCARDED=588761\n",
      " SKIPPED=23199646\n",
      "*** CUBES=4700\n",
      " MATCHED=68066\n",
      " NOT MATCHED=172764\n",
      " DISCARDED=616197\n",
      " SKIPPED=23685746\n",
      "*** CUBES=4800\n",
      " MATCHED=70395\n",
      " NOT MATCHED=181146\n",
      " DISCARDED=647718\n",
      " SKIPPED=24166646\n",
      "*** CUBES=4900\n",
      " MATCHED=72495\n",
      " NOT MATCHED=185747\n",
      " DISCARDED=665033\n",
      " SKIPPED=24665546\n",
      "*** CUBES=5000\n",
      " MATCHED=74425\n",
      " NOT MATCHED=189932\n",
      " DISCARDED=680766\n",
      " SKIPPED=25166446\n",
      "*** CUBES=5100\n",
      " MATCHED=74785\n",
      " NOT MATCHED=190350\n",
      " DISCARDED=682336\n",
      " SKIPPED=25685346\n",
      "*** CUBES=5200\n",
      " MATCHED=74785\n",
      " NOT MATCHED=190350\n",
      " DISCARDED=682336\n",
      " SKIPPED=26206246\n",
      "*** MATCHED=74785\n",
      "*** NOT MATCHED=190350\n",
      "*** DISCARDED=682336\n",
      "*** SKIPPED=26258336\n",
      "#####################################################################\n",
      "TRAINING:\n",
      "MODEL:/media/1KGB_ILAN/papyrus/models/front_tear_model2.ckpt\n",
      "#####################################################################\n",
      "INFO:tensorflow:Restoring parameters from /media/1KGB_ILAN/papyrus/models/front_tear_model1.ckpt\n",
      "Model restored.\n",
      "step 49, training accuracy 0.78\n",
      "step 99, training accuracy 0.75\n",
      "step 149, training accuracy 0.70\n",
      "step 199, training accuracy 0.85\n",
      "step 249, training accuracy 0.77\n",
      "step 299, training accuracy 0.80\n",
      "step 349, training accuracy 0.80\n",
      "step 399, training accuracy 0.88\n",
      "step 449, training accuracy 0.80\n",
      "step 499, training accuracy 0.80\n",
      "step 549, training accuracy 0.82\n",
      "step 599, training accuracy 0.90\n",
      "step 649, training accuracy 0.85\n",
      "step 699, training accuracy 0.85\n",
      "step 749, training accuracy 0.88\n",
      "step 799, training accuracy 0.87\n",
      "step 849, training accuracy 0.82\n",
      "step 899, training accuracy 0.73\n",
      "step 949, training accuracy 0.82\n",
      "step 999, training accuracy 0.92\n",
      "step 1049, training accuracy 0.83\n",
      "step 1099, training accuracy 0.83\n",
      "step 1149, training accuracy 0.78\n",
      "step 1199, training accuracy 0.88\n",
      "step 1249, training accuracy 0.95\n",
      "step 1299, training accuracy 0.88\n",
      "step 1349, training accuracy 0.88\n",
      "step 1399, training accuracy 0.85\n",
      "step 1449, training accuracy 0.92\n",
      "step 1499, training accuracy 0.88\n",
      "step 1549, training accuracy 0.87\n",
      "step 1599, training accuracy 0.88\n",
      "step 1649, training accuracy 0.83\n",
      "step 1699, training accuracy 0.80\n",
      "step 1749, training accuracy 0.90\n",
      "step 1799, training accuracy 0.95\n",
      "step 1849, training accuracy 0.93\n",
      "step 1899, training accuracy 0.90\n",
      "step 1949, training accuracy 0.93\n",
      "step 1999, training accuracy 0.90\n",
      "Optimization Finished!\n",
      "step 1999, training accuracy 0.9\n",
      "Model saved in file: /media/1KGB_ILAN/papyrus/models/front_tear_model2.ckpt\n",
      "#####################################################################\n",
      "TRAINING ENDED\n",
      "#####################################################################\n",
      " \n",
      " \n"
     ]
    }
   ],
   "source": [
    "# RUN2 - take 2nd large pieces and train on it\n",
    "# train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "#    parallelize_build_train_set(cubes_set, ROOT_FOLDER + \"train_concats2/\", 1, 7, PARALLEL_build_train_set_for_binary_labeling)\n",
    "cubes_set = pre_process_training(\"PX303-Fg004-R-C01-R01-D05032015-T115135-ML638__006.jpg\", \\\n",
    "                                 100, -1, 400, -1)\n",
    "train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "    NEW_build_train_set_for_binary_labeling(cubes_set, cubes_set, CUBE_SIZE, ROOT_FOLDER + \"train_concats3/\", 12, 7)\n",
    "tf.reset_default_graph()\n",
    "model_tf_deep(250)\n",
    "train(train_imgs, train_lbls, ROOT_FOLDER + \"models/front_tear_model2.ckpt\", ROOT_FOLDER + \"models/front_tear_model1.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-01T23:05:02.443698Z",
     "start_time": "2019-06-01T22:09:12.687507Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# cubes_set = pre_process_training(\"PX303-Fg004-V-C01-R01-D08032015-T110817-ML638__006.jpg\", \\\n",
    "#                                  100, -1, 400, -1, max_cols=8, max_rows=5)\n",
    "# tf.reset_default_graph()\n",
    "# model_tf_deep(250)\n",
    "# len_cubes = len(cubes_set)\n",
    "# bat_cubes = 1000\n",
    "# # COPY MANUALLY!!!\n",
    "# #shutil.copy(ROOT_FOLDER + \"models/tear_model1.ckpt\", ROOT_FOLDER + \"models/tear_model2.ckpt\")\n",
    "# for start_cubes in range(0, len_cubes, bat_cubes):\n",
    "#     end_cubes = ((start+bat_cubes) if start+bat_cubes < len_cubes else (len_cubes - 1))\n",
    "#     BATCHES = int((end_cubes - start_cubes) / 10) # smaller train batches\n",
    "#     train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "#         NEW_build_train_set_for_binary_labeling(cubes_set[start_cubes:end_cubes], cubes_set, CUBE_SIZE, \n",
    "#                                                 ROOT_FOLDER + \"train_concats2/\", 1, 5)\n",
    "#     train(train_imgs, train_lbls, ROOT_FOLDER + \"models/tear_model2.ckpt\", ROOT_FOLDER + \"models/tear_model2.ckpt\")  \n",
    "# BATCHES = 2000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# MULTI PROCESS\n",
    "import multiprocessing\n",
    "from itertools import product, repeat\n",
    "\n",
    "num_cores = 4# multiprocessing.cpu_count()\n",
    "num_partitions = num_cores * 4\n",
    "\n",
    "def chunkify(lst,n):\n",
    "    return [lst[i::n] for i in range(n)]\n",
    "\n",
    "def parallelize_build_train_set(cubes_set, folder, enrich_factor, tolerance_factor, func):\n",
    "    adjs_list = chunkify(cubes_set, num_partitions)\n",
    "    print('list:', adjs_list)\n",
    "    # test block for running serialy    \n",
    "    # results = []\n",
    "    # for item in zip(np.arange(num_partitions), df_split):\n",
    "    #    results.append(func(item))\n",
    "    # df = pd.concat(results, ignore_index=True)\n",
    "\n",
    "    # parallel block\n",
    "    pool = multiprocessing.Pool(num_cores)\n",
    "    # import pdb; pdb.set_trace()\n",
    "    args = zip(repeat(cubes_set), adjs_list, repeat(CUBE_SIZE), \\\n",
    "               repeat(ROOT_FOLDER + folder), repeat(enrich_factor), repeat(tolerance_factor))\n",
    "\n",
    "    results = pool.map(func, args)\n",
    "    train_imgs = []\n",
    "    train_lbls = []\n",
    "    train_x_delta = []\n",
    "    train_y_delta = []\n",
    "    is_enriched = []\n",
    "    \n",
    "    for result in results:\n",
    "        train_imgs.append(result[0])\n",
    "        train_lbls.append(result[1])\n",
    "        train_x_delta.append(result[2])\n",
    "        train_y_delta.append(result[3])\n",
    "        is_enriched.append(result[4])\n",
    "    \n",
    "    pool.close()\n",
    "    pool.join()\n",
    "\n",
    "    return train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# Parallel RUN2 - take 2nd large pieces and train on it\n",
    "cubes_set = pre_process_training(\"PX303-Fg004-V-C01-R01-D08032015-T110817-ML638__006.jpg\", 100, -1, 400, -1, max_cols=9, max_rows=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# RE-RUN2 - take 2nd large pieces and train on it\n",
    "train_imgs, train_lbls, is_enriched = \\\n",
    "    load_train_from_disk(ROOT_FOLDER + \"train_concats2/\")\n",
    "print(len(train_imgs))\n",
    "tf.reset_default_graph()\n",
    "model_tf_deep(250)\n",
    "train(train_imgs, train_lbls, ROOT_FOLDER + \"model_binary/tear_model2.ckpt\", ROOT_FOLDER + \"model_binary/tear_model1.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-20T06:32:03.991172Z",
     "start_time": "2019-10-20T06:01:00.829864Z"
    },
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#####################################################################\n",
      "PRE_PROCESS:PX303-Fg006-R-C02-R02-D08032015-T114223-ML638__006.jpg\n",
      "#####################################################################\n",
      "PRE_PROCESS:::TEAR_2X3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/util/dtype.py:141: UserWarning: Possible precision loss when converting from float64 to uint16\n",
      "  .format(dtypeobj_in, dtypeobj_out))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PRE_PROCESS:::TEAR_2X4\n",
      "PRE_PROCESS:::TEAR_2X5\n",
      "PRE_PROCESS:::TEAR_2X6\n",
      "PRE_PROCESS:::TEAR_2X7\n",
      "PRE_PROCESS:::TEAR_3X3\n",
      "PRE_PROCESS:::TEAR_3X4\n",
      "PRE_PROCESS:::TEAR_3X5\n",
      "PRE_PROCESS:::TEAR_3X6\n",
      "PRE_PROCESS:::TEAR_3X7\n",
      "*** CUBES=100\n",
      " MATCHED=1381\n",
      " NOT MATCHED=3750\n",
      " DISCARDED=13625\n",
      " SKIPPED=412236\n",
      "*** CUBES=200\n",
      " MATCHED=4219\n",
      " NOT MATCHED=7265\n",
      " DISCARDED=26388\n",
      " SKIPPED=829868\n",
      "*** CUBES=300\n",
      " MATCHED=4219\n",
      " NOT MATCHED=7265\n",
      " DISCARDED=26388\n",
      " SKIPPED=1263868\n",
      "*** CUBES=400\n",
      " MATCHED=4320\n",
      " NOT MATCHED=8256\n",
      " DISCARDED=29980\n",
      " SKIPPED=1693280\n",
      "*** CUBES=500\n",
      " MATCHED=4990\n",
      " NOT MATCHED=11451\n",
      " DISCARDED=41551\n",
      " SKIPPED=2112480\n",
      "*** CUBES=600\n",
      " MATCHED=6025\n",
      " NOT MATCHED=13234\n",
      " DISCARDED=48017\n",
      " SKIPPED=2538192\n",
      "*** CUBES=700\n",
      " MATCHED=6025\n",
      " NOT MATCHED=13234\n",
      " DISCARDED=48017\n",
      " SKIPPED=2972192\n",
      "*** CUBES=800\n",
      " MATCHED=7175\n",
      " NOT MATCHED=16680\n",
      " DISCARDED=60647\n",
      " SKIPPED=3390064\n",
      "*** CUBES=900\n",
      " MATCHED=9541\n",
      " NOT MATCHED=19758\n",
      " DISCARDED=71931\n",
      " SKIPPED=3809616\n",
      "*** CUBES=1000\n",
      " MATCHED=9541\n",
      " NOT MATCHED=19758\n",
      " DISCARDED=71931\n",
      " SKIPPED=4243616\n",
      "*** CUBES=1100\n",
      " MATCHED=9755\n",
      " NOT MATCHED=21318\n",
      " DISCARDED=77807\n",
      " SKIPPED=4670164\n",
      "*** CUBES=1200\n",
      " MATCHED=11784\n",
      " NOT MATCHED=24705\n",
      " DISCARDED=90535\n",
      " SKIPPED=5087964\n",
      "*** CUBES=1300\n",
      " MATCHED=12546\n",
      " NOT MATCHED=25617\n",
      " DISCARDED=93967\n",
      " SKIPPED=5517590\n",
      "*** CUBES=1400\n",
      " MATCHED=12628\n",
      " NOT MATCHED=25990\n",
      " DISCARDED=95108\n",
      " SKIPPED=5950072\n",
      "*** CUBES=1500\n",
      " MATCHED=13289\n",
      " NOT MATCHED=29380\n",
      " DISCARDED=105475\n",
      " SKIPPED=6370272\n",
      "*** CUBES=1600\n",
      " MATCHED=14295\n",
      " NOT MATCHED=31103\n",
      " DISCARDED=110744\n",
      " SKIPPED=6797234\n",
      "*** CUBES=1700\n",
      " MATCHED=14462\n",
      " NOT MATCHED=31927\n",
      " DISCARDED=113820\n",
      " SKIPPED=7227329\n",
      "*** CUBES=1800\n",
      " MATCHED=16592\n",
      " NOT MATCHED=39413\n",
      " DISCARDED=141762\n",
      " SKIPPED=7625829\n",
      "*** CUBES=1900\n",
      " MATCHED=19295\n",
      " NOT MATCHED=46434\n",
      " DISCARDED=167984\n",
      " SKIPPED=8026499\n",
      "*** CUBES=2000\n",
      " MATCHED=20182\n",
      " NOT MATCHED=50624\n",
      " DISCARDED=183753\n",
      " SKIPPED=8440499\n",
      "*** CUBES=2100\n",
      " MATCHED=20899\n",
      " NOT MATCHED=52339\n",
      " DISCARDED=190205\n",
      " SKIPPED=8866299\n",
      "*** CUBES=2200\n",
      " MATCHED=20899\n",
      " NOT MATCHED=52339\n",
      " DISCARDED=190205\n",
      " SKIPPED=9300299\n",
      "*** CUBES=2300\n",
      " MATCHED=21722\n",
      " NOT MATCHED=56820\n",
      " DISCARDED=206992\n",
      " SKIPPED=9713000\n",
      "*** CUBES=2400\n",
      " MATCHED=23868\n",
      " NOT MATCHED=64403\n",
      " DISCARDED=235433\n",
      " SKIPPED=10110900\n",
      "*** CUBES=2500\n",
      " MATCHED=25997\n",
      " NOT MATCHED=69251\n",
      " DISCARDED=253612\n",
      " SKIPPED=10521796\n",
      "*** CUBES=2600\n",
      " MATCHED=28668\n",
      " NOT MATCHED=73032\n",
      " DISCARDED=267790\n",
      " SKIPPED=10937746\n",
      "*** CUBES=2700\n",
      " MATCHED=28668\n",
      " NOT MATCHED=73032\n",
      " DISCARDED=267790\n",
      " SKIPPED=11371746\n",
      "*** CUBES=2800\n",
      " MATCHED=28818\n",
      " NOT MATCHED=74215\n",
      " DISCARDED=272031\n",
      " SKIPPED=11800316\n",
      "*** CUBES=2900\n",
      " MATCHED=31098\n",
      " NOT MATCHED=82097\n",
      " DISCARDED=300265\n",
      " SKIPPED=12198116\n",
      "*** CUBES=3000\n",
      " MATCHED=33361\n",
      " NOT MATCHED=88809\n",
      " DISCARDED=324328\n",
      " SKIPPED=12601256\n",
      "*** CUBES=3100\n",
      " MATCHED=35755\n",
      " NOT MATCHED=92790\n",
      " DISCARDED=338657\n",
      " SKIPPED=13016856\n",
      "*** CUBES=3200\n",
      " MATCHED=37100\n",
      " NOT MATCHED=94700\n",
      " DISCARDED=345530\n",
      " SKIPPED=13442024\n",
      "*** CUBES=3300\n",
      " MATCHED=37100\n",
      " NOT MATCHED=94700\n",
      " DISCARDED=345530\n",
      " SKIPPED=13876024\n",
      "*** CUBES=3400\n",
      " MATCHED=38330\n",
      " NOT MATCHED=99721\n",
      " DISCARDED=364261\n",
      " SKIPPED=14286224\n",
      "*** CUBES=3500\n",
      " MATCHED=40610\n",
      " NOT MATCHED=107103\n",
      " DISCARDED=391783\n",
      " SKIPPED=14685224\n",
      "*** CUBES=3600\n",
      " MATCHED=42603\n",
      " NOT MATCHED=110992\n",
      " DISCARDED=406623\n",
      " SKIPPED=15100416\n",
      "*** CUBES=3700\n",
      " MATCHED=44642\n",
      " NOT MATCHED=114001\n",
      " DISCARDED=418147\n",
      " SKIPPED=15519800\n",
      "*** CUBES=3800\n",
      " MATCHED=44642\n",
      " NOT MATCHED=114001\n",
      " DISCARDED=418147\n",
      " SKIPPED=15953800\n",
      "*** CUBES=3900\n",
      " MATCHED=45241\n",
      " NOT MATCHED=117270\n",
      " DISCARDED=428121\n",
      " SKIPPED=16374528\n",
      "*** CUBES=4000\n",
      " MATCHED=47390\n",
      " NOT MATCHED=125047\n",
      " DISCARDED=451847\n",
      " SKIPPED=16776928\n",
      "*** CUBES=4100\n",
      " MATCHED=48778\n",
      " NOT MATCHED=130132\n",
      " DISCARDED=467344\n",
      " SKIPPED=17190278\n",
      "*** CUBES=4200\n",
      " MATCHED=50003\n",
      " NOT MATCHED=133098\n",
      " DISCARDED=476387\n",
      " SKIPPED=17612208\n",
      "*** CUBES=4300\n",
      " MATCHED=50003\n",
      " NOT MATCHED=133098\n",
      " DISCARDED=476387\n",
      " SKIPPED=18046208\n",
      "*** MATCHED=50003\n",
      "*** NOT MATCHED=133098\n",
      "*** DISCARDED=476387\n",
      "*** SKIPPED=18224148\n",
      "WARNING:tensorflow:From <ipython-input-24-6d6c00010ca0>:97: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
      "\n",
      "#####################################################################\n",
      "TRAINING:\n",
      "MODEL:/media/1KGB_ILAN/papyrus/models/front_tear_model3.ckpt\n",
      "#####################################################################\n",
      "INFO:tensorflow:Restoring parameters from /media/1KGB_ILAN/papyrus/models/front_tear_model2.ckpt\n",
      "Model restored.\n",
      "step 49, training accuracy 0.90\n",
      "step 99, training accuracy 0.78\n",
      "step 149, training accuracy 0.87\n",
      "step 199, training accuracy 0.85\n",
      "step 249, training accuracy 0.77\n",
      "step 299, training accuracy 0.83\n",
      "step 349, training accuracy 0.78\n",
      "step 399, training accuracy 0.82\n",
      "step 449, training accuracy 0.82\n",
      "step 499, training accuracy 0.80\n",
      "step 549, training accuracy 0.88\n",
      "step 599, training accuracy 0.92\n",
      "step 649, training accuracy 0.78\n",
      "step 699, training accuracy 0.90\n",
      "step 749, training accuracy 0.82\n",
      "step 799, training accuracy 0.80\n",
      "step 849, training accuracy 0.95\n",
      "step 899, training accuracy 0.83\n",
      "step 949, training accuracy 0.92\n",
      "step 999, training accuracy 0.88\n",
      "step 1049, training accuracy 0.87\n",
      "step 1099, training accuracy 0.93\n",
      "step 1149, training accuracy 0.88\n",
      "step 1199, training accuracy 0.93\n",
      "step 1249, training accuracy 0.90\n",
      "step 1299, training accuracy 0.95\n",
      "step 1349, training accuracy 0.93\n",
      "step 1399, training accuracy 0.90\n",
      "step 1449, training accuracy 0.93\n",
      "step 1499, training accuracy 0.95\n",
      "step 1549, training accuracy 1.00\n",
      "step 1599, training accuracy 0.92\n",
      "step 1649, training accuracy 0.97\n",
      "step 1699, training accuracy 0.92\n",
      "step 1749, training accuracy 0.92\n",
      "step 1799, training accuracy 0.95\n",
      "step 1849, training accuracy 0.98\n",
      "step 1899, training accuracy 0.90\n",
      "step 1949, training accuracy 0.93\n",
      "step 1999, training accuracy 0.92\n",
      "Optimization Finished!\n",
      "step 1999, training accuracy 0.916667\n",
      "Model saved in file: /media/1KGB_ILAN/papyrus/models/front_tear_model3.ckpt\n",
      "#####################################################################\n",
      "TRAINING ENDED\n",
      "#####################################################################\n",
      " \n",
      " \n"
     ]
    }
   ],
   "source": [
    "# OPTIONAL RUN3 - take 3rd large pieces and train on it OR TEST in next block\n",
    "\n",
    "cubes_set = pre_process_training(\"PX303-Fg006-R-C02-R02-D08032015-T114223-ML638__006.jpg\", 0, 6200, 0, 4400)\n",
    "train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "    NEW_build_train_set_for_binary_labeling(cubes_set, cubes_set, CUBE_SIZE, ROOT_FOLDER + \"train_concats3/\", 12, 7)\n",
    "tf.reset_default_graph()\n",
    "model_tf_deep(250)    \n",
    "train(train_imgs, train_lbls, ROOT_FOLDER + \"models/front_tear_model3.ckpt\", ROOT_FOLDER + \"models/front_tear_model2.ckpt\")\n",
    "# train(train_imgs, train_lbls, ROOT_FOLDER + \"models/tear_model1.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-19T20:53:51.053127Z",
     "start_time": "2019-10-19T20:28:05.003521Z"
    },
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#####################################################################\n",
      "PRE_PROCESS:PX303-Fg006-R-C02-R02-D08032015-T114223-ML638__006.jpg\n",
      "#####################################################################\n",
      "PRE_PROCESS:::TEAR_2X3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ilan/venv35/lib/python3.5/site-packages/skimage/util/dtype.py:141: UserWarning: Possible precision loss when converting from float64 to uint16\n",
      "  .format(dtypeobj_in, dtypeobj_out))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PRE_PROCESS:::TEAR_2X4\n",
      "PRE_PROCESS:::TEAR_2X5\n",
      "PRE_PROCESS:::TEAR_2X6\n",
      "PRE_PROCESS:::TEAR_2X7\n",
      "PRE_PROCESS:::TEAR_3X3\n",
      "PRE_PROCESS:::TEAR_3X4\n",
      "PRE_PROCESS:::TEAR_3X5\n",
      "PRE_PROCESS:::TEAR_3X6\n",
      "PRE_PROCESS:::TEAR_3X7\n",
      "*** CUBES=100\n",
      " MATCHED=396\n",
      " NOT MATCHED=3359\n",
      " DISCARDED=12649\n",
      " SKIPPED=414612\n",
      "*** CUBES=200\n",
      " MATCHED=1930\n",
      " NOT MATCHED=6505\n",
      " DISCARDED=24493\n",
      " SKIPPED=834546\n",
      "*** CUBES=300\n",
      " MATCHED=1930\n",
      " NOT MATCHED=6505\n",
      " DISCARDED=24493\n",
      " SKIPPED=1269546\n",
      "*** CUBES=400\n",
      " MATCHED=2116\n",
      " NOT MATCHED=7852\n",
      " DISCARDED=29929\n",
      " SKIPPED=1697751\n",
      "*** CUBES=500\n",
      " MATCHED=3417\n",
      " NOT MATCHED=10840\n",
      " DISCARDED=41970\n",
      " SKIPPED=2117651\n",
      "*** CUBES=600\n",
      " MATCHED=3729\n",
      " NOT MATCHED=12128\n",
      " DISCARDED=47157\n",
      " SKIPPED=2546158\n",
      "*** CUBES=700\n",
      " MATCHED=3761\n",
      " NOT MATCHED=12314\n",
      " DISCARDED=47821\n",
      " SKIPPED=2980306\n",
      "*** CUBES=800\n",
      " MATCHED=4612\n",
      " NOT MATCHED=15402\n",
      " DISCARDED=58886\n",
      " SKIPPED=3401106\n",
      "*** CUBES=900\n",
      " MATCHED=5186\n",
      " NOT MATCHED=17845\n",
      " DISCARDED=67627\n",
      " SKIPPED=3824888\n",
      "*** CUBES=1000\n",
      " MATCHED=5186\n",
      " NOT MATCHED=17845\n",
      " DISCARDED=67627\n",
      " SKIPPED=4259888\n",
      "*** CUBES=1100\n",
      " MATCHED=5756\n",
      " NOT MATCHED=20048\n",
      " DISCARDED=76047\n",
      " SKIPPED=4684223\n",
      "*** CUBES=1200\n",
      " MATCHED=6353\n",
      " NOT MATCHED=22699\n",
      " DISCARDED=86176\n",
      " SKIPPED=5106398\n",
      "*** CUBES=1300\n",
      " MATCHED=6353\n",
      " NOT MATCHED=22699\n",
      " DISCARDED=86176\n",
      " SKIPPED=5541398\n",
      "*** CUBES=1400\n",
      " MATCHED=6813\n",
      " NOT MATCHED=25348\n",
      " DISCARDED=93917\n",
      " SKIPPED=5965968\n",
      "*** CUBES=1500\n",
      " MATCHED=7796\n",
      " NOT MATCHED=28977\n",
      " DISCARDED=104521\n",
      " SKIPPED=6386664\n",
      "*** CUBES=1600\n",
      " MATCHED=7796\n",
      " NOT MATCHED=28977\n",
      " DISCARDED=104521\n",
      " SKIPPED=6821664\n",
      "*** CUBES=1700\n",
      " MATCHED=8978\n",
      " NOT MATCHED=33585\n",
      " DISCARDED=121969\n",
      " SKIPPED=7234554\n",
      "*** CUBES=1800\n",
      " MATCHED=11105\n",
      " NOT MATCHED=41967\n",
      " DISCARDED=153688\n",
      " SKIPPED=7629354\n",
      "*** CUBES=1900\n",
      " MATCHED=12597\n",
      " NOT MATCHED=47913\n",
      " DISCARDED=176272\n",
      " SKIPPED=8035754\n",
      "*** CUBES=2000\n",
      " MATCHED=14622\n",
      " NOT MATCHED=52098\n",
      " DISCARDED=192194\n",
      " SKIPPED=8450554\n",
      "*** CUBES=2100\n",
      " MATCHED=15516\n",
      " NOT MATCHED=53852\n",
      " DISCARDED=198882\n",
      " SKIPPED=8877070\n",
      "*** CUBES=2200\n",
      " MATCHED=15516\n",
      " NOT MATCHED=53852\n",
      " DISCARDED=198882\n",
      " SKIPPED=9312070\n",
      "*** CUBES=2300\n",
      " MATCHED=15998\n",
      " NOT MATCHED=57992\n",
      " DISCARDED=214876\n",
      " SKIPPED=9726910\n",
      "*** CUBES=2400\n",
      " MATCHED=17949\n",
      " NOT MATCHED=65372\n",
      " DISCARDED=243399\n",
      " SKIPPED=10125910\n",
      "*** CUBES=2500\n",
      " MATCHED=19501\n",
      " NOT MATCHED=70577\n",
      " DISCARDED=263488\n",
      " SKIPPED=10535534\n",
      "*** CUBES=2600\n",
      " MATCHED=21334\n",
      " NOT MATCHED=74559\n",
      " DISCARDED=278813\n",
      " SKIPPED=10951134\n",
      "*** CUBES=2700\n",
      " MATCHED=21384\n",
      " NOT MATCHED=74639\n",
      " DISCARDED=279119\n",
      " SKIPPED=11385746\n",
      "*** CUBES=2800\n",
      " MATCHED=21409\n",
      " NOT MATCHED=74959\n",
      " DISCARDED=280258\n",
      " SKIPPED=11819286\n",
      "*** CUBES=2900\n",
      " MATCHED=22950\n",
      " NOT MATCHED=82942\n",
      " DISCARDED=308692\n",
      " SKIPPED=12217786\n",
      "*** CUBES=3000\n",
      " MATCHED=24493\n",
      " NOT MATCHED=90043\n",
      " DISCARDED=333980\n",
      " SKIPPED=12620312\n",
      "*** CUBES=3100\n",
      " MATCHED=26167\n",
      " NOT MATCHED=94023\n",
      " DISCARDED=348110\n",
      " SKIPPED=13037112\n",
      "*** CUBES=3200\n",
      " MATCHED=27301\n",
      " NOT MATCHED=96451\n",
      " DISCARDED=356724\n",
      " SKIPPED=13461010\n",
      "*** CUBES=3300\n",
      " MATCHED=27301\n",
      " NOT MATCHED=96451\n",
      " DISCARDED=356724\n",
      " SKIPPED=13896010\n",
      "*** CUBES=3400\n",
      " MATCHED=27712\n",
      " NOT MATCHED=100492\n",
      " DISCARDED=372030\n",
      " SKIPPED=14311630\n",
      "*** CUBES=3500\n",
      " MATCHED=29275\n",
      " NOT MATCHED=107574\n",
      " DISCARDED=398855\n",
      " SKIPPED=14712630\n",
      "*** CUBES=3600\n",
      " MATCHED=30823\n",
      " NOT MATCHED=111821\n",
      " DISCARDED=415396\n",
      " SKIPPED=15126752\n",
      "*** CUBES=3700\n",
      " MATCHED=31890\n",
      " NOT MATCHED=114724\n",
      " DISCARDED=426840\n",
      " SKIPPED=15547334\n",
      "*** CUBES=3800\n",
      " MATCHED=31890\n",
      " NOT MATCHED=114724\n",
      " DISCARDED=426840\n",
      " SKIPPED=15982334\n",
      "*** CUBES=3900\n",
      " MATCHED=32129\n",
      " NOT MATCHED=118080\n",
      " DISCARDED=437032\n",
      " SKIPPED=16403763\n",
      "*** CUBES=4000\n",
      " MATCHED=33110\n",
      " NOT MATCHED=126260\n",
      " DISCARDED=461883\n",
      " SKIPPED=16805663\n",
      "*** CUBES=4100\n",
      " MATCHED=34523\n",
      " NOT MATCHED=131240\n",
      " DISCARDED=477030\n",
      " SKIPPED=17220443\n",
      "*** CUBES=4200\n",
      " MATCHED=35699\n",
      " NOT MATCHED=134625\n",
      " DISCARDED=487337\n",
      " SKIPPED=17641673\n",
      "*** CUBES=4300\n",
      " MATCHED=35699\n",
      " NOT MATCHED=134625\n",
      " DISCARDED=487337\n",
      " SKIPPED=18076673\n",
      "*** MATCHED=35699\n",
      "*** NOT MATCHED=134625\n",
      "*** DISCARDED=487337\n",
      "*** SKIPPED=18298523\n",
      "WARNING:tensorflow:From <ipython-input-6-6d6c00010ca0>:97: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
      "\n",
      "170324\n",
      "INFO:tensorflow:Restoring parameters from /media/1KGB_ILAN/papyrus/models/front_tear_model2.ckpt\n",
      "Model restored.\n",
      ">>> step 100\n",
      "step 0-99, precision 0.000000, recall 0.000000, f_score 0.000000\n",
      "=== total 100 match 6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ilan/venv35/lib/python3.5/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 due to no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> step 200\n",
      "step 100-199, precision 0.000000, recall 0.000000, f_score 0.000000\n",
      "=== total 200 match 22\n",
      ">>> step 300\n",
      "step 200-299, precision 0.000000, recall 0.000000, f_score 0.000000\n",
      "=== total 300 match 34\n",
      ">>> step 400\n",
      "step 300-399, precision 0.000000, recall 0.000000, f_score 0.000000\n",
      "=== total 400 match 46\n",
      ">>> step 500\n",
      "step 400-499, precision 0.000000, recall 0.000000, f_score 0.000000\n",
      "=== total 500 match 58\n",
      ">>> step 600\n",
      "step 500-599, precision 0.000000, recall 0.000000, f_score 0.000000\n",
      "=== total 600 match 70\n",
      ">>> step 700\n",
      "step 600-699, precision 0.000000, recall 0.000000, f_score 0.000000\n",
      "=== total 700 match 77\n",
      ">>> step 800\n",
      "step 700-799, precision 0.000000, recall 0.000000, f_score 0.000000\n",
      "=== total 800 match 84\n",
      ">>> step 900\n",
      "step 800-899, precision 0.021739, recall 0.030303, f_score 0.025316\n",
      "=== total 900 match 92\n",
      ">>> step 1000\n",
      "step 900-999, precision 0.020833, recall 0.025000, f_score 0.022727\n",
      "=== total 1000 match 96\n",
      ">>> step 1100\n",
      "step 1000-1099, precision 0.039216, recall 0.036036, f_score 0.037559\n",
      "=== total 1100 match 102\n",
      ">>> step 1200\n",
      "step 1100-1199, precision 0.034783, recall 0.036036, f_score 0.035398\n",
      "=== total 1200 match 115\n",
      ">>> step 1300\n",
      "step 1200-1299, precision 0.032258, recall 0.032258, f_score 0.032258\n",
      "=== total 1300 match 124\n",
      ">>> step 1400\n",
      "step 1300-1399, precision 0.029197, recall 0.030534, f_score 0.029851\n",
      "=== total 1400 match 137\n",
      ">>> step 1500\n",
      "step 1400-1499, precision 0.026846, recall 0.030303, f_score 0.028470\n",
      "=== total 1500 match 149\n",
      ">>> step 1600\n",
      "step 1500-1599, precision 0.025478, recall 0.030303, f_score 0.027682\n",
      "=== total 1600 match 157\n",
      ">>> step 1700\n",
      "step 1600-1699, precision 0.024242, recall 0.028777, f_score 0.026316\n",
      "=== total 1700 match 165\n",
      ">>> step 1800\n",
      "step 1700-1799, precision 0.022727, recall 0.028777, f_score 0.025397\n",
      "=== total 1800 match 176\n",
      ">>> step 1900\n",
      "step 1800-1899, precision 0.021277, recall 0.028777, f_score 0.024465\n",
      "=== total 1900 match 188\n",
      ">>> step 2000\n",
      "step 1900-1999, precision 0.020101, recall 0.028777, f_score 0.023669\n",
      "=== total 2000 match 199\n",
      ">>> step 2100\n",
      "step 2000-2099, precision 0.019324, recall 0.026316, f_score 0.022284\n",
      "=== total 2100 match 207\n",
      ">>> step 2200\n",
      "step 2100-2199, precision 0.018265, recall 0.026316, f_score 0.021563\n",
      "=== total 2200 match 219\n",
      ">>> step 2300\n",
      "step 2200-2299, precision 0.068826, recall 0.103030, f_score 0.082524\n",
      "=== total 2300 match 247\n",
      ">>> step 2400\n",
      "step 2300-2399, precision 0.065891, recall 0.103030, f_score 0.080378\n",
      "=== total 2400 match 258\n",
      ">>> step 2500\n",
      "step 2400-2499, precision 0.144781, recall 0.225131, f_score 0.176230\n",
      "=== total 2500 match 297\n",
      ">>> step 2600\n",
      "step 2500-2599, precision 0.140065, recall 0.225131, f_score 0.172691\n",
      "=== total 2600 match 307\n",
      ">>> step 2700\n",
      "step 2600-2699, precision 0.175758, recall 0.278846, f_score 0.215613\n",
      "=== total 2700 match 330\n",
      ">>> step 2800\n",
      "step 2700-2799, precision 0.168605, recall 0.267281, f_score 0.206774\n",
      "=== total 2800 match 344\n",
      ">>> step 2900\n",
      "step 2800-2899, precision 0.222222, recall 0.345679, f_score 0.270531\n",
      "=== total 2900 match 378\n",
      ">>> step 3000\n",
      "step 2900-2999, precision 0.242424, recall 0.341637, f_score 0.283604\n",
      "=== total 3000 match 396\n",
      ">>> step 3100\n",
      "step 3000-3099, precision 0.291572, recall 0.401254, f_score 0.337731\n",
      "=== total 3100 match 439\n",
      ">>> step 3200\n",
      "step 3100-3199, precision 0.286353, recall 0.401254, f_score 0.334204\n",
      "=== total 3200 match 447\n",
      ">>> step 3300\n",
      "step 3200-3299, precision 0.284141, recall 0.361345, f_score 0.318126\n",
      "=== total 3300 match 454\n",
      ">>> step 3400\n",
      "step 3300-3399, precision 0.286022, recall 0.359459, f_score 0.318563\n",
      "=== total 3400 match 465\n",
      ">>> step 3500\n",
      "step 3400-3499, precision 0.290188, recall 0.351010, f_score 0.317714\n",
      "=== total 3500 match 479\n",
      ">>> step 3600\n",
      "step 3500-3599, precision 0.283096, recall 0.351010, f_score 0.313416\n",
      "=== total 3600 match 491\n",
      ">>> step 3700\n",
      "step 3600-3699, precision 0.278557, recall 0.351010, f_score 0.310615\n",
      "=== total 3700 match 499\n",
      ">>> step 3800\n",
      "step 3700-3799, precision 0.300567, recall 0.382212, f_score 0.336508\n",
      "=== total 3800 match 529\n",
      ">>> step 3900\n",
      "step 3800-3899, precision 0.292279, recall 0.382212, f_score 0.331250\n",
      "=== total 3900 match 544\n",
      ">>> step 4000\n",
      "step 3900-3999, precision 0.287004, recall 0.365517, f_score 0.321537\n",
      "=== total 4000 match 554\n",
      ">>> step 4100\n",
      "step 4000-4099, precision 0.306507, recall 0.393407, f_score 0.344562\n",
      "=== total 4100 match 584\n",
      ">>> step 4200\n",
      "step 4100-4199, precision 0.301855, recall 0.393407, f_score 0.341603\n",
      "=== total 4200 match 593\n",
      ">>> step 4300\n",
      "step 4200-4299, precision 0.318255, recall 0.409563, f_score 0.358182\n",
      "=== total 4300 match 619\n",
      ">>> step 4400\n",
      "step 4300-4399, precision 0.314195, recall 0.409563, f_score 0.355596\n",
      "=== total 4400 match 627\n",
      ">>> step 4500\n",
      "step 4400-4499, precision 0.309748, recall 0.409563, f_score 0.352731\n",
      "=== total 4500 match 636\n",
      ">>> step 4600\n",
      "step 4500-4599, precision 0.307810, recall 0.414433, f_score 0.353251\n",
      "=== total 4600 match 653\n",
      ">>> step 4700\n",
      "step 4600-4699, precision 0.322254, recall 0.432171, f_score 0.369205\n",
      "=== total 4700 match 692\n",
      ">>> step 4800\n",
      "step 4700-4799, precision 0.328691, recall 0.442777, f_score 0.377298\n",
      "=== total 4800 match 718\n",
      ">>> step 4900\n",
      "step 4800-4899, precision 0.347480, recall 0.468694, f_score 0.399086\n",
      "=== total 4900 match 754\n",
      ">>> step 5000\n",
      "step 4900-4999, precision 0.399758, recall 0.526232, f_score 0.454358\n",
      "=== total 5000 match 828\n",
      ">>> step 5100\n",
      "step 5000-5099, precision 0.407018, recall 0.509517, f_score 0.452536\n",
      "=== total 5100 match 855\n",
      ">>> step 5200\n",
      "step 5100-5199, precision 0.411565, recall 0.520057, f_score 0.459494\n",
      "=== total 5200 match 882\n",
      ">>> step 5300\n",
      "step 5200-5299, precision 0.441489, recall 0.550398, f_score 0.489965\n",
      "=== total 5300 match 940\n",
      ">>> step 5400\n",
      "step 5300-5399, precision 0.439331, recall 0.553360, f_score 0.489796\n",
      "=== total 5400 match 956\n",
      ">>> step 5500\n",
      "step 5400-5499, precision 0.448241, recall 0.568153, f_score 0.501124\n",
      "=== total 5500 match 995\n",
      ">>> step 5600\n",
      "step 5500-5599, precision 0.471483, recall 0.594012, f_score 0.525702\n",
      "=== total 5600 match 1052\n",
      ">>> step 5700\n",
      "step 5600-5699, precision 0.465291, recall 0.594012, f_score 0.521831\n",
      "=== total 5700 match 1066\n",
      ">>> step 5800\n",
      "step 5700-5799, precision 0.471545, recall 0.606272, f_score 0.530488\n",
      "=== total 5800 match 1107\n",
      ">>> step 5900\n",
      "step 5800-5899, precision 0.487931, recall 0.621295, f_score 0.546596\n",
      "=== total 5900 match 1160\n",
      ">>> step 6000\n",
      "step 5900-5999, precision 0.503704, recall 0.636837, f_score 0.562500\n",
      "=== total 6000 match 1215\n",
      ">>> step 6100\n",
      "step 6000-6099, precision 0.521260, recall 0.646484, f_score 0.577158\n",
      "=== total 6100 match 1270\n",
      ">>> step 6200\n",
      "step 6100-6199, precision 0.533080, recall 0.644894, f_score 0.583680\n",
      "=== total 6200 match 1315\n",
      ">>> step 6300\n",
      "step 6200-6299, precision 0.536478, recall 0.652330, f_score 0.588759\n",
      "=== total 6300 match 1357\n",
      ">>> step 6400\n",
      "step 6300-6399, precision 0.549435, recall 0.661003, f_score 0.600077\n",
      "=== total 6400 match 1416\n",
      ">>> step 6500\n",
      "step 6400-6499, precision 0.556772, recall 0.667760, f_score 0.607236\n",
      "=== total 6500 match 1462\n",
      ">>> step 6600\n",
      "step 6500-6599, precision 0.563333, recall 0.669042, f_score 0.611654\n",
      "=== total 6600 match 1500\n",
      ">>> step 6700\n",
      "step 6600-6699, precision 0.574823, recall 0.681645, f_score 0.623693\n",
      "=== total 6700 match 1557\n",
      ">>> step 6800\n",
      "step 6700-6799, precision 0.576826, recall 0.678519, f_score 0.623553\n",
      "=== total 6800 match 1588\n",
      ">>> step 6900\n",
      "step 6800-6899, precision 0.573658, recall 0.677229, f_score 0.621156\n",
      "=== total 6900 match 1602\n",
      ">>> step 7000\n",
      "step 6900-6999, precision 0.583133, recall 0.687989, f_score 0.631236\n",
      "=== total 7000 match 1660\n",
      ">>> step 7100\n",
      "step 7000-7099, precision 0.586085, recall 0.693650, f_score 0.635347\n",
      "=== total 7100 match 1696\n",
      ">>> step 7200\n",
      "step 7100-7199, precision 0.593337, recall 0.696561, f_score 0.640819\n",
      "=== total 7200 match 1741\n",
      ">>> step 7300\n",
      "step 7200-7299, precision 0.599777, recall 0.701892, f_score 0.646829\n",
      "=== total 7300 match 1794\n",
      ">>> step 7400\n",
      "step 7300-7399, precision 0.601847, recall 0.707987, f_score 0.650617\n",
      "=== total 7400 match 1841\n",
      ">>> step 7500\n",
      "step 7400-7499, precision 0.602902, recall 0.699501, f_score 0.647619\n",
      "=== total 7500 match 1861\n",
      ">>> step 7600\n",
      "step 7500-7599, precision 0.604134, recall 0.700246, f_score 0.648649\n",
      "=== total 7600 match 1887\n",
      ">>> step 7700\n",
      "step 7600-7699, precision 0.603872, recall 0.697703, f_score 0.647405\n",
      "=== total 7700 match 1911\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> step 7800\n",
      "step 7700-7799, precision 0.611478, recall 0.706573, f_score 0.655595\n",
      "=== total 7800 match 1969\n",
      ">>> step 7900\n",
      "step 7800-7899, precision 0.608081, recall 0.705744, f_score 0.653283\n",
      "=== total 7900 match 1980\n",
      ">>> step 8000\n",
      "step 7900-7999, precision 0.613175, recall 0.696288, f_score 0.652094\n",
      "=== total 8000 match 2019\n",
      ">>> step 8100\n",
      "step 8000-8099, precision 0.611521, recall 0.688470, f_score 0.647718\n",
      "=== total 8100 match 2031\n",
      ">>> step 8200\n",
      "step 8100-8199, precision 0.612966, recall 0.692350, f_score 0.650244\n",
      "=== total 8200 match 2067\n",
      ">>> step 8300\n",
      "step 8200-8299, precision 0.619473, recall 0.700532, f_score 0.657514\n",
      "=== total 8300 match 2126\n",
      ">>> step 8400\n",
      "step 8300-8399, precision 0.626147, recall 0.707254, f_score 0.664234\n",
      "=== total 8400 match 2180\n",
      ">>> step 8500\n",
      "step 8400-8499, precision 0.623857, recall 0.707254, f_score 0.662943\n",
      "=== total 8500 match 2188\n",
      ">>> step 8600\n",
      "step 8500-8599, precision 0.619285, recall 0.706247, f_score 0.659913\n",
      "=== total 8600 match 2209\n",
      ">>> step 8700\n",
      "step 8600-8699, precision 0.613453, recall 0.706247, f_score 0.656587\n",
      "=== total 8700 match 2230\n",
      ">>> step 8800\n",
      "step 8700-8799, precision 0.613174, recall 0.706572, f_score 0.656568\n",
      "=== total 8800 match 2262\n",
      ">>> step 8900\n",
      "step 8800-8899, precision 0.609938, recall 0.706572, f_score 0.654709\n",
      "=== total 8900 match 2274\n",
      ">>> step 9000\n",
      "step 8900-8999, precision 0.608848, recall 0.703441, f_score 0.652735\n",
      "=== total 9000 match 2283\n",
      ">>> step 9100\n",
      "step 9000-9099, precision 0.605744, recall 0.699497, f_score 0.649254\n",
      "=== total 9100 match 2298\n",
      ">>> step 9200\n",
      "step 9100-9199, precision 0.603381, recall 0.699497, f_score 0.647894\n",
      "=== total 9200 match 2307\n",
      ">>> step 9300\n",
      "step 9200-9299, precision 0.597425, recall 0.699497, f_score 0.644444\n",
      "=== total 9300 match 2330\n",
      ">>> step 9400\n",
      "step 9300-9399, precision 0.592845, recall 0.699497, f_score 0.641770\n",
      "=== total 9400 match 2348\n",
      ">>> step 9500\n",
      "step 9400-9499, precision 0.590581, recall 0.699497, f_score 0.640442\n",
      "=== total 9500 match 2357\n",
      ">>> step 9600\n",
      "step 9500-9599, precision 0.587094, recall 0.699497, f_score 0.638386\n",
      "=== total 9600 match 2371\n",
      ">>> step 9700\n",
      "step 9600-9699, precision 0.584138, recall 0.699497, f_score 0.636634\n",
      "=== total 9700 match 2383\n",
      ">>> step 9800\n",
      "step 9700-9799, precision 0.584820, recall 0.698020, f_score 0.636425\n",
      "=== total 9800 match 2411\n",
      ">>> step 9900\n",
      "step 9800-9899, precision 0.583882, recall 0.687318, f_score 0.631392\n",
      "=== total 9900 match 2432\n",
      ">>> step 10000\n",
      "step 9900-9999, precision 0.589516, recall 0.690926, f_score 0.636205\n",
      "=== total 10000 match 2480\n",
      ">>> step 10100\n",
      "step 10000-10099, precision 0.588871, recall 0.679132, f_score 0.630789\n",
      "=== total 10100 match 2498\n",
      ">>> step 10200\n",
      "step 10100-10199, precision 0.586056, recall 0.679132, f_score 0.629170\n",
      "=== total 10200 match 2510\n",
      ">>> step 10300\n",
      "step 10200-10299, precision 0.583927, recall 0.670455, f_score 0.624207\n",
      "=== total 10300 match 2526\n",
      ">>> step 10400\n",
      "step 10300-10399, precision 0.585127, recall 0.666815, f_score 0.623306\n",
      "=== total 10400 match 2555\n",
      ">>> step 10500\n",
      "step 10400-10499, precision 0.583496, recall 0.657168, f_score 0.618144\n",
      "=== total 10500 match 2569\n",
      ">>> step 10600\n",
      "step 10500-10599, precision 0.584971, recall 0.647611, f_score 0.614699\n",
      "=== total 10600 match 2595\n",
      ">>> step 10700\n",
      "step 10600-10699, precision 0.587452, recall 0.645363, f_score 0.615048\n",
      "=== total 10700 match 2630\n",
      ">>> step 10800\n",
      "step 10700-10799, precision 0.587703, recall 0.634623, f_score 0.610262\n",
      "=== total 10800 match 2651\n",
      ">>> step 10900\n",
      "step 10800-10899, precision 0.588827, recall 0.633667, f_score 0.610425\n",
      "=== total 10900 match 2685\n",
      ">>> step 11000\n",
      "step 10900-10999, precision 0.589583, recall 0.628346, f_score 0.608348\n",
      "=== total 11000 match 2707\n",
      ">>> step 11100\n",
      "step 11000-11099, precision 0.589404, recall 0.623346, f_score 0.605900\n",
      "=== total 11100 match 2718\n",
      ">>> step 11200\n",
      "step 11100-11199, precision 0.586813, recall 0.617103, f_score 0.601577\n",
      "=== total 11200 match 2730\n",
      ">>> step 11300\n",
      "step 11200-11299, precision 0.584425, recall 0.612510, f_score 0.598138\n",
      "=== total 11300 match 2748\n",
      ">>> step 11400\n",
      "step 11300-11399, precision 0.584444, recall 0.609006, f_score 0.596472\n",
      "=== total 11400 match 2777\n",
      ">>> step 11500\n",
      "step 11400-11499, precision 0.583753, recall 0.601927, f_score 0.592701\n",
      "=== total 11500 match 2782\n",
      ">>> step 11600\n",
      "step 11500-11599, precision 0.582496, recall 0.594872, f_score 0.588619\n",
      "=== total 11600 match 2788\n",
      ">>> step 11700\n",
      "step 11600-11699, precision 0.586219, recall 0.596763, f_score 0.591444\n",
      "=== total 11700 match 2830\n",
      ">>> step 11800\n",
      "step 11700-11799, precision 0.584853, recall 0.595714, f_score 0.590234\n",
      "=== total 11800 match 2852\n",
      ">>> step 11900\n",
      "step 11800-11899, precision 0.591143, recall 0.599791, f_score 0.595436\n",
      "=== total 11900 match 2913\n",
      ">>> step 12000\n",
      "step 11900-11999, precision 0.589377, recall 0.599377, f_score 0.594335\n",
      "=== total 12000 match 2937\n",
      ">>> step 12100\n",
      "step 12000-12099, precision 0.591919, recall 0.595932, f_score 0.593919\n",
      "=== total 12100 match 2970\n",
      ">>> step 12200\n",
      "step 12100-12199, precision 0.592272, recall 0.595845, f_score 0.594053\n",
      "=== total 12200 match 3002\n",
      ">>> step 12300\n",
      "step 12200-12299, precision 0.593699, recall 0.599007, f_score 0.596341\n",
      "=== total 12300 match 3047\n",
      ">>> step 12400\n",
      "step 12300-12399, precision 0.590731, recall 0.596572, f_score 0.593637\n",
      "=== total 12400 match 3064\n",
      ">>> step 12500\n",
      "step 12400-12499, precision 0.589702, recall 0.593160, f_score 0.591426\n",
      "=== total 12500 match 3088\n",
      ">>> step 12600\n",
      "step 12500-12599, precision 0.586285, recall 0.593160, f_score 0.589702\n",
      "=== total 12600 match 3106\n",
      ">>> step 12700\n",
      "step 12600-12699, precision 0.585801, recall 0.595662, f_score 0.590690\n",
      "=== total 12700 match 3141\n",
      ">>> step 12800\n",
      "step 12700-12799, precision 0.586608, recall 0.599037, f_score 0.592757\n",
      "=== total 12800 match 3181\n",
      ">>> step 12900\n",
      "step 12800-12899, precision 0.588892, recall 0.603114, f_score 0.595918\n",
      "=== total 12900 match 3223\n",
      ">>> step 13000\n",
      "step 12900-12999, precision 0.585441, recall 0.603114, f_score 0.594146\n",
      "=== total 13000 match 3242\n",
      ">>> step 13100\n",
      "step 13000-13099, precision 0.582387, recall 0.603114, f_score 0.592569\n",
      "=== total 13100 match 3259\n",
      ">>> step 13200\n",
      "step 13100-13199, precision 0.581332, recall 0.604872, f_score 0.592868\n",
      "=== total 13200 match 3289\n",
      ">>> step 13300\n",
      "step 13200-13299, precision 0.580217, recall 0.605984, f_score 0.592821\n",
      "=== total 13300 match 3316\n",
      ">>> step 13400\n",
      "step 13300-13399, precision 0.581049, recall 0.609185, f_score 0.594784\n",
      "=== total 13400 match 3356\n",
      ">>> step 13500\n",
      "step 13400-13499, precision 0.578120, recall 0.609185, f_score 0.593246\n",
      "=== total 13500 match 3373\n",
      ">>> step 13600\n",
      "step 13500-13599, precision 0.581117, recall 0.613770, f_score 0.596997\n",
      "=== total 13600 match 3421\n",
      ">>> step 13700\n",
      "step 13600-13699, precision 0.581206, recall 0.615479, f_score 0.597852\n",
      "=== total 13700 match 3448\n",
      ">>> step 13800\n",
      "step 13700-13799, precision 0.579781, recall 0.616539, f_score 0.597595\n",
      "=== total 13800 match 3472\n",
      ">>> step 13900\n",
      "step 13800-13899, precision 0.578722, recall 0.616530, f_score 0.597028\n",
      "=== total 13900 match 3506\n",
      ">>> step 14000\n",
      "step 13900-13999, precision 0.583170, recall 0.622269, f_score 0.602085\n",
      "=== total 14000 match 3565\n",
      ">>> step 14100\n",
      "step 14000-14099, precision 0.582848, recall 0.623701, f_score 0.602582\n",
      "=== total 14100 match 3603\n",
      ">>> step 14200\n",
      "step 14100-14199, precision 0.583219, recall 0.625184, f_score 0.603473\n",
      "=== total 14200 match 3635\n",
      ">>> step 14300\n",
      "step 14200-14299, precision 0.583697, recall 0.626573, f_score 0.604375\n",
      "=== total 14300 match 3668\n",
      ">>> step 14400\n",
      "step 14300-14399, precision 0.579902, recall 0.626573, f_score 0.602335\n",
      "=== total 14400 match 3692\n",
      ">>> step 14500\n",
      "step 14400-14499, precision 0.582620, recall 0.630680, f_score 0.605698\n",
      "=== total 14500 match 3740\n",
      ">>> step 14600\n",
      "step 14500-14599, precision 0.581340, recall 0.628268, f_score 0.603893\n",
      "=== total 14600 match 3762\n",
      ">>> step 14700\n",
      "step 14600-14699, precision 0.578794, recall 0.622052, f_score 0.599644\n",
      "=== total 14700 match 3782\n",
      ">>> step 14800\n",
      "step 14700-14799, precision 0.575750, recall 0.622052, f_score 0.598006\n",
      "=== total 14800 match 3802\n",
      ">>> step 14900\n",
      "step 14800-14899, precision 0.578235, recall 0.625458, f_score 0.600920\n",
      "=== total 14900 match 3841\n",
      ">>> step 15000\n",
      "step 14900-14999, precision 0.575240, recall 0.625458, f_score 0.599298\n",
      "=== total 15000 match 3861\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> step 15100\n",
      "step 15000-15099, precision 0.575071, recall 0.626543, f_score 0.599705\n",
      "=== total 15100 match 3883\n",
      ">>> step 15200\n",
      "step 15100-15199, precision 0.575128, recall 0.626186, f_score 0.599572\n",
      "=== total 15200 match 3900\n",
      ">>> step 15300\n",
      "step 15200-15299, precision 0.572704, recall 0.622056, f_score 0.596361\n",
      "=== total 15300 match 3920\n",
      ">>> step 15400\n",
      "step 15300-15399, precision 0.570376, recall 0.622056, f_score 0.595096\n",
      "=== total 15400 match 3936\n",
      ">>> step 15500\n",
      "step 15400-15499, precision 0.570493, recall 0.621333, f_score 0.594829\n",
      "=== total 15500 match 3972\n",
      ">>> step 15600\n",
      "step 15500-15599, precision 0.568490, recall 0.621333, f_score 0.593738\n",
      "=== total 15600 match 3986\n",
      ">>> step 15700\n",
      "step 15600-15699, precision 0.566075, recall 0.621333, f_score 0.592418\n",
      "=== total 15700 match 4003\n",
      ">>> step 15800\n",
      "step 15700-15799, precision 0.570234, recall 0.625472, f_score 0.596577\n",
      "=== total 15800 match 4065\n",
      ">>> step 15900\n",
      "step 15800-15899, precision 0.571533, recall 0.627782, f_score 0.598339\n",
      "=== total 15900 match 4096\n",
      ">>> step 16000\n",
      "step 15900-15999, precision 0.570697, recall 0.624600, f_score 0.596433\n",
      "=== total 16000 match 4102\n",
      ">>> step 16100\n",
      "step 16000-16099, precision 0.569864, recall 0.622441, f_score 0.594993\n",
      "=== total 16100 match 4108\n",
      ">>> step 16200\n",
      "step 16100-16199, precision 0.569586, recall 0.620297, f_score 0.593861\n",
      "=== total 16200 match 4110\n",
      ">>> step 16300\n",
      "step 16200-16299, precision 0.569033, recall 0.620297, f_score 0.593560\n",
      "=== total 16300 match 4114\n",
      ">>> step 16400\n",
      "step 16300-16399, precision 0.568999, recall 0.618431, f_score 0.592686\n",
      "=== total 16400 match 4116\n",
      ">>> step 16500\n",
      "step 16400-16499, precision 0.568723, recall 0.616316, f_score 0.591564\n",
      "=== total 16500 match 4118\n",
      ">>> step 16600\n",
      "step 16500-16599, precision 0.567620, recall 0.616316, f_score 0.590966\n",
      "=== total 16600 match 4126\n",
      ">>> step 16700\n",
      "step 16600-16699, precision 0.567070, recall 0.616316, f_score 0.590668\n",
      "=== total 16700 match 4130\n",
      ">>> step 16800\n",
      "step 16700-16799, precision 0.566933, recall 0.610214, f_score 0.587778\n",
      "=== total 16800 match 4131\n",
      ">>> step 16900\n",
      "step 16800-16899, precision 0.566763, recall 0.605426, f_score 0.585457\n",
      "=== total 16900 match 4134\n",
      ">>> step 17000\n",
      "step 16900-16999, precision 0.566763, recall 0.602469, f_score 0.584071\n",
      "=== total 17000 match 4134\n",
      ">>> step 17100\n",
      "step 17000-17099, precision 0.565942, recall 0.602469, f_score 0.583634\n",
      "=== total 17100 match 4140\n",
      ">>> step 17200\n",
      "step 17100-17199, precision 0.565259, recall 0.602469, f_score 0.583271\n",
      "=== total 17200 match 4145\n",
      ">>> step 17300\n",
      "step 17200-17299, precision 0.565123, recall 0.602469, f_score 0.583199\n",
      "=== total 17300 match 4146\n",
      ">>> step 17400\n",
      "step 17300-17399, precision 0.564714, recall 0.602469, f_score 0.582981\n",
      "=== total 17400 match 4149\n",
      ">>> step 17500\n",
      "step 17400-17499, precision 0.564170, recall 0.602469, f_score 0.582691\n",
      "=== total 17500 match 4153\n",
      ">>> step 17600\n",
      "step 17500-17599, precision 0.563763, recall 0.602469, f_score 0.582474\n",
      "=== total 17600 match 4156\n",
      ">>> step 17700\n",
      "step 17600-17699, precision 0.562951, recall 0.602469, f_score 0.582039\n",
      "=== total 17700 match 4162\n",
      ">>> step 17800\n",
      "step 17700-17799, precision 0.562815, recall 0.602469, f_score 0.581967\n",
      "=== total 17800 match 4163\n",
      ">>> step 17900\n",
      "step 17800-17899, precision 0.562275, recall 0.600461, f_score 0.580741\n",
      "=== total 17900 match 4167\n",
      ">>> step 18000\n",
      "step 17900-17999, precision 0.561736, recall 0.596487, f_score 0.578590\n",
      "=== total 18000 match 4171\n",
      ">>> step 18100\n",
      "step 18000-18099, precision 0.561467, recall 0.594519, f_score 0.577520\n",
      "=== total 18100 match 4173\n",
      ">>> step 18200\n",
      "step 18100-18199, precision 0.560660, recall 0.594519, f_score 0.577094\n",
      "=== total 18200 match 4179\n",
      ">>> step 18300\n",
      "step 18200-18299, precision 0.560124, recall 0.590623, f_score 0.574969\n",
      "=== total 18300 match 4183\n",
      ">>> step 18400\n",
      "step 18300-18399, precision 0.559589, recall 0.586777, f_score 0.572861\n",
      "=== total 18400 match 4187\n",
      ">>> step 18500\n",
      "step 18400-18499, precision 0.558922, recall 0.582981, f_score 0.570698\n",
      "=== total 18500 match 4192\n",
      ">>> step 18600\n",
      "step 18500-18599, precision 0.558522, recall 0.577520, f_score 0.567862\n",
      "=== total 18600 match 4195\n",
      ">>> step 18700\n",
      "step 18600-18699, precision 0.557990, recall 0.573843, f_score 0.565805\n",
      "=== total 18700 match 4199\n",
      ">>> step 18800\n",
      "step 18700-18799, precision 0.559659, recall 0.572813, f_score 0.566160\n",
      "=== total 18800 match 4224\n",
      ">>> step 18900\n",
      "step 18800-18899, precision 0.561337, recall 0.570745, f_score 0.566002\n",
      "=== total 18900 match 4247\n",
      ">>> step 19000\n",
      "step 18900-18999, precision 0.560809, recall 0.567619, f_score 0.564194\n",
      "=== total 19000 match 4251\n",
      ">>> step 19100\n",
      "step 19000-19099, precision 0.564479, recall 0.570186, f_score 0.567318\n",
      "=== total 19100 match 4296\n",
      ">>> step 19200\n",
      "step 19100-19199, precision 0.563953, recall 0.568448, f_score 0.566192\n",
      "=== total 19200 match 4300\n",
      ">>> step 19300\n",
      "step 19200-19299, precision 0.562776, recall 0.568448, f_score 0.565598\n",
      "=== total 19300 match 4309\n",
      ">>> step 19400\n",
      "step 19300-19399, precision 0.562486, recall 0.565238, f_score 0.563858\n",
      "=== total 19400 match 4313\n",
      ">>> step 19500\n",
      "step 19400-19499, precision 0.562630, recall 0.559650, f_score 0.561136\n",
      "=== total 19500 match 4319\n",
      ">>> step 19600\n",
      "step 19500-19599, precision 0.562500, recall 0.553279, f_score 0.557851\n",
      "=== total 19600 match 4320\n",
      ">>> step 19700\n",
      "step 19600-19699, precision 0.562240, recall 0.547051, f_score 0.554541\n",
      "=== total 19700 match 4322\n",
      ">>> step 19800\n",
      "step 19700-19799, precision 0.563423, recall 0.541805, f_score 0.552402\n",
      "=== total 19800 match 4336\n",
      ">>> step 19900\n",
      "step 19800-19899, precision 0.566758, recall 0.542688, f_score 0.554462\n",
      "=== total 19900 match 4374\n",
      ">>> step 20000\n",
      "step 19900-19999, precision 0.566370, recall 0.537511, f_score 0.551563\n",
      "=== total 20000 match 4377\n",
      ">>> step 20100\n",
      "step 20000-20099, precision 0.565723, recall 0.536000, f_score 0.550461\n",
      "=== total 20100 match 4382\n",
      ">>> step 20200\n",
      "step 20100-20199, precision 0.564950, recall 0.536000, f_score 0.550094\n",
      "=== total 20200 match 4388\n",
      ">>> step 20300\n",
      "step 20200-20299, precision 0.563153, recall 0.536000, f_score 0.549241\n",
      "=== total 20300 match 4402\n",
      ">>> step 20400\n",
      "step 20300-20399, precision 0.561495, recall 0.536000, f_score 0.548451\n",
      "=== total 20400 match 4415\n",
      ">>> step 20500\n",
      "step 20400-20499, precision 0.561388, recall 0.537301, f_score 0.549080\n",
      "=== total 20500 match 4439\n",
      ">>> step 20600\n",
      "step 20500-20599, precision 0.561911, recall 0.538594, f_score 0.550005\n",
      "=== total 20600 match 4458\n",
      ">>> step 20700\n",
      "step 20600-20699, precision 0.561439, recall 0.539386, f_score 0.550192\n",
      "=== total 20700 match 4476\n",
      ">>> step 20800\n",
      "step 20700-20799, precision 0.560721, recall 0.539713, f_score 0.550016\n",
      "=== total 20800 match 4496\n",
      ">>> step 20900\n",
      "step 20800-20899, precision 0.559973, recall 0.539713, f_score 0.549657\n",
      "=== total 20900 match 4502\n",
      ">>> step 21000\n",
      "step 20900-20999, precision 0.559228, recall 0.539713, f_score 0.549297\n",
      "=== total 21000 match 4508\n",
      ">>> step 21100\n",
      "step 21000-21099, precision 0.559083, recall 0.541071, f_score 0.549930\n",
      "=== total 21100 match 4536\n",
      ">>> step 21200\n",
      "step 21100-21199, precision 0.561162, recall 0.544049, f_score 0.552473\n",
      "=== total 21200 match 4578\n",
      ">>> step 21300\n",
      "step 21200-21299, precision 0.563338, recall 0.547479, f_score 0.555295\n",
      "=== total 21300 match 4626\n",
      ">>> step 21400\n",
      "step 21300-21399, precision 0.561880, recall 0.547479, f_score 0.554586\n",
      "=== total 21400 match 4638\n",
      ">>> step 21500\n",
      "step 21400-21499, precision 0.561033, recall 0.547479, f_score 0.554173\n",
      "=== total 21500 match 4645\n",
      ">>> step 21600\n",
      "step 21500-21599, precision 0.561211, recall 0.547454, f_score 0.554248\n",
      "=== total 21600 match 4656\n",
      ">>> step 21700\n",
      "step 21600-21699, precision 0.559889, recall 0.547454, f_score 0.553602\n",
      "=== total 21700 match 4667\n",
      ">>> step 21800\n",
      "step 21700-21799, precision 0.558930, recall 0.545284, f_score 0.552023\n",
      "=== total 21800 match 4675\n",
      ">>> step 21900\n",
      "step 21800-21899, precision 0.557946, recall 0.543587, f_score 0.550673\n",
      "=== total 21900 match 4694\n",
      ">>> step 22000\n",
      "step 21900-21999, precision 0.559458, recall 0.543362, f_score 0.551293\n",
      "=== total 22000 match 4726\n",
      ">>> step 22100\n",
      "step 22000-22099, precision 0.558842, recall 0.543344, f_score 0.550984\n",
      "=== total 22100 match 4733\n",
      ">>> step 22200\n",
      "step 22100-22199, precision 0.557704, recall 0.542092, f_score 0.549788\n",
      "=== total 22200 match 4757\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> step 22300\n",
      "step 22200-22299, precision 0.557236, recall 0.542092, f_score 0.549560\n",
      "=== total 22300 match 4761\n",
      ">>> step 22400\n",
      "step 22300-22399, precision 0.558038, recall 0.543293, f_score 0.550566\n",
      "=== total 22400 match 4790\n",
      ">>> step 22500\n",
      "step 22400-22499, precision 0.559644, recall 0.545436, f_score 0.552449\n",
      "=== total 22500 match 4837\n",
      ">>> step 22600\n",
      "step 22500-22599, precision 0.561011, recall 0.545982, f_score 0.553394\n",
      "=== total 22600 match 4868\n",
      ">>> step 22700\n",
      "step 22600-22699, precision 0.563160, recall 0.549544, f_score 0.556269\n",
      "=== total 22700 match 4924\n",
      ">>> step 22800\n",
      "step 22700-22799, precision 0.565792, recall 0.552838, f_score 0.559240\n",
      "=== total 22800 match 4993\n",
      ">>> step 22900\n",
      "step 22800-22899, precision 0.566409, recall 0.554088, f_score 0.560181\n",
      "=== total 22900 match 5037\n",
      ">>> step 23000\n",
      "step 22900-22999, precision 0.566528, recall 0.550906, f_score 0.558608\n",
      "=== total 23000 match 5043\n",
      ">>> step 23100\n",
      "step 23000-23099, precision 0.566079, recall 0.550906, f_score 0.558390\n",
      "=== total 23100 match 5047\n",
      ">>> step 23200\n",
      "step 23100-23199, precision 0.564848, recall 0.550164, f_score 0.557409\n",
      "=== total 23200 match 5058\n",
      ">>> step 23300\n",
      "step 23200-23299, precision 0.564179, recall 0.548053, f_score 0.555999\n",
      "=== total 23300 match 5064\n",
      ">>> step 23400\n",
      "step 23300-23399, precision 0.563622, recall 0.548053, f_score 0.555728\n",
      "=== total 23400 match 5069\n",
      ">>> step 23500\n",
      "step 23400-23499, precision 0.563178, recall 0.547318, f_score 0.555135\n",
      "=== total 23500 match 5073\n",
      ">>> step 23600\n",
      "step 23500-23599, precision 0.562685, recall 0.546341, f_score 0.554392\n",
      "=== total 23600 match 5081\n",
      ">>> step 23700\n",
      "step 23600-23699, precision 0.561690, recall 0.546341, f_score 0.553909\n",
      "=== total 23700 match 5090\n",
      ">>> step 23800\n",
      "step 23700-23799, precision 0.561359, recall 0.546341, f_score 0.553748\n",
      "=== total 23800 match 5093\n",
      ">>> step 23900\n",
      "step 23800-23899, precision 0.560918, recall 0.544364, f_score 0.552517\n",
      "=== total 23900 match 5097\n",
      ">>> step 24000\n",
      "step 23900-23999, precision 0.560039, recall 0.544364, f_score 0.552090\n",
      "=== total 24000 match 5105\n",
      ">>> step 24100\n",
      "step 24000-24099, precision 0.560281, recall 0.545385, f_score 0.552733\n",
      "=== total 24100 match 5126\n",
      ">>> step 24200\n",
      "step 24100-24199, precision 0.559837, recall 0.544886, f_score 0.552260\n",
      "=== total 24200 match 5139\n",
      ">>> step 24300\n",
      "step 24200-24299, precision 0.559806, recall 0.544579, f_score 0.552087\n",
      "=== total 24300 match 5150\n",
      ">>> step 24400\n",
      "step 24300-24399, precision 0.559263, recall 0.544579, f_score 0.551823\n",
      "=== total 24400 match 5155\n",
      ">>> step 24500\n",
      "step 24400-24499, precision 0.558806, recall 0.544665, f_score 0.551645\n",
      "=== total 24500 match 5161\n",
      ">>> step 24600\n",
      "step 24500-24599, precision 0.559011, recall 0.545112, f_score 0.551974\n",
      "=== total 24600 match 5177\n",
      ">>> step 24700\n",
      "step 24600-24699, precision 0.558579, recall 0.545112, f_score 0.551764\n",
      "=== total 24700 match 5181\n",
      ">>> step 24800\n",
      "step 24700-24799, precision 0.558682, recall 0.543393, f_score 0.550931\n",
      "=== total 24800 match 5189\n",
      ">>> step 24900\n",
      "step 24800-24899, precision 0.558422, recall 0.539320, f_score 0.548704\n",
      "=== total 24900 match 5195\n",
      ">>> step 25000\n",
      "step 24900-24999, precision 0.558207, recall 0.535734, f_score 0.546740\n",
      "=== total 25000 match 5197\n",
      ">>> step 25100\n",
      "step 25000-25099, precision 0.557885, recall 0.532880, f_score 0.545096\n",
      "=== total 25100 match 5200\n",
      ">>> step 25200\n",
      "step 25100-25199, precision 0.557858, recall 0.531056, f_score 0.544127\n",
      "=== total 25200 match 5211\n",
      ">>> step 25300\n",
      "step 25200-25299, precision 0.557644, recall 0.526440, f_score 0.541593\n",
      "=== total 25300 match 5213\n",
      ">>> step 25400\n",
      "step 25300-25399, precision 0.556683, recall 0.525203, f_score 0.540485\n",
      "=== total 25400 match 5222\n",
      ">>> step 25500\n",
      "step 25400-25499, precision 0.556746, recall 0.519650, f_score 0.537559\n",
      "=== total 25500 match 5225\n",
      ">>> step 25600\n",
      "step 25500-25599, precision 0.556746, recall 0.516696, f_score 0.535974\n",
      "=== total 25600 match 5225\n",
      ">>> step 25700\n",
      "step 25600-25699, precision 0.556640, recall 0.510709, f_score 0.532686\n",
      "=== total 25700 match 5226\n",
      ">>> step 25800\n",
      "step 25700-25799, precision 0.556681, recall 0.505907, f_score 0.530081\n",
      "=== total 25800 match 5231\n",
      ">>> step 25900\n",
      "step 25800-25899, precision 0.556044, recall 0.505907, f_score 0.529792\n",
      "=== total 25900 match 5237\n",
      ">>> step 26000\n",
      "step 25900-25999, precision 0.555195, recall 0.505907, f_score 0.529406\n",
      "=== total 26000 match 5245\n",
      ">>> step 26100\n",
      "step 26000-26099, precision 0.554455, recall 0.505907, f_score 0.529070\n",
      "=== total 26100 match 5252\n",
      ">>> step 26200\n",
      "step 26100-26199, precision 0.553507, recall 0.504679, f_score 0.527967\n",
      "=== total 26200 match 5261\n",
      ">>> step 26300\n",
      "step 26200-26299, precision 0.552771, recall 0.503545, f_score 0.527011\n",
      "=== total 26300 match 5268\n",
      ">>> step 26400\n",
      "step 26300-26399, precision 0.552247, recall 0.502329, f_score 0.526107\n",
      "=== total 26400 match 5273\n",
      ">>> step 26500\n",
      "step 26400-26499, precision 0.551306, recall 0.502329, f_score 0.525679\n",
      "=== total 26500 match 5282\n",
      ">>> step 26600\n",
      "step 26500-26599, precision 0.550473, recall 0.502329, f_score 0.525300\n",
      "=== total 26600 match 5290\n",
      ">>> step 26700\n",
      "step 26600-26699, precision 0.550161, recall 0.502329, f_score 0.525158\n",
      "=== total 26700 match 5293\n",
      ">>> step 26800\n",
      "step 26700-26799, precision 0.549745, recall 0.500516, f_score 0.523977\n",
      "=== total 26800 match 5297\n",
      ">>> step 26900\n",
      "step 26800-26899, precision 0.549020, recall 0.500516, f_score 0.523647\n",
      "=== total 26900 match 5304\n",
      ">>> step 27000\n",
      "step 26900-26999, precision 0.548193, recall 0.500516, f_score 0.523270\n",
      "=== total 27000 match 5312\n",
      ">>> step 27100\n",
      "step 27000-27099, precision 0.549269, recall 0.502227, f_score 0.524696\n",
      "=== total 27100 match 5338\n",
      ">>> step 27200\n",
      "step 27100-27199, precision 0.552695, recall 0.506621, f_score 0.528656\n",
      "=== total 27200 match 5399\n",
      ">>> step 27300\n",
      "step 27200-27299, precision 0.552184, recall 0.506621, f_score 0.528422\n",
      "=== total 27300 match 5404\n",
      ">>> step 27400\n",
      "step 27300-27399, precision 0.552874, recall 0.507268, f_score 0.529090\n",
      "=== total 27400 match 5428\n",
      ">>> step 27500\n",
      "step 27400-27499, precision 0.554540, recall 0.510084, f_score 0.531384\n",
      "=== total 27500 match 5473\n",
      ">>> step 27600\n",
      "step 27500-27599, precision 0.554237, recall 0.510084, f_score 0.531245\n",
      "=== total 27600 match 5476\n",
      ">>> step 27700\n",
      "step 27600-27699, precision 0.553126, recall 0.510084, f_score 0.530734\n",
      "=== total 27700 match 5487\n",
      ">>> step 27800\n",
      "step 27700-27799, precision 0.553863, recall 0.511643, f_score 0.531917\n",
      "=== total 27800 match 5514\n",
      ">>> step 27900\n",
      "step 27800-27899, precision 0.553461, recall 0.511643, f_score 0.531732\n",
      "=== total 27900 match 5518\n",
      ">>> step 28000\n",
      "step 27900-27999, precision 0.552960, recall 0.511386, f_score 0.531361\n",
      "=== total 28000 match 5523\n",
      ">>> step 28100\n",
      "step 28000-28099, precision 0.553088, recall 0.509058, f_score 0.530160\n",
      "=== total 28100 match 5538\n",
      ">>> step 28200\n",
      "step 28100-28199, precision 0.553291, recall 0.506354, f_score 0.528783\n",
      "=== total 28200 match 5545\n",
      ">>> step 28300\n",
      "step 28200-28299, precision 0.552892, recall 0.504191, f_score 0.527420\n",
      "=== total 28300 match 5549\n",
      ">>> step 28400\n",
      "step 28300-28399, precision 0.552518, recall 0.502700, f_score 0.526433\n",
      "=== total 28400 match 5560\n",
      ">>> step 28500\n",
      "step 28400-28499, precision 0.551805, recall 0.500733, f_score 0.525030\n",
      "=== total 28500 match 5569\n",
      ">>> step 28600\n",
      "step 28500-28599, precision 0.551013, recall 0.499512, f_score 0.524000\n",
      "=== total 28600 match 5577\n",
      ">>> step 28700\n",
      "step 28600-28699, precision 0.550896, recall 0.494769, f_score 0.521326\n",
      "=== total 28700 match 5580\n",
      ">>> step 28800\n",
      "step 28700-28799, precision 0.550107, recall 0.492155, f_score 0.519520\n",
      "=== total 28800 match 5588\n",
      ">>> step 28900\n",
      "step 28800-28899, precision 0.549321, recall 0.487240, f_score 0.516422\n",
      "=== total 28900 match 5596\n",
      ">>> step 29000\n",
      "step 28900-28999, precision 0.548617, recall 0.484023, f_score 0.514300\n",
      "=== total 29000 match 5605\n",
      ">>> step 29100\n",
      "step 29000-29099, precision 0.548226, recall 0.484023, f_score 0.514128\n",
      "=== total 29100 match 5609\n",
      ">>> step 29200\n",
      "step 29100-29199, precision 0.546958, recall 0.483491, f_score 0.513270\n",
      "=== total 29200 match 5622\n",
      ">>> step 29300\n",
      "step 29200-29299, precision 0.545955, recall 0.483273, f_score 0.512705\n",
      "=== total 29300 match 5636\n",
      ">>> step 29400\n",
      "step 29300-29399, precision 0.544988, recall 0.483273, f_score 0.512278\n",
      "=== total 29400 match 5646\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> step 29500\n",
      "step 29400-29499, precision 0.544682, recall 0.482899, f_score 0.511933\n",
      "=== total 29500 match 5651\n",
      ">>> step 29600\n",
      "step 29500-29599, precision 0.544989, recall 0.483563, f_score 0.512442\n",
      "=== total 29600 match 5668\n",
      ">>> step 29700\n",
      "step 29600-29699, precision 0.544221, recall 0.483563, f_score 0.512102\n",
      "=== total 29700 match 5676\n",
      ">>> step 29800\n",
      "step 29700-29799, precision 0.543360, recall 0.483563, f_score 0.511720\n",
      "=== total 29800 match 5685\n",
      ">>> step 29900\n",
      "step 29800-29899, precision 0.542596, recall 0.483563, f_score 0.511382\n",
      "=== total 29900 match 5693\n",
      ">>> step 30000\n",
      "step 29900-29999, precision 0.541455, recall 0.483563, f_score 0.510874\n",
      "=== total 30000 match 5705\n",
      ">>> step 30100\n",
      "step 30000-30099, precision 0.540035, recall 0.483563, f_score 0.510241\n",
      "=== total 30100 match 5720\n",
      ">>> step 30200\n",
      "step 30100-30199, precision 0.538435, recall 0.483034, f_score 0.509232\n",
      "=== total 30200 match 5737\n",
      ">>> step 30300\n",
      "step 30200-30299, precision 0.537591, recall 0.482505, f_score 0.508561\n",
      "=== total 30300 match 5746\n",
      ">>> step 30400\n",
      "step 30300-30399, precision 0.536564, recall 0.481453, f_score 0.507517\n",
      "=== total 30400 match 5757\n",
      ">>> step 30500\n",
      "step 30400-30499, precision 0.535820, recall 0.480330, f_score 0.506560\n",
      "=== total 30500 match 5765\n",
      ">>> step 30600\n",
      "step 30500-30599, precision 0.534799, recall 0.479361, f_score 0.505565\n",
      "=== total 30600 match 5776\n",
      ">>> step 30700\n",
      "step 30600-30699, precision 0.533218, recall 0.477958, f_score 0.504078\n",
      "=== total 30700 match 5795\n",
      ">>> step 30800\n",
      "step 30700-30799, precision 0.532391, recall 0.477958, f_score 0.503709\n",
      "=== total 30800 match 5804\n",
      ">>> step 30900\n",
      "step 30800-30899, precision 0.532499, recall 0.479167, f_score 0.504427\n",
      "=== total 30900 match 5831\n",
      ">>> step 31000\n",
      "step 30900-30999, precision 0.532981, recall 0.480486, f_score 0.505374\n",
      "=== total 31000 match 5867\n",
      ">>> step 31100\n",
      "step 31000-31099, precision 0.533740, recall 0.479805, f_score 0.505338\n",
      "=== total 31100 match 5898\n",
      ">>> step 31200\n",
      "step 31100-31199, precision 0.533266, recall 0.479160, f_score 0.504767\n",
      "=== total 31200 match 5907\n",
      ">>> step 31300\n",
      "step 31200-31299, precision 0.532634, recall 0.478215, f_score 0.503960\n",
      "=== total 31300 match 5914\n",
      ">>> step 31400\n",
      "step 31300-31399, precision 0.532481, recall 0.478452, f_score 0.504022\n",
      "=== total 31400 match 5942\n",
      ">>> step 31500\n",
      "step 31400-31499, precision 0.532011, recall 0.476879, f_score 0.502939\n",
      "=== total 31500 match 5951\n",
      ">>> step 31600\n",
      "step 31500-31599, precision 0.531890, recall 0.473621, f_score 0.501067\n",
      "=== total 31600 match 5958\n",
      ">>> step 31700\n",
      "step 31600-31699, precision 0.531669, recall 0.472384, f_score 0.500276\n",
      "=== total 31700 match 5968\n",
      ">>> step 31800\n",
      "step 31700-31799, precision 0.531370, recall 0.470867, f_score 0.499293\n",
      "=== total 31800 match 5977\n",
      ">>> step 31900\n",
      "step 31800-31899, precision 0.530864, recall 0.468837, f_score 0.497927\n",
      "=== total 31900 match 5994\n",
      ">>> step 32000\n",
      "step 31900-31999, precision 0.530068, recall 0.468837, f_score 0.497576\n",
      "=== total 32000 match 6003\n",
      ">>> step 32100\n",
      "step 32000-32099, precision 0.528220, recall 0.468837, f_score 0.496761\n",
      "=== total 32100 match 6024\n",
      ">>> step 32200\n",
      "step 32100-32199, precision 0.527847, recall 0.468810, f_score 0.496580\n",
      "=== total 32200 match 6051\n",
      ">>> step 32300\n",
      "step 32200-32299, precision 0.528339, recall 0.468736, f_score 0.496756\n",
      "=== total 32300 match 6087\n",
      ">>> step 32400\n",
      "step 32300-32399, precision 0.527273, recall 0.467538, f_score 0.495612\n",
      "=== total 32400 match 6105\n",
      ">>> step 32500\n",
      "step 32400-32499, precision 0.527574, recall 0.469388, f_score 0.496783\n",
      "=== total 32500 match 6147\n",
      ">>> step 32600\n",
      "step 32500-32599, precision 0.526119, recall 0.467695, f_score 0.495190\n",
      "=== total 32600 match 6164\n",
      ">>> step 32700\n",
      "step 32600-32699, precision 0.524834, recall 0.465156, f_score 0.493197\n",
      "=== total 32700 match 6181\n",
      ">>> step 32800\n",
      "step 32700-32799, precision 0.525132, recall 0.467143, f_score 0.494443\n",
      "=== total 32800 match 6227\n",
      ">>> step 32900\n",
      "step 32800-32899, precision 0.525678, recall 0.468714, f_score 0.495565\n",
      "=== total 32900 match 6270\n",
      ">>> step 33000\n",
      "step 32900-32999, precision 0.526873, recall 0.471495, f_score 0.497648\n",
      "=== total 33000 match 6326\n",
      ">>> step 33100\n",
      "step 33000-33099, precision 0.529181, recall 0.473005, f_score 0.499519\n",
      "=== total 33100 match 6374\n",
      ">>> step 33200\n",
      "step 33100-33199, precision 0.527773, recall 0.472806, f_score 0.498780\n",
      "=== total 33200 match 6391\n",
      ">>> step 33300\n",
      "step 33200-33299, precision 0.526620, recall 0.472806, f_score 0.498264\n",
      "=== total 33300 match 6405\n",
      ">>> step 33400\n",
      "step 33300-33399, precision 0.525389, recall 0.472343, f_score 0.497456\n",
      "=== total 33400 match 6420\n",
      ">>> step 33500\n",
      "step 33400-33499, precision 0.524572, recall 0.470629, f_score 0.496139\n",
      "=== total 33500 match 6430\n",
      ">>> step 33600\n",
      "step 33500-33599, precision 0.523595, recall 0.470432, f_score 0.495592\n",
      "=== total 33600 match 6442\n",
      ">>> step 33700\n",
      "step 33600-33699, precision 0.522608, recall 0.470056, f_score 0.494941\n",
      "=== total 33700 match 6458\n",
      ">>> step 33800\n",
      "step 33700-33799, precision 0.521638, recall 0.470056, f_score 0.494505\n",
      "=== total 33800 match 6470\n",
      ">>> step 33900\n",
      "step 33800-33899, precision 0.520894, recall 0.469558, f_score 0.493896\n",
      "=== total 33900 match 6485\n",
      ">>> step 34000\n",
      "step 33900-33999, precision 0.519373, recall 0.469558, f_score 0.493211\n",
      "=== total 34000 match 6504\n",
      ">>> step 34100\n",
      "step 34000-34099, precision 0.518496, recall 0.468321, f_score 0.492133\n",
      "=== total 34100 match 6515\n",
      ">>> step 34200\n",
      "step 34100-34199, precision 0.516356, recall 0.468321, f_score 0.491167\n",
      "=== total 34200 match 6542\n",
      ">>> step 34300\n",
      "step 34200-34299, precision 0.515794, recall 0.465244, f_score 0.489217\n",
      "=== total 34300 match 6553\n",
      ">>> step 34400\n",
      "step 34300-34399, precision 0.514616, recall 0.465244, f_score 0.488686\n",
      "=== total 34400 match 6568\n",
      ">>> step 34500\n",
      "step 34400-34499, precision 0.513444, recall 0.465244, f_score 0.488157\n",
      "=== total 34500 match 6583\n",
      ">>> step 34600\n",
      "step 34500-34599, precision 0.512044, recall 0.465244, f_score 0.487523\n",
      "=== total 34600 match 6601\n",
      ">>> step 34700\n",
      "step 34600-34699, precision 0.512338, recall 0.467078, f_score 0.488662\n",
      "=== total 34700 match 6646\n",
      ">>> step 34800\n",
      "step 34700-34799, precision 0.511523, recall 0.468027, f_score 0.488809\n",
      "=== total 34800 match 6682\n",
      ">>> step 34900\n",
      "step 34800-34899, precision 0.512271, recall 0.469914, f_score 0.490179\n",
      "=== total 34900 match 6723\n",
      ">>> step 35000\n",
      "step 34900-34999, precision 0.511335, recall 0.470420, f_score 0.490025\n",
      "=== total 35000 match 6749\n",
      ">>> step 35100\n",
      "step 35000-35099, precision 0.512077, recall 0.472290, f_score 0.491379\n",
      "=== total 35100 match 6790\n",
      ">>> step 35200\n",
      "step 35100-35199, precision 0.513289, recall 0.475000, f_score 0.493403\n",
      "=== total 35200 match 6848\n",
      ">>> step 35300\n",
      "step 35200-35299, precision 0.513301, recall 0.476133, f_score 0.494019\n",
      "=== total 35300 match 6879\n",
      ">>> step 35400\n",
      "step 35300-35399, precision 0.514636, recall 0.478803, f_score 0.496073\n",
      "=== total 35400 match 6935\n",
      ">>> step 35500\n",
      "step 35400-35499, precision 0.514847, recall 0.480198, f_score 0.496919\n",
      "=== total 35500 match 6971\n",
      ">>> step 35600\n",
      "step 35500-35599, precision 0.514110, recall 0.480198, f_score 0.496576\n",
      "=== total 35600 match 6981\n",
      ">>> step 35700\n",
      "step 35600-35699, precision 0.514372, recall 0.480754, f_score 0.496995\n",
      "=== total 35700 match 6993\n",
      ">>> step 35800\n",
      "step 35700-35799, precision 0.514936, recall 0.482345, f_score 0.498108\n",
      "=== total 35800 match 7030\n",
      ">>> step 35900\n",
      "step 35800-35899, precision 0.518962, recall 0.486326, f_score 0.502114\n",
      "=== total 35900 match 7093\n",
      ">>> step 36000\n",
      "step 35900-35999, precision 0.518160, recall 0.487274, f_score 0.502243\n",
      "=== total 36000 match 7131\n",
      ">>> step 36100\n",
      "step 36000-36099, precision 0.518167, recall 0.488152, f_score 0.502711\n",
      "=== total 36100 match 7156\n",
      ">>> step 36200\n",
      "step 36100-36199, precision 0.519338, recall 0.489766, f_score 0.504119\n",
      "=== total 36200 match 7188\n",
      ">>> step 36300\n",
      "step 36200-36299, precision 0.519989, recall 0.491501, f_score 0.505344\n",
      "=== total 36300 match 7229\n",
      ">>> step 36400\n",
      "step 36300-36399, precision 0.521548, recall 0.494213, f_score 0.507513\n",
      "=== total 36400 match 7286\n",
      ">>> step 36500\n",
      "step 36400-36499, precision 0.521674, recall 0.495982, f_score 0.508504\n",
      "=== total 36500 match 7336\n",
      ">>> step 36600\n",
      "step 36500-36599, precision 0.522309, recall 0.498323, f_score 0.510034\n",
      "=== total 36600 match 7396\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> step 36700\n",
      "step 36600-36699, precision 0.523976, recall 0.500385, f_score 0.511909\n",
      "=== total 36700 match 7445\n",
      ">>> step 36800\n",
      "step 36700-36799, precision 0.523062, recall 0.500385, f_score 0.511472\n",
      "=== total 36800 match 7458\n",
      ">>> step 36900\n",
      "step 36800-36899, precision 0.523854, recall 0.502300, f_score 0.512851\n",
      "=== total 36900 match 7504\n",
      ">>> step 37000\n",
      "step 36900-36999, precision 0.522193, recall 0.503503, f_score 0.512678\n",
      "=== total 37000 match 7570\n",
      ">>> step 37100\n",
      "step 37000-37099, precision 0.522956, recall 0.506332, f_score 0.514510\n",
      "=== total 37100 match 7645\n",
      ">>> step 37200\n",
      "step 37100-37199, precision 0.524359, recall 0.509313, f_score 0.516726\n",
      "=== total 37200 match 7718\n",
      ">>> step 37300\n",
      "step 37200-37299, precision 0.521319, recall 0.509313, f_score 0.515246\n",
      "=== total 37300 match 7763\n",
      ">>> step 37400\n",
      "step 37300-37399, precision 0.518447, recall 0.509313, f_score 0.513840\n",
      "=== total 37400 match 7806\n",
      ">>> step 37500\n",
      "step 37400-37499, precision 0.517188, recall 0.509313, f_score 0.513220\n",
      "=== total 37500 match 7825\n",
      ">>> step 37600\n",
      "step 37500-37599, precision 0.516567, recall 0.510411, f_score 0.513471\n",
      "=== total 37600 match 7877\n",
      ">>> step 37700\n",
      "step 37600-37699, precision 0.515194, recall 0.510411, f_score 0.512791\n",
      "=== total 37700 match 7898\n",
      ">>> step 37800\n",
      "step 37700-37799, precision 0.516109, recall 0.513214, f_score 0.514657\n",
      "=== total 37800 match 7977\n",
      ">>> step 37900\n",
      "step 37800-37899, precision 0.514689, recall 0.513214, f_score 0.513950\n",
      "=== total 37900 match 7999\n",
      ">>> step 38000\n",
      "step 37900-37999, precision 0.512256, recall 0.513214, f_score 0.512734\n",
      "=== total 38000 match 8037\n",
      ">>> step 38100\n",
      "step 38000-38099, precision 0.511874, recall 0.511556, f_score 0.511715\n",
      "=== total 38100 match 8043\n",
      ">>> step 38200\n",
      "step 38100-38199, precision 0.513494, recall 0.514571, f_score 0.514032\n",
      "=== total 38200 match 8115\n",
      ">>> step 38300\n",
      "step 38200-38299, precision 0.510662, recall 0.514571, f_score 0.512609\n",
      "=== total 38300 match 8160\n",
      ">>> step 38400\n",
      "step 38300-38399, precision 0.508481, recall 0.514571, f_score 0.511508\n",
      "=== total 38400 match 8195\n",
      ">>> step 38500\n",
      "step 38400-38499, precision 0.506441, recall 0.514571, f_score 0.510474\n",
      "=== total 38500 match 8228\n",
      ">>> step 38600\n",
      "step 38500-38599, precision 0.505851, recall 0.516125, f_score 0.510936\n",
      "=== total 38600 match 8289\n",
      ">>> step 38700\n",
      "step 38600-38699, precision 0.504573, recall 0.516125, f_score 0.510284\n",
      "=== total 38700 match 8310\n",
      ">>> step 38800\n",
      "step 38700-38799, precision 0.504838, recall 0.517005, f_score 0.510849\n",
      "=== total 38800 match 8371\n",
      ">>> step 38900\n",
      "step 38800-38899, precision 0.502916, recall 0.517005, f_score 0.509863\n",
      "=== total 38900 match 8403\n",
      ">>> step 39000\n",
      "step 38900-38999, precision 0.504129, recall 0.519577, f_score 0.511737\n",
      "=== total 39000 match 8476\n",
      ">>> step 39100\n",
      "step 39000-39099, precision 0.505324, recall 0.522249, f_score 0.513647\n",
      "=== total 39100 match 8547\n",
      ">>> step 39200\n",
      "step 39100-39199, precision 0.503318, recall 0.522480, f_score 0.512720\n",
      "=== total 39200 match 8589\n",
      ">>> step 39300\n",
      "step 39200-39299, precision 0.501450, recall 0.522480, f_score 0.511749\n",
      "=== total 39300 match 8621\n",
      ">>> step 39400\n",
      "step 39300-39399, precision 0.503230, recall 0.524525, f_score 0.513657\n",
      "=== total 39400 match 8670\n",
      ">>> step 39500\n",
      "step 39400-39499, precision 0.501948, recall 0.525552, f_score 0.513479\n",
      "=== total 39500 match 8728\n",
      ">>> step 39600\n",
      "step 39500-39599, precision 0.502216, recall 0.527705, f_score 0.514645\n",
      "=== total 39600 match 8799\n",
      ">>> step 39700\n",
      "step 39600-39699, precision 0.501874, recall 0.526071, f_score 0.513688\n",
      "=== total 39700 match 8805\n",
      ">>> step 39800\n",
      "step 39700-39799, precision 0.503663, recall 0.528876, f_score 0.515961\n",
      "=== total 39800 match 8873\n",
      ">>> step 39900\n",
      "step 39800-39899, precision 0.501402, recall 0.528876, f_score 0.514773\n",
      "=== total 39900 match 8913\n",
      ">>> step 40000\n",
      "step 39900-39999, precision 0.503009, recall 0.531129, f_score 0.516687\n",
      "=== total 40000 match 8972\n",
      ">>> step 40100\n",
      "step 40000-40099, precision 0.501722, recall 0.531176, f_score 0.516029\n",
      "=== total 40100 match 8999\n",
      ">>> step 40200\n",
      "step 40100-40199, precision 0.499557, recall 0.531176, f_score 0.514882\n",
      "=== total 40200 match 9038\n",
      ">>> step 40300\n",
      "step 40200-40299, precision 0.501318, recall 0.533809, f_score 0.517054\n",
      "=== total 40300 match 9102\n",
      ">>> step 40400\n",
      "step 40300-40399, precision 0.499890, recall 0.533918, f_score 0.516344\n",
      "=== total 40400 match 9132\n",
      ">>> step 40500\n",
      "step 40400-40499, precision 0.498580, recall 0.533918, f_score 0.515644\n",
      "=== total 40500 match 9156\n",
      ">>> step 40600\n",
      "step 40500-40599, precision 0.497331, recall 0.533918, f_score 0.514975\n",
      "=== total 40600 match 9179\n",
      ">>> step 40700\n",
      "step 40600-40699, precision 0.498488, recall 0.536628, f_score 0.516855\n",
      "=== total 40700 match 9258\n",
      ">>> step 40800\n",
      "step 40700-40799, precision 0.498231, recall 0.538346, f_score 0.517512\n",
      "=== total 40800 match 9327\n",
      ">>> step 40900\n",
      "step 40800-40899, precision 0.495627, recall 0.538346, f_score 0.516104\n",
      "=== total 40900 match 9376\n",
      ">>> step 41000\n",
      "step 40900-40999, precision 0.493784, recall 0.538346, f_score 0.515103\n",
      "=== total 41000 match 9411\n",
      ">>> step 41100\n",
      "step 41000-41099, precision 0.494295, recall 0.540051, f_score 0.516161\n",
      "=== total 41100 match 9466\n",
      ">>> step 41200\n",
      "step 41100-41199, precision 0.495327, recall 0.541997, f_score 0.517612\n",
      "=== total 41200 match 9523\n",
      ">>> step 41300\n",
      "step 41200-41299, precision 0.493169, recall 0.542628, f_score 0.516718\n",
      "=== total 41300 match 9589\n",
      ">>> step 41400\n",
      "step 41300-41399, precision 0.494616, recall 0.545071, f_score 0.518619\n",
      "=== total 41400 match 9658\n",
      ">>> step 41500\n",
      "step 41400-41499, precision 0.492932, recall 0.545071, f_score 0.517692\n",
      "=== total 41500 match 9691\n",
      ">>> step 41600\n",
      "step 41500-41599, precision 0.493187, recall 0.546921, f_score 0.518666\n",
      "=== total 41600 match 9761\n",
      ">>> step 41700\n",
      "step 41600-41699, precision 0.491525, recall 0.546921, f_score 0.517746\n",
      "=== total 41700 match 9794\n",
      ">>> step 41800\n",
      "step 41700-41799, precision 0.492906, recall 0.549480, f_score 0.519658\n",
      "=== total 41800 match 9868\n",
      ">>> step 41900\n",
      "step 41800-41899, precision 0.491710, recall 0.549480, f_score 0.518993\n",
      "=== total 41900 match 9892\n",
      ">>> step 42000\n",
      "step 41900-41999, precision 0.491604, recall 0.550687, f_score 0.519471\n",
      "=== total 42000 match 9945\n",
      ">>> step 42100\n",
      "step 42000-42099, precision 0.490863, recall 0.550687, f_score 0.519057\n",
      "=== total 42100 match 9960\n",
      ">>> step 42200\n",
      "step 42100-42199, precision 0.492275, recall 0.553203, f_score 0.520964\n",
      "=== total 42200 match 10033\n",
      ">>> step 42300\n",
      "step 42200-42299, precision 0.490126, recall 0.553203, f_score 0.519758\n",
      "=== total 42300 match 10077\n",
      ">>> step 42400\n",
      "step 42300-42399, precision 0.488092, recall 0.553203, f_score 0.518612\n",
      "=== total 42400 match 10119\n",
      ">>> step 42500\n",
      "step 42400-42499, precision 0.489454, recall 0.555692, f_score 0.520474\n",
      "=== total 42500 match 10193\n",
      ">>> step 42600\n",
      "step 42500-42599, precision 0.489223, recall 0.557024, f_score 0.520926\n",
      "=== total 42600 match 10253\n",
      ">>> step 42700\n",
      "step 42600-42699, precision 0.488747, recall 0.558042, f_score 0.521101\n",
      "=== total 42700 match 10308\n",
      ">>> step 42800\n",
      "step 42700-42799, precision 0.486810, recall 0.558042, f_score 0.519998\n",
      "=== total 42800 match 10349\n",
      ">>> step 42900\n",
      "step 42800-42899, precision 0.484889, recall 0.558042, f_score 0.518900\n",
      "=== total 42900 match 10390\n",
      ">>> step 43000\n",
      "step 42900-42999, precision 0.485523, recall 0.559311, f_score 0.519811\n",
      "=== total 43000 match 10430\n",
      ">>> step 43100\n",
      "step 43000-43099, precision 0.485772, recall 0.560119, f_score 0.520303\n",
      "=== total 43100 match 10472\n",
      ">>> step 43200\n",
      "step 43100-43199, precision 0.485370, recall 0.560874, f_score 0.520397\n",
      "=== total 43200 match 10526\n",
      ">>> step 43300\n",
      "step 43200-43299, precision 0.485392, recall 0.560460, f_score 0.520232\n",
      "=== total 43300 match 10542\n",
      ">>> step 43400\n",
      "step 43300-43399, precision 0.486436, recall 0.562527, f_score 0.521722\n",
      "=== total 43400 match 10616\n",
      ">>> step 43500\n",
      "step 43400-43499, precision 0.484792, recall 0.562527, f_score 0.520775\n",
      "=== total 43500 match 10652\n",
      ">>> step 43600\n",
      "step 43500-43599, precision 0.483159, recall 0.562527, f_score 0.519831\n",
      "=== total 43600 match 10688\n",
      ">>> step 43700\n",
      "step 43600-43699, precision 0.481312, recall 0.562527, f_score 0.518760\n",
      "=== total 43700 match 10729\n",
      ">>> step 43800\n",
      "step 43700-43799, precision 0.483091, recall 0.564897, f_score 0.520801\n",
      "=== total 43800 match 10793\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> step 43900\n",
      "step 43800-43899, precision 0.484530, recall 0.567026, f_score 0.522542\n",
      "=== total 43900 match 10860\n",
      ">>> step 44000\n",
      "step 43900-43999, precision 0.484418, recall 0.568096, f_score 0.522931\n",
      "=== total 44000 match 10910\n",
      ">>> step 44100\n",
      "step 44000-44099, precision 0.484406, recall 0.569346, f_score 0.523453\n",
      "=== total 44100 match 10966\n",
      ">>> step 44200\n",
      "step 44100-44199, precision 0.482602, recall 0.569346, f_score 0.522398\n",
      "=== total 44200 match 11007\n",
      ">>> step 44300\n",
      "step 44200-44299, precision 0.482593, recall 0.570497, f_score 0.522876\n",
      "=== total 44300 match 11059\n",
      ">>> step 44400\n",
      "step 44300-44399, precision 0.482672, recall 0.571642, f_score 0.523403\n",
      "=== total 44400 match 11109\n",
      ">>> step 44500\n",
      "step 44400-44499, precision 0.482926, recall 0.572826, f_score 0.524048\n",
      "=== total 44500 match 11157\n",
      ">>> step 44600\n",
      "step 44500-44599, precision 0.481329, recall 0.572826, f_score 0.523107\n",
      "=== total 44600 match 11194\n",
      ">>> step 44700\n",
      "step 44600-44699, precision 0.480000, recall 0.572826, f_score 0.522321\n",
      "=== total 44700 match 11225\n",
      ">>> step 44800\n",
      "step 44700-44799, precision 0.478550, recall 0.572826, f_score 0.521461\n",
      "=== total 44800 match 11259\n",
      ">>> step 44900\n",
      "step 44800-44899, precision 0.478311, recall 0.574003, f_score 0.521806\n",
      "=== total 44900 match 11319\n",
      ">>> step 45000\n",
      "step 44900-44999, precision 0.478158, recall 0.575174, f_score 0.522198\n",
      "=== total 45000 match 11377\n",
      ">>> step 45100\n",
      "step 45000-45099, precision 0.478543, recall 0.576250, f_score 0.522871\n",
      "=== total 45100 match 11418\n",
      ">>> step 45200\n",
      "step 45100-45199, precision 0.476955, recall 0.576250, f_score 0.521922\n",
      "=== total 45200 match 11456\n",
      ">>> step 45300\n",
      "step 45200-45299, precision 0.475461, recall 0.576250, f_score 0.521026\n",
      "=== total 45300 match 11492\n",
      ">>> step 45400\n",
      "step 45300-45399, precision 0.475531, recall 0.577408, f_score 0.521541\n",
      "=== total 45400 match 11545\n",
      ">>> step 45500\n",
      "step 45400-45499, precision 0.474462, recall 0.577408, f_score 0.520898\n",
      "=== total 45500 match 11571\n",
      ">>> step 45600\n",
      "step 45500-45599, precision 0.475892, recall 0.579487, f_score 0.522605\n",
      "=== total 45600 match 11635\n",
      ">>> step 45700\n",
      "step 45600-45699, precision 0.474681, recall 0.579619, f_score 0.521928\n",
      "=== total 45700 match 11671\n",
      ">>> step 45800\n",
      "step 45700-45799, precision 0.474685, recall 0.580891, f_score 0.522445\n",
      "=== total 45800 match 11732\n",
      ">>> step 45900\n",
      "step 45800-45899, precision 0.474699, recall 0.581790, f_score 0.522817\n",
      "=== total 45900 match 11778\n",
      ">>> step 46000\n",
      "step 45900-45999, precision 0.475885, recall 0.583351, f_score 0.524166\n",
      "=== total 46000 match 11839\n",
      ">>> step 46100\n",
      "step 46000-46099, precision 0.476771, recall 0.584569, f_score 0.525196\n",
      "=== total 46100 match 11903\n",
      ">>> step 46200\n",
      "step 46100-46199, precision 0.476832, recall 0.585679, f_score 0.525680\n",
      "=== total 46200 match 11956\n",
      ">>> step 46300\n",
      "step 46200-46299, precision 0.475361, recall 0.585679, f_score 0.524785\n",
      "=== total 46300 match 11993\n",
      ">>> step 46400\n",
      "step 46300-46399, precision 0.476269, recall 0.587332, f_score 0.526002\n",
      "=== total 46400 match 12052\n",
      ">>> step 46500\n",
      "step 46400-46499, precision 0.475785, recall 0.587989, f_score 0.525970\n",
      "=== total 46500 match 12100\n",
      ">>> step 46600\n",
      "step 46500-46599, precision 0.476852, recall 0.589689, f_score 0.527302\n",
      "=== total 46600 match 12161\n",
      ">>> step 46700\n",
      "step 46600-46699, precision 0.476596, recall 0.590669, f_score 0.527536\n",
      "=== total 46700 match 12220\n",
      ">>> step 46800\n",
      "step 46700-46799, precision 0.477191, recall 0.591120, f_score 0.528081\n",
      "=== total 46800 match 12276\n",
      ">>> step 46900\n",
      "step 46800-46899, precision 0.475641, recall 0.591120, f_score 0.527130\n",
      "=== total 46900 match 12316\n",
      ">>> step 47000\n",
      "step 46900-46999, precision 0.476537, recall 0.592846, f_score 0.528366\n",
      "=== total 47000 match 12381\n",
      ">>> step 47100\n",
      "step 47000-47099, precision 0.475684, recall 0.593173, f_score 0.527971\n",
      "=== total 47100 match 12420\n",
      ">>> step 47200\n",
      "step 47100-47199, precision 0.474653, recall 0.593173, f_score 0.527335\n",
      "=== total 47200 match 12447\n",
      ">>> step 47300\n",
      "step 47200-47299, precision 0.475654, recall 0.594662, f_score 0.528542\n",
      "=== total 47300 match 12507\n",
      ">>> step 47400\n",
      "step 47300-47399, precision 0.473986, recall 0.594662, f_score 0.527511\n",
      "=== total 47400 match 12551\n",
      ">>> step 47500\n",
      "step 47400-47499, precision 0.473044, recall 0.594662, f_score 0.526926\n",
      "=== total 47500 match 12576\n",
      ">>> step 47600\n",
      "step 47500-47599, precision 0.472255, recall 0.594662, f_score 0.526437\n",
      "=== total 47600 match 12597\n",
      ">>> step 47700\n",
      "step 47600-47699, precision 0.473443, recall 0.596678, f_score 0.527965\n",
      "=== total 47700 match 12671\n",
      ">>> step 47800\n",
      "step 47700-47799, precision 0.473589, recall 0.597718, f_score 0.528462\n",
      "=== total 47800 match 12722\n",
      ">>> step 47900\n",
      "step 47800-47899, precision 0.472327, recall 0.597718, f_score 0.527676\n",
      "=== total 47900 match 12756\n",
      ">>> step 48000\n",
      "step 47900-47999, precision 0.471145, recall 0.597718, f_score 0.526937\n",
      "=== total 48000 match 12788\n",
      ">>> step 48100\n",
      "step 48000-48099, precision 0.472432, recall 0.599704, f_score 0.528514\n",
      "=== total 48100 match 12859\n",
      ">>> step 48200\n",
      "step 48100-48199, precision 0.472379, recall 0.600453, f_score 0.528772\n",
      "=== total 48200 match 12907\n",
      ">>> step 48300\n",
      "step 48200-48299, precision 0.472570, recall 0.600786, f_score 0.529020\n",
      "=== total 48300 match 12942\n",
      ">>> step 48400\n",
      "step 48300-48399, precision 0.471843, recall 0.601138, f_score 0.528701\n",
      "=== total 48400 match 12981\n",
      ">>> step 48500\n",
      "step 48400-48499, precision 0.471971, recall 0.601841, f_score 0.529052\n",
      "=== total 48500 match 13022\n",
      ">>> step 48600\n",
      "step 48500-48599, precision 0.470750, recall 0.601919, f_score 0.528315\n",
      "=== total 48600 match 13060\n",
      ">>> step 48700\n",
      "step 48600-48699, precision 0.471812, recall 0.603489, f_score 0.529588\n",
      "=== total 48700 match 13126\n",
      ">>> step 48800\n",
      "step 48700-48799, precision 0.470785, recall 0.603017, f_score 0.528759\n",
      "=== total 48800 match 13161\n",
      ">>> step 48900\n",
      "step 48800-48899, precision 0.469394, recall 0.603017, f_score 0.527881\n",
      "=== total 48900 match 13200\n",
      ">>> step 49000\n",
      "step 48900-48999, precision 0.469368, recall 0.603922, f_score 0.528211\n",
      "=== total 49000 match 13254\n",
      ">>> step 49100\n",
      "step 49000-49099, precision 0.468543, recall 0.604114, f_score 0.527761\n",
      "=== total 49100 match 13288\n",
      ">>> step 49200\n",
      "step 49100-49199, precision 0.469385, recall 0.605062, f_score 0.528657\n",
      "=== total 49200 match 13343\n",
      ">>> step 49300\n",
      "step 49200-49299, precision 0.468157, recall 0.605062, f_score 0.527877\n",
      "=== total 49300 match 13378\n",
      ">>> step 49400\n",
      "step 49300-49399, precision 0.467353, recall 0.605062, f_score 0.527366\n",
      "=== total 49400 match 13401\n",
      ">>> step 49500\n",
      "step 49400-49499, precision 0.466647, recall 0.605520, f_score 0.527089\n",
      "=== total 49500 match 13447\n",
      ">>> step 49600\n",
      "step 49500-49599, precision 0.467546, recall 0.606672, f_score 0.528100\n",
      "=== total 49600 match 13496\n",
      ">>> step 49700\n",
      "step 49600-49699, precision 0.468603, recall 0.608363, f_score 0.529414\n",
      "=== total 49700 match 13568\n",
      ">>> step 49800\n",
      "step 49700-49799, precision 0.467294, recall 0.608363, f_score 0.528578\n",
      "=== total 49800 match 13606\n",
      ">>> step 49900\n",
      "step 49800-49899, precision 0.466197, recall 0.608363, f_score 0.527876\n",
      "=== total 49900 match 13638\n",
      ">>> step 50000\n",
      "step 49900-49999, precision 0.464970, recall 0.608363, f_score 0.527088\n",
      "=== total 50000 match 13674\n",
      ">>> step 50100\n",
      "step 50000-50099, precision 0.465408, recall 0.609335, f_score 0.527734\n",
      "=== total 50100 match 13717\n",
      ">>> step 50200\n",
      "step 50100-50199, precision 0.466435, recall 0.611190, f_score 0.529090\n",
      "=== total 50200 match 13794\n",
      ">>> step 50300\n",
      "step 50200-50299, precision 0.467393, recall 0.612845, f_score 0.530327\n",
      "=== total 50300 match 13862\n",
      ">>> step 50400\n",
      "step 50300-50399, precision 0.466676, recall 0.613028, f_score 0.529933\n",
      "=== total 50400 match 13894\n",
      ">>> step 50500\n",
      "step 50400-50499, precision 0.465437, recall 0.613028, f_score 0.529133\n",
      "=== total 50500 match 13931\n",
      ">>> step 50600\n",
      "step 50500-50599, precision 0.464703, recall 0.613028, f_score 0.528659\n",
      "=== total 50600 match 13953\n",
      ">>> step 50700\n",
      "step 50600-50699, precision 0.465753, recall 0.614574, f_score 0.529913\n",
      "=== total 50700 match 14016\n",
      ">>> step 50800\n",
      "step 50700-50799, precision 0.465903, recall 0.615442, f_score 0.530333\n",
      "=== total 50800 match 14063\n",
      ">>> step 50900\n",
      "step 50800-50899, precision 0.466416, recall 0.616559, f_score 0.531080\n",
      "=== total 50900 match 14114\n",
      ">>> step 51000\n",
      "step 50900-50999, precision 0.467804, recall 0.618346, f_score 0.532643\n",
      "=== total 51000 match 14179\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> step 51100\n",
      "step 51000-51099, precision 0.466883, recall 0.618346, f_score 0.532045\n",
      "=== total 51100 match 14207\n",
      ">>> step 51200\n",
      "step 51100-51199, precision 0.465768, recall 0.618346, f_score 0.531320\n",
      "=== total 51200 match 14241\n",
      ">>> step 51300\n",
      "step 51200-51299, precision 0.467073, recall 0.619282, f_score 0.532514\n",
      "=== total 51300 match 14289\n",
      ">>> step 51400\n",
      "step 51300-51399, precision 0.467192, recall 0.620198, f_score 0.532930\n",
      "=== total 51400 match 14341\n",
      ">>> step 51500\n",
      "step 51400-51499, precision 0.465892, recall 0.620198, f_score 0.532084\n",
      "=== total 51500 match 14381\n",
      ">>> step 51600\n",
      "step 51500-51599, precision 0.464665, recall 0.620198, f_score 0.531282\n",
      "=== total 51600 match 14419\n",
      ">>> step 51700\n",
      "step 51600-51699, precision 0.463347, recall 0.620198, f_score 0.530420\n",
      "=== total 51700 match 14460\n",
      ">>> step 51800\n",
      "step 51700-51799, precision 0.462478, recall 0.620374, f_score 0.529914\n",
      "=== total 51800 match 14498\n",
      ">>> step 51900\n",
      "step 51800-51899, precision 0.462395, recall 0.621110, f_score 0.530128\n",
      "=== total 51900 match 14546\n",
      ">>> step 52000\n",
      "step 51900-51999, precision 0.463755, recall 0.622759, f_score 0.531623\n",
      "=== total 52000 match 14609\n",
      ">>> step 52100\n",
      "step 52000-52099, precision 0.462615, recall 0.622759, f_score 0.530873\n",
      "=== total 52100 match 14645\n",
      ">>> step 52200\n",
      "step 52100-52199, precision 0.461607, recall 0.622759, f_score 0.530208\n",
      "=== total 52200 match 14677\n",
      ">>> step 52300\n",
      "step 52200-52299, precision 0.460807, recall 0.623071, f_score 0.529793\n",
      "=== total 52300 match 14722\n",
      ">>> step 52400\n",
      "step 52300-52399, precision 0.460242, recall 0.623659, f_score 0.529632\n",
      "=== total 52400 match 14777\n",
      ">>> step 52500\n",
      "step 52400-52499, precision 0.459793, recall 0.623855, f_score 0.529405\n",
      "=== total 52500 match 14811\n",
      ">>> step 52600\n",
      "step 52500-52599, precision 0.459545, recall 0.623186, f_score 0.528999\n",
      "=== total 52600 match 14856\n",
      ">>> step 52700\n",
      "step 52600-52699, precision 0.458434, recall 0.623186, f_score 0.528262\n",
      "=== total 52700 match 14892\n",
      ">>> step 52800\n",
      "step 52700-52799, precision 0.458749, recall 0.622990, f_score 0.528401\n",
      "=== total 52800 match 14945\n",
      ">>> step 52900\n",
      "step 52800-52899, precision 0.459756, recall 0.624695, f_score 0.529682\n",
      "=== total 52900 match 15021\n",
      ">>> step 53000\n",
      "step 52900-52999, precision 0.459798, recall 0.625542, f_score 0.530015\n",
      "=== total 53000 match 15074\n",
      ">>> step 53100\n",
      "step 53000-53099, precision 0.459931, recall 0.626385, f_score 0.530405\n",
      "=== total 53100 match 15124\n",
      ">>> step 53200\n",
      "step 53100-53199, precision 0.460129, recall 0.627257, f_score 0.530850\n",
      "=== total 53200 match 15174\n",
      ">>> step 53300\n",
      "step 53200-53299, precision 0.459161, recall 0.627257, f_score 0.530205\n",
      "=== total 53300 match 15206\n",
      ">>> step 53400\n",
      "step 53300-53399, precision 0.460198, recall 0.628209, f_score 0.531236\n",
      "=== total 53400 match 15263\n",
      ">>> step 53500\n",
      "step 53400-53499, precision 0.461418, recall 0.629864, f_score 0.532641\n",
      "=== total 53500 match 15331\n",
      ">>> step 53600\n",
      "step 53500-53599, precision 0.461258, recall 0.629830, f_score 0.532522\n",
      "=== total 53600 match 15371\n",
      ">>> step 53700\n",
      "step 53600-53699, precision 0.461294, recall 0.630651, f_score 0.532839\n",
      "=== total 53700 match 15424\n",
      ">>> step 53800\n",
      "step 53700-53799, precision 0.461479, recall 0.631467, f_score 0.533254\n",
      "=== total 53800 match 15472\n",
      ">>> step 53900\n",
      "step 53800-53899, precision 0.460785, recall 0.631737, f_score 0.532886\n",
      "=== total 53900 match 15517\n",
      ">>> step 54000\n",
      "step 53900-53999, precision 0.461459, recall 0.632561, f_score 0.533630\n",
      "=== total 54000 match 15568\n",
      ">>> step 54100\n",
      "step 54000-54099, precision 0.460336, recall 0.632561, f_score 0.532878\n",
      "=== total 54100 match 15606\n",
      ">>> step 54200\n",
      "step 54100-54199, precision 0.459548, recall 0.632788, f_score 0.532430\n",
      "=== total 54200 match 15648\n",
      ">>> step 54300\n",
      "step 54200-54299, precision 0.460177, recall 0.633979, f_score 0.533274\n",
      "=== total 54300 match 15707\n",
      ">>> step 54400\n",
      "step 54300-54399, precision 0.459037, recall 0.633979, f_score 0.532508\n",
      "=== total 54400 match 15746\n",
      ">>> step 54500\n",
      "step 54400-54499, precision 0.457990, recall 0.633979, f_score 0.531803\n",
      "=== total 54500 match 15782\n",
      ">>> step 54600\n",
      "step 54500-54599, precision 0.458352, recall 0.634940, f_score 0.532385\n",
      "=== total 54600 match 15835\n",
      ">>> step 54700\n",
      "step 54600-54699, precision 0.458110, recall 0.635578, f_score 0.532446\n",
      "=== total 54700 match 15887\n",
      ">>> step 54800\n",
      "step 54700-54799, precision 0.458046, recall 0.636403, f_score 0.532692\n",
      "=== total 54800 match 15946\n",
      ">>> step 54900\n",
      "step 54800-54899, precision 0.457157, recall 0.636403, f_score 0.532090\n",
      "=== total 54900 match 15977\n",
      ">>> step 55000\n",
      "step 54900-54999, precision 0.457246, recall 0.637099, f_score 0.532393\n",
      "=== total 55000 match 16022\n",
      ">>> step 55100\n",
      "step 55000-55099, precision 0.457424, recall 0.637980, f_score 0.532821\n",
      "=== total 55100 match 16077\n",
      ">>> step 55200\n",
      "step 55100-55199, precision 0.458537, recall 0.639544, f_score 0.534122\n",
      "=== total 55200 match 16147\n",
      ">>> step 55300\n",
      "step 55200-55299, precision 0.457517, recall 0.639544, f_score 0.533429\n",
      "=== total 55300 match 16183\n",
      ">>> step 55400\n",
      "step 55300-55399, precision 0.457495, recall 0.640193, f_score 0.533640\n",
      "=== total 55400 match 16245\n",
      ">>> step 55500\n",
      "step 55400-55499, precision 0.456707, recall 0.640193, f_score 0.533104\n",
      "=== total 55500 match 16273\n",
      ">>> step 55600\n",
      "step 55500-55599, precision 0.457940, recall 0.641564, f_score 0.534419\n",
      "=== total 55600 match 16334\n",
      ">>> step 55700\n",
      "step 55600-55699, precision 0.457630, recall 0.641278, f_score 0.534109\n",
      "=== total 55700 match 16356\n",
      ">>> step 55800\n",
      "step 55700-55799, precision 0.456792, recall 0.641278, f_score 0.533538\n",
      "=== total 55800 match 16386\n",
      ">>> step 55900\n",
      "step 55800-55899, precision 0.456929, recall 0.642076, f_score 0.533907\n",
      "=== total 55900 match 16438\n",
      ">>> step 56000\n",
      "step 55900-55999, precision 0.456014, recall 0.642076, f_score 0.533281\n",
      "=== total 56000 match 16471\n",
      ">>> step 56100\n",
      "step 56000-56099, precision 0.457041, recall 0.643429, f_score 0.534450\n",
      "=== total 56100 match 16539\n",
      ">>> step 56200\n",
      "step 56100-56199, precision 0.456186, recall 0.643429, f_score 0.533865\n",
      "=== total 56200 match 16570\n",
      ">>> step 56300\n",
      "step 56200-56299, precision 0.455526, recall 0.643429, f_score 0.533413\n",
      "=== total 56300 match 16594\n",
      ">>> step 56400\n",
      "step 56300-56399, precision 0.454622, recall 0.643429, f_score 0.532793\n",
      "=== total 56400 match 16627\n",
      ">>> step 56500\n",
      "step 56400-56499, precision 0.455847, recall 0.644940, f_score 0.534152\n",
      "=== total 56500 match 16692\n",
      ">>> step 56600\n",
      "step 56500-56599, precision 0.456413, recall 0.645990, f_score 0.534901\n",
      "=== total 56600 match 16748\n",
      ">>> step 56700\n",
      "step 56600-56699, precision 0.456110, recall 0.646438, f_score 0.534846\n",
      "=== total 56700 match 16792\n",
      ">>> step 56800\n",
      "step 56700-56799, precision 0.455378, recall 0.646438, f_score 0.534343\n",
      "=== total 56800 match 16819\n",
      ">>> step 56900\n",
      "step 56800-56899, precision 0.454675, recall 0.646438, f_score 0.533858\n",
      "=== total 56900 match 16845\n",
      ">>> step 57000\n",
      "step 56900-56999, precision 0.454330, recall 0.647212, f_score 0.533884\n",
      "=== total 57000 match 16915\n",
      ">>> step 57100\n",
      "step 57000-57099, precision 0.454481, recall 0.647607, f_score 0.534123\n",
      "=== total 57100 match 16971\n",
      ">>> step 57200\n",
      "step 57100-57199, precision 0.454807, recall 0.647901, f_score 0.534448\n",
      "=== total 57200 match 17005\n",
      ">>> step 57300\n",
      "step 57200-57299, precision 0.455628, recall 0.648739, f_score 0.535300\n",
      "=== total 57300 match 17049\n",
      ">>> step 57400\n",
      "step 57300-57399, precision 0.454561, recall 0.648739, f_score 0.534563\n",
      "=== total 57400 match 17089\n",
      ">>> step 57500\n",
      "step 57400-57499, precision 0.454136, recall 0.648739, f_score 0.534269\n",
      "=== total 57500 match 17105\n",
      ">>> step 57600\n",
      "step 57500-57599, precision 0.454328, recall 0.646956, f_score 0.533795\n",
      "=== total 57600 match 17122\n",
      ">>> step 57700\n",
      "step 57600-57699, precision 0.454360, recall 0.646527, f_score 0.533671\n",
      "=== total 57700 match 17145\n",
      ">>> step 57800\n",
      "step 57700-57799, precision 0.454286, recall 0.646502, f_score 0.533612\n",
      "=== total 57800 match 17150\n",
      ">>> step 57900\n",
      "step 57800-57899, precision 0.454371, recall 0.645073, f_score 0.533183\n",
      "=== total 57900 match 17160\n",
      ">>> step 58000\n",
      "step 57900-57999, precision 0.454318, recall 0.645073, f_score 0.533146\n",
      "=== total 58000 match 17162\n",
      ">>> step 58100\n",
      "step 58000-58099, precision 0.454185, recall 0.645073, f_score 0.533055\n",
      "=== total 58100 match 17167\n",
      ">>> step 58200\n",
      "step 58100-58199, precision 0.453879, recall 0.644107, f_score 0.532515\n",
      "=== total 58200 match 17194\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> step 58300\n",
      "step 58200-58299, precision 0.453679, recall 0.643794, f_score 0.532270\n",
      "=== total 58300 match 17206\n",
      ">>> step 58400\n",
      "step 58300-58399, precision 0.453590, recall 0.643351, f_score 0.532057\n",
      "=== total 58400 match 17216\n",
      ">>> step 58500\n",
      "step 58400-58499, precision 0.453696, recall 0.642440, f_score 0.531818\n",
      "=== total 58500 match 17223\n",
      ">>> step 58600\n",
      "step 58500-58599, precision 0.453669, recall 0.641122, f_score 0.531348\n",
      "=== total 58600 match 17224\n",
      ">>> step 58700\n",
      "step 58600-58699, precision 0.453596, recall 0.639892, f_score 0.530874\n",
      "=== total 58700 match 17229\n",
      ">>> step 58800\n",
      "step 58700-58799, precision 0.453133, recall 0.638070, f_score 0.529930\n",
      "=== total 58800 match 17251\n",
      ">>> step 58900\n",
      "step 58800-58899, precision 0.452713, recall 0.636823, f_score 0.529213\n",
      "=== total 58900 match 17267\n",
      ">>> step 59000\n",
      "step 58900-58999, precision 0.452604, recall 0.635802, f_score 0.528785\n",
      "=== total 59000 match 17280\n",
      ">>> step 59100\n",
      "step 59000-59099, precision 0.452055, recall 0.633844, f_score 0.527733\n",
      "=== total 59100 match 17301\n",
      ">>> step 59200\n",
      "step 59100-59199, precision 0.451967, recall 0.632805, f_score 0.527313\n",
      "=== total 59200 match 17311\n",
      ">>> step 59300\n",
      "step 59200-59299, precision 0.451836, recall 0.632805, f_score 0.527224\n",
      "=== total 59300 match 17316\n",
      ">>> step 59400\n",
      "step 59300-59399, precision 0.451457, recall 0.631689, f_score 0.526578\n",
      "=== total 59400 match 17335\n",
      ">>> step 59500\n",
      "step 59400-59499, precision 0.451457, recall 0.631689, f_score 0.526578\n",
      "=== total 59500 match 17335\n",
      ">>> step 59600\n",
      "step 59500-59599, precision 0.451542, recall 0.630241, f_score 0.526132\n",
      "=== total 59600 match 17345\n",
      ">>> step 59700\n",
      "step 59600-59699, precision 0.451464, recall 0.630241, f_score 0.526079\n",
      "=== total 59700 match 17348\n",
      ">>> step 59800\n",
      "step 59700-59799, precision 0.451734, recall 0.629101, f_score 0.525864\n",
      "=== total 59800 match 17362\n",
      ">>> step 59900\n",
      "step 59800-59899, precision 0.451708, recall 0.628144, f_score 0.525512\n",
      "=== total 59900 match 17363\n",
      ">>> step 60000\n",
      "step 59900-59999, precision 0.451604, recall 0.627641, f_score 0.525265\n",
      "=== total 60000 match 17367\n",
      ">>> step 60100\n",
      "step 60000-60099, precision 0.451492, recall 0.626376, f_score 0.524747\n",
      "=== total 60100 match 17389\n",
      ">>> step 60200\n",
      "step 60100-60199, precision 0.451089, recall 0.625887, f_score 0.524302\n",
      "=== total 60200 match 17409\n",
      ">>> step 60300\n",
      "step 60200-60299, precision 0.450924, recall 0.624881, f_score 0.523838\n",
      "=== total 60300 match 17422\n",
      ">>> step 60400\n",
      "step 60300-60399, precision 0.450872, recall 0.624881, f_score 0.523803\n",
      "=== total 60400 match 17424\n",
      ">>> step 60500\n",
      "step 60400-60499, precision 0.450923, recall 0.624355, f_score 0.523653\n",
      "=== total 60500 match 17442\n",
      ">>> step 60600\n",
      "step 60500-60599, precision 0.450776, recall 0.623594, f_score 0.523286\n",
      "=== total 60600 match 17461\n",
      ">>> step 60700\n",
      "step 60600-60699, precision 0.450544, recall 0.623594, f_score 0.523129\n",
      "=== total 60700 match 17470\n",
      ">>> step 60800\n",
      "step 60700-60799, precision 0.450521, recall 0.622117, f_score 0.522593\n",
      "=== total 60800 match 17482\n",
      ">>> step 60900\n",
      "step 60800-60899, precision 0.450634, recall 0.621758, f_score 0.522543\n",
      "=== total 60900 match 17502\n",
      ">>> step 61000\n",
      "step 60900-60999, precision 0.450325, recall 0.621758, f_score 0.522335\n",
      "=== total 61000 match 17514\n",
      ">>> step 61100\n",
      "step 61000-61099, precision 0.451032, recall 0.622502, f_score 0.523073\n",
      "=== total 61100 match 17542\n",
      ">>> step 61200\n",
      "step 61100-61199, precision 0.452304, recall 0.623805, f_score 0.524388\n",
      "=== total 61200 match 17601\n",
      ">>> step 61300\n",
      "step 61200-61299, precision 0.452409, recall 0.622054, f_score 0.523839\n",
      "=== total 61300 match 17619\n",
      ">>> step 61400\n",
      "step 61300-61399, precision 0.452332, recall 0.621036, f_score 0.523426\n",
      "=== total 61400 match 17622\n",
      ">>> step 61500\n",
      "step 61400-61499, precision 0.453646, recall 0.622429, f_score 0.524800\n",
      "=== total 61500 match 17679\n",
      ">>> step 61600\n",
      "step 61500-61599, precision 0.454191, recall 0.623190, f_score 0.525436\n",
      "=== total 61600 match 17715\n",
      ">>> step 61700\n",
      "step 61600-61699, precision 0.454038, recall 0.623190, f_score 0.525333\n",
      "=== total 61700 match 17721\n",
      ">>> step 61800\n",
      "step 61700-61799, precision 0.454192, recall 0.622710, f_score 0.525266\n",
      "=== total 61800 match 17737\n",
      ">>> step 61900\n",
      "step 61800-61899, precision 0.455394, recall 0.623778, f_score 0.526449\n",
      "=== total 61900 match 17789\n",
      ">>> step 62000\n",
      "step 61900-61999, precision 0.455368, recall 0.623778, f_score 0.526432\n",
      "=== total 62000 match 17790\n",
      ">>> step 62100\n",
      "step 62000-62099, precision 0.455933, recall 0.624529, f_score 0.527077\n",
      "=== total 62100 match 17825\n",
      ">>> step 62200\n",
      "step 62100-62199, precision 0.456475, recall 0.624205, f_score 0.527324\n",
      "=== total 62200 match 17852\n",
      ">>> step 62300\n",
      "step 62200-62299, precision 0.456454, recall 0.623900, f_score 0.527200\n",
      "=== total 62300 match 17866\n",
      ">>> step 62400\n",
      "step 62300-62399, precision 0.457640, recall 0.624714, f_score 0.528282\n",
      "=== total 62400 match 17918\n",
      ">>> step 62500\n",
      "step 62400-62499, precision 0.458213, recall 0.623881, f_score 0.528365\n",
      "=== total 62500 match 17948\n",
      ">>> step 62600\n",
      "step 62500-62599, precision 0.458431, recall 0.622252, f_score 0.527925\n",
      "=== total 62600 match 17970\n",
      ">>> step 62700\n",
      "step 62600-62699, precision 0.458489, recall 0.621654, f_score 0.527748\n",
      "=== total 62700 match 17983\n",
      ">>> step 62800\n",
      "step 62700-62799, precision 0.459066, recall 0.621226, f_score 0.527975\n",
      "=== total 62800 match 18017\n",
      ">>> step 62900\n",
      "step 62800-62899, precision 0.459128, recall 0.619454, f_score 0.527375\n",
      "=== total 62900 match 18032\n",
      ">>> step 63000\n",
      "step 62900-62999, precision 0.459095, recall 0.617443, f_score 0.526624\n",
      "=== total 63000 match 18042\n",
      ">>> step 63100\n",
      "step 63000-63099, precision 0.459811, recall 0.618268, f_score 0.527394\n",
      "=== total 63100 match 18077\n",
      ">>> step 63200\n",
      "step 63100-63199, precision 0.460285, recall 0.618864, f_score 0.527923\n",
      "=== total 63200 match 18104\n",
      ">>> step 63300\n",
      "step 63200-63299, precision 0.460439, recall 0.617462, f_score 0.527514\n",
      "=== total 63300 match 18124\n",
      ">>> step 63400\n",
      "step 63300-63399, precision 0.460312, recall 0.617462, f_score 0.527430\n",
      "=== total 63400 match 18129\n",
      ">>> step 63500\n",
      "step 63400-63499, precision 0.460428, recall 0.616349, f_score 0.527100\n",
      "=== total 63500 match 18144\n",
      ">>> step 63600\n",
      "step 63500-63599, precision 0.461065, recall 0.616879, f_score 0.527710\n",
      "=== total 63600 match 18184\n",
      ">>> step 63700\n",
      "step 63600-63699, precision 0.460938, recall 0.616879, f_score 0.527627\n",
      "=== total 63700 match 18189\n",
      ">>> step 63800\n",
      "step 63700-63799, precision 0.461175, recall 0.615464, f_score 0.527264\n",
      "=== total 63800 match 18210\n",
      ">>> step 63900\n",
      "step 63800-63899, precision 0.461720, recall 0.614930, f_score 0.527424\n",
      "=== total 63900 match 18234\n",
      ">>> step 64000\n",
      "step 63900-63999, precision 0.462334, recall 0.615659, f_score 0.528093\n",
      "=== total 64000 match 18266\n",
      ">>> step 64100\n",
      "step 64000-64099, precision 0.462708, recall 0.614658, f_score 0.527968\n",
      "=== total 64100 match 18288\n",
      ">>> step 64200\n",
      "step 64100-64199, precision 0.462607, recall 0.613500, f_score 0.527474\n",
      "=== total 64200 match 18292\n",
      ">>> step 64300\n",
      "step 64200-64299, precision 0.462656, recall 0.612248, f_score 0.527043\n",
      "=== total 64300 match 18303\n",
      ">>> step 64400\n",
      "step 64300-64399, precision 0.463150, recall 0.612289, f_score 0.527378\n",
      "=== total 64400 match 18331\n",
      ">>> step 64500\n",
      "step 64400-64499, precision 0.463232, recall 0.612204, f_score 0.527400\n",
      "=== total 64500 match 18345\n",
      ">>> step 64600\n",
      "step 64500-64599, precision 0.463307, recall 0.611623, f_score 0.527233\n",
      "=== total 64600 match 18355\n",
      ">>> step 64700\n",
      "step 64600-64699, precision 0.463193, recall 0.609428, f_score 0.526342\n",
      "=== total 64700 match 18366\n",
      ">>> step 64800\n",
      "step 64700-64799, precision 0.463092, recall 0.608469, f_score 0.525919\n",
      "=== total 64800 match 18370\n",
      ">>> step 64900\n",
      "step 64800-64899, precision 0.463346, recall 0.608832, f_score 0.526218\n",
      "=== total 64900 match 18388\n",
      ">>> step 65000\n",
      "step 64900-64999, precision 0.464026, recall 0.608943, f_score 0.526698\n",
      "=== total 65000 match 18430\n",
      ">>> step 65100\n",
      "step 65000-65099, precision 0.465224, recall 0.610331, f_score 0.527989\n",
      "=== total 65100 match 18490\n",
      ">>> step 65200\n",
      "step 65100-65199, precision 0.465149, recall 0.610331, f_score 0.527941\n",
      "=== total 65200 match 18493\n",
      ">>> step 65300\n",
      "step 65200-65299, precision 0.465214, recall 0.610734, f_score 0.528133\n",
      "=== total 65300 match 18542\n",
      ">>> step 65400\n",
      "step 65300-65399, precision 0.466205, recall 0.612110, f_score 0.529286\n",
      "=== total 65400 match 18627\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> step 65500\n",
      "step 65400-65499, precision 0.467180, recall 0.613635, f_score 0.530485\n",
      "=== total 65500 match 18708\n",
      ">>> step 65600\n",
      "step 65500-65599, precision 0.466959, recall 0.613662, f_score 0.530352\n",
      "=== total 65600 match 18719\n",
      ">>> step 65700\n",
      "step 65600-65699, precision 0.467589, recall 0.614663, f_score 0.531133\n",
      "=== total 65700 match 18790\n",
      ">>> step 65800\n",
      "step 65700-65799, precision 0.467613, recall 0.614525, f_score 0.531096\n",
      "=== total 65800 match 18819\n",
      ">>> step 65900\n",
      "step 65800-65899, precision 0.467538, recall 0.614525, f_score 0.531048\n",
      "=== total 65900 match 18822\n",
      ">>> step 66000\n",
      "step 65900-65999, precision 0.467635, recall 0.614875, f_score 0.531242\n",
      "=== total 66000 match 18863\n",
      ">>> step 66100\n",
      "step 66000-66099, precision 0.467620, recall 0.613434, f_score 0.530693\n",
      "=== total 66100 match 18885\n",
      ">>> step 66200\n",
      "step 66100-66199, precision 0.467323, recall 0.613434, f_score 0.530502\n",
      "=== total 66200 match 18897\n",
      ">>> step 66300\n",
      "step 66200-66299, precision 0.467096, recall 0.613230, f_score 0.530279\n",
      "=== total 66300 match 18934\n",
      ">>> step 66400\n",
      "step 66300-66399, precision 0.466793, recall 0.612025, f_score 0.529633\n",
      "=== total 66400 match 18972\n",
      ">>> step 66500\n",
      "step 66400-66499, precision 0.466102, recall 0.611299, f_score 0.528917\n",
      "=== total 66500 match 19013\n",
      ">>> step 66600\n",
      "step 66500-66599, precision 0.465967, recall 0.610680, f_score 0.528598\n",
      "=== total 66600 match 19070\n",
      ">>> step 66700\n",
      "step 66600-66699, precision 0.466990, recall 0.612146, f_score 0.529806\n",
      "=== total 66700 match 19146\n",
      ">>> step 66800\n",
      "step 66700-66799, precision 0.467653, recall 0.612916, f_score 0.530521\n",
      "=== total 66800 match 19198\n",
      ">>> step 66900\n",
      "step 66800-66899, precision 0.467414, recall 0.613272, f_score 0.530500\n",
      "=== total 66900 match 19257\n",
      ">>> step 67000\n",
      "step 66900-66999, precision 0.468415, recall 0.614164, f_score 0.531478\n",
      "=== total 67000 match 19329\n",
      ">>> step 67100\n",
      "step 67000-67099, precision 0.468184, recall 0.614052, f_score 0.531288\n",
      "=== total 67100 match 19377\n",
      ">>> step 67200\n",
      "step 67100-67199, precision 0.469070, recall 0.615353, f_score 0.532345\n",
      "=== total 67200 match 19447\n",
      ">>> step 67300\n",
      "step 67200-67299, precision 0.469239, recall 0.614997, f_score 0.532320\n",
      "=== total 67300 match 19489\n",
      ">>> step 67400\n",
      "step 67300-67399, precision 0.469035, recall 0.615100, f_score 0.532228\n",
      "=== total 67400 match 19506\n",
      ">>> step 67500\n",
      "step 67400-67499, precision 0.470032, recall 0.616390, f_score 0.533353\n",
      "=== total 67500 match 19571\n",
      ">>> step 67600\n",
      "step 67500-67599, precision 0.470008, recall 0.616390, f_score 0.533337\n",
      "=== total 67600 match 19572\n",
      ">>> step 67700\n",
      "step 67600-67699, precision 0.470753, recall 0.617537, f_score 0.534246\n",
      "=== total 67700 match 19643\n",
      ">>> step 67800\n",
      "step 67700-67799, precision 0.470134, recall 0.617067, f_score 0.533672\n",
      "=== total 67800 match 19688\n",
      ">>> step 67900\n",
      "step 67800-67899, precision 0.470245, recall 0.617602, f_score 0.533943\n",
      "=== total 67900 match 19728\n",
      ">>> step 68000\n",
      "step 67900-67999, precision 0.470758, recall 0.618308, f_score 0.534538\n",
      "=== total 68000 match 19800\n",
      ">>> step 68100\n",
      "step 68000-68099, precision 0.470191, recall 0.617881, f_score 0.534013\n",
      "=== total 68100 match 19843\n",
      ">>> step 68200\n",
      "step 68100-68199, precision 0.469984, recall 0.617235, f_score 0.533638\n",
      "=== total 68200 match 19873\n",
      ">>> step 68300\n",
      "step 68200-68299, precision 0.470889, recall 0.618496, f_score 0.534692\n",
      "=== total 68300 match 19941\n",
      ">>> step 68400\n",
      "step 68300-68399, precision 0.470836, recall 0.617833, f_score 0.534410\n",
      "=== total 68400 match 19956\n",
      ">>> step 68500\n",
      "step 68400-68499, precision 0.471353, recall 0.618635, f_score 0.535043\n",
      "=== total 68500 match 20002\n",
      ">>> step 68600\n",
      "step 68500-68599, precision 0.471595, recall 0.618378, f_score 0.535103\n",
      "=== total 68600 match 20049\n",
      ">>> step 68700\n",
      "step 68600-68699, precision 0.471595, recall 0.618378, f_score 0.535103\n",
      "=== total 68700 match 20049\n",
      ">>> step 68800\n",
      "step 68700-68799, precision 0.472045, recall 0.619250, f_score 0.535719\n",
      "=== total 68800 match 20104\n",
      ">>> step 68900\n",
      "step 68800-68899, precision 0.473012, recall 0.620784, f_score 0.536916\n",
      "=== total 68900 match 20194\n",
      ">>> step 69000\n",
      "step 68900-68999, precision 0.472903, recall 0.620858, f_score 0.536873\n",
      "=== total 69000 match 20205\n",
      ">>> step 69100\n",
      "step 69000-69099, precision 0.472632, recall 0.621173, f_score 0.536816\n",
      "=== total 69100 match 20261\n",
      ">>> step 69200\n",
      "step 69100-69199, precision 0.473394, recall 0.622398, f_score 0.537765\n",
      "=== total 69200 match 20334\n",
      ">>> step 69300\n",
      "step 69200-69299, precision 0.473960, recall 0.622905, f_score 0.538320\n",
      "=== total 69300 match 20392\n",
      ">>> step 69400\n",
      "step 69300-69399, precision 0.474019, recall 0.623045, f_score 0.538410\n",
      "=== total 69400 match 20419\n",
      ">>> step 69500\n",
      "step 69400-69499, precision 0.473764, recall 0.623045, f_score 0.538246\n",
      "=== total 69500 match 20430\n",
      ">>> step 69600\n",
      "step 69500-69599, precision 0.473697, recall 0.623505, f_score 0.538374\n",
      "=== total 69600 match 20473\n",
      ">>> step 69700\n",
      "step 69600-69699, precision 0.472843, recall 0.623505, f_score 0.537822\n",
      "=== total 69700 match 20510\n",
      ">>> step 69800\n",
      "step 69700-69799, precision 0.472486, recall 0.623820, f_score 0.537708\n",
      "=== total 69800 match 20553\n",
      ">>> step 69900\n",
      "step 69800-69899, precision 0.471889, recall 0.623820, f_score 0.537321\n",
      "=== total 69900 match 20579\n",
      ">>> step 70000\n",
      "step 69900-69999, precision 0.471751, recall 0.623820, f_score 0.537232\n",
      "=== total 70000 match 20585\n",
      ">>> step 70100\n",
      "step 70000-70099, precision 0.471339, recall 0.623820, f_score 0.536964\n",
      "=== total 70100 match 20603\n",
      ">>> step 70200\n",
      "step 70100-70199, precision 0.470941, recall 0.624134, f_score 0.536822\n",
      "=== total 70200 match 20648\n",
      ">>> step 70300\n",
      "step 70200-70299, precision 0.470531, recall 0.624134, f_score 0.536556\n",
      "=== total 70300 match 20666\n",
      ">>> step 70400\n",
      "step 70300-70399, precision 0.470031, recall 0.624134, f_score 0.536230\n",
      "=== total 70400 match 20688\n",
      ">>> step 70500\n",
      "step 70400-70499, precision 0.469441, recall 0.624134, f_score 0.535846\n",
      "=== total 70500 match 20714\n",
      ">>> step 70600\n",
      "step 70500-70599, precision 0.469311, recall 0.624736, f_score 0.535983\n",
      "=== total 70600 match 20773\n",
      ">>> step 70700\n",
      "step 70600-70699, precision 0.468854, recall 0.625048, f_score 0.535800\n",
      "=== total 70700 match 20821\n",
      ">>> step 70800\n",
      "step 70700-70799, precision 0.468534, recall 0.624433, f_score 0.535365\n",
      "=== total 70800 match 20848\n",
      ">>> step 70900\n",
      "step 70800-70899, precision 0.467883, recall 0.624433, f_score 0.534940\n",
      "=== total 70900 match 20877\n",
      ">>> step 71000\n",
      "step 70900-70999, precision 0.467495, recall 0.624888, f_score 0.534853\n",
      "=== total 71000 match 20935\n",
      ">>> step 71100\n",
      "step 71000-71099, precision 0.466714, recall 0.624888, f_score 0.534342\n",
      "=== total 71100 match 20970\n",
      ">>> step 71200\n",
      "step 71100-71199, precision 0.466492, recall 0.624888, f_score 0.534196\n",
      "=== total 71200 match 20980\n",
      ">>> step 71300\n",
      "step 71200-71299, precision 0.466483, recall 0.625486, f_score 0.534408\n",
      "=== total 71300 match 21034\n",
      ">>> step 71400\n",
      "step 71300-71399, precision 0.466128, recall 0.625486, f_score 0.534175\n",
      "=== total 71400 match 21050\n",
      ">>> step 71500\n",
      "step 71400-71499, precision 0.465575, recall 0.625486, f_score 0.533812\n",
      "=== total 71500 match 21075\n",
      ">>> step 71600\n",
      "step 71500-71599, precision 0.465596, recall 0.625796, f_score 0.533938\n",
      "=== total 71600 match 21102\n",
      ">>> step 71700\n",
      "step 71600-71699, precision 0.465089, recall 0.625796, f_score 0.533605\n",
      "=== total 71700 match 21125\n",
      ">>> step 71800\n",
      "step 71700-71799, precision 0.464497, recall 0.625963, f_score 0.533275\n",
      "=== total 71800 match 21167\n",
      ">>> step 71900\n",
      "step 71800-71899, precision 0.464014, recall 0.625963, f_score 0.532958\n",
      "=== total 71900 match 21189\n",
      ">>> step 72000\n",
      "step 71900-71999, precision 0.464186, recall 0.626557, f_score 0.533286\n",
      "=== total 72000 match 21235\n",
      ">>> step 72100\n",
      "step 72000-72099, precision 0.463859, recall 0.626866, f_score 0.533182\n",
      "=== total 72100 match 21278\n",
      ">>> step 72200\n",
      "step 72100-72199, precision 0.463741, recall 0.627103, f_score 0.533189\n",
      "=== total 72200 match 21305\n",
      ">>> step 72300\n",
      "step 72200-72299, precision 0.463143, recall 0.627173, f_score 0.532819\n",
      "=== total 72300 match 21339\n",
      ">>> step 72400\n",
      "step 72300-72399, precision 0.462731, recall 0.627173, f_score 0.532547\n",
      "=== total 72400 match 21358\n",
      ">>> step 72500\n",
      "step 72400-72499, precision 0.462082, recall 0.627173, f_score 0.532117\n",
      "=== total 72500 match 21388\n",
      ">>> step 72600\n",
      "step 72500-72599, precision 0.461521, recall 0.627173, f_score 0.531744\n",
      "=== total 72600 match 21414\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> step 72700\n",
      "step 72600-72699, precision 0.461417, recall 0.627622, f_score 0.531837\n",
      "=== total 72700 match 21460\n",
      ">>> step 72800\n",
      "step 72700-72799, precision 0.461249, recall 0.628070, f_score 0.531886\n",
      "=== total 72800 match 21509\n",
      ">>> step 72900\n",
      "step 72800-72899, precision 0.460606, recall 0.628070, f_score 0.531458\n",
      "=== total 72900 match 21539\n",
      ">>> step 73000\n",
      "step 72900-72999, precision 0.460034, recall 0.628235, f_score 0.531136\n",
      "=== total 73000 match 21581\n",
      ">>> step 73100\n",
      "step 73000-73099, precision 0.460382, recall 0.628822, f_score 0.531578\n",
      "=== total 73100 match 21619\n",
      ">>> step 73200\n",
      "step 73100-73199, precision 0.459808, recall 0.628822, f_score 0.531195\n",
      "=== total 73200 match 21646\n",
      ">>> step 73300\n",
      "step 73200-73299, precision 0.460692, recall 0.629991, f_score 0.532202\n",
      "=== total 73300 match 21713\n",
      ">>> step 73400\n",
      "step 73300-73399, precision 0.460764, recall 0.630596, f_score 0.532466\n",
      "=== total 73400 match 21766\n",
      ">>> step 73500\n",
      "step 73400-73499, precision 0.461179, recall 0.631282, f_score 0.532987\n",
      "=== total 73500 match 21818\n",
      ">>> step 73600\n",
      "step 73500-73599, precision 0.460937, recall 0.631628, f_score 0.532949\n",
      "=== total 73600 match 21862\n",
      ">>> step 73700\n",
      "step 73600-73699, precision 0.460431, recall 0.631628, f_score 0.532611\n",
      "=== total 73700 match 21886\n",
      ">>> step 73800\n",
      "step 73700-73799, precision 0.461051, recall 0.632642, f_score 0.533386\n",
      "=== total 73800 match 21952\n",
      ">>> step 73900\n",
      "step 73800-73899, precision 0.460485, recall 0.632642, f_score 0.533007\n",
      "=== total 73900 match 21979\n",
      ">>> step 74000\n",
      "step 73900-73999, precision 0.460087, recall 0.632642, f_score 0.532740\n",
      "=== total 74000 match 21998\n",
      ">>> step 74100\n",
      "step 74000-74099, precision 0.459607, recall 0.632642, f_score 0.532418\n",
      "=== total 74100 match 22021\n",
      ">>> step 74200\n",
      "step 74100-74199, precision 0.459695, recall 0.633113, f_score 0.532644\n",
      "=== total 74200 match 22069\n",
      ">>> step 74300\n",
      "step 74200-74299, precision 0.460423, recall 0.634163, f_score 0.533504\n",
      "=== total 74300 match 22134\n",
      ">>> step 74400\n",
      "step 74300-74399, precision 0.459793, recall 0.634254, f_score 0.533114\n",
      "=== total 74400 match 22173\n",
      ">>> step 74500\n",
      "step 74400-74499, precision 0.459400, recall 0.634254, f_score 0.532849\n",
      "=== total 74500 match 22192\n",
      ">>> step 74600\n",
      "step 74500-74599, precision 0.459261, recall 0.634550, f_score 0.532860\n",
      "=== total 74600 match 22227\n",
      ">>> step 74700\n",
      "step 74600-74699, precision 0.460103, recall 0.635682, f_score 0.533826\n",
      "=== total 74700 match 22295\n",
      ">>> step 74800\n",
      "step 74700-74799, precision 0.460155, recall 0.636268, f_score 0.534067\n",
      "=== total 74800 match 22349\n",
      ">>> step 74900\n",
      "step 74800-74899, precision 0.460887, recall 0.637390, f_score 0.534955\n",
      "=== total 74900 match 22422\n",
      ">>> step 75000\n",
      "step 74900-74999, precision 0.460977, recall 0.637970, f_score 0.535221\n",
      "=== total 75000 match 22474\n",
      ">>> step 75100\n",
      "step 75000-75099, precision 0.461620, recall 0.638949, f_score 0.535999\n",
      "=== total 75100 match 22538\n",
      ">>> step 75200\n",
      "step 75100-75199, precision 0.461027, recall 0.639082, f_score 0.535645\n",
      "=== total 75200 match 22580\n",
      ">>> step 75300\n",
      "step 75200-75299, precision 0.461790, recall 0.640054, f_score 0.536502\n",
      "=== total 75300 match 22638\n",
      ">>> step 75400\n",
      "step 75300-75399, precision 0.461525, recall 0.640186, f_score 0.536369\n",
      "=== total 75400 match 22664\n",
      ">>> step 75500\n",
      "step 75400-75499, precision 0.462344, recall 0.641152, f_score 0.537262\n",
      "=== total 75500 match 22719\n",
      ">>> step 75600\n",
      "step 75500-75599, precision 0.461877, recall 0.641152, f_score 0.536946\n",
      "=== total 75600 match 22742\n",
      ">>> step 75700\n",
      "step 75600-75699, precision 0.461613, recall 0.641152, f_score 0.536767\n",
      "=== total 75700 match 22755\n",
      ">>> step 75800\n",
      "step 75700-75799, precision 0.461505, recall 0.641503, f_score 0.536817\n",
      "=== total 75800 match 22795\n",
      ">>> step 75900\n",
      "step 75800-75899, precision 0.461154, recall 0.641721, f_score 0.536656\n",
      "=== total 75900 match 22834\n",
      ">>> step 76000\n",
      "step 75900-75999, precision 0.460590, recall 0.641721, f_score 0.536274\n",
      "=== total 76000 match 22862\n",
      ">>> step 76100\n",
      "step 76000-76099, precision 0.460860, recall 0.642288, f_score 0.536655\n",
      "=== total 76100 match 22905\n",
      ">>> step 76200\n",
      "step 76100-76199, precision 0.460257, recall 0.642288, f_score 0.536246\n",
      "=== total 76200 match 22935\n",
      ">>> step 76300\n",
      "step 76200-76299, precision 0.461047, recall 0.643312, f_score 0.537139\n",
      "=== total 76300 match 23002\n",
      ">>> step 76400\n",
      "step 76300-76399, precision 0.460687, recall 0.643533, f_score 0.536971\n",
      "=== total 76400 match 23046\n",
      ">>> step 76500\n",
      "step 76400-76499, precision 0.460612, recall 0.643813, f_score 0.537018\n",
      "=== total 76500 match 23078\n",
      ">>> step 76600\n",
      "step 76500-76599, precision 0.460317, recall 0.644093, f_score 0.536915\n",
      "=== total 76600 match 23121\n",
      ">>> step 76700\n",
      "step 76600-76699, precision 0.460874, recall 0.644846, f_score 0.537555\n",
      "=== total 76700 match 23169\n",
      ">>> step 76800\n",
      "step 76700-76799, precision 0.460627, recall 0.645167, f_score 0.537499\n",
      "=== total 76800 match 23214\n",
      ">>> step 76900\n",
      "step 76800-76899, precision 0.460151, recall 0.645167, f_score 0.537175\n",
      "=== total 76900 match 23238\n",
      ">>> step 77000\n",
      "step 76900-76999, precision 0.459696, recall 0.645167, f_score 0.536865\n",
      "=== total 77000 match 23261\n",
      ">>> step 77100\n",
      "step 77000-77099, precision 0.460457, recall 0.646174, f_score 0.537732\n",
      "=== total 77100 match 23329\n",
      ">>> step 77200\n",
      "step 77100-77199, precision 0.461275, recall 0.647235, f_score 0.538657\n",
      "=== total 77200 match 23396\n",
      ">>> step 77300\n",
      "step 77200-77299, precision 0.460862, recall 0.647235, f_score 0.538375\n",
      "=== total 77300 match 23417\n",
      ">>> step 77400\n",
      "step 77300-77399, precision 0.460390, recall 0.647235, f_score 0.538053\n",
      "=== total 77400 match 23441\n",
      ">>> step 77500\n",
      "step 77400-77499, precision 0.460589, recall 0.647665, f_score 0.538337\n",
      "=== total 77500 match 23483\n",
      ">>> step 77600\n",
      "step 77500-77599, precision 0.461460, recall 0.648716, f_score 0.539296\n",
      "=== total 77600 match 23547\n",
      ">>> step 77700\n",
      "step 77600-77699, precision 0.460951, recall 0.648716, f_score 0.538948\n",
      "=== total 77700 match 23573\n",
      ">>> step 77800\n",
      "step 77700-77799, precision 0.461643, recall 0.649762, f_score 0.539781\n",
      "=== total 77800 match 23646\n",
      ">>> step 77900\n",
      "step 77800-77899, precision 0.461610, recall 0.650303, f_score 0.539946\n",
      "=== total 77900 match 23704\n",
      ">>> step 78000\n",
      "step 77900-77999, precision 0.462237, recall 0.651339, f_score 0.540732\n",
      "=== total 78000 match 23780\n",
      ">>> step 78100\n",
      "step 78000-78099, precision 0.461577, recall 0.651339, f_score 0.540280\n",
      "=== total 78100 match 23814\n",
      ">>> step 78200\n",
      "step 78100-78199, precision 0.460977, recall 0.651339, f_score 0.539869\n",
      "=== total 78200 match 23845\n",
      ">>> step 78300\n",
      "step 78200-78299, precision 0.461680, recall 0.652369, f_score 0.540705\n",
      "=== total 78300 match 23917\n",
      ">>> step 78400\n",
      "step 78300-78399, precision 0.461025, recall 0.652369, f_score 0.540255\n",
      "=== total 78400 match 23951\n",
      ">>> step 78500\n",
      "step 78400-78499, precision 0.460429, recall 0.652369, f_score 0.539846\n",
      "=== total 78500 match 23982\n",
      ">>> step 78600\n",
      "step 78500-78599, precision 0.459988, recall 0.652369, f_score 0.539542\n",
      "=== total 78600 match 24005\n",
      ">>> step 78700\n",
      "step 78600-78699, precision 0.459921, recall 0.652902, f_score 0.539679\n",
      "=== total 78700 match 24065\n",
      ">>> step 78800\n",
      "step 78700-78799, precision 0.460563, recall 0.653923, f_score 0.540470\n",
      "=== total 78800 match 24140\n",
      ">>> step 78900\n",
      "step 78800-78899, precision 0.460087, recall 0.653923, f_score 0.540141\n",
      "=== total 78900 match 24165\n",
      ">>> step 79000\n",
      "step 78900-78999, precision 0.459406, recall 0.653943, f_score 0.539679\n",
      "=== total 79000 match 24203\n",
      ">>> step 79100\n",
      "step 79000-79099, precision 0.459465, recall 0.654533, f_score 0.539920\n",
      "=== total 79100 match 24263\n",
      ">>> step 79200\n",
      "step 79100-79199, precision 0.460015, recall 0.655463, f_score 0.540616\n",
      "=== total 79200 match 24334\n",
      ">>> step 79300\n",
      "step 79200-79299, precision 0.460078, recall 0.655928, f_score 0.540818\n",
      "=== total 79300 match 24385\n",
      ">>> step 79400\n",
      "step 79300-79399, precision 0.460617, recall 0.656931, f_score 0.541532\n",
      "=== total 79400 match 24465\n",
      ">>> step 79500\n",
      "step 79400-79499, precision 0.460381, recall 0.657101, f_score 0.541426\n",
      "=== total 79500 match 24521\n",
      ">>> step 79600\n",
      "step 79500-79599, precision 0.461047, recall 0.658096, f_score 0.542225\n",
      "=== total 79600 match 24594\n",
      ">>> step 79700\n",
      "step 79600-79699, precision 0.460430, recall 0.658096, f_score 0.541797\n",
      "=== total 79700 match 24627\n",
      ">>> step 79800\n",
      "step 79700-79799, precision 0.461186, recall 0.659086, f_score 0.542656\n",
      "=== total 79800 match 24695\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> step 79900\n",
      "step 79800-79899, precision 0.460590, recall 0.659086, f_score 0.542243\n",
      "=== total 79900 match 24727\n",
      ">>> step 80000\n",
      "step 79900-79999, precision 0.460949, recall 0.659547, f_score 0.542648\n",
      "=== total 80000 match 24788\n",
      ">>> step 80100\n",
      "step 80000-80099, precision 0.460484, recall 0.659547, f_score 0.542326\n",
      "=== total 80100 match 24813\n",
      ">>> step 80200\n",
      "step 80100-80199, precision 0.459947, recall 0.659547, f_score 0.541953\n",
      "=== total 80200 match 24842\n",
      ">>> step 80300\n",
      "step 80200-80299, precision 0.459875, recall 0.659901, f_score 0.542023\n",
      "=== total 80300 match 24885\n",
      ">>> step 80400\n",
      "step 80300-80399, precision 0.459348, recall 0.660058, f_score 0.541709\n",
      "=== total 80400 match 24931\n",
      ">>> step 80500\n",
      "step 80400-80499, precision 0.458924, recall 0.660058, f_score 0.541415\n",
      "=== total 80500 match 24954\n",
      ">>> step 80600\n",
      "step 80500-80599, precision 0.458918, recall 0.660566, f_score 0.541581\n",
      "=== total 80600 match 25011\n",
      ">>> step 80700\n",
      "step 80600-80699, precision 0.458298, recall 0.660586, f_score 0.541156\n",
      "=== total 80700 match 25047\n",
      ">>> step 80800\n",
      "step 80700-80799, precision 0.458990, recall 0.661540, f_score 0.541959\n",
      "=== total 80800 match 25116\n",
      ">>> step 80900\n",
      "step 80800-80899, precision 0.458375, recall 0.661448, f_score 0.541498\n",
      "=== total 80900 match 25165\n",
      ">>> step 81000\n",
      "step 80900-80999, precision 0.458054, recall 0.661701, f_score 0.541359\n",
      "=== total 81000 match 25211\n",
      ">>> step 81100\n",
      "step 81000-81099, precision 0.457749, recall 0.661895, f_score 0.541211\n",
      "=== total 81100 match 25254\n",
      ">>> step 81200\n",
      "step 81100-81199, precision 0.458363, recall 0.662726, f_score 0.541918\n",
      "=== total 81200 match 25314\n",
      ">>> step 81300\n",
      "step 81200-81299, precision 0.457916, recall 0.662860, f_score 0.541650\n",
      "=== total 81300 match 25354\n",
      ">>> step 81400\n",
      "step 81300-81399, precision 0.457501, recall 0.662860, f_score 0.541360\n",
      "=== total 81400 match 25377\n",
      ">>> step 81500\n",
      "step 81400-81499, precision 0.456892, recall 0.662880, f_score 0.540940\n",
      "=== total 81500 match 25413\n",
      ">>> step 81600\n",
      "step 81500-81599, precision 0.457470, recall 0.663820, f_score 0.541658\n",
      "=== total 81600 match 25488\n",
      ">>> step 81700\n",
      "step 81600-81699, precision 0.457994, recall 0.664774, f_score 0.542343\n",
      "=== total 81700 match 25568\n",
      ">>> step 81800\n",
      "step 81700-81799, precision 0.457422, recall 0.664774, f_score 0.541941\n",
      "=== total 81800 match 25600\n",
      ">>> step 81900\n",
      "step 81800-81899, precision 0.456858, recall 0.664812, f_score 0.541558\n",
      "=== total 81900 match 25636\n",
      ">>> step 82000\n",
      "step 81900-81999, precision 0.456887, recall 0.665382, f_score 0.541768\n",
      "=== total 82000 match 25700\n",
      ">>> step 82100\n",
      "step 82000-82099, precision 0.457604, recall 0.666328, f_score 0.542585\n",
      "=== total 82100 match 25769\n",
      ">>> step 82200\n",
      "step 82100-82199, precision 0.457019, recall 0.666328, f_score 0.542173\n",
      "=== total 82200 match 25802\n",
      ">>> step 82300\n",
      "step 82200-82299, precision 0.457644, recall 0.667268, f_score 0.542925\n",
      "=== total 82300 match 25876\n",
      ">>> step 82400\n",
      "step 82300-82399, precision 0.457553, recall 0.667754, f_score 0.543021\n",
      "=== total 82400 match 25938\n",
      ">>> step 82500\n",
      "step 82400-82499, precision 0.458120, recall 0.668687, f_score 0.543729\n",
      "=== total 82500 match 26015\n",
      ">>> step 82600\n",
      "step 82500-82599, precision 0.457593, recall 0.668687, f_score 0.543357\n",
      "=== total 82600 match 26045\n",
      ">>> step 82700\n",
      "step 82600-82699, precision 0.457066, recall 0.668687, f_score 0.542986\n",
      "=== total 82700 match 26075\n",
      ">>> step 82800\n",
      "step 82700-82799, precision 0.457825, recall 0.669613, f_score 0.543827\n",
      "=== total 82800 match 26141\n",
      ">>> step 82900\n",
      "step 82800-82899, precision 0.457318, recall 0.669613, f_score 0.543469\n",
      "=== total 82900 match 26170\n",
      ">>> step 83000\n",
      "step 82900-82999, precision 0.456933, recall 0.669613, f_score 0.543198\n",
      "=== total 83000 match 26192\n",
      ">>> step 83100\n",
      "step 83000-83099, precision 0.456585, recall 0.669613, f_score 0.542951\n",
      "=== total 83100 match 26212\n",
      ">>> step 83200\n",
      "step 83100-83199, precision 0.456601, recall 0.670093, f_score 0.543120\n",
      "=== total 83200 match 26268\n",
      ">>> step 83300\n",
      "step 83200-83299, precision 0.457095, recall 0.671012, f_score 0.543772\n",
      "=== total 83300 match 26349\n",
      ">>> step 83400\n",
      "step 83300-83399, precision 0.456437, recall 0.671012, f_score 0.543306\n",
      "=== total 83400 match 26387\n",
      ">>> step 83500\n",
      "step 83400-83499, precision 0.455891, recall 0.671049, f_score 0.542931\n",
      "=== total 83500 match 26423\n",
      ">>> step 83600\n",
      "step 83500-83599, precision 0.455922, recall 0.671598, f_score 0.543132\n",
      "=== total 83600 match 26487\n",
      ">>> step 83700\n",
      "step 83600-83699, precision 0.456394, recall 0.672399, f_score 0.543729\n",
      "=== total 83700 match 26556\n",
      ">>> step 83800\n",
      "step 83700-83799, precision 0.456428, recall 0.672871, f_score 0.543908\n",
      "=== total 83800 match 26611\n",
      ">>> step 83900\n",
      "step 83800-83899, precision 0.457087, recall 0.673775, f_score 0.544671\n",
      "=== total 83900 match 26682\n",
      ">>> step 84000\n",
      "step 83900-83999, precision 0.457102, recall 0.674243, f_score 0.544835\n",
      "=== total 84000 match 26738\n",
      ">>> step 84100\n",
      "step 84000-84099, precision 0.457688, recall 0.675139, f_score 0.545543\n",
      "=== total 84100 match 26813\n",
      ">>> step 84200\n",
      "step 84100-84199, precision 0.457211, recall 0.675139, f_score 0.545204\n",
      "=== total 84200 match 26841\n",
      ">>> step 84300\n",
      "step 84200-84299, precision 0.457761, recall 0.676030, f_score 0.545885\n",
      "=== total 84300 match 26918\n",
      ">>> step 84400\n",
      "step 84300-84399, precision 0.457217, recall 0.676030, f_score 0.545499\n",
      "=== total 84400 match 26950\n",
      ">>> step 84500\n",
      "step 84400-84499, precision 0.457661, recall 0.676810, f_score 0.546069\n",
      "=== total 84500 match 27020\n",
      ">>> step 84600\n",
      "step 84500-84599, precision 0.457221, recall 0.676810, f_score 0.545755\n",
      "=== total 84600 match 27046\n",
      ">>> step 84700\n",
      "step 84600-84699, precision 0.456698, recall 0.676810, f_score 0.545382\n",
      "=== total 84700 match 27077\n",
      ">>> step 84800\n",
      "step 84700-84799, precision 0.456831, recall 0.677269, f_score 0.545627\n",
      "=== total 84800 match 27126\n",
      ">>> step 84900\n",
      "step 84800-84899, precision 0.456293, recall 0.677269, f_score 0.545243\n",
      "=== total 84900 match 27158\n",
      ">>> step 85000\n",
      "step 84900-84999, precision 0.455923, recall 0.677269, f_score 0.544979\n",
      "=== total 85000 match 27180\n",
      ">>> step 85100\n",
      "step 85000-85099, precision 0.455857, recall 0.677727, f_score 0.545079\n",
      "=== total 85100 match 27241\n",
      ">>> step 85200\n",
      "step 85100-85199, precision 0.455356, recall 0.677727, f_score 0.544721\n",
      "=== total 85200 match 27271\n",
      ">>> step 85300\n",
      "step 85200-85299, precision 0.455985, recall 0.678604, f_score 0.545455\n",
      "=== total 85300 match 27343\n",
      ">>> step 85400\n",
      "step 85300-85399, precision 0.455704, recall 0.678723, f_score 0.545292\n",
      "=== total 85400 match 27384\n",
      ">>> step 85500\n",
      "step 85400-85499, precision 0.455633, recall 0.679055, f_score 0.545347\n",
      "=== total 85500 match 27430\n",
      ">>> step 85600\n",
      "step 85500-85599, precision 0.455274, recall 0.679177, f_score 0.545130\n",
      "=== total 85600 match 27467\n",
      ">>> step 85700\n",
      "step 85600-85699, precision 0.455833, recall 0.679959, f_score 0.545783\n",
      "=== total 85700 match 27532\n",
      ">>> step 85800\n",
      "step 85700-85799, precision 0.455370, recall 0.680045, f_score 0.545478\n",
      "=== total 85800 match 27571\n",
      ">>> step 85900\n",
      "step 85800-85899, precision 0.455023, recall 0.680045, f_score 0.545230\n",
      "=== total 85900 match 27592\n",
      ">>> step 86000\n",
      "step 85900-85999, precision 0.454513, recall 0.680045, f_score 0.544863\n",
      "=== total 86000 match 27623\n",
      ">>> step 86100\n",
      "step 86000-86099, precision 0.455202, recall 0.680910, f_score 0.545636\n",
      "=== total 86100 match 27691\n",
      ">>> step 86200\n",
      "step 86100-86199, precision 0.455855, recall 0.681769, f_score 0.546381\n",
      "=== total 86200 match 27761\n",
      ">>> step 86300\n",
      "step 86200-86299, precision 0.455380, recall 0.681769, f_score 0.546039\n",
      "=== total 86300 match 27790\n",
      ">>> step 86400\n",
      "step 86300-86399, precision 0.455098, recall 0.681923, f_score 0.545886\n",
      "=== total 86400 match 27827\n",
      ">>> step 86500\n",
      "step 86400-86499, precision 0.455096, recall 0.682351, f_score 0.546022\n",
      "=== total 86500 match 27882\n",
      ">>> step 86600\n",
      "step 86500-86599, precision 0.455723, recall 0.683169, f_score 0.546734\n",
      "=== total 86600 match 27949\n",
      ">>> step 86700\n",
      "step 86600-86699, precision 0.455478, recall 0.683169, f_score 0.546559\n",
      "=== total 86700 match 27964\n",
      ">>> step 86800\n",
      "step 86700-86799, precision 0.455384, recall 0.683014, f_score 0.546441\n",
      "=== total 86800 match 27983\n",
      ">>> step 86900\n",
      "step 86800-86899, precision 0.455693, recall 0.683504, f_score 0.546820\n",
      "=== total 86900 match 28032\n",
      ">>> step 87000\n",
      "step 86900-86999, precision 0.455586, recall 0.683538, f_score 0.546754\n",
      "=== total 87000 match 28043\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> step 87100\n",
      "step 87000-87099, precision 0.455630, recall 0.683665, f_score 0.546827\n",
      "=== total 87100 match 28071\n",
      ">>> step 87200\n",
      "step 87100-87199, precision 0.455634, recall 0.683243, f_score 0.546694\n",
      "=== total 87200 match 28073\n",
      ">>> step 87300\n",
      "step 87200-87299, precision 0.455526, recall 0.683362, f_score 0.546654\n",
      "=== total 87300 match 28095\n",
      ">>> step 87400\n",
      "step 87300-87399, precision 0.455518, recall 0.683581, f_score 0.546719\n",
      "=== total 87400 match 28124\n",
      ">>> step 87500\n",
      "step 87400-87499, precision 0.455405, recall 0.683581, f_score 0.546638\n",
      "=== total 87500 match 28131\n",
      ">>> step 87600\n",
      "step 87500-87599, precision 0.455659, recall 0.683902, f_score 0.546923\n",
      "=== total 87600 match 28157\n",
      ">>> step 87700\n",
      "step 87600-87699, precision 0.455500, recall 0.684014, f_score 0.546844\n",
      "=== total 87700 match 28191\n",
      ">>> step 87800\n",
      "step 87700-87799, precision 0.455769, recall 0.684127, f_score 0.547074\n",
      "=== total 87800 match 28227\n",
      ">>> step 87900\n",
      "step 87800-87899, precision 0.455656, recall 0.684127, f_score 0.546993\n",
      "=== total 87900 match 28234\n",
      ">>> step 88000\n",
      "step 87900-87999, precision 0.455592, recall 0.684127, f_score 0.546946\n",
      "=== total 88000 match 28238\n",
      ">>> step 88100\n",
      "step 88000-88099, precision 0.456428, recall 0.684799, f_score 0.547764\n",
      "=== total 88100 match 28298\n",
      ">>> step 88200\n",
      "step 88100-88199, precision 0.456951, recall 0.684497, f_score 0.548044\n",
      "=== total 88200 match 28340\n",
      ">>> step 88300\n",
      "step 88200-88299, precision 0.457036, recall 0.684697, f_score 0.548169\n",
      "=== total 88300 match 28361\n",
      ">>> step 88400\n",
      "step 88300-88399, precision 0.457487, recall 0.685229, f_score 0.548664\n",
      "=== total 88400 match 28403\n",
      ">>> step 88500\n",
      "step 88400-88499, precision 0.457422, recall 0.685229, f_score 0.548617\n",
      "=== total 88500 match 28407\n",
      ">>> step 88600\n",
      "step 88500-88599, precision 0.457342, recall 0.684291, f_score 0.548258\n",
      "=== total 88600 match 28412\n",
      ">>> step 88700\n",
      "step 88600-88699, precision 0.458118, recall 0.685120, f_score 0.549082\n",
      "=== total 88700 match 28473\n",
      ">>> step 88800\n",
      "step 88700-88799, precision 0.458114, recall 0.685042, f_score 0.549054\n",
      "=== total 88800 match 28482\n",
      ">>> step 88900\n",
      "step 88800-88899, precision 0.458390, recall 0.684238, f_score 0.548994\n",
      "=== total 88900 match 28515\n",
      ">>> step 89000\n",
      "step 88900-88999, precision 0.459137, recall 0.684548, f_score 0.549629\n",
      "=== total 89000 match 28571\n",
      ">>> step 89100\n",
      "step 89000-89099, precision 0.458934, recall 0.683654, f_score 0.549195\n",
      "=== total 89100 match 28588\n",
      ">>> step 89200\n",
      "step 89100-89199, precision 0.459057, recall 0.682657, f_score 0.548961\n",
      "=== total 89200 match 28613\n",
      ">>> step 89300\n",
      "step 89200-89299, precision 0.459856, recall 0.683479, f_score 0.549799\n",
      "=== total 89300 match 28672\n",
      ">>> step 89400\n",
      "step 89300-89399, precision 0.459876, recall 0.682710, f_score 0.549564\n",
      "=== total 89400 match 28686\n",
      ">>> step 89500\n",
      "step 89400-89499, precision 0.459898, recall 0.682488, f_score 0.549508\n",
      "=== total 89500 match 28702\n",
      ">>> step 89600\n",
      "step 89500-89599, precision 0.460175, recall 0.682659, f_score 0.549761\n",
      "=== total 89600 match 28726\n",
      ">>> step 89700\n",
      "step 89600-89699, precision 0.460453, recall 0.683068, f_score 0.550091\n",
      "=== total 89700 match 28763\n",
      ">>> step 89800\n",
      "step 89700-89799, precision 0.460595, recall 0.683345, f_score 0.550283\n",
      "=== total 89800 match 28791\n",
      ">>> step 89900\n",
      "step 89800-89899, precision 0.461141, recall 0.683194, f_score 0.550623\n",
      "=== total 89900 match 28835\n",
      ">>> step 90000\n",
      "step 89900-89999, precision 0.461061, recall 0.682317, f_score 0.550281\n",
      "=== total 90000 match 28840\n",
      ">>> step 90100\n",
      "step 90000-90099, precision 0.461245, recall 0.682212, f_score 0.550378\n",
      "=== total 90100 match 28861\n",
      ">>> step 90200\n",
      "step 90100-90199, precision 0.461794, recall 0.682830, f_score 0.550970\n",
      "=== total 90200 match 28909\n",
      ">>> step 90300\n",
      "step 90200-90299, precision 0.461698, recall 0.682830, f_score 0.550902\n",
      "=== total 90300 match 28915\n",
      ">>> step 90400\n",
      "step 90300-90399, precision 0.462481, recall 0.683309, f_score 0.551615\n",
      "=== total 90400 match 28972\n",
      ">>> step 90500\n",
      "step 90400-90499, precision 0.462385, recall 0.681848, f_score 0.551070\n",
      "=== total 90500 match 28978\n",
      ">>> step 90600\n",
      "step 90500-90599, precision 0.462740, recall 0.682269, f_score 0.551459\n",
      "=== total 90600 match 29012\n",
      ">>> step 90700\n",
      "step 90600-90699, precision 0.463473, recall 0.683074, f_score 0.552243\n",
      "=== total 90700 match 29074\n",
      ">>> step 90800\n",
      "step 90700-90799, precision 0.463409, recall 0.683074, f_score 0.552198\n",
      "=== total 90800 match 29078\n",
      ">>> step 90900\n",
      "step 90800-90899, precision 0.463411, recall 0.681604, f_score 0.551718\n",
      "=== total 90900 match 29080\n",
      ">>> step 91000\n",
      "step 90900-90999, precision 0.464202, recall 0.682357, f_score 0.552526\n",
      "=== total 91000 match 29136\n",
      ">>> step 91100\n",
      "step 91000-91099, precision 0.464155, recall 0.682357, f_score 0.552492\n",
      "=== total 91100 match 29139\n",
      ">>> step 91200\n",
      "step 91100-91199, precision 0.464709, recall 0.682965, f_score 0.553084\n",
      "=== total 91200 match 29186\n",
      ">>> step 91300\n",
      "step 91200-91299, precision 0.464645, recall 0.682484, f_score 0.552881\n",
      "=== total 91300 match 29190\n",
      ">>> step 91400\n",
      "step 91300-91399, precision 0.465431, recall 0.683230, f_score 0.553682\n",
      "=== total 91400 match 29246\n",
      ">>> step 91500\n",
      "step 91400-91499, precision 0.465950, recall 0.683833, f_score 0.554247\n",
      "=== total 91500 match 29295\n",
      ">>> step 91600\n",
      "step 91500-91599, precision 0.465902, recall 0.683354, f_score 0.554056\n",
      "=== total 91600 match 29298\n",
      ">>> step 91700\n",
      "step 91600-91699, precision 0.466132, recall 0.682497, f_score 0.553937\n",
      "=== total 91700 match 29320\n",
      ">>> step 91800\n",
      "step 91700-91799, precision 0.466430, recall 0.682909, f_score 0.554283\n",
      "=== total 91800 match 29357\n",
      ">>> step 91900\n",
      "step 91800-91899, precision 0.466853, recall 0.683414, f_score 0.554747\n",
      "=== total 91900 match 29399\n",
      ">>> step 92000\n",
      "step 91900-91999, precision 0.466837, recall 0.683414, f_score 0.554736\n",
      "=== total 92000 match 29400\n",
      ">>> step 92100\n",
      "step 92000-92099, precision 0.466805, recall 0.683414, f_score 0.554714\n",
      "=== total 92100 match 29402\n",
      ">>> step 92200\n",
      "step 92100-92199, precision 0.466757, recall 0.683414, f_score 0.554680\n",
      "=== total 92200 match 29405\n",
      ">>> step 92300\n",
      "step 92200-92299, precision 0.466898, recall 0.683415, f_score 0.554779\n",
      "=== total 92300 match 29424\n",
      ">>> step 92400\n",
      "step 92300-92399, precision 0.467813, recall 0.684211, f_score 0.555688\n",
      "=== total 92400 match 29484\n",
      ">>> step 92500\n",
      "step 92400-92499, precision 0.468016, recall 0.684007, f_score 0.555764\n",
      "=== total 92500 match 29499\n",
      ">>> step 92600\n",
      "step 92500-92599, precision 0.468729, recall 0.684153, f_score 0.556314\n",
      "=== total 92600 match 29548\n",
      ">>> step 92700\n",
      "step 92600-92699, precision 0.469051, recall 0.684003, f_score 0.556491\n",
      "=== total 92700 match 29581\n",
      ">>> step 92800\n",
      "step 92700-92799, precision 0.469773, recall 0.684780, f_score 0.557256\n",
      "=== total 92800 match 29642\n",
      ">>> step 92900\n",
      "step 92800-92899, precision 0.470343, recall 0.685047, f_score 0.557746\n",
      "=== total 92900 match 29689\n",
      ">>> step 93000\n",
      "step 92900-92999, precision 0.470299, recall 0.685078, f_score 0.557725\n",
      "=== total 93000 match 29696\n",
      ">>> step 93100\n",
      "step 93000-93099, precision 0.470267, recall 0.684205, f_score 0.557414\n",
      "=== total 93100 match 29698\n",
      ">>> step 93200\n",
      "step 93100-93199, precision 0.470428, recall 0.684592, f_score 0.557655\n",
      "=== total 93200 match 29741\n",
      ">>> step 93300\n",
      "step 93200-93299, precision 0.470940, recall 0.684596, f_score 0.558016\n",
      "=== total 93300 match 29783\n",
      ">>> step 93400\n",
      "step 93300-93399, precision 0.470977, recall 0.683075, f_score 0.557536\n",
      "=== total 93400 match 29787\n",
      ">>> step 93500\n",
      "step 93400-93499, precision 0.470782, recall 0.683152, f_score 0.557425\n",
      "=== total 93500 match 29810\n",
      ">>> step 93600\n",
      "step 93500-93599, precision 0.471485, recall 0.683845, f_score 0.558148\n",
      "=== total 93600 match 29861\n",
      ">>> step 93700\n",
      "step 93600-93699, precision 0.471421, recall 0.683845, f_score 0.558104\n",
      "=== total 93700 match 29865\n",
      ">>> step 93800\n",
      "step 93700-93799, precision 0.471530, recall 0.684147, f_score 0.558280\n",
      "=== total 93800 match 29909\n",
      ">>> step 93900\n",
      "step 93800-93899, precision 0.472349, recall 0.684911, f_score 0.559109\n",
      "=== total 93900 match 29963\n",
      ">>> step 94000\n",
      "step 93900-93999, precision 0.472255, recall 0.684414, f_score 0.558877\n",
      "=== total 94000 match 29969\n",
      ">>> step 94100\n",
      "step 94000-94099, precision 0.472345, recall 0.682789, f_score 0.558397\n",
      "=== total 94100 match 29976\n",
      ">>> step 94200\n",
      "step 94100-94199, precision 0.472350, recall 0.682799, f_score 0.558404\n",
      "=== total 94200 match 30018\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> step 94300\n",
      "step 94200-94299, precision 0.473159, recall 0.683652, f_score 0.559255\n",
      "=== total 94300 match 30085\n",
      ">>> step 94400\n",
      "step 94300-94399, precision 0.473609, recall 0.683384, f_score 0.559479\n",
      "=== total 94400 match 30124\n",
      ">>> step 94500\n",
      "step 94400-94499, precision 0.473620, recall 0.683251, f_score 0.559442\n",
      "=== total 94500 match 30155\n",
      ">>> step 94600\n",
      "step 94500-94599, precision 0.474268, recall 0.683841, f_score 0.560092\n",
      "=== total 94600 match 30196\n",
      ">>> step 94700\n",
      "step 94600-94699, precision 0.474360, recall 0.682593, f_score 0.559737\n",
      "=== total 94700 match 30207\n",
      ">>> step 94800\n",
      "step 94700-94799, precision 0.474329, recall 0.682593, f_score 0.559716\n",
      "=== total 94800 match 30209\n",
      ">>> step 94900\n",
      "step 94800-94899, precision 0.474395, recall 0.682272, f_score 0.559653\n",
      "=== total 94900 match 30228\n",
      ">>> step 95000\n",
      "step 94900-94999, precision 0.474418, recall 0.682095, f_score 0.559610\n",
      "=== total 95000 match 30256\n",
      ">>> step 95100\n",
      "step 95000-95099, precision 0.474463, recall 0.682319, f_score 0.559716\n",
      "=== total 95100 match 30289\n",
      ">>> step 95200\n",
      "step 95100-95199, precision 0.475051, recall 0.682513, f_score 0.560191\n",
      "=== total 95200 match 30342\n",
      ">>> step 95300\n",
      "step 95200-95299, precision 0.475099, recall 0.682179, f_score 0.560112\n",
      "=== total 95300 match 30360\n",
      ">>> step 95400\n",
      "step 95300-95399, precision 0.475146, recall 0.682271, f_score 0.560175\n",
      "=== total 95400 match 30397\n",
      ">>> step 95500\n",
      "step 95400-95499, precision 0.475668, recall 0.682766, f_score 0.560705\n",
      "=== total 95500 match 30433\n",
      ">>> step 95600\n",
      "step 95500-95599, precision 0.475605, recall 0.682766, f_score 0.560662\n",
      "=== total 95600 match 30437\n",
      ">>> step 95700\n",
      "step 95600-95699, precision 0.475778, recall 0.681213, f_score 0.560257\n",
      "=== total 95700 match 30447\n",
      ">>> step 95800\n",
      "step 95700-95799, precision 0.475798, recall 0.680171, f_score 0.559918\n",
      "=== total 95800 match 30452\n",
      ">>> step 95900\n",
      "step 95800-95899, precision 0.475903, recall 0.680416, f_score 0.560074\n",
      "=== total 95900 match 30502\n",
      ">>> step 96000\n",
      "step 95900-95999, precision 0.476621, recall 0.681163, f_score 0.560824\n",
      "=== total 96000 match 30561\n",
      ">>> step 96100\n",
      "step 96000-96099, precision 0.476527, recall 0.681163, f_score 0.560759\n",
      "=== total 96100 match 30567\n",
      ">>> step 96200\n",
      "step 96100-96199, precision 0.476464, recall 0.681313, f_score 0.560766\n",
      "=== total 96200 match 30592\n",
      ">>> step 96300\n",
      "step 96200-96299, precision 0.476223, recall 0.681536, f_score 0.560675\n",
      "=== total 96300 match 30639\n",
      ">>> step 96400\n",
      "step 96300-96399, precision 0.476021, recall 0.681589, f_score 0.560553\n",
      "=== total 96400 match 30673\n",
      ">>> step 96500\n",
      "step 96400-96499, precision 0.475587, recall 0.681589, f_score 0.560252\n",
      "=== total 96500 match 30701\n",
      ">>> step 96600\n",
      "step 96500-96599, precision 0.475092, recall 0.681589, f_score 0.559908\n",
      "=== total 96600 match 30733\n",
      ">>> step 96700\n",
      "step 96600-96699, precision 0.474999, recall 0.681871, f_score 0.559939\n",
      "=== total 96700 match 30779\n",
      ">>> step 96800\n",
      "step 96700-96799, precision 0.474645, recall 0.681839, f_score 0.559681\n",
      "=== total 96800 match 30802\n",
      ">>> step 96900\n",
      "step 96800-96899, precision 0.474003, recall 0.681598, f_score 0.559154\n",
      "=== total 96900 match 30850\n",
      ">>> step 97000\n",
      "step 96900-96999, precision 0.474127, recall 0.681689, f_score 0.559271\n",
      "=== total 97000 match 30882\n",
      ">>> step 97100\n",
      "step 97000-97099, precision 0.473652, recall 0.681689, f_score 0.558940\n",
      "=== total 97100 match 30913\n",
      ">>> step 97200\n",
      "step 97100-97199, precision 0.473349, recall 0.681370, f_score 0.558622\n",
      "=== total 97200 match 30937\n",
      ">>> step 97300\n",
      "step 97200-97299, precision 0.472997, recall 0.681370, f_score 0.558377\n",
      "=== total 97300 match 30960\n",
      ">>> step 97400\n",
      "step 97300-97399, precision 0.472684, recall 0.681562, f_score 0.558224\n",
      "=== total 97400 match 31008\n",
      ">>> step 97500\n",
      "step 97400-97499, precision 0.472201, recall 0.681244, f_score 0.557779\n",
      "=== total 97500 match 31044\n",
      ">>> step 97600\n",
      "step 97500-97599, precision 0.471860, recall 0.681436, f_score 0.557606\n",
      "=== total 97600 match 31094\n",
      ">>> step 97700\n",
      "step 97600-97699, precision 0.471738, recall 0.681436, f_score 0.557521\n",
      "=== total 97700 match 31102\n",
      ">>> step 97800\n",
      "step 97700-97799, precision 0.471238, recall 0.681436, f_score 0.557172\n",
      "=== total 97800 match 31135\n",
      ">>> step 97900\n",
      "step 97800-97899, precision 0.470603, recall 0.681436, f_score 0.556728\n",
      "=== total 97900 match 31177\n",
      ">>> step 98000\n",
      "step 97900-97999, precision 0.470149, recall 0.681350, f_score 0.556381\n",
      "=== total 98000 match 31222\n",
      ">>> step 98100\n",
      "step 98000-98099, precision 0.469999, recall 0.681350, f_score 0.556276\n",
      "=== total 98100 match 31232\n",
      ">>> step 98200\n",
      "step 98100-98199, precision 0.469854, recall 0.681512, f_score 0.556229\n",
      "=== total 98200 match 31265\n",
      ">>> step 98300\n",
      "step 98200-98299, precision 0.469633, recall 0.681881, f_score 0.556196\n",
      "=== total 98300 match 31333\n",
      ">>> step 98400\n",
      "step 98300-98399, precision 0.469465, recall 0.681867, f_score 0.556074\n",
      "=== total 98400 match 31374\n",
      ">>> step 98500\n",
      "step 98400-98499, precision 0.469513, recall 0.681959, f_score 0.556138\n",
      "=== total 98500 match 31407\n",
      ">>> step 98600\n",
      "step 98500-98599, precision 0.469124, recall 0.682148, f_score 0.555928\n",
      "=== total 98600 match 31465\n",
      ">>> step 98700\n",
      "step 98600-98699, precision 0.468864, recall 0.682301, f_score 0.555796\n",
      "=== total 98700 match 31523\n",
      ">>> step 98800\n",
      "step 98700-98799, precision 0.468609, recall 0.682577, f_score 0.555708\n",
      "=== total 98800 match 31585\n",
      ">>> step 98900\n",
      "step 98800-98899, precision 0.468260, recall 0.682738, f_score 0.555516\n",
      "=== total 98900 match 31632\n",
      ">>> step 99000\n",
      "step 98900-98999, precision 0.467949, recall 0.682738, f_score 0.555297\n",
      "=== total 99000 match 31653\n",
      ">>> step 99100\n",
      "step 99000-99099, precision 0.467669, recall 0.682738, f_score 0.555100\n",
      "=== total 99100 match 31672\n",
      ">>> step 99200\n",
      "step 99100-99199, precision 0.467344, recall 0.682738, f_score 0.554871\n",
      "=== total 99200 match 31694\n",
      ">>> step 99300\n",
      "step 99200-99299, precision 0.467324, recall 0.683118, f_score 0.554982\n",
      "=== total 99300 match 31751\n",
      ">>> step 99400\n",
      "step 99300-99399, precision 0.467059, recall 0.683118, f_score 0.554795\n",
      "=== total 99400 match 31769\n",
      ">>> step 99500\n",
      "step 99400-99499, precision 0.467174, recall 0.683307, f_score 0.554939\n",
      "=== total 99500 match 31789\n",
      ">>> step 99600\n",
      "step 99500-99599, precision 0.467142, recall 0.683497, f_score 0.554979\n",
      "=== total 99600 match 31819\n",
      ">>> step 99700\n",
      "step 99600-99699, precision 0.466937, recall 0.683497, f_score 0.554834\n",
      "=== total 99700 match 31833\n",
      ">>> step 99800\n",
      "step 99700-99799, precision 0.467566, recall 0.684107, f_score 0.555479\n",
      "=== total 99800 match 31880\n",
      ">>> step 99900\n",
      "step 99800-99899, precision 0.467350, recall 0.684136, f_score 0.555336\n",
      "=== total 99900 match 31899\n",
      ">>> step 100000\n",
      "step 99900-99999, precision 0.467520, recall 0.684512, f_score 0.555580\n",
      "=== total 100000 match 31943\n",
      ">>> step 100100\n",
      "step 100000-100099, precision 0.467775, recall 0.684974, f_score 0.555913\n",
      "=== total 100100 match 31994\n",
      ">>> step 100200\n",
      "step 100100-100199, precision 0.467264, recall 0.684974, f_score 0.555551\n",
      "=== total 100200 match 32029\n",
      ">>> step 100300\n",
      "step 100200-100299, precision 0.467753, recall 0.685519, f_score 0.556076\n",
      "=== total 100300 match 32081\n",
      ">>> step 100400\n",
      "step 100300-100399, precision 0.467291, recall 0.685648, f_score 0.555792\n",
      "=== total 100400 match 32132\n",
      ">>> step 100500\n",
      "step 100400-100499, precision 0.466951, recall 0.685424, f_score 0.555477\n",
      "=== total 100500 match 32164\n",
      ">>> step 100600\n",
      "step 100500-100599, precision 0.466559, recall 0.685424, f_score 0.555200\n",
      "=== total 100600 match 32191\n",
      ">>> step 100700\n",
      "step 100600-100699, precision 0.466683, recall 0.685751, f_score 0.555396\n",
      "=== total 100700 match 32236\n",
      ">>> step 100800\n",
      "step 100700-100799, precision 0.467032, recall 0.686203, f_score 0.555791\n",
      "=== total 100800 match 32289\n",
      ">>> step 100900\n",
      "step 100800-100899, precision 0.466867, recall 0.686210, f_score 0.555676\n",
      "=== total 100900 match 32339\n",
      ">>> step 101000\n",
      "step 100900-100999, precision 0.466391, recall 0.686210, f_score 0.555339\n",
      "=== total 101000 match 32372\n",
      ">>> step 101100\n",
      "step 101000-101099, precision 0.466218, recall 0.686210, f_score 0.555216\n",
      "=== total 101100 match 32384\n",
      ">>> step 101200\n",
      "step 101100-101199, precision 0.466160, recall 0.686210, f_score 0.555176\n",
      "=== total 101200 match 32388\n",
      ">>> step 101300\n",
      "step 101200-101299, precision 0.466187, recall 0.686015, f_score 0.555131\n",
      "=== total 101300 match 32399\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> step 101400\n",
      "step 101300-101399, precision 0.465939, recall 0.685900, f_score 0.554917\n",
      "=== total 101400 match 32427\n",
      ">>> step 101500\n",
      "step 101400-101499, precision 0.466166, recall 0.686270, f_score 0.555199\n",
      "=== total 101500 match 32467\n",
      ">>> step 101600\n",
      "step 101500-101599, precision 0.466238, recall 0.686668, f_score 0.555381\n",
      "=== total 101600 match 32522\n",
      ">>> step 101700\n",
      "step 101600-101699, precision 0.466749, recall 0.687319, f_score 0.555956\n",
      "=== total 101700 match 32585\n",
      ">>> step 101800\n",
      "step 101700-101799, precision 0.466548, recall 0.687319, f_score 0.555813\n",
      "=== total 101800 match 32599\n",
      ">>> step 101900\n",
      "step 101800-101899, precision 0.467022, recall 0.688024, f_score 0.556380\n",
      "=== total 101900 match 32673\n",
      ">>> step 102000\n",
      "step 101900-101999, precision 0.467622, recall 0.688726, f_score 0.557035\n",
      "=== total 102000 match 32738\n",
      ">>> step 102100\n",
      "step 102000-102099, precision 0.467718, recall 0.689011, f_score 0.557197\n",
      "=== total 102100 match 32789\n",
      ">>> step 102200\n",
      "step 102100-102199, precision 0.467704, recall 0.689256, f_score 0.557267\n",
      "=== total 102200 match 32837\n",
      ">>> step 102300\n",
      "step 102200-102299, precision 0.467764, recall 0.689293, f_score 0.557322\n",
      "=== total 102300 match 32867\n",
      ">>> step 102400\n",
      "step 102300-102399, precision 0.468345, recall 0.689988, f_score 0.557961\n",
      "=== total 102400 match 32933\n",
      ">>> step 102500\n",
      "step 102400-102499, precision 0.468061, recall 0.689988, f_score 0.557759\n",
      "=== total 102500 match 32953\n",
      ">>> step 102600\n",
      "step 102500-102599, precision 0.467890, recall 0.689988, f_score 0.557638\n",
      "=== total 102600 match 32965\n",
      ">>> step 102700\n",
      "step 102600-102699, precision 0.467408, recall 0.689988, f_score 0.557296\n",
      "=== total 102700 match 32999\n",
      ">>> step 102800\n",
      "step 102700-102799, precision 0.467344, recall 0.690304, f_score 0.557353\n",
      "=== total 102800 match 33057\n",
      ">>> step 102900\n",
      "step 102800-102899, precision 0.466822, recall 0.690304, f_score 0.556982\n",
      "=== total 102900 match 33094\n",
      ">>> step 103000\n",
      "step 102900-102999, precision 0.466395, recall 0.690439, f_score 0.556722\n",
      "=== total 103000 match 33150\n",
      ">>> step 103100\n",
      "step 103000-103099, precision 0.465945, recall 0.690439, f_score 0.556401\n",
      "=== total 103100 match 33182\n",
      ">>> step 103200\n",
      "step 103100-103199, precision 0.465999, recall 0.690798, f_score 0.556556\n",
      "=== total 103200 match 33234\n",
      ">>> step 103300\n",
      "step 103200-103299, precision 0.465592, recall 0.690798, f_score 0.556266\n",
      "=== total 103300 match 33263\n",
      ">>> step 103400\n",
      "step 103300-103399, precision 0.465412, recall 0.690991, f_score 0.556199\n",
      "=== total 103400 match 33306\n",
      ">>> step 103500\n",
      "step 103400-103499, precision 0.465641, recall 0.691404, f_score 0.556497\n",
      "=== total 103500 match 33354\n",
      ">>> step 103600\n",
      "step 103500-103599, precision 0.465487, recall 0.691582, f_score 0.556445\n",
      "=== total 103600 match 33393\n",
      ">>> step 103700\n",
      "step 103600-103699, precision 0.465405, recall 0.691870, f_score 0.556479\n",
      "=== total 103700 match 33444\n",
      ">>> step 103800\n",
      "step 103700-103799, precision 0.465225, recall 0.691939, f_score 0.556373\n",
      "=== total 103800 match 33487\n",
      ">>> step 103900\n",
      "step 103800-103899, precision 0.465229, recall 0.692239, f_score 0.556473\n",
      "=== total 103900 match 33534\n",
      ">>> step 104000\n",
      "step 103900-103999, precision 0.465246, recall 0.692621, f_score 0.556608\n",
      "=== total 104000 match 33593\n",
      ">>> step 104100\n",
      "step 104000-104099, precision 0.464958, recall 0.692730, f_score 0.556437\n",
      "=== total 104100 match 33631\n",
      ">>> step 104200\n",
      "step 104100-104199, precision 0.464458, recall 0.692798, f_score 0.556101\n",
      "=== total 104200 match 33678\n",
      ">>> step 104300\n",
      "step 104200-104299, precision 0.464471, recall 0.693152, f_score 0.556224\n",
      "=== total 104300 match 33733\n",
      ">>> step 104400\n",
      "step 104300-104399, precision 0.464605, recall 0.693572, f_score 0.556455\n",
      "=== total 104400 match 33790\n",
      ">>> step 104500\n",
      "step 104400-104499, precision 0.464518, recall 0.693896, f_score 0.556497\n",
      "=== total 104500 match 33848\n",
      ">>> step 104600\n",
      "step 104500-104599, precision 0.464339, recall 0.694011, f_score 0.556406\n",
      "=== total 104600 match 33889\n",
      ">>> step 104700\n",
      "step 104600-104699, precision 0.464011, recall 0.694011, f_score 0.556170\n",
      "=== total 104700 match 33913\n",
      ">>> step 104800\n",
      "step 104700-104799, precision 0.463751, recall 0.694011, f_score 0.555983\n",
      "=== total 104800 match 33932\n",
      ">>> step 104900\n",
      "step 104800-104899, precision 0.463546, recall 0.694011, f_score 0.555836\n",
      "=== total 104900 match 33947\n",
      ">>> step 105000\n",
      "step 104900-104999, precision 0.463223, recall 0.693946, f_score 0.555583\n",
      "=== total 105000 match 33975\n",
      ">>> step 105100\n",
      "step 105000-105099, precision 0.463087, recall 0.694141, f_score 0.555548\n",
      "=== total 105100 match 34026\n",
      ">>> step 105200\n",
      "step 105100-105199, precision 0.463105, recall 0.694424, f_score 0.555651\n",
      "=== total 105200 match 34070\n",
      ">>> step 105300\n",
      "step 105200-105299, precision 0.463222, recall 0.694813, f_score 0.555860\n",
      "=== total 105300 match 34124\n",
      ">>> step 105400\n",
      "step 105300-105399, precision 0.463682, recall 0.695482, f_score 0.556405\n",
      "=== total 105400 match 34198\n",
      ">>> step 105500\n",
      "step 105400-105499, precision 0.463492, recall 0.695482, f_score 0.556269\n",
      "=== total 105500 match 34212\n",
      ">>> step 105600\n",
      "step 105500-105599, precision 0.463896, recall 0.696149, f_score 0.556773\n",
      "=== total 105600 match 34290\n",
      ">>> step 105700\n",
      "step 105600-105699, precision 0.464409, recall 0.696746, f_score 0.557334\n",
      "=== total 105700 match 34349\n",
      ">>> step 105800\n",
      "step 105700-105799, precision 0.464533, recall 0.697103, f_score 0.557537\n",
      "=== total 105800 match 34398\n",
      ">>> step 105900\n",
      "step 105800-105899, precision 0.464537, recall 0.697473, f_score 0.557657\n",
      "=== total 105900 match 34458\n",
      ">>> step 106000\n",
      "step 105900-105999, precision 0.464379, recall 0.697336, f_score 0.557500\n",
      "=== total 106000 match 34502\n",
      ">>> step 106100\n",
      "step 106000-106099, precision 0.464898, recall 0.697994, f_score 0.558085\n",
      "=== total 106100 match 34571\n",
      ">>> step 106200\n",
      "step 106100-106199, precision 0.464603, recall 0.697994, f_score 0.557872\n",
      "=== total 106200 match 34593\n",
      ">>> step 106300\n",
      "step 106200-106299, precision 0.464240, recall 0.697994, f_score 0.557610\n",
      "=== total 106300 match 34620\n",
      ">>> step 106400\n",
      "step 106300-106399, precision 0.463919, recall 0.697994, f_score 0.557378\n",
      "=== total 106400 match 34644\n",
      ">>> step 106500\n",
      "step 106400-106499, precision 0.464066, recall 0.698334, f_score 0.557593\n",
      "=== total 106500 match 34689\n",
      ">>> step 106600\n",
      "step 106500-106599, precision 0.463639, recall 0.698334, f_score 0.557285\n",
      "=== total 106600 match 34721\n",
      ">>> step 106700\n",
      "step 106600-106699, precision 0.463372, recall 0.698334, f_score 0.557092\n",
      "=== total 106700 match 34741\n",
      ">>> step 106800\n",
      "step 106700-106799, precision 0.463347, recall 0.698674, f_score 0.557182\n",
      "=== total 106800 match 34799\n",
      ">>> step 106900\n",
      "step 106800-106899, precision 0.463054, recall 0.698674, f_score 0.556970\n",
      "=== total 106900 match 34821\n",
      ">>> step 107000\n",
      "step 106900-106999, precision 0.462615, recall 0.698674, f_score 0.556653\n",
      "=== total 107000 match 34854\n",
      ">>> step 107100\n",
      "step 107000-107099, precision 0.463065, recall 0.699204, f_score 0.557147\n",
      "=== total 107100 match 34913\n",
      ">>> step 107200\n",
      "step 107100-107199, precision 0.462694, recall 0.699204, f_score 0.556878\n",
      "=== total 107200 match 34941\n",
      ">>> step 107300\n",
      "step 107200-107299, precision 0.462657, recall 0.699542, f_score 0.556958\n",
      "=== total 107300 match 35000\n",
      ">>> step 107400\n",
      "step 107300-107399, precision 0.462765, recall 0.699879, f_score 0.557143\n",
      "=== total 107400 match 35048\n",
      ">>> step 107500\n",
      "step 107400-107499, precision 0.462442, recall 0.699918, f_score 0.556921\n",
      "=== total 107500 match 35079\n",
      ">>> step 107600\n",
      "step 107500-107599, precision 0.463029, recall 0.700525, f_score 0.557539\n",
      "=== total 107600 match 35136\n",
      ">>> step 107700\n",
      "step 107600-107699, precision 0.462634, recall 0.700525, f_score 0.557253\n",
      "=== total 107700 match 35166\n",
      ">>> step 107800\n",
      "step 107700-107799, precision 0.462373, recall 0.700693, f_score 0.557116\n",
      "=== total 107800 match 35214\n",
      ">>> step 107900\n",
      "step 107800-107899, precision 0.462402, recall 0.700950, f_score 0.557219\n",
      "=== total 107900 match 35255\n",
      ">>> step 108000\n",
      "step 107900-107999, precision 0.462310, recall 0.701182, f_score 0.557225\n",
      "=== total 108000 match 35301\n",
      ">>> step 108100\n",
      "step 108000-108099, precision 0.462335, recall 0.701489, f_score 0.557340\n",
      "=== total 108100 match 35351\n",
      ">>> step 108200\n",
      "step 108100-108199, precision 0.462290, recall 0.701771, f_score 0.557397\n",
      "=== total 108200 match 35402\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> step 108300\n",
      "step 108200-108299, precision 0.461990, recall 0.701771, f_score 0.557178\n",
      "=== total 108300 match 35425\n",
      ">>> step 108400\n",
      "step 108300-108399, precision 0.461690, recall 0.701771, f_score 0.556960\n",
      "=== total 108400 match 35448\n",
      ">>> step 108500\n",
      "step 108400-108499, precision 0.461443, recall 0.701771, f_score 0.556780\n",
      "=== total 108500 match 35467\n",
      ">>> step 108600\n",
      "step 108500-108599, precision 0.461422, recall 0.702103, f_score 0.556869\n",
      "=== total 108600 match 35525\n",
      ">>> step 108700\n",
      "step 108600-108699, precision 0.461110, recall 0.702103, f_score 0.556642\n",
      "=== total 108700 match 35549\n",
      ">>> step 108800\n",
      "step 108700-108799, precision 0.461592, recall 0.702697, f_score 0.557180\n",
      "=== total 108800 match 35618\n",
      ">>> step 108900\n",
      "step 108800-108899, precision 0.461767, recall 0.702606, f_score 0.557279\n",
      "=== total 108900 match 35676\n",
      ">>> step 109000\n",
      "step 108900-108999, precision 0.461495, recall 0.702606, f_score 0.557081\n",
      "=== total 109000 match 35697\n",
      ">>> step 109100\n",
      "step 109000-109099, precision 0.461921, recall 0.702898, f_score 0.557483\n",
      "=== total 109100 match 35755\n",
      ">>> step 109200\n",
      "step 109100-109199, precision 0.461805, recall 0.703006, f_score 0.557432\n",
      "=== total 109200 match 35803\n",
      ">>> step 109300\n",
      "step 109200-109299, precision 0.461689, recall 0.702935, f_score 0.557325\n",
      "=== total 109300 match 35851\n",
      ">>> step 109400\n",
      "step 109300-109399, precision 0.462284, recall 0.703564, f_score 0.557957\n",
      "=== total 109400 match 35913\n",
      ">>> step 109500\n",
      "step 109400-109499, precision 0.462274, recall 0.703890, f_score 0.558052\n",
      "=== total 109500 match 35970\n",
      ">>> step 109600\n",
      "step 109500-109599, precision 0.462668, recall 0.704389, f_score 0.558495\n",
      "=== total 109600 match 36041\n",
      ">>> step 109700\n",
      "step 109600-109699, precision 0.462398, recall 0.704389, f_score 0.558299\n",
      "=== total 109700 match 36062\n",
      ">>> step 109800\n",
      "step 109700-109799, precision 0.462142, recall 0.704389, f_score 0.558112\n",
      "=== total 109800 match 36082\n",
      ">>> step 109900\n",
      "step 109800-109899, precision 0.461835, recall 0.704389, f_score 0.557888\n",
      "=== total 109900 match 36106\n",
      ">>> step 110000\n",
      "step 109900-109999, precision 0.461502, recall 0.704389, f_score 0.557646\n",
      "=== total 110000 match 36132\n",
      ">>> step 110100\n",
      "step 110000-110099, precision 0.461513, recall 0.704542, f_score 0.557701\n",
      "=== total 110100 match 36168\n",
      ">>> step 110200\n",
      "step 110100-110199, precision 0.461598, recall 0.704756, f_score 0.557831\n",
      "=== total 110200 match 36209\n",
      ">>> step 110300\n",
      "step 110200-110299, precision 0.461910, recall 0.705167, f_score 0.558187\n",
      "=== total 110300 match 36256\n",
      ">>> step 110400\n",
      "step 110300-110399, precision 0.461632, recall 0.705060, f_score 0.557950\n",
      "=== total 110400 match 36280\n",
      ">>> step 110500\n",
      "step 110400-110499, precision 0.461714, recall 0.705068, f_score 0.558013\n",
      "=== total 110500 match 36306\n",
      ">>> step 110600\n",
      "step 110500-110599, precision 0.461397, recall 0.705068, f_score 0.557781\n",
      "=== total 110600 match 36331\n",
      ">>> step 110700\n",
      "step 110600-110699, precision 0.461498, recall 0.705381, f_score 0.557953\n",
      "=== total 110700 match 36388\n",
      ">>> step 110800\n",
      "step 110700-110799, precision 0.461283, recall 0.705381, f_score 0.557796\n",
      "=== total 110800 match 36405\n",
      ">>> step 110900\n",
      "step 110800-110899, precision 0.461541, recall 0.705776, f_score 0.558108\n",
      "=== total 110900 match 36454\n",
      ">>> step 111000\n",
      "step 110900-110999, precision 0.461351, recall 0.705776, f_score 0.557969\n",
      "=== total 111000 match 36469\n",
      ">>> step 111100\n",
      "step 111000-111099, precision 0.461871, recall 0.706099, f_score 0.558451\n",
      "=== total 111100 match 36521\n",
      ">>> step 111200\n",
      "step 111100-111199, precision 0.461694, recall 0.706099, f_score 0.558321\n",
      "=== total 111200 match 36535\n",
      ">>> step 111300\n",
      "step 111200-111299, precision 0.461837, recall 0.706419, f_score 0.558525\n",
      "=== total 111300 match 36580\n",
      ">>> step 111400\n",
      "step 111300-111399, precision 0.461597, recall 0.706419, f_score 0.558350\n",
      "=== total 111400 match 36599\n",
      ">>> step 111500\n",
      "step 111400-111499, precision 0.461375, recall 0.706737, f_score 0.558287\n",
      "=== total 111500 match 36673\n",
      ">>> step 111600\n",
      "step 111500-111599, precision 0.461819, recall 0.707349, f_score 0.558803\n",
      "=== total 111600 match 36746\n",
      ">>> step 111700\n",
      "step 111600-111699, precision 0.461823, recall 0.707533, f_score 0.558863\n",
      "=== total 111700 match 36789\n",
      ">>> step 111800\n",
      "step 111700-111799, precision 0.461668, recall 0.707582, f_score 0.558765\n",
      "=== total 111800 match 36810\n",
      ">>> step 111900\n",
      "step 111800-111899, precision 0.461543, recall 0.707582, f_score 0.558673\n",
      "=== total 111900 match 36820\n",
      ">>> step 112000\n",
      "step 111900-111999, precision 0.461405, recall 0.707582, f_score 0.558572\n",
      "=== total 112000 match 36831\n",
      ">>> step 112100\n",
      "step 112000-112099, precision 0.461079, recall 0.707582, f_score 0.558334\n",
      "=== total 112100 match 36857\n",
      ">>> step 112200\n",
      "step 112100-112199, precision 0.461167, recall 0.707772, f_score 0.558457\n",
      "=== total 112200 match 36889\n",
      ">>> step 112300\n",
      "step 112200-112299, precision 0.461057, recall 0.707857, f_score 0.558403\n",
      "=== total 112300 match 36913\n",
      ">>> step 112400\n",
      "step 112300-112399, precision 0.460807, recall 0.707857, f_score 0.558220\n",
      "=== total 112400 match 36933\n",
      ">>> step 112500\n",
      "step 112400-112499, precision 0.461324, recall 0.708463, f_score 0.558787\n",
      "=== total 112500 match 37000\n",
      ">>> step 112600\n",
      "step 112500-112599, precision 0.461489, recall 0.708765, f_score 0.559002\n",
      "=== total 112600 match 37041\n",
      ">>> step 112700\n",
      "step 112600-112699, precision 0.461553, recall 0.709067, f_score 0.559143\n",
      "=== total 112700 match 37090\n",
      ">>> step 112800\n",
      "step 112700-112799, precision 0.461605, recall 0.709338, f_score 0.559265\n",
      "=== total 112800 match 37140\n",
      ">>> step 112900\n",
      "step 112800-112899, precision 0.462117, recall 0.709938, f_score 0.559828\n",
      "=== total 112900 match 37207\n",
      ">>> step 113000\n",
      "step 112900-112999, precision 0.462676, recall 0.710495, f_score 0.560411\n",
      "=== total 113000 match 37268\n",
      ">>> step 113100\n",
      "step 113000-113099, precision 0.462827, recall 0.710805, f_score 0.560618\n",
      "=== total 113100 match 37312\n",
      ">>> step 113200\n",
      "step 113100-113199, precision 0.462901, recall 0.711102, f_score 0.560765\n",
      "=== total 113200 match 37360\n",
      ">>> step 113300\n",
      "step 113200-113299, precision 0.462982, recall 0.711275, f_score 0.560878\n",
      "=== total 113300 match 37401\n",
      ">>> step 113400\n",
      "step 113300-113399, precision 0.462772, recall 0.711275, f_score 0.560724\n",
      "=== total 113400 match 37418\n",
      ">>> step 113500\n",
      "step 113400-113499, precision 0.462562, recall 0.711275, f_score 0.560570\n",
      "=== total 113500 match 37435\n",
      ">>> step 113600\n",
      "step 113500-113599, precision 0.462650, recall 0.711358, f_score 0.560660\n",
      "=== total 113600 match 37443\n",
      ">>> step 113700\n",
      "step 113600-113699, precision 0.462638, recall 0.710979, f_score 0.560533\n",
      "=== total 113700 match 37444\n",
      ">>> step 113800\n",
      "step 113700-113799, precision 0.462613, recall 0.710979, f_score 0.560515\n",
      "=== total 113800 match 37446\n",
      ">>> step 113900\n",
      "step 113800-113899, precision 0.462543, recall 0.710332, f_score 0.560263\n",
      "=== total 113900 match 37456\n",
      ">>> step 114000\n",
      "step 113900-113999, precision 0.462628, recall 0.710067, f_score 0.560243\n",
      "=== total 114000 match 37475\n",
      ">>> step 114100\n",
      "step 114000-114099, precision 0.462749, recall 0.709791, f_score 0.560245\n",
      "=== total 114100 match 37489\n",
      ">>> step 114200\n",
      "step 114100-114199, precision 0.462712, recall 0.709414, f_score 0.560101\n",
      "=== total 114200 match 37492\n",
      ">>> step 114300\n",
      "step 114200-114299, precision 0.462687, recall 0.709037, f_score 0.559965\n",
      "=== total 114300 match 37494\n",
      ">>> step 114400\n",
      "step 114300-114399, precision 0.462724, recall 0.708774, f_score 0.559910\n",
      "=== total 114400 match 37517\n",
      ">>> step 114500\n",
      "step 114400-114499, precision 0.462711, recall 0.708774, f_score 0.559901\n",
      "=== total 114500 match 37518\n",
      ">>> step 114600\n",
      "step 114500-114599, precision 0.462842, recall 0.708806, f_score 0.560006\n",
      "=== total 114600 match 37529\n",
      ">>> step 114700\n",
      "step 114600-114699, precision 0.462944, recall 0.708439, f_score 0.559966\n",
      "=== total 114700 match 37538\n",
      ">>> step 114800\n",
      "step 114700-114799, precision 0.462874, recall 0.707943, f_score 0.559760\n",
      "=== total 114800 match 37548\n",
      ">>> step 114900\n",
      "step 114800-114899, precision 0.462965, recall 0.707833, f_score 0.559793\n",
      "=== total 114900 match 37573\n",
      ">>> step 115000\n",
      "step 114900-114999, precision 0.463171, recall 0.707675, f_score 0.559893\n",
      "=== total 115000 match 37606\n",
      ">>> step 115100\n",
      "step 115000-115099, precision 0.463113, recall 0.707038, f_score 0.559652\n",
      "=== total 115100 match 37615\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> step 115200\n",
      "step 115100-115199, precision 0.463113, recall 0.707038, f_score 0.559652\n",
      "=== total 115200 match 37615\n",
      ">>> step 115300\n",
      "step 115200-115299, precision 0.463113, recall 0.706293, f_score 0.559418\n",
      "=== total 115300 match 37615\n",
      ">>> step 115400\n",
      "step 115300-115399, precision 0.463113, recall 0.705920, f_score 0.559301\n",
      "=== total 115400 match 37615\n",
      ">>> step 115500\n",
      "step 115400-115499, precision 0.463298, recall 0.705947, f_score 0.559444\n",
      "=== total 115500 match 37641\n",
      ">>> step 115600\n",
      "step 115500-115599, precision 0.463343, recall 0.705966, f_score 0.559483\n",
      "=== total 115600 match 37646\n",
      ">>> step 115700\n",
      "step 115600-115699, precision 0.463589, recall 0.705621, f_score 0.559554\n",
      "=== total 115700 match 37667\n",
      ">>> step 115800\n",
      "step 115700-115799, precision 0.463604, recall 0.704777, f_score 0.559300\n",
      "=== total 115800 match 37683\n",
      ">>> step 115900\n",
      "step 115800-115899, precision 0.463604, recall 0.704095, f_score 0.559085\n",
      "=== total 115900 match 37683\n",
      ">>> step 116000\n",
      "step 115900-115999, precision 0.463606, recall 0.703087, f_score 0.558768\n",
      "=== total 116000 match 37685\n",
      ">>> step 116100\n",
      "step 116000-116099, precision 0.463575, recall 0.702840, f_score 0.558667\n",
      "=== total 116100 match 37694\n",
      ">>> step 116200\n",
      "step 116100-116199, precision 0.463652, recall 0.701807, f_score 0.558397\n",
      "=== total 116200 match 37705\n",
      ">>> step 116300\n",
      "step 116200-116299, precision 0.463717, recall 0.700777, f_score 0.558117\n",
      "=== total 116300 match 37717\n",
      ">>> step 116400\n",
      "step 116300-116399, precision 0.463692, recall 0.700777, f_score 0.558099\n",
      "=== total 116400 match 37719\n",
      ">>> step 116500\n",
      "step 116400-116499, precision 0.463692, recall 0.700777, f_score 0.558099\n",
      "=== total 116500 match 37719\n",
      ">>> step 116600\n",
      "step 116500-116599, precision 0.463692, recall 0.700048, f_score 0.557868\n",
      "=== total 116600 match 37719\n",
      ">>> step 116700\n",
      "step 116600-116699, precision 0.463692, recall 0.698650, f_score 0.557424\n",
      "=== total 116700 match 37719\n",
      ">>> step 116800\n",
      "step 116700-116799, precision 0.463749, recall 0.697417, f_score 0.557072\n",
      "=== total 116800 match 37723\n",
      ">>> step 116900\n",
      "step 116800-116899, precision 0.463749, recall 0.696029, f_score 0.556629\n",
      "=== total 116900 match 37723\n",
      ">>> step 117000\n",
      "step 116900-116999, precision 0.463777, recall 0.694727, f_score 0.556232\n",
      "=== total 117000 match 37725\n",
      ">>> step 117100\n",
      "step 117000-117099, precision 0.463779, recall 0.693390, f_score 0.555804\n",
      "=== total 117100 match 37727\n",
      ">>> step 117200\n",
      "step 117100-117199, precision 0.463852, recall 0.692420, f_score 0.555545\n",
      "=== total 117200 match 37734\n",
      ">>> step 117300\n",
      "step 117200-117299, precision 0.463911, recall 0.691742, f_score 0.555369\n",
      "=== total 117300 match 37740\n",
      ">>> step 117400\n",
      "step 117300-117399, precision 0.463911, recall 0.691742, f_score 0.555369\n",
      "=== total 117400 match 37740\n",
      ">>> step 117500\n",
      "step 117400-117499, precision 0.463911, recall 0.691387, f_score 0.555254\n",
      "=== total 117500 match 37740\n",
      ">>> step 117600\n",
      "step 117500-117599, precision 0.463927, recall 0.690893, f_score 0.555106\n",
      "=== total 117600 match 37743\n",
      ">>> step 117700\n",
      "step 117600-117699, precision 0.463943, recall 0.690672, f_score 0.555047\n",
      "=== total 117700 match 37746\n",
      ">>> step 117800\n",
      "step 117700-117799, precision 0.463933, recall 0.689352, f_score 0.554613\n",
      "=== total 117800 match 37749\n",
      ">>> step 117900\n",
      "step 117800-117899, precision 0.463896, recall 0.688648, f_score 0.554358\n",
      "=== total 117900 match 37752\n",
      ">>> step 118000\n",
      "step 117900-117999, precision 0.463896, recall 0.687944, f_score 0.554130\n",
      "=== total 118000 match 37752\n",
      ">>> step 118100\n",
      "step 118000-118099, precision 0.463910, recall 0.686635, f_score 0.553715\n",
      "=== total 118100 match 37753\n",
      ">>> step 118200\n",
      "step 118100-118199, precision 0.463873, recall 0.685936, f_score 0.553461\n",
      "=== total 118200 match 37756\n",
      ">>> step 118300\n",
      "step 118200-118299, precision 0.463836, recall 0.685238, f_score 0.553208\n",
      "=== total 118300 match 37759\n",
      ">>> step 118400\n",
      "step 118300-118399, precision 0.463824, recall 0.684274, f_score 0.552885\n",
      "=== total 118400 match 37760\n",
      ">>> step 118500\n",
      "step 118400-118499, precision 0.463812, recall 0.683847, f_score 0.552736\n",
      "=== total 118500 match 37761\n",
      ">>> step 118600\n",
      "step 118500-118599, precision 0.463812, recall 0.683847, f_score 0.552736\n",
      "=== total 118600 match 37761\n",
      ">>> step 118700\n",
      "step 118600-118699, precision 0.463812, recall 0.683153, f_score 0.552510\n",
      "=== total 118700 match 37761\n",
      ">>> step 118800\n",
      "step 118700-118799, precision 0.463834, recall 0.682018, f_score 0.552153\n",
      "=== total 118800 match 37770\n",
      ">>> step 118900\n",
      "step 118800-118899, precision 0.463945, recall 0.681198, f_score 0.551963\n",
      "=== total 118900 match 37789\n",
      ">>> step 119000\n",
      "step 118900-118999, precision 0.464349, recall 0.681118, f_score 0.552223\n",
      "=== total 119000 match 37825\n",
      ">>> step 119100\n",
      "step 119000-119099, precision 0.464718, recall 0.681078, f_score 0.552470\n",
      "=== total 119100 match 37866\n",
      ">>> step 119200\n",
      "step 119100-119199, precision 0.464718, recall 0.679920, f_score 0.552089\n",
      "=== total 119200 match 37866\n",
      ">>> step 119300\n",
      "step 119200-119299, precision 0.464718, recall 0.678582, f_score 0.551647\n",
      "=== total 119300 match 37866\n",
      ">>> step 119400\n",
      "step 119300-119399, precision 0.464720, recall 0.677967, f_score 0.551445\n",
      "=== total 119400 match 37868\n",
      ">>> step 119500\n",
      "step 119400-119499, precision 0.464720, recall 0.677967, f_score 0.551445\n",
      "=== total 119500 match 37868\n",
      ">>> step 119600\n",
      "step 119500-119599, precision 0.464720, recall 0.677289, f_score 0.551221\n",
      "=== total 119600 match 37868\n",
      ">>> step 119700\n",
      "step 119600-119699, precision 0.464780, recall 0.676999, f_score 0.551167\n",
      "=== total 119700 match 37876\n",
      ">>> step 119800\n",
      "step 119700-119799, precision 0.464882, recall 0.677032, f_score 0.551250\n",
      "=== total 119800 match 37887\n",
      ">>> step 119900\n",
      "step 119800-119899, precision 0.464995, recall 0.676041, f_score 0.551001\n",
      "=== total 119900 match 37895\n",
      ">>> step 120000\n",
      "step 119900-119999, precision 0.465124, recall 0.675750, f_score 0.550995\n",
      "=== total 120000 match 37906\n",
      ">>> step 120100\n",
      "step 120000-120099, precision 0.465124, recall 0.675078, f_score 0.550771\n",
      "=== total 120100 match 37906\n",
      ">>> step 120200\n",
      "step 120100-120199, precision 0.465087, recall 0.673788, f_score 0.550315\n",
      "=== total 120200 match 37909\n",
      ">>> step 120300\n",
      "step 120200-120299, precision 0.465087, recall 0.673119, f_score 0.550092\n",
      "=== total 120300 match 37909\n",
      ">>> step 120400\n",
      "step 120300-120399, precision 0.465370, recall 0.673214, f_score 0.550321\n",
      "=== total 120400 match 37929\n",
      ">>> step 120500\n",
      "step 120400-120499, precision 0.465345, recall 0.672317, f_score 0.550004\n",
      "=== total 120500 match 37931\n",
      ">>> step 120600\n",
      "step 120500-120599, precision 0.465345, recall 0.671728, f_score 0.549807\n",
      "=== total 120600 match 37931\n",
      ">>> step 120700\n",
      "step 120600-120699, precision 0.465345, recall 0.671728, f_score 0.549807\n",
      "=== total 120700 match 37931\n",
      ">>> step 120800\n",
      "step 120700-120799, precision 0.465345, recall 0.671064, f_score 0.549584\n",
      "=== total 120800 match 37931\n",
      ">>> step 120900\n",
      "step 120800-120899, precision 0.465271, recall 0.669791, f_score 0.549106\n",
      "=== total 120900 match 37937\n",
      ">>> step 121000\n",
      "step 120900-120999, precision 0.465247, recall 0.668523, f_score 0.548662\n",
      "=== total 121000 match 37939\n",
      ">>> step 121100\n",
      "step 121000-121099, precision 0.465186, recall 0.667385, f_score 0.548236\n",
      "=== total 121100 match 37944\n",
      ">>> step 121200\n",
      "step 121100-121199, precision 0.465100, recall 0.666000, f_score 0.547708\n",
      "=== total 121200 match 37951\n",
      ">>> step 121300\n",
      "step 121200-121299, precision 0.465051, recall 0.664746, f_score 0.547250\n",
      "=== total 121300 match 37955\n",
      ">>> step 121400\n",
      "step 121300-121399, precision 0.465042, recall 0.663721, f_score 0.546897\n",
      "=== total 121400 match 37960\n",
      ">>> step 121500\n",
      "step 121400-121499, precision 0.464993, recall 0.662924, f_score 0.546592\n",
      "=== total 121500 match 37964\n",
      ">>> step 121600\n",
      "step 121500-121599, precision 0.464956, recall 0.662924, f_score 0.546566\n",
      "=== total 121600 match 37967\n",
      ">>> step 121700\n",
      "step 121600-121699, precision 0.465012, recall 0.662652, f_score 0.546512\n",
      "=== total 121700 match 37984\n",
      ">>> step 121800\n",
      "step 121700-121799, precision 0.464965, recall 0.662629, f_score 0.546472\n",
      "=== total 121800 match 38005\n",
      ">>> step 121900\n",
      "step 121800-121899, precision 0.464894, recall 0.662021, f_score 0.546216\n",
      "=== total 121900 match 38013\n",
      ">>> step 122000\n",
      "step 121900-121999, precision 0.464832, recall 0.661724, f_score 0.546073\n",
      "=== total 122000 match 38018\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> step 122100\n",
      "step 122000-122099, precision 0.464919, recall 0.661194, f_score 0.545951\n",
      "=== total 122100 match 38026\n",
      ">>> step 122200\n",
      "step 122100-122199, precision 0.465149, recall 0.661204, f_score 0.546113\n",
      "=== total 122200 match 38076\n",
      ">>> step 122300\n",
      "step 122200-122299, precision 0.465213, recall 0.661432, f_score 0.546236\n",
      "=== total 122300 match 38118\n",
      ">>> step 122400\n",
      "step 122300-122399, precision 0.465552, recall 0.661624, f_score 0.546535\n",
      "=== total 122400 match 38144\n",
      ">>> step 122500\n",
      "step 122400-122499, precision 0.465515, recall 0.661304, f_score 0.546400\n",
      "=== total 122500 match 38147\n",
      ">>> step 122600\n",
      "step 122500-122599, precision 0.465595, recall 0.661105, f_score 0.546387\n",
      "=== total 122600 match 38207\n",
      ">>> step 122700\n",
      "step 122600-122699, precision 0.465934, recall 0.661063, f_score 0.546607\n",
      "=== total 122700 match 38235\n",
      ">>> step 122800\n",
      "step 122700-122799, precision 0.465934, recall 0.661063, f_score 0.546607\n",
      "=== total 122800 match 38235\n",
      ">>> step 122900\n",
      "step 122800-122899, precision 0.465825, recall 0.661063, f_score 0.546531\n",
      "=== total 122900 match 38244\n",
      ">>> step 123000\n",
      "step 122900-122999, precision 0.465714, recall 0.660649, f_score 0.546313\n",
      "=== total 123000 match 38266\n",
      ">>> step 123100\n",
      "step 123000-123099, precision 0.465875, recall 0.660759, f_score 0.546462\n",
      "=== total 123100 match 38330\n",
      ">>> step 123200\n",
      "step 123100-123199, precision 0.465870, recall 0.659686, f_score 0.546091\n",
      "=== total 123200 match 38339\n",
      ">>> step 123300\n",
      "step 123200-123299, precision 0.466075, recall 0.659687, f_score 0.546232\n",
      "=== total 123300 match 38393\n",
      ">>> step 123400\n",
      "step 123300-123399, precision 0.466073, recall 0.659209, f_score 0.546067\n",
      "=== total 123400 match 38436\n",
      ">>> step 123500\n",
      "step 123400-123499, precision 0.466412, recall 0.659761, f_score 0.546489\n",
      "=== total 123500 match 38511\n",
      ">>> step 123600\n",
      "step 123500-123599, precision 0.466463, recall 0.659334, f_score 0.546377\n",
      "=== total 123600 match 38554\n",
      ">>> step 123700\n",
      "step 123600-123699, precision 0.466660, recall 0.659317, f_score 0.546506\n",
      "=== total 123700 match 38572\n",
      ">>> step 123800\n",
      "step 123700-123799, precision 0.466587, recall 0.659317, f_score 0.546456\n",
      "=== total 123800 match 38578\n",
      ">>> step 123900\n",
      "step 123800-123899, precision 0.466406, recall 0.659317, f_score 0.546332\n",
      "=== total 123900 match 38593\n",
      ">>> step 124000\n",
      "step 123900-123999, precision 0.466106, recall 0.659296, f_score 0.546119\n",
      "=== total 124000 match 38635\n",
      ">>> step 124100\n",
      "step 124000-124099, precision 0.465708, recall 0.659296, f_score 0.545846\n",
      "=== total 124100 match 38668\n",
      ">>> step 124200\n",
      "step 124100-124199, precision 0.465468, recall 0.659383, f_score 0.545711\n",
      "=== total 124200 match 38703\n",
      ">>> step 124300\n",
      "step 124200-124299, precision 0.465425, recall 0.659476, f_score 0.545713\n",
      "=== total 124300 match 38756\n",
      ">>> step 124400\n",
      "step 124300-124399, precision 0.464897, recall 0.659476, f_score 0.545350\n",
      "=== total 124400 match 38800\n",
      ">>> step 124500\n",
      "step 124400-124499, precision 0.464789, recall 0.659476, f_score 0.545276\n",
      "=== total 124500 match 38809\n",
      ">>> step 124600\n",
      "step 124500-124599, precision 0.464801, recall 0.659638, f_score 0.545339\n",
      "=== total 124600 match 38836\n",
      ">>> step 124700\n",
      "step 124600-124699, precision 0.464713, recall 0.659727, f_score 0.545309\n",
      "=== total 124700 match 38867\n",
      ">>> step 124800\n",
      "step 124700-124799, precision 0.464414, recall 0.659727, f_score 0.545103\n",
      "=== total 124800 match 38892\n",
      ">>> step 124900\n",
      "step 124800-124899, precision 0.464064, recall 0.659560, f_score 0.544805\n",
      "=== total 124900 match 38930\n",
      ">>> step 125000\n",
      "step 124900-124999, precision 0.464011, recall 0.659612, f_score 0.544786\n",
      "=== total 125000 match 38956\n",
      ">>> step 125100\n",
      "step 125000-125099, precision 0.463677, recall 0.659612, f_score 0.544556\n",
      "=== total 125100 match 38984\n",
      ">>> step 125200\n",
      "step 125100-125199, precision 0.463321, recall 0.659612, f_score 0.544310\n",
      "=== total 125200 match 39014\n",
      ">>> step 125300\n",
      "step 125200-125299, precision 0.463072, recall 0.659773, f_score 0.544194\n",
      "=== total 125300 match 39063\n",
      ">>> step 125400\n",
      "step 125300-125399, precision 0.462990, recall 0.660009, f_score 0.544217\n",
      "=== total 125400 match 39111\n",
      ">>> step 125500\n",
      "step 125400-125499, precision 0.462801, recall 0.660009, f_score 0.544086\n",
      "=== total 125500 match 39127\n",
      ">>> step 125600\n",
      "step 125500-125599, precision 0.462719, recall 0.660244, f_score 0.544109\n",
      "=== total 125600 match 39175\n",
      ">>> step 125700\n",
      "step 125600-125699, precision 0.462696, recall 0.660479, f_score 0.544173\n",
      "=== total 125700 match 39218\n",
      ">>> step 125800\n",
      "step 125700-125799, precision 0.462837, recall 0.660788, f_score 0.544376\n",
      "=== total 125800 match 39260\n",
      ">>> step 125900\n",
      "step 125800-125899, precision 0.462472, recall 0.660788, f_score 0.544123\n",
      "=== total 125900 match 39291\n",
      ">>> step 126000\n",
      "step 125900-125999, precision 0.462155, recall 0.660788, f_score 0.543903\n",
      "=== total 126000 match 39318\n",
      ">>> step 126100\n",
      "step 126000-126099, precision 0.461896, recall 0.660788, f_score 0.543724\n",
      "=== total 126100 match 39340\n",
      ">>> step 126200\n",
      "step 126100-126199, precision 0.461603, recall 0.660788, f_score 0.543521\n",
      "=== total 126200 match 39365\n",
      ">>> step 126300\n",
      "step 126200-126299, precision 0.461322, recall 0.660788, f_score 0.543326\n",
      "=== total 126300 match 39389\n",
      ">>> step 126400\n",
      "step 126300-126399, precision 0.461052, recall 0.660475, f_score 0.543034\n",
      "=== total 126400 match 39412\n",
      ">>> step 126500\n",
      "step 126400-126499, precision 0.460971, recall 0.660475, f_score 0.542977\n",
      "=== total 126500 match 39419\n",
      ">>> step 126600\n",
      "step 126500-126599, precision 0.460626, recall 0.660464, f_score 0.542734\n",
      "=== total 126600 match 39455\n",
      ">>> step 126700\n",
      "step 126600-126699, precision 0.460512, recall 0.660409, f_score 0.542636\n",
      "=== total 126700 match 39493\n",
      ">>> step 126800\n",
      "step 126700-126799, precision 0.460209, recall 0.660409, f_score 0.542426\n",
      "=== total 126800 match 39519\n",
      ">>> step 126900\n",
      "step 126800-126899, precision 0.459977, recall 0.660509, f_score 0.542299\n",
      "=== total 126900 match 39565\n",
      ">>> step 127000\n",
      "step 126900-126999, precision 0.459771, recall 0.660571, f_score 0.542176\n",
      "=== total 127000 match 39598\n",
      ">>> step 127100\n",
      "step 127000-127099, precision 0.459448, recall 0.660464, f_score 0.541915\n",
      "=== total 127100 match 39628\n",
      ">>> step 127200\n",
      "step 127100-127199, precision 0.459251, recall 0.659985, f_score 0.541617\n",
      "=== total 127200 match 39645\n",
      ">>> step 127300\n",
      "step 127200-127299, precision 0.459257, recall 0.660231, f_score 0.541704\n",
      "=== total 127300 match 39688\n",
      ">>> step 127400\n",
      "step 127300-127399, precision 0.459124, recall 0.660403, f_score 0.541670\n",
      "=== total 127400 match 39730\n",
      ">>> step 127500\n",
      "step 127400-127499, precision 0.458824, recall 0.660403, f_score 0.541461\n",
      "=== total 127500 match 39756\n",
      ">>> step 127600\n",
      "step 127500-127599, precision 0.458766, recall 0.660403, f_score 0.541421\n",
      "=== total 127600 match 39761\n",
      ">>> step 127700\n",
      "step 127600-127699, precision 0.458589, recall 0.660394, f_score 0.541294\n",
      "=== total 127700 match 39796\n",
      ">>> step 127800\n",
      "step 127700-127799, precision 0.458439, recall 0.660394, f_score 0.541190\n",
      "=== total 127800 match 39809\n",
      ">>> step 127900\n",
      "step 127800-127899, precision 0.458685, recall 0.660752, f_score 0.541481\n",
      "=== total 127900 match 39864\n",
      ">>> step 128000\n",
      "step 127900-127999, precision 0.458620, recall 0.660842, f_score 0.541466\n",
      "=== total 128000 match 39911\n",
      ">>> step 128100\n",
      "step 128000-128099, precision 0.458404, recall 0.660941, f_score 0.541349\n",
      "=== total 128100 match 39956\n",
      ">>> step 128200\n",
      "step 128100-128199, precision 0.458267, recall 0.660941, f_score 0.541253\n",
      "=== total 128200 match 39968\n",
      ">>> step 128300\n",
      "step 128200-128299, precision 0.457957, recall 0.660941, f_score 0.541037\n",
      "=== total 128300 match 39995\n",
      ">>> step 128400\n",
      "step 128300-128399, precision 0.457705, recall 0.660941, f_score 0.540861\n",
      "=== total 128400 match 40017\n",
      ">>> step 128500\n",
      "step 128400-128499, precision 0.457523, recall 0.660941, f_score 0.540734\n",
      "=== total 128500 match 40033\n",
      ">>> step 128600\n",
      "step 128500-128599, precision 0.457283, recall 0.660941, f_score 0.540566\n",
      "=== total 128600 match 40054\n",
      ">>> step 128700\n",
      "step 128600-128699, precision 0.457533, recall 0.661332, f_score 0.540872\n",
      "=== total 128700 match 40102\n",
      ">>> step 128800\n",
      "step 128700-128799, precision 0.457351, recall 0.661332, f_score 0.540744\n",
      "=== total 128800 match 40118\n",
      ">>> step 128900\n",
      "step 128800-128899, precision 0.457889, recall 0.661941, f_score 0.541325\n",
      "=== total 128900 match 40180\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> step 129000\n",
      "step 128900-128999, precision 0.457989, recall 0.662257, f_score 0.541500\n",
      "=== total 129000 match 40228\n",
      ">>> step 129100\n",
      "step 129000-129099, precision 0.457796, recall 0.662257, f_score 0.541365\n",
      "=== total 129100 match 40245\n",
      ">>> step 129200\n",
      "step 129100-129199, precision 0.457684, recall 0.661675, f_score 0.541092\n",
      "=== total 129200 match 40257\n",
      ">>> step 129300\n",
      "step 129200-129299, precision 0.457588, recall 0.661345, f_score 0.540914\n",
      "=== total 129300 match 40283\n",
      ">>> step 129400\n",
      "step 129300-129399, precision 0.457247, recall 0.661345, f_score 0.540676\n",
      "=== total 129400 match 40313\n",
      ">>> step 129500\n",
      "step 129400-129499, precision 0.457325, recall 0.661660, f_score 0.540836\n",
      "=== total 129500 match 40363\n",
      ">>> step 129600\n",
      "step 129500-129599, precision 0.457331, recall 0.661152, f_score 0.540670\n",
      "=== total 129600 match 40369\n",
      ">>> step 129700\n",
      "step 129600-129699, precision 0.457349, recall 0.661216, f_score 0.540704\n",
      "=== total 129700 match 40409\n",
      ">>> step 129800\n",
      "step 129700-129799, precision 0.457111, recall 0.661216, f_score 0.540538\n",
      "=== total 129800 match 40430\n",
      ">>> step 129900\n",
      "step 129800-129899, precision 0.456716, recall 0.661216, f_score 0.540262\n",
      "=== total 129900 match 40465\n",
      ">>> step 130000\n",
      "step 129900-129999, precision 0.456751, recall 0.660816, f_score 0.540153\n",
      "=== total 130000 match 40475\n",
      ">>> step 130100\n",
      "step 130000-130099, precision 0.457308, recall 0.661422, f_score 0.540745\n",
      "=== total 130100 match 40535\n",
      ">>> step 130200\n",
      "step 130100-130199, precision 0.457263, recall 0.661422, f_score 0.540713\n",
      "=== total 130200 match 40539\n",
      ">>> step 130300\n",
      "step 130200-130299, precision 0.457457, recall 0.661273, f_score 0.540799\n",
      "=== total 130300 match 40559\n",
      ">>> step 130400\n",
      "step 130300-130399, precision 0.457981, recall 0.661733, f_score 0.541319\n",
      "=== total 130400 match 40613\n",
      ">>> step 130500\n",
      "step 130400-130499, precision 0.457970, recall 0.661122, f_score 0.541107\n",
      "=== total 130500 match 40614\n",
      ">>> step 130600\n",
      "step 130500-130599, precision 0.458504, recall 0.661675, f_score 0.541665\n",
      "=== total 130600 match 40667\n",
      ">>> step 130700\n",
      "step 130600-130699, precision 0.458299, recall 0.661723, f_score 0.541537\n",
      "=== total 130700 match 40694\n",
      ">>> step 130800\n",
      "step 130700-130799, precision 0.458073, recall 0.661723, f_score 0.541380\n",
      "=== total 130800 match 40714\n",
      ">>> step 130900\n",
      "step 130800-130899, precision 0.457759, recall 0.661723, f_score 0.541160\n",
      "=== total 130900 match 40742\n",
      ">>> step 131000\n",
      "step 130900-130999, precision 0.457534, recall 0.661723, f_score 0.541003\n",
      "=== total 131000 match 40762\n",
      ">>> step 131100\n",
      "step 131000-131099, precision 0.457220, recall 0.661723, f_score 0.540783\n",
      "=== total 131100 match 40790\n",
      ">>> step 131200\n",
      "step 131100-131199, precision 0.457250, recall 0.661999, f_score 0.540897\n",
      "=== total 131200 match 40842\n",
      ">>> step 131300\n",
      "step 131200-131299, precision 0.456892, recall 0.661999, f_score 0.540646\n",
      "=== total 131300 match 40874\n",
      ">>> step 131400\n",
      "step 131300-131399, precision 0.457157, recall 0.662419, f_score 0.540972\n",
      "=== total 131400 match 40940\n",
      ">>> step 131500\n",
      "step 131400-131499, precision 0.457178, recall 0.662730, f_score 0.541090\n",
      "=== total 131500 match 40995\n",
      ">>> step 131600\n",
      "step 131500-131599, precision 0.457104, recall 0.662496, f_score 0.540960\n",
      "=== total 131600 match 41006\n",
      ">>> step 131700\n",
      "step 131600-131699, precision 0.456792, recall 0.662496, f_score 0.540742\n",
      "=== total 131700 match 41034\n",
      ">>> step 131800\n",
      "step 131700-131799, precision 0.456609, recall 0.661994, f_score 0.540446\n",
      "=== total 131800 match 41057\n",
      ">>> step 131900\n",
      "step 131800-131899, precision 0.456242, recall 0.661866, f_score 0.540147\n",
      "=== total 131900 match 41101\n",
      ">>> step 132000\n",
      "step 131900-131999, precision 0.455987, recall 0.661866, f_score 0.539968\n",
      "=== total 132000 match 41124\n",
      ">>> step 132100\n",
      "step 132000-132099, precision 0.456009, recall 0.662176, f_score 0.540087\n",
      "=== total 132100 match 41179\n",
      ">>> step 132200\n",
      "step 132100-132199, precision 0.456101, recall 0.662169, f_score 0.540148\n",
      "=== total 132200 match 41208\n",
      ">>> step 132300\n",
      "step 132200-132299, precision 0.456006, recall 0.662196, f_score 0.540091\n",
      "=== total 132300 match 41256\n",
      ">>> step 132400\n",
      "step 132300-132399, precision 0.455719, recall 0.662196, f_score 0.539890\n",
      "=== total 132400 match 41282\n",
      ">>> step 132500\n",
      "step 132400-132499, precision 0.455521, recall 0.662196, f_score 0.539750\n",
      "=== total 132500 match 41300\n",
      ">>> step 132600\n",
      "step 132500-132599, precision 0.455539, recall 0.662259, f_score 0.539784\n",
      "=== total 132600 match 41340\n",
      ">>> step 132700\n",
      "step 132600-132699, precision 0.455909, recall 0.662641, f_score 0.540171\n",
      "=== total 132700 match 41403\n",
      ">>> step 132800\n",
      "step 132700-132799, precision 0.455799, recall 0.662641, f_score 0.540094\n",
      "=== total 132800 match 41413\n",
      ">>> step 132900\n",
      "step 132800-132899, precision 0.455977, recall 0.662741, f_score 0.540252\n",
      "=== total 132900 match 41467\n",
      ">>> step 133000\n",
      "step 132900-132999, precision 0.456246, recall 0.662946, f_score 0.540509\n",
      "=== total 133000 match 41528\n",
      ">>> step 133100\n",
      "step 133000-133099, precision 0.456162, recall 0.662763, f_score 0.540389\n",
      "=== total 133100 match 41562\n",
      ">>> step 133200\n",
      "step 133100-133199, precision 0.456540, recall 0.663060, f_score 0.540753\n",
      "=== total 133200 match 41613\n",
      ">>> step 133300\n",
      "step 133200-133299, precision 0.456373, recall 0.663107, f_score 0.540651\n",
      "=== total 133300 match 41637\n",
      ">>> step 133400\n",
      "step 133300-133399, precision 0.456187, recall 0.663107, f_score 0.540521\n",
      "=== total 133400 match 41654\n",
      ">>> step 133500\n",
      "step 133400-133499, precision 0.455924, recall 0.663107, f_score 0.540336\n",
      "=== total 133500 match 41678\n",
      ">>> step 133600\n",
      "step 133500-133599, precision 0.455694, recall 0.663107, f_score 0.540175\n",
      "=== total 133600 match 41699\n",
      ">>> step 133700\n",
      "step 133600-133699, precision 0.455356, recall 0.663107, f_score 0.539937\n",
      "=== total 133700 match 41730\n",
      ">>> step 133800\n",
      "step 133700-133799, precision 0.455566, recall 0.663483, f_score 0.540209\n",
      "=== total 133800 match 41781\n",
      ">>> step 133900\n",
      "step 133800-133899, precision 0.455571, recall 0.663753, f_score 0.540302\n",
      "=== total 133900 match 41831\n",
      ">>> step 134000\n",
      "step 133900-133999, precision 0.455587, recall 0.664080, f_score 0.540422\n",
      "=== total 134000 match 41891\n",
      ">>> step 134100\n",
      "step 134000-134099, precision 0.455684, recall 0.664372, f_score 0.540586\n",
      "=== total 134100 match 41937\n",
      ">>> step 134200\n",
      "step 134100-134199, precision 0.455782, recall 0.664675, f_score 0.540756\n",
      "=== total 134200 match 41985\n",
      ">>> step 134300\n",
      "step 134200-134299, precision 0.455576, recall 0.664675, f_score 0.540611\n",
      "=== total 134300 match 42004\n",
      ">>> step 134400\n",
      "step 134300-134399, precision 0.455525, recall 0.664665, f_score 0.540572\n",
      "=== total 134400 match 42046\n",
      ">>> step 134500\n",
      "step 134400-134499, precision 0.455168, recall 0.664665, f_score 0.540320\n",
      "=== total 134500 match 42079\n",
      ">>> step 134600\n",
      "step 134500-134599, precision 0.455355, recall 0.664933, f_score 0.540541\n",
      "=== total 134600 match 42121\n",
      ">>> step 134700\n",
      "step 134600-134699, precision 0.455444, recall 0.665201, f_score 0.540692\n",
      "=== total 134700 match 42172\n",
      ">>> step 134800\n",
      "step 134700-134799, precision 0.455422, recall 0.665432, f_score 0.540753\n",
      "=== total 134800 match 42218\n",
      ">>> step 134900\n",
      "step 134800-134899, precision 0.455228, recall 0.665432, f_score 0.540616\n",
      "=== total 134900 match 42236\n",
      ">>> step 135000\n",
      "step 134900-134999, precision 0.454915, recall 0.665432, f_score 0.540395\n",
      "=== total 135000 match 42265\n",
      ">>> step 135100\n",
      "step 135000-135099, precision 0.455080, recall 0.665802, f_score 0.540634\n",
      "=== total 135100 match 42320\n",
      ">>> step 135200\n",
      "step 135100-135199, precision 0.455530, recall 0.666379, f_score 0.541141\n",
      "=== total 135200 match 42388\n",
      ">>> step 135300\n",
      "step 135200-135299, precision 0.455266, recall 0.666402, f_score 0.540962\n",
      "=== total 135300 match 42417\n",
      ">>> step 135400\n",
      "step 135300-135399, precision 0.455513, recall 0.666816, f_score 0.541273\n",
      "=== total 135400 match 42473\n",
      ">>> step 135500\n",
      "step 135400-135499, precision 0.455988, recall 0.667320, f_score 0.541775\n",
      "=== total 135500 match 42534\n",
      ">>> step 135600\n",
      "step 135500-135599, precision 0.456245, recall 0.667618, f_score 0.542055\n",
      "=== total 135600 match 42567\n",
      ">>> step 135700\n",
      "step 135600-135699, precision 0.456498, recall 0.668188, f_score 0.542421\n",
      "=== total 135700 match 42653\n",
      ">>> step 135800\n",
      "step 135700-135799, precision 0.456241, recall 0.668188, f_score 0.542239\n",
      "=== total 135800 match 42677\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> step 135900\n",
      "step 135800-135899, precision 0.455942, recall 0.668188, f_score 0.542028\n",
      "=== total 135900 match 42705\n",
      ">>> step 136000\n",
      "step 135900-135999, precision 0.455782, recall 0.668188, f_score 0.541915\n",
      "=== total 136000 match 42720\n",
      ">>> step 136100\n",
      "step 136000-136099, precision 0.455547, recall 0.668188, f_score 0.541749\n",
      "=== total 136100 match 42742\n",
      ">>> step 136200\n",
      "step 136100-136199, precision 0.455442, recall 0.668347, f_score 0.541727\n",
      "=== total 136200 match 42787\n",
      ">>> step 136300\n",
      "step 136200-136299, precision 0.455354, recall 0.668450, f_score 0.541699\n",
      "=== total 136300 match 42815\n",
      ">>> step 136400\n",
      "step 136300-136399, precision 0.455121, recall 0.668450, f_score 0.541533\n",
      "=== total 136400 match 42837\n",
      ">>> step 136500\n",
      "step 136400-136499, precision 0.455542, recall 0.668949, f_score 0.541995\n",
      "=== total 136500 match 42894\n",
      ">>> step 136600\n",
      "step 136500-136599, precision 0.455468, recall 0.669072, f_score 0.541984\n",
      "=== total 136600 match 42947\n",
      ">>> step 136700\n",
      "step 136600-136699, precision 0.455281, recall 0.669185, f_score 0.541888\n",
      "=== total 136700 match 42991\n",
      ">>> step 136800\n",
      "step 136700-136799, precision 0.455086, recall 0.669332, f_score 0.541798\n",
      "=== total 136800 match 43038\n",
      ">>> step 136900\n",
      "step 136800-136899, precision 0.454944, recall 0.669479, f_score 0.541746\n",
      "=== total 136900 match 43080\n",
      ">>> step 137000\n",
      "step 136900-136999, precision 0.454739, recall 0.669284, f_score 0.541537\n",
      "=== total 137000 match 43106\n",
      ">>> step 137100\n",
      "step 137000-137099, precision 0.454486, recall 0.669284, f_score 0.541357\n",
      "=== total 137100 match 43130\n",
      ">>> step 137200\n",
      "step 137100-137199, precision 0.454463, recall 0.669509, f_score 0.541414\n",
      "=== total 137200 match 43185\n",
      ">>> step 137300\n",
      "step 137200-137299, precision 0.454436, recall 0.669791, f_score 0.541487\n",
      "=== total 137300 match 43247\n",
      ">>> step 137400\n",
      "step 137300-137399, precision 0.454367, recall 0.669652, f_score 0.541393\n",
      "=== total 137400 match 43280\n",
      ">>> step 137500\n",
      "step 137400-137499, precision 0.454147, recall 0.669652, f_score 0.541236\n",
      "=== total 137500 match 43301\n",
      ">>> step 137600\n",
      "step 137500-137599, precision 0.453926, recall 0.669652, f_score 0.541080\n",
      "=== total 137600 match 43322\n",
      ">>> step 137700\n",
      "step 137600-137699, precision 0.453908, recall 0.669944, f_score 0.541162\n",
      "=== total 137700 match 43381\n",
      ">>> step 137800\n",
      "step 137700-137799, precision 0.454422, recall 0.670505, f_score 0.541710\n",
      "=== total 137800 match 43442\n",
      ">>> step 137900\n",
      "step 137800-137899, precision 0.454109, recall 0.670561, f_score 0.541506\n",
      "=== total 137900 match 43483\n",
      ">>> step 138000\n",
      "step 137900-137999, precision 0.454335, recall 0.670962, f_score 0.541797\n",
      "=== total 138000 match 43545\n",
      ">>> step 138100\n",
      "step 138000-138099, precision 0.454814, recall 0.671520, f_score 0.542320\n",
      "=== total 138100 match 43609\n",
      ">>> step 138200\n",
      "step 138100-138199, precision 0.454722, recall 0.671808, f_score 0.542349\n",
      "=== total 138200 match 43675\n",
      ">>> step 138300\n",
      "step 138200-138299, precision 0.455221, recall 0.672363, f_score 0.542884\n",
      "=== total 138300 match 43737\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-19-0cb334589f57>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset_default_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mmodel_tf_deep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m250\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mvalidate2_for_cross_validation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_imgs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_lbls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mis_enriched\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mROOT_FOLDER\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"models/front_tear_model2.ckpt\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-14-e4129b99881f>\u001b[0m in \u001b[0;36mvalidate2_for_cross_validation\u001b[0;34m(test_imgs, test_lbls, is_enriched, model, max_samples)\u001b[0m\n\u001b[1;32m     57\u001b[0m         \u001b[0mmy_prediction\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_conv\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m         \u001b[0mmy_target\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 59\u001b[0;31m         \u001b[0mv1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmy_prediction\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mx_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0my_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeep_prob\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m1.0\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     60\u001b[0m         \u001b[0mv2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmy_target\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mx_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0my_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeep_prob\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m1.0\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     61\u001b[0m         \u001b[0mv1t\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv1t\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venv35/lib/python3.5/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36meval\u001b[0;34m(self, feed_dict, session)\u001b[0m\n\u001b[1;32m    711\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    712\u001b[0m     \"\"\"\n\u001b[0;32m--> 713\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_eval_using_default_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    714\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    715\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venv35/lib/python3.5/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m_eval_using_default_session\u001b[0;34m(tensors, feed_dict, graph, session)\u001b[0m\n\u001b[1;32m   5155\u001b[0m                        \u001b[0;34m\"the tensor's graph is different from the session's \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5156\u001b[0m                        \"graph.\")\n\u001b[0;32m-> 5157\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5158\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5159\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venv35/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    927\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    928\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 929\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    930\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    931\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venv35/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1150\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1151\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1152\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1153\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1154\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venv35/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1326\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1327\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1328\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1329\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1330\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venv35/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1332\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1333\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1334\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1335\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1336\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venv35/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1315\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_run_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1316\u001b[0m       \u001b[0;31m# Ensure any changes to the graph are reflected in the runtime.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1317\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1318\u001b[0m       return self._call_tf_sessionrun(\n\u001b[1;32m   1319\u001b[0m           options, feed_dict, fetch_list, target_list, run_metadata)\n",
      "\u001b[0;32m~/venv35/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_extend_graph\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1350\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1351\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_graph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session_run_lock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1352\u001b[0;31m       \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mExtendSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1353\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1354\u001b[0m   \u001b[0;31m# The threshold to run garbage collection to delete dead tensors.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# TEST3 - take 1 piece and cross- validate on this (uncomment all for full test run)\n",
    "cubes_set = pre_process_training(\"PX303-Fg006-R-C02-R02-D08032015-T114223-ML638__006.jpg\", 0, 6200, 0, 4400)\n",
    "train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "    NEW_build_train_set_for_binary_labeling(cubes_set, cubes_set, CUBE_SIZE, ROOT_FOLDER + \"train_concats3/\", 1, 7)\n",
    "tf.reset_default_graph()\n",
    "model_tf_deep(250, 1)    \n",
    "validate2_for_cross_validation(train_imgs, train_lbls, is_enriched, ROOT_FOLDER + \"models/front_tear_model2.ckpt\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# RE-TEST3 - take 1 piece and cross- validate on this (uncomment all for full test run)\n",
    "train_imgs, train_lbls, is_enriched = \\\n",
    "    load_train_from_disk(ROOT_FOLDER + \"train_concats3/\")\n",
    "# tf.reset_default_graph()\n",
    "# model_tf_deep(250, 1)    \n",
    "# validate2_for_cross_validation(train_imgs, train_lbls, is_enriched, ROOT_FOLDER + \"model_binary/tear_model3.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "#### STOP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "len(train_imgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum(x[1] == 1 for x in train_lbls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "#####################################################################\n",
    "TRAINING:\n",
    "MODEL:/Users/il239838/Downloads/private/Thesis/Papyrus/model_binary_new_X6/tear_model1.ckpt\n",
    "#####################################################################\n",
    "step 49, training accuracy 0.78\n",
    "step 99, training accuracy 0.84\n",
    "step 149, training accuracy 0.9\n",
    "step 199, training accuracy 0.92\n",
    "step 249, training accuracy 0.88\n",
    "step 299, training accuracy 0.84\n",
    "step 349, training accuracy 0.92\n",
    "step 399, training accuracy 0.86\n",
    "Optimization Finished!\n",
    "step 399, training accuracy 0.86\n",
    "Model saved in file: /Users/il239838/Downloads/private/Thesis/Papyrus/model_binary_new_X6/tear_model1.ckpt\n",
    "#####################################################################\n",
    "TRAINING ENDED\n",
    "#####################################################################\n",
    "deper network\n",
    "#####################################################################\n",
    "TRAINING:\n",
    "MODEL:/Users/il239838/Downloads/private/Thesis/Papyrus/model_binary_new_X6/tear_model1.ckpt\n",
    "#####################################################################\n",
    "step 49, training accuracy 0.84\n",
    "step 99, training accuracy 0.94\n",
    "step 149, training accuracy 0.86\n",
    "step 199, training accuracy 0.92\n",
    "step 249, training accuracy 0.88\n",
    "step 299, training accuracy 0.92\n",
    "step 349, training accuracy 0.96\n",
    "step 399, training accuracy 0.96\n",
    "Optimization Finished!\n",
    "step 399, training accuracy 0.96\n",
    "Model saved in file: /Users/il239838/Downloads/private/Thesis/Papyrus/model_binary_new_X6/tear_model1.ckpt\n",
    "#####################################################################\n",
    "TRAINING ENDED\n",
    "#####################################################################\n",
    "deeper network on GCP\n",
    "#####################################################################\n",
    "TRAINING:\n",
    "MODEL:/home/il239838/files/model_binary/tear_model1.ckpt\n",
    "#####################################################################\n",
    "WARNING:tensorflow:From /home/il239838/miniconda3/envs/carnd-term1/lib/python3.5/site-packages/tensorflow/python/util/tf_should_use.py:118: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n",
    "Instructions for updating:\n",
    "Use `tf.global_variables_initializer` instead.\n",
    "step 49, training accuracy 0.82\n",
    "step 99, training accuracy 0.92\n",
    "step 149, training accuracy 0.72\n",
    "step 199, training accuracy 0.8\n",
    "step 249, training accuracy 0.88\n",
    "step 299, training accuracy 0.88\n",
    "step 349, training accuracy 0.94\n",
    "step 399, training accuracy 0.84\n",
    "Optimization Finished!\n",
    "step 399, training accuracy 0.84\n",
    "Model saved in file: /home/il239838/files/model_binary/tear_model1.ckpt\n",
    "#####################################################################\n",
    "TRAINING ENDED\n",
    "#####################################################################\n",
    "  \n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "#####################################################################\n",
    "TRAINING:\n",
    "MODEL:/Users/il239838/Downloads/private/Thesis/Papyrus/model_binary_new_X6/tear_model2.ckpt\n",
    "#####################################################################\n",
    "INFO:tensorflow:Restoring parameters from /Users/il239838/Downloads/private/Thesis/Papyrus/model_binary_new_X6/tear_model1.ckpt\n",
    "Model restored.\n",
    "step 49, training accuracy 0.76\n",
    "step 99, training accuracy 0.82\n",
    "step 149, training accuracy 0.96\n",
    "step 199, training accuracy 0.86\n",
    "step 249, training accuracy 0.76\n",
    "step 299, training accuracy 0.82\n",
    "step 349, training accuracy 0.86\n",
    "step 399, training accuracy 0.88\n",
    "Optimization Finished!\n",
    "step 399, training accuracy 0.88\n",
    "Model saved in file: /Users/il239838/Downloads/private/Thesis/Papyrus/model_binary_new_X6/tear_model2.ckpt\n",
    "#####################################################################\n",
    "TRAINING ENDED\n",
    "#####################################################################\n",
    "deeper network\n",
    "#####################################################################\n",
    "TRAINING:\n",
    "MODEL:/Users/il239838/Downloads/private/Thesis/Papyrus/model_binary_new_X6/tear_model2.ckpt\n",
    "#####################################################################\n",
    "INFO:tensorflow:Restoring parameters from /Users/il239838/Downloads/private/Thesis/Papyrus/model_binary_new_X6/tear_model1.ckpt\n",
    "Model restored.\n",
    "step 49, training accuracy 0.88\n",
    "step 99, training accuracy 0.88\n",
    "step 149, training accuracy 0.88\n",
    "step 199, training accuracy 0.88\n",
    "step 249, training accuracy 0.92\n",
    "step 299, training accuracy 0.92\n",
    "step 349, training accuracy 0.86\n",
    "step 399, training accuracy 0.94\n",
    "Optimization Finished!\n",
    "step 399, training accuracy 0.94\n",
    "Model saved in file: /Users/il239838/Downloads/private/Thesis/Papyrus/model_binary_new_X6/tear_model2.ckpt\n",
    "#####################################################################\n",
    "TRAINING ENDED\n",
    "#####################################################################\n",
    "deeper network on GCP\n",
    "#####################################################################\n",
    "TRAINING:\n",
    "MODEL:/home/il239838/files/model_binary/tear_model2.ckpt\n",
    "#####################################################################\n",
    "INFO:tensorflow:Restoring parameters from /home/il239838/files/model_binary/tear_model1.ckpt\n",
    "Model restored.\n",
    "step 49, training accuracy 0.88\n",
    "step 99, training accuracy 0.96\n",
    "step 149, training accuracy 0.84\n",
    "step 199, training accuracy 0.84\n",
    "step 249, training accuracy 0.84\n",
    "step 299, training accuracy 0.92\n",
    "step 349, training accuracy 0.98\n",
    "step 399, training accuracy 0.9\n",
    "Optimization Finished!\n",
    "step 399, training accuracy 0.9\n",
    "Model saved in file: /home/il239838/files/model_binary/tear_model2.ckpt\n",
    "#####################################################################\n",
    "TRAINING ENDED\n",
    "#####################################################################\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "#####################################################################\n",
    ">>> step 100\n",
    "step 0-99, precision 0.695652, recall 0.640000, f_score 0.666667\n",
    ">>> step 200\n",
    "step 100-199, precision 0.512500, recall 0.732143, f_score 0.602941\n",
    ">>> step 300\n",
    "step 200-299, precision 0.350427, recall 0.732143, f_score 0.473988\n",
    ">>> step 400\n",
    "step 300-399, precision 0.408805, recall 0.677083, f_score 0.509804\n",
    ">>> step 500\n",
    "step 400-499, precision 0.366834, recall 0.651786, f_score 0.469453\n",
    ">>> step 600\n",
    "step 500-599, precision 0.349282, recall 0.651786, f_score 0.454829\n",
    ">>> step 700\n",
    "step 600-699, precision 0.376471, recall 0.662069, f_score 0.480000\n",
    ">>> step 800\n",
    "step 700-799, precision 0.383142, recall 0.595238, f_score 0.466200\n",
    ">>> step 900\n",
    "step 800-899, precision 0.394649, recall 0.617801, f_score 0.481633\n",
    ">>> step 1000\n",
    "step 900-999, precision 0.407821, recall 0.651786, f_score 0.501718\n",
    ">>> step 1100\n",
    "step 1000-1099, precision 0.404432, recall 0.651786, f_score 0.499145\n",
    ">>> step 1200\n",
    "step 1100-1199, precision 0.423559, recall 0.657588, f_score 0.515244\n",
    ">>> step 1300\n",
    "step 1200-1299, precision 0.408983, recall 0.662835, f_score 0.505848\n",
    ">>> step 1400\n",
    "step 1300-1399, precision 0.405640, recall 0.667857, f_score 0.504723\n",
    ">>> step 1500\n",
    "step 1400-1499, precision 0.415020, recall 0.670927, f_score 0.512821\n",
    ">>> step 1600\n",
    "step 1500-1599, precision 0.402852, recall 0.672619, f_score 0.503902\n",
    ">>> step 1700\n",
    "step 1600-1699, precision 0.401681, recall 0.647696, f_score 0.495851\n",
    ">>> step 1800\n",
    "step 1700-1799, precision 0.387987, recall 0.647696, f_score 0.485279\n",
    ">>> step 1900\n",
    "step 1800-1899, precision 0.385484, recall 0.647696, f_score 0.483316\n",
    ">>> step 2000\n",
    "step 1900-1999, precision 0.388802, recall 0.647668, f_score 0.485909\n",
    ">>> step 2100\n",
    "step 2000-2099, precision 0.379161, recall 0.650124, f_score 0.478976\n",
    ">>> step 2200\n",
    "step 2100-2199, precision 0.374286, recall 0.650124, f_score 0.475068\n",
    ">>> step 2300\n",
    "step 2200-2299, precision 0.381868, recall 0.651054, f_score 0.481385\n",
    ">>> step 2400\n",
    "step 2300-2399, precision 0.392622, recall 0.659292, f_score 0.492155\n",
    ">>> step 2500\n",
    "step 2400-2499, precision 0.393604, recall 0.665281, f_score 0.494590\n",
    ">>> step 2600\n",
    "step 2500-2599, precision 0.392601, recall 0.664646, f_score 0.493623\n",
    ">>> step 2700\n",
    "step 2600-2699, precision 0.389810, recall 0.664646, f_score 0.491412\n",
    ">>> step 2800\n",
    "step 2700-2799, precision 0.384884, recall 0.655446, f_score 0.484982\n",
    ">>> step 2900\n",
    "step 2800-2899, precision 0.390572, recall 0.664122, f_score 0.491873\n",
    ">>> step 3000\n",
    "step 2900-2999, precision 0.401064, recall 0.679279, f_score 0.504348\n",
    ">>> step 3100\n",
    "step 3000-3099, precision 0.418534, recall 0.695431, f_score 0.522568\n",
    ">>> step 3200\n",
    "step 3100-3199, precision 0.430129, recall 0.704545, f_score 0.534154\n",
    ">>> step 3300\n",
    "step 3200-3299, precision 0.446036, recall 0.716258, f_score 0.549735\n",
    ">>> step 3400\n",
    "step 3300-3399, precision 0.448563, recall 0.722388, f_score 0.553459\n",
    ">>> step 3500\n",
    "step 3400-3499, precision 0.455204, recall 0.719599, f_score 0.557650\n",
    ">>> step 3600\n",
    "step 3500-3599, precision 0.456560, recall 0.718271, f_score 0.558266\n",
    ">>> step 3700\n",
    "step 3600-3699, precision 0.457841, recall 0.724000, f_score 0.560950\n",
    ">>> step 3800\n",
    "step 3700-3799, precision 0.455285, recall 0.720721, f_score 0.558047\n",
    ">>> step 3900\n",
    "step 3800-3899, precision 0.459969, recall 0.724351, f_score 0.562650\n",
    ">>> step 4000\n",
    "step 3900-3999, precision 0.445289, recall 0.724351, f_score 0.551529\n",
    ">>> step 4100\n",
    "step 4000-4099, precision 0.442598, recall 0.724351, f_score 0.549461\n",
    ">>> step 4200\n",
    "step 4100-4199, precision 0.445596, recall 0.725301, f_score 0.552040\n",
    ">>> step 4300\n",
    "step 4200-4299, precision 0.429793, recall 0.724760, f_score 0.539597\n",
    ">>> step 4400\n",
    "step 4300-4399, precision 0.426450, recall 0.724760, f_score 0.536955\n",
    ">>> step 4500\n",
    "step 4400-4499, precision 0.427491, recall 0.727485, f_score 0.538528\n",
    ">>> step 4600\n",
    "step 4500-4599, precision 0.427310, recall 0.723820, f_score 0.537377\n",
    ">>> step 4700\n",
    "step 4600-4699, precision 0.422999, recall 0.723820, f_score 0.533956\n",
    ">>> step 4800\n",
    "step 4700-4799, precision 0.419893, recall 0.723820, f_score 0.531474\n",
    ">>> step 4900\n",
    "step 4800-4899, precision 0.423740, recall 0.723669, f_score 0.534504\n",
    ">>> step 5000\n",
    "step 4900-4999, precision 0.428387, recall 0.725683, f_score 0.538742\n",
    ">>> step 5100\n",
    "step 5000-5099, precision 0.417348, recall 0.725683, f_score 0.529928\n",
    ">>> step 5200\n",
    "step 5100-5199, precision 0.414482, recall 0.725683, f_score 0.527612\n",
    ">>> step 5300\n",
    "step 5200-5299, precision 0.412935, recall 0.725683, f_score 0.526358\n",
    ">>> step 5400\n",
    "step 5300-5399, precision 0.418093, recall 0.720759, f_score 0.529207\n",
    ">>> step 5500\n",
    "step 5400-5499, precision 0.409934, recall 0.720294, f_score 0.522502\n",
    ">>> step 5600\n",
    "step 5500-5599, precision 0.409280, recall 0.719665, f_score 0.521805\n",
    ">>> step 5700\n",
    "step 5600-5699, precision 0.411176, recall 0.716189, f_score 0.522422\n",
    ">>> step 5800\n",
    "step 5700-5799, precision 0.413056, recall 0.707921, f_score 0.521707\n",
    ">>> step 5900\n",
    "step 5800-5899, precision 0.408427, recall 0.701061, f_score 0.516152\n",
    ">>> step 6000\n",
    "step 5900-5999, precision 0.404113, recall 0.701061, f_score 0.512694\n",
    ">>> step 6100\n",
    "step 6000-6099, precision 0.399565, recall 0.700382, f_score 0.508839\n",
    ">>> step 6200\n",
    "step 6100-6199, precision 0.398913, recall 0.700382, f_score 0.508310\n",
    ">>> step 6300\n",
    "step 6200-6299, precision 0.402681, recall 0.704503, f_score 0.512453\n",
    ">>> step 6400\n",
    "step 6300-6399, precision 0.404178, recall 0.709441, f_score 0.514970\n",
    ">>> step 6500\n",
    "step 6400-6499, precision 0.402955, recall 0.713255, f_score 0.514974\n",
    ">>> step 6600\n",
    "step 6500-6599, precision 0.402532, recall 0.713645, f_score 0.514730\n",
    ">>> step 6700\n",
    "step 6600-6699, precision 0.413011, recall 0.712585, f_score 0.522933\n",
    ">>> step 6800\n",
    "step 6700-6799, precision 0.416506, recall 0.713813, f_score 0.526059\n",
    ">>> step 6900\n",
    "step 6800-6899, precision 0.421002, recall 0.712439, f_score 0.529253\n",
    ">>> step 7000\n",
    "step 6900-6999, precision 0.428773, recall 0.709176, f_score 0.534427\n",
    ">>> step 7100\n",
    "step 7000-7099, precision 0.430497, recall 0.704718, f_score 0.534488\n",
    ">>> step 7200\n",
    "step 7100-7199, precision 0.437929, recall 0.695273, f_score 0.537381\n",
    ">>> step 7300\n",
    "step 7200-7299, precision 0.440254, recall 0.693133, f_score 0.538483\n",
    ">>> step 7400\n",
    "step 7300-7399, precision 0.441348, recall 0.690577, f_score 0.538525\n",
    ">>> step 7500\n",
    "step 7400-7499, precision 0.439821, recall 0.688375, f_score 0.536719\n",
    ">>> step 7600\n",
    "step 7500-7599, precision 0.440693, recall 0.683196, f_score 0.535782\n",
    ">>> step 7700\n",
    "step 7600-7699, precision 0.440618, recall 0.676152, f_score 0.533547\n",
    ">>> step 7800\n",
    "step 7700-7799, precision 0.435618, recall 0.676152, f_score 0.529865\n",
    ">>> step 7900\n",
    "step 7800-7899, precision 0.431701, recall 0.674044, f_score 0.526316\n",
    ">>> step 8000\n",
    "step 7900-7999, precision 0.429482, recall 0.668651, f_score 0.523021\n",
    ">>> step 8100\n",
    "step 8000-8099, precision 0.428027, recall 0.667768, f_score 0.521672\n",
    ">>> step 8200\n",
    "step 8100-8199, precision 0.429717, recall 0.664057, f_score 0.521784\n",
    ">>> step 8300\n",
    "step 8200-8299, precision 0.428451, recall 0.664057, f_score 0.520849\n",
    ">>> step 8400\n",
    "step 8300-8399, precision 0.429589, recall 0.660438, f_score 0.520569\n",
    ">>> step 8500\n",
    "step 8400-8499, precision 0.428870, recall 0.660438, f_score 0.520041\n",
    ">>> step 8600\n",
    "step 8500-8599, precision 0.427618, recall 0.660438, f_score 0.519119\n",
    ">>> step 8700\n",
    "step 8600-8699, precision 0.424528, recall 0.655478, f_score 0.515310\n",
    ">>> step 8800\n",
    "step 8700-8799, precision 0.421074, recall 0.655478, f_score 0.512757\n",
    ">>> step 8900\n",
    "step 8800-8899, precision 0.420032, recall 0.650814, f_score 0.510555\n",
    ">>> step 9000\n",
    "step 8900-8999, precision 0.415667, recall 0.650814, f_score 0.507317\n",
    ">>> step 9100\n",
    "step 9000-9099, precision 0.414201, recall 0.646154, f_score 0.504808\n",
    ">>> step 9200\n",
    "step 9100-9199, precision 0.414510, recall 0.642944, f_score 0.504053\n",
    ">>> step 9300\n",
    "step 9200-9299, precision 0.413793, recall 0.639138, f_score 0.502352\n",
    ">>> step 9400\n",
    "step 9300-9399, precision 0.408101, recall 0.639138, f_score 0.498134\n",
    ">>> step 9500\n",
    "step 9400-9499, precision 0.407789, recall 0.639138, f_score 0.497902\n",
    ">>> step 9600\n",
    "step 9500-9599, precision 0.408248, recall 0.639218, f_score 0.498268\n",
    ">>> step 9700\n",
    "step 9600-9699, precision 0.406097, recall 0.639218, f_score 0.496663\n",
    ">>> step 9800\n",
    "step 9700-9799, precision 0.408482, recall 0.639487, f_score 0.498524\n",
    ">>> step 9900\n",
    "step 9800-9899, precision 0.407298, recall 0.639098, f_score 0.497524\n",
    ">>> step 10000\n",
    "step 9900-9999, precision 0.406250, recall 0.639098, f_score 0.496741\n",
    ">>> step 10100\n",
    "step 10000-10099, precision 0.407407, recall 0.636312, f_score 0.496758\n",
    ">>> step 10200\n",
    "step 10100-10199, precision 0.408892, recall 0.636415, f_score 0.497892\n",
    ">>> step 10300\n",
    "step 10200-10299, precision 0.407852, recall 0.636415, f_score 0.497120\n",
    ">>> step 10400\n",
    "step 10300-10399, precision 0.408796, recall 0.637079, f_score 0.498024\n",
    ">>> step 10500\n",
    "step 10400-10499, precision 0.406452, recall 0.637079, f_score 0.496280\n",
    ">>> step 10600\n",
    "step 10500-10599, precision 0.405540, recall 0.637989, f_score 0.495875\n",
    ">>> step 10700\n",
    "step 10600-10699, precision 0.403583, recall 0.636918, f_score 0.494087\n",
    ">>> step 10800\n",
    "step 10700-10799, precision 0.401748, recall 0.636918, f_score 0.492710\n",
    ">>> step 10900\n",
    "step 10800-10899, precision 0.402490, recall 0.636066, f_score 0.493011\n",
    ">>> step 11000\n",
    "step 10900-10999, precision 0.400619, recall 0.633496, f_score 0.490836\n",
    ">>> step 11100\n",
    "step 11000-11099, precision 0.396798, recall 0.633496, f_score 0.487958\n",
    ">>> step 11200\n",
    "step 11100-11199, precision 0.396944, recall 0.629510, f_score 0.486880\n",
    ">>> step 11300\n",
    "step 11200-11299, precision 0.397928, recall 0.628496, f_score 0.487316\n",
    ">>> step 11400\n",
    "step 11300-11399, precision 0.398474, recall 0.625521, f_score 0.486826\n",
    ">>> step 11500\n",
    "step 11400-11499, precision 0.397233, recall 0.620690, f_score 0.484435\n",
    ">>> step 11600\n",
    "step 11500-11599, precision 0.396189, recall 0.620690, f_score 0.483658\n",
    ">>> step 11700\n",
    "step 11600-11699, precision 0.397906, recall 0.620725, f_score 0.484945\n",
    ">>> step 11800\n",
    "step 11700-11799, precision 0.397856, recall 0.618999, f_score 0.484381\n",
    ">>> step 11900\n",
    "step 11800-11899, precision 0.396184, recall 0.618999, f_score 0.483139\n",
    ">>> step 12000\n",
    "step 11900-11999, precision 0.394779, recall 0.618999, f_score 0.482094\n",
    ">>> step 12100\n",
    "step 12000-12099, precision 0.397894, recall 0.619473, f_score 0.484554\n",
    ">>> step 12200\n",
    "step 12100-12199, precision 0.402704, recall 0.621242, f_score 0.488652\n",
    ">>> step 12300\n",
    "step 12200-12299, precision 0.408271, recall 0.618464, f_score 0.491852\n",
    ">>> step 12400\n",
    "step 12300-12399, precision 0.411149, recall 0.615207, f_score 0.492893\n",
    ">>> step 12500\n",
    "step 12400-12499, precision 0.414962, recall 0.612991, f_score 0.494902\n",
    ">>> step 12600\n",
    "step 12500-12599, precision 0.416262, recall 0.606275, f_score 0.493614\n",
    ">>> step 12700\n",
    "step 12600-12699, precision 0.419520, recall 0.603456, f_score 0.494951\n",
    ">>> step 12800\n",
    "step 12700-12799, precision 0.418013, recall 0.603456, f_score 0.493901\n",
    ">>> step 12900\n",
    "step 12800-12899, precision 0.417014, recall 0.600943, f_score 0.492362\n",
    ">>> step 13000\n",
    "step 12900-12999, precision 0.412596, recall 0.600943, f_score 0.489269\n",
    ">>> step 13100\n",
    "step 13000-13099, precision 0.410480, recall 0.599745, f_score 0.487383\n",
    ">>> step 13200\n",
    "step 13100-13199, precision 0.408577, recall 0.599745, f_score 0.486039\n",
    ">>> step 13300\n",
    "step 13200-13299, precision 0.407397, recall 0.599745, f_score 0.485203\n",
    ">>> step 13400\n",
    "step 13300-13399, precision 0.406178, recall 0.599409, f_score 0.484228\n",
    ">>> step 13500\n",
    "step 13400-13499, precision 0.401925, recall 0.599409, f_score 0.481193\n",
    ">>> step 13600\n",
    "step 13500-13599, precision 0.401584, recall 0.599409, f_score 0.480948\n",
    ">>> step 13700\n",
    "step 13600-13699, precision 0.400391, recall 0.600335, f_score 0.480389\n",
    ">>> step 13800\n",
    "step 13700-13799, precision 0.399224, recall 0.598753, f_score 0.479042\n",
    ">>> step 13900\n",
    "step 13800-13899, precision 0.395553, recall 0.598919, f_score 0.476442\n",
    ">>> step 14000\n",
    "step 13900-13999, precision 0.393915, recall 0.598432, f_score 0.475098\n",
    ">>> step 14100\n",
    "step 14000-14099, precision 0.393915, recall 0.598432, f_score 0.475098\n",
    ">>> step 14200\n",
    "step 14100-14199, precision 0.392741, recall 0.598432, f_score 0.474244\n",
    ">>> step 14300\n",
    "step 14200-14299, precision 0.393562, recall 0.597536, f_score 0.474560\n",
    ">>> step 14400\n",
    "step 14300-14399, precision 0.390675, recall 0.594374, f_score 0.471463\n",
    ">>> step 14500\n",
    "step 14400-14499, precision 0.389944, recall 0.592924, f_score 0.470474\n",
    ">>> step 14600\n",
    "step 14500-14599, precision 0.386943, recall 0.592924, f_score 0.468283\n",
    ">>> step 14700\n",
    "step 14600-14699, precision 0.386532, recall 0.592924, f_score 0.467983\n",
    ">>> step 14800\n",
    "step 14700-14799, precision 0.387326, recall 0.595152, f_score 0.469258\n",
    ">>> step 14900\n",
    "step 14800-14899, precision 0.386903, recall 0.595343, f_score 0.469007\n",
    ">>> step 15000\n",
    "step 14900-14999, precision 0.388658, recall 0.595694, f_score 0.470403\n",
    ">>> step 15100\n",
    "step 15000-15099, precision 0.387264, recall 0.595304, f_score 0.469260\n",
    ">>> step 15200\n",
    "step 15100-15199, precision 0.386923, recall 0.596679, f_score 0.469435\n",
    ">>> step 15300\n",
    "step 15200-15299, precision 0.387583, recall 0.597950, f_score 0.470315\n",
    ">>> step 15400\n",
    "step 15300-15399, precision 0.384228, recall 0.599214, f_score 0.468222\n",
    ">>> step 15500\n",
    "step 15400-15499, precision 0.383070, recall 0.599214, f_score 0.467361\n",
    ">>> step 15600\n",
    "step 15500-15599, precision 0.384500, recall 0.599143, f_score 0.468403\n",
    ">>> step 15700\n",
    "step 15600-15699, precision 0.386607, recall 0.601239, f_score 0.470606\n",
    ">>> step 15800\n",
    "step 15700-15799, precision 0.382607, recall 0.601239, f_score 0.467630\n",
    ">>> step 15900\n",
    "step 15800-15899, precision 0.382626, recall 0.603309, f_score 0.468269\n",
    ">>> step 16000\n",
    "step 15900-15999, precision 0.383248, recall 0.603296, f_score 0.468731\n",
    ">>> step 16100\n",
    "step 16000-16099, precision 0.382317, recall 0.603296, f_score 0.468034\n",
    ">>> step 16200\n",
    "step 16100-16199, precision 0.381946, recall 0.603296, f_score 0.467756\n",
    ">>> step 16300\n",
    "step 16200-16299, precision 0.379460, recall 0.603296, f_score 0.465887\n",
    ">>> step 16400\n",
    "step 16300-16399, precision 0.382472, recall 0.604669, f_score 0.468563\n",
    ">>> step 16500\n",
    "step 16400-16499, precision 0.385124, recall 0.605644, f_score 0.470843\n",
    ">>> step 16600\n",
    "step 16500-16599, precision 0.386226, recall 0.609174, f_score 0.472732\n",
    ">>> step 16700\n",
    "step 16600-16699, precision 0.385509, recall 0.609174, f_score 0.472195\n",
    ">>> step 16800\n",
    "step 16700-16799, precision 0.387558, recall 0.609641, f_score 0.473870\n",
    ">>> step 16900\n",
    "step 16800-16899, precision 0.389587, recall 0.611689, f_score 0.476004\n",
    ">>> step 17000\n",
    "step 16900-16999, precision 0.392534, recall 0.612858, f_score 0.478555\n",
    ">>> step 17100\n",
    "step 17000-17099, precision 0.393763, recall 0.611711, f_score 0.479115\n",
    ">>> step 17200\n",
    "step 17100-17199, precision 0.392442, recall 0.611711, f_score 0.478136\n",
    ">>> step 17300\n",
    "step 17200-17299, precision 0.394895, recall 0.614296, f_score 0.480746\n",
    ">>> step 17400\n",
    "step 17300-17399, precision 0.397101, recall 0.616644, f_score 0.483100\n",
    ">>> step 17500\n",
    "step 17400-17499, precision 0.401132, recall 0.617085, f_score 0.486208\n",
    ">>> step 17600\n",
    "step 17500-17599, precision 0.399220, recall 0.617085, f_score 0.484801\n",
    ">>> step 17700\n",
    "step 17600-17699, precision 0.400259, recall 0.617539, f_score 0.485707\n",
    ">>> step 17800\n",
    "step 17700-17799, precision 0.402402, recall 0.618734, f_score 0.487653\n",
    ">>> step 17900\n",
    "step 17800-17899, precision 0.403036, recall 0.617830, f_score 0.487836\n",
    ">>> step 18000\n",
    "step 17900-17999, precision 0.404838, recall 0.618076, f_score 0.489231\n",
    ">>> step 18100\n",
    "step 18000-18099, precision 0.407634, recall 0.618758, f_score 0.491482\n",
    ">>> step 18200\n",
    "step 18100-18199, precision 0.406473, recall 0.618682, f_score 0.490614\n",
    ">>> step 18300\n",
    "step 18200-18299, precision 0.407004, recall 0.619139, f_score 0.491144\n",
    ">>> step 18400\n",
    "step 18300-18399, precision 0.406982, recall 0.619472, f_score 0.491232\n",
    ">>> step 18500\n",
    "step 18400-18499, precision 0.405737, recall 0.619093, f_score 0.490206\n",
    ">>> step 18600\n",
    "step 18500-18599, precision 0.404076, recall 0.620025, f_score 0.489282\n",
    ">>> step 18700\n",
    "step 18600-18699, precision 0.401760, recall 0.620025, f_score 0.487581\n",
    ">>> step 18800\n",
    "step 18700-18799, precision 0.399308, recall 0.620025, f_score 0.485771\n",
    ">>> step 18900\n",
    "step 18800-18899, precision 0.398062, recall 0.620321, f_score 0.484938\n",
    ">>> step 19000\n",
    "step 18900-18999, precision 0.396462, recall 0.620321, f_score 0.483748\n",
    ">>> step 19100\n",
    "step 19000-19099, precision 0.396994, recall 0.620614, f_score 0.484234\n",
    ">>> step 19200\n",
    "step 19100-19199, precision 0.395224, recall 0.620431, f_score 0.482859\n",
    ">>> step 19300\n",
    "step 19200-19299, precision 0.393585, recall 0.620281, f_score 0.481589\n",
    ">>> step 19400\n",
    "step 19300-19399, precision 0.393508, recall 0.620281, f_score 0.481531\n",
    ">>> step 19500\n",
    "step 19400-19499, precision 0.393243, recall 0.621160, f_score 0.481597\n",
    ">>> step 19600\n",
    "step 19500-19599, precision 0.392549, recall 0.621160, f_score 0.481077\n",
    ">>> step 19700\n",
    "step 19600-19699, precision 0.392088, recall 0.621160, f_score 0.480730\n",
    ">>> step 19800\n",
    "step 19700-19799, precision 0.391704, recall 0.621160, f_score 0.480442\n",
    ">>> step 19900\n",
    "step 19800-19899, precision 0.391016, recall 0.621160, f_score 0.479923\n",
    ">>> step 20000\n",
    "step 19900-19999, precision 0.390358, recall 0.620519, f_score 0.479236\n",
    ">>> step 20100\n",
    "step 20000-20099, precision 0.390504, recall 0.620764, f_score 0.479419\n",
    ">>> step 20200\n",
    "step 20100-20199, precision 0.386887, recall 0.621114, f_score 0.476787\n",
    ">>> step 20300\n",
    "step 20200-20299, precision 0.384308, recall 0.621114, f_score 0.474824\n",
    ">>> step 20400\n",
    "step 20300-20399, precision 0.384161, recall 0.621114, f_score 0.474712\n",
    ">>> step 20500\n",
    "step 20400-20499, precision 0.383577, recall 0.621114, f_score 0.474266\n",
    ">>> step 20600\n",
    "step 20500-20599, precision 0.383660, recall 0.622861, f_score 0.474837\n",
    ">>> step 20700\n",
    "step 20600-20699, precision 0.383569, recall 0.624886, f_score 0.475355\n",
    ">>> step 20800\n",
    "step 20700-20799, precision 0.382785, recall 0.624886, f_score 0.474752\n",
    ">>> step 20900\n",
    "step 20800-20899, precision 0.384644, recall 0.627704, f_score 0.476995\n",
    ">>> step 21000\n",
    "step 20900-20999, precision 0.383092, recall 0.627704, f_score 0.475800\n",
    ">>> step 21100\n",
    "step 21000-21099, precision 0.383464, recall 0.626977, f_score 0.475878\n",
    ">>> step 21200\n",
    "step 21100-21199, precision 0.383045, recall 0.626977, f_score 0.475555\n",
    ">>> step 21300\n",
    "step 21200-21299, precision 0.383430, recall 0.626852, f_score 0.475816\n",
    ">>> step 21400\n",
    "step 21300-21399, precision 0.384685, recall 0.628119, f_score 0.477146\n",
    ">>> step 21500\n",
    "step 21400-21499, precision 0.383581, recall 0.628119, f_score 0.476296\n",
    ">>> step 21600\n",
    "step 21500-21599, precision 0.384506, recall 0.629446, f_score 0.477391\n",
    ">>> step 21700\n",
    "step 21600-21699, precision 0.382462, recall 0.629446, f_score 0.475813\n",
    ">>> step 21800\n",
    "step 21700-21799, precision 0.382716, recall 0.628439, f_score 0.475721\n",
    ">>> step 21900\n",
    "step 21800-21899, precision 0.380835, recall 0.628439, f_score 0.474265\n",
    ">>> step 22000\n",
    "step 21900-21999, precision 0.380235, recall 0.628439, f_score 0.473799\n",
    ">>> step 22100\n",
    "step 22000-22099, precision 0.380420, recall 0.629084, f_score 0.474126\n",
    ">>> step 22200\n",
    "step 22100-22199, precision 0.378838, recall 0.629758, f_score 0.473086\n",
    ">>> step 22300\n",
    "step 22200-22299, precision 0.378196, recall 0.628481, f_score 0.472225\n",
    ">>> step 22400\n",
    "step 22300-22399, precision 0.377388, recall 0.627647, f_score 0.471359\n",
    ">>> step 22500\n",
    "step 22400-22499, precision 0.377378, recall 0.626815, f_score 0.471117\n",
    ">>> step 22600\n",
    "step 22500-22599, precision 0.377937, recall 0.628183, f_score 0.471939\n",
    ">>> step 22700\n",
    "step 22600-22699, precision 0.376335, recall 0.628183, f_score 0.470688\n",
    ">>> step 22800\n",
    "step 22700-22799, precision 0.376016, recall 0.628183, f_score 0.470439\n",
    ">>> step 22900\n",
    "step 22800-22899, precision 0.375968, recall 0.629473, f_score 0.470762\n",
    ">>> step 23000\n",
    "step 22900-22999, precision 0.375210, recall 0.629473, f_score 0.470167\n",
    ">>> step 23100\n",
    "step 23000-23099, precision 0.374708, recall 0.630191, f_score 0.469973\n",
    ">>> step 23200\n",
    "step 23100-23199, precision 0.376144, recall 0.631564, f_score 0.471484\n",
    ">>> step 23300\n",
    "step 23200-23299, precision 0.376630, recall 0.632834, f_score 0.472219\n",
    ">>> step 23400\n",
    "step 23300-23399, precision 0.375021, recall 0.632834, f_score 0.470952\n",
    ">>> step 23500\n",
    "step 23400-23499, precision 0.374286, recall 0.633527, f_score 0.470564\n",
    ">>> step 23600\n",
    "step 23500-23599, precision 0.374068, recall 0.634763, f_score 0.470732\n",
    ">>> step 23700\n",
    "step 23600-23699, precision 0.373814, recall 0.635494, f_score 0.470731\n",
    ">>> step 23800\n",
    "step 23700-23799, precision 0.372496, recall 0.635494, f_score 0.469685\n",
    ">>> step 23900\n",
    "step 23800-23899, precision 0.372906, recall 0.635918, f_score 0.470127\n",
    ">>> step 24000\n",
    "step 23900-23999, precision 0.372976, recall 0.634865, f_score 0.469894\n",
    ">>> step 24100\n",
    "step 24000-24099, precision 0.373713, recall 0.634750, f_score 0.470448\n",
    ">>> step 24200\n",
    "step 24100-24199, precision 0.373123, recall 0.634750, f_score 0.469979\n",
    ">>> step 24300\n",
    "step 24200-24299, precision 0.374528, recall 0.635369, f_score 0.471263\n",
    ">>> step 24400\n",
    "step 24300-24399, precision 0.374765, recall 0.635397, f_score 0.471458"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"test %.2f\"%0.12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "for i in range(len(cubes_set)):\n",
    "    if (cubes_set[i][\"file\"]) == \"PX303-Fg001-V-C01-R01_TEAR_8X5_PIECE_2X4\":\n",
    "        plt.imshow(cubes_set[i][\"cube\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "for i in range(len(cubes_set)):\n",
    "    if (cubes_set[i][\"file\"]) == \"PX303-Fg001-V-C01-R01_TEAR_8X5_PIECE_4X4\":\n",
    "        plt.imshow(cubes_set[i][\"cube\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# When trained on a ratio of 1:2 non-match:match - and tested on the same the results are excellend\n",
    "*** MATCHED=19065\n",
    "*** NOT MATCHED=8183\n",
    "*** DISCARDED=1979027\n",
    "TOTAL 27300, precision 0.815258, recall 0.979806, f_score 0.889990\n",
    "TOTAL true = 19065\n",
    "TOTAL error rate = 0.169158\n",
    "\n",
    "# When trained on a ratio of 1:2 non-match:match - and tested on 6:1 (much more non-match) the results are poor\n",
    "# coverage is still excellent but precision is bad - means a lot of false-positive\n",
    "# this usually implies that we'll get a lot of trash in the validation set\n",
    "# hence next attempt will be to train on 6:1 and see it the test is better\n",
    "*** MATCHED=19065\n",
    "*** NOT MATCHED=125210\n",
    "*** DISCARDED=1862000\n",
    "precision 0.237849, recall 0.973684, f_score 0.382309\n",
    "\n",
    "# When trained on a ratio of 9:1 non-match:match - and tested on 6:1 the results are poor\n",
    "# coverage is around 70% and precision is 35 - means a lot of false-positive\n",
    "# this usually implies that we'll get a lot of trash in the validation set\n",
    "# hence next attempt will be to train on 6:1 and see it the test is better\n",
    "*** MATCHED=19065\n",
    "*** NOT MATCHED=125210\n",
    "*** DISCARDED=1862000\n",
    "precision 0.355357, recall 0.737720, f_score 0.479663\n",
    "\n",
    "\n",
    "# Test from 0ct-07\n",
    "=== total 144000 match 62339\n",
    "TOTAL 144000, precision 0.286081, recall 0.952518, f_score 0.440009\n",
    "TOTAL true = 18723\n",
    "TOTAL error rate = 0.315236"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "#debug\n",
    "\n",
    "short_name = \"PX303-Fg006-V-C02-R02\"\n",
    "# image = read_and_crop(img_name, x_start, x_end, y_start, y_end)\n",
    "\n",
    "cubes_set = []\n",
    "col_cut = 3\n",
    "row_cut = 2\n",
    "pieces = []\n",
    "piece = {}\n",
    "image = img.imread(ROOT_FOLDER + \"test_fragments/PX303Fg006_3X2_1X1.jpg\")\n",
    "image = exposure.equalize_adapthist(image, clip_limit=0.03)\n",
    "piece[\"cut\"] = image[:,:,0]\n",
    "piece[\"col\"] = 1\n",
    "piece[\"row\"] = 1\n",
    "piece[\"col_px\"] = 0\n",
    "piece[\"row_px\"] = 0\n",
    "pieces.append(piece)\n",
    "piece = {}\n",
    "image = img.imread(ROOT_FOLDER + \"test_fragments/PX303Fg006_3X2_2X1.jpg\")\n",
    "image = exposure.equalize_adapthist(image, clip_limit=0.03)\n",
    "piece[\"cut\"] = image[:,:,0]\n",
    "piece[\"col\"] = 2\n",
    "piece[\"row\"] = 1\n",
    "piece[\"col_px\"] = 2300\n",
    "piece[\"row_px\"] = 0\n",
    "pieces.append(piece)\n",
    "\n",
    "for piece in pieces:\n",
    "    # print(\"PRE_PROCESS:::\"+\"PIECE_\"+str(piece[\"col\"])+\"X\"+str(piece[\"row\"]))\n",
    "    fragment_name = short_name + \"_TEAR_\"+str(col_cut)+\"X\"+str(row_cut)+\"_PIECE_\"+str(piece[\"col\"])+\"X\"+str(piece[\"row\"])\n",
    "    fragment_file_name = short_name + \"_\"+str(col_cut)+\"X\"+str(row_cut)+\"_\"+str(piece[\"col\"])+\"X\"+str(piece[\"row\"])\n",
    "    cubes = VAL_slice_TEAR_to_static_slices(fragment_name, piece)\n",
    "    # import pdb; pdb.set_trace()\n",
    "    for cube in cubes:\n",
    "        cube[\"tear\"] = str(col_cut)+\"X\"+str(row_cut)\n",
    "        cube[\"piece_col\"] = piece[\"col\"]\n",
    "        cube[\"piece_row\"] = piece[\"row\"]\n",
    "    # print(\"File: %s >>> cubes: %d\"%(file_, len(cubes)))\n",
    "    cubes_set.extend(cubes)\n",
    "    \n",
    "train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "    NEW_build_train_set_for_binary_labeling(cubes_set, CUBE_SIZE, ROOT_FOLDER + \"train_concats3/\", 1, 7)\n",
    "import pdb; pdb.set_trace()    \n",
    "tf.reset_default_graph()\n",
    "model_tf_deep(250, 1)    \n",
    "validate2_for_cross_validation(train_imgs, train_lbls, is_enriched, ROOT_FOLDER + \"model_binary/tear_model3.ckpt\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2019-10-19T20:03:23.448Z"
    }
   },
   "outputs": [],
   "source": [
    "Out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T10:32:29.152185Z",
     "start_time": "2019-04-03T09:38:59.768856Z"
    },
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "#TRAIN 3\n",
    "cubes_set = pre_process_training(\"PX303-Fg006-V-C02-R02-D08032015-T115622-ML638__006.jpg\", 0, 6200, 0, 4400, max_cols=8, max_rows=4)\n",
    "train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "    NEW_build_train_set_for_binary_labeling(cubes_set, cubes_set, CUBE_SIZE, ROOT_FOLDER + \"train_concats3/\", 2, 7)\n",
    "tf.reset_default_graph()\n",
    "model_tf_deep(250)    \n",
    "train(train_imgs, train_lbls, ROOT_FOLDER + \"models/tear_model1.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T13:33:46.532102Z",
     "start_time": "2019-04-03T12:32:32.371972Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "#TRAIN 2\n",
    "cubes_set = pre_process_training(\"PX303-Fg004-V-C01-R01-D08032015-T110817-ML638__006.jpg\", \\\n",
    "                                 100, -1, 400, -1, max_cols=8, max_rows=4)\n",
    "train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "    NEW_build_train_set_for_binary_labeling(cubes_set, cubes_set, CUBE_SIZE, ROOT_FOLDER + \"train_concats3/\", 2, 7)\n",
    "tf.reset_default_graph()\n",
    "model_tf_deep(250)\n",
    "train(train_imgs, train_lbls, ROOT_FOLDER + \"models/tear_model2.ckpt\", ROOT_FOLDER + \"models/tear_model1.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T13:49:29.331063Z",
     "start_time": "2019-04-03T13:34:34.742578Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# validate 1\n",
    "cubes_set = pre_process_training(\"PX303-Fg001-V-C01-R01-D05032015-T112520-ML638__006.jpg\", max_cols=8, max_rows=4)\n",
    "train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "  NEW_build_train_set_for_binary_labeling(cubes_set, cubes_set, CUBE_SIZE, ROOT_FOLDER + \"train_concats3/\", 1, 5)\n",
    "tf.reset_default_graph()\n",
    "model_tf_deep(250, 1)    \n",
    "validate2_for_cross_validation(train_imgs, train_lbls, is_enriched, ROOT_FOLDER + \"models/tear_model2.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############2_3_val_1\n",
    "=== total 91300 match 12738 /// 1,5 => 1,5\n",
    "TOTAL 91300, precision 0.475899, recall 0.349899, f_score 0.403286\n",
    "TOTAL true = 17325\n",
    "TOTAL error rate = 0.196484\n",
    "=== total 91900 match 12457 /// 1,7 => 1,5\n",
    "TOTAL 91900, precision 0.511760, recall 0.364765, f_score 0.425937\n",
    "TOTAL true = 17477\n",
    "TOTAL error rate = 0.186986\n",
    "=== total 91000 match 20063 /// 2,7 => 1,5\n",
    "TOTAL 91000, precision 0.516672, recall 0.600266, f_score 0.555341\n",
    "TOTAL true = 17269\n",
    "TOTAL error rate = 0.182418\n",
    "=== total 91100 match 19002 /// 3,7 => 1,5\n",
    "TOTAL 91100, precision 0.501474, recall 0.552022, f_score 0.525535\n",
    "TOTAL true = 17262\n",
    "TOTAL error rate = 0.188869\n",
    "=== total 90800 match 15016 /// 4,7 => 1,5\n",
    "TOTAL 90800, precision 0.472030, recall 0.416280, f_score 0.442406\n",
    "TOTAL true = 17027\n",
    "TOTAL error rate = 0.196773\n",
    "=== total 91400 match 23705 /// 2,7 => 1,5\n",
    "TOTAL 91400, precision 0.453449, recall 0.622842, f_score 0.524815\n",
    "TOTAL true = 17258\n",
    "TOTAL error rate = 0.212965"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#############1_2_val_3\n",
    "=== total 536900 match 120543 ## previously - not sure which params\n",
    "TOTAL 536900, precision 0.348216, recall 0.684915, f_score 0.461700\n",
    "TOTAL true = 61285\n",
    "TOTAL error rate = 0.182302\n",
    "=== total 538600 match 208051 /// 2,7 => 1,5\n",
    "TOTAL 538600, precision 0.279508, recall 0.944471, f_score 0.431359\n",
    "TOTAL true = 61571\n",
    "TOTAL error rate = 0.284660"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#############1_3_val_2\n",
    "=== total 329000 match 111791 /// 2,7 => 1,5\n",
    "TOTAL 329000, precision 0.344285, recall 0.757131, f_score 0.473334\n",
    "TOTAL true = 50834\n",
    "TOTAL error rate = 0.260331"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T20:47:01.403854Z",
     "start_time": "2019-04-03T19:52:46.394114Z"
    }
   },
   "outputs": [],
   "source": [
    "#TRAIN 3\n",
    "cubes_set = pre_process_training(\"PX303-Fg006-V-C02-R02-D08032015-T115622-ML638__006.jpg\", 0, 6200, 0, 4400, max_cols=8, max_rows=4)\n",
    "train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "    NEW_build_train_set_for_binary_labeling(cubes_set, cubes_set, CUBE_SIZE, ROOT_FOLDER + \"train_concats3/\", 2, 7)\n",
    "tf.reset_default_graph()\n",
    "model_tf_deep(250)    \n",
    "train(train_imgs, train_lbls, ROOT_FOLDER + \"models/tear_model2.ckpt\", ROOT_FOLDER + \"models/tear_model1.ckpt\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-04T05:24:20.228262Z",
     "start_time": "2019-04-04T04:18:30.422157Z"
    }
   },
   "outputs": [],
   "source": [
    "# validate 2\n",
    "cubes_set = pre_process_training(\"PX303-Fg004-V-C01-R01-D08032015-T110817-ML638__006.jpg\", max_cols=8, max_rows=4)\n",
    "train_imgs, train_lbls, train_x_delta, train_y_delta, is_enriched = \\\n",
    "  NEW_build_train_set_for_binary_labeling(cubes_set, cubes_set, CUBE_SIZE, ROOT_FOLDER + \"train_concats3/\", 1, 5)\n",
    "tf.reset_default_graph()\n",
    "model_tf_deep(250, 1)    \n",
    "validate2_for_cross_validation(train_imgs, train_lbls, is_enriched, ROOT_FOLDER + \"models/tear_model2.ckpt\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-04T05:53:22.125080Z",
     "start_time": "2019-04-04T05:34:40.304989Z"
    }
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "model_tf_deep(250)    \n",
    "train(train_imgs, train_lbls, ROOT_FOLDER + \"models/tear_model3.ckpt\", ROOT_FOLDER + \"models/tear_model2.ckpt\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-02T11:50:30.106916Z",
     "start_time": "2019-10-02T11:50:30.100682Z"
    }
   },
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "can't assign to operator (<ipython-input-3-898be71fcaa8>, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-3-898be71fcaa8>\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m    tensorboard --logdir='/media/1KGB_ILAN/papyrus/models'\u001b[0m\n\u001b[0m                                                          ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m can't assign to operator\n"
     ]
    }
   ],
   "source": [
    "tensorboard --logdir='/media/1KGB_ILAN/papyrus/models'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Raw Cell Format",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}