candidate_3.2.4.01.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\eric\\Anaconda2\\envs\\exts-aml2\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import wget\n",
    "from sklearn.model_selection import train_test_split\n",
    "import tensorflow as tf\n",
    "from training_utils import download_file, get_batches, read_and_decode_single_example, load_validation_data, \\\n",
    "    download_data, evaluate_model, get_training_data, load_weights, flatten, _scale_input_data, augment, _conv2d_batch_norm, standardize, _read_images\n",
    "import argparse\n",
    "from tensorboard import summary as summary_lib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "epochs = 5\n",
    "dataset = 100\n",
    "init_model = None\n",
    "restore_model = None\n",
    "how = \"mask\"\n",
    "action = \"train\"\n",
    "threshold = 0.5\n",
    "freeze = False\n",
    "stop = False\n",
    "contrast = 0\n",
    "normalize = False\n",
    "size = 320\n",
    "weight = 75\n",
    "distort = False\n",
    "version = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps per epoch: 37\n",
      "Number of classes: 2\n",
      "Image crop size: 320\n"
     ]
    }
   ],
   "source": [
    "# figure out how to label the model name\n",
    "if how == \"label\":\n",
    "    model_label = \"l\"\n",
    "elif how == \"normal\":\n",
    "    model_label = \"b\"\n",
    "elif how == \"mask\":\n",
    "    model_label = \"m\"\n",
    "else:\n",
    "    model_label = \"x\"\n",
    "\n",
    "# precalculated pixel mean of images\n",
    "mu = 104.1353\n",
    "\n",
    "# download the data\n",
    "download_data(what=dataset)\n",
    "\n",
    "## config\n",
    "batch_size = 16\n",
    "\n",
    "if dataset != 100:\n",
    "    train_files, total_records = get_training_data(what=dataset)\n",
    "else:\n",
    "    total_records = len(os.listdir(os.path.join(\"data\", \"train_images\")))\n",
    "\n",
    "## Hyperparameters\n",
    "epsilon = 1e-8\n",
    "\n",
    "# learning rate\n",
    "epochs_per_decay = 10\n",
    "decay_factor = 0.85\n",
    "staircase = True\n",
    "\n",
    "# if we are retraining some layers start with smaller learning rate\n",
    "if not stop and not freeze:\n",
    "    starting_rate = 0.001\n",
    "else:\n",
    "    starting_rate = 0.0001\n",
    "\n",
    "# learning rate decay variables\n",
    "steps_per_epoch = int(total_records / batch_size)\n",
    "print(\"Steps per epoch:\", steps_per_epoch)\n",
    "\n",
    "# lambdas\n",
    "lamC = 0.00001\n",
    "lamF = 0.00250\n",
    "\n",
    "# use dropout\n",
    "dropout = True\n",
    "fcdropout_rate = 0.25\n",
    "convdropout_rate = 0.001\n",
    "pooldropout_rate = 0.1\n",
    "\n",
    "if how == \"label\":\n",
    "    num_classes = 5\n",
    "elif how == \"normal\":\n",
    "    num_classes = 2\n",
    "elif how == \"mass\":\n",
    "    num_classes = 3\n",
    "elif how == \"benign\":\n",
    "    num_classes = 3\n",
    "elif how == \"mask\":\n",
    "    num_classes = 2\n",
    "\n",
    "print(\"Number of classes:\", num_classes)\n",
    "print(\"Image crop size:\", size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def _read_images(image_dir, crop_size, scale_by=0.66):\n",
    "    filenames = tf.train.match_filenames_once(image_dir + \"*.png\")\n",
    "    filename_queue = tf.train.string_input_producer(filenames)\n",
    "\n",
    "    # create the reader\n",
    "    image_reader = tf.WholeFileReader()\n",
    "\n",
    "    # Read a whole file from the queue\n",
    "    filename, image_file = image_reader.read(filename_queue)\n",
    "\n",
    "    # decode the image\n",
    "    raw_image = tf.image.decode_png(image_file)\n",
    "    \n",
    "    # figure out size of raw crop by dividing size by scale\n",
    "    image_size = int(crop_size // scale_by)\n",
    "\n",
    "    # crop the image\n",
    "    raw_image = tf.random_crop(raw_image, size=[image_size, image_size, 3])\n",
    "\n",
    "    # resize the image\n",
    "    resized_image = tf.image.resize_images(raw_image, size=[crop_size, crop_size])\n",
    "\n",
    "    # extract the image and label from the channels and resize them for convnet\n",
    "    image = tf.reshape(resized_image[:, :, 0], [crop_size, crop_size, 1])\n",
    "    label = tf.reshape(resized_image[:, :, 1], [crop_size, crop_size, 1])\n",
    "    \n",
    "    label = tf.cast(label, dtype=tf.int32)\n",
    "    \n",
    "    return image, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Scale of 0 disables regularizer.\n",
      "INFO:tensorflow:Scale of 0 disables regularizer.\n",
      "Graph created...\n"
     ]
    }
   ],
   "source": [
    "## Build the graph\n",
    "graph = tf.Graph()\n",
    "\n",
    "model_name = \"model_s3.2.3.01\" + model_label + \".\" + str(dataset) + str(version)\n",
    "## Change Log\n",
    "# 0.0.0.4 - increase pool3 to 3x3 with stride 3\n",
    "# 0.0.0.6 - reduce pool 3 stride back to 2\n",
    "# 0.0.0.7 - reduce lambda for l2 reg\n",
    "# 0.0.0.8 - increase conv1 to 7x7 stride 2\n",
    "# 0.0.0.9 - disable per image normalization\n",
    "# 0.0.0.10 - commented out batch norm in conv layers, added conv4 and changed stride of convs to 1, increased FC lambda\n",
    "# 0.0.0.11 - turn dropout for conv layers on\n",
    "# 0.0.0.12 - added batch norm after pooling layers, increase pool dropout, decrease conv dropout, added extra conv layer to reduce data dimensionality\n",
    "# 0.0.0.13 - added precision and f1 summaries\n",
    "# 0.0.0.14 - fixing batch normalization, I don't think it's going to work after each pool\n",
    "# 0.0.0.15 - reduced xentropy weighting term\n",
    "# 0.0.0.17 - replaced initial 5x5 conv layers with 3 3x3 layers\n",
    "# 0.0.0.18 - changed stride of first conv to 2 from 1\n",
    "# 0.0.0.19 - doubled units in two fc layers\n",
    "# 0.0.0.20 - lowered learning rate, put a batch norm back in\n",
    "# 0.0.0.21 - put all batch norms back in\n",
    "# 0.0.0.22 - increased lambdaC, removed dropout from conv layers\n",
    "# 1.0.0.23 - added extra conv layers\n",
    "# 1.0.0.27 - updates to training code and metrics\n",
    "# 1.0.0.28 - using weighted x-entropy to improve recall\n",
    "# 1.0.0.29 - updated code to work training to classify for multiple classes\n",
    "# 1.0.0.29f - putting weighted x-entropy back\n",
    "# 1.0.0.30b - changed some hyperparameters\n",
    "# 1.0.0.31l - added decision threshold to predictions\n",
    "# 1.0.0.33 - scaling input data\n",
    "# 1.0.0.34 - centering data by 127, not by mean\n",
    "# 1.0.0.35 - not centering data, just scaling it\n",
    "# 2.0.0.35 - turning into fcn\n",
    "# 2.0.0.36 - scaling and centering data?\n",
    "# 3.0.0.36 - adjusting to do segmentation instead of classification\n",
    "# 3.0.0.37 - trying to get this to train faster\n",
    "# 3.0.0.38 - adding tiny value to logits to avoid xe of NaN\n",
    "# 3.0.0.39 - doing metrics per pixel instead of per image\n",
    "# 3.0.0.40 - adjusted graph so we can do online data augmentation and labels will be transformed in same way as images\n",
    "# 3.1.0.40 - adding some layers back in that were previously removed to take more advantage of pre-trained model\n",
    "# 3.1.0.41 - changed skip connections to try to make it a bit more stable\n",
    "# 3.1.0.42 - changed one more skip connection\n",
    "# 3.1.0.43 - trying to not restore batch norm to see if that helps with NaN at test time\n",
    "# 3.1.0.44 - increased size of upconv filters to try to reduce patchiness of result, removed fc layer 3 as it was losing a lot of data\n",
    "# 3.1.0.45 - adding some dropout to try to regularize\n",
    "# 3.2.0.45 - restructuring to accept 320x320 images as input\n",
    "# 3.2.0.46 - increased sizes of upsample filters\n",
    "# 3.2.0.47 - changed number of filters again to speed up training\n",
    "# 3.2.1.48 - adding extra skip connection to try to get better predictions\n",
    "# 3.2.1.49 - renamed one upconv layer so they can be isolated and trained\n",
    "# 3.2.2.01 - tweaking the upsampling layers\n",
    "# 3.2.3.01 - going to train from scratch so adding some extras layers and such\n",
    "# 3.2.4.01 - switching from tf records to reading entire images and taking random crops for more training data\n",
    "\n",
    "with graph.as_default():\n",
    "    training = tf.placeholder(dtype=tf.bool, name=\"is_training\")\n",
    "    is_testing = tf.placeholder(dtype=bool, shape=(), name=\"is_testing\")\n",
    "\n",
    "    # create global step for decaying learning rate\n",
    "    global_step = tf.Variable(0, trainable=False)\n",
    "\n",
    "    learning_rate = tf.train.exponential_decay(starting_rate,\n",
    "                                               global_step,\n",
    "                                               steps_per_epoch * epochs_per_decay,\n",
    "                                               decay_factor,\n",
    "                                               staircase=staircase)\n",
    "\n",
    "    with tf.name_scope('inputs') as scope:\n",
    "        with tf.device('/cpu:0'):\n",
    "            image, label = _read_images(\"./data/train_images/\", size, scale_by=0.66)\n",
    "            X_def, y_def = tf.train.batch([image, label], batch_size=batch_size)\n",
    "\n",
    "            # image, label = read_and_decode_single_example(train_files, label_type=how, normalize=False, distort=False, size=640)\n",
    "            # X_def, y_def = tf.train.shuffle_batch([image, label], batch_size=batch_size, capacity=2000, seed=None, min_after_dequeue=1000)\n",
    "\n",
    "            # Placeholders\n",
    "            X = tf.placeholder_with_default(X_def, shape=[None, size, size, 1])\n",
    "            y = tf.placeholder_with_default(y_def, shape=[None, size, size, 1])\n",
    "\n",
    "            X_fl = tf.cast(X, tf.float32)\n",
    "\n",
    "            # optional online data augmentation\n",
    "            if distort:\n",
    "                X_dis, y_adj = augment(X_fl, y, horizontal_flip=True, augment_labels=True, vertical_flip=True, mixup=0)\n",
    "            else:\n",
    "                y_adj = y\n",
    "                X_dis = X_fl\n",
    "\n",
    "            # cast to float and scale input data\n",
    "            X_adj = _scale_input_data(X_dis, contrast=contrast, mu=127.0, scale=255.0)\n",
    "\n",
    "    # Convolutional layer 1\n",
    "    with tf.name_scope('conv1') as scope:\n",
    "        conv1 = tf.layers.conv2d(\n",
    "            X_adj,\n",
    "            filters=32,\n",
    "            kernel_size=(3, 3),\n",
    "            strides=(2, 2),\n",
    "            padding='SAME',\n",
    "            activation=None,\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=100),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv1'\n",
    "        )\n",
    "\n",
    "        conv1 = tf.layers.batch_normalization(\n",
    "            conv1,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            fused=True,\n",
    "            name='bn1'\n",
    "        )\n",
    "\n",
    "        # apply relu\n",
    "        conv1_bn_relu = tf.nn.relu(conv1, name='relu1')\n",
    "\n",
    "    with tf.name_scope('conv1.1') as scope:\n",
    "        conv11 = tf.layers.conv2d(\n",
    "            conv1_bn_relu,\n",
    "            filters=32,\n",
    "            kernel_size=(3, 3),\n",
    "            strides=(1, 1),\n",
    "            padding='SAME',\n",
    "            activation=None,\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=101),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv1.1'\n",
    "        )\n",
    "\n",
    "        conv11 = tf.layers.batch_normalization(\n",
    "            conv11,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            fused=True,\n",
    "            name='bn1.1'\n",
    "        )\n",
    "\n",
    "        # apply relu\n",
    "        conv11 = tf.nn.relu(conv11, name='relu1.1')\n",
    "\n",
    "\n",
    "    with tf.name_scope('conv1.2') as scope:\n",
    "        conv12 = tf.layers.conv2d(\n",
    "            conv11,\n",
    "            filters=32,\n",
    "            kernel_size=(3, 3),\n",
    "            strides=(1, 1),\n",
    "            padding='SAME',\n",
    "            activation=None,\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=1101),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv1.2'\n",
    "        )\n",
    "\n",
    "        conv12 = tf.layers.batch_normalization(\n",
    "            conv12,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            fused=True,\n",
    "            name='bn1.2'\n",
    "        )\n",
    "\n",
    "        # apply relu\n",
    "        conv12_relu = tf.nn.relu(conv12, name='relu1.1')\n",
    "\n",
    "    # Max pooling layer 1\n",
    "    with tf.name_scope('pool1') as scope:\n",
    "        pool1 = tf.layers.max_pooling2d(\n",
    "            conv12_relu,\n",
    "            pool_size=(3, 3),\n",
    "            strides=(2, 2),\n",
    "            padding='SAME',\n",
    "            name='pool1'\n",
    "        )\n",
    "\n",
    "        # optional dropout\n",
    "        if dropout:\n",
    "            pool1 = tf.layers.dropout(pool1, rate=pooldropout_rate, seed=103, training=training)\n",
    "\n",
    "    # Convolutional layer 2\n",
    "    with tf.name_scope('conv2.1') as scope:\n",
    "        conv2 = tf.layers.conv2d(\n",
    "            pool1,\n",
    "            filters=64,\n",
    "            kernel_size=(3, 3),\n",
    "            strides=(1, 1),\n",
    "            padding='SAME',\n",
    "            activation=None,\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=104),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv2.1'\n",
    "        )\n",
    "\n",
    "        conv2 = tf.layers.batch_normalization(\n",
    "            conv2,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            fused=True,\n",
    "            name='bn2.1'\n",
    "        )\n",
    "\n",
    "        # apply relu\n",
    "        conv2 = tf.nn.relu(conv2, name='relu2.1')\n",
    "\n",
    "    # Convolutional layer 2\n",
    "    with tf.name_scope('conv2.2') as scope:\n",
    "        conv22 = tf.layers.conv2d(\n",
    "            conv2,\n",
    "            filters=64,\n",
    "            kernel_size=(3, 3),\n",
    "            strides=(1, 1),\n",
    "            padding='SAME',\n",
    "            activation=None,\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=1104),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv2.2'\n",
    "        )\n",
    "\n",
    "        conv22 = tf.layers.batch_normalization(\n",
    "            conv22,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            fused=True,\n",
    "            name='bn2.2'\n",
    "        )\n",
    "\n",
    "        # apply relu\n",
    "        conv22_relu = tf.nn.relu(conv22, name='relu2.2')\n",
    "\n",
    "    # Max pooling layer 2\n",
    "    with tf.name_scope('pool2') as scope:\n",
    "        pool2 = tf.layers.max_pooling2d(\n",
    "            conv22_relu,\n",
    "            pool_size=(2, 2),\n",
    "            strides=(2, 2),\n",
    "            padding='SAME',\n",
    "            name='pool2'\n",
    "        )\n",
    "\n",
    "        # optional dropout\n",
    "        if dropout:\n",
    "            pool2 = tf.layers.dropout(pool2, rate=pooldropout_rate, seed=106, training=training)\n",
    "\n",
    "    # Convolutional layer 3\n",
    "    with tf.name_scope('conv3.1') as scope:\n",
    "        conv3 = tf.layers.conv2d(\n",
    "            pool2,\n",
    "            filters=128,\n",
    "            kernel_size=(3, 3),\n",
    "            strides=(1, 1),\n",
    "            padding='SAME',\n",
    "            activation=None,\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=107),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv3.1'\n",
    "        )\n",
    "\n",
    "        conv3 = tf.layers.batch_normalization(\n",
    "            conv3,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            fused=True,\n",
    "            name='bn3.1'\n",
    "        )\n",
    "\n",
    "        # apply relu\n",
    "        conv3 = tf.nn.relu(conv3, name='relu3.1')\n",
    "\n",
    "    # Convolutional layer 3\n",
    "    with tf.name_scope('conv3.2') as scope:\n",
    "        conv32 = tf.layers.conv2d(\n",
    "            conv3,\n",
    "            filters=128,\n",
    "            kernel_size=(3, 3),\n",
    "            strides=(1, 1),\n",
    "            padding='SAME',\n",
    "            activation=None,\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=1107),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv3.2'\n",
    "        )\n",
    "\n",
    "        conv32 = tf.layers.batch_normalization(\n",
    "            conv32,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            fused=True,\n",
    "            name='bn3.2'\n",
    "        )\n",
    "\n",
    "        # apply relu\n",
    "        conv32 = tf.nn.relu(conv32, name='relu3.2')\n",
    "\n",
    "    # Max pooling layer 3\n",
    "    with tf.name_scope('pool3') as scope:\n",
    "        pool3 = tf.layers.max_pooling2d(\n",
    "            conv32,\n",
    "            pool_size=(2, 2),\n",
    "            strides=(2, 2),\n",
    "            padding='SAME',\n",
    "            name='pool3'\n",
    "        )\n",
    "\n",
    "        if dropout:\n",
    "            pool3 = tf.layers.dropout(pool3, rate=pooldropout_rate, seed=109, training=training)\n",
    "\n",
    "    # Convolutional layer 4\n",
    "    with tf.name_scope('conv4') as scope:\n",
    "        conv4 = tf.layers.conv2d(\n",
    "            pool3,\n",
    "            filters=256,\n",
    "            kernel_size=(3, 3),\n",
    "            strides=(1, 1),\n",
    "            padding='SAME',\n",
    "            activation=None,\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=110),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv4'\n",
    "        )\n",
    "\n",
    "        conv4 = tf.layers.batch_normalization(\n",
    "            conv4,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            fused=True,\n",
    "            name='bn4'\n",
    "        )\n",
    "\n",
    "        # apply relu\n",
    "        conv4_bn_relu = tf.nn.relu(conv4, name='relu4')\n",
    "\n",
    "    with tf.name_scope('conv4.1') as scope:\n",
    "        conv41 = tf.layers.conv2d(\n",
    "            conv4_bn_relu,\n",
    "            filters=256,\n",
    "            kernel_size=(3, 3),\n",
    "            strides=(1, 1),\n",
    "            padding='SAME',\n",
    "            activation=None,\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=1710),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv4.1'\n",
    "        )\n",
    "\n",
    "        conv41 = tf.layers.batch_normalization(\n",
    "            conv41,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            fused=True,\n",
    "            name='bn4.1'\n",
    "        )\n",
    "\n",
    "        # apply relu\n",
    "        conv41_bn_relu = tf.nn.relu(conv41, name='relu4.1')\n",
    "\n",
    "    # Max pooling layer 4\n",
    "    with tf.name_scope('pool4') as scope:\n",
    "            pool4 = tf.layers.max_pooling2d(\n",
    "                conv41_bn_relu,\n",
    "                pool_size=(2, 2),\n",
    "                strides=(2, 2),\n",
    "                padding='SAME',\n",
    "                name='pool4'\n",
    "            )\n",
    "\n",
    "            if dropout:\n",
    "                pool4 = tf.layers.dropout(pool4, rate=pooldropout_rate, seed=112, training=training)\n",
    "\n",
    "    # Convolutional layer 4\n",
    "    with tf.name_scope('conv5') as scope:\n",
    "        conv5 = tf.layers.conv2d(\n",
    "            pool4,\n",
    "            filters=512,\n",
    "            kernel_size=(3, 3),\n",
    "            strides=(1, 1),\n",
    "            padding='SAME',\n",
    "            activation=None,\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=113),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv5'\n",
    "        )\n",
    "\n",
    "        conv5 = tf.layers.batch_normalization(\n",
    "            conv5,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            fused=True,\n",
    "            name='bn5'\n",
    "        )\n",
    "\n",
    "        # apply relu\n",
    "        conv5_bn_relu = tf.nn.relu(conv5, name='relu5')\n",
    "\n",
    "    with tf.name_scope('conv5.1') as scope:\n",
    "        conv51 = tf.layers.conv2d(\n",
    "            conv5_bn_relu,\n",
    "            filters=512,\n",
    "            kernel_size=(3, 3),\n",
    "            strides=(1, 1),\n",
    "            padding='SAME',\n",
    "            activation=None,\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=1193),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv5.1'\n",
    "        )\n",
    "\n",
    "        conv51 = tf.layers.batch_normalization(\n",
    "            conv51,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            fused=True,\n",
    "            name='bn5.1'\n",
    "        )\n",
    "\n",
    "        # apply relu\n",
    "        conv51_bn_relu = tf.nn.relu(conv51, name='relu5.1')\n",
    "\n",
    "    # Max pooling layer 5\n",
    "    with tf.name_scope('pool5') as scope:\n",
    "        pool5 = tf.layers.max_pooling2d(\n",
    "            conv51_bn_relu,\n",
    "            pool_size=(2, 2),\n",
    "            strides=(2, 2),\n",
    "            padding='SAME',\n",
    "            name='pool5'\n",
    "        )\n",
    "\n",
    "        if dropout:\n",
    "            pool5 = tf.layers.dropout(pool5, rate=pooldropout_rate, seed=115, training=training)\n",
    "\n",
    "    if stop:\n",
    "        pool5 = tf.stop_gradient(pool5, name=\"pool5_freeze\")\n",
    "\n",
    "    fc1 = _conv2d_batch_norm(pool5, 2048, kernel_size=(5, 5), stride=(5, 5), training=training, epsilon=1e-8,\n",
    "                             padding=\"VALID\", seed=1013, lambd=lamC, name=\"fc_1\")\n",
    "\n",
    "    fc1= tf.layers.dropout(fc1, rate=fcdropout_rate, seed=11537, training=training)\n",
    "\n",
    "    fc2 = _conv2d_batch_norm(fc1, 2048, kernel_size=(1, 1), stride=(1, 1), training=training, epsilon=1e-8,\n",
    "                             padding=\"VALID\", seed=1014, lambd=lamC, name=\"fc_2\")\n",
    "\n",
    "    fc2 = tf.layers.dropout(fc2, rate=fcdropout_rate, seed=12537, training=training)\n",
    "\n",
    "    # upsample back to 5x5\n",
    "    with tf.name_scope('up_conv1') as scope:\n",
    "        unpool1 = tf.layers.conv2d_transpose(\n",
    "            fc2,\n",
    "            filters=512,\n",
    "            kernel_size=(5, 5),\n",
    "            strides=(5, 5),\n",
    "            padding='SAME',\n",
    "            activation=None,\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=11435),\n",
    "            kernel_regularizer=None,\n",
    "            name='up_conv1'\n",
    "        )\n",
    "\n",
    "        unpool1 = unpool1 + pool5\n",
    "\n",
    "    # upsample to 10x10\n",
    "    with tf.name_scope('up_conv2') as scope:\n",
    "        unpool2 = tf.layers.conv2d_transpose(\n",
    "            unpool1,\n",
    "            filters=256,\n",
    "            kernel_size=(4, 4),\n",
    "            strides=(2, 2),\n",
    "            padding='SAME',\n",
    "            activation=None,\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=11435),\n",
    "            kernel_regularizer=None,\n",
    "            name='up_conv2'\n",
    "        )\n",
    "\n",
    "        # skip connection\n",
    "        unpool2 = unpool2 + pool4\n",
    "\n",
    "        unpool2 = tf.nn.elu(unpool2, name=\"up_conv2_relu\")\n",
    "\n",
    "        if dropout:\n",
    "            unpool2 = tf.layers.dropout(unpool2, rate=convdropout_rate, seed=13537, training=training)\n",
    "\n",
    "    # upsample to 20x20\n",
    "    with tf.name_scope('up_conv3') as scope:\n",
    "        unpool3 = tf.layers.conv2d_transpose(\n",
    "            unpool2,\n",
    "            filters=128,\n",
    "            kernel_size=(4, 4),\n",
    "            strides=(2, 2),\n",
    "            padding='SAME',\n",
    "            activation=None,\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=19317),\n",
    "            kernel_regularizer=None,\n",
    "            name='up_conv3'\n",
    "        )\n",
    "\n",
    "        # skip connection\n",
    "        unpool3 = unpool3 + pool3\n",
    "\n",
    "        unpool3 = tf.nn.elu(unpool3, name='relu6.5')\n",
    "\n",
    "\n",
    "    # upsample to 40x40\n",
    "    with tf.name_scope('up_conv4') as scope:\n",
    "        unpool4 = tf.layers.conv2d_transpose(\n",
    "            unpool3,\n",
    "            filters=64,\n",
    "            kernel_size=(4, 4),\n",
    "            strides=(2, 2),\n",
    "            padding='SAME',\n",
    "            activation=None,\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=11728),\n",
    "            kernel_regularizer=None,\n",
    "            name='up_conv4'\n",
    "        )\n",
    "\n",
    "        if dropout:\n",
    "            unpool4 = tf.layers.dropout(unpool4, rate=convdropout_rate, seed=14537, training=training)\n",
    "\n",
    "        unpool4 = unpool4 + pool2\n",
    "\n",
    "        unpool4 = tf.nn.elu(unpool4, name='up_relu4')\n",
    "\n",
    "    # upsample to 80x80\n",
    "    with tf.name_scope('up_conv5') as scope:\n",
    "        unpool5 = tf.layers.conv2d_transpose(\n",
    "            unpool4,\n",
    "            filters=32,\n",
    "            kernel_size=(4, 4),\n",
    "            strides=(2, 2),\n",
    "            padding='SAME',\n",
    "            activation=None,\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=11756),\n",
    "            kernel_regularizer=None,\n",
    "            name='up_conv5'\n",
    "        )\n",
    "\n",
    "        if dropout:\n",
    "            unpool5 = tf.layers.dropout(unpool5, rate=pooldropout_rate, seed=14537, training=training)\n",
    "\n",
    "        # skip connection\n",
    "        unpool5 = unpool5 + pool1\n",
    "\n",
    "        # activation\n",
    "        unpool5 = tf.nn.elu(unpool5, name='relu10')\n",
    "\n",
    "    conv6 = _conv2d_batch_norm(unpool5, 16, kernel_size=(3, 3), stride=(1, 1), training=training, lambd=0.0,\n",
    "                               name=\"up_conv6\", activation=\"elu\")\n",
    "\n",
    "    # upsample to 160x160\n",
    "    with tf.name_scope('up_conv7') as scope:\n",
    "        unpool7 = tf.layers.conv2d_transpose(\n",
    "            conv6,\n",
    "            filters=32,\n",
    "            kernel_size=(4, 4),\n",
    "            strides=(2, 2),\n",
    "            padding='SAME',\n",
    "            activation=None,\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=11756),\n",
    "            kernel_regularizer=None,\n",
    "            name='up_conv7'\n",
    "        )\n",
    "\n",
    "        if dropout:\n",
    "            unpool7 = tf.layers.dropout(unpool7, rate=pooldropout_rate, seed=14557, training=training)\n",
    "\n",
    "            unpool7 = unpool7 + conv1\n",
    "\n",
    "        # activation\n",
    "        unpool7 = tf.nn.elu(unpool7, name='relu11')\n",
    "\n",
    "    # one last conv layer before logits\n",
    "    conv8 = _conv2d_batch_norm(unpool7, 16, kernel_size=(3,3), stride=(1,1), training=training, lambd=0.0, name=\"up_conv8\", activation=\"elu\")\n",
    "\n",
    "    # upsample to 320x320\n",
    "    with tf.name_scope('logits') as scope:\n",
    "        logits = tf.layers.conv2d_transpose(\n",
    "            conv8,\n",
    "            filters=2,\n",
    "            kernel_size=(4, 4),\n",
    "            strides=(2, 2),\n",
    "            padding='SAME',\n",
    "            activation=None,\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=11793),\n",
    "            kernel_regularizer=None,\n",
    "            name='logits'\n",
    "        )\n",
    "\n",
    "    # get the fully connected variables so we can only train them when retraining the network\n",
    "    fc_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, \"up_\")\n",
    "    tr_logits =  tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, \"logits\")\n",
    "\n",
    "    with tf.variable_scope('conv1', reuse=True):\n",
    "        conv_kernels1 = tf.get_variable('kernel')\n",
    "        kernel_transposed = tf.transpose(conv_kernels1, [3, 0, 1, 2])\n",
    "\n",
    "    with tf.variable_scope('visualization'):\n",
    "        tf.summary.image('conv1/filters', kernel_transposed, max_outputs=32, collections=[\"kernels\"])\n",
    "\n",
    "    # This will weight the positive examples higher so as to improve recall\n",
    "    weights = tf.multiply(tf.cast(weight, tf.float32), tf.cast(tf.greater(y_adj, 0), tf.float32)) + 1\n",
    "\n",
    "    mean_ce = tf.reduce_mean(tf.losses.sparse_softmax_cross_entropy(labels=y_adj, logits=(logits + 1e-10), weights=weights))\n",
    "\n",
    "    # Add in l2 loss\n",
    "    loss = mean_ce + tf.losses.get_regularization_loss()\n",
    "\n",
    "    # Adam optimizer\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "\n",
    "    # Minimize cross-entropy - freeze certain layers depending on input\n",
    "    if freeze:\n",
    "        train_op = optimizer.minimize(loss, global_step=global_step, var_list=fc_vars + tr_logits)\n",
    "    else:\n",
    "        train_op = optimizer.minimize(loss, global_step=global_step)\n",
    "\n",
    "    # predictions = tf.reshape(tf.argmax(logits, axis=-1, output_type=tf.int32), (-1, 320,320))\n",
    "    # if we reshape the predictions it won't work with images of other sizes\n",
    "    predictions = tf.argmax(logits, axis=-1, output_type=tf.int32)\n",
    "\n",
    "    # squash the predictions into a per image prediction - negative images will have a max of 0\n",
    "    pred_sum = tf.reduce_sum(predictions, axis=[1, 2])\n",
    "    image_predictions = tf.cast(tf.greater(pred_sum, (size * size // 750)), dtype=tf.uint8)\n",
    "    image_truth = tf.reduce_max(y_adj, axis=[1, 2])\n",
    "    # image_predictions = tf.reduce_max(predictions, axis=[1,2,3])\n",
    "\n",
    "    # set a threshold on the predictions so we ignore images with only a few positive pixels\n",
    "    pred_sum = tf.reduce_sum(predictions, axis=[1, 2])\n",
    "    image_predictions = tf.cast(tf.greater(pred_sum, (size*size//750)),dtype=tf.uint8)\n",
    "\n",
    "    # get the accuracy per pixel\n",
    "    accuracy, acc_op = tf.metrics.accuracy(\n",
    "        labels=y_adj,\n",
    "        predictions=predictions,\n",
    "        updates_collections=[tf.GraphKeys.UPDATE_OPS, 'metrics_ops'],\n",
    "        name=\"accuracy\",\n",
    "    )\n",
    "    # calculate recall and precision per pixel\n",
    "    recall, rec_op = tf.metrics.recall(labels=y_adj, predictions=predictions,\n",
    "                                       updates_collections=[tf.GraphKeys.UPDATE_OPS, 'metrics_ops'],\n",
    "                                       name=\"pixel_recall\")\n",
    "    precision, prec_op = tf.metrics.precision(labels=y_adj, predictions=predictions,\n",
    "                                              updates_collections=[tf.GraphKeys.UPDATE_OPS, 'metrics_ops'],\n",
    "                                              name=\"pixel_precision\")\n",
    "\n",
    "    f1_score = 2 * ((precision * recall) / (precision + recall))\n",
    "\n",
    "    # per image metrics\n",
    "    image_accuracy, image_acc_op = tf.metrics.accuracy(\n",
    "        labels=image_truth,\n",
    "        predictions=image_predictions,\n",
    "        updates_collections=[tf.GraphKeys.UPDATE_OPS, 'metrics_ops'],\n",
    "        name=\"image_accuracy\",\n",
    "    )\n",
    "\n",
    "    image_recall, image_rec_op = tf.metrics.recall(labels=image_truth, predictions=image_predictions,\n",
    "                                       updates_collections=[tf.GraphKeys.UPDATE_OPS, 'metrics_ops'], name=\"image_recall\")\n",
    "    image_precision, image_prec_op = tf.metrics.precision(labels=image_truth, predictions=image_predictions,\n",
    "                                              updates_collections=[tf.GraphKeys.UPDATE_OPS, 'metrics_ops'], name=\"image_precision\")\n",
    "\n",
    "\n",
    "    tf.summary.scalar('recall_1', recall, collections=[\"summaries\"])\n",
    "    tf.summary.scalar('recall_per_image', image_recall, collections=[\"summaries\"])\n",
    "    tf.summary.scalar('precision_1', precision, collections=[\"summaries\"])\n",
    "    tf.summary.scalar('precision_per_image', image_precision, collections=[\"summaries\"])\n",
    "    tf.summary.scalar('f1_score', f1_score, collections=[\"summaries\"])\n",
    "\n",
    "    # Create summary hooks\n",
    "    tf.summary.scalar('accuracy', accuracy, collections=[\"summaries\"])\n",
    "    tf.summary.scalar('accuracy_per_image', image_accuracy, collections=[\"summaries\"])\n",
    "    tf.summary.scalar('cross_entropy', mean_ce, collections=[\"summaries\"])\n",
    "    tf.summary.scalar('learning_rate', learning_rate, collections=[\"summaries\"])\n",
    "\n",
    "    # add this so that the batch norm gets run\n",
    "    extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "\n",
    "    # collect the metrics ops into one op so we can run that at test time\n",
    "    metrics_op = tf.get_collection('metrics_ops')\n",
    "\n",
    "    # Merge all the summaries\n",
    "    merged = tf.summary.merge_all(\"summaries\")\n",
    "    kernel_summaries = tf.summary.merge_all(\"kernels\")\n",
    "\n",
    "    print(\"Graph created...\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## CONFIGURE OPTIONS\n",
    "if init_model is not None:\n",
    "    if os.path.exists(os.path.join(\"model\", init_model + '.ckpt.index')):\n",
    "        init = False\n",
    "    else:\n",
    "        init = True\n",
    "elif restore_model is not None:\n",
    "    if os.path.exists(os.path.join(\"model\", restore_model + '.ckpt.index')):\n",
    "        init = False\n",
    "    else:\n",
    "        init = True\n",
    "else:\n",
    "    if os.path.exists(os.path.join(\"model\", model_name + '.ckpt.index')):\n",
    "        init = False\n",
    "    else:\n",
    "        init = True\n",
    "\n",
    "meta_data_every = 1\n",
    "log_to_tensorboard = True\n",
    "print_every = 5  # how often to print metrics\n",
    "checkpoint_every = 1  # how often to save model in epochs\n",
    "use_gpu = False  # whether or not to use the GPU\n",
    "print_metrics = True  # whether to print or plot metrics, if False a plot will be created and updated every epoch\n",
    "\n",
    "# Initialize metrics or load them from disk if they exist\n",
    "if os.path.exists(os.path.join(\"data\", model_name + \"train_acc.npy\")):\n",
    "    train_acc_values = np.load(os.path.join(\"data\", model_name + \"train_acc.npy\")).tolist()\n",
    "else:\n",
    "    train_acc_values = []\n",
    "\n",
    "if os.path.exists(os.path.join(\"data\", model_name + \"train_loss.npy\")):\n",
    "    train_cost_values = np.load(os.path.join(\"data\", model_name + \"train_loss.npy\")).tolist()\n",
    "else:\n",
    "    train_cost_values = []\n",
    "\n",
    "if os.path.exists(os.path.join(\"data\", model_name + \"train_lr.npy\")):\n",
    "    train_lr_values = np.load(os.path.join(\"data\", model_name + \"train_lr.npy\")).tolist()\n",
    "else:\n",
    "    train_lr_values = []\n",
    "\n",
    "if os.path.exists(os.path.join(\"data\", model_name + \"train_recall.npy\")):\n",
    "    train_recall_values = np.load(os.path.join(\"data\", model_name + \"train_recall.npy\")).tolist()\n",
    "else:\n",
    "    train_recall_values = []\n",
    "\n",
    "if os.path.exists(os.path.join(\"data\", model_name + \"cv_acc.npy\")):\n",
    "    valid_acc_values = np.load(os.path.join(\"data\", model_name + \"cv_acc.npy\")).tolist()\n",
    "else:\n",
    "    valid_acc_values = []\n",
    "\n",
    "if os.path.exists(os.path.join(\"data\", model_name + \"cv_loss.npy\")):\n",
    "    valid_cost_values = np.load(os.path.join(\"data\", model_name + \"cv_loss.npy\")).tolist()\n",
    "else:\n",
    "    valid_cost_values = []\n",
    "\n",
    "if os.path.exists(os.path.join(\"data\", model_name + \"cv_recall.npy\")):\n",
    "    valid_recall_values = np.load(os.path.join(\"data\", model_name + \"cv_recall.npy\")).tolist()\n",
    "else:\n",
    "    valid_recall_values = []\n",
    "\n",
    "config = tf.ConfigProto()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing model...\n",
      "Training model model_s3.2.3.01m.100 ...\n"
     ]
    }
   ],
   "source": [
    "## train the model\n",
    "with tf.Session(graph=graph, config=config) as sess:\n",
    "    if log_to_tensorboard:\n",
    "        train_writer = tf.summary.FileWriter('./logs/tr_' + model_name, sess.graph)\n",
    "        test_writer = tf.summary.FileWriter('./logs/te_' + model_name)\n",
    "    \n",
    "    if not print_metrics:\n",
    "        # create a plot to be updated as model is trained\n",
    "        f, ax = plt.subplots(1,4,figsize=(24,5))\n",
    "    \n",
    "    # create the saver\n",
    "    saver = tf.train.Saver()\n",
    "    sess.run(tf.local_variables_initializer())\n",
    "    \n",
    "    # If the model is new initialize variables, else restore the session\n",
    "    if init:\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "        print(\"Initializing model...\")\n",
    "    else:\n",
    "        # if we are initializing with the weights from another model load it\n",
    "        if init_model is not None:\n",
    "            # initialize the global variables\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "\n",
    "            # create the initializer function to initialize the weights\n",
    "            init_fn = load_weights(init_model, exclude=[\"fc3\", \"logits\", \"bn_conv6\", \"up_conv7\",   \"bn_up_conv8\",\"bn_up_conv6\",\"bn_up_conv7\",\"conv_up_conv6\", \"conv_up_conv8\",\"up_conv1\",\"up_conv2\",\"up_conv5\",\"up_conv6\", \"accuracy\", \"up_conv4\", \"up_conv3\", \"global_step\"])\n",
    "\n",
    "            # run the initializer\n",
    "            init_fn(sess)\n",
    "\n",
    "            ## reload some weights from one checkpoint and some from a different one\n",
    "            # init_fn = load_weights(\"model_s3.2.1.48m.12\", exclude=[\"conv_up_conv7\", \"bn_up_conv7\", \"fc3\", \"conv5\", \"accuracy\", \"bn5\"])\n",
    "            # init_fn(sess)\n",
    "            #\n",
    "            # init_fn = load_weights(\"model_s3.2.0.47m.12\", include=[\"conv5\", \"bn5\"])\n",
    "            # init_fn(sess)\n",
    "            #\n",
    "            # # reset the global step\n",
    "            initial_global_step = global_step.assign(0)\n",
    "            sess.run(initial_global_step)\n",
    "\n",
    "            print(\"Initializing weights from model\", init_model)\n",
    "\n",
    "            # reset init model so we don't do this again\n",
    "            init_model = None\n",
    "        elif restore_model is not None:\n",
    "            saver.restore(sess, './model/' + restore_model + '.ckpt')\n",
    "            print(\"Restoring model\", restore_model)\n",
    "\n",
    "            initial_global_step = global_step.assign(0)\n",
    "            sess.run(initial_global_step)\n",
    "        # otherwise load this model\n",
    "        else:\n",
    "            saver.restore(sess, './model/' + model_name + '.ckpt')\n",
    "            print(\"Restoring model\", model_name)\n",
    "\n",
    "    # start the queue runners\n",
    "    coord = tf.train.Coordinator()\n",
    "    threads = tf.train.start_queue_runners(coord=coord)\n",
    "\n",
    "    # if we are training the model\n",
    "    if action == \"train\":\n",
    "\n",
    "        print(\"Training model\", model_name, \"...\")\n",
    "\n",
    "        for epoch in range(epochs):\n",
    "            sess.run(tf.local_variables_initializer())\n",
    "\n",
    "            # Accuracy values (train) after each batch\n",
    "            batch_acc = []\n",
    "            batch_cost = []\n",
    "            batch_recall = []\n",
    "\n",
    "            for i in range(steps_per_epoch):\n",
    "                # create the metadata\n",
    "                run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)\n",
    "                run_metadata = tf.RunMetadata()\n",
    "\n",
    "                # Run training op and update ops\n",
    "                if (i % 50 != 0) or (i == 0):\n",
    "                    # log the kernel images once per epoch\n",
    "                    if (i == (steps_per_epoch - 1)) and log_to_tensorboard:\n",
    "                        _, _, image_summary, step = sess.run(\n",
    "                            [train_op, extra_update_ops, kernel_summaries, global_step],\n",
    "                            feed_dict={\n",
    "                                training: True,\n",
    "                            },\n",
    "                            options=run_options,\n",
    "                            run_metadata=run_metadata)\n",
    "\n",
    "                        # write the summary\n",
    "                        train_writer.add_summary(image_summary, step)\n",
    "                    else:\n",
    "                        _, _, step = sess.run(\n",
    "                            [train_op, extra_update_ops, global_step],\n",
    "                                feed_dict={\n",
    "                                    training: True,\n",
    "                                },\n",
    "                                options=run_options,\n",
    "                                run_metadata=run_metadata)\n",
    "\n",
    "                # every 50th step get the metrics\n",
    "                else:\n",
    "                    _, _, precision_value, summary, acc_value, cost_value, recall_value, step, lr = sess.run(\n",
    "                        [train_op, extra_update_ops, prec_op, merged, accuracy, mean_ce, rec_op, global_step, learning_rate],\n",
    "                        feed_dict={\n",
    "                            training: True,\n",
    "                        },\n",
    "                        options=run_options,\n",
    "                        run_metadata=run_metadata)\n",
    "\n",
    "                    # Save accuracy (current batch)\n",
    "                    batch_acc.append(acc_value)\n",
    "                    batch_cost.append(cost_value)\n",
    "                    batch_recall.append(recall_value)\n",
    "\n",
    "                    # log the summaries to tensorboard every 50 steps\n",
    "                    if log_to_tensorboard:\n",
    "                        # write the summary\n",
    "                        train_writer.add_summary(summary, step)\n",
    "\n",
    "                # only log the meta data once per epoch\n",
    "                if i == 1:\n",
    "                    train_writer.add_run_metadata(run_metadata, 'step %d' % step)\n",
    "\n",
    "            # save checkpoint every nth epoch\n",
    "            if (epoch % checkpoint_every == 0):\n",
    "                print(\"Saving checkpoint\")\n",
    "                save_path = saver.save(sess, './model/' + model_name + '.ckpt')\n",
    "\n",
    "                # Now that model is saved set init to false so we reload it next time\n",
    "                init = False\n",
    "\n",
    "            # init batch arrays\n",
    "            batch_cv_acc = []\n",
    "            batch_cv_loss = []\n",
    "            batch_cv_recall = []\n",
    "\n",
    "            # initialize the local variables so we have metrics only on the evaluation\n",
    "            sess.run(tf.local_variables_initializer())\n",
    "\n",
    "            print(\"Evaluating model...\")\n",
    "            # load the test data\n",
    "            X_cv, y_cv = load_validation_data(percentage=1, how=how, which=dataset)\n",
    "\n",
    "            # evaluate the test data\n",
    "            for X_batch, y_batch in get_batches(X_cv, y_cv, batch_size, distort=False):\n",
    "                _, valid_acc, valid_recall, valid_cost = sess.run(\n",
    "                    [metrics_op, accuracy, recall, mean_ce],\n",
    "                    feed_dict={\n",
    "                        X: X_batch,\n",
    "                        y: y_batch,\n",
    "                        training: False\n",
    "                    })\n",
    "\n",
    "                batch_cv_acc.append(valid_acc)\n",
    "                batch_cv_loss.append(valid_cost)\n",
    "                batch_cv_recall.append(valid_recall)\n",
    "\n",
    "            # Write average of validation data to summary logs\n",
    "            if log_to_tensorboard:\n",
    "                # evaluate once more to get the summary, which will then be written to tensorboard\n",
    "                summary, cv_accuracy = sess.run(\n",
    "                    [merged, accuracy],\n",
    "                    feed_dict={\n",
    "                        X: X_cv[0:2],\n",
    "                        y: y_cv[0:2],\n",
    "                        training: False\n",
    "                    })\n",
    "\n",
    "            test_writer.add_summary(summary, step)\n",
    "            # test_writer.add_summary(other_summaries, step)\n",
    "            step += 1\n",
    "\n",
    "            # delete the test data to save memory\n",
    "            del (X_cv)\n",
    "            del (y_cv)\n",
    "\n",
    "            print(\"Done evaluating...\")\n",
    "\n",
    "            # take the mean of the values to add to the metrics\n",
    "            valid_acc_values.append(np.mean(batch_cv_acc))\n",
    "            train_acc_values.append(np.mean(batch_acc))\n",
    "\n",
    "            valid_cost_values.append(np.mean(batch_cv_loss))\n",
    "            train_cost_values.append(np.mean(batch_cost))\n",
    "\n",
    "            valid_recall_values.append(np.mean(batch_cv_recall))\n",
    "            train_recall_values.append(np.mean(batch_recall))\n",
    "\n",
    "            train_lr_values.append(lr)\n",
    "\n",
    "            # save the metrics\n",
    "            np.save(os.path.join(\"data\", model_name + \"train_acc.npy\"), train_acc_values)\n",
    "            np.save(os.path.join(\"data\", model_name + \"cv_acc.npy\"), valid_acc_values)\n",
    "\n",
    "            np.save(os.path.join(\"data\", model_name + \"train_loss.npy\"), train_cost_values)\n",
    "            np.save(os.path.join(\"data\", model_name + \"cv_loss.npy\"), valid_cost_values)\n",
    "\n",
    "            np.save(os.path.join(\"data\", model_name + \"train_recall.npy\"), train_recall_values)\n",
    "            np.save(os.path.join(\"data\", model_name + \"cv_recall.npy\"), valid_recall_values)\n",
    "\n",
    "            np.save(os.path.join(\"data\", model_name + \"train_lr.npy\"), train_lr_values)\n",
    "\n",
    "            # Print progress every nth epoch to keep output to reasonable amount\n",
    "            if (epoch % print_every == 0):\n",
    "                print(\n",
    "                'Epoch {:02d} - step {} - cv acc: {:.4f} - train acc: {:.3f} (mean)'.format(\n",
    "                    epoch, step, np.mean(batch_cv_acc), np.mean(batch_acc)\n",
    "                ))\n",
    "\n",
    "    # stop the coordinator\n",
    "    coord.request_stop()\n",
    "\n",
    "    # Wait for threads to stop\n",
    "    coord.join(threads)\n",
    "\n",
    "    sess.run(tf.local_variables_initializer())\n",
    "    print(\"Evaluating on test data\")\n",
    "\n",
    "    # evaluate the test data\n",
    "    X_te, y_te = load_validation_data(how=how, data=\"test\", which=dataset)\n",
    "\n",
    "    test_accuracy = []\n",
    "    test_recall = []\n",
    "    test_predictions = []\n",
    "    ground_truth = []\n",
    "    for X_batch, y_batch in get_batches(X_te, y_te, batch_size, distort=False):\n",
    "        yhat, test_acc_value, test_recall_value, test_prec_value = sess.run([predictions, acc_op, rec_op, prec_op], feed_dict=\n",
    "        {\n",
    "            X: X_batch,\n",
    "            y: y_batch,\n",
    "            training: False\n",
    "        })\n",
    "\n",
    "        test_accuracy.append(test_acc_value)\n",
    "        test_recall.append(test_recall_value)\n",
    "        test_predictions.append(yhat)\n",
    "        ground_truth.append(y_batch)\n",
    "\n",
    "    # print the results\n",
    "    print(\"Mean Test Accuracy:\", np.mean(test_accuracy))\n",
    "    print(\"Mean Test Recall:\", np.mean(test_recall))\n",
    "\n",
    "    # unlist the predictions and truth\n",
    "    test_predictions = flatten(test_predictions)\n",
    "    ground_truth = flatten(ground_truth)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}