crop_cbis_images_13.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Same as 12 but creating images double the size to fine tune the network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import PIL\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "import os\n",
    "import random\n",
    "import shutil\n",
    "import re\n",
    "from scipy.misc import imresize\n",
    "from mammo_utils import create_mask, half_image, get_fuzzy_offset, progress, clean_name, random_flip_image\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## randomly rotate an image\n",
    "def random_rotate_image(img, mask):\n",
    "    rotations = np.random.randint(low=-3, high=3)\n",
    "    rotated_img = np.rot90(img, rotations)\n",
    "    rotated_mask = np.rot90(mask, rotations)\n",
    "    return rotated_img, rotated_mask\n",
    "\n",
    "## randomly flip an image left-right, up-down or both and return it\n",
    "def random_flip_image(img, mask):\n",
    "    fliplr = np.random.binomial(1,0.5)\n",
    "    flipud = np.random.binomial(1,0.5)\n",
    "    \n",
    "    if fliplr:\n",
    "        img = np.flip(img, 1)\n",
    "        mask = np.flip(mask, 1)\n",
    "    if flipud:\n",
    "        img = np.flip(img, 0)\n",
    "        mask = np.flip(mask, 0)\n",
    "        \n",
    "    return random_rotate_image(img, mask)\n",
    "\n",
    "def get_roi_edges(center_col, center_row, img_height, img_width, fuzz_offset_w=0, fuzz_offset_h=0, scale_factor=1, slice_size=640):\n",
    "    # slice margin\n",
    "    slice_margin = slice_size // 3\n",
    "    \n",
    "    # figure out the new center of the ROI\n",
    "    center_col_scaled = int(center_col * scale_factor)\n",
    "    center_row_scaled = int(center_row * scale_factor)\n",
    "    \n",
    "    start_col = center_col_scaled - slice_margin + fuzz_offset_h\n",
    "    end_col = start_col + slice_size\n",
    "    \n",
    "    if start_col < 0:\n",
    "        start_col = 20\n",
    "        end_col = start_col + slice_size\n",
    "    elif end_col > (img_width - 20):\n",
    "        end_col = img_width - 20\n",
    "        start_col = end_col - slice_size\n",
    "        \n",
    "    start_row = center_row_scaled - slice_margin + fuzz_offset_w\n",
    "    end_row = start_row + slice_size\n",
    "    \n",
    "    if start_row < 0:\n",
    "        start_row = 20\n",
    "        end_row = start_row + slice_size\n",
    "    elif end_row > (img_height - 20):\n",
    "        end_row = img_height - 20\n",
    "        start_row = end_row - slice_size           \n",
    "     \n",
    "    return start_row, end_row, start_col, end_col\n",
    "\n",
    "def random_crop_image(img, mask, slice_size=1196, tile_size=640):\n",
    "    img_h = img.shape[0]\n",
    "    img_w = img.shape[1]\n",
    "    \n",
    "    # make sure the image is big enough to use\n",
    "    if (img_h < slice_size) or (img_w < slice_size):\n",
    "        print(\"Error - image is wrong size!\", img.shape)\n",
    "        return np.array([0]), np.array([0])\n",
    "    \n",
    "    # pick a random place to start the crop so that the crop will be the right size\n",
    "    start_row = np.random.randint(low=0, high=(img_h - slice_size))\n",
    "    start_col = np.random.randint(low=0, high=(img_w - slice_size))\n",
    "    \n",
    "    end_row = start_row + slice_size\n",
    "    end_col = start_col + slice_size\n",
    "    \n",
    "    # crop the image and randomly rotate it\n",
    "    cropped_img = img[start_row:end_row, start_col:end_col]\n",
    "    cropped_mask = mask[start_row:end_row, start_col:end_col]\n",
    "    \n",
    "    cropped_img, cropped_mask = random_flip_image(cropped_img, cropped_mask)\n",
    "    \n",
    "    # make sure the image is the right size and not mostly black\n",
    "    if (cropped_img.shape[0] == cropped_img.shape[1]) and (np.sum(cropped_img <= 10) <= (slice_size * slice_size * 0.9)):\n",
    "        # resize it and return it\n",
    "        cropped_img = imresize(cropped_img, (tile_size,tile_size))\n",
    "        cropped_mask = imresize(cropped_mask, (tile_size,tile_size))\n",
    "        return cropped_img.reshape((tile_size, tile_size, 1)), cropped_mask.reshape((tile_size, tile_size, 1))\n",
    "    \n",
    "    # else repeat until the image is the right size\n",
    "    else:\n",
    "        rtn_image, rtn_mask = random_crop_image(img, mask)\n",
    "        return rtn_image, rtn_mask\n",
    "    \n",
    "#########################################################\n",
    "## For abnormal images, extract the ROI based on the mask\n",
    "## Extract each ROI in 3 ways:\n",
    "##     1. At size, centered with random offset\n",
    "##     2. With 1.5 times context and random flip and rotate\n",
    "##     3. Zoomed so it all fits\n",
    "def create_cbis_slices(mask_dir, image_dir, labels, slice_size=640, debug=True):\n",
    "    # initialize return variables\n",
    "    slices_list = []\n",
    "    label_list = []\n",
    "    filenames_list = []\n",
    "    roi_sizes = []\n",
    "    \n",
    "    # we will extract the tiles at regular size and then size them down to 320x320\n",
    "    full_slice_size = 1196\n",
    "    \n",
    "    # get list of files in the directory\n",
    "    mask_files = os.listdir(mask_dir)\n",
    "    counter = 0\n",
    "    \n",
    "    # display the progress bar\n",
    "    if debug is None:\n",
    "        progress(counter, len(mask_files), 'WORKING')\n",
    "        \n",
    "        \n",
    "    # loop through the masks\n",
    "    for mask in mask_files:\n",
    "        \n",
    "        # update the progress bar\n",
    "        counter += 1\n",
    "        if debug is None:\n",
    "            progress(counter, len(mask_files), mask)\n",
    "            \n",
    "        # get the image name\n",
    "        base_image_file = clean_name(mask)\n",
    "        \n",
    "        # try some variations to open the image\n",
    "        try:\n",
    "            full_image = PIL.Image.open(image_dir + \"/\" + base_image_file + '_FULL.jpg')\n",
    "        except:\n",
    "            try:\n",
    "                full_image = PIL.Image.open(image_dir + \"/\" + base_image_file + \"000000.jpg\")\n",
    "            except:\n",
    "                try:\n",
    "                    full_image = PIL.Image.open(image_dir + \"/\" + base_image_file + \"000001.jpg\")\n",
    "                except:\n",
    "                    print(\"Error FileNotFound:\", base_image_file)\n",
    "                    continue\n",
    "        \n",
    "        mask_image = PIL.Image.open(os.path.join(mask_dir, mask))\n",
    "        \n",
    "        # get the label\n",
    "        try:\n",
    "            label = labels.loc[base_image_file + \".jpg\"]['CLASS']\n",
    "        except:\n",
    "            print(\"Label not found\", base_image_file)\n",
    "            continue\n",
    "        \n",
    "        # turn the image into an array and drop the unneeded dimensions\n",
    "        full_image_arr = np.array(full_image)[:,:,0]\n",
    "        mask_image = np.array(mask_image)[:,:,0]\n",
    "        \n",
    "        # get the mask\n",
    "        center_row, center_col, too_big, full_image_arr, mask_size = create_mask(mask_dir + \"/\" + mask, full_image_arr, half=False, output=debug)\n",
    "        \n",
    "        # get the height and width of the image\n",
    "        image_h, image_w = full_image_arr.shape\n",
    "        \n",
    "        # get the roi size from the mask\n",
    "        try:\n",
    "            mask_height = mask_size[0]\n",
    "            mask_width = mask_size[1]\n",
    "            roi_size = np.max([mask_height, mask_width])\n",
    "            if debug:\n",
    "                print(\"Mask\", mask, \" Height:\", mask_height, \"Width:\", mask_width)\n",
    "        except:\n",
    "            print(\"Mask Size Error:\", mask_size, \"for\", mask)\n",
    "            \n",
    "        # add the ROI size the the list - we will use this for cropping the normal images\n",
    "        roi_sizes.append(roi_size)\n",
    "        \n",
    "        # if there is a problem with the location of the ROI skip this one\n",
    "        if (center_row == 0) and (center_col == 0):\n",
    "            print(\"Error, skipping\", mask)\n",
    "            continue\n",
    "        \n",
    "        ###########################################################################\n",
    "        ## Extract the ROI depending on it's size\n",
    "        # if the ROI is smaller than a slice extract it with some padding\n",
    "        if roi_size < full_slice_size:\n",
    "            if debug:\n",
    "                print(\"ROI small\", mask)\n",
    "            \n",
    "            ## Make sure the size of the ROI is at least as big as a tile will be\n",
    "            adj_mask_height = int(np.max([full_slice_size * 1.4, mask_height]))\n",
    "            adj_mask_width = int(np.max([full_slice_size * 1.4, mask_width]))\n",
    "\n",
    "            ## Extract the full ROI with 20% padding on either side\n",
    "            start_row = int(np.max([center_row - (adj_mask_height // 2), 0]))\n",
    "            end_row = start_row + adj_mask_height\n",
    "            if end_row > image_h:\n",
    "                end_row = image_h\n",
    "                start_row = image_h - adj_mask_height\n",
    "\n",
    "            start_col = int(np.max([center_col - (adj_mask_width // 2), 0]))\n",
    "            end_col = start_col + adj_mask_width\n",
    "            if end_col > image_w:\n",
    "                end_col = image_w\n",
    "                start_col = image_w - adj_mask_width\n",
    "\n",
    "            # extract the ROI and randomly flip it\n",
    "            cropped_roi = full_image_arr[start_row:end_row, start_col:end_col]\n",
    "            cropped_mask = mask_image[start_row:end_row, start_col:end_col]\n",
    "            roi_img, roi_mask = random_flip_image(cropped_roi, cropped_mask)\n",
    "            \n",
    "        # else extract the ROI with less padding\n",
    "        else:\n",
    "            if debug:\n",
    "                print(\"ROI Big\", mask)\n",
    "            \n",
    "            # we will still use a small amount of padding as it is necessary for the random cropping\n",
    "            adj_mask_height = int(np.max([full_slice_size * 1.15, mask_height]))\n",
    "            adj_mask_width = int(np.max([full_slice_size * 1.15, mask_width]))\n",
    "            \n",
    "            start_row = np.max([center_row - (adj_mask_height // 2), 0])\n",
    "            end_row = start_row + adj_mask_height\n",
    "            if end_row > image_h:\n",
    "                end_row = image_h\n",
    "                start_row = image_h - adj_mask_height\n",
    "\n",
    "            start_col = np.max([center_col - (adj_mask_width // 2), 0])\n",
    "            end_col = start_col + adj_mask_width\n",
    "            if end_col > image_w:\n",
    "                end_col = image_w\n",
    "                start_col = image_w - adj_mask_width\n",
    "\n",
    "            # extract the ROI and randomly flip it\n",
    "            cropped_roi = full_image_arr[start_row:end_row, start_col:end_col]\n",
    "            cropped_mask = mask_image[start_row:end_row, start_col:end_col]\n",
    "            roi_img, roi_mask = random_flip_image(cropped_roi, cropped_mask)\n",
    "        \n",
    "        ###########################################################################\n",
    "        ## Now we have an image that is just the ROI, with or without padding\n",
    "        ## Take 2 random crops from it\n",
    "        slice_1, label_1 = random_crop_image(roi_img, roi_mask)\n",
    "        slice_2, label_2 = random_crop_image(roi_img, roi_mask)\n",
    "        \n",
    "        # for each of the slices, if it is sized properly add it to the list\n",
    "        if (slice_1.shape[0] == slice_size) and (slice_1.shape[1] == slice_size):\n",
    "            if np.sum(slice_1 <= 10) <= (slice_size * slice_size * 0.9):\n",
    "                slices_list.append(slice_1)\n",
    "                label_list.append(label_1)\n",
    "                filenames_list.append(base_image_file + \".jpg\")\n",
    "        \n",
    "        if (slice_2.shape[0] == slice_size) and (slice_2.shape[1] == slice_size):\n",
    "            if np.sum(slice_2 <= 10) <= (slice_size * slice_size * 0.9):\n",
    "                slices_list.append(slice_2)\n",
    "                label_list.append(label_2)\n",
    "                filenames_list.append(base_image_file + \".jpg\")\n",
    "        \n",
    "        # if the ROI is 1.5 times bigger than the slice take one more crop\n",
    "        if roi_size > (full_slice_size * 1.5):\n",
    "            slice_2, label_2 = random_crop_image(roi_img, roi_mask)\n",
    "            \n",
    "            if (slice_2.shape[0] == slice_size) and (slice_2.shape[1] == slice_size):\n",
    "                if np.sum(slice_2 <= 10) <= (slice_size * slice_size * 0.9):\n",
    "                    slices_list.append(slice_2)\n",
    "                    label_list.append(label_2)\n",
    "                    filenames_list.append(base_image_file + \".jpg\")\n",
    "\n",
    "        # if the ROI is double the size of the target image take one more crop\n",
    "#         if roi_size > (full_slice_size * 2):\n",
    "#             #print(\"Super big\")\n",
    "#             slice_2, label_2 = random_crop_image(roi_img, roi_mask)\n",
    "            \n",
    "#             if (slice_2.shape[0] == slice_size) and (slice_2.shape[1] == slice_size):\n",
    "#                 if np.sum(slice_2 <= 10) <= (slice_size * slice_size * 0.9):\n",
    "#                     slices_list.append(slice_2)\n",
    "#                     label_list.append(label_2)\n",
    "#                     filenames_list.append(base_image_file + \".jpg\")\n",
    "        \n",
    "        ###########################################################################\n",
    "        ## Now we will take just some random crops of the image, ignoring the ROI\n",
    "        for foo in range(4):       \n",
    "            try:\n",
    "                slice_4, label_4 = random_crop_image(full_image_arr, mask_image)\n",
    "\n",
    "                # make sure the size is right\n",
    "                if (slice_4.shape[0] == slice_size) and (slice_4.shape[1] == slice_size):\n",
    "                    # make sure the image isn't all black, less than 80% or 90% is good\n",
    "                    if np.sum(slice_4 <= 10) <= (slice_size * slice_size * 0.8):\n",
    "                        slices_list.append(slice_4)\n",
    "                        label_list.append(label_4)\n",
    "                        filenames_list.append(base_image_file + \".jpg\")\n",
    "            except:\n",
    "                continue\n",
    "\n",
    "    # return the data\n",
    "    return np.array(slices_list), np.array(label_list), np.array(filenames_list), roi_sizes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def plot_random_samples(images, labels, names, N=20):\n",
    "    idxs = random.sample(range(len(images)), k=N)\n",
    "\n",
    "    for idx in idxs:\n",
    "        f, ax = plt.subplots(1, 2, figsize=(8, 4))\n",
    "        ax[0].imshow(images[idx].reshape(640,640))\n",
    "        ax[0].set_title(names[idx] + \" IM - \" + str(idx))\n",
    "        ax[1].imshow(labels[idx].reshape(640,640))\n",
    "        ax[1].set_title(names[idx] + \" LA - \" + str(idx))\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calc Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load the labels\n",
    "test_labels = pd.read_pickle(os.path.join(\"data\", \"test_labels.pkl\"))\n",
    "test_labels['IMAGE_NAME2'] = test_labels.index\n",
    "test_labels = test_labels.drop_duplicates(['IMAGE_NAME2'])\n",
    "\n",
    "## use a copy on the local drive to make testing faster\n",
    "mask_dir = \"C:\\\\Users\\\\eric\\\\Documents\\\\Courses\\\\Applied ML 2\\\\mammography\\\\data\\\\cbis-ddsm\\\\Calc Test All Mask JPEGs\"\n",
    "image_dir = \"C:\\\\Users\\\\eric\\\\Documents\\\\Courses\\\\Applied ML 2\\\\mammography\\\\data\\\\cbis-ddsm\\\\Calc Test All Full JPEGs\"\n",
    "\n",
    "calc_test_slices, calc_test_labels, calc_testfilenames, calc_test_roi_sizes = create_cbis_slices(mask_dir, image_dir, labels=test_labels, debug=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(\"Calc Test Slices shape:\", calc_test_slices.shape)\n",
    "print(\"Calc Test Labels:\", calc_test_labels.shape)\n",
    "print(\"Calc Test File Name List:\", len(calc_testfilenames))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plot_random_samples(calc_test_slices, calc_test_labels, calc_testfilenames, N=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# scale the image data\n",
    "# calc_test_slices = (calc_test_slices - 127.0) / 255.0\n",
    "calc_test_slices = calc_test_slices.astype(np.uint8)\n",
    "\n",
    "# relabel the labels\n",
    "calc_test_labels[calc_test_labels > 0] = 1\n",
    "calc_test_labels = calc_test_labels.astype(np.uint8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.mean(calc_test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save(os.path.join(\"data\", \"calc_test_slices13.npy\"), calc_test_slices)\n",
    "np.save(os.path.join(\"data\", \"calc_test_filenames13.npy\"), calc_testfilenames)\n",
    "np.save(os.path.join(\"data\", \"calc_test_labels13.npy\"), np.array(calc_test_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del(calc_test_slices, calc_test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mass Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load the labels\n",
    "test_labels = pd.read_pickle(os.path.join(\"data\", \"test_labels.pkl\"))\n",
    "test_labels['IMAGE_NAME2'] = test_labels.index\n",
    "test_labels = test_labels.drop_duplicates(['IMAGE_NAME2'])\n",
    "\n",
    "## use a copy on the local drive to make testing faster\n",
    "mask_dir = \"C:\\\\Users\\\\eric\\\\Documents\\\\Courses\\\\Applied ML 2\\\\mammography\\\\data\\\\cbis-ddsm\\\\Mass Test All Mask JPEGs\"\n",
    "image_dir = \"C:\\\\Users\\\\eric\\\\Documents\\\\Courses\\\\Applied ML 2\\\\mammography\\\\data\\\\cbis-ddsm\\\\Mass Test All Full JPEGs\"\n",
    "\n",
    "mass_test_slices, mass_test_labels, mass_testfilenames, mass_test_roi_sizes = create_cbis_slices(mask_dir, image_dir, labels=test_labels, debug=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(\"Mass Test Slices shape:\", mass_test_slices.shape)\n",
    "print(\"Mass Test Labels:\", len(mass_test_labels))\n",
    "print(\"Mass Test File Name List:\", len(mass_testfilenames))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plot_random_samples(mass_test_slices, mass_test_labels, mass_testfilenames, N=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# scale the image data\n",
    "# mass_test_slices = (mass_test_slices - 127.0) / 255.0\n",
    "mass_test_slices = mass_test_slices.astype(np.uint8)\n",
    "\n",
    "# relabel the labels\n",
    "mass_test_labels[mass_test_labels > 0] = 1\n",
    "mass_test_labels = mass_test_labels.astype(np.uint8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.mean(mass_test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save(os.path.join(\"data\", \"mass_test_slices13.npy\"), mass_test_slices)\n",
    "np.save(os.path.join(\"data\", \"mass_test_filenames13.npy\"), mass_testfilenames)\n",
    "np.save(os.path.join(\"data\", \"mass_test_labels13.npy\"), mass_test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del(mass_test_slices, mass_test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calc Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_labels = pd.read_pickle(os.path.join(\"data\", \"train_labels.pkl\"))\n",
    "train_labels['IMAGE_NAME2'] = train_labels.index\n",
    "train_labels = train_labels.drop_duplicates(['IMAGE_NAME2'])\n",
    "\n",
    "## use a copy on the local drive to make testing faster\n",
    "mask_dir = \"C:\\\\Users\\\\eric\\\\Documents\\\\Courses\\\\Applied ML 2\\\\mammography\\\\data\\\\cbis-ddsm\\\\Calc Train All Mask JPEGs Full\"\n",
    "image_dir = \"C:\\\\Users\\\\eric\\\\Documents\\\\Courses\\\\Applied ML 2\\\\mammography\\\\data\\\\cbis-ddsm\\\\Calc Train All Full JPEGs\"\n",
    "\n",
    "calc_train_slices, calc_train_labels, calc_trainfilenames, calc_train_roi_sizes = create_cbis_slices(mask_dir, image_dir, labels=train_labels, debug=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(\"Calc Train Slices shape:\", calc_train_slices.shape)\n",
    "print(\"Calc Train Labels:\", calc_train_labels.shape)\n",
    "print(\"Calc Train File Name List:\", len(calc_trainfilenames))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plot_random_samples(calc_train_slices, calc_train_labels, calc_trainfilenames, N=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# scale the image data\n",
    "# mass_test_slices = (mass_test_slices - 127.0) / 255.0\n",
    "calc_train_slices = calc_train_slices.astype(np.uint8)\n",
    "\n",
    "# relabel the labels\n",
    "calc_train_labels[calc_train_labels > 0] = 1\n",
    "calc_train_labels = calc_train_labels.astype(np.uint8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.07468652443807633"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(calc_train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save(os.path.join(\"data\", \"calc_train_slices13.npy\"), calc_train_slices)\n",
    "np.save(os.path.join(\"data\", \"calc_train_filenames13.npy\"), calc_trainfilenames)\n",
    "np.save(os.path.join(\"data\", \"calc_train_labels13.npy\"), calc_train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del(calc_train_slices, calc_train_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mass Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_labels = pd.read_pickle(os.path.join(\"data\", \"train_labels.pkl\"))\n",
    "train_labels['IMAGE_NAME2'] = train_labels.index\n",
    "train_labels = train_labels.drop_duplicates(['IMAGE_NAME2'])\n",
    "\n",
    "## use a copy on the local drive to make testing faster\n",
    "mask_dir = \"C:\\\\Users\\\\eric\\\\Documents\\\\Courses\\\\Applied ML 2\\\\mammography\\\\data\\\\cbis-ddsm\\\\Mass Train All Mask JPEGs Full\"\n",
    "image_dir = \"C:\\\\Users\\\\eric\\\\Documents\\\\Courses\\\\Applied ML 2\\\\mammography\\\\data\\\\cbis-ddsm\\\\Mass Train All JPEGs Full\"\n",
    "\n",
    "mass_train_slices, mass_train_labels, mass_trainfilenames, mass_train_roi_sizes = create_cbis_slices(mask_dir, image_dir, labels=train_labels, debug=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mass Train Slices shape: (6716, 640, 640, 1)\n",
      "Mass Train Labels: (6716, 640, 640, 1)\n",
      "Mass Train File Name List: 6716\n"
     ]
    }
   ],
   "source": [
    "print(\"Mass Train Slices shape:\", mass_train_slices.shape)\n",
    "print(\"Mass Train Labels:\", mass_train_labels.shape)\n",
    "print(\"Mass Train File Name List:\", len(mass_train_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plot_random_samples(mass_train_slices, mass_train_labels, mass_train_labels, N=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# scale the image data\n",
    "# mass_test_slices = (mass_test_slices - 127.0) / 255.0\n",
    "mass_train_slices = mass_train_slices.astype(np.uint8)\n",
    "\n",
    "# relabel the labels\n",
    "mass_train_labels[mass_train_labels > 0] = 1\n",
    "mass_train_slices = mass_train_slices.astype(np.uint8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.02646542320228745"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(mass_train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save(os.path.join(\"data\", \"mass_train_slices13.npy\"), mass_train_slices)\n",
    "np.save(os.path.join(\"data\", \"mass_train_filenames13.npy\"), mass_trainfilenames)\n",
    "np.save(os.path.join(\"data\", \"mass_train_labels13.npy\"), mass_train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del(mass_train_slices, mass_train_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combine Slices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train slices: (13548, 640, 640, 1)\n",
      "Train filenames: (13548,)\n"
     ]
    }
   ],
   "source": [
    "mass_train_slices = np.load(os.path.join(\"data\", \"mass_train_slices13.npy\"))\n",
    "calc_train_slices = np.load(os.path.join(\"data\", \"calc_train_slices13.npy\"))\n",
    "\n",
    "mass_train_names = np.load(os.path.join(\"data\", \"mass_train_filenames13.npy\"))\n",
    "calc_train_names = np.load(os.path.join(\"data\", \"calc_train_filenames13.npy\"))\n",
    "\n",
    "train_names = np.concatenate([mass_train_names, calc_train_names], axis=0)\n",
    "train_slices = np.concatenate([mass_train_slices, calc_train_slices], axis=0)\n",
    "\n",
    "print(\"Train slices:\", train_slices.shape)\n",
    "print(\"Train filenames:\", train_names.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save(os.path.join(\"data\", \"cbis_train_slices13.npy\"), train_slices)\n",
    "np.save(os.path.join(\"data\", \"cbis_train_names13.npy\"), train_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del(train_slices, mass_train_slices, calc_train_slices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train labels: (13548, 640, 640, 1)\n"
     ]
    }
   ],
   "source": [
    "mass_train_labels = np.load(os.path.join(\"data\", \"mass_train_labels13.npy\"))\n",
    "calc_train_labels = np.load(os.path.join(\"data\", \"calc_train_labels13.npy\"))\n",
    "\n",
    "train_labels = np.concatenate([mass_train_labels, calc_train_labels], axis=0)\n",
    "print(\"Train labels:\", train_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save(os.path.join(\"data\", \"cbis_train_labels13.npy\"), train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del(train_labels, mass_train_labels , calc_train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mass_test_labels = np.load(os.path.join(\"data\", \"mass_test_labels13.npy\"))\n",
    "calc_test_labels = np.load(os.path.join(\"data\", \"calc_test_labels13.npy\"))\n",
    "\n",
    "mass_test_slices = np.load(os.path.join(\"data\", \"mass_test_slices13.npy\"))\n",
    "calc_test_slices = np.load(os.path.join(\"data\", \"calc_test_slices13.npy\"))\n",
    "\n",
    "mass_test_names = np.load(os.path.join(\"data\", \"mass_test_filenames13.npy\"))\n",
    "calc_test_names = np.load(os.path.join(\"data\", \"calc_test_filenames13.npy\"))\n",
    "\n",
    "test_slices = np.concatenate([mass_test_slices, calc_test_slices], axis=0)\n",
    "test_labels = np.concatenate([mass_test_labels, calc_test_labels], axis=0)\n",
    "test_names = np.concatenate([mass_test_names, calc_test_names], axis=0)\n",
    "\n",
    "print(\"Test labels:\", test_labels.shape)\n",
    "print(\"Test slices:\", test_slices.shape)\n",
    "print(\"Test filenames:\", test_names.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save(os.path.join(\"data\", \"cbis_test_slices13.npy\"), test_slices)\n",
    "np.save(os.path.join(\"data\", \"cbis_test_labels13.npy\"), test_labels)\n",
    "np.save(os.path.join(\"data\", \"cbis_test_names13.npy\"), test_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del(test_labels, test_slices, mass_test_labels , mass_test_slices, calc_test_labels, calc_test_slices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}