03_TF_NN_my.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "code_folding": [
     0
    ],
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# import  input_data and read the data set\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "code_folding": [
     0
    ],
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define util functions\n",
    "\n",
    "# initialize a shaped matrix of weights with random values\n",
    "def weight_variable(shape):\n",
    "  initial = tf.truncated_normal(shape, stddev=0.1)\n",
    "  return tf.Variable(initial)\n",
    "\n",
    "# initialize a shaped matrix of bias with random values\n",
    "def bias_variable(shape):\n",
    "  initial = tf.constant(0.1, shape=shape)\n",
    "  return tf.Variable(initial)\n",
    "\n",
    "def conv2d(x, W):\n",
    "  return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')\n",
    "\n",
    "def max_pool_2x2(x):\n",
    "  return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],\n",
    "                        strides=[1, 2, 2, 1], padding='SAME')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "code_folding": [
     0
    ],
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Define the model\n",
    "\n",
    "# A simple model for predicting y. Will not be used here...(see previous example)\n",
    "# x = tf.placeholder(tf.float32, [None, 784])\n",
    "# W = tf.Variable(tf.zeros([784, 10]))\n",
    "# b = tf.Variable(tf.zeros([10]))\n",
    "# y = tf.nn.softmax(tf.matmul(x, W) + b)\n",
    "# y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "# foundation of the model - the input layer of the image 28x28=784\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "x_image = tf.reshape(x, [-1,28,28,1]) # reshape to 28x28, 1 is the number of color channels\n",
    "\n",
    "# the target digits of the model\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "# first convolutional layer: one input image and 32 output filters of 5x5\n",
    "W_conv1 = weight_variable([5, 5, 1, 32])\n",
    "b_conv1 = bias_variable([32])\n",
    "h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)\n",
    "h_pool1 = max_pool_2x2(h_conv1) # size is reduced to 14x14\n",
    "\n",
    "# second convolutional layer: 32 input (filtered) images and 64 output filters of 5x5\n",
    "W_conv2 = weight_variable([5, 5, 32, 64])\n",
    "b_conv2 = bias_variable([64])\n",
    "h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)\n",
    "h_pool2 = max_pool_2x2(h_conv2) # size is reduced to 7x7\n",
    "h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64]) # shape as an array \n",
    "\n",
    "# third layer - fully connected with input 7x7x64 and output 1024\n",
    "W_fc1 = weight_variable([7 * 7 * 64, 1024])\n",
    "b_fc1 = bias_variable([1024])\n",
    "h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)\n",
    "\n",
    "# a drop layer with probability \n",
    "keep_prob = tf.placeholder(tf.float32)\n",
    "h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)\n",
    "\n",
    "# final layer - softmax reduction 10 outputs\n",
    "W_fc2 = weight_variable([1024, 10])\n",
    "b_fc2 = bias_variable([10])\n",
    "y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2\n",
    "\n",
    "# minimize loss function\n",
    "cross_entropy = tf.reduce_mean(\n",
    "  tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))\n",
    "\n",
    "# define train step and rate\n",
    "train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.16470589,  0.60392159,  1.        ,  0.59607846,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.37254903,  0.88627458,  0.99215692,\n",
       "         0.99215692,  0.88627458,  0.        ,  0.        ,  0.36078432,\n",
       "         0.0509804 ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.01960784,  0.29803923,\n",
       "         0.97647065,  0.99215692,  0.99215692,  0.99215692,  0.88627458,\n",
       "         0.        ,  0.41176474,  0.98431379,  0.85490203,  0.34117648,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.4666667 ,  0.99215692,  0.99215692,  0.99215692,\n",
       "         0.99215692,  0.83529419,  0.4039216 ,  0.        ,  0.49019611,\n",
       "         0.99215692,  0.99215692,  0.9333334 ,  0.34117648,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.01960784,  0.64705884,\n",
       "         0.99215692,  0.99215692,  0.99215692,  0.88235301,  0.23137257,\n",
       "         0.        ,  0.        ,  0.49019611,  0.99215692,  0.99215692,\n",
       "         0.99215692,  0.94117653,  0.54509807,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.01176471,  0.4784314 ,  0.99215692,  0.99215692,  0.99215692,\n",
       "         0.75294125,  0.07450981,  0.        ,  0.        ,  0.        ,\n",
       "         0.19607845,  0.61960787,  0.98431379,  0.99215692,  0.99215692,\n",
       "         0.37647063,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.21960786,  0.99215692,\n",
       "         0.99215692,  0.99215692,  0.76862752,  0.0509804 ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.86274517,  0.99215692,  0.99215692,  0.37647063,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.36470589,  0.84705889,  0.99215692,  0.99215692,  0.76862752,\n",
       "         0.02745098,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.86274517,  0.99215692,\n",
       "         0.99215692,  0.37647063,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.13725491,  0.89019614,  0.99215692,\n",
       "         0.99215692,  0.75294125,  0.0509804 ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.14509805,  0.90980399,  0.99215692,  0.93725497,  0.24313727,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.81568635,  0.99215692,  0.99215692,  0.99215692,  0.4039216 ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.54901963,  0.99215692,\n",
       "         0.99215692,  0.83137262,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.33725491,  0.97254908,  0.99215692,\n",
       "         0.95294124,  0.72941178,  0.10980393,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.09411766,  0.95294124,  0.99215692,  0.99215692,  0.72941178,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.46274513,  0.99215692,  0.99215692,  0.77647066,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.49019611,  0.99215692,\n",
       "         0.99215692,  0.96470594,  0.26274511,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.46274513,  0.99215692,\n",
       "         0.99215692,  0.47058827,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.00392157,  0.50196081,  0.99215692,  0.99215692,  0.74901962,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.46274513,  0.99215692,  0.99215692,  0.2392157 ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.29411766,  0.99215692,\n",
       "         0.99215692,  0.99215692,  0.49019611,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.46274513,\n",
       "         0.99215692,  0.99215692,  0.74901962,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.07843138,\n",
       "         0.4666667 ,  0.97254908,  0.99215692,  0.99215692,  0.8588236 ,\n",
       "         0.1254902 ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.46274513,  0.99215692,  0.99215692,\n",
       "         0.84705889,  0.10588236,  0.        ,  0.        ,  0.        ,\n",
       "         0.11764707,  0.47450984,  0.88627458,  0.99215692,  0.99215692,\n",
       "         0.99215692,  0.99215692,  0.20392159,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.09803922,  0.80000007,  0.99215692,  0.99215692,  0.85098046,\n",
       "         0.78039223,  0.78039223,  0.78039223,  0.85490203,  0.99215692,\n",
       "         0.99215692,  0.99215692,  0.99215692,  0.87450987,  0.52941179,\n",
       "         0.03137255,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.31764707,\n",
       "         0.96862751,  0.99215692,  0.99215692,  0.99215692,  0.99215692,\n",
       "         0.99215692,  0.99215692,  0.99215692,  0.99215692,  0.97647065,\n",
       "         0.81568635,  0.15294118,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.76862752,  0.92941183,\n",
       "         0.99215692,  0.99215692,  0.99215692,  0.99215692,  0.99215692,\n",
       "         0.99215692,  0.83137262,  0.33333334,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.10196079,  0.45490199,  0.77647066,\n",
       "         0.99215692,  0.4666667 ,  0.45490199,  0.45490199,  0.12156864,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ]], dtype=float32)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "mnist.train.next_batch(1)[0]\n",
    "# plt.imshow(mnist.train.next_batch(1)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "code_folding": [
     0
    ],
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 0, training accuracy 0.04\n",
      "step 100, training accuracy 0.89\n",
      "step 200, training accuracy 0.94\n",
      "step 300, training accuracy 0.95\n",
      "step 400, training accuracy 0.91\n",
      "step 500, training accuracy 0.96\n",
      "step 600, training accuracy 0.95\n",
      "step 700, training accuracy 0.97\n",
      "step 800, training accuracy 0.93\n",
      "step 900, training accuracy 0.98\n",
      "step 1000, training accuracy 0.96\n",
      "step 1100, training accuracy 0.98\n",
      "step 1200, training accuracy 0.97\n",
      "step 1300, training accuracy 0.99\n",
      "step 1400, training accuracy 0.98\n",
      "step 1500, training accuracy 0.98\n",
      "step 1600, training accuracy 0.98\n",
      "step 1700, training accuracy 0.98\n",
      "step 1800, training accuracy 0.96\n",
      "step 1900, training accuracy 0.99\n",
      "step 2000, training accuracy 0.99\n",
      "step 2100, training accuracy 0.98\n",
      "step 2200, training accuracy 0.98\n",
      "step 2300, training accuracy 0.99\n",
      "step 2400, training accuracy 1\n",
      "step 2500, training accuracy 1\n",
      "step 2600, training accuracy 0.99\n",
      "step 2700, training accuracy 1\n",
      "step 2800, training accuracy 0.99\n",
      "step 2900, training accuracy 0.99\n",
      "step 3000, training accuracy 1\n",
      "step 3100, training accuracy 1\n",
      "step 3200, training accuracy 0.96\n",
      "step 3300, training accuracy 0.99\n",
      "step 3400, training accuracy 1\n",
      "step 3500, training accuracy 1\n",
      "step 3600, training accuracy 0.99\n",
      "step 3700, training accuracy 0.99\n",
      "step 3800, training accuracy 1\n",
      "step 3900, training accuracy 0.99\n",
      "step 4000, training accuracy 1\n",
      "step 4100, training accuracy 1\n",
      "step 4200, training accuracy 1\n",
      "step 4300, training accuracy 0.99\n",
      "step 4400, training accuracy 0.99\n",
      "step 4500, training accuracy 0.99\n",
      "step 4600, training accuracy 1\n",
      "step 4700, training accuracy 1\n",
      "step 4800, training accuracy 1\n",
      "step 4900, training accuracy 0.99\n"
     ]
    }
   ],
   "source": [
    "# Train the model\n",
    "\n",
    "# start session and initialize variables\n",
    "sess = tf.InteractiveSession()\n",
    "sess.run(tf.initialize_all_variables())\n",
    "\n",
    "# evaluate the prediction and the accuracy on the train test - needed only for printing during the training\n",
    "correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "# run the train batches\n",
    "for i in range(5000):\n",
    "    batch = mnist.train.next_batch(100)\n",
    "    # print the accuracy thus far\n",
    "    if i%100 == 0:\n",
    "        train_accuracy = accuracy.eval(feed_dict={\n",
    "            x:batch[0], y_: batch[1], keep_prob: 1.0})\n",
    "        print(\"step %d, training accuracy %g\"%(i, train_accuracy))\n",
    "    # train\n",
    "    train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "code_folding": [
     0
    ],
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test accuracy 0.9896\n"
     ]
    }
   ],
   "source": [
    "# Test the model\n",
    "print(\"test accuracy %g\"%accuracy.eval(feed_dict={\n",
    "    x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0}))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}