Added layers

lab-11-1-mnist_cnn.py

@@ -43,7 +43,7 @@
 L2 = tf.nn.relu(L2)
 L2 = tf.nn.max_pool(L2, ksize=[1, 2, 2, 1],
                     strides=[1, 2, 2, 1], padding='SAME')
-L2 = tf.reshape(L2, [-1, 7 * 7 * 64])
+L2_flat = tf.reshape(L2, [-1, 7 * 7 * 64])
 '''
 Tensor("Conv2D_1:0", shape=(?, 14, 14, 64), dtype=float32)
 Tensor("Relu_1:0", shape=(?, 14, 14, 64), dtype=float32)
@@ -55,11 +55,11 @@
 W3 = tf.get_variable("W3", shape=[7 * 7 * 64, 10],
                      initializer=tf.contrib.layers.xavier_initializer())
 b = tf.Variable(tf.random_normal([10]))
-hypothesis = tf.matmul(L2, W3) + b
+logits = tf.matmul(L2_flat, W3) + b
 
 # define cost/loss & optimizer
 cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
-    logits=hypothesis, labels=Y))
+    logits=logits, labels=Y))
 optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
 
 # initialize
@@ -83,7 +83,7 @@
 print('Learning Finished!')
 
 # Test model and check accuracy
-correct_prediction = tf.equal(tf.argmax(hypothesis, 1), tf.argmax(Y, 1))
+correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(Y, 1))
 accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
 print('Accuracy:', sess.run(accuracy, feed_dict={
       X: mnist.test.images, Y: mnist.test.labels}))
@@ -92,7 +92,7 @@
 r = random.randint(0, mnist.test.num_examples - 1)
 print("Label: ", sess.run(tf.argmax(mnist.test.labels[r:r + 1], 1)))
 print("Prediction: ", sess.run(
-    tf.argmax(hypothesis, 1), feed_dict={X: mnist.test.images[r:r + 1]}))
+    tf.argmax(logits, 1), feed_dict={X: mnist.test.images[r:r + 1]}))
 
 # plt.imshow(mnist.test.images[r:r + 1].
 #           reshape(28, 28), cmap='Greys', interpolation='nearest')

lab-11-2-mnist_deep_cnn.py

@@ -66,7 +66,7 @@
 L3 = tf.nn.max_pool(L3, ksize=[1, 2, 2, 1], strides=[
                     1, 2, 2, 1], padding='SAME')
 L3 = tf.nn.dropout(L3, keep_prob=keep_prob)
-L3 = tf.reshape(L3, [-1, 128 * 4 * 4])
+L3_flat = tf.reshape(L3, [-1, 128 * 4 * 4])
 '''
 Tensor("Conv2D_2:0", shape=(?, 7, 7, 128), dtype=float32)
 Tensor("Relu_2:0", shape=(?, 7, 7, 128), dtype=float32)
@@ -79,7 +79,7 @@
 W4 = tf.get_variable("W4", shape=[128 * 4 * 4, 625],
                      initializer=tf.contrib.layers.xavier_initializer())
 b4 = tf.Variable(tf.random_normal([625]))
-L4 = tf.nn.relu(tf.matmul(L3, W4) + b4)
+L4 = tf.nn.relu(tf.matmul(L3_flat, W4) + b4)
 L4 = tf.nn.dropout(L4, keep_prob=keep_prob)
 '''
 Tensor("Relu_3:0", shape=(?, 625), dtype=float32)
@@ -90,14 +90,14 @@
 W5 = tf.get_variable("W5", shape=[625, 10],
                      initializer=tf.contrib.layers.xavier_initializer())
 b5 = tf.Variable(tf.random_normal([10]))
-hypothesis = tf.matmul(L4, W5) + b5
+logits = tf.matmul(L4, W5) + b5
 '''
 Tensor("add_1:0", shape=(?, 10), dtype=float32)
 '''
 
 # define cost/loss & optimizer
 cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
-    logits=hypothesis, labels=Y))
+    logits=logits, labels=Y))
 optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
 
 # initialize
@@ -121,7 +121,7 @@
 print('Learning Finished!')
 
 # Test model and check accuracy
-correct_prediction = tf.equal(tf.argmax(hypothesis, 1), tf.argmax(Y, 1))
+correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(Y, 1))
 accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
 print('Accuracy:', sess.run(accuracy, feed_dict={
       X: mnist.test.images, Y: mnist.test.labels, keep_prob: 1}))
@@ -130,7 +130,7 @@
 r = random.randint(0, mnist.test.num_examples - 1)
 print("Label: ", sess.run(tf.argmax(mnist.test.labels[r:r + 1], 1)))
 print("Prediction: ", sess.run(
-    tf.argmax(hypothesis, 1), feed_dict={X: mnist.test.images[r:r + 1], keep_prob: 1}))
+    tf.argmax(logits, 1), feed_dict={X: mnist.test.images[r:r + 1], keep_prob: 1}))
 
 # plt.imshow(mnist.test.images[r:r + 1].
 #           reshape(28, 28), cmap='Greys', interpolation='nearest')

lab-11-3-mnist_cnn_class.py

@@ -78,7 +78,8 @@ def _build_net(self):
             L3 = tf.nn.max_pool(L3, ksize=[1, 2, 2, 1], strides=[
                                 1, 2, 2, 1], padding='SAME')
             L3 = tf.nn.dropout(L3, keep_prob=self.keep_prob)
-            L3 = tf.reshape(L3, [-1, 128 * 4 * 4])
+
+            L3_flat = tf.reshape(L3, [-1, 128 * 4 * 4])
             '''
             Tensor("Conv2D_2:0", shape=(?, 7, 7, 128), dtype=float32)
             Tensor("Relu_2:0", shape=(?, 7, 7, 128), dtype=float32)
@@ -91,7 +92,7 @@ def _build_net(self):
             W4 = tf.get_variable("W4", shape=[128 * 4 * 4, 625],
                                  initializer=tf.contrib.layers.xavier_initializer())
             b4 = tf.Variable(tf.random_normal([625]))
-            L4 = tf.nn.relu(tf.matmul(L3, W4) + b4)
+            L4 = tf.nn.relu(tf.matmul(L3_flat, W4) + b4)
             L4 = tf.nn.dropout(L4, keep_prob=self.keep_prob)
             '''
             Tensor("Relu_3:0", shape=(?, 625), dtype=float32)
@@ -102,23 +103,23 @@ def _build_net(self):
             W5 = tf.get_variable("W5", shape=[625, 10],
                                  initializer=tf.contrib.layers.xavier_initializer())
             b5 = tf.Variable(tf.random_normal([10]))
-            self.logit = tf.matmul(L4, W5) + b5
+            self.logits = tf.matmul(L4, W5) + b5
             '''
             Tensor("add_1:0", shape=(?, 10), dtype=float32)
             '''
 
         # define cost/loss & optimizer
         self.cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
-            logits=self.logit, labels=self.Y))
+            logits=self.logits, labels=self.Y))
         self.optimizer = tf.train.AdamOptimizer(
             learning_rate=learning_rate).minimize(self.cost)
 
         correct_prediction = tf.equal(
-            tf.argmax(self.logit, 1), tf.argmax(self.Y, 1))
+            tf.argmax(self.logits, 1), tf.argmax(self.Y, 1))
         self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
 
     def predict(self, x_test, keep_prop=1.0):
-        return self.sess.run(self.logit, feed_dict={self.X: x_test, self.keep_prob: keep_prop})
+        return self.sess.run(self.logits, feed_dict={self.X: x_test, self.keep_prob: keep_prop})
 
     def get_accuracy(self, x_test, y_test, keep_prop=1.0):
         return self.sess.run(self.accuracy, feed_dict={self.X: x_test, self.Y: y_test, self.keep_prob: keep_prop})

lab-11-4-mnist_cnn_ensemble.py

@@ -77,7 +77,8 @@ def _build_net(self):
             L3 = tf.nn.max_pool(L3, ksize=[1, 2, 2, 1], strides=[
                                 1, 2, 2, 1], padding='SAME')
             L3 = tf.nn.dropout(L3, keep_prob=self.keep_prob)
-            L3 = tf.reshape(L3, [-1, 128 * 4 * 4])
+
+            L3_flat = tf.reshape(L3, [-1, 128 * 4 * 4])
             '''
             Tensor("Conv2D_2:0", shape=(?, 7, 7, 128), dtype=float32)
             Tensor("Relu_2:0", shape=(?, 7, 7, 128), dtype=float32)
@@ -90,7 +91,7 @@ def _build_net(self):
             W4 = tf.get_variable("W4", shape=[128 * 4 * 4, 625],
                                  initializer=tf.contrib.layers.xavier_initializer())
             b4 = tf.Variable(tf.random_normal([625]))
-            L4 = tf.nn.relu(tf.matmul(L3, W4) + b4)
+            L4 = tf.nn.relu(tf.matmul(L3_flat, W4) + b4)
             L4 = tf.nn.dropout(L4, keep_prob=self.keep_prob)
             '''
             Tensor("Relu_3:0", shape=(?, 625), dtype=float32)
@@ -101,23 +102,23 @@ def _build_net(self):
             W5 = tf.get_variable("W5", shape=[625, 10],
                                  initializer=tf.contrib.layers.xavier_initializer())
             b5 = tf.Variable(tf.random_normal([10]))
-            self.logit = tf.matmul(L4, W5) + b5
+            self.logits = tf.matmul(L4, W5) + b5
             '''
             Tensor("add_1:0", shape=(?, 10), dtype=float32)
             '''
 
         # define cost/loss & optimizer
         self.cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
-            logits=self.logit, labels=self.Y))
+            logits=self.logits, labels=self.Y))
         self.optimizer = tf.train.AdamOptimizer(
             learning_rate=learning_rate).minimize(self.cost)
 
         correct_prediction = tf.equal(
-            tf.argmax(self.logit, 1), tf.argmax(self.Y, 1))
+            tf.argmax(self.logits, 1), tf.argmax(self.Y, 1))
         self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
 
     def predict(self, x_test, keep_prop=1.0):
-        return self.sess.run(self.logit, feed_dict={self.X: x_test, self.keep_prob: keep_prop})
+        return self.sess.run(self.logits, feed_dict={self.X: x_test, self.keep_prob: keep_prop})
 
     def get_accuracy(self, x_test, y_test, keep_prop=1.0):
         return self.sess.run(self.accuracy, feed_dict={self.X: x_test, self.Y: y_test, self.keep_prob: keep_prop})
@@ -136,6 +137,8 @@ def train(self, x_data, y_data, keep_prop=0.7):
 
 sess.run(tf.global_variables_initializer())
 
+print('Learning Started!')
+
 # train my model
 for epoch in range(training_epochs):
     avg_cost_list = np.zeros(len(models))

lab-11-5-mnist_cnn_ensemble_layers.py

@@ -0,0 +1,178 @@
+# Lab 11 MNIST and Deep learning CNN
+# https://www.tensorflow.org/tutorials/layers
+import tensorflow as tf
+import numpy as np
+
+from tensorflow.examples.tutorials.mnist import input_data
+
+tf.set_random_seed(777)  # reproducibility
+
+mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
+# Check out https://www.tensorflow.org/get_started/mnist/beginners for
+# more information about the mnist dataset
+
+# hyper parameters
+learning_rate = 0.001
+training_epochs = 20
+batch_size = 100
+
+
+class Model:
+
+    def __init__(self, sess, name):
+        self.sess = sess
+        self.name = name
+        self._build_net()
+
+    def _build_net(self):
+        with tf.variable_scope(self.name):
+            # dropout (keep_prob) rate  0.7~0.5 on training, but should be 1
+            # for testing
+            self.training = tf.placeholder(tf.bool)
+
+            # input place holders
+            self.X = tf.placeholder(tf.float32, [None, 784])
+
+            # img 28x28x1 (black/white), Input Layer
+            X_img = tf.reshape(self.X, [-1, 28, 28, 1])
+            self.Y = tf.placeholder(tf.float32, [None, 10])
+
+            # L1 ImgIn shape=(?, 28, 28, 1)
+            # W1 = tf.Variable(tf.random_normal([3, 3, 1, 32], stddev=0.01))
+            #    Conv     -> (?, 28, 28, 32)
+            #    Pool     -> (?, 14, 14, 32)
+            # L1 = tf.nn.conv2d(X_img, W1, strides=[1, 1, 1, 1], padding='SAME')
+            # L1 = tf.nn.relu(L1)
+
+            # Convolutional Layer #1
+            conv1 = tf.layers.conv2d(inputs=X_img, filters=32, kernel_size=[3, 3],
+                                     padding="SAME", activation=tf.nn.relu)
+
+            #  L1 = tf.nn.max_pool(L1, ksize=[1, 2, 2, 1],
+            #                    strides=[1, 2, 2, 1], padding='SAME')
+
+            # Pooling Layer #1
+            pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2],
+                                            padding="SAME", strides=2)
+
+            # L1 = tf.nn.dropout(L1, keep_prob=self.keep_prob)
+            dropout1 = tf.layers.dropout(inputs=pool1,
+                                         rate=0.7, training=self.training)
+
+            # Convolutional Layer #2 and Pooling Layer #2
+            conv2 = tf.layers.conv2d(inputs=dropout1, filters=64, kernel_size=[3, 3],
+                                     padding="SAME", activation=tf.nn.relu)
+            pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2],
+                                            padding="SAME", strides=2)
+            dropout2 = tf.layers.dropout(inputs=pool2,
+                                         rate=0.7, training=self.training)
+
+            # Convolutional Layer #2 and Pooling Layer #2
+            conv3 = tf.layers.conv2d(inputs=dropout2, filters=128, kernel_size=[3, 3],
+                                     padding="same", activation=tf.nn.relu)
+            pool3 = tf.layers.max_pooling2d(inputs=conv3, pool_size=[2, 2],
+                                            padding="same", strides=2)
+            dropout3 = tf.layers.dropout(inputs=pool3,
+                                         rate=0.7, training=self.training)
+
+            flat = tf.reshape(dropout3, [-1, 128 * 4 * 4])
+            '''
+            Tensor("Conv2D_2:0", shape=(?, 7, 7, 128), dtype=float32)
+            Tensor("Relu_2:0", shape=(?, 7, 7, 128), dtype=float32)
+            Tensor("MaxPool_2:0", shape=(?, 4, 4, 128), dtype=float32)
+            Tensor("dropout_2/mul:0", shape=(?, 4, 4, 128), dtype=float32)
+            Tensor("Reshape_1:0", shape=(?, 2048), dtype=float32)
+            '''
+
+            # # Dense Layer: 4x4x128 inputs -> 625 outputs
+            # W4 = tf.get_variable("W4", shape=[128 * 4 * 4, 625],
+            #                     initializer=tf.contrib.layers.xavier_initializer())
+            # b4 = tf.Variable(tf.random_normal([625]))
+            # L4 = tf.nn.relu(tf.matmul(L3, W4) + b4)
+            # L4 = tf.nn.dropout(L4, keep_prob=self.keep_prob)
+
+            dense4 = tf.layers.dense(inputs=flat,
+                                     units=625, activation=tf.nn.relu)
+            dropout4 = tf.layers.dropout(inputs=dense4,
+                                         rate=0.5, training=self.training)
+
+            # Logits Layer: L5 Final FC 625 inputs -> 10 outputs
+            self.logits = tf.layers.dense(inputs=dropout4, units=10)
+
+        # define cost/loss & optimizer
+        self.cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
+            logits=self.logits, labels=self.Y))
+        self.optimizer = tf.train.AdamOptimizer(
+            learning_rate=learning_rate).minimize(self.cost)
+
+        correct_prediction = tf.equal(
+            tf.argmax(self.logits, 1), tf.argmax(self.Y, 1))
+        self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
+
+    def predict(self, x_test, training=False):
+        return self.sess.run(self.logits,
+                             feed_dict={self.X: x_test, self.training: training})
+
+    def get_accuracy(self, x_test, y_test, training=False):
+        return self.sess.run(self.accuracy,
+                             feed_dict={self.X: x_test,
+                                        self.Y: y_test, self.training: training})
+
+    def train(self, x_data, y_data, training=True):
+        return self.sess.run([self.cost, self.optimizer], feed_dict={
+            self.X: x_data, self.Y: y_data, self.training: training})
+
+# initialize
+sess = tf.Session()
+
+models = []
+num_models = 2
+for m in range(num_models):
+    models.append(Model(sess, "model" + str(m)))
+
+sess.run(tf.global_variables_initializer())
+
+print('Learning Started!')
+
+# train my model
+for epoch in range(training_epochs):
+    avg_cost_list = np.zeros(len(models))
+    total_batch = int(mnist.train.num_examples / batch_size)
+    for i in range(total_batch):
+        batch_xs, batch_ys = mnist.train.next_batch(batch_size)
+
+        # train each model
+        for m_idx, m in enumerate(models):
+            c, _ = m.train(batch_xs, batch_ys)
+            avg_cost_list[m_idx] += c / total_batch
+
+    print('Epoch:', '%04d' % (epoch + 1), 'cost =', avg_cost_list)
+
+print('Learning Finished!')
+
+# Test model and check accuracy
+test_size = len(mnist.test.labels)
+predictions = np.zeros(test_size * 10).reshape(test_size, 10)
+for m_idx, m in enumerate(models):
+    print(m_idx, 'Accuracy:', m.get_accuracy(
+        mnist.test.images, mnist.test.labels))
+    p = m.predict(mnist.test.images)
+    predictions += p
+
+ensemble_correct_prediction = tf.equal(
+    tf.argmax(predictions, 1), tf.argmax(mnist.test.labels, 1))
+ensemble_accuracy = tf.reduce_mean(
+    tf.cast(ensemble_correct_prediction, tf.float32))
+print('Ensemble accuracy:', sess.run(ensemble_accuracy))
+
+'''
+0 Accuracy: 0.9933
+1 Accuracy: 0.9946
+2 Accuracy: 0.9934
+3 Accuracy: 0.9935
+4 Accuracy: 0.9935
+5 Accuracy: 0.9949
+6 Accuracy: 0.9941
+
+Ensemble accuracy: 0.9952
+'''