EvaluationModels_Mnist2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 221
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6087,
     "status": "ok",
     "timestamp": 1565406480620,
     "user": {
      "displayName": "Krishan Rajaratnam",
      "photoUrl": "https://lh6.googleusercontent.com/-Za30nR0ehyQ/AAAAAAAAAAI/AAAAAAAAGMw/jV8ldTp8928/s64/photo.jpg",
      "userId": "09748671397076320798"
     },
     "user_tz": 240
    },
    "id": "aWICguuPjcck",
    "outputId": "449cea5c-6f1b-41d9-fa50-7ed89c1cb0b5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: keras==2.1.6 in /usr/local/lib/python3.6/dist-packages (2.1.6)\n",
      "Requirement already satisfied: hyperopt==0.1 in /usr/local/lib/python3.6/dist-packages (0.1)\n",
      "Requirement already satisfied: networkx==1.11 in /usr/local/lib/python3.6/dist-packages (1.11)\n",
      "Requirement already satisfied: numpy>=1.9.1 in /usr/local/lib/python3.6/dist-packages (from keras==2.1.6) (1.16.4)\n",
      "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.6/dist-packages (from keras==2.1.6) (1.12.0)\n",
      "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras==2.1.6) (2.8.0)\n",
      "Requirement already satisfied: scipy>=0.14 in /usr/local/lib/python3.6/dist-packages (from keras==2.1.6) (1.3.0)\n",
      "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from keras==2.1.6) (3.13)\n",
      "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from hyperopt==0.1) (0.16.0)\n",
      "Requirement already satisfied: nose in /usr/local/lib/python3.6/dist-packages (from hyperopt==0.1) (1.3.7)\n",
      "Requirement already satisfied: pymongo in /usr/local/lib/python3.6/dist-packages (from hyperopt==0.1) (3.8.0)\n",
      "Requirement already satisfied: decorator>=3.4.0 in /usr/local/lib/python3.6/dist-packages (from networkx==1.11) (4.4.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install keras==2.1.6 hyperopt==0.1 networkx==1.11\n",
    "#Note: This works with Python 3.6.8. For Python 3.7, use hyperopt 0.1.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 122
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 44307,
     "status": "ok",
     "timestamp": 1565566876314,
     "user": {
      "displayName": "Krishan Rajaratnam",
      "photoUrl": "https://lh6.googleusercontent.com/-Za30nR0ehyQ/AAAAAAAAAAI/AAAAAAAAGMw/jV8ldTp8928/s64/photo.jpg",
      "userId": "09748671397076320798"
     },
     "user_tz": 240
    },
    "id": "E5RcVIBdbZFL",
    "outputId": "a4c704fe-d4b2-4bb5-cde6-343bb55dac9a"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import csv \n",
    "\n",
    "from hyperopt import hp\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4574,
     "status": "ok",
     "timestamp": 1565566883477,
     "user": {
      "displayName": "Krishan Rajaratnam",
      "photoUrl": "https://lh6.googleusercontent.com/-Za30nR0ehyQ/AAAAAAAAAAI/AAAAAAAAGMw/jV8ldTp8928/s64/photo.jpg",
      "userId": "09748671397076320798"
     },
     "user_tz": 240
    },
    "id": "QtTpiZKPbZGF",
    "outputId": "fd53ebda-e6f8-4bdc-e488-a9cbe355d94d"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "from os import listdir\n",
    "import sys\n",
    "\n",
    "#Append local dir to path to import custom modules.\n",
    "local_dir = \"\"\n",
    "sys.path.append(local_dir + \"models\")\n",
    "\n",
    "#Sometimes the following modues won't be imported in Colab. Try uploading a new module and adding/removing sys.path.append to get it to work.\n",
    "from simpleCNN import simple_cnn_model\n",
    "from deepNN import deep_nn_model\n",
    "from processModel import MetaModel\n",
    "\n",
    "#load MNIST\n",
    "from keras.datasets import mnist\n",
    "from keras import backend as K\n",
    "\n",
    "import keras\n",
    "from keras import layers\n",
    "from keras.layers import Input, Dense, Activation\n",
    "from keras.models import Model\n",
    "from keras.preprocessing import image\n",
    "from keras import regularizers\n",
    "from keras.optimizers import Adam, Nadam, Adagrad, Adadelta\n",
    "from keras.activations import softmax\n",
    "from keras.losses import sparse_categorical_crossentropy\n",
    "\n",
    "from keras.wrappers.scikit_learn import KerasClassifier, KerasRegressor\n",
    "\n",
    "# getting rid of annoying warnings.\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 119
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3140,
     "status": "ok",
     "timestamp": 1565566901390,
     "user": {
      "displayName": "Krishan Rajaratnam",
      "photoUrl": "https://lh6.googleusercontent.com/-Za30nR0ehyQ/AAAAAAAAAAI/AAAAAAAAGMw/jV8ldTp8928/s64/photo.jpg",
      "userId": "09748671397076320798"
     },
     "user_tz": 240
    },
    "id": "1sNDAamZbZGV",
    "outputId": "71d57b58-7890-47db-9b73-2309a613205d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://s3.amazonaws.com/img-datasets/mnist.npz\n",
      "11493376/11490434 [==============================] - 1s 0us/step\n",
      "x_train shape: (48000, 28, 28, 1)\n",
      "48000 train samples\n",
      "12000 validation samples\n",
      "10000 test samples\n"
     ]
    }
   ],
   "source": [
    "#Preprocess the data\n",
    "\n",
    "# input image dimensions\n",
    "img_rows, img_cols = 28, 28\n",
    "\n",
    "#Number of possible target values for y.\n",
    "classes = 10\n",
    "#File to log model results\n",
    "model_res_file = 'modelResults_mnist.csv'\n",
    "\n",
    "#Load and setup the meta-model.\n",
    "meta_model = MetaModel(local_dir, model_res_file)\n",
    "\n",
    "# the data, split between train and test sets\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "\n",
    "if K.image_data_format() == 'channels_first':\n",
    "    X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)\n",
    "    X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols)\n",
    "    input_shape = (1, img_rows, img_cols)\n",
    "else:\n",
    "    X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1)\n",
    "    X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1)\n",
    "    input_shape = (img_rows, img_cols, 1)\n",
    "\n",
    "def plot_train_val(history):\n",
    "  \"\"\"Method to plot the out of a call to keras model.fit.\n",
    "  \"\"\"\n",
    "  acc = history.history['acc']\n",
    "  val_acc = history.history['val_acc']\n",
    "  loss = history.history['loss']\n",
    "  val_loss = history.history['val_loss']\n",
    "  epochs = range(1, len(acc) + 1)\n",
    "  plt.plot(epochs, acc, 'bo', label='Training acc')\n",
    "  plt.plot(epochs, val_acc, 'b', label='Validation acc')\n",
    "  plt.title('Training and validation accuracy')\n",
    "  plt.legend()\n",
    "  plt.figure()\n",
    "  plt.plot(epochs, loss, 'bo', label='Training loss')\n",
    "  plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
    "  plt.title('Training and validation loss')\n",
    "  plt.legend()\n",
    "  plt.show()\n",
    "    \n",
    "X_train = X_train.astype('float32')\n",
    "X_test = X_test.astype('float32')\n",
    "X_train /= 255\n",
    "X_test /= 255\n",
    "#Get the cross validation data.\n",
    "X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.20, random_state=42)\n",
    "print('x_train shape:', X_train.shape)\n",
    "print(X_train.shape[0], 'train samples')\n",
    "print(X_val.shape[0], 'validation samples')\n",
    "print(X_test.shape[0], 'test samples')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hgRG9a_j_fGS"
   },
   "outputs": [],
   "source": [
    "#Code to write the columns for the modelResults csv. This info will be logged\n",
    "#after the models are trained and evaluated.\n",
    "with open(local_dir + '/' + model_res_file, mode='w') as file:\n",
    "  writer = csv.writer(file, delimiter=',', quotechar='\"', quoting=csv.QUOTE_MINIMAL)\n",
    "  writer.writerow(['networkType', 'trainAcc', 'trainTime (s)', 'testAcc', 'layers', 'epochs', 'activations', 'optim', 'learning_rate','reg_param', 'dropout_prob', 'normTech' , 'misc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "cellView": "both",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4212632,
     "status": "ok",
     "timestamp": 1565410711808,
     "user": {
      "displayName": "Krishan Rajaratnam",
      "photoUrl": "https://lh6.googleusercontent.com/-Za30nR0ehyQ/AAAAAAAAAAI/AAAAAAAAGMw/jV8ldTp8928/s64/photo.jpg",
      "userId": "09748671397076320798"
     },
     "user_tz": 240
    },
    "id": "NUectloXeoWS",
    "outputId": "f33d13f3-4ad7-4aac-cd8e-5355fc5054ff"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0810 03:08:17.972030 140433291319168 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:508: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "W0810 03:08:17.991092 140433291319168 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3144: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "W0810 03:08:18.007775 140433291319168 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3837: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "W0810 03:08:18.042321 140433291319168 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:168: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
      "\n",
      "W0810 03:08:18.043622 140433291319168 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:175: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'classes': 10, 'activation': 'prelu', 'batch_size': 64, 'epochs': 50, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.07489338120701139, 'lr': 0.007202338280154828}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0810 03:08:18.558062 140433291319168 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1801: The name tf.nn.fused_batch_norm is deprecated. Please use tf.compat.v1.nn.fused_batch_norm instead.\n",
      "\n",
      "W0810 03:08:18.651619 140433291319168 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3661: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n",
      "\n",
      "W0810 03:08:18.699115 140433291319168 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/optimizers.py:757: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n",
      "W0810 03:08:18.819005 140433291319168 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 10s 212us/step - loss: 4.6239 - acc: 0.6894 - val_loss: 0.1152 - val_acc: 0.9667\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 7s 146us/step - loss: 0.0812 - acc: 0.9756 - val_loss: 0.0644 - val_acc: 0.9832\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 7s 146us/step - loss: 0.0615 - acc: 0.9814 - val_loss: 0.0573 - val_acc: 0.9833\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 7s 144us/step - loss: 0.0544 - acc: 0.9838 - val_loss: 0.0638 - val_acc: 0.9818\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 7s 144us/step - loss: 0.0490 - acc: 0.9844 - val_loss: 0.0551 - val_acc: 0.9841\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 7s 144us/step - loss: 0.0437 - acc: 0.9863 - val_loss: 0.0607 - val_acc: 0.9841\n",
      "Epoch 7/50\n",
      "48000/48000 [==============================] - 7s 143us/step - loss: 0.0461 - acc: 0.9858 - val_loss: 0.0691 - val_acc: 0.9836\n",
      "12000/12000 [==============================] - 1s 97us/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0810 03:09:12.197845 140433291319168 nn_ops.py:4224] Large dropout rate: 0.555952 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'classes': 10, 'activation': 'prelu', 'batch_size': 64, 'epochs': 50, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.5559519906886558, 'lr': 0.003116704142995985}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0810 03:09:12.322598 140433291319168 nn_ops.py:4224] Large dropout rate: 0.555952 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 8s 159us/step - loss: 1.8338 - acc: 0.8486 - val_loss: 1.6472 - val_acc: 0.8812\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.8064 - acc: 0.9320 - val_loss: 0.0708 - val_acc: 0.9794\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 7s 149us/step - loss: 0.0632 - acc: 0.9808 - val_loss: 0.0580 - val_acc: 0.9830\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 7s 149us/step - loss: 0.0555 - acc: 0.9834 - val_loss: 0.0665 - val_acc: 0.9805\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 7s 149us/step - loss: 0.0480 - acc: 0.9851 - val_loss: 0.0617 - val_acc: 0.9819\n",
      "12000/12000 [==============================] - 1s 100us/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0810 03:09:50.380141 140433291319168 nn_ops.py:4224] Large dropout rate: 0.544313 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'classes': 10, 'activation': 'prelu', 'batch_size': 64, 'epochs': 50, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.544312716464993, 'lr': 0.006583617073750991}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0810 03:09:50.502453 140433291319168 nn_ops.py:4224] Large dropout rate: 0.544313 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 8s 160us/step - loss: 4.8996 - acc: 0.6728 - val_loss: 4.8579 - val_acc: 0.6857\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.3081 - acc: 0.9541 - val_loss: 0.0723 - val_acc: 0.9771\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 7s 153us/step - loss: 0.0725 - acc: 0.9776 - val_loss: 0.0903 - val_acc: 0.9742\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.0648 - acc: 0.9803 - val_loss: 0.0667 - val_acc: 0.9810\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 7s 150us/step - loss: 0.0596 - acc: 0.9813 - val_loss: 0.0785 - val_acc: 0.9778\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 7s 155us/step - loss: 0.0578 - acc: 0.9818 - val_loss: 0.0742 - val_acc: 0.9802\n",
      "12000/12000 [==============================] - 1s 102us/step\n",
      "48000/48000 [==============================] - 5s 100us/step\n",
      "10000/10000 [==============================] - 1s 99us/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0810 03:10:42.555482 140433291319168 nn_ops.py:4224] Large dropout rate: 0.556546 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss = 0.06667118707485498\n",
      "Test Accuracy = 0.9802\n",
      "{'classes': 10, 'activation': 'prelu', 'batch_size': 64, 'epochs': 50, 'optimizer': 'adadelta', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.5565458516255086, 'lr': 0.007009178982546022}\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 8s 164us/step - loss: 1.7464 - acc: 0.4464 - val_loss: 0.9525 - val_acc: 0.6860\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.7488 - acc: 0.7562 - val_loss: 0.6125 - val_acc: 0.7976\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.5418 - acc: 0.8267 - val_loss: 0.4930 - val_acc: 0.8417\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.4556 - acc: 0.8570 - val_loss: 0.4213 - val_acc: 0.8666\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.3985 - acc: 0.8754 - val_loss: 0.3902 - val_acc: 0.8758\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.3649 - acc: 0.8867 - val_loss: 0.3535 - val_acc: 0.8902\n",
      "Epoch 7/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.3347 - acc: 0.8961 - val_loss: 0.3247 - val_acc: 0.9026\n",
      "Epoch 8/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.3141 - acc: 0.9031 - val_loss: 0.3039 - val_acc: 0.9046\n",
      "Epoch 9/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.2963 - acc: 0.9088 - val_loss: 0.2859 - val_acc: 0.9113\n",
      "Epoch 10/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.2767 - acc: 0.9148 - val_loss: 0.2757 - val_acc: 0.9161\n",
      "Epoch 11/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.2651 - acc: 0.9194 - val_loss: 0.2622 - val_acc: 0.9214\n",
      "Epoch 12/50\n",
      "48000/48000 [==============================] - 7s 150us/step - loss: 0.2527 - acc: 0.9233 - val_loss: 0.2450 - val_acc: 0.9298\n",
      "Epoch 13/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.2409 - acc: 0.9269 - val_loss: 0.2380 - val_acc: 0.9300\n",
      "Epoch 14/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.2309 - acc: 0.9288 - val_loss: 0.2275 - val_acc: 0.9327\n",
      "Epoch 15/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.2200 - acc: 0.9331 - val_loss: 0.2237 - val_acc: 0.9330\n",
      "Epoch 16/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.2143 - acc: 0.9351 - val_loss: 0.2148 - val_acc: 0.9371\n",
      "Epoch 17/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.2060 - acc: 0.9387 - val_loss: 0.2106 - val_acc: 0.9383\n",
      "Epoch 18/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.2016 - acc: 0.9393 - val_loss: 0.2008 - val_acc: 0.9387\n",
      "Epoch 19/50\n",
      "48000/48000 [==============================] - 7s 150us/step - loss: 0.1944 - acc: 0.9410 - val_loss: 0.1943 - val_acc: 0.9420\n",
      "Epoch 20/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.1866 - acc: 0.9441 - val_loss: 0.1913 - val_acc: 0.9425\n",
      "Epoch 21/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.1813 - acc: 0.9450 - val_loss: 0.1842 - val_acc: 0.9479\n",
      "Epoch 22/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.1761 - acc: 0.9471 - val_loss: 0.1790 - val_acc: 0.9466\n",
      "Epoch 23/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.1733 - acc: 0.9482 - val_loss: 0.1768 - val_acc: 0.9466\n",
      "Epoch 24/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.1673 - acc: 0.9490 - val_loss: 0.1711 - val_acc: 0.9496\n",
      "Epoch 25/50\n",
      "48000/48000 [==============================] - 7s 150us/step - loss: 0.1640 - acc: 0.9511 - val_loss: 0.1680 - val_acc: 0.9506\n",
      "Epoch 26/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.1583 - acc: 0.9525 - val_loss: 0.1634 - val_acc: 0.9526\n",
      "Epoch 27/50\n",
      "48000/48000 [==============================] - 7s 150us/step - loss: 0.1563 - acc: 0.9531 - val_loss: 0.1619 - val_acc: 0.9522\n",
      "Epoch 28/50\n",
      "48000/48000 [==============================] - 7s 149us/step - loss: 0.1518 - acc: 0.9553 - val_loss: 0.1562 - val_acc: 0.9550\n",
      "Epoch 29/50\n",
      "48000/48000 [==============================] - 7s 150us/step - loss: 0.1512 - acc: 0.9547 - val_loss: 0.1524 - val_acc: 0.9562\n",
      "Epoch 30/50\n",
      "48000/48000 [==============================] - 7s 149us/step - loss: 0.1456 - acc: 0.9571 - val_loss: 0.1496 - val_acc: 0.9566\n",
      "Epoch 31/50\n",
      "48000/48000 [==============================] - 7s 150us/step - loss: 0.1424 - acc: 0.9572 - val_loss: 0.1444 - val_acc: 0.9580\n",
      "Epoch 32/50\n",
      "48000/48000 [==============================] - 7s 149us/step - loss: 0.1413 - acc: 0.9582 - val_loss: 0.1409 - val_acc: 0.9591\n",
      "Epoch 33/50\n",
      "48000/48000 [==============================] - 7s 149us/step - loss: 0.1386 - acc: 0.9593 - val_loss: 0.1386 - val_acc: 0.9597\n",
      "Epoch 34/50\n",
      "48000/48000 [==============================] - 7s 149us/step - loss: 0.1345 - acc: 0.9599 - val_loss: 0.1414 - val_acc: 0.9593\n",
      "Epoch 35/50\n",
      "48000/48000 [==============================] - 7s 148us/step - loss: 0.1326 - acc: 0.9607 - val_loss: 0.1386 - val_acc: 0.9579\n",
      "12000/12000 [==============================] - 1s 96us/step\n",
      "{'classes': 10, 'activation': 'prelu', 'batch_size': 64, 'epochs': 50, 'optimizer': 'adadelta', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.05403044061302784, 'lr': 0.010059716392771194}\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 8s 166us/step - loss: 1.2936 - acc: 0.6043 - val_loss: 0.6561 - val_acc: 0.8315\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 7s 155us/step - loss: 0.5074 - acc: 0.8691 - val_loss: 0.4161 - val_acc: 0.8915\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 8s 157us/step - loss: 0.3702 - acc: 0.8998 - val_loss: 0.3336 - val_acc: 0.9107\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.3104 - acc: 0.9161 - val_loss: 0.2884 - val_acc: 0.9229\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 7s 155us/step - loss: 0.2735 - acc: 0.9246 - val_loss: 0.2578 - val_acc: 0.9303\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 8s 157us/step - loss: 0.2477 - acc: 0.9316 - val_loss: 0.2352 - val_acc: 0.9358\n",
      "Epoch 7/50\n",
      "48000/48000 [==============================] - 8s 157us/step - loss: 0.2272 - acc: 0.9373 - val_loss: 0.2188 - val_acc: 0.9419\n",
      "Epoch 8/50\n",
      "48000/48000 [==============================] - 7s 155us/step - loss: 0.2111 - acc: 0.9419 - val_loss: 0.2044 - val_acc: 0.9447\n",
      "Epoch 9/50\n",
      "48000/48000 [==============================] - 7s 155us/step - loss: 0.1977 - acc: 0.9462 - val_loss: 0.1925 - val_acc: 0.9489\n",
      "Epoch 10/50\n",
      "48000/48000 [==============================] - 7s 156us/step - loss: 0.1862 - acc: 0.9494 - val_loss: 0.1817 - val_acc: 0.9507\n",
      "Epoch 11/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.1763 - acc: 0.9524 - val_loss: 0.1731 - val_acc: 0.9527\n",
      "Epoch 12/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.1672 - acc: 0.9546 - val_loss: 0.1654 - val_acc: 0.9553\n",
      "Epoch 13/50\n",
      "48000/48000 [==============================] - 7s 155us/step - loss: 0.1596 - acc: 0.9569 - val_loss: 0.1580 - val_acc: 0.9577\n",
      "Epoch 14/50\n",
      "48000/48000 [==============================] - 7s 155us/step - loss: 0.1527 - acc: 0.9586 - val_loss: 0.1520 - val_acc: 0.9588\n",
      "Epoch 15/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.1463 - acc: 0.9607 - val_loss: 0.1466 - val_acc: 0.9607\n",
      "Epoch 16/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.1406 - acc: 0.9619 - val_loss: 0.1410 - val_acc: 0.9623\n",
      "Epoch 17/50\n",
      "48000/48000 [==============================] - 7s 153us/step - loss: 0.1351 - acc: 0.9637 - val_loss: 0.1375 - val_acc: 0.9628\n",
      "Epoch 18/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.1303 - acc: 0.9651 - val_loss: 0.1326 - val_acc: 0.9648\n",
      "Epoch 19/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.1264 - acc: 0.9659 - val_loss: 0.1289 - val_acc: 0.9654\n",
      "Epoch 20/50\n",
      "48000/48000 [==============================] - 7s 153us/step - loss: 0.1220 - acc: 0.9672 - val_loss: 0.1249 - val_acc: 0.9667\n",
      "Epoch 21/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.1187 - acc: 0.9679 - val_loss: 0.1218 - val_acc: 0.9673\n",
      "Epoch 22/50\n",
      "48000/48000 [==============================] - 7s 153us/step - loss: 0.1149 - acc: 0.9695 - val_loss: 0.1184 - val_acc: 0.9675\n",
      "Epoch 23/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.1115 - acc: 0.9701 - val_loss: 0.1159 - val_acc: 0.9692\n",
      "Epoch 24/50\n",
      "48000/48000 [==============================] - 7s 153us/step - loss: 0.1085 - acc: 0.9707 - val_loss: 0.1131 - val_acc: 0.9696\n",
      "Epoch 25/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.1062 - acc: 0.9718 - val_loss: 0.1114 - val_acc: 0.9698\n",
      "Epoch 26/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.1036 - acc: 0.9720 - val_loss: 0.1083 - val_acc: 0.9702\n",
      "Epoch 27/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.1005 - acc: 0.9726 - val_loss: 0.1065 - val_acc: 0.9718\n",
      "Epoch 28/50\n",
      "48000/48000 [==============================] - 7s 153us/step - loss: 0.0984 - acc: 0.9739 - val_loss: 0.1037 - val_acc: 0.9721\n",
      "Epoch 29/50\n",
      "48000/48000 [==============================] - 7s 153us/step - loss: 0.0962 - acc: 0.9743 - val_loss: 0.1011 - val_acc: 0.9727\n",
      "Epoch 30/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.0943 - acc: 0.9746 - val_loss: 0.1002 - val_acc: 0.9727\n",
      "Epoch 31/50\n",
      "48000/48000 [==============================] - 7s 153us/step - loss: 0.0921 - acc: 0.9755 - val_loss: 0.0983 - val_acc: 0.9734\n",
      "Epoch 32/50\n",
      "48000/48000 [==============================] - 7s 153us/step - loss: 0.0906 - acc: 0.9758 - val_loss: 0.0970 - val_acc: 0.9738\n",
      "Epoch 33/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.0887 - acc: 0.9762 - val_loss: 0.0960 - val_acc: 0.9739\n",
      "Epoch 34/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.0871 - acc: 0.9767 - val_loss: 0.0942 - val_acc: 0.9742\n",
      "Epoch 35/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.0855 - acc: 0.9775 - val_loss: 0.0928 - val_acc: 0.9747\n",
      "Epoch 36/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.0840 - acc: 0.9776 - val_loss: 0.0913 - val_acc: 0.9750\n",
      "Epoch 37/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.0823 - acc: 0.9783 - val_loss: 0.0901 - val_acc: 0.9754\n",
      "Epoch 38/50\n",
      "48000/48000 [==============================] - 7s 153us/step - loss: 0.0813 - acc: 0.9786 - val_loss: 0.0889 - val_acc: 0.9753\n",
      "Epoch 39/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.0795 - acc: 0.9788 - val_loss: 0.0875 - val_acc: 0.9758\n",
      "Epoch 40/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.0783 - acc: 0.9790 - val_loss: 0.0866 - val_acc: 0.9756\n",
      "Epoch 41/50\n",
      "48000/48000 [==============================] - 7s 153us/step - loss: 0.0771 - acc: 0.9796 - val_loss: 0.0856 - val_acc: 0.9765\n",
      "Epoch 42/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.0762 - acc: 0.9798 - val_loss: 0.0846 - val_acc: 0.9762\n",
      "Epoch 43/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.0749 - acc: 0.9802 - val_loss: 0.0834 - val_acc: 0.9762\n",
      "Epoch 44/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.0737 - acc: 0.9806 - val_loss: 0.0826 - val_acc: 0.9767\n",
      "Epoch 45/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.0732 - acc: 0.9809 - val_loss: 0.0817 - val_acc: 0.9782\n",
      "Epoch 46/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.0721 - acc: 0.9811 - val_loss: 0.0812 - val_acc: 0.9765\n",
      "Epoch 47/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.0710 - acc: 0.9813 - val_loss: 0.0798 - val_acc: 0.9780\n",
      "Epoch 48/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.0698 - acc: 0.9816 - val_loss: 0.0793 - val_acc: 0.9780\n",
      "Epoch 49/50\n",
      "48000/48000 [==============================] - 7s 156us/step - loss: 0.0691 - acc: 0.9818 - val_loss: 0.0788 - val_acc: 0.9775\n",
      "Epoch 50/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.0684 - acc: 0.9820 - val_loss: 0.0778 - val_acc: 0.9778\n",
      "12000/12000 [==============================] - 1s 98us/step\n",
      "{'classes': 10, 'activation': 'prelu', 'batch_size': 64, 'epochs': 50, 'optimizer': 'adadelta', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.2551778586301004, 'lr': 0.007673302839809885}\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 8s 165us/step - loss: 1.5420 - acc: 0.5270 - val_loss: 0.7672 - val_acc: 0.7780\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.5938 - acc: 0.8318 - val_loss: 0.4877 - val_acc: 0.8619\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 7s 153us/step - loss: 0.4304 - acc: 0.8775 - val_loss: 0.3914 - val_acc: 0.8901\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.3627 - acc: 0.8962 - val_loss: 0.3423 - val_acc: 0.9023\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.3215 - acc: 0.9086 - val_loss: 0.3086 - val_acc: 0.9129\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 7s 155us/step - loss: 0.2923 - acc: 0.9165 - val_loss: 0.2820 - val_acc: 0.9200\n",
      "Epoch 7/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.2709 - acc: 0.9229 - val_loss: 0.2652 - val_acc: 0.9242\n",
      "Epoch 8/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.2531 - acc: 0.9271 - val_loss: 0.2466 - val_acc: 0.9307\n",
      "Epoch 9/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.2381 - acc: 0.9331 - val_loss: 0.2348 - val_acc: 0.9337\n",
      "Epoch 10/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.2267 - acc: 0.9353 - val_loss: 0.2205 - val_acc: 0.9383\n",
      "Epoch 11/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.2135 - acc: 0.9396 - val_loss: 0.2118 - val_acc: 0.9423\n",
      "Epoch 12/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.2034 - acc: 0.9426 - val_loss: 0.2019 - val_acc: 0.9452\n",
      "Epoch 13/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.1947 - acc: 0.9447 - val_loss: 0.1925 - val_acc: 0.9466\n",
      "Epoch 14/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.1859 - acc: 0.9481 - val_loss: 0.1868 - val_acc: 0.9473\n",
      "Epoch 15/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.1782 - acc: 0.9497 - val_loss: 0.1786 - val_acc: 0.9508\n",
      "Epoch 16/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.1721 - acc: 0.9514 - val_loss: 0.1735 - val_acc: 0.9517\n",
      "Epoch 17/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.1653 - acc: 0.9546 - val_loss: 0.1685 - val_acc: 0.9537\n",
      "Epoch 18/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.1598 - acc: 0.9546 - val_loss: 0.1626 - val_acc: 0.9543\n",
      "Epoch 19/50\n",
      "48000/48000 [==============================] - 7s 153us/step - loss: 0.1542 - acc: 0.9568 - val_loss: 0.1583 - val_acc: 0.9563\n",
      "Epoch 20/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.1487 - acc: 0.9585 - val_loss: 0.1523 - val_acc: 0.9582\n",
      "Epoch 21/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.1447 - acc: 0.9592 - val_loss: 0.1492 - val_acc: 0.9592\n",
      "Epoch 22/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.1399 - acc: 0.9610 - val_loss: 0.1430 - val_acc: 0.9598\n",
      "Epoch 23/50\n",
      "48000/48000 [==============================] - 7s 155us/step - loss: 0.1357 - acc: 0.9621 - val_loss: 0.1405 - val_acc: 0.9612\n",
      "Epoch 24/50\n",
      "48000/48000 [==============================] - 7s 153us/step - loss: 0.1326 - acc: 0.9630 - val_loss: 0.1364 - val_acc: 0.9627\n",
      "Epoch 25/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.1292 - acc: 0.9640 - val_loss: 0.1331 - val_acc: 0.9647\n",
      "Epoch 26/50\n",
      "48000/48000 [==============================] - 7s 153us/step - loss: 0.1251 - acc: 0.9646 - val_loss: 0.1306 - val_acc: 0.9637\n",
      "Epoch 27/50\n",
      "48000/48000 [==============================] - 7s 153us/step - loss: 0.1223 - acc: 0.9659 - val_loss: 0.1263 - val_acc: 0.9658\n",
      "Epoch 28/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.1192 - acc: 0.9672 - val_loss: 0.1242 - val_acc: 0.9666\n",
      "Epoch 29/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.1165 - acc: 0.9676 - val_loss: 0.1212 - val_acc: 0.9662\n",
      "Epoch 30/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.1151 - acc: 0.9684 - val_loss: 0.1207 - val_acc: 0.9678\n",
      "Epoch 31/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.1115 - acc: 0.9694 - val_loss: 0.1193 - val_acc: 0.9667\n",
      "Epoch 32/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.1089 - acc: 0.9700 - val_loss: 0.1163 - val_acc: 0.9677\n",
      "Epoch 33/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.1068 - acc: 0.9712 - val_loss: 0.1146 - val_acc: 0.9682\n",
      "Epoch 34/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.1046 - acc: 0.9716 - val_loss: 0.1117 - val_acc: 0.9690\n",
      "Epoch 35/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.1030 - acc: 0.9723 - val_loss: 0.1098 - val_acc: 0.9688\n",
      "Epoch 36/50\n",
      "48000/48000 [==============================] - 7s 155us/step - loss: 0.1015 - acc: 0.9723 - val_loss: 0.1085 - val_acc: 0.9702\n",
      "Epoch 37/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.0996 - acc: 0.9724 - val_loss: 0.1067 - val_acc: 0.9699\n",
      "Epoch 38/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.0971 - acc: 0.9737 - val_loss: 0.1046 - val_acc: 0.9711\n",
      "Epoch 39/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.0960 - acc: 0.9730 - val_loss: 0.1039 - val_acc: 0.9698\n",
      "Epoch 40/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 0.0946 - acc: 0.9741 - val_loss: 0.1024 - val_acc: 0.9713\n",
      "Epoch 41/50\n",
      "48000/48000 [==============================] - 7s 155us/step - loss: 0.0931 - acc: 0.9744 - val_loss: 0.1009 - val_acc: 0.9728\n",
      "Epoch 42/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.0911 - acc: 0.9749 - val_loss: 0.0987 - val_acc: 0.9737\n",
      "Epoch 43/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.0898 - acc: 0.9761 - val_loss: 0.0993 - val_acc: 0.9722\n",
      "Epoch 44/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.0884 - acc: 0.9760 - val_loss: 0.0958 - val_acc: 0.9742\n",
      "Epoch 45/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.0877 - acc: 0.9760 - val_loss: 0.0945 - val_acc: 0.9740\n",
      "Epoch 46/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.0859 - acc: 0.9766 - val_loss: 0.0946 - val_acc: 0.9732\n",
      "Epoch 47/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.0851 - acc: 0.9768 - val_loss: 0.0938 - val_acc: 0.9735\n",
      "Epoch 48/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.0841 - acc: 0.9767 - val_loss: 0.0937 - val_acc: 0.9739\n",
      "Epoch 49/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.0829 - acc: 0.9768 - val_loss: 0.0911 - val_acc: 0.9733\n",
      "Epoch 50/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 0.0818 - acc: 0.9772 - val_loss: 0.0911 - val_acc: 0.9742\n",
      "12000/12000 [==============================] - 1s 97us/step\n",
      "48000/48000 [==============================] - 5s 99us/step\n",
      "10000/10000 [==============================] - 1s 101us/step\n",
      "Loss = 0.06597911177426577\n",
      "Test Accuracy = 0.9798\n",
      "{'classes': 10, 'activation': 'prelu', 'batch_size': 64, 'epochs': 50, 'optimizer': 'adagrad', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.1325615491685135, 'lr': 0.015168793934237043}\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 7s 154us/step - loss: 7.7614 - acc: 0.4981 - val_loss: 6.5598 - val_acc: 0.5817\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 7s 140us/step - loss: 4.9149 - acc: 0.6832 - val_loss: 4.7844 - val_acc: 0.6916\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 7s 139us/step - loss: 4.7863 - acc: 0.6953 - val_loss: 4.7773 - val_acc: 0.6943\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 7s 139us/step - loss: 4.7806 - acc: 0.6969 - val_loss: 4.7743 - val_acc: 0.6947\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 7s 140us/step - loss: 4.7764 - acc: 0.6979 - val_loss: 4.7710 - val_acc: 0.6952\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 7s 138us/step - loss: 4.7734 - acc: 0.6991 - val_loss: 4.7693 - val_acc: 0.6963\n",
      "Epoch 7/50\n",
      "48000/48000 [==============================] - 7s 138us/step - loss: 4.7716 - acc: 0.6998 - val_loss: 4.7726 - val_acc: 0.6955\n",
      "Epoch 8/50\n",
      "48000/48000 [==============================] - 7s 139us/step - loss: 4.7695 - acc: 0.7006 - val_loss: 4.7683 - val_acc: 0.6971\n",
      "Epoch 9/50\n",
      "48000/48000 [==============================] - 7s 138us/step - loss: 4.7683 - acc: 0.7011 - val_loss: 4.7706 - val_acc: 0.6966\n",
      "Epoch 10/50\n",
      "48000/48000 [==============================] - 7s 139us/step - loss: 4.7670 - acc: 0.7011 - val_loss: 4.7689 - val_acc: 0.6967\n",
      "12000/12000 [==============================] - 1s 100us/step\n",
      "{'classes': 10, 'activation': 'prelu', 'batch_size': 64, 'epochs': 50, 'optimizer': 'adagrad', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.20704447025845518, 'lr': 0.01052519904399246}\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 7s 155us/step - loss: 8.2896 - acc: 0.4722 - val_loss: 8.1472 - val_acc: 0.4880\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 7s 141us/step - loss: 8.1011 - acc: 0.4926 - val_loss: 8.1393 - val_acc: 0.4906\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 7s 141us/step - loss: 8.0967 - acc: 0.4939 - val_loss: 8.1377 - val_acc: 0.4908\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 7s 141us/step - loss: 8.0938 - acc: 0.4950 - val_loss: 8.1358 - val_acc: 0.4914\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 7s 141us/step - loss: 8.0918 - acc: 0.4956 - val_loss: 8.1351 - val_acc: 0.4918\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 7s 140us/step - loss: 8.0906 - acc: 0.4960 - val_loss: 8.1340 - val_acc: 0.4921\n",
      "Epoch 7/50\n",
      "48000/48000 [==============================] - 7s 141us/step - loss: 8.0895 - acc: 0.4968 - val_loss: 8.1336 - val_acc: 0.4923\n",
      "Epoch 8/50\n",
      "48000/48000 [==============================] - 7s 139us/step - loss: 8.0889 - acc: 0.4968 - val_loss: 8.1348 - val_acc: 0.4918\n",
      "Epoch 9/50\n",
      "48000/48000 [==============================] - 7s 139us/step - loss: 8.0883 - acc: 0.4969 - val_loss: 8.1329 - val_acc: 0.4923\n",
      "Epoch 10/50\n",
      "48000/48000 [==============================] - 7s 138us/step - loss: 8.0878 - acc: 0.4971 - val_loss: 8.1327 - val_acc: 0.4923\n",
      "Epoch 11/50\n",
      "48000/48000 [==============================] - 7s 139us/step - loss: 8.0875 - acc: 0.4973 - val_loss: 8.1334 - val_acc: 0.4923\n",
      "Epoch 12/50\n",
      "48000/48000 [==============================] - 7s 138us/step - loss: 8.0868 - acc: 0.4975 - val_loss: 8.1325 - val_acc: 0.4926\n",
      "Epoch 13/50\n",
      "48000/48000 [==============================] - 7s 139us/step - loss: 8.0866 - acc: 0.4976 - val_loss: 8.1323 - val_acc: 0.4926\n",
      "Epoch 14/50\n",
      "48000/48000 [==============================] - 7s 139us/step - loss: 8.0862 - acc: 0.4976 - val_loss: 8.1330 - val_acc: 0.4925\n",
      "Epoch 15/50\n",
      "48000/48000 [==============================] - 7s 139us/step - loss: 8.0862 - acc: 0.4976 - val_loss: 8.1323 - val_acc: 0.4927\n",
      "Epoch 16/50\n",
      "48000/48000 [==============================] - 7s 138us/step - loss: 8.0857 - acc: 0.4977 - val_loss: 8.1324 - val_acc: 0.4923\n",
      "Epoch 17/50\n",
      "48000/48000 [==============================] - 7s 139us/step - loss: 8.0855 - acc: 0.4980 - val_loss: 8.1326 - val_acc: 0.4927\n",
      "12000/12000 [==============================] - 1s 98us/step\n",
      "{'classes': 10, 'activation': 'prelu', 'batch_size': 64, 'epochs': 50, 'optimizer': 'adagrad', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.1611620085069388, 'lr': 0.005029553101854687}\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 7s 156us/step - loss: 0.8724 - acc: 0.9020 - val_loss: 0.1196 - val_acc: 0.9683\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 7s 138us/step - loss: 0.0999 - acc: 0.9726 - val_loss: 0.0899 - val_acc: 0.9761\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 7s 138us/step - loss: 0.0770 - acc: 0.9786 - val_loss: 0.0772 - val_acc: 0.9788\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 7s 139us/step - loss: 0.0656 - acc: 0.9825 - val_loss: 0.0723 - val_acc: 0.9788\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 7s 138us/step - loss: 0.0579 - acc: 0.9848 - val_loss: 0.0669 - val_acc: 0.9801\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 7s 139us/step - loss: 0.0529 - acc: 0.9857 - val_loss: 0.0614 - val_acc: 0.9834\n",
      "Epoch 7/50\n",
      "48000/48000 [==============================] - 7s 143us/step - loss: 0.0490 - acc: 0.9866 - val_loss: 0.0600 - val_acc: 0.9829\n",
      "Epoch 8/50\n",
      "48000/48000 [==============================] - 7s 140us/step - loss: 0.0456 - acc: 0.9874 - val_loss: 0.0587 - val_acc: 0.9834\n",
      "Epoch 9/50\n",
      "48000/48000 [==============================] - 7s 138us/step - loss: 0.0427 - acc: 0.9885 - val_loss: 0.0564 - val_acc: 0.9836\n",
      "Epoch 10/50\n",
      "48000/48000 [==============================] - 7s 138us/step - loss: 0.0414 - acc: 0.9886 - val_loss: 0.0550 - val_acc: 0.9838\n",
      "Epoch 11/50\n",
      "48000/48000 [==============================] - 7s 138us/step - loss: 0.0391 - acc: 0.9897 - val_loss: 0.0544 - val_acc: 0.9834\n",
      "Epoch 12/50\n",
      "48000/48000 [==============================] - 7s 138us/step - loss: 0.0374 - acc: 0.9897 - val_loss: 0.0538 - val_acc: 0.9840\n",
      "Epoch 13/50\n",
      "48000/48000 [==============================] - 7s 138us/step - loss: 0.0360 - acc: 0.9903 - val_loss: 0.0519 - val_acc: 0.9848\n",
      "Epoch 14/50\n",
      "48000/48000 [==============================] - 7s 139us/step - loss: 0.0343 - acc: 0.9910 - val_loss: 0.0513 - val_acc: 0.9853\n",
      "Epoch 15/50\n",
      "48000/48000 [==============================] - 7s 139us/step - loss: 0.0334 - acc: 0.9911 - val_loss: 0.0504 - val_acc: 0.9856\n",
      "Epoch 16/50\n",
      "48000/48000 [==============================] - 7s 137us/step - loss: 0.0316 - acc: 0.9918 - val_loss: 0.0508 - val_acc: 0.9855\n",
      "Epoch 17/50\n",
      "48000/48000 [==============================] - 7s 137us/step - loss: 0.0314 - acc: 0.9917 - val_loss: 0.0522 - val_acc: 0.9854\n",
      "12000/12000 [==============================] - 1s 101us/step\n",
      "48000/48000 [==============================] - 5s 100us/step\n",
      "10000/10000 [==============================] - 1s 100us/step\n",
      "Loss = 0.04161801513135433\n",
      "Test Accuracy = 0.986\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 64, 'epochs': 50, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.03789402093272474, 'lr': 0.009270372136268783}\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 7s 142us/step - loss: 2.3339 - acc: 0.8234 - val_loss: 0.0857 - val_acc: 0.9744\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 6s 125us/step - loss: 0.0645 - acc: 0.9809 - val_loss: 0.0716 - val_acc: 0.9801\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 6s 126us/step - loss: 0.0475 - acc: 0.9853 - val_loss: 0.0604 - val_acc: 0.9820\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 6s 125us/step - loss: 0.0403 - acc: 0.9869 - val_loss: 0.0615 - val_acc: 0.9832\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 6s 126us/step - loss: 0.0345 - acc: 0.9890 - val_loss: 0.0582 - val_acc: 0.9842\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 6s 126us/step - loss: 0.0315 - acc: 0.9905 - val_loss: 0.0666 - val_acc: 0.9819\n",
      "Epoch 7/50\n",
      "48000/48000 [==============================] - 6s 125us/step - loss: 0.0281 - acc: 0.9902 - val_loss: 0.0568 - val_acc: 0.9848\n",
      "Epoch 8/50\n",
      "48000/48000 [==============================] - 6s 126us/step - loss: 0.0247 - acc: 0.9921 - val_loss: 0.0589 - val_acc: 0.9865\n",
      "Epoch 9/50\n",
      "48000/48000 [==============================] - 6s 125us/step - loss: 0.0221 - acc: 0.9925 - val_loss: 0.0627 - val_acc: 0.9862\n",
      "12000/12000 [==============================] - 1s 93us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 64, 'epochs': 50, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.056754919269764746, 'lr': 0.01575541560900057}\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 7s 142us/step - loss: 7.9154 - acc: 0.4545 - val_loss: 0.3367 - val_acc: 0.9002\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 6s 126us/step - loss: 0.3027 - acc: 0.9086 - val_loss: 0.2543 - val_acc: 0.9255\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 6s 125us/step - loss: 0.2621 - acc: 0.9201 - val_loss: 0.2325 - val_acc: 0.9290\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 6s 126us/step - loss: 0.2434 - acc: 0.9260 - val_loss: 0.2386 - val_acc: 0.9269\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 6s 124us/step - loss: 0.2345 - acc: 0.9287 - val_loss: 0.2310 - val_acc: 0.9290\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 6s 125us/step - loss: 0.2284 - acc: 0.9303 - val_loss: 0.2333 - val_acc: 0.9253\n",
      "Epoch 7/50\n",
      "48000/48000 [==============================] - 6s 125us/step - loss: 0.2235 - acc: 0.9323 - val_loss: 0.2079 - val_acc: 0.9363\n",
      "Epoch 8/50\n",
      "48000/48000 [==============================] - 6s 125us/step - loss: 0.2197 - acc: 0.9333 - val_loss: 0.2214 - val_acc: 0.9344\n",
      "Epoch 9/50\n",
      "48000/48000 [==============================] - 6s 126us/step - loss: 0.2158 - acc: 0.9340 - val_loss: 0.2222 - val_acc: 0.9308\n",
      "12000/12000 [==============================] - 1s 90us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 64, 'epochs': 50, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.5592016005389866, 'lr': 0.010102523270378229}\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 7s 144us/step - loss: 11.1642 - acc: 0.3050 - val_loss: 10.4545 - val_acc: 0.3133\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 6s 126us/step - loss: 1.1420 - acc: 0.8794 - val_loss: 0.1411 - val_acc: 0.9539\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 6s 127us/step - loss: 0.1412 - acc: 0.9565 - val_loss: 0.1417 - val_acc: 0.9566\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 6s 126us/step - loss: 0.1287 - acc: 0.9605 - val_loss: 0.1361 - val_acc: 0.9573\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 6s 126us/step - loss: 0.1206 - acc: 0.9626 - val_loss: 0.1155 - val_acc: 0.9655\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 6s 126us/step - loss: 0.1145 - acc: 0.9644 - val_loss: 0.1198 - val_acc: 0.9634\n",
      "Epoch 7/50\n",
      "48000/48000 [==============================] - 6s 127us/step - loss: 0.1080 - acc: 0.9658 - val_loss: 0.1053 - val_acc: 0.9677\n",
      "Epoch 8/50\n",
      "48000/48000 [==============================] - 6s 126us/step - loss: 0.1041 - acc: 0.9675 - val_loss: 0.1109 - val_acc: 0.9677\n",
      "Epoch 9/50\n",
      "48000/48000 [==============================] - 6s 126us/step - loss: 0.1045 - acc: 0.9680 - val_loss: 0.1117 - val_acc: 0.9662\n",
      "12000/12000 [==============================] - 1s 92us/step\n",
      "48000/48000 [==============================] - 4s 93us/step\n",
      "10000/10000 [==============================] - 1s 94us/step\n",
      "Loss = 0.06275897115222906\n",
      "Test Accuracy = 0.9837\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 64, 'epochs': 50, 'optimizer': 'adadelta', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.10390998661244255, 'lr': 0.0025907930922493636}\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 7s 148us/step - loss: 2.0078 - acc: 0.3057 - val_loss: 1.4297 - val_acc: 0.5547\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 1.1238 - acc: 0.6706 - val_loss: 0.9145 - val_acc: 0.7427\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.7778 - acc: 0.7858 - val_loss: 0.6855 - val_acc: 0.8119\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 6s 128us/step - loss: 0.6137 - acc: 0.8310 - val_loss: 0.5673 - val_acc: 0.8442\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.5230 - acc: 0.8575 - val_loss: 0.4932 - val_acc: 0.8669\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.4640 - acc: 0.8723 - val_loss: 0.4449 - val_acc: 0.8773\n",
      "Epoch 7/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.4232 - acc: 0.8830 - val_loss: 0.4070 - val_acc: 0.8897\n",
      "Epoch 8/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.3928 - acc: 0.8915 - val_loss: 0.3805 - val_acc: 0.8959\n",
      "Epoch 9/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.3684 - acc: 0.8981 - val_loss: 0.3569 - val_acc: 0.9022\n",
      "Epoch 10/50\n",
      "48000/48000 [==============================] - 6s 133us/step - loss: 0.3481 - acc: 0.9031 - val_loss: 0.3397 - val_acc: 0.9069\n",
      "Epoch 11/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.3315 - acc: 0.9077 - val_loss: 0.3230 - val_acc: 0.9113\n",
      "Epoch 12/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.3168 - acc: 0.9114 - val_loss: 0.3096 - val_acc: 0.9160\n",
      "Epoch 13/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.3045 - acc: 0.9156 - val_loss: 0.2988 - val_acc: 0.9184\n",
      "Epoch 14/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.2934 - acc: 0.9183 - val_loss: 0.2871 - val_acc: 0.9225\n",
      "Epoch 15/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.2837 - acc: 0.9211 - val_loss: 0.2780 - val_acc: 0.9253\n",
      "Epoch 16/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.2744 - acc: 0.9230 - val_loss: 0.2699 - val_acc: 0.9267\n",
      "Epoch 17/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.2664 - acc: 0.9259 - val_loss: 0.2624 - val_acc: 0.9286\n",
      "Epoch 18/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.2581 - acc: 0.9286 - val_loss: 0.2535 - val_acc: 0.9333\n",
      "Epoch 19/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.2505 - acc: 0.9297 - val_loss: 0.2461 - val_acc: 0.9347\n",
      "Epoch 20/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.2447 - acc: 0.9320 - val_loss: 0.2403 - val_acc: 0.9356\n",
      "Epoch 21/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.2381 - acc: 0.9340 - val_loss: 0.2351 - val_acc: 0.9368\n",
      "Epoch 22/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.2320 - acc: 0.9358 - val_loss: 0.2288 - val_acc: 0.9391\n",
      "Epoch 23/50\n",
      "48000/48000 [==============================] - 6s 128us/step - loss: 0.2262 - acc: 0.9379 - val_loss: 0.2241 - val_acc: 0.9406\n",
      "Epoch 24/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.2216 - acc: 0.9390 - val_loss: 0.2189 - val_acc: 0.9417\n",
      "Epoch 25/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.2166 - acc: 0.9402 - val_loss: 0.2142 - val_acc: 0.9425\n",
      "Epoch 26/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.2124 - acc: 0.9410 - val_loss: 0.2110 - val_acc: 0.9427\n",
      "Epoch 27/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.2076 - acc: 0.9434 - val_loss: 0.2061 - val_acc: 0.9441\n",
      "Epoch 28/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.2036 - acc: 0.9436 - val_loss: 0.2018 - val_acc: 0.9466\n",
      "Epoch 29/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1996 - acc: 0.9454 - val_loss: 0.1979 - val_acc: 0.9480\n",
      "Epoch 30/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1957 - acc: 0.9462 - val_loss: 0.1943 - val_acc: 0.9490\n",
      "Epoch 31/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1919 - acc: 0.9474 - val_loss: 0.1913 - val_acc: 0.9489\n",
      "Epoch 32/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1888 - acc: 0.9478 - val_loss: 0.1882 - val_acc: 0.9501\n",
      "Epoch 33/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1853 - acc: 0.9486 - val_loss: 0.1866 - val_acc: 0.9508\n",
      "Epoch 34/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1819 - acc: 0.9502 - val_loss: 0.1825 - val_acc: 0.9518\n",
      "Epoch 35/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1790 - acc: 0.9505 - val_loss: 0.1790 - val_acc: 0.9525\n",
      "Epoch 36/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1762 - acc: 0.9521 - val_loss: 0.1757 - val_acc: 0.9530\n",
      "Epoch 37/50\n",
      "48000/48000 [==============================] - 6s 128us/step - loss: 0.1732 - acc: 0.9528 - val_loss: 0.1733 - val_acc: 0.9540\n",
      "Epoch 38/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1706 - acc: 0.9532 - val_loss: 0.1720 - val_acc: 0.9545\n",
      "Epoch 39/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1682 - acc: 0.9536 - val_loss: 0.1676 - val_acc: 0.9548\n",
      "Epoch 40/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1657 - acc: 0.9543 - val_loss: 0.1666 - val_acc: 0.9550\n",
      "Epoch 41/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1628 - acc: 0.9550 - val_loss: 0.1650 - val_acc: 0.9568\n",
      "Epoch 42/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1602 - acc: 0.9559 - val_loss: 0.1619 - val_acc: 0.9573\n",
      "Epoch 43/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1581 - acc: 0.9563 - val_loss: 0.1599 - val_acc: 0.9575\n",
      "Epoch 44/50\n",
      "48000/48000 [==============================] - 6s 128us/step - loss: 0.1557 - acc: 0.9571 - val_loss: 0.1582 - val_acc: 0.9583\n",
      "Epoch 45/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1537 - acc: 0.9573 - val_loss: 0.1558 - val_acc: 0.9581\n",
      "Epoch 46/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1515 - acc: 0.9578 - val_loss: 0.1536 - val_acc: 0.9602\n",
      "Epoch 47/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1495 - acc: 0.9590 - val_loss: 0.1519 - val_acc: 0.9596\n",
      "Epoch 48/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1479 - acc: 0.9593 - val_loss: 0.1502 - val_acc: 0.9597\n",
      "Epoch 49/50\n",
      "48000/48000 [==============================] - 6s 128us/step - loss: 0.1463 - acc: 0.9592 - val_loss: 0.1475 - val_acc: 0.9611\n",
      "Epoch 50/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1440 - acc: 0.9603 - val_loss: 0.1465 - val_acc: 0.9613\n",
      "12000/12000 [==============================] - 1s 92us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 64, 'epochs': 50, 'optimizer': 'adadelta', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.21242927027550662, 'lr': 0.010895033662315368}\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 7s 152us/step - loss: 1.1838 - acc: 0.6370 - val_loss: 0.5960 - val_acc: 0.8417\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.4785 - acc: 0.8690 - val_loss: 0.3985 - val_acc: 0.8902\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.3654 - acc: 0.8970 - val_loss: 0.3308 - val_acc: 0.9067\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.3129 - acc: 0.9118 - val_loss: 0.2954 - val_acc: 0.9161\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.2801 - acc: 0.9208 - val_loss: 0.2671 - val_acc: 0.9260\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.2571 - acc: 0.9278 - val_loss: 0.2458 - val_acc: 0.9324\n",
      "Epoch 7/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.2361 - acc: 0.9332 - val_loss: 0.2286 - val_acc: 0.9337\n",
      "Epoch 8/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.2200 - acc: 0.9384 - val_loss: 0.2146 - val_acc: 0.9396\n",
      "Epoch 9/50\n",
      "48000/48000 [==============================] - 6s 133us/step - loss: 0.2060 - acc: 0.9423 - val_loss: 0.2025 - val_acc: 0.9430\n",
      "Epoch 10/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.1940 - acc: 0.9456 - val_loss: 0.1929 - val_acc: 0.9458\n",
      "Epoch 11/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1836 - acc: 0.9488 - val_loss: 0.1831 - val_acc: 0.9474\n",
      "Epoch 12/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1740 - acc: 0.9514 - val_loss: 0.1731 - val_acc: 0.9529\n",
      "Epoch 13/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1658 - acc: 0.9533 - val_loss: 0.1661 - val_acc: 0.9541\n",
      "Epoch 14/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1576 - acc: 0.9562 - val_loss: 0.1601 - val_acc: 0.9553\n",
      "Epoch 15/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1504 - acc: 0.9586 - val_loss: 0.1543 - val_acc: 0.9571\n",
      "Epoch 16/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1439 - acc: 0.9602 - val_loss: 0.1481 - val_acc: 0.9592\n",
      "Epoch 17/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1386 - acc: 0.9614 - val_loss: 0.1434 - val_acc: 0.9597\n",
      "Epoch 18/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1331 - acc: 0.9629 - val_loss: 0.1378 - val_acc: 0.9615\n",
      "Epoch 19/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1280 - acc: 0.9648 - val_loss: 0.1330 - val_acc: 0.9635\n",
      "Epoch 20/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1239 - acc: 0.9659 - val_loss: 0.1294 - val_acc: 0.9645\n",
      "Epoch 21/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1193 - acc: 0.9673 - val_loss: 0.1262 - val_acc: 0.9653\n",
      "Epoch 22/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1160 - acc: 0.9679 - val_loss: 0.1224 - val_acc: 0.9655\n",
      "Epoch 23/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1127 - acc: 0.9691 - val_loss: 0.1190 - val_acc: 0.9677\n",
      "Epoch 24/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1088 - acc: 0.9703 - val_loss: 0.1164 - val_acc: 0.9667\n",
      "Epoch 25/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1057 - acc: 0.9710 - val_loss: 0.1130 - val_acc: 0.9695\n",
      "Epoch 26/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1037 - acc: 0.9714 - val_loss: 0.1117 - val_acc: 0.9685\n",
      "Epoch 27/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1009 - acc: 0.9725 - val_loss: 0.1078 - val_acc: 0.9702\n",
      "Epoch 28/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.0977 - acc: 0.9738 - val_loss: 0.1063 - val_acc: 0.9705\n",
      "Epoch 29/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.0957 - acc: 0.9732 - val_loss: 0.1041 - val_acc: 0.9706\n",
      "Epoch 30/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.0930 - acc: 0.9746 - val_loss: 0.1024 - val_acc: 0.9723\n",
      "Epoch 31/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.0910 - acc: 0.9754 - val_loss: 0.0997 - val_acc: 0.9716\n",
      "Epoch 32/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.0889 - acc: 0.9759 - val_loss: 0.0993 - val_acc: 0.9736\n",
      "Epoch 33/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.0872 - acc: 0.9767 - val_loss: 0.0969 - val_acc: 0.9741\n",
      "Epoch 34/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.0853 - acc: 0.9772 - val_loss: 0.0959 - val_acc: 0.9737\n",
      "Epoch 35/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.0842 - acc: 0.9768 - val_loss: 0.0916 - val_acc: 0.9740\n",
      "Epoch 36/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.0821 - acc: 0.9779 - val_loss: 0.0911 - val_acc: 0.9748\n",
      "Epoch 37/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.0809 - acc: 0.9778 - val_loss: 0.0906 - val_acc: 0.9753\n",
      "Epoch 38/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.0797 - acc: 0.9784 - val_loss: 0.0891 - val_acc: 0.9752\n",
      "Epoch 39/50\n",
      "48000/48000 [==============================] - 6s 128us/step - loss: 0.0782 - acc: 0.9787 - val_loss: 0.0877 - val_acc: 0.9760\n",
      "Epoch 40/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.0762 - acc: 0.9794 - val_loss: 0.0880 - val_acc: 0.9758\n",
      "Epoch 41/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.0748 - acc: 0.9793 - val_loss: 0.0869 - val_acc: 0.9755\n",
      "Epoch 42/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.0743 - acc: 0.9795 - val_loss: 0.0843 - val_acc: 0.9755\n",
      "Epoch 43/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.0735 - acc: 0.9798 - val_loss: 0.0841 - val_acc: 0.9760\n",
      "Epoch 44/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.0721 - acc: 0.9799 - val_loss: 0.0829 - val_acc: 0.9771\n",
      "Epoch 45/50\n",
      "48000/48000 [==============================] - 6s 128us/step - loss: 0.0707 - acc: 0.9805 - val_loss: 0.0813 - val_acc: 0.9772\n",
      "Epoch 46/50\n",
      "48000/48000 [==============================] - 6s 128us/step - loss: 0.0703 - acc: 0.9811 - val_loss: 0.0809 - val_acc: 0.9782\n",
      "Epoch 47/50\n",
      "48000/48000 [==============================] - 6s 128us/step - loss: 0.0692 - acc: 0.9807 - val_loss: 0.0796 - val_acc: 0.9769\n",
      "Epoch 48/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.0680 - acc: 0.9810 - val_loss: 0.0785 - val_acc: 0.9771\n",
      "Epoch 49/50\n",
      "48000/48000 [==============================] - 6s 128us/step - loss: 0.0674 - acc: 0.9815 - val_loss: 0.0792 - val_acc: 0.9781\n",
      "Epoch 50/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.0662 - acc: 0.9821 - val_loss: 0.0786 - val_acc: 0.9769\n",
      "12000/12000 [==============================] - 1s 97us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 64, 'epochs': 50, 'optimizer': 'adadelta', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.5224676648679641, 'lr': 0.011202641687237388}\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 1.4233 - acc: 0.5493 - val_loss: 0.6804 - val_acc: 0.7807\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.5411 - acc: 0.8294 - val_loss: 0.4379 - val_acc: 0.8618\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.3982 - acc: 0.8759 - val_loss: 0.3523 - val_acc: 0.8911\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.3329 - acc: 0.8996 - val_loss: 0.3062 - val_acc: 0.9066\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.2931 - acc: 0.9122 - val_loss: 0.2783 - val_acc: 0.9137\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.2651 - acc: 0.9214 - val_loss: 0.2523 - val_acc: 0.9247\n",
      "Epoch 7/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.2441 - acc: 0.9273 - val_loss: 0.2286 - val_acc: 0.9305\n",
      "Epoch 8/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.2239 - acc: 0.9330 - val_loss: 0.2123 - val_acc: 0.9368\n",
      "Epoch 9/50\n",
      "48000/48000 [==============================] - 6s 134us/step - loss: 0.2115 - acc: 0.9378 - val_loss: 0.2026 - val_acc: 0.9412\n",
      "Epoch 10/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1977 - acc: 0.9411 - val_loss: 0.1916 - val_acc: 0.9420\n",
      "Epoch 11/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1857 - acc: 0.9454 - val_loss: 0.1803 - val_acc: 0.9471\n",
      "Epoch 12/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1787 - acc: 0.9461 - val_loss: 0.1762 - val_acc: 0.9466\n",
      "Epoch 13/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1694 - acc: 0.9492 - val_loss: 0.1694 - val_acc: 0.9493\n",
      "Epoch 14/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1614 - acc: 0.9524 - val_loss: 0.1558 - val_acc: 0.9547\n",
      "Epoch 15/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1541 - acc: 0.9542 - val_loss: 0.1547 - val_acc: 0.9537\n",
      "Epoch 16/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1493 - acc: 0.9555 - val_loss: 0.1527 - val_acc: 0.9563\n",
      "Epoch 17/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1430 - acc: 0.9584 - val_loss: 0.1419 - val_acc: 0.9585\n",
      "Epoch 18/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1394 - acc: 0.9579 - val_loss: 0.1424 - val_acc: 0.9596\n",
      "Epoch 19/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1360 - acc: 0.9600 - val_loss: 0.1359 - val_acc: 0.9603\n",
      "Epoch 20/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1313 - acc: 0.9613 - val_loss: 0.1324 - val_acc: 0.9620\n",
      "Epoch 21/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1266 - acc: 0.9624 - val_loss: 0.1314 - val_acc: 0.9623\n",
      "Epoch 22/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1238 - acc: 0.9634 - val_loss: 0.1242 - val_acc: 0.9655\n",
      "Epoch 23/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1208 - acc: 0.9641 - val_loss: 0.1247 - val_acc: 0.9640\n",
      "Epoch 24/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1177 - acc: 0.9654 - val_loss: 0.1225 - val_acc: 0.9645\n",
      "Epoch 25/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1149 - acc: 0.9656 - val_loss: 0.1160 - val_acc: 0.9682\n",
      "Epoch 26/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1123 - acc: 0.9664 - val_loss: 0.1148 - val_acc: 0.9688\n",
      "Epoch 27/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1095 - acc: 0.9671 - val_loss: 0.1162 - val_acc: 0.9668\n",
      "Epoch 28/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1089 - acc: 0.9677 - val_loss: 0.1090 - val_acc: 0.9686\n",
      "Epoch 29/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1042 - acc: 0.9690 - val_loss: 0.1092 - val_acc: 0.9685\n",
      "Epoch 30/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1036 - acc: 0.9696 - val_loss: 0.1069 - val_acc: 0.9706\n",
      "Epoch 31/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.1002 - acc: 0.9699 - val_loss: 0.1046 - val_acc: 0.9693\n",
      "Epoch 32/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.0987 - acc: 0.9706 - val_loss: 0.1017 - val_acc: 0.9713\n",
      "Epoch 33/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.0969 - acc: 0.9714 - val_loss: 0.1035 - val_acc: 0.9720\n",
      "Epoch 34/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.0946 - acc: 0.9718 - val_loss: 0.1031 - val_acc: 0.9698\n",
      "12000/12000 [==============================] - 1s 93us/step\n",
      "48000/48000 [==============================] - 5s 94us/step\n",
      "10000/10000 [==============================] - 1s 93us/step\n",
      "Loss = 0.06561726584509014\n",
      "Test Accuracy = 0.98\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 64, 'epochs': 50, 'optimizer': 'adagrad', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.2133052595057478, 'lr': 0.00507844566050672}\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 7s 141us/step - loss: 0.4070 - acc: 0.9270 - val_loss: 0.1221 - val_acc: 0.9661\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 0.0993 - acc: 0.9724 - val_loss: 0.0948 - val_acc: 0.9751\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 0.0778 - acc: 0.9784 - val_loss: 0.0792 - val_acc: 0.9780\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 0.0670 - acc: 0.9812 - val_loss: 0.0742 - val_acc: 0.9795\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 6s 120us/step - loss: 0.0593 - acc: 0.9835 - val_loss: 0.0673 - val_acc: 0.9810\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 0.0547 - acc: 0.9847 - val_loss: 0.0687 - val_acc: 0.9804\n",
      "Epoch 7/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 0.0515 - acc: 0.9858 - val_loss: 0.0630 - val_acc: 0.9812\n",
      "Epoch 8/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 0.0486 - acc: 0.9865 - val_loss: 0.0601 - val_acc: 0.9822\n",
      "Epoch 9/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 0.0454 - acc: 0.9874 - val_loss: 0.0587 - val_acc: 0.9824\n",
      "Epoch 10/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 0.0435 - acc: 0.9879 - val_loss: 0.0560 - val_acc: 0.9828\n",
      "Epoch 11/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 0.0417 - acc: 0.9889 - val_loss: 0.0556 - val_acc: 0.9835\n",
      "Epoch 12/50\n",
      "48000/48000 [==============================] - 6s 120us/step - loss: 0.0399 - acc: 0.9891 - val_loss: 0.0548 - val_acc: 0.9833\n",
      "Epoch 13/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 0.0384 - acc: 0.9892 - val_loss: 0.0553 - val_acc: 0.9832\n",
      "Epoch 14/50\n",
      "48000/48000 [==============================] - 6s 120us/step - loss: 0.0369 - acc: 0.9897 - val_loss: 0.0544 - val_acc: 0.9838\n",
      "Epoch 15/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 0.0354 - acc: 0.9905 - val_loss: 0.0521 - val_acc: 0.9840\n",
      "Epoch 16/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 0.0347 - acc: 0.9907 - val_loss: 0.0519 - val_acc: 0.9849\n",
      "Epoch 17/50\n",
      "48000/48000 [==============================] - 6s 120us/step - loss: 0.0332 - acc: 0.9906 - val_loss: 0.0508 - val_acc: 0.9846\n",
      "Epoch 18/50\n",
      "48000/48000 [==============================] - 6s 120us/step - loss: 0.0321 - acc: 0.9914 - val_loss: 0.0511 - val_acc: 0.9846\n",
      "Epoch 19/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 0.0317 - acc: 0.9911 - val_loss: 0.0512 - val_acc: 0.9851\n",
      "12000/12000 [==============================] - 1s 95us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 64, 'epochs': 50, 'optimizer': 'adagrad', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.2354797380310846, 'lr': 0.0064022270124550005}\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 7s 141us/step - loss: 2.4082 - acc: 0.8166 - val_loss: 1.6843 - val_acc: 0.8772\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 1.6610 - acc: 0.8821 - val_loss: 1.6630 - val_acc: 0.8838\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 6s 118us/step - loss: 1.6455 - acc: 0.8870 - val_loss: 1.6578 - val_acc: 0.8851\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 1.6380 - acc: 0.8893 - val_loss: 1.6504 - val_acc: 0.8872\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 6s 121us/step - loss: 0.2353 - acc: 0.9694 - val_loss: 0.0690 - val_acc: 0.9812\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 0.0530 - acc: 0.9850 - val_loss: 0.0607 - val_acc: 0.9830\n",
      "Epoch 7/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 0.0473 - acc: 0.9862 - val_loss: 0.0587 - val_acc: 0.9822\n",
      "Epoch 8/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 0.0430 - acc: 0.9878 - val_loss: 0.0553 - val_acc: 0.9828\n",
      "Epoch 9/50\n",
      "48000/48000 [==============================] - 6s 118us/step - loss: 0.0403 - acc: 0.9886 - val_loss: 0.0527 - val_acc: 0.9848\n",
      "Epoch 10/50\n",
      "48000/48000 [==============================] - 6s 118us/step - loss: 0.0376 - acc: 0.9892 - val_loss: 0.0563 - val_acc: 0.9832\n",
      "Epoch 11/50\n",
      "48000/48000 [==============================] - 6s 118us/step - loss: 0.0356 - acc: 0.9900 - val_loss: 0.0526 - val_acc: 0.9842\n",
      "Epoch 12/50\n",
      "48000/48000 [==============================] - 6s 118us/step - loss: 0.0338 - acc: 0.9900 - val_loss: 0.0513 - val_acc: 0.9847\n",
      "Epoch 13/50\n",
      "48000/48000 [==============================] - 6s 118us/step - loss: 0.0320 - acc: 0.9913 - val_loss: 0.0493 - val_acc: 0.9857\n",
      "Epoch 14/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 0.0316 - acc: 0.9913 - val_loss: 0.0518 - val_acc: 0.9844\n",
      "Epoch 15/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 0.0302 - acc: 0.9918 - val_loss: 0.0483 - val_acc: 0.9857\n",
      "Epoch 16/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 0.0290 - acc: 0.9919 - val_loss: 0.0481 - val_acc: 0.9862\n",
      "Epoch 17/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 0.0276 - acc: 0.9927 - val_loss: 0.0476 - val_acc: 0.9861\n",
      "Epoch 18/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 0.0269 - acc: 0.9928 - val_loss: 0.0493 - val_acc: 0.9854\n",
      "Epoch 19/50\n",
      "48000/48000 [==============================] - 6s 118us/step - loss: 0.0266 - acc: 0.9926 - val_loss: 0.0498 - val_acc: 0.9853\n",
      "12000/12000 [==============================] - 1s 95us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 64, 'epochs': 50, 'optimizer': 'adagrad', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.03289654934603687, 'lr': 0.0062854552799091745}\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 7s 143us/step - loss: 0.4413 - acc: 0.9298 - val_loss: 0.1078 - val_acc: 0.9705\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 0.0867 - acc: 0.9760 - val_loss: 0.0803 - val_acc: 0.9775\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 0.0671 - acc: 0.9811 - val_loss: 0.0717 - val_acc: 0.9803\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 0.0570 - acc: 0.9845 - val_loss: 0.0645 - val_acc: 0.9818\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 0.0509 - acc: 0.9864 - val_loss: 0.0589 - val_acc: 0.9827\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 6s 120us/step - loss: 0.0463 - acc: 0.9876 - val_loss: 0.0570 - val_acc: 0.9834\n",
      "Epoch 7/50\n",
      "48000/48000 [==============================] - 6s 120us/step - loss: 0.0429 - acc: 0.9886 - val_loss: 0.0544 - val_acc: 0.9843\n",
      "Epoch 8/50\n",
      "48000/48000 [==============================] - 6s 120us/step - loss: 0.0395 - acc: 0.9896 - val_loss: 0.0523 - val_acc: 0.9842\n",
      "Epoch 9/50\n",
      "48000/48000 [==============================] - 6s 120us/step - loss: 0.0374 - acc: 0.9902 - val_loss: 0.0529 - val_acc: 0.9842\n",
      "Epoch 10/50\n",
      "48000/48000 [==============================] - 6s 119us/step - loss: 0.0354 - acc: 0.9906 - val_loss: 0.0528 - val_acc: 0.9838\n",
      "12000/12000 [==============================] - 1s 95us/step\n",
      "48000/48000 [==============================] - 5s 96us/step\n",
      "10000/10000 [==============================] - 1s 96us/step\n",
      "Loss = 0.041548941666819154\n",
      "Test Accuracy = 0.9862\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 64, 'epochs': 50, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.09974106885919276, 'lr': 0.01481177431122598}\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 7s 155us/step - loss: 0.8472 - acc: 0.9003 - val_loss: 0.1344 - val_acc: 0.9617\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 6s 128us/step - loss: 0.1443 - acc: 0.9599 - val_loss: 0.1295 - val_acc: 0.9643\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 6s 128us/step - loss: 0.1370 - acc: 0.9637 - val_loss: 0.1572 - val_acc: 0.9624\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 6s 128us/step - loss: 0.1270 - acc: 0.9658 - val_loss: 0.1271 - val_acc: 0.9664\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 6s 128us/step - loss: 0.1263 - acc: 0.9679 - val_loss: 0.1527 - val_acc: 0.9645\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 6s 128us/step - loss: 0.1434 - acc: 0.9659 - val_loss: 0.1183 - val_acc: 0.9696\n",
      "Epoch 7/50\n",
      "48000/48000 [==============================] - 6s 128us/step - loss: 0.1141 - acc: 0.9709 - val_loss: 0.1420 - val_acc: 0.9662\n",
      "Epoch 8/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1275 - acc: 0.9718 - val_loss: 0.1638 - val_acc: 0.9623\n",
      "12000/12000 [==============================] - 1s 96us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 64, 'epochs': 50, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.04970092779073536, 'lr': 0.006145926259900306}\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 8s 158us/step - loss: 0.2787 - acc: 0.9310 - val_loss: 0.1400 - val_acc: 0.9616\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1113 - acc: 0.9687 - val_loss: 0.0974 - val_acc: 0.9737\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 6s 128us/step - loss: 0.1020 - acc: 0.9721 - val_loss: 0.1119 - val_acc: 0.9686\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 6s 128us/step - loss: 0.0891 - acc: 0.9758 - val_loss: 0.1108 - val_acc: 0.9730\n",
      "12000/12000 [==============================] - 1s 97us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 64, 'epochs': 50, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.44339634420746776, 'lr': 0.0037068682021205255}\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 8s 160us/step - loss: 0.2158 - acc: 0.9393 - val_loss: 0.0938 - val_acc: 0.9716\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.0895 - acc: 0.9733 - val_loss: 0.0945 - val_acc: 0.9716\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.0753 - acc: 0.9773 - val_loss: 0.0797 - val_acc: 0.9766\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.0719 - acc: 0.9790 - val_loss: 0.0855 - val_acc: 0.9779\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.0699 - acc: 0.9795 - val_loss: 0.0893 - val_acc: 0.9774\n",
      "12000/12000 [==============================] - 1s 98us/step\n",
      "48000/48000 [==============================] - 5s 98us/step\n",
      "10000/10000 [==============================] - 1s 98us/step\n",
      "Loss = 0.08175238422370748\n",
      "Test Accuracy = 0.9781\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 64, 'epochs': 50, 'optimizer': 'adadelta', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.2691197874035688, 'lr': 0.007835846881945946}\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 8s 163us/step - loss: 1.4352 - acc: 0.5719 - val_loss: 0.8736 - val_acc: 0.7818\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.6880 - acc: 0.8276 - val_loss: 0.5646 - val_acc: 0.8538\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.5032 - acc: 0.8677 - val_loss: 0.4557 - val_acc: 0.8787\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.4248 - acc: 0.8852 - val_loss: 0.3974 - val_acc: 0.8935\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.3799 - acc: 0.8945 - val_loss: 0.3615 - val_acc: 0.9011\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.3508 - acc: 0.9024 - val_loss: 0.3380 - val_acc: 0.9040\n",
      "Epoch 7/50\n",
      "48000/48000 [==============================] - 7s 136us/step - loss: 0.3272 - acc: 0.9088 - val_loss: 0.3156 - val_acc: 0.9097\n",
      "Epoch 8/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.3108 - acc: 0.9127 - val_loss: 0.3004 - val_acc: 0.9160\n",
      "Epoch 9/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.2968 - acc: 0.9166 - val_loss: 0.2877 - val_acc: 0.9185\n",
      "Epoch 10/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.2826 - acc: 0.9208 - val_loss: 0.2759 - val_acc: 0.9232\n",
      "Epoch 11/50\n",
      "48000/48000 [==============================] - 6s 128us/step - loss: 0.2720 - acc: 0.9236 - val_loss: 0.2661 - val_acc: 0.9267\n",
      "Epoch 12/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.2612 - acc: 0.9270 - val_loss: 0.2566 - val_acc: 0.9287\n",
      "Epoch 13/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.2524 - acc: 0.9289 - val_loss: 0.2475 - val_acc: 0.9305\n",
      "Epoch 14/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.2455 - acc: 0.9304 - val_loss: 0.2416 - val_acc: 0.9316\n",
      "Epoch 15/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.2370 - acc: 0.9333 - val_loss: 0.2350 - val_acc: 0.9328\n",
      "Epoch 16/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.2298 - acc: 0.9355 - val_loss: 0.2275 - val_acc: 0.9347\n",
      "Epoch 17/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.2245 - acc: 0.9373 - val_loss: 0.2212 - val_acc: 0.9382\n",
      "Epoch 18/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.2179 - acc: 0.9393 - val_loss: 0.2159 - val_acc: 0.9397\n",
      "Epoch 19/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.2129 - acc: 0.9408 - val_loss: 0.2104 - val_acc: 0.9422\n",
      "Epoch 20/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.2068 - acc: 0.9416 - val_loss: 0.2051 - val_acc: 0.9430\n",
      "Epoch 21/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.2011 - acc: 0.9445 - val_loss: 0.2010 - val_acc: 0.9423\n",
      "Epoch 22/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1968 - acc: 0.9454 - val_loss: 0.1968 - val_acc: 0.9454\n",
      "Epoch 23/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1918 - acc: 0.9468 - val_loss: 0.1926 - val_acc: 0.9473\n",
      "Epoch 24/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1875 - acc: 0.9478 - val_loss: 0.1876 - val_acc: 0.9473\n",
      "Epoch 25/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1837 - acc: 0.9491 - val_loss: 0.1830 - val_acc: 0.9504\n",
      "Epoch 26/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1791 - acc: 0.9499 - val_loss: 0.1812 - val_acc: 0.9485\n",
      "Epoch 27/50\n",
      "48000/48000 [==============================] - 6s 128us/step - loss: 0.1761 - acc: 0.9511 - val_loss: 0.1760 - val_acc: 0.9520\n",
      "Epoch 28/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1712 - acc: 0.9530 - val_loss: 0.1729 - val_acc: 0.9531\n",
      "Epoch 29/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1679 - acc: 0.9535 - val_loss: 0.1695 - val_acc: 0.9527\n",
      "Epoch 30/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1650 - acc: 0.9547 - val_loss: 0.1681 - val_acc: 0.9532\n",
      "Epoch 31/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1620 - acc: 0.9551 - val_loss: 0.1643 - val_acc: 0.9553\n",
      "Epoch 32/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1587 - acc: 0.9564 - val_loss: 0.1620 - val_acc: 0.9563\n",
      "Epoch 33/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.1551 - acc: 0.9576 - val_loss: 0.1603 - val_acc: 0.9555\n",
      "Epoch 34/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.1528 - acc: 0.9581 - val_loss: 0.1571 - val_acc: 0.9561\n",
      "Epoch 35/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.1504 - acc: 0.9590 - val_loss: 0.1528 - val_acc: 0.9586\n",
      "Epoch 36/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.1475 - acc: 0.9597 - val_loss: 0.1502 - val_acc: 0.9597\n",
      "Epoch 37/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1451 - acc: 0.9607 - val_loss: 0.1498 - val_acc: 0.9585\n",
      "Epoch 38/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.1428 - acc: 0.9610 - val_loss: 0.1490 - val_acc: 0.9592\n",
      "Epoch 39/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1404 - acc: 0.9619 - val_loss: 0.1450 - val_acc: 0.9595\n",
      "Epoch 40/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1385 - acc: 0.9625 - val_loss: 0.1407 - val_acc: 0.9625\n",
      "Epoch 41/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1356 - acc: 0.9636 - val_loss: 0.1393 - val_acc: 0.9623\n",
      "Epoch 42/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.1341 - acc: 0.9636 - val_loss: 0.1391 - val_acc: 0.9638\n",
      "Epoch 43/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.1325 - acc: 0.9635 - val_loss: 0.1374 - val_acc: 0.9630\n",
      "Epoch 44/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1299 - acc: 0.9642 - val_loss: 0.1357 - val_acc: 0.9627\n",
      "Epoch 45/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.1277 - acc: 0.9653 - val_loss: 0.1325 - val_acc: 0.9627\n",
      "Epoch 46/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.1268 - acc: 0.9650 - val_loss: 0.1324 - val_acc: 0.9633\n",
      "Epoch 47/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1246 - acc: 0.9667 - val_loss: 0.1310 - val_acc: 0.9635\n",
      "Epoch 48/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.1232 - acc: 0.9667 - val_loss: 0.1290 - val_acc: 0.9640\n",
      "Epoch 49/50\n",
      "48000/48000 [==============================] - 6s 129us/step - loss: 0.1209 - acc: 0.9674 - val_loss: 0.1272 - val_acc: 0.9647\n",
      "Epoch 50/50\n",
      "48000/48000 [==============================] - 6s 130us/step - loss: 0.1195 - acc: 0.9676 - val_loss: 0.1268 - val_acc: 0.9647\n",
      "12000/12000 [==============================] - 1s 96us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 64, 'epochs': 50, 'optimizer': 'adadelta', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.47481806825567874, 'lr': 0.013951550012444155}\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 8s 163us/step - loss: 1.1765 - acc: 0.6375 - val_loss: 0.6524 - val_acc: 0.8129\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.5286 - acc: 0.8513 - val_loss: 0.4477 - val_acc: 0.8692\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.4091 - acc: 0.8828 - val_loss: 0.3751 - val_acc: 0.8914\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.3577 - acc: 0.8958 - val_loss: 0.3341 - val_acc: 0.9030\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.3232 - acc: 0.9059 - val_loss: 0.3019 - val_acc: 0.9104\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 6s 135us/step - loss: 0.2968 - acc: 0.9140 - val_loss: 0.2817 - val_acc: 0.9186\n",
      "Epoch 7/50\n",
      "48000/48000 [==============================] - 6s 135us/step - loss: 0.2793 - acc: 0.9186 - val_loss: 0.2670 - val_acc: 0.9237\n",
      "Epoch 8/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.2627 - acc: 0.9231 - val_loss: 0.2527 - val_acc: 0.9281\n",
      "Epoch 9/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.2485 - acc: 0.9284 - val_loss: 0.2386 - val_acc: 0.9308\n",
      "Epoch 10/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.2362 - acc: 0.9316 - val_loss: 0.2291 - val_acc: 0.9327\n",
      "Epoch 11/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.2243 - acc: 0.9362 - val_loss: 0.2201 - val_acc: 0.9365\n",
      "Epoch 12/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.2143 - acc: 0.9386 - val_loss: 0.2105 - val_acc: 0.9409\n",
      "Epoch 13/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.2062 - acc: 0.9410 - val_loss: 0.2045 - val_acc: 0.9412\n",
      "Epoch 14/50\n",
      "48000/48000 [==============================] - 6s 133us/step - loss: 0.1965 - acc: 0.9443 - val_loss: 0.1951 - val_acc: 0.9446\n",
      "Epoch 15/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.1897 - acc: 0.9459 - val_loss: 0.1882 - val_acc: 0.9454\n",
      "Epoch 16/50\n",
      "48000/48000 [==============================] - 6s 133us/step - loss: 0.1810 - acc: 0.9491 - val_loss: 0.1807 - val_acc: 0.9492\n",
      "Epoch 17/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.1764 - acc: 0.9495 - val_loss: 0.1757 - val_acc: 0.9507\n",
      "Epoch 18/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.1691 - acc: 0.9515 - val_loss: 0.1700 - val_acc: 0.9523\n",
      "Epoch 19/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.1638 - acc: 0.9538 - val_loss: 0.1661 - val_acc: 0.9531\n",
      "Epoch 20/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.1584 - acc: 0.9559 - val_loss: 0.1588 - val_acc: 0.9553\n",
      "Epoch 21/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.1531 - acc: 0.9581 - val_loss: 0.1549 - val_acc: 0.9563\n",
      "Epoch 22/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.1493 - acc: 0.9574 - val_loss: 0.1513 - val_acc: 0.9579\n",
      "Epoch 23/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.1449 - acc: 0.9591 - val_loss: 0.1480 - val_acc: 0.9573\n",
      "Epoch 24/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.1412 - acc: 0.9605 - val_loss: 0.1435 - val_acc: 0.9583\n",
      "Epoch 25/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.1379 - acc: 0.9618 - val_loss: 0.1380 - val_acc: 0.9619\n",
      "Epoch 26/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.1332 - acc: 0.9630 - val_loss: 0.1361 - val_acc: 0.9613\n",
      "Epoch 27/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.1300 - acc: 0.9634 - val_loss: 0.1325 - val_acc: 0.9625\n",
      "Epoch 28/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.1281 - acc: 0.9639 - val_loss: 0.1304 - val_acc: 0.9647\n",
      "Epoch 29/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.1247 - acc: 0.9658 - val_loss: 0.1278 - val_acc: 0.9633\n",
      "Epoch 30/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.1221 - acc: 0.9654 - val_loss: 0.1284 - val_acc: 0.9637\n",
      "Epoch 31/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.1182 - acc: 0.9667 - val_loss: 0.1235 - val_acc: 0.9651\n",
      "Epoch 32/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.1160 - acc: 0.9683 - val_loss: 0.1188 - val_acc: 0.9677\n",
      "Epoch 33/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.1142 - acc: 0.9684 - val_loss: 0.1188 - val_acc: 0.9667\n",
      "Epoch 34/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.1117 - acc: 0.9691 - val_loss: 0.1165 - val_acc: 0.9677\n",
      "Epoch 35/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.1100 - acc: 0.9694 - val_loss: 0.1131 - val_acc: 0.9686\n",
      "Epoch 36/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.1071 - acc: 0.9703 - val_loss: 0.1109 - val_acc: 0.9690\n",
      "Epoch 37/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.1056 - acc: 0.9703 - val_loss: 0.1092 - val_acc: 0.9700\n",
      "Epoch 38/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.1037 - acc: 0.9713 - val_loss: 0.1090 - val_acc: 0.9690\n",
      "Epoch 39/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.1014 - acc: 0.9715 - val_loss: 0.1080 - val_acc: 0.9699\n",
      "Epoch 40/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.0997 - acc: 0.9732 - val_loss: 0.1052 - val_acc: 0.9718\n",
      "Epoch 41/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.0986 - acc: 0.9732 - val_loss: 0.1047 - val_acc: 0.9703\n",
      "Epoch 42/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.0970 - acc: 0.9737 - val_loss: 0.1036 - val_acc: 0.9710\n",
      "Epoch 43/50\n",
      "48000/48000 [==============================] - 6s 134us/step - loss: 0.0955 - acc: 0.9730 - val_loss: 0.1012 - val_acc: 0.9718\n",
      "Epoch 44/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.0947 - acc: 0.9737 - val_loss: 0.0999 - val_acc: 0.9728\n",
      "Epoch 45/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.0926 - acc: 0.9745 - val_loss: 0.0984 - val_acc: 0.9716\n",
      "Epoch 46/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.0916 - acc: 0.9743 - val_loss: 0.0975 - val_acc: 0.9720\n",
      "Epoch 47/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.0897 - acc: 0.9755 - val_loss: 0.0945 - val_acc: 0.9737\n",
      "Epoch 48/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.0880 - acc: 0.9766 - val_loss: 0.0959 - val_acc: 0.9729\n",
      "Epoch 49/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.0882 - acc: 0.9758 - val_loss: 0.0951 - val_acc: 0.9726\n",
      "12000/12000 [==============================] - 1s 98us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 64, 'epochs': 50, 'optimizer': 'adadelta', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.48735975657962205, 'lr': 0.004380681557645655}\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 8s 165us/step - loss: 2.0804 - acc: 0.2950 - val_loss: 1.4062 - val_acc: 0.5547\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 6s 133us/step - loss: 1.0885 - acc: 0.6825 - val_loss: 0.8802 - val_acc: 0.7532\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.7482 - acc: 0.7939 - val_loss: 0.6588 - val_acc: 0.8164\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.5968 - acc: 0.8334 - val_loss: 0.5527 - val_acc: 0.8442\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 6s 134us/step - loss: 0.5119 - acc: 0.8556 - val_loss: 0.4848 - val_acc: 0.8620\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 6s 135us/step - loss: 0.4634 - acc: 0.8678 - val_loss: 0.4409 - val_acc: 0.8754\n",
      "Epoch 7/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.4243 - acc: 0.8796 - val_loss: 0.4086 - val_acc: 0.8822\n",
      "Epoch 8/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.3973 - acc: 0.8866 - val_loss: 0.3838 - val_acc: 0.8888\n",
      "Epoch 9/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.3741 - acc: 0.8916 - val_loss: 0.3675 - val_acc: 0.8942\n",
      "Epoch 10/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.3572 - acc: 0.8970 - val_loss: 0.3499 - val_acc: 0.8981\n",
      "Epoch 11/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.3443 - acc: 0.9008 - val_loss: 0.3358 - val_acc: 0.9048\n",
      "Epoch 12/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.3329 - acc: 0.9034 - val_loss: 0.3251 - val_acc: 0.9043\n",
      "Epoch 13/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.3210 - acc: 0.9068 - val_loss: 0.3162 - val_acc: 0.9082\n",
      "Epoch 14/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.3098 - acc: 0.9103 - val_loss: 0.3053 - val_acc: 0.9110\n",
      "Epoch 15/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.3017 - acc: 0.9111 - val_loss: 0.2977 - val_acc: 0.9141\n",
      "Epoch 16/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.2946 - acc: 0.9152 - val_loss: 0.2882 - val_acc: 0.9186\n",
      "Epoch 17/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.2875 - acc: 0.9164 - val_loss: 0.2817 - val_acc: 0.9182\n",
      "Epoch 18/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.2804 - acc: 0.9177 - val_loss: 0.2739 - val_acc: 0.9196\n",
      "Epoch 19/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.2753 - acc: 0.9203 - val_loss: 0.2669 - val_acc: 0.9238\n",
      "Epoch 20/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.2686 - acc: 0.9220 - val_loss: 0.2614 - val_acc: 0.9244\n",
      "Epoch 21/50\n",
      "48000/48000 [==============================] - 6s 133us/step - loss: 0.2627 - acc: 0.9235 - val_loss: 0.2585 - val_acc: 0.9245\n",
      "Epoch 22/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.2580 - acc: 0.9248 - val_loss: 0.2526 - val_acc: 0.9270\n",
      "Epoch 23/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.2510 - acc: 0.9267 - val_loss: 0.2486 - val_acc: 0.9291\n",
      "Epoch 24/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.2466 - acc: 0.9282 - val_loss: 0.2446 - val_acc: 0.9287\n",
      "Epoch 25/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.2419 - acc: 0.9302 - val_loss: 0.2429 - val_acc: 0.9315\n",
      "Epoch 26/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.2376 - acc: 0.9307 - val_loss: 0.2353 - val_acc: 0.9317\n",
      "Epoch 27/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.2336 - acc: 0.9332 - val_loss: 0.2320 - val_acc: 0.9294\n",
      "Epoch 28/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.2304 - acc: 0.9325 - val_loss: 0.2299 - val_acc: 0.9322\n",
      "Epoch 29/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.2263 - acc: 0.9347 - val_loss: 0.2254 - val_acc: 0.9350\n",
      "Epoch 30/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.2222 - acc: 0.9362 - val_loss: 0.2219 - val_acc: 0.9363\n",
      "Epoch 31/50\n",
      "48000/48000 [==============================] - 6s 131us/step - loss: 0.2188 - acc: 0.9366 - val_loss: 0.2180 - val_acc: 0.9359\n",
      "Epoch 32/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.2163 - acc: 0.9381 - val_loss: 0.2153 - val_acc: 0.9376\n",
      "Epoch 33/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.2128 - acc: 0.9391 - val_loss: 0.2105 - val_acc: 0.9401\n",
      "Epoch 34/50\n",
      "48000/48000 [==============================] - 6s 133us/step - loss: 0.2105 - acc: 0.9385 - val_loss: 0.2110 - val_acc: 0.9411\n",
      "Epoch 35/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.2049 - acc: 0.9409 - val_loss: 0.2079 - val_acc: 0.9411\n",
      "Epoch 36/50\n",
      "48000/48000 [==============================] - 6s 133us/step - loss: 0.2029 - acc: 0.9412 - val_loss: 0.2051 - val_acc: 0.9399\n",
      "Epoch 37/50\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.2009 - acc: 0.9429 - val_loss: 0.2016 - val_acc: 0.9428\n",
      "Epoch 38/50\n",
      "48000/48000 [==============================] - 6s 133us/step - loss: 0.1996 - acc: 0.9428 - val_loss: 0.1978 - val_acc: 0.9452\n",
      "Epoch 39/50\n",
      "48000/48000 [==============================] - 6s 133us/step - loss: 0.1964 - acc: 0.9434 - val_loss: 0.1978 - val_acc: 0.9441\n",
      "Epoch 40/50\n",
      "48000/48000 [==============================] - 6s 134us/step - loss: 0.1920 - acc: 0.9444 - val_loss: 0.1949 - val_acc: 0.9443\n",
      "Epoch 41/50\n",
      "48000/48000 [==============================] - 6s 134us/step - loss: 0.1893 - acc: 0.9451 - val_loss: 0.1896 - val_acc: 0.9457\n",
      "Epoch 42/50\n",
      "48000/48000 [==============================] - 7s 137us/step - loss: 0.1882 - acc: 0.9452 - val_loss: 0.1875 - val_acc: 0.9456\n",
      "Epoch 43/50\n",
      "48000/48000 [==============================] - 6s 133us/step - loss: 0.1857 - acc: 0.9467 - val_loss: 0.1857 - val_acc: 0.9465\n",
      "Epoch 44/50\n",
      "48000/48000 [==============================] - 6s 134us/step - loss: 0.1819 - acc: 0.9484 - val_loss: 0.1838 - val_acc: 0.9497\n",
      "Epoch 45/50\n",
      "48000/48000 [==============================] - 6s 134us/step - loss: 0.1822 - acc: 0.9474 - val_loss: 0.1817 - val_acc: 0.9484\n",
      "Epoch 46/50\n",
      "48000/48000 [==============================] - 6s 133us/step - loss: 0.1795 - acc: 0.9481 - val_loss: 0.1797 - val_acc: 0.9486\n",
      "Epoch 47/50\n",
      "48000/48000 [==============================] - 6s 134us/step - loss: 0.1762 - acc: 0.9493 - val_loss: 0.1786 - val_acc: 0.9495\n",
      "Epoch 48/50\n",
      "48000/48000 [==============================] - 6s 134us/step - loss: 0.1744 - acc: 0.9499 - val_loss: 0.1757 - val_acc: 0.9505\n",
      "Epoch 49/50\n",
      "48000/48000 [==============================] - 6s 133us/step - loss: 0.1732 - acc: 0.9511 - val_loss: 0.1755 - val_acc: 0.9495\n",
      "Epoch 50/50\n",
      "48000/48000 [==============================] - 6s 134us/step - loss: 0.1721 - acc: 0.9507 - val_loss: 0.1715 - val_acc: 0.9507\n",
      "12000/12000 [==============================] - 1s 100us/step\n",
      "48000/48000 [==============================] - 5s 99us/step\n",
      "10000/10000 [==============================] - 1s 104us/step\n",
      "Loss = 0.0846389085970819\n",
      "Test Accuracy = 0.9748\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 64, 'epochs': 50, 'optimizer': 'adagrad', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.13249221258033286, 'lr': 0.004153097464621249}\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 8s 159us/step - loss: 0.2927 - acc: 0.9157 - val_loss: 0.1915 - val_acc: 0.9469\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 6s 123us/step - loss: 0.1737 - acc: 0.9517 - val_loss: 0.1550 - val_acc: 0.9595\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 6s 126us/step - loss: 0.1414 - acc: 0.9613 - val_loss: 0.1323 - val_acc: 0.9657\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 6s 124us/step - loss: 0.1224 - acc: 0.9666 - val_loss: 0.1203 - val_acc: 0.9680\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 0.1095 - acc: 0.9705 - val_loss: 0.1085 - val_acc: 0.9702\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 6s 121us/step - loss: 0.1001 - acc: 0.9734 - val_loss: 0.1019 - val_acc: 0.9730\n",
      "Epoch 7/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 0.0936 - acc: 0.9750 - val_loss: 0.0971 - val_acc: 0.9748\n",
      "Epoch 8/50\n",
      "48000/48000 [==============================] - 6s 123us/step - loss: 0.0870 - acc: 0.9773 - val_loss: 0.0914 - val_acc: 0.9758\n",
      "Epoch 9/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 0.0828 - acc: 0.9786 - val_loss: 0.0868 - val_acc: 0.9759\n",
      "Epoch 10/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 0.0787 - acc: 0.9794 - val_loss: 0.0849 - val_acc: 0.9761\n",
      "Epoch 11/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 0.0749 - acc: 0.9802 - val_loss: 0.0807 - val_acc: 0.9786\n",
      "Epoch 12/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 0.0722 - acc: 0.9812 - val_loss: 0.0794 - val_acc: 0.9797\n",
      "Epoch 13/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 0.0691 - acc: 0.9823 - val_loss: 0.0762 - val_acc: 0.9798\n",
      "Epoch 14/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 0.0680 - acc: 0.9820 - val_loss: 0.0754 - val_acc: 0.9788\n",
      "Epoch 15/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 0.0646 - acc: 0.9828 - val_loss: 0.0743 - val_acc: 0.9792\n",
      "Epoch 16/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 0.0625 - acc: 0.9836 - val_loss: 0.0722 - val_acc: 0.9808\n",
      "Epoch 17/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 0.0611 - acc: 0.9835 - val_loss: 0.0710 - val_acc: 0.9799\n",
      "Epoch 18/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 0.0598 - acc: 0.9847 - val_loss: 0.0684 - val_acc: 0.9812\n",
      "Epoch 19/50\n",
      "48000/48000 [==============================] - 6s 121us/step - loss: 0.0581 - acc: 0.9847 - val_loss: 0.0685 - val_acc: 0.9817\n",
      "Epoch 20/50\n",
      "48000/48000 [==============================] - 6s 121us/step - loss: 0.0567 - acc: 0.9851 - val_loss: 0.0674 - val_acc: 0.9817\n",
      "Epoch 21/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 0.0554 - acc: 0.9855 - val_loss: 0.0652 - val_acc: 0.9822\n",
      "Epoch 22/50\n",
      "48000/48000 [==============================] - 6s 121us/step - loss: 0.0541 - acc: 0.9860 - val_loss: 0.0659 - val_acc: 0.9811\n",
      "Epoch 23/50\n",
      "48000/48000 [==============================] - 6s 121us/step - loss: 0.0532 - acc: 0.9863 - val_loss: 0.0656 - val_acc: 0.9818\n",
      "12000/12000 [==============================] - 1s 104us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 64, 'epochs': 50, 'optimizer': 'adagrad', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.33840033940071834, 'lr': 0.014293772361968776}\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 8s 158us/step - loss: 8.2959 - acc: 0.4658 - val_loss: 6.6190 - val_acc: 0.5683\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 5.3181 - acc: 0.6524 - val_loss: 3.2412 - val_acc: 0.7802\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 6s 121us/step - loss: 3.2214 - acc: 0.7854 - val_loss: 3.2222 - val_acc: 0.7867\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 6s 121us/step - loss: 3.0037 - acc: 0.8017 - val_loss: 1.6448 - val_acc: 0.8818\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 6s 123us/step - loss: 1.6178 - acc: 0.8876 - val_loss: 1.6360 - val_acc: 0.8853\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 1.6104 - acc: 0.8900 - val_loss: 1.6309 - val_acc: 0.8859\n",
      "Epoch 7/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 1.6054 - acc: 0.8915 - val_loss: 1.6322 - val_acc: 0.8864\n",
      "Epoch 8/50\n",
      "48000/48000 [==============================] - 6s 121us/step - loss: 1.6024 - acc: 0.8920 - val_loss: 1.6261 - val_acc: 0.8882\n",
      "Epoch 9/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 1.6002 - acc: 0.8929 - val_loss: 1.6265 - val_acc: 0.8879\n",
      "Epoch 10/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 1.5992 - acc: 0.8933 - val_loss: 1.6257 - val_acc: 0.8882\n",
      "Epoch 11/50\n",
      "48000/48000 [==============================] - 6s 123us/step - loss: 1.5972 - acc: 0.8936 - val_loss: 1.6232 - val_acc: 0.8882\n",
      "Epoch 12/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 1.5957 - acc: 0.8941 - val_loss: 1.6225 - val_acc: 0.8892\n",
      "Epoch 13/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 1.5941 - acc: 0.8945 - val_loss: 1.6237 - val_acc: 0.8889\n",
      "Epoch 14/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 1.5934 - acc: 0.8953 - val_loss: 1.6206 - val_acc: 0.8888\n",
      "Epoch 15/50\n",
      "48000/48000 [==============================] - 6s 123us/step - loss: 1.5923 - acc: 0.8950 - val_loss: 1.6219 - val_acc: 0.8892\n",
      "Epoch 16/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 1.5919 - acc: 0.8953 - val_loss: 1.6207 - val_acc: 0.8893\n",
      "12000/12000 [==============================] - 1s 100us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 64, 'epochs': 50, 'optimizer': 'adagrad', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7fb8fd6cfe48>], 'dropout_prob': 0.5280522074385521, 'lr': 0.0087813847779937}\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 8s 164us/step - loss: 4.4849 - acc: 0.6914 - val_loss: 3.1833 - val_acc: 0.7790\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 6s 124us/step - loss: 3.1228 - acc: 0.7886 - val_loss: 3.1515 - val_acc: 0.7895\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 3.1031 - acc: 0.7943 - val_loss: 3.1439 - val_acc: 0.7905\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 3.0936 - acc: 0.7975 - val_loss: 3.1369 - val_acc: 0.7946\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 3.0880 - acc: 0.7987 - val_loss: 3.1314 - val_acc: 0.7947\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 3.0834 - acc: 0.7999 - val_loss: 3.1292 - val_acc: 0.7963\n",
      "Epoch 7/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 3.0814 - acc: 0.8006 - val_loss: 3.1270 - val_acc: 0.7960\n",
      "Epoch 8/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 3.0793 - acc: 0.8015 - val_loss: 3.1249 - val_acc: 0.7976\n",
      "Epoch 9/50\n",
      "48000/48000 [==============================] - 6s 124us/step - loss: 3.0769 - acc: 0.8017 - val_loss: 3.1240 - val_acc: 0.7980\n",
      "Epoch 10/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 3.0747 - acc: 0.8030 - val_loss: 3.1226 - val_acc: 0.7984\n",
      "Epoch 11/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 3.0733 - acc: 0.8032 - val_loss: 3.1237 - val_acc: 0.7975\n",
      "Epoch 12/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 3.0730 - acc: 0.8034 - val_loss: 3.1205 - val_acc: 0.7987\n",
      "Epoch 13/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 3.0717 - acc: 0.8037 - val_loss: 3.1217 - val_acc: 0.7976\n",
      "Epoch 14/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 3.0699 - acc: 0.8043 - val_loss: 3.1187 - val_acc: 0.7994\n",
      "Epoch 15/50\n",
      "48000/48000 [==============================] - 6s 125us/step - loss: 3.0700 - acc: 0.8040 - val_loss: 3.1208 - val_acc: 0.7983\n",
      "Epoch 16/50\n",
      "48000/48000 [==============================] - 6s 128us/step - loss: 3.0691 - acc: 0.8045 - val_loss: 3.1182 - val_acc: 0.7995\n",
      "Epoch 17/50\n",
      "48000/48000 [==============================] - 6s 123us/step - loss: 3.0690 - acc: 0.8043 - val_loss: 3.1190 - val_acc: 0.7994\n",
      "Epoch 18/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 3.0681 - acc: 0.8045 - val_loss: 3.1178 - val_acc: 0.7989\n",
      "Epoch 19/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 3.0677 - acc: 0.8048 - val_loss: 3.1191 - val_acc: 0.7988\n",
      "Epoch 20/50\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 3.0666 - acc: 0.8051 - val_loss: 3.1182 - val_acc: 0.7990\n",
      "12000/12000 [==============================] - 1s 98us/step\n",
      "48000/48000 [==============================] - 5s 100us/step\n",
      "10000/10000 [==============================] - 1s 100us/step\n",
      "Loss = 0.05984352563619614\n",
      "Test Accuracy = 0.9823\n"
     ]
    }
   ],
   "source": [
    "#A simple single stage convolutional network.\n",
    "\n",
    "#The neural network architectures will be built using combinations of the following parameters.\n",
    "params = {'classes' : [classes],\n",
    "          'activation' : ['prelu', 'relu', 'tanh'],\n",
    "          'batch_size': [64],\n",
    "          'epochs': [50],\n",
    "          'optimizer': ['rmsprop', 'adadelta', 'adagrad'],\n",
    "          'loss': ['sparse_categorical_crossentropy'],\n",
    "          'reg_param': [0],\n",
    "          'callbacks' : [[keras.callbacks.EarlyStopping(monitor='val_loss',\n",
    "                              min_delta=0,\n",
    "                              patience=2,\n",
    "                              verbose=0, mode='auto')]]}\n",
    "\n",
    "#Hyperopt will optimize the following parameters in the given ranges.\n",
    "search_space = {'lr': hp.loguniform('lr', -6, -4), 'dropout_prob' : hp.uniform('dropout_prob', 0.0, 0.6)}\n",
    "\n",
    "#The call to process_model will compile, train and evaulate the above models and optimize the hyper parameters.\n",
    "#The output will be saved to a file and the output details will be recorded in the model_res_file. \n",
    "model_name = \"simpleCNN\"\n",
    "meta_model.max_evals = 3\n",
    "model_results = meta_model.process_model(params, search_space, simple_cnn_model, model_name, [X_train, X_val, X_test, y_train, y_val, y_test], \n",
    "              [model_name, 0, 0, 0, 'layers', 'epochs', 'activation', 'optimizer', 'lr','reg_param', 'dropout_prob', np.NaN , np.NaN])\n",
    "\n",
    "#Now lets plot the results.\n",
    "#history = model_results[0][0]\n",
    "#plot_train_val(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 170
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2607987,
     "status": "ok",
     "timestamp": 1565386375362,
     "user": {
      "displayName": "Krishan Rajaratnam",
      "photoUrl": "https://lh6.googleusercontent.com/-Za30nR0ehyQ/AAAAAAAAAAI/AAAAAAAAGMw/jV8ldTp8928/s64/photo.jpg",
      "userId": "09748671397076320798"
     },
     "user_tz": 240
    },
    "id": "-lnFLmjM2j4w",
    "outputId": "d2498581-6194-4959-e51f-5daa8ffddab0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3/3 [00:51<00:00, 17.66s/it, best loss: 0.1948245023091634]\n",
      "100%|██████████| 3/3 [09:40<00:00, 193.47s/it, best loss: 0.24558969349662463]\n",
      "100%|██████████| 3/3 [04:07<00:00, 82.91s/it, best loss: 0.07493167001754046]\n",
      "100%|██████████| 3/3 [01:09<00:00, 21.24s/it, best loss: 0.28629955342567215]\n",
      "100%|██████████| 3/3 [09:45<00:00, 194.29s/it, best loss: 0.10016030426323413]\n",
      "100%|██████████| 3/3 [02:47<00:00, 59.36s/it, best loss: 0.07556310591295672]\n",
      "100%|██████████| 3/3 [00:48<00:00, 16.27s/it, best loss: 0.36509187636400264]\n",
      "100%|██████████| 3/3 [08:35<00:00, 182.43s/it, best loss: 0.25183458972970646]\n",
      "100%|██████████| 3/3 [04:39<00:00, 88.41s/it, best loss: 0.0881148839861465]\n"
     ]
    }
   ],
   "source": [
    "#A basic single layer neural network\n",
    "\n",
    "#Reshape the data for processing by neural nets\n",
    "X_train = X_train.reshape(X_train.shape[0], img_rows * img_cols)\n",
    "X_val = X_val.reshape(X_val.shape[0],  img_rows * img_cols)\n",
    "X_test = X_test.reshape(X_test.shape[0],  img_rows * img_cols)\n",
    "\n",
    "params = {'layers': [[500]],\n",
    "          'classes' : [classes],\n",
    "          'activation' : ['sigmoid', 'relu', 'tanh'],\n",
    "          'batch_size': [64],\n",
    "          'epochs': [50],\n",
    "          'optimizer': ['sgd', 'adadelta', 'adagrad'],\n",
    "          'loss': ['sparse_categorical_crossentropy'],\n",
    "          'reg_param': [0],\n",
    "          'callbacks' : [[keras.callbacks.EarlyStopping(monitor='val_loss',\n",
    "                                                        \n",
    "                              min_delta=0,\n",
    "                              patience=2,\n",
    "                              verbose=0, mode='auto')]]}\n",
    "\n",
    "search_space = {'lr': hp.loguniform('lr', -5, -3), 'dropout_prob' : hp.uniform('dropout_prob', 0.0, 0.5)}\n",
    "\n",
    "#Compile the model\n",
    "model_name = \"simpleNN\"\n",
    "meta_model.max_evals = 3\n",
    "meta_model.verbose = 0\n",
    "model_results = meta_model.process_model(params, search_space, deep_nn_model, model_name, [X_train, X_val, X_test, y_train, y_val, y_test], \n",
    "              [model_name, 0, 0, 0, 'layers','epochs', 'activation', 'optimizer', 'lr','reg_param', 'dropout_prob', np.NaN , np.NaN])\n",
    "\n",
    "#Now lets plot the results.\n",
    "#history = model_results[0][0]\n",
    "#plot_train_val(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "021pZecbZKOg"
   },
   "outputs": [],
   "source": [
    "#A basic multi-layer neural network\n",
    "\n",
    "#Reshape the data for processing by neural nets\n",
    "X_train = X_train.reshape(X_train.shape[0], img_rows * img_cols)\n",
    "X_val = X_val.reshape(X_val.shape[0],  img_rows * img_cols)\n",
    "X_test = X_test.reshape(X_test.shape[0],  img_rows * img_cols)\n",
    "\n",
    "params = {'layers': [[1000, 500, 500, 500]],\n",
    "          'classes' : [classes],\n",
    "          'activation' : ['sigmoid', 'relu', 'tanh'],\n",
    "          'batch_size': [64],\n",
    "          'epochs': [50],\n",
    "          'optimizer': ['sgd', 'adadelta', 'adagrad'],\n",
    "          'loss': ['sparse_categorical_crossentropy'],\n",
    "          'reg_param' : [0],\n",
    "          'callbacks' : [[keras.callbacks.EarlyStopping(monitor='val_loss',\n",
    "                              min_delta=0,\n",
    "                              patience=2,\n",
    "                              verbose=0, mode='auto')]]}\n",
    "\n",
    "search_space = {'lr': hp.loguniform('lr', -6, -4), 'dropout_prob' : hp.uniform('dropout_prob', 0.0, 0.6)}\n",
    "\n",
    "#Compile the model\n",
    "model_name = \"deepNN\"\n",
    "meta_model.max_evals = 3\n",
    "meta_model.verbose = 0\n",
    "model_results = meta_model.process_model(params, search_space, deep_nn_model, model_name, [X_train, X_val, X_test, y_train, y_val, y_test], \n",
    "              [model_name, 0, 0, 0, 'layers', 'epochs', 'activation', 'optimizer', 'lr','reg_param', 'dropout_prob', np.NaN , np.NaN])\n",
    "\n",
    "#Now lets plot the results.\n",
    "#history = model_results[0][0]\n",
    "#plot_train_val(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1081,
     "status": "ok",
     "timestamp": 1565567090852,
     "user": {
      "displayName": "Krishan Rajaratnam",
      "photoUrl": "https://lh6.googleusercontent.com/-Za30nR0ehyQ/AAAAAAAAAAI/AAAAAAAAGMw/jV8ldTp8928/s64/photo.jpg",
      "userId": "09748671397076320798"
     },
     "user_tz": 240
    },
    "id": "8PN20m2IwjqX",
    "outputId": "5bacc73b-c308-4b26-d23f-861a967a0aa1"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>networkType</th>\n",
       "      <th>trainAcc</th>\n",
       "      <th>trainTime (s)</th>\n",
       "      <th>testAcc</th>\n",
       "      <th>layers</th>\n",
       "      <th>epochs</th>\n",
       "      <th>activations</th>\n",
       "      <th>optim</th>\n",
       "      <th>learning_rate</th>\n",
       "      <th>reg_param</th>\n",
       "      <th>dropout_prob</th>\n",
       "      <th>normTech</th>\n",
       "      <th>misc</th>\n",
       "      <th>testAccPerT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>simpleCNN</td>\n",
       "      <td>0.991625</td>\n",
       "      <td>128.453790</td>\n",
       "      <td>0.9862</td>\n",
       "      <td>NaN</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>adagrad</td>\n",
       "      <td>0.005078</td>\n",
       "      <td>0</td>\n",
       "      <td>0.213305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.767747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>simpleCNN</td>\n",
       "      <td>0.991896</td>\n",
       "      <td>127.392681</td>\n",
       "      <td>0.9860</td>\n",
       "      <td>NaN</td>\n",
       "      <td>50</td>\n",
       "      <td>prelu</td>\n",
       "      <td>adagrad</td>\n",
       "      <td>0.005030</td>\n",
       "      <td>0</td>\n",
       "      <td>0.161162</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.773985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>simpleCNN</td>\n",
       "      <td>0.994375</td>\n",
       "      <td>65.084619</td>\n",
       "      <td>0.9837</td>\n",
       "      <td>NaN</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>rmsprop</td>\n",
       "      <td>0.009270</td>\n",
       "      <td>0</td>\n",
       "      <td>0.037894</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.511417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>simpleCNN</td>\n",
       "      <td>0.987083</td>\n",
       "      <td>159.142339</td>\n",
       "      <td>0.9823</td>\n",
       "      <td>NaN</td>\n",
       "      <td>50</td>\n",
       "      <td>tanh</td>\n",
       "      <td>adagrad</td>\n",
       "      <td>0.004153</td>\n",
       "      <td>0</td>\n",
       "      <td>0.132492</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.617246</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>simpleCNN</td>\n",
       "      <td>0.985812</td>\n",
       "      <td>41.673621</td>\n",
       "      <td>0.9802</td>\n",
       "      <td>NaN</td>\n",
       "      <td>50</td>\n",
       "      <td>prelu</td>\n",
       "      <td>rmsprop</td>\n",
       "      <td>0.003117</td>\n",
       "      <td>0</td>\n",
       "      <td>0.555952</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.352087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>simpleCNN</td>\n",
       "      <td>0.982187</td>\n",
       "      <td>366.359355</td>\n",
       "      <td>0.9800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>adadelta</td>\n",
       "      <td>0.010895</td>\n",
       "      <td>0</td>\n",
       "      <td>0.212429</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.267497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>simpleCNN</td>\n",
       "      <td>0.982083</td>\n",
       "      <td>417.585114</td>\n",
       "      <td>0.9798</td>\n",
       "      <td>NaN</td>\n",
       "      <td>50</td>\n",
       "      <td>prelu</td>\n",
       "      <td>adadelta</td>\n",
       "      <td>0.010060</td>\n",
       "      <td>0</td>\n",
       "      <td>0.054030</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.234635</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.998708</td>\n",
       "      <td>82.006778</td>\n",
       "      <td>0.9794</td>\n",
       "      <td>200</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>adagrad</td>\n",
       "      <td>0.021706</td>\n",
       "      <td>0</td>\n",
       "      <td>0.002780</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.194292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>deepNN</td>\n",
       "      <td>0.999250</td>\n",
       "      <td>89.528609</td>\n",
       "      <td>0.9791</td>\n",
       "      <td>1000-500-500-500</td>\n",
       "      <td>50</td>\n",
       "      <td>tanh</td>\n",
       "      <td>adagrad</td>\n",
       "      <td>0.009503</td>\n",
       "      <td>0</td>\n",
       "      <td>0.031815</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.093617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.996854</td>\n",
       "      <td>60.324102</td>\n",
       "      <td>0.9788</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>adagrad</td>\n",
       "      <td>0.044943</td>\n",
       "      <td>0</td>\n",
       "      <td>0.189095</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.622569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>simpleCNN</td>\n",
       "      <td>0.980458</td>\n",
       "      <td>38.673631</td>\n",
       "      <td>0.9781</td>\n",
       "      <td>NaN</td>\n",
       "      <td>50</td>\n",
       "      <td>tanh</td>\n",
       "      <td>rmsprop</td>\n",
       "      <td>0.003707</td>\n",
       "      <td>0</td>\n",
       "      <td>0.443396</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.529113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.992458</td>\n",
       "      <td>83.005636</td>\n",
       "      <td>0.9775</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>adagrad</td>\n",
       "      <td>0.038847</td>\n",
       "      <td>0</td>\n",
       "      <td>0.269767</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.177631</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>deepNN</td>\n",
       "      <td>0.993625</td>\n",
       "      <td>69.414156</td>\n",
       "      <td>0.9761</td>\n",
       "      <td>1000-500-500-500</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>adagrad</td>\n",
       "      <td>0.014893</td>\n",
       "      <td>0</td>\n",
       "      <td>0.314792</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.406197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>simpleCNN</td>\n",
       "      <td>0.975917</td>\n",
       "      <td>370.800954</td>\n",
       "      <td>0.9748</td>\n",
       "      <td>NaN</td>\n",
       "      <td>50</td>\n",
       "      <td>tanh</td>\n",
       "      <td>adadelta</td>\n",
       "      <td>0.013952</td>\n",
       "      <td>0</td>\n",
       "      <td>0.474818</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.262890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.992417</td>\n",
       "      <td>70.773251</td>\n",
       "      <td>0.9735</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>tanh</td>\n",
       "      <td>adagrad</td>\n",
       "      <td>0.032645</td>\n",
       "      <td>0</td>\n",
       "      <td>0.206737</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.375520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.989708</td>\n",
       "      <td>30.119569</td>\n",
       "      <td>0.9723</td>\n",
       "      <td>200</td>\n",
       "      <td>50</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.008676</td>\n",
       "      <td>0</td>\n",
       "      <td>0.039979</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.228134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.981646</td>\n",
       "      <td>247.256589</td>\n",
       "      <td>0.9709</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>adadelta</td>\n",
       "      <td>0.045467</td>\n",
       "      <td>0</td>\n",
       "      <td>0.129459</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.392669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.981146</td>\n",
       "      <td>131.708493</td>\n",
       "      <td>0.9704</td>\n",
       "      <td>200</td>\n",
       "      <td>50</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>adagrad</td>\n",
       "      <td>0.018916</td>\n",
       "      <td>0</td>\n",
       "      <td>0.101433</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.736779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>deepNN</td>\n",
       "      <td>0.971688</td>\n",
       "      <td>200.194306</td>\n",
       "      <td>0.9650</td>\n",
       "      <td>1000-500-500-500</td>\n",
       "      <td>50</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>adagrad</td>\n",
       "      <td>0.005531</td>\n",
       "      <td>0</td>\n",
       "      <td>0.143509</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.482032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.977938</td>\n",
       "      <td>65.780685</td>\n",
       "      <td>0.9648</td>\n",
       "      <td>50</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>adagrad</td>\n",
       "      <td>0.026247</td>\n",
       "      <td>0</td>\n",
       "      <td>0.037190</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.466692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.972688</td>\n",
       "      <td>184.267622</td>\n",
       "      <td>0.9639</td>\n",
       "      <td>200</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>adadelta</td>\n",
       "      <td>0.049686</td>\n",
       "      <td>0</td>\n",
       "      <td>0.089821</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.523098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.973792</td>\n",
       "      <td>61.045387</td>\n",
       "      <td>0.9612</td>\n",
       "      <td>200</td>\n",
       "      <td>50</td>\n",
       "      <td>tanh</td>\n",
       "      <td>adagrad</td>\n",
       "      <td>0.029675</td>\n",
       "      <td>0</td>\n",
       "      <td>0.356438</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.574566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.980875</td>\n",
       "      <td>32.123013</td>\n",
       "      <td>0.9610</td>\n",
       "      <td>50</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.009015</td>\n",
       "      <td>0</td>\n",
       "      <td>0.009621</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.991625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>deepNN</td>\n",
       "      <td>0.966063</td>\n",
       "      <td>397.130157</td>\n",
       "      <td>0.9606</td>\n",
       "      <td>1000-500-500-500</td>\n",
       "      <td>50</td>\n",
       "      <td>tanh</td>\n",
       "      <td>adadelta</td>\n",
       "      <td>0.007354</td>\n",
       "      <td>0</td>\n",
       "      <td>0.077252</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.241885</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>deepNN</td>\n",
       "      <td>0.967063</td>\n",
       "      <td>33.903111</td>\n",
       "      <td>0.9598</td>\n",
       "      <td>1000-500-500-500</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.009032</td>\n",
       "      <td>0</td>\n",
       "      <td>0.013524</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.831009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>deepNN</td>\n",
       "      <td>0.960646</td>\n",
       "      <td>119.375441</td>\n",
       "      <td>0.9543</td>\n",
       "      <td>50-25-25-25</td>\n",
       "      <td>50</td>\n",
       "      <td>tanh</td>\n",
       "      <td>adagrad</td>\n",
       "      <td>0.004221</td>\n",
       "      <td>0</td>\n",
       "      <td>0.016827</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.799411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.963875</td>\n",
       "      <td>91.175906</td>\n",
       "      <td>0.9535</td>\n",
       "      <td>50</td>\n",
       "      <td>50</td>\n",
       "      <td>tanh</td>\n",
       "      <td>adagrad</td>\n",
       "      <td>0.016927</td>\n",
       "      <td>0</td>\n",
       "      <td>0.146942</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.045781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.957979</td>\n",
       "      <td>23.458261</td>\n",
       "      <td>0.9478</td>\n",
       "      <td>200</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.008842</td>\n",
       "      <td>0</td>\n",
       "      <td>0.244561</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.040368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.960812</td>\n",
       "      <td>34.997603</td>\n",
       "      <td>0.9476</td>\n",
       "      <td>50</td>\n",
       "      <td>50</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.007866</td>\n",
       "      <td>0</td>\n",
       "      <td>0.233522</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.707614</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.953229</td>\n",
       "      <td>21.134256</td>\n",
       "      <td>0.9449</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.012580</td>\n",
       "      <td>0</td>\n",
       "      <td>0.452530</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.470940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>deepNN</td>\n",
       "      <td>0.955458</td>\n",
       "      <td>28.508818</td>\n",
       "      <td>0.9447</td>\n",
       "      <td>50-25-25-25</td>\n",
       "      <td>50</td>\n",
       "      <td>tanh</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.007582</td>\n",
       "      <td>0</td>\n",
       "      <td>0.005818</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.313712</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>deepNN</td>\n",
       "      <td>0.949500</td>\n",
       "      <td>67.742311</td>\n",
       "      <td>0.9436</td>\n",
       "      <td>50-25-25-25</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>adagrad</td>\n",
       "      <td>0.007162</td>\n",
       "      <td>0</td>\n",
       "      <td>0.071327</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.392926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.952750</td>\n",
       "      <td>47.365074</td>\n",
       "      <td>0.9411</td>\n",
       "      <td>50</td>\n",
       "      <td>50</td>\n",
       "      <td>tanh</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.009664</td>\n",
       "      <td>0</td>\n",
       "      <td>0.150509</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.986907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>deepNN</td>\n",
       "      <td>0.941417</td>\n",
       "      <td>401.623071</td>\n",
       "      <td>0.9396</td>\n",
       "      <td>1000-500-500-500</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>adadelta</td>\n",
       "      <td>0.003525</td>\n",
       "      <td>0</td>\n",
       "      <td>0.191603</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.233951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.945521</td>\n",
       "      <td>17.786103</td>\n",
       "      <td>0.9367</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.017177</td>\n",
       "      <td>0</td>\n",
       "      <td>0.118600</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.266471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.942479</td>\n",
       "      <td>19.332285</td>\n",
       "      <td>0.9359</td>\n",
       "      <td>200</td>\n",
       "      <td>50</td>\n",
       "      <td>tanh</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.013908</td>\n",
       "      <td>0</td>\n",
       "      <td>0.069080</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.841125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.931042</td>\n",
       "      <td>245.228300</td>\n",
       "      <td>0.9317</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>adadelta</td>\n",
       "      <td>0.048444</td>\n",
       "      <td>0</td>\n",
       "      <td>0.105768</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.379932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.932833</td>\n",
       "      <td>142.254990</td>\n",
       "      <td>0.9312</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>tanh</td>\n",
       "      <td>adadelta</td>\n",
       "      <td>0.049766</td>\n",
       "      <td>0</td>\n",
       "      <td>0.432530</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.654599</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.931271</td>\n",
       "      <td>96.858346</td>\n",
       "      <td>0.9292</td>\n",
       "      <td>50</td>\n",
       "      <td>50</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>adagrad</td>\n",
       "      <td>0.011601</td>\n",
       "      <td>0</td>\n",
       "      <td>0.225089</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.959339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.930312</td>\n",
       "      <td>18.489639</td>\n",
       "      <td>0.9274</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>tanh</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.009964</td>\n",
       "      <td>0</td>\n",
       "      <td>0.264271</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.015782</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.926625</td>\n",
       "      <td>210.577913</td>\n",
       "      <td>0.9252</td>\n",
       "      <td>200</td>\n",
       "      <td>50</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>adadelta</td>\n",
       "      <td>0.046882</td>\n",
       "      <td>0</td>\n",
       "      <td>0.274665</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.439362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.923854</td>\n",
       "      <td>202.206347</td>\n",
       "      <td>0.9243</td>\n",
       "      <td>50</td>\n",
       "      <td>50</td>\n",
       "      <td>tanh</td>\n",
       "      <td>adadelta</td>\n",
       "      <td>0.019481</td>\n",
       "      <td>0</td>\n",
       "      <td>0.159309</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.457107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.925083</td>\n",
       "      <td>187.663996</td>\n",
       "      <td>0.9243</td>\n",
       "      <td>50</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>adadelta</td>\n",
       "      <td>0.028072</td>\n",
       "      <td>0</td>\n",
       "      <td>0.210736</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.492529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.922500</td>\n",
       "      <td>201.306828</td>\n",
       "      <td>0.9232</td>\n",
       "      <td>200</td>\n",
       "      <td>50</td>\n",
       "      <td>tanh</td>\n",
       "      <td>adadelta</td>\n",
       "      <td>0.010860</td>\n",
       "      <td>0</td>\n",
       "      <td>0.155209</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.458603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>deepNN</td>\n",
       "      <td>0.918125</td>\n",
       "      <td>27.215357</td>\n",
       "      <td>0.9119</td>\n",
       "      <td>1000-500-500-500</td>\n",
       "      <td>50</td>\n",
       "      <td>tanh</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.004017</td>\n",
       "      <td>0</td>\n",
       "      <td>0.145234</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.350682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>deepNN</td>\n",
       "      <td>0.912354</td>\n",
       "      <td>46.370231</td>\n",
       "      <td>0.9039</td>\n",
       "      <td>50-25-25-25</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.003349</td>\n",
       "      <td>0</td>\n",
       "      <td>0.253146</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.949311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>simpleNN</td>\n",
       "      <td>0.900604</td>\n",
       "      <td>209.663918</td>\n",
       "      <td>0.9036</td>\n",
       "      <td>50</td>\n",
       "      <td>50</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>adadelta</td>\n",
       "      <td>0.021242</td>\n",
       "      <td>0</td>\n",
       "      <td>0.110413</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.430975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>deepNN</td>\n",
       "      <td>0.899792</td>\n",
       "      <td>261.443288</td>\n",
       "      <td>0.9000</td>\n",
       "      <td>50-25-25-25</td>\n",
       "      <td>50</td>\n",
       "      <td>tanh</td>\n",
       "      <td>adadelta</td>\n",
       "      <td>0.005280</td>\n",
       "      <td>0</td>\n",
       "      <td>0.040969</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.344243</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>deepNN</td>\n",
       "      <td>0.897521</td>\n",
       "      <td>52.946338</td>\n",
       "      <td>0.8903</td>\n",
       "      <td>50-25-25-25</td>\n",
       "      <td>50</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.007728</td>\n",
       "      <td>0</td>\n",
       "      <td>0.336836</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.681514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>deepNN</td>\n",
       "      <td>0.895563</td>\n",
       "      <td>127.973625</td>\n",
       "      <td>0.8876</td>\n",
       "      <td>1000-500-500-500</td>\n",
       "      <td>50</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.009623</td>\n",
       "      <td>0</td>\n",
       "      <td>0.479470</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.693580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>deepNN</td>\n",
       "      <td>0.870938</td>\n",
       "      <td>393.965554</td>\n",
       "      <td>0.8748</td>\n",
       "      <td>1000-500-500-500</td>\n",
       "      <td>50</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>adadelta</td>\n",
       "      <td>0.017219</td>\n",
       "      <td>0</td>\n",
       "      <td>0.348957</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.222050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>deepNN</td>\n",
       "      <td>0.834354</td>\n",
       "      <td>138.409112</td>\n",
       "      <td>0.8381</td>\n",
       "      <td>50-25-25-25</td>\n",
       "      <td>50</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>adagrad</td>\n",
       "      <td>0.008990</td>\n",
       "      <td>0</td>\n",
       "      <td>0.252561</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.605524</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>deepNN</td>\n",
       "      <td>0.717250</td>\n",
       "      <td>280.135365</td>\n",
       "      <td>0.7206</td>\n",
       "      <td>50-25-25-25</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>adadelta</td>\n",
       "      <td>0.005182</td>\n",
       "      <td>0</td>\n",
       "      <td>0.204509</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.257233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>deepNN</td>\n",
       "      <td>0.581937</td>\n",
       "      <td>274.244618</td>\n",
       "      <td>0.5863</td>\n",
       "      <td>50-25-25-25</td>\n",
       "      <td>50</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>adadelta</td>\n",
       "      <td>0.009620</td>\n",
       "      <td>0</td>\n",
       "      <td>0.075231</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.213787</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   networkType  trainAcc  trainTime (s)  ...  normTech misc  testAccPerT\n",
       "50   simpleCNN  0.991625     128.453790  ...       NaN  NaN     0.767747\n",
       "47   simpleCNN  0.991896     127.392681  ...       NaN  NaN     0.773985\n",
       "48   simpleCNN  0.994375      65.084619  ...       NaN  NaN     1.511417\n",
       "53   simpleCNN  0.987083     159.142339  ...       NaN  NaN     0.617246\n",
       "45   simpleCNN  0.985812      41.673621  ...       NaN  NaN     2.352087\n",
       "49   simpleCNN  0.982187     366.359355  ...       NaN  NaN     0.267497\n",
       "46   simpleCNN  0.982083     417.585114  ...       NaN  NaN     0.234635\n",
       "32    simpleNN  0.998708      82.006778  ...       NaN  NaN     1.194292\n",
       "17      deepNN  0.999250      89.528609  ...       NaN  NaN     1.093617\n",
       "41    simpleNN  0.996854      60.324102  ...       NaN  NaN     1.622569\n",
       "51   simpleCNN  0.980458      38.673631  ...       NaN  NaN     2.529113\n",
       "38    simpleNN  0.992458      83.005636  ...       NaN  NaN     1.177631\n",
       "14      deepNN  0.993625      69.414156  ...       NaN  NaN     1.406197\n",
       "52   simpleCNN  0.975917     370.800954  ...       NaN  NaN     0.262890\n",
       "44    simpleNN  0.992417      70.773251  ...       NaN  NaN     1.375520\n",
       "27    simpleNN  0.989708      30.119569  ...       NaN  NaN     3.228134\n",
       "40    simpleNN  0.981646     247.256589  ...       NaN  NaN     0.392669\n",
       "29    simpleNN  0.981146     131.708493  ...       NaN  NaN     0.736779\n",
       "11      deepNN  0.971688     200.194306  ...       NaN  NaN     0.482032\n",
       "23    simpleNN  0.977938      65.780685  ...       NaN  NaN     1.466692\n",
       "31    simpleNN  0.972688     184.267622  ...       NaN  NaN     0.523098\n",
       "35    simpleNN  0.973792      61.045387  ...       NaN  NaN     1.574566\n",
       "21    simpleNN  0.980875      32.123013  ...       NaN  NaN     2.991625\n",
       "16      deepNN  0.966063     397.130157  ...       NaN  NaN     0.241885\n",
       "12      deepNN  0.967063      33.903111  ...       NaN  NaN     2.831009\n",
       "8       deepNN  0.960646     119.375441  ...       NaN  NaN     0.799411\n",
       "26    simpleNN  0.963875      91.175906  ...       NaN  NaN     1.045781\n",
       "30    simpleNN  0.957979      23.458261  ...       NaN  NaN     4.040368\n",
       "18    simpleNN  0.960812      34.997603  ...       NaN  NaN     2.707614\n",
       "36    simpleNN  0.953229      21.134256  ...       NaN  NaN     4.470940\n",
       "6       deepNN  0.955458      28.508818  ...       NaN  NaN     3.313712\n",
       "5       deepNN  0.949500      67.742311  ...       NaN  NaN     1.392926\n",
       "24    simpleNN  0.952750      47.365074  ...       NaN  NaN     1.986907\n",
       "13      deepNN  0.941417     401.623071  ...       NaN  NaN     0.233951\n",
       "39    simpleNN  0.945521      17.786103  ...       NaN  NaN     5.266471\n",
       "33    simpleNN  0.942479      19.332285  ...       NaN  NaN     4.841125\n",
       "37    simpleNN  0.931042     245.228300  ...       NaN  NaN     0.379932\n",
       "43    simpleNN  0.932833     142.254990  ...       NaN  NaN     0.654599\n",
       "20    simpleNN  0.931271      96.858346  ...       NaN  NaN     0.959339\n",
       "42    simpleNN  0.930312      18.489639  ...       NaN  NaN     5.015782\n",
       "28    simpleNN  0.926625     210.577913  ...       NaN  NaN     0.439362\n",
       "25    simpleNN  0.923854     202.206347  ...       NaN  NaN     0.457107\n",
       "22    simpleNN  0.925083     187.663996  ...       NaN  NaN     0.492529\n",
       "34    simpleNN  0.922500     201.306828  ...       NaN  NaN     0.458603\n",
       "15      deepNN  0.918125      27.215357  ...       NaN  NaN     3.350682\n",
       "3       deepNN  0.912354      46.370231  ...       NaN  NaN     1.949311\n",
       "19    simpleNN  0.900604     209.663918  ...       NaN  NaN     0.430975\n",
       "7       deepNN  0.899792     261.443288  ...       NaN  NaN     0.344243\n",
       "0       deepNN  0.897521      52.946338  ...       NaN  NaN     1.681514\n",
       "9       deepNN  0.895563     127.973625  ...       NaN  NaN     0.693580\n",
       "10      deepNN  0.870938     393.965554  ...       NaN  NaN     0.222050\n",
       "2       deepNN  0.834354     138.409112  ...       NaN  NaN     0.605524\n",
       "4       deepNN  0.717250     280.135365  ...       NaN  NaN     0.257233\n",
       "1       deepNN  0.581937     274.244618  ...       NaN  NaN     0.213787\n",
       "\n",
       "[54 rows x 14 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Display the model results after sorting by test accuracy.\n",
    "import pandas as pd\n",
    "data = pd.read_csv(local_dir + '/' + model_res_file)\n",
    "data['testAccPerT'] = 100 * data['testAcc'] / data['trainTime (s)']\n",
    "data = data.sort_values('testAcc', ascending = False)\n",
    "data.head(200)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "EvaluationModels_Mnist2.ipynb",
   "provenance": [
    {
     "file_id": "1_xKB15zZV41Ekp_uQI_HBfy8qUyKfquP",
     "timestamp": 1564695172021
    },
    {
     "file_id": "1n2GXcGNTUet8aFhhQpcs_c6MfR2E9lBT",
     "timestamp": 1564589553925
    },
    {
     "file_id": "1wHjKuYRr_epugI6eXkZ2_nBiBFFl6Q3s",
     "timestamp": 1564415521703
    }
   ],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}