EvaluateModels_cifar10.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 221
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4431,
     "status": "ok",
     "timestamp": 1566502519871,
     "user": {
      "displayName": "Krishan Rajaratnam",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mAto1HIjJ0u4TNrl13hFRazE_mhv2-W5G8WFYoicQ=s64",
      "userId": "09748671397076320798"
     },
     "user_tz": 240
    },
    "id": "d0L1uNwUlVpk",
    "outputId": "40e275b4-6138-43ca-f20d-f39f8da6d102"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: keras==2.1.6 in c:\\users\\krishan\\anaconda3\\lib\\site-packages (2.1.6)\n",
      "Requirement already satisfied: hyperopt==0.1.2 in c:\\users\\krishan\\anaconda3\\lib\\site-packages (0.1.2)\n",
      "Requirement already satisfied: networkx==1.11 in c:\\users\\krishan\\anaconda3\\lib\\site-packages (1.11)\n",
      "Requirement already satisfied: six>=1.9.0 in c:\\users\\krishan\\anaconda3\\lib\\site-packages (from keras==2.1.6) (1.12.0)\n",
      "Requirement already satisfied: numpy>=1.9.1 in c:\\users\\krishan\\anaconda3\\lib\\site-packages (from keras==2.1.6) (1.15.4)\n",
      "Requirement already satisfied: h5py in c:\\users\\krishan\\anaconda3\\lib\\site-packages (from keras==2.1.6) (2.8.0)\n",
      "Requirement already satisfied: scipy>=0.14 in c:\\users\\krishan\\anaconda3\\lib\\site-packages (from keras==2.1.6) (1.1.0)\n",
      "Requirement already satisfied: pyyaml in c:\\users\\krishan\\anaconda3\\lib\\site-packages (from keras==2.1.6) (3.13)\n",
      "Requirement already satisfied: tqdm in c:\\users\\krishan\\anaconda3\\lib\\site-packages (from hyperopt==0.1.2) (4.28.1)\n",
      "Requirement already satisfied: pymongo in c:\\users\\krishan\\anaconda3\\lib\\site-packages (from hyperopt==0.1.2) (3.8.0)\n",
      "Requirement already satisfied: future in c:\\users\\krishan\\anaconda3\\lib\\site-packages (from hyperopt==0.1.2) (0.17.1)\n",
      "Requirement already satisfied: decorator>=3.4.0 in c:\\users\\krishan\\anaconda3\\lib\\site-packages (from networkx==1.11) (4.3.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install keras==2.1.6 hyperopt==0.1 networkx==1.11\n",
    "#Note: This works with Python 3.6.8. For Python 3.7, use hyperopt 0.1.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6763,
     "status": "ok",
     "timestamp": 1566502533691,
     "user": {
      "displayName": "Krishan Rajaratnam",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mAto1HIjJ0u4TNrl13hFRazE_mhv2-W5G8WFYoicQ=s64",
      "userId": "09748671397076320798"
     },
     "user_tz": 240
    },
    "id": "QCWNJG4El5AZ",
    "outputId": "0f0f5e4f-dc1c-45ea-f5b5-4f2cac75f5c2"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import csv \n",
    "import time\n",
    "\n",
    "import os\n",
    "from os import listdir\n",
    "import sys\n",
    "\n",
    "#Append local dir to path to import custom modules.\n",
    "local_dir = \"\"\n",
    "sys.path.append(os.path.join(local_dir, \"models\"))\n",
    "\n",
    "#Sometimes the following modues won't be imported in Colab. Try uploading a new module and adding/removing sys.path.append to get it to work.\n",
    "from simpleCNN import simple_cnn_model\n",
    "from deepNN import deep_nn_model\n",
    "from zeroNN import zero_nn_model\n",
    "from deepCNN import deep_cnn_model\n",
    "from VGG import vgg_model\n",
    "from processModel import MetaModel\n",
    "\n",
    "#load CIFAR10\n",
    "from keras.datasets import cifar10\n",
    "from keras import backend as K\n",
    "\n",
    "import keras\n",
    "from keras import layers\n",
    "from keras.layers import Input, Dense, Activation\n",
    "from keras.models import Model\n",
    "from keras.preprocessing import image\n",
    "from keras import regularizers\n",
    "from keras.optimizers import Adam, Nadam, Adagrad, Adadelta\n",
    "from keras.activations import softmax\n",
    "from keras.losses import sparse_categorical_crossentropy\n",
    "\n",
    "from hyperopt import hp\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 119
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 17074,
     "status": "ok",
     "timestamp": 1566502556001,
     "user": {
      "displayName": "Krishan Rajaratnam",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mAto1HIjJ0u4TNrl13hFRazE_mhv2-W5G8WFYoicQ=s64",
      "userId": "09748671397076320798"
     },
     "user_tz": 240
    },
    "id": "fftXnPKenXdt",
    "outputId": "75070538-13d3-4052-9a2f-6124ce61261b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (40000, 32, 32, 3)\n",
      "40000 train samples\n",
      "10000 validation samples\n",
      "10000 test samples\n"
     ]
    }
   ],
   "source": [
    "#Preprocess the data\n",
    "\n",
    "# input image dimensions\n",
    "img_rows, img_cols, channels = 32, 32, 3\n",
    "\n",
    "#Number of possible target values for y.\n",
    "classes = 10\n",
    "#File to log model results\n",
    "model_res_file = 'modelResults_Cifar10.csv'\n",
    "\n",
    "#Load and setup the meta-model.\n",
    "meta_model = MetaModel(local_dir, model_res_file)\n",
    "\n",
    "# the data, split between train and test sets\n",
    "(X_train, y_train), (X_test, y_test) = cifar10.load_data()\n",
    "\n",
    "if K.image_data_format() == 'channels_first':\n",
    "    X_train = X_train.reshape(X_train.shape[0], channels, img_rows, img_cols)\n",
    "    X_test = X_test.reshape(X_test.shape[0], channels, img_rows, img_cols)\n",
    "    input_shape = (channels, img_rows, img_cols)\n",
    "else:\n",
    "    X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, channels)\n",
    "    X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, channels)\n",
    "    input_shape = (img_rows, img_cols, channels)\n",
    "  \n",
    "X_train = X_train.astype('float32')\n",
    "X_test = X_test.astype('float32')\n",
    "X_train /= 255\n",
    "X_test /= 255\n",
    "#Get the cross validation data.\n",
    "X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.20, random_state=42)\n",
    "print('x_train shape:', X_train.shape)\n",
    "print(X_train.shape[0], 'train samples')\n",
    "print(X_val.shape[0], 'validation samples')\n",
    "print(X_test.shape[0], 'test samples')\n",
    "\n",
    "def plot_train_val(history):\n",
    "  \"\"\"Method to plot the out of a call to keras model.fit.\n",
    "  \"\"\"\n",
    "  acc = history.history['acc']\n",
    "  val_acc = history.history['val_acc']\n",
    "  loss = history.history['loss']\n",
    "  val_loss = history.history['val_loss']\n",
    "  epochs = range(1, len(acc) + 1)\n",
    "  plt.plot(epochs, acc, 'bo', label='Training acc')\n",
    "  plt.plot(epochs, val_acc, 'b', label='Validation acc')\n",
    "  plt.title('Training and validation accuracy')\n",
    "  plt.legend()\n",
    "  plt.figure()\n",
    "  plt.plot(epochs, loss, 'bo', label='Training loss')\n",
    "  plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
    "  plt.title('Training and validation loss')\n",
    "  plt.legend()\n",
    "  plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hqBA9r1BsMQB"
   },
   "outputs": [],
   "source": [
    "#Code to write the columns for the modelResults csv. This info will be logged\n",
    "#after the models are trained and evaluated.\n",
    "with open(os.path.join(local_dir, model_res_file), mode='w') as file:\n",
    "  writer = csv.writer(file, delimiter=',', quotechar='\"', quoting=csv.QUOTE_MINIMAL)\n",
    "  writer.writerow(['networkType', 'trainAcc', 'trainTime (s)', 'testAcc', 'layers', 'epochs', 'activations', 'optim', 'learning_rate','reg_param', 'dropout_prob', 'normTech' , 'misc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 179912,
     "status": "ok",
     "timestamp": 1565894515245,
     "user": {
      "displayName": "Krishan Rajaratnam",
      "photoUrl": "https://lh6.googleusercontent.com/-Za30nR0ehyQ/AAAAAAAAAAI/AAAAAAAAGMw/jV8ldTp8928/s64/photo.jpg",
      "userId": "09748671397076320798"
     },
     "user_tz": 240
    },
    "id": "RHbe_QmMsT_C",
    "outputId": "bab71420-fb77-40cd-bb85-a71a4d8c2147",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#First try logistic regression\n",
    "\n",
    "#Reshape the data for processing by neural nets\n",
    "X_train = X_train.reshape(X_train.shape[0], img_rows * img_cols * channels)\n",
    "X_val = X_val.reshape(X_val.shape[0],  img_rows * img_cols * channels)\n",
    "X_test = X_test.reshape(X_test.shape[0],  img_rows * img_cols * channels)\n",
    "\n",
    "#The neural network architectures will be built using combinations of the following parameters.\n",
    "params = {'classes' : [classes],\n",
    "          'activation' : ['softmax'],\n",
    "          'batch_size': [1024],\n",
    "          'epochs': [1],\n",
    "          'optimizer': ['adagrad', 'adadelta', 'rmsprop', 'adam', 'nadam'],\n",
    "          'loss': ['sparse_categorical_crossentropy'],\n",
    "          'reg_param': [0],\n",
    "          'callbacks' : [[keras.callbacks.EarlyStopping(monitor='val_loss',\n",
    "                              min_delta=0,\n",
    "                              patience=2,\n",
    "                              verbose=0, mode='auto')]]}\n",
    "\n",
    "#Hyperopt will optimize the following parameters in the given ranges.\n",
    "search_space = {'lr': hp.loguniform('lr', -6, -4)}\n",
    "\n",
    "#The call to process_model will compile, train and evaulate the above models and optimize the hyper parameters.\n",
    "#The output will be saved to a file and the output details will be recorded in the model_res_file. \n",
    "model_name = \"logisticRegression\"\n",
    "meta_model.max_evals = 1\n",
    "model_results = meta_model.process_model(params, search_space, zero_nn_model, model_name, [X_train, X_val, X_test, y_train, y_val, y_test], \n",
    "              [model_name, 0, 0, 0, np.NaN, 'epochs', 'activation', 'optimizer', 'lr','reg_param', 'dropout_prob', np.NaN , np.NaN])\n",
    "\n",
    "#Now lets plot the results.\n",
    "history = model_results[0][0]\n",
    "plot_train_val(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1654021,
     "status": "ok",
     "timestamp": 1565896503355,
     "user": {
      "displayName": "Krishan Rajaratnam",
      "photoUrl": "https://lh6.googleusercontent.com/-Za30nR0ehyQ/AAAAAAAAAAI/AAAAAAAAGMw/jV8ldTp8928/s64/photo.jpg",
      "userId": "09748671397076320798"
     },
     "user_tz": 240
    },
    "id": "IO97wHE12cz6",
    "outputId": "879af991-a5fd-4fe1-80e0-2af9db96eddf"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0815 18:47:26.960574 139890393814912 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3144: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.3754557343337546, 'lr': 0.00852465672827463}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0815 18:47:27.310944 139890393814912 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1801: The name tf.nn.fused_batch_norm is deprecated. Please use tf.compat.v1.nn.fused_batch_norm instead.\n",
      "\n",
      "W0815 18:47:27.416089 139890393814912 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3661: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 6s 147us/step - loss: 11.0231 - acc: 0.2158 - val_loss: 11.8123 - val_acc: 0.1509\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 10.2140 - acc: 0.2723 - val_loss: 11.7119 - val_acc: 0.1264\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 9.1901 - acc: 0.2960 - val_loss: 10.7507 - val_acc: 0.1567\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 5.6641 - acc: 0.3332 - val_loss: 3.4723 - val_acc: 0.1951\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 2.5977 - acc: 0.3248 - val_loss: 2.0160 - val_acc: 0.3128\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.7623 - acc: 0.3776 - val_loss: 1.6553 - val_acc: 0.4178\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.5974 - acc: 0.4377 - val_loss: 1.7235 - val_acc: 0.3888\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.5016 - acc: 0.4688 - val_loss: 1.6641 - val_acc: 0.4226\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.4458 - acc: 0.4899 - val_loss: 4.8002 - val_acc: 0.1521\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.3992 - acc: 0.5086 - val_loss: 1.6140 - val_acc: 0.4416\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.3572 - acc: 0.5258 - val_loss: 1.5983 - val_acc: 0.4324\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.3333 - acc: 0.5314 - val_loss: 1.5952 - val_acc: 0.4212\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.3070 - acc: 0.5438 - val_loss: 1.3980 - val_acc: 0.5053\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.2972 - acc: 0.5467 - val_loss: 1.4023 - val_acc: 0.5063\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.2775 - acc: 0.5560 - val_loss: 1.3746 - val_acc: 0.5178\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.2799 - acc: 0.5546 - val_loss: 1.3656 - val_acc: 0.5158\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.2574 - acc: 0.5641 - val_loss: 1.5472 - val_acc: 0.4587\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.2626 - acc: 0.5603 - val_loss: 1.3256 - val_acc: 0.5325\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.2448 - acc: 0.5664 - val_loss: 1.3853 - val_acc: 0.5057\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.2435 - acc: 0.5657 - val_loss: 1.4051 - val_acc: 0.5111\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.2296 - acc: 0.5713 - val_loss: 1.5027 - val_acc: 0.4581\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.2229 - acc: 0.5745 - val_loss: 1.5875 - val_acc: 0.4773\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.2235 - acc: 0.5747 - val_loss: 1.3511 - val_acc: 0.5364\n",
      "Epoch 24/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.2268 - acc: 0.5756 - val_loss: 1.3861 - val_acc: 0.5056\n",
      "10000/10000 [==============================] - 1s 101us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.24457317279668722, 'lr': 0.0026806947768226795}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 3s 84us/step - loss: 3.8598 - acc: 0.2806 - val_loss: 2.9402 - val_acc: 0.2601\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.8816 - acc: 0.4099 - val_loss: 1.9716 - val_acc: 0.3380\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.5338 - acc: 0.4641 - val_loss: 2.5933 - val_acc: 0.2355\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.4303 - acc: 0.4995 - val_loss: 1.7208 - val_acc: 0.4088\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.3862 - acc: 0.5173 - val_loss: 2.6991 - val_acc: 0.2703\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.2994 - acc: 0.5460 - val_loss: 2.4834 - val_acc: 0.2623\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.2425 - acc: 0.5689 - val_loss: 1.6666 - val_acc: 0.4425\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.1833 - acc: 0.5902 - val_loss: 2.3274 - val_acc: 0.3035\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.1168 - acc: 0.6146 - val_loss: 1.9181 - val_acc: 0.3816\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.0837 - acc: 0.6261 - val_loss: 1.4558 - val_acc: 0.4968\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.0360 - acc: 0.6445 - val_loss: 2.0369 - val_acc: 0.3686\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.0232 - acc: 0.6477 - val_loss: 1.4069 - val_acc: 0.5155\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.9859 - acc: 0.6617 - val_loss: 1.2343 - val_acc: 0.5753\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.9589 - acc: 0.6710 - val_loss: 1.4765 - val_acc: 0.4926\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.9592 - acc: 0.6699 - val_loss: 1.3538 - val_acc: 0.5299\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.9359 - acc: 0.6779 - val_loss: 1.1966 - val_acc: 0.5863\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.9243 - acc: 0.6818 - val_loss: 1.1983 - val_acc: 0.5827\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.9192 - acc: 0.6834 - val_loss: 1.7075 - val_acc: 0.4269\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.9052 - acc: 0.6856 - val_loss: 1.4363 - val_acc: 0.5035\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.8928 - acc: 0.6948 - val_loss: 1.1940 - val_acc: 0.5780\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.8853 - acc: 0.6928 - val_loss: 1.4203 - val_acc: 0.5334\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.8632 - acc: 0.7037 - val_loss: 1.5129 - val_acc: 0.5082\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.8569 - acc: 0.7058 - val_loss: 1.2976 - val_acc: 0.5616\n",
      "Epoch 24/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.8487 - acc: 0.7072 - val_loss: 1.3322 - val_acc: 0.5602\n",
      "Epoch 25/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.8561 - acc: 0.7044 - val_loss: 1.1967 - val_acc: 0.5924\n",
      "Epoch 26/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.8333 - acc: 0.7124 - val_loss: 1.4852 - val_acc: 0.5124\n",
      "10000/10000 [==============================] - 1s 98us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.5517648001033064, 'lr': 0.008339290309801933}"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0815 18:49:45.981048 139890393814912 nn_ops.py:4224] Large dropout rate: 0.551765 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0815 18:49:46.118086 139890393814912 nn_ops.py:4224] Large dropout rate: 0.551765 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 3s 86us/step - loss: 9.7549 - acc: 0.2210 - val_loss: 9.9028 - val_acc: 0.2024\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 8.9508 - acc: 0.2704 - val_loss: 9.0637 - val_acc: 0.1835\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 6.8874 - acc: 0.2802 - val_loss: 13.4499 - val_acc: 0.0998\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 4.1604 - acc: 0.2724 - val_loss: 4.6644 - val_acc: 0.1934\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 2.4172 - acc: 0.2840 - val_loss: 2.1669 - val_acc: 0.2316\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.7860 - acc: 0.3689 - val_loss: 2.0426 - val_acc: 0.2555\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.6266 - acc: 0.4235 - val_loss: 1.9026 - val_acc: 0.2866\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.5391 - acc: 0.4556 - val_loss: 1.7922 - val_acc: 0.3330\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.4848 - acc: 0.4753 - val_loss: 1.7552 - val_acc: 0.3633\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.4508 - acc: 0.4896 - val_loss: 1.6723 - val_acc: 0.3871\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.4069 - acc: 0.5075 - val_loss: 1.8048 - val_acc: 0.3309\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.3841 - acc: 0.5154 - val_loss: 1.6093 - val_acc: 0.4240\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.3432 - acc: 0.5289 - val_loss: 1.6149 - val_acc: 0.3958\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.3366 - acc: 0.5303 - val_loss: 1.7047 - val_acc: 0.3627\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.3283 - acc: 0.5339 - val_loss: 1.6136 - val_acc: 0.4279\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.2968 - acc: 0.5444 - val_loss: 1.6395 - val_acc: 0.4188\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.2944 - acc: 0.5458 - val_loss: 1.5181 - val_acc: 0.4580\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.2790 - acc: 0.5537 - val_loss: 1.4285 - val_acc: 0.4927\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.2626 - acc: 0.5583 - val_loss: 1.8099 - val_acc: 0.3522\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.2592 - acc: 0.5609 - val_loss: 1.6635 - val_acc: 0.4257\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.2583 - acc: 0.5607 - val_loss: 1.5298 - val_acc: 0.4610\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.2298 - acc: 0.5709 - val_loss: 1.4894 - val_acc: 0.4620\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.2225 - acc: 0.5743 - val_loss: 1.3950 - val_acc: 0.5190\n",
      "Epoch 24/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.2149 - acc: 0.5764 - val_loss: 1.3745 - val_acc: 0.5094\n",
      "Epoch 25/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.2056 - acc: 0.5804 - val_loss: 1.5333 - val_acc: 0.4406\n",
      "Epoch 26/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.2003 - acc: 0.5826 - val_loss: 1.4788 - val_acc: 0.4757\n",
      "Epoch 27/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.1893 - acc: 0.5854 - val_loss: 1.3881 - val_acc: 0.5206\n",
      "Epoch 28/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.1942 - acc: 0.5854 - val_loss: 1.5166 - val_acc: 0.4553\n",
      "Epoch 29/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.1802 - acc: 0.5924 - val_loss: 1.4823 - val_acc: 0.4670\n",
      "Epoch 30/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.1720 - acc: 0.5946 - val_loss: 1.4554 - val_acc: 0.4850\n",
      "10000/10000 [==============================] - 1s 98us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.5413636321061072, 'lr': 0.004195951103893687}"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0815 18:51:07.043039 139890393814912 nn_ops.py:4224] Large dropout rate: 0.541364 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0815 18:51:07.176871 139890393814912 nn_ops.py:4224] Large dropout rate: 0.541364 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 4s 88us/step - loss: 8.6128 - acc: 0.2194 - val_loss: 7.6952 - val_acc: 0.2282\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 7.3066 - acc: 0.2979 - val_loss: 7.6075 - val_acc: 0.2364\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 7.2595 - acc: 0.3222 - val_loss: 7.3664 - val_acc: 0.2663\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 7.1121 - acc: 0.3533 - val_loss: 7.4145 - val_acc: 0.2563\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 6.2649 - acc: 0.3774 - val_loss: 7.5948 - val_acc: 0.2066\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 5.6395 - acc: 0.4202 - val_loss: 6.0801 - val_acc: 0.3132\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 5.0055 - acc: 0.4353 - val_loss: 5.2062 - val_acc: 0.2639\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 4.1663 - acc: 0.4771 - val_loss: 4.2705 - val_acc: 0.4017\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 3.1946 - acc: 0.4997 - val_loss: 3.0451 - val_acc: 0.4189\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.8798 - acc: 0.5231 - val_loss: 1.4657 - val_acc: 0.4945\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.2377 - acc: 0.5778 - val_loss: 1.4582 - val_acc: 0.4867\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.1576 - acc: 0.6010 - val_loss: 1.6434 - val_acc: 0.4026\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.1290 - acc: 0.6137 - val_loss: 1.3500 - val_acc: 0.5442\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.1059 - acc: 0.6211 - val_loss: 1.3270 - val_acc: 0.5373\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.1001 - acc: 0.6225 - val_loss: 1.6210 - val_acc: 0.4087\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.0698 - acc: 0.6336 - val_loss: 1.3076 - val_acc: 0.5590\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.0393 - acc: 0.6434 - val_loss: 1.2246 - val_acc: 0.5817\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.0443 - acc: 0.6418 - val_loss: 1.2584 - val_acc: 0.5665\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.0263 - acc: 0.6481 - val_loss: 1.2632 - val_acc: 0.5701\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.0085 - acc: 0.6528 - val_loss: 1.2493 - val_acc: 0.5651\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.0032 - acc: 0.6548 - val_loss: 1.7013 - val_acc: 0.4144\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.0123 - acc: 0.6502 - val_loss: 1.1943 - val_acc: 0.5943\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.9784 - acc: 0.6625 - val_loss: 1.2046 - val_acc: 0.5984\n",
      "Epoch 24/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.9719 - acc: 0.6642 - val_loss: 1.3409 - val_acc: 0.5204\n",
      "Epoch 25/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.9684 - acc: 0.6682 - val_loss: 1.1674 - val_acc: 0.6144\n",
      "Epoch 26/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.9520 - acc: 0.6728 - val_loss: 1.3476 - val_acc: 0.5443\n",
      "Epoch 27/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.9510 - acc: 0.6735 - val_loss: 1.3371 - val_acc: 0.5469\n",
      "Epoch 28/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.9422 - acc: 0.6746 - val_loss: 1.1593 - val_acc: 0.6080\n",
      "Epoch 29/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.9237 - acc: 0.6839 - val_loss: 1.1303 - val_acc: 0.6225\n",
      "Epoch 30/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.9267 - acc: 0.6792 - val_loss: 1.6634 - val_acc: 0.4373\n",
      "Epoch 31/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.9263 - acc: 0.6796 - val_loss: 1.1312 - val_acc: 0.6207\n",
      "Epoch 32/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.9077 - acc: 0.6893 - val_loss: 1.2249 - val_acc: 0.5714\n",
      "Epoch 33/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.9061 - acc: 0.6883 - val_loss: 1.1493 - val_acc: 0.6253\n",
      "Epoch 34/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.9049 - acc: 0.6904 - val_loss: 1.2620 - val_acc: 0.5373\n",
      "Epoch 35/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.8958 - acc: 0.6902 - val_loss: 1.3649 - val_acc: 0.5389\n",
      "10000/10000 [==============================] - 1s 106us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.3171580262734339, 'lr': 0.006156502981830486}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 4s 90us/step - loss: 9.3142 - acc: 0.2290 - val_loss: 10.8594 - val_acc: 0.1560\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 7.3883 - acc: 0.3048 - val_loss: 7.8762 - val_acc: 0.2115\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 6.9566 - acc: 0.3521 - val_loss: 6.6665 - val_acc: 0.2549\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 5.9184 - acc: 0.3728 - val_loss: 7.4749 - val_acc: 0.1849\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 5.3603 - acc: 0.4107 - val_loss: 6.2790 - val_acc: 0.2297\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 3.9441 - acc: 0.4305 - val_loss: 8.0358 - val_acc: 0.1292\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.8698 - acc: 0.4498 - val_loss: 9.1092 - val_acc: 0.1517\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.4206 - acc: 0.5100 - val_loss: 5.5568 - val_acc: 0.1726\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.3636 - acc: 0.5340 - val_loss: 3.7705 - val_acc: 0.2084\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.3041 - acc: 0.5524 - val_loss: 3.4593 - val_acc: 0.2600\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.2395 - acc: 0.5746 - val_loss: 2.6323 - val_acc: 0.2998\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.2064 - acc: 0.5852 - val_loss: 2.6049 - val_acc: 0.2859\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.1744 - acc: 0.5958 - val_loss: 2.4543 - val_acc: 0.3148\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.1425 - acc: 0.6070 - val_loss: 1.8826 - val_acc: 0.3978\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.1025 - acc: 0.6200 - val_loss: 1.5852 - val_acc: 0.4464\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.0853 - acc: 0.6288 - val_loss: 1.9804 - val_acc: 0.3671\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.0507 - acc: 0.6395 - val_loss: 1.9786 - val_acc: 0.3757\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.0469 - acc: 0.6421 - val_loss: 1.3165 - val_acc: 0.5439\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.0111 - acc: 0.6521 - val_loss: 1.3590 - val_acc: 0.5292\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.9994 - acc: 0.6580 - val_loss: 1.3731 - val_acc: 0.5353\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.9791 - acc: 0.6623 - val_loss: 1.3280 - val_acc: 0.5535\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.9840 - acc: 0.6614 - val_loss: 1.4190 - val_acc: 0.4983\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.9562 - acc: 0.6728 - val_loss: 1.4352 - val_acc: 0.5078\n",
      "Epoch 24/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.9488 - acc: 0.6741 - val_loss: 1.2591 - val_acc: 0.5605\n",
      "Epoch 25/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.9332 - acc: 0.6776 - val_loss: 1.3248 - val_acc: 0.5410\n",
      "Epoch 26/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.9362 - acc: 0.6791 - val_loss: 1.2343 - val_acc: 0.5786\n",
      "Epoch 27/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.9062 - acc: 0.6869 - val_loss: 1.1173 - val_acc: 0.6149\n",
      "Epoch 28/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.8849 - acc: 0.6963 - val_loss: 1.4558 - val_acc: 0.5315\n",
      "Epoch 29/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.8909 - acc: 0.6968 - val_loss: 1.3507 - val_acc: 0.5368\n",
      "Epoch 30/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.8887 - acc: 0.6939 - val_loss: 1.1996 - val_acc: 0.5975\n",
      "Epoch 31/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.8639 - acc: 0.7031 - val_loss: 1.1067 - val_acc: 0.6244\n",
      "Epoch 32/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.8471 - acc: 0.7104 - val_loss: 1.3059 - val_acc: 0.5626\n",
      "Epoch 33/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.8647 - acc: 0.7021 - val_loss: 1.5540 - val_acc: 0.5132\n",
      "Epoch 34/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.8395 - acc: 0.7117 - val_loss: 1.3606 - val_acc: 0.5540\n",
      "Epoch 35/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.8253 - acc: 0.7165 - val_loss: 1.3185 - val_acc: 0.5484\n",
      "Epoch 36/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.8317 - acc: 0.7120 - val_loss: 1.3478 - val_acc: 0.5729\n",
      "Epoch 37/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.8231 - acc: 0.7161 - val_loss: 1.4963 - val_acc: 0.4844\n",
      "10000/10000 [==============================] - 1s 100us/step\n",
      "40000/40000 [==============================] - 4s 102us/step\n",
      "10000/10000 [==============================] - 1s 117us/step\n",
      "Loss = 1.3665396448135376\n",
      "Test Accuracy = 0.5411\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.08288376130991659, 'lr': 0.016053190903759885}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 4s 94us/step - loss: 11.8776 - acc: 0.1809 - val_loss: 12.9671 - val_acc: 0.1092\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 8.3396 - acc: 0.1842 - val_loss: 5.1567 - val_acc: 0.1151\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 2.3115 - acc: 0.1007 - val_loss: 2.3028 - val_acc: 0.1023\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 2.3029 - acc: 0.1011 - val_loss: 2.3029 - val_acc: 0.0973\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 2.3028 - acc: 0.0993 - val_loss: 2.3028 - val_acc: 0.0996\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 2.3028 - acc: 0.0972 - val_loss: 2.3030 - val_acc: 0.0933\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 2.3028 - acc: 0.1000 - val_loss: 2.3029 - val_acc: 0.0933\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 2.3029 - acc: 0.0996 - val_loss: 2.3027 - val_acc: 0.0979\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 2.3029 - acc: 0.0964 - val_loss: 2.3030 - val_acc: 0.0933\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 2.3028 - acc: 0.0975 - val_loss: 2.3027 - val_acc: 0.0979\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 2.3028 - acc: 0.0987 - val_loss: 2.3030 - val_acc: 0.0973\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 2.3029 - acc: 0.0996 - val_loss: 2.3029 - val_acc: 0.0933\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 2.3028 - acc: 0.1007 - val_loss: 2.3033 - val_acc: 0.0933\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 2.3030 - acc: 0.0985 - val_loss: 2.3030 - val_acc: 0.0933\n",
      "10000/10000 [==============================] - 1s 106us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.43204219061433824, 'lr': 0.002978068202513929}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 4s 95us/step - loss: 6.3001 - acc: 0.2017 - val_loss: 7.1980 - val_acc: 0.1476\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 3.3672 - acc: 0.3229 - val_loss: 3.4741 - val_acc: 0.2434\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 2.8943 - acc: 0.3822 - val_loss: 4.3147 - val_acc: 0.1300\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.8188 - acc: 0.4291 - val_loss: 2.3826 - val_acc: 0.2151\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.5439 - acc: 0.4719 - val_loss: 2.0641 - val_acc: 0.2736\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.4517 - acc: 0.4958 - val_loss: 1.6902 - val_acc: 0.4053\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.4027 - acc: 0.5179 - val_loss: 1.5903 - val_acc: 0.4514\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.3821 - acc: 0.5256 - val_loss: 1.7610 - val_acc: 0.3767\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.3116 - acc: 0.5462 - val_loss: 1.5444 - val_acc: 0.4499\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.2599 - acc: 0.5618 - val_loss: 1.8262 - val_acc: 0.3535\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.2651 - acc: 0.5669 - val_loss: 1.4609 - val_acc: 0.4841\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.1724 - acc: 0.5931 - val_loss: 1.6137 - val_acc: 0.4365\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.1797 - acc: 0.5945 - val_loss: 1.5839 - val_acc: 0.4672\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.1603 - acc: 0.6007 - val_loss: 1.4577 - val_acc: 0.4838\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.1004 - acc: 0.6216 - val_loss: 1.5859 - val_acc: 0.4734\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.0860 - acc: 0.6269 - val_loss: 1.3557 - val_acc: 0.5082\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.0603 - acc: 0.6346 - val_loss: 1.4081 - val_acc: 0.4938\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.0440 - acc: 0.6424 - val_loss: 1.5011 - val_acc: 0.4716\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.0768 - acc: 0.6324 - val_loss: 1.1981 - val_acc: 0.5835\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.0073 - acc: 0.6534 - val_loss: 1.5289 - val_acc: 0.4852\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.9904 - acc: 0.6610 - val_loss: 1.2726 - val_acc: 0.5568\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.9787 - acc: 0.6662 - val_loss: 1.2919 - val_acc: 0.5422\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.9740 - acc: 0.6640 - val_loss: 1.4745 - val_acc: 0.4904\n",
      "Epoch 24/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.9593 - acc: 0.6707 - val_loss: 1.2817 - val_acc: 0.5289\n",
      "Epoch 25/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.9484 - acc: 0.6760 - val_loss: 1.1407 - val_acc: 0.6091\n",
      "Epoch 26/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.9418 - acc: 0.6764 - val_loss: 1.3014 - val_acc: 0.5421\n",
      "Epoch 27/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.9186 - acc: 0.6843 - val_loss: 1.4913 - val_acc: 0.4773\n",
      "Epoch 28/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.9263 - acc: 0.6824 - val_loss: 1.3092 - val_acc: 0.5480\n",
      "Epoch 29/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.9101 - acc: 0.6886 - val_loss: 1.4047 - val_acc: 0.5233\n",
      "Epoch 30/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.9080 - acc: 0.6898 - val_loss: 1.1951 - val_acc: 0.5836\n",
      "Epoch 31/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.8917 - acc: 0.6943 - val_loss: 1.1669 - val_acc: 0.5956\n",
      "10000/10000 [==============================] - 1s 101us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.31491251055971675, 'lr': 0.005902475883816378}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 4s 99us/step - loss: 9.0473 - acc: 0.2129 - val_loss: 8.8898 - val_acc: 0.1803\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 8.4719 - acc: 0.2766 - val_loss: 7.9035 - val_acc: 0.2164\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 7.3220 - acc: 0.3002 - val_loss: 6.9476 - val_acc: 0.2751\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 6.2812 - acc: 0.2930 - val_loss: 4.5898 - val_acc: 0.1724\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 3.7960 - acc: 0.3193 - val_loss: 2.1121 - val_acc: 0.2940\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.7330 - acc: 0.4057 - val_loss: 2.6434 - val_acc: 0.2522\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.5049 - acc: 0.4751 - val_loss: 1.9342 - val_acc: 0.3302\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.4016 - acc: 0.5145 - val_loss: 2.1207 - val_acc: 0.3090\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.3321 - acc: 0.5383 - val_loss: 1.5457 - val_acc: 0.4339\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.2964 - acc: 0.5494 - val_loss: 1.7716 - val_acc: 0.3896\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.2448 - acc: 0.5697 - val_loss: 1.6077 - val_acc: 0.4102\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.2121 - acc: 0.5791 - val_loss: 1.5666 - val_acc: 0.4526\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.1743 - acc: 0.5936 - val_loss: 1.3402 - val_acc: 0.5151\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.1498 - acc: 0.6030 - val_loss: 2.1292 - val_acc: 0.3864\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.1412 - acc: 0.6078 - val_loss: 1.2846 - val_acc: 0.5460\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.1076 - acc: 0.6168 - val_loss: 1.3170 - val_acc: 0.5372\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.0898 - acc: 0.6239 - val_loss: 1.2363 - val_acc: 0.5659\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.0737 - acc: 0.6318 - val_loss: 1.5268 - val_acc: 0.4695\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.0622 - acc: 0.6346 - val_loss: 1.4693 - val_acc: 0.5155\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.0323 - acc: 0.6437 - val_loss: 1.3529 - val_acc: 0.5335\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.0236 - acc: 0.6485 - val_loss: 1.5306 - val_acc: 0.4796\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 1.0144 - acc: 0.6512 - val_loss: 1.3594 - val_acc: 0.5243\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.9933 - acc: 0.6570 - val_loss: 1.2969 - val_acc: 0.5479\n",
      "10000/10000 [==============================] - 1s 102us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.3037489895160158, 'lr': 0.0043100948750029846}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 4s 99us/step - loss: 7.2042 - acc: 0.2319 - val_loss: 7.3686 - val_acc: 0.1665\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 5.8184 - acc: 0.3436 - val_loss: 6.5540 - val_acc: 0.2011\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 5.6862 - acc: 0.3881 - val_loss: 6.2899 - val_acc: 0.2548\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 5.6067 - acc: 0.4173 - val_loss: 6.8681 - val_acc: 0.1844\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 5.5809 - acc: 0.4325 - val_loss: 5.9248 - val_acc: 0.3055\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 5.5360 - acc: 0.4461 - val_loss: 5.9337 - val_acc: 0.3127\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 5.5033 - acc: 0.4584 - val_loss: 5.7228 - val_acc: 0.3701\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 5.4449 - acc: 0.4801 - val_loss: 5.5781 - val_acc: 0.4378\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 5.4478 - acc: 0.4806 - val_loss: 5.6059 - val_acc: 0.4259\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 5.4221 - acc: 0.4891 - val_loss: 5.6176 - val_acc: 0.4251\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 5.4060 - acc: 0.4955 - val_loss: 5.5572 - val_acc: 0.4545\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 5.3938 - acc: 0.4996 - val_loss: 5.7793 - val_acc: 0.3774\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 5.3714 - acc: 0.5078 - val_loss: 5.8454 - val_acc: 0.3646\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 5.3627 - acc: 0.5125 - val_loss: 5.4724 - val_acc: 0.4789\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 5.3476 - acc: 0.5161 - val_loss: 5.8951 - val_acc: 0.3459\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 5.3489 - acc: 0.5133 - val_loss: 5.5249 - val_acc: 0.4560\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 5.0184 - acc: 0.5121 - val_loss: 5.8511 - val_acc: 0.2637\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 4.0404 - acc: 0.5334 - val_loss: 4.3764 - val_acc: 0.4099\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 3.0959 - acc: 0.5382 - val_loss: 3.1481 - val_acc: 0.3972\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 2.5359 - acc: 0.5863 - val_loss: 2.6960 - val_acc: 0.5008\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 2.4954 - acc: 0.5997 - val_loss: 2.5766 - val_acc: 0.5575\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 2.4744 - acc: 0.6049 - val_loss: 2.7139 - val_acc: 0.5003\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 2.4527 - acc: 0.6134 - val_loss: 2.6288 - val_acc: 0.5384\n",
      "Epoch 24/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 2.4457 - acc: 0.6141 - val_loss: 2.5520 - val_acc: 0.5691\n",
      "Epoch 25/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 2.4281 - acc: 0.6197 - val_loss: 2.6899 - val_acc: 0.5155\n",
      "Epoch 26/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 2.4216 - acc: 0.6212 - val_loss: 2.6125 - val_acc: 0.5527\n",
      "Epoch 27/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 2.4070 - acc: 0.6277 - val_loss: 2.9856 - val_acc: 0.4217\n",
      "Epoch 28/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 2.4048 - acc: 0.6289 - val_loss: 2.6876 - val_acc: 0.5221\n",
      "Epoch 29/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 2.3956 - acc: 0.6307 - val_loss: 2.6273 - val_acc: 0.5326\n",
      "Epoch 30/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 2.3780 - acc: 0.6375 - val_loss: 2.9847 - val_acc: 0.4378\n",
      "10000/10000 [==============================] - 1s 101us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.05756435585476527, 'lr': 0.01815667567575375}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 4s 99us/step - loss: 11.8578 - acc: 0.1989 - val_loss: 12.2633 - val_acc: 0.1938\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 9.4915 - acc: 0.2206 - val_loss: 3.3265 - val_acc: 0.1164\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 2.3100 - acc: 0.0988 - val_loss: 2.3036 - val_acc: 0.0933\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 2.3030 - acc: 0.1011 - val_loss: 2.3029 - val_acc: 0.0933\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 2.3028 - acc: 0.0998 - val_loss: 2.3029 - val_acc: 0.0973\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 2.3030 - acc: 0.0998 - val_loss: 2.3029 - val_acc: 0.0994\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 2.3029 - acc: 0.0999 - val_loss: 2.3029 - val_acc: 0.0994\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 2.3029 - acc: 0.1001 - val_loss: 2.3030 - val_acc: 0.1023\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 2.3030 - acc: 0.1008 - val_loss: 2.3032 - val_acc: 0.0973\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 2.3029 - acc: 0.1002 - val_loss: 2.3031 - val_acc: 0.0979\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 2.3029 - acc: 0.1009 - val_loss: 2.3030 - val_acc: 0.0994\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 2.3028 - acc: 0.0982 - val_loss: 2.3029 - val_acc: 0.1030\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 2.3031 - acc: 0.1001 - val_loss: 2.3031 - val_acc: 0.0973\n",
      "10000/10000 [==============================] - 1s 101us/step\n",
      "40000/40000 [==============================] - 4s 101us/step\n",
      "10000/10000 [==============================] - 1s 112us/step\n",
      "Loss = 1.169975454711914\n",
      "Test Accuracy = 0.6038\n",
      "{'classes': 10, 'activation': 'sigmoid', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.5433638123400395, 'lr': 0.008082201584940288}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0815 18:59:39.397234 139890393814912 nn_ops.py:4224] Large dropout rate: 0.543364 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 4s 101us/step - loss: 14.0013 - acc: 0.1006 - val_loss: 14.4692 - val_acc: 0.1023\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 14.5156 - acc: 0.0994 - val_loss: 14.4692 - val_acc: 0.1023\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 14.5156 - acc: 0.0994 - val_loss: 14.4692 - val_acc: 0.1023\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 14.5156 - acc: 0.0994 - val_loss: 14.4692 - val_acc: 0.1023\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 14.5156 - acc: 0.0994 - val_loss: 14.4692 - val_acc: 0.1023\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 14.5156 - acc: 0.0994 - val_loss: 14.4692 - val_acc: 0.1023\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 14.5156 - acc: 0.0994 - val_loss: 14.4692 - val_acc: 0.1023\n",
      "10000/10000 [==============================] - 1s 106us/step\n",
      "{'classes': 10, 'activation': 'sigmoid', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.516960239503922, 'lr': 0.004586219403549131}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 4s 103us/step - loss: 12.6610 - acc: 0.1297 - val_loss: 12.9897 - val_acc: 0.1528\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 13.0310 - acc: 0.1458 - val_loss: 12.9857 - val_acc: 0.1626\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9894 - acc: 0.1577 - val_loss: 13.0246 - val_acc: 0.1237\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 12.9887 - acc: 0.1585 - val_loss: 13.2414 - val_acc: 0.1023\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 13.1375 - acc: 0.1410 - val_loss: 12.9717 - val_acc: 0.1653\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9856 - acc: 0.1620 - val_loss: 12.9730 - val_acc: 0.1666\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9756 - acc: 0.1662 - val_loss: 13.0689 - val_acc: 0.1118\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 12.9817 - acc: 0.1636 - val_loss: 13.0722 - val_acc: 0.1184\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 12.9687 - acc: 0.1698 - val_loss: 12.9869 - val_acc: 0.1465\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9618 - acc: 0.1727 - val_loss: 12.9509 - val_acc: 0.1770\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9553 - acc: 0.1756 - val_loss: 12.9356 - val_acc: 0.1886\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9470 - acc: 0.1789 - val_loss: 12.9347 - val_acc: 0.1851\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 12.9580 - acc: 0.1757 - val_loss: 12.9614 - val_acc: 0.1679\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 12.9892 - acc: 0.1687 - val_loss: 13.1479 - val_acc: 0.1071\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9774 - acc: 0.1729 - val_loss: 12.9609 - val_acc: 0.1669\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9347 - acc: 0.1847 - val_loss: 12.9262 - val_acc: 0.1889\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9564 - acc: 0.1777 - val_loss: 12.9444 - val_acc: 0.1767\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9613 - acc: 0.1761 - val_loss: 13.2110 - val_acc: 0.1108\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9455 - acc: 0.1817 - val_loss: 12.9399 - val_acc: 0.1798\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9440 - acc: 0.1813 - val_loss: 12.9464 - val_acc: 0.1742\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9359 - acc: 0.1847 - val_loss: 12.9768 - val_acc: 0.1616\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9597 - acc: 0.1764 - val_loss: 12.9499 - val_acc: 0.1739\n",
      "10000/10000 [==============================] - 1s 105us/step\n",
      "{'classes': 10, 'activation': 'sigmoid', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.5055944549804372, 'lr': 0.010523858865420845}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 4s 107us/step - loss: 13.8971 - acc: 0.1017 - val_loss: 14.5401 - val_acc: 0.0979\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 14.4978 - acc: 0.1005 - val_loss: 14.5401 - val_acc: 0.0979\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 14.4978 - acc: 0.1005 - val_loss: 14.5401 - val_acc: 0.0979\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 14.4978 - acc: 0.1005 - val_loss: 14.5401 - val_acc: 0.0979\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 14.4978 - acc: 0.1005 - val_loss: 14.5401 - val_acc: 0.0979\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 14.4978 - acc: 0.1005 - val_loss: 14.5401 - val_acc: 0.0979\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 14.4978 - acc: 0.1005 - val_loss: 14.5401 - val_acc: 0.0979\n",
      "10000/10000 [==============================] - 1s 105us/step\n",
      "{'classes': 10, 'activation': 'sigmoid', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.21653206382235193, 'lr': 0.005808954713135917}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 4s 108us/step - loss: 12.0026 - acc: 0.1201 - val_loss: 11.9331 - val_acc: 0.1012\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 11.6348 - acc: 0.1461 - val_loss: 11.6508 - val_acc: 0.1144\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 11.6117 - acc: 0.1481 - val_loss: 11.5781 - val_acc: 0.1336\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 11.6085 - acc: 0.1505 - val_loss: 11.5994 - val_acc: 0.1436\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 11.5983 - acc: 0.1560 - val_loss: 11.7209 - val_acc: 0.1078\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 11.6496 - acc: 0.1483 - val_loss: 11.6024 - val_acc: 0.1279\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 11.6069 - acc: 0.1560 - val_loss: 11.5598 - val_acc: 0.1488\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 11.6258 - acc: 0.1564 - val_loss: 11.6347 - val_acc: 0.1430\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 11.6216 - acc: 0.1571 - val_loss: 11.5359 - val_acc: 0.1513\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 11.5920 - acc: 0.1646 - val_loss: 11.5136 - val_acc: 0.1639\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 11.5748 - acc: 0.1697 - val_loss: 11.5715 - val_acc: 0.1477\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 11.5779 - acc: 0.1697 - val_loss: 11.8527 - val_acc: 0.1349\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 11.6087 - acc: 0.1716 - val_loss: 11.7161 - val_acc: 0.1138\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 11.5425 - acc: 0.1848 - val_loss: 11.7959 - val_acc: 0.1074\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 11.5922 - acc: 0.1700 - val_loss: 11.6002 - val_acc: 0.1677\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 11.5488 - acc: 0.1852 - val_loss: 11.5596 - val_acc: 0.1635\n",
      "10000/10000 [==============================] - 1s 103us/step\n",
      "{'classes': 10, 'activation': 'sigmoid', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.0385641552881645, 'lr': 0.004474191718012783}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 4s 111us/step - loss: 10.4849 - acc: 0.1784 - val_loss: 10.4234 - val_acc: 0.1716\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 10.1024 - acc: 0.2311 - val_loss: 10.2709 - val_acc: 0.1730\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 10.0446 - acc: 0.2473 - val_loss: 10.1682 - val_acc: 0.2001\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 10.0421 - acc: 0.2505 - val_loss: 10.1516 - val_acc: 0.2085\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 10.0153 - acc: 0.2617 - val_loss: 10.0679 - val_acc: 0.2459\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 10.0092 - acc: 0.2650 - val_loss: 10.1075 - val_acc: 0.2297\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 9.9856 - acc: 0.2732 - val_loss: 10.4435 - val_acc: 0.2073\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 10.0285 - acc: 0.2634 - val_loss: 10.0721 - val_acc: 0.2491\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 9.9723 - acc: 0.2801 - val_loss: 10.0826 - val_acc: 0.2424\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 9.9600 - acc: 0.2837 - val_loss: 10.1376 - val_acc: 0.2300\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 10.0112 - acc: 0.2719 - val_loss: 10.5578 - val_acc: 0.1387\n",
      "10000/10000 [==============================] - 1s 105us/step\n",
      "40000/40000 [==============================] - 4s 103us/step\n",
      "10000/10000 [==============================] - 1s 111us/step\n",
      "Loss = 10.544600828552246\n",
      "Test Accuracy = 0.1305\n",
      "{'classes': 10, 'activation': 'sigmoid', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.45809590837531666, 'lr': 0.017329157336070412}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 4s 112us/step - loss: 14.0058 - acc: 0.1022 - val_loss: 14.6143 - val_acc: 0.0933\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 14.4793 - acc: 0.1017 - val_loss: 14.6143 - val_acc: 0.0933\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 14.4793 - acc: 0.1017 - val_loss: 14.6143 - val_acc: 0.0933\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 14.4793 - acc: 0.1017 - val_loss: 14.6143 - val_acc: 0.0933\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 14.4793 - acc: 0.1017 - val_loss: 14.6143 - val_acc: 0.0933\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 14.4793 - acc: 0.1017 - val_loss: 14.6143 - val_acc: 0.0933\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 14.4793 - acc: 0.1017 - val_loss: 14.6143 - val_acc: 0.0933\n",
      "10000/10000 [==============================] - 1s 105us/step\n",
      "{'classes': 10, 'activation': 'sigmoid', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.408931354589519, 'lr': 0.0056001235851507025}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 5s 116us/step - loss: 11.8976 - acc: 0.1271 - val_loss: 11.5831 - val_acc: 0.1618\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 11.6729 - acc: 0.1530 - val_loss: 11.5607 - val_acc: 0.1636\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 11.6053 - acc: 0.1667 - val_loss: 11.5454 - val_acc: 0.1520\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 11.5815 - acc: 0.1682 - val_loss: 11.5189 - val_acc: 0.1623\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 11.5611 - acc: 0.1726 - val_loss: 11.6043 - val_acc: 0.1540\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 11.5474 - acc: 0.1802 - val_loss: 11.4828 - val_acc: 0.1977\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 11.5456 - acc: 0.1816 - val_loss: 11.5008 - val_acc: 0.1801\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 11.5953 - acc: 0.1737 - val_loss: 11.5339 - val_acc: 0.1558\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 11.5412 - acc: 0.1861 - val_loss: 11.5968 - val_acc: 0.1383\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 11.5359 - acc: 0.1880 - val_loss: 11.5073 - val_acc: 0.1768\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 11.5818 - acc: 0.1888 - val_loss: 11.6382 - val_acc: 0.1740\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 11.5225 - acc: 0.2000 - val_loss: 11.4450 - val_acc: 0.2082\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 11.4922 - acc: 0.2028 - val_loss: 11.5374 - val_acc: 0.1642\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 11.5145 - acc: 0.1990 - val_loss: 11.4519 - val_acc: 0.1970\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 11.5472 - acc: 0.1946 - val_loss: 11.4470 - val_acc: 0.2068\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 11.4757 - acc: 0.2099 - val_loss: 11.4338 - val_acc: 0.2156\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 11.4906 - acc: 0.2044 - val_loss: 11.6437 - val_acc: 0.1396\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 11.4820 - acc: 0.2086 - val_loss: 11.5363 - val_acc: 0.1669\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 11.4822 - acc: 0.2077 - val_loss: 11.5715 - val_acc: 0.1891\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 11.5631 - acc: 0.1946 - val_loss: 11.4667 - val_acc: 0.2011\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 11.4827 - acc: 0.2103 - val_loss: 11.6335 - val_acc: 0.1889\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 11.5882 - acc: 0.1986 - val_loss: 11.5202 - val_acc: 0.1743\n",
      "10000/10000 [==============================] - 1s 106us/step\n",
      "{'classes': 10, 'activation': 'sigmoid', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.4110757255309516, 'lr': 0.006672745114358988}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 5s 119us/step - loss: 12.6332 - acc: 0.1134 - val_loss: 13.1216 - val_acc: 0.1334\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 13.0228 - acc: 0.1306 - val_loss: 13.1402 - val_acc: 0.1162\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 13.0457 - acc: 0.1273 - val_loss: 13.1514 - val_acc: 0.1173\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 13.0231 - acc: 0.1348 - val_loss: 13.1595 - val_acc: 0.1078\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 13.0676 - acc: 0.1291 - val_loss: 13.1130 - val_acc: 0.1401\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 13.0055 - acc: 0.1388 - val_loss: 13.1115 - val_acc: 0.1410\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 13.0145 - acc: 0.1400 - val_loss: 13.2203 - val_acc: 0.1027\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 13.0020 - acc: 0.1417 - val_loss: 13.1030 - val_acc: 0.1465\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9846 - acc: 0.1459 - val_loss: 13.1374 - val_acc: 0.1204\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9751 - acc: 0.1502 - val_loss: 13.1520 - val_acc: 0.1146\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 13.0035 - acc: 0.1429 - val_loss: 13.1032 - val_acc: 0.1468\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9555 - acc: 0.1611 - val_loss: 13.1697 - val_acc: 0.1220\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 13.0515 - acc: 0.1366 - val_loss: 13.2034 - val_acc: 0.1151\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9930 - acc: 0.1507 - val_loss: 13.0918 - val_acc: 0.1547\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9676 - acc: 0.1563 - val_loss: 13.1027 - val_acc: 0.1494\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9582 - acc: 0.1602 - val_loss: 13.0992 - val_acc: 0.1491\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9810 - acc: 0.1535 - val_loss: 13.1754 - val_acc: 0.1137\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 12.9552 - acc: 0.1619 - val_loss: 13.1014 - val_acc: 0.1460\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 12.9559 - acc: 0.1612 - val_loss: 13.0890 - val_acc: 0.1534\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 12.9442 - acc: 0.1662 - val_loss: 13.2705 - val_acc: 0.1057\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9486 - acc: 0.1648 - val_loss: 13.1148 - val_acc: 0.1457\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 13.0038 - acc: 0.1525 - val_loss: 13.2389 - val_acc: 0.1137\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9473 - acc: 0.1662 - val_loss: 13.0756 - val_acc: 0.1647\n",
      "Epoch 24/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 12.9656 - acc: 0.1608 - val_loss: 13.1243 - val_acc: 0.1376\n",
      "Epoch 25/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 12.9525 - acc: 0.1651 - val_loss: 13.0741 - val_acc: 0.1670\n",
      "Epoch 26/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 12.9409 - acc: 0.1689 - val_loss: 13.0944 - val_acc: 0.1538\n",
      "Epoch 27/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9305 - acc: 0.1738 - val_loss: 13.0748 - val_acc: 0.1647\n",
      "Epoch 28/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9419 - acc: 0.1686 - val_loss: 13.1106 - val_acc: 0.1421\n",
      "Epoch 29/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 12.9361 - acc: 0.1708 - val_loss: 13.0719 - val_acc: 0.1667\n",
      "Epoch 30/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9572 - acc: 0.1657 - val_loss: 13.3930 - val_acc: 0.1029\n",
      "Epoch 31/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 12.9380 - acc: 0.1710 - val_loss: 13.0693 - val_acc: 0.1678\n",
      "Epoch 32/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 12.9311 - acc: 0.1740 - val_loss: 13.2834 - val_acc: 0.1127\n",
      "Epoch 33/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 12.9393 - acc: 0.1707 - val_loss: 13.0954 - val_acc: 0.1540\n",
      "Epoch 34/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 12.9380 - acc: 0.1710 - val_loss: 13.0679 - val_acc: 0.1676\n",
      "Epoch 35/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 12.9353 - acc: 0.1729 - val_loss: 13.2392 - val_acc: 0.1177\n",
      "Epoch 36/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9330 - acc: 0.1736 - val_loss: 13.0845 - val_acc: 0.1592\n",
      "Epoch 37/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 12.9321 - acc: 0.1742 - val_loss: 13.1237 - val_acc: 0.1452\n",
      "Epoch 38/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 12.9337 - acc: 0.1735 - val_loss: 13.1527 - val_acc: 0.1310\n",
      "Epoch 39/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 12.9316 - acc: 0.1732 - val_loss: 13.1012 - val_acc: 0.1518\n",
      "Epoch 40/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 12.9323 - acc: 0.1745 - val_loss: 13.2121 - val_acc: 0.1164\n",
      "10000/10000 [==============================] - 1s 103us/step\n",
      "{'classes': 10, 'activation': 'sigmoid', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.4046229491429527, 'lr': 0.003098052105428273}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 5s 122us/step - loss: 10.3010 - acc: 0.1535 - val_loss: 10.1951 - val_acc: 0.1937\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 10.0992 - acc: 0.2180 - val_loss: 10.1094 - val_acc: 0.2298\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 10.0807 - acc: 0.2251 - val_loss: 10.0965 - val_acc: 0.2350\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 10.0622 - acc: 0.2327 - val_loss: 10.1048 - val_acc: 0.2186\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 10.0656 - acc: 0.2341 - val_loss: 10.1564 - val_acc: 0.2018\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 10.0753 - acc: 0.2302 - val_loss: 10.1110 - val_acc: 0.2249\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 10.0514 - acc: 0.2411 - val_loss: 10.2330 - val_acc: 0.1976\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 10.1208 - acc: 0.2255 - val_loss: 10.0616 - val_acc: 0.2534\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 10.0344 - acc: 0.2488 - val_loss: 10.1244 - val_acc: 0.2012\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 10.0679 - acc: 0.2384 - val_loss: 10.0996 - val_acc: 0.2193\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 10.0468 - acc: 0.2462 - val_loss: 10.0712 - val_acc: 0.2363\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 10.0031 - acc: 0.2622 - val_loss: 10.1795 - val_acc: 0.1672\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 9.9996 - acc: 0.2656 - val_loss: 10.1706 - val_acc: 0.1776\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 10.0368 - acc: 0.2517 - val_loss: 10.3537 - val_acc: 0.1529\n",
      "10000/10000 [==============================] - 1s 104us/step\n",
      "{'classes': 10, 'activation': 'sigmoid', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.17780086739531917, 'lr': 0.016366707570086354}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 5s 124us/step - loss: 13.9884 - acc: 0.0985 - val_loss: 14.4418 - val_acc: 0.1040\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 14.5224 - acc: 0.0990 - val_loss: 14.4418 - val_acc: 0.1040\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 14.5224 - acc: 0.0990 - val_loss: 14.4418 - val_acc: 0.1040\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 14.5224 - acc: 0.0990 - val_loss: 14.4418 - val_acc: 0.1040\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 14.5224 - acc: 0.0990 - val_loss: 14.4418 - val_acc: 0.1040\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 14.5224 - acc: 0.0990 - val_loss: 14.4418 - val_acc: 0.1040\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 14.5224 - acc: 0.0990 - val_loss: 14.4418 - val_acc: 0.1040\n",
      "10000/10000 [==============================] - 1s 107us/step\n",
      "40000/40000 [==============================] - 5s 113us/step\n",
      "10000/10000 [==============================] - 1s 122us/step\n",
      "Loss = 10.344166860961915\n",
      "Test Accuracy = 0.1532\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.23396052932260447, 'lr': 0.007221643065142851}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 5s 129us/step - loss: 7.2148 - acc: 0.2222 - val_loss: 11.1422 - val_acc: 0.1576\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 4.8761 - acc: 0.3118 - val_loss: 12.6076 - val_acc: 0.1059\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 3.0397 - acc: 0.3820 - val_loss: 9.4433 - val_acc: 0.1662\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.6367 - acc: 0.4542 - val_loss: 5.0095 - val_acc: 0.1308\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.4699 - acc: 0.4910 - val_loss: 4.8674 - val_acc: 0.1392\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.4119 - acc: 0.5077 - val_loss: 5.0027 - val_acc: 0.1450\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.3702 - acc: 0.5201 - val_loss: 5.5019 - val_acc: 0.1319\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.3047 - acc: 0.5402 - val_loss: 4.5319 - val_acc: 0.1601\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.2631 - acc: 0.5594 - val_loss: 4.9603 - val_acc: 0.1106\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.2446 - acc: 0.5664 - val_loss: 4.6768 - val_acc: 0.1125\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.1863 - acc: 0.5854 - val_loss: 6.6362 - val_acc: 0.1313\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.1501 - acc: 0.5963 - val_loss: 3.1103 - val_acc: 0.2151\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.1061 - acc: 0.6121 - val_loss: 3.7718 - val_acc: 0.1966\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.1022 - acc: 0.6147 - val_loss: 3.9013 - val_acc: 0.1777\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.0702 - acc: 0.6229 - val_loss: 4.4194 - val_acc: 0.1772\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.0431 - acc: 0.6379 - val_loss: 2.5169 - val_acc: 0.3235\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.0262 - acc: 0.6401 - val_loss: 4.5325 - val_acc: 0.1884\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.0385 - acc: 0.6371 - val_loss: 1.9764 - val_acc: 0.3845\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.0306 - acc: 0.6402 - val_loss: 2.6122 - val_acc: 0.2840\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.9991 - acc: 0.6519 - val_loss: 3.6719 - val_acc: 0.2709\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.9993 - acc: 0.6513 - val_loss: 4.1379 - val_acc: 0.2057\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.0107 - acc: 0.6497 - val_loss: 3.4150 - val_acc: 0.2392\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.9760 - acc: 0.6594 - val_loss: 2.3024 - val_acc: 0.3896\n",
      "Epoch 24/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.9786 - acc: 0.6594 - val_loss: 1.8174 - val_acc: 0.4347\n",
      "Epoch 25/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.9779 - acc: 0.6571 - val_loss: 1.8453 - val_acc: 0.4063\n",
      "Epoch 26/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.9808 - acc: 0.6595 - val_loss: 2.8292 - val_acc: 0.3035\n",
      "Epoch 27/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.9551 - acc: 0.6684 - val_loss: 2.6885 - val_acc: 0.3378\n",
      "Epoch 28/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.9553 - acc: 0.6680 - val_loss: 2.1819 - val_acc: 0.3596\n",
      "Epoch 29/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.9361 - acc: 0.6765 - val_loss: 1.5608 - val_acc: 0.4776\n",
      "Epoch 30/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.9591 - acc: 0.6662 - val_loss: 1.7155 - val_acc: 0.4360\n",
      "Epoch 31/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.9363 - acc: 0.6760 - val_loss: 2.2843 - val_acc: 0.3892\n",
      "Epoch 32/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.9457 - acc: 0.6688 - val_loss: 2.6972 - val_acc: 0.2917\n",
      "Epoch 33/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.9409 - acc: 0.6726 - val_loss: 1.7118 - val_acc: 0.4714\n",
      "Epoch 34/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.9402 - acc: 0.6714 - val_loss: 3.0097 - val_acc: 0.2541\n",
      "Epoch 35/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.9203 - acc: 0.6805 - val_loss: 1.8417 - val_acc: 0.4404\n",
      "10000/10000 [==============================] - 1s 108us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.3313272517075206, 'lr': 0.014170877449407943}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 5s 132us/step - loss: 9.8430 - acc: 0.2343 - val_loss: 9.3182 - val_acc: 0.2071\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 7.1611 - acc: 0.2860 - val_loss: 14.0310 - val_acc: 0.1044\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 2.2305 - acc: 0.3906 - val_loss: 8.5319 - val_acc: 0.1816\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.5776 - acc: 0.4565 - val_loss: 6.1442 - val_acc: 0.1906\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.4602 - acc: 0.4895 - val_loss: 3.7778 - val_acc: 0.2299\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.4104 - acc: 0.5025 - val_loss: 3.2852 - val_acc: 0.2424\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.3617 - acc: 0.5163 - val_loss: 3.5253 - val_acc: 0.2305\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.3562 - acc: 0.5198 - val_loss: 3.4479 - val_acc: 0.2379\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.3141 - acc: 0.5366 - val_loss: 4.5546 - val_acc: 0.1786\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.2735 - acc: 0.5512 - val_loss: 4.2002 - val_acc: 0.1613\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.2463 - acc: 0.5624 - val_loss: 3.8745 - val_acc: 0.1948\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.2208 - acc: 0.5747 - val_loss: 3.4307 - val_acc: 0.2216\n",
      "10000/10000 [==============================] - 1s 107us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.061431309451076464, 'lr': 0.006032655350264376}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 5s 131us/step - loss: 2.5893 - acc: 0.2924 - val_loss: 2.7139 - val_acc: 0.2603\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.7002 - acc: 0.4095 - val_loss: 3.7679 - val_acc: 0.1836\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.5361 - acc: 0.4617 - val_loss: 2.6000 - val_acc: 0.2904\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.4699 - acc: 0.4881 - val_loss: 4.2304 - val_acc: 0.2549\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.4576 - acc: 0.4974 - val_loss: 2.3382 - val_acc: 0.3753\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.3373 - acc: 0.5317 - val_loss: 2.6311 - val_acc: 0.3428\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.2714 - acc: 0.5568 - val_loss: 2.2927 - val_acc: 0.3472\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.2404 - acc: 0.5658 - val_loss: 1.6549 - val_acc: 0.4638\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.2057 - acc: 0.5821 - val_loss: 1.6368 - val_acc: 0.4651\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.1541 - acc: 0.5954 - val_loss: 2.1799 - val_acc: 0.3919\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.1480 - acc: 0.5997 - val_loss: 2.3839 - val_acc: 0.3316\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.1211 - acc: 0.6102 - val_loss: 2.0752 - val_acc: 0.4349\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.1045 - acc: 0.6143 - val_loss: 1.8608 - val_acc: 0.4604\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.0593 - acc: 0.6318 - val_loss: 1.9332 - val_acc: 0.4362\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.0567 - acc: 0.6326 - val_loss: 2.1574 - val_acc: 0.4276\n",
      "10000/10000 [==============================] - 1s 108us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.4544525820397381, 'lr': 0.00597123469444552}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 5s 136us/step - loss: 6.8821 - acc: 0.2278 - val_loss: 8.8373 - val_acc: 0.1634\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 3.7486 - acc: 0.3116 - val_loss: 10.2861 - val_acc: 0.1189\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 3.0992 - acc: 0.3800 - val_loss: 10.3765 - val_acc: 0.1242\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 2.9091 - acc: 0.4275 - val_loss: 10.8591 - val_acc: 0.0921\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 2.7754 - acc: 0.4580 - val_loss: 11.9137 - val_acc: 0.1179\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 2.2404 - acc: 0.4703 - val_loss: 3.6904 - val_acc: 0.2207\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.3721 - acc: 0.5195 - val_loss: 3.0293 - val_acc: 0.2585\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.2943 - acc: 0.5458 - val_loss: 3.5284 - val_acc: 0.2285\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.2582 - acc: 0.5571 - val_loss: 3.6624 - val_acc: 0.2700\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.2019 - acc: 0.5770 - val_loss: 3.4646 - val_acc: 0.2664\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.1702 - acc: 0.5874 - val_loss: 4.3450 - val_acc: 0.1726\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.1701 - acc: 0.5876 - val_loss: 4.5206 - val_acc: 0.2091\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.1048 - acc: 0.6111 - val_loss: 2.7861 - val_acc: 0.2864\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.0990 - acc: 0.6139 - val_loss: 4.3854 - val_acc: 0.1605\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.0946 - acc: 0.6146 - val_loss: 4.0647 - val_acc: 0.1884\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.0846 - acc: 0.6199 - val_loss: 3.5777 - val_acc: 0.2480\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.0417 - acc: 0.6361 - val_loss: 3.4048 - val_acc: 0.2544\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.0429 - acc: 0.6333 - val_loss: 3.3010 - val_acc: 0.2529\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.0415 - acc: 0.6370 - val_loss: 2.7788 - val_acc: 0.2581\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.0169 - acc: 0.6459 - val_loss: 2.9978 - val_acc: 0.2491\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.0189 - acc: 0.6443 - val_loss: 3.1288 - val_acc: 0.2640\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.0051 - acc: 0.6485 - val_loss: 2.8742 - val_acc: 0.2725\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.0001 - acc: 0.6501 - val_loss: 4.1555 - val_acc: 0.1809\n",
      "Epoch 24/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.9926 - acc: 0.6517 - val_loss: 2.7992 - val_acc: 0.2658\n",
      "Epoch 25/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.9751 - acc: 0.6594 - val_loss: 3.1781 - val_acc: 0.2389\n",
      "10000/10000 [==============================] - 1s 107us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.0436343603198178, 'lr': 0.007702135877788225}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 6s 138us/step - loss: 3.0186 - acc: 0.2842 - val_loss: 3.3361 - val_acc: 0.2321\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.8112 - acc: 0.3871 - val_loss: 2.5019 - val_acc: 0.2808\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.6676 - acc: 0.4296 - val_loss: 2.9000 - val_acc: 0.2758\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.5505 - acc: 0.4634 - val_loss: 3.3331 - val_acc: 0.2418\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.5057 - acc: 0.4776 - val_loss: 3.8213 - val_acc: 0.2608\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.4720 - acc: 0.4921 - val_loss: 4.6861 - val_acc: 0.2406\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.4132 - acc: 0.5098 - val_loss: 3.1288 - val_acc: 0.2499\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.3450 - acc: 0.5315 - val_loss: 2.7031 - val_acc: 0.2903\n",
      "10000/10000 [==============================] - 1s 107us/step\n",
      "40000/40000 [==============================] - 4s 105us/step\n",
      "10000/10000 [==============================] - 1s 115us/step\n",
      "Loss = 1.8626601154327393\n",
      "Test Accuracy = 0.4277\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.5769392665632171, 'lr': 0.0031990309288565267}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 6s 140us/step - loss: 6.1388 - acc: 0.2190 - val_loss: 8.3872 - val_acc: 0.1484\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 4.2325 - acc: 0.2942 - val_loss: 7.7762 - val_acc: 0.1206\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 2.3400 - acc: 0.3597 - val_loss: 4.4677 - val_acc: 0.2170\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.6196 - acc: 0.4408 - val_loss: 6.7681 - val_acc: 0.1251\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.5065 - acc: 0.4725 - val_loss: 3.2081 - val_acc: 0.2341\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.4271 - acc: 0.4975 - val_loss: 3.6002 - val_acc: 0.1788\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.4118 - acc: 0.5106 - val_loss: 3.2995 - val_acc: 0.2181\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.3469 - acc: 0.5292 - val_loss: 5.1721 - val_acc: 0.1572\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.2764 - acc: 0.5538 - val_loss: 3.2684 - val_acc: 0.2354\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.2292 - acc: 0.5686 - val_loss: 4.3412 - val_acc: 0.1498\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.1943 - acc: 0.5805 - val_loss: 3.6760 - val_acc: 0.2459\n",
      "10000/10000 [==============================] - 1s 110us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.3915606456409716, 'lr': 0.00892415675428787}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 9.5315 - acc: 0.2162 - val_loss: 13.5639 - val_acc: 0.1023\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 6.5483 - acc: 0.2953 - val_loss: 13.3531 - val_acc: 0.0974\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 3.9923 - acc: 0.3514 - val_loss: 8.8242 - val_acc: 0.1434\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.6805 - acc: 0.4242 - val_loss: 6.8632 - val_acc: 0.1959\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.4708 - acc: 0.4790 - val_loss: 7.0745 - val_acc: 0.1821\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.4151 - acc: 0.5027 - val_loss: 6.0759 - val_acc: 0.1674\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.3620 - acc: 0.5176 - val_loss: 5.0387 - val_acc: 0.1552\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.3188 - acc: 0.5354 - val_loss: 6.4324 - val_acc: 0.1718\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.2773 - acc: 0.5461 - val_loss: 5.4128 - val_acc: 0.1677\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.2488 - acc: 0.5591 - val_loss: 4.6923 - val_acc: 0.1749\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.2069 - acc: 0.5732 - val_loss: 7.5253 - val_acc: 0.1315\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.1878 - acc: 0.5814 - val_loss: 4.7663 - val_acc: 0.2173\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.1670 - acc: 0.5905 - val_loss: 4.7333 - val_acc: 0.1870\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.1186 - acc: 0.6101 - val_loss: 3.8949 - val_acc: 0.1759\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.1110 - acc: 0.6099 - val_loss: 5.9149 - val_acc: 0.2026\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.0798 - acc: 0.6222 - val_loss: 4.5098 - val_acc: 0.1844\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.0596 - acc: 0.6305 - val_loss: 2.4803 - val_acc: 0.2955\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 1.0529 - acc: 0.6324 - val_loss: 4.6831 - val_acc: 0.1643\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.0457 - acc: 0.6341 - val_loss: 4.7072 - val_acc: 0.1930\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.0292 - acc: 0.6426 - val_loss: 4.2142 - val_acc: 0.1390\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.0292 - acc: 0.6433 - val_loss: 3.7666 - val_acc: 0.2537\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.0066 - acc: 0.6477 - val_loss: 4.2227 - val_acc: 0.1463\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.0058 - acc: 0.6499 - val_loss: 4.1893 - val_acc: 0.1874\n",
      "10000/10000 [==============================] - 1s 107us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.20853656681220395, 'lr': 0.00290261737746798}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 6s 146us/step - loss: 2.2354 - acc: 0.3088 - val_loss: 2.3967 - val_acc: 0.2913\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.6480 - acc: 0.4180 - val_loss: 4.7866 - val_acc: 0.1836\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.5126 - acc: 0.4663 - val_loss: 4.6075 - val_acc: 0.2032\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.3910 - acc: 0.5085 - val_loss: 4.4968 - val_acc: 0.1949\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.3218 - acc: 0.5347 - val_loss: 4.5555 - val_acc: 0.1817\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.2578 - acc: 0.5568 - val_loss: 6.0808 - val_acc: 0.1704\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.2591 - acc: 0.5571 - val_loss: 3.8144 - val_acc: 0.1956\n",
      "10000/10000 [==============================] - 1s 109us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.00858911659885928, 'lr': 0.004777535870000538}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 6s 148us/step - loss: 2.2957 - acc: 0.3118 - val_loss: 2.0730 - val_acc: 0.3132\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.6708 - acc: 0.4178 - val_loss: 3.2581 - val_acc: 0.2050\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.5274 - acc: 0.4673 - val_loss: 3.5087 - val_acc: 0.1955\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.4204 - acc: 0.5060 - val_loss: 4.7023 - val_acc: 0.1653\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.3852 - acc: 0.5211 - val_loss: 2.9973 - val_acc: 0.2777\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.2823 - acc: 0.5520 - val_loss: 2.1044 - val_acc: 0.3673\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.2467 - acc: 0.5640 - val_loss: 2.6935 - val_acc: 0.3174\n",
      "10000/10000 [==============================] - 1s 108us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'reg_param': 0, 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f3a1fd0be80>], 'dropout_prob': 0.17874324937786606, 'lr': 0.006827210431543594}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 6s 149us/step - loss: 3.4061 - acc: 0.2742 - val_loss: 3.1752 - val_acc: 0.2558\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.8762 - acc: 0.3710 - val_loss: 6.5582 - val_acc: 0.1802\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.6666 - acc: 0.4287 - val_loss: 6.7280 - val_acc: 0.1530\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.6014 - acc: 0.4520 - val_loss: 6.6929 - val_acc: 0.1774\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.4870 - acc: 0.4859 - val_loss: 6.1678 - val_acc: 0.2298\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.3991 - acc: 0.5156 - val_loss: 6.4820 - val_acc: 0.1914\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.3584 - acc: 0.5279 - val_loss: 5.9535 - val_acc: 0.1849\n",
      "10000/10000 [==============================] - 1s 109us/step\n",
      "40000/40000 [==============================] - 4s 108us/step\n",
      "10000/10000 [==============================] - 1s 119us/step\n",
      "Loss = 2.6941247436523437\n",
      "Test Accuracy = 0.3236\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXl4FGW2xt9D2BfZFQSSAAYhBAIh\nggoCskgQFURUFmdEx0FUXNC5XkYdtzvovTMKM85wVcbrjAqCqAPiGGFcEAEXEjYFAoQtEEAIW1gC\nEsi5f5yqpNPppbq7uru6c37P0091V31Vdbo6eevU+c53PmJmKIqiKPFFjWgboCiKotiPiruiKEoc\nouKuKIoSh6i4K4qixCEq7oqiKHGIiruiKEocouIexxBRAhGdIqJEO9tGEyK6jIhsz98loiFEtNvl\n81YiusZK2yDO9QYRPRHs/opihZrRNkCpgIhOuXysD+BnABeMz/cy89xAjsfMFwA0tLttdYCZL7fj\nOER0D4A7mHmgy7HvsePYiuILFXcHwczl4mp4hvcw8+fe2hNRTWY+HwnbFMUf+vfoLDQsE0MQ0e+J\n6D0imkdEJwHcQURXEdF3RHSciA4Q0StEVMtoX5OImIiSjc9zjO2fEtFJIvqWiNoH2tbYPpyIthFR\nMRH9hYhWEdFEL3ZbsfFeItpORMeI6BWXfROIaCYRHSGinQCyfFyfJ4lovtu6WUQ0w3h/DxHlGd9n\nh+FVeztWIRENNN7XJ6J3DNs2Aejl1vYpItppHHcTEd1krO8G4K8ArjFCXoddru2zLvtPNr77ESJa\nREStrVybQK6zaQ8RfU5ER4noJyJ63OU8vzOuyQkiyiWiSz2FwIhopfk7G9fza+M8RwE8RUQpRLTM\nOMdh47o1dtk/yfiORcb2PxNRXcPmLi7tWhNRCRE19/Z9FT8ws74c+AKwG8AQt3W/B3AOwI2QG3M9\nAFcA6AN5CusAYBuAKUb7mgAYQLLxeQ6AwwAyAdQC8B6AOUG0vRjASQAjjW2PAigFMNHLd7Fi40cA\nGgNIBnDU/O4ApgDYBKAtgOYAvpY/W4/n6QDgFIAGLsc+BCDT+Hyj0YYADAJwBkB3Y9sQALtdjlUI\nYKDx/iUAXwFoCiAJwGa3trcBaG38JuMNGy4xtt0D4Cs3O+cAeNZ4f51hYw8AdQH8L4AvrVybAK9z\nYwAHATwMoA6AiwD0Nrb9FsAGACnGd+gBoBmAy9yvNYCV5u9sfLfzAO4DkAD5e+wEYDCA2sbfySoA\nL7l8n43G9WxgtO9rbJsNYLrLeR4DsDDa/4ex/Iq6Afry8sN4F/cv/ez3GwDvG+89CfZrLm1vArAx\niLZ3A1jhso0AHIAXcbdo45Uu2/8J4DfG+68h4Slz2/XuguN27O8AjDfeDwew1UfbfwF4wHjvS9z3\nuP4WAO53bevhuBsBjDDe+xP3twC84LLtIkg/S1t/1ybA6/wLADle2u0w7XVbb0Xcd/qxYYx5XgDX\nAPgJQIKHdn0B7AJAxuf1AEbb/X9VnV4alok99rp+IKLORPSJ8Zh9AsDzAFr42P8nl/cl8N2J6q3t\npa52sPw3Fno7iEUbLZ0LQIEPewHgXQDjjPfjjc+mHTcQ0fdGyOA4xGv2da1MWvuygYgmEtEGI7Rw\nHEBni8cF5PuVH4+ZTwA4BqCNSxtLv5mf69wOIuKe8LXNH+5/j62IaAER7TNs+IebDbtZOu8rwcyr\nIE8B/YgoDUAigE+CtEmBxtxjEfc0wNchnuJlzHwRgKchnnQ4OQDxLAEARESoLEbuhGLjAYgomPhL\n1VwAYAgRtYGEjd41bKwH4AMAL0JCJk0A/NuiHT95s4GIOgB4FRKaaG4cd4vLcf2lbe6HhHrM4zWC\nhH/2WbDLHV/XeS+Ajl7287bttGFTfZd1rdzauH+//4FkeXUzbJjoZkMSESV4seNtAHdAnjIWMPPP\nXtopFlBxj30aASgGcNrokLo3Auf8F4AMIrqRiGpC4rgtw2TjAgCPEFEbo3PtP301ZuafIKGDf0BC\nMvnGpjqQOHARgAtEdAMkNmzVhieIqAnJOIApLtsaQgSuCHKf+zXEczc5CKCta8emG/MA/IqIuhNR\nHcjNZwUze30S8oGv67wYQCIRTSGiOkR0ERH1Nra9AeD3RNSRhB5E1AxyU/sJ0nGfQEST4HIj8mHD\naQDFRNQOEhoy+RbAEQAvkHRS1yOivi7b34GEccZDhF4JARX32OcxAHdCOjhfh3R8hhVmPgjgdgAz\nIP+sHQGsg3hsdtv4KoAvAPwIIAfiffvjXUgMvTwkw8zHAUwFsBDSKTkGcpOywjOQJ4jdAD6Fi/Aw\n8w8A/gJgtdHmcgDfu+z7GYB8AAeJyDW8Yu6/BBI+WWjsnwhggkW73PF6nZm5GMBQALdAbjjbAAww\nNv8RwCLIdT4B6dysa4Tbfg3gCUjn+mVu380TzwDoDbnJLAbwoYsN5wHcAKALxIvfA/kdzO27Ib/z\nz8z8TYDfXXHD7LxQlKAxHrP3AxjDzCuibY8SuxDR25BO2mejbUuso4OYlKAgoixIZsoZSCpdKcR7\nVZSgMPovRgLoFm1b4gENyyjB0g/ATkiseRiAm7UDTAkWInoRkmv/AjPvibY98YCGZRRFUeIQ9dwV\nRVHikKjF3Fu0aMHJycnROr2iKEpMsmbNmsPM7Cv1GEAUxT05ORm5ubnROr2iKEpMQkT+RmkD0LCM\noihKXGJJ3Ikoi2Rmmu1ENM3D9plEtN54bTPqayiKoihRwm9YxhigMgsyuq0QQA4RLWbmzWYbZp7q\n0v5BAD3DYKuiKIpiESsx994AtjPzTgAgmQxhJKSmtSfGQYYgB0xpaSkKCwtx9uzZYHZXIkTdunXR\ntm1b1KrlrVyKoijRxoq4t0Hlsp6FkAkBqkBESQDaA/jSy/ZJACYBQGJi1eJ+hYWFaNSoEZKTkyGF\nBhWnwcw4cuQICgsL0b59e/87KIoSFezuUB0L4ANP9ZoBgJlnM3MmM2e2bFk1k+fs2bNo3ry5CruD\nISI0b95cn64UxY25c4HkZKBGDVnO9TCdvZU2dmFF3Pehci3rtvBea3ospIRp0KiwOx/9jZR4wg5R\nnjsXmDQJKCgAmGU5aVLldlba2Iq/qZogoZudkHBLbUj9h64e2nWGlEQlK1NA9erVi93ZvHlzlXWK\nM9HfSokH5sxhrl+fWeRWXvXry/pA2iQlVd5uvpKSAmtjBQC5bMc0eyw1mKcAWAogDzJDyiYiep6M\nWd4NxgKYb5w8Jjly5Ah69OiBHj16oFWrVmjTpk3553Pnzlk6xl133YWtW7f6bDNr1izMDefzmKLE\nOJEKXzz5JFBSUnldSYmsD6TNHi+lzlzXW2ljK1buAOF42eG5z5kjdz0iWbreSUPlmWee4T/+8Y9V\n1peVlfGFCxfsO1GMop67Ei6seMp2QeTZmyYKrE1Meu5OJZLxq+3btyM1NRUTJkxA165dceDAAUya\nNAmZmZno2rUrnn/++fK2/fr1w/r163H+/Hk0adIE06ZNQ3p6Oq666iocOnQIAPDUU0/hT3/6U3n7\nadOmoXfv3rj88svxzTcyAc3p06dxyy23IDU1FWPGjEFmZibWr19fxbZnnnkGV1xxBdLS0jB58mQz\nRIZt27Zh0KBBSE9PR0ZGBnbv3g0AeOGFF9CtWzekp6fjSVfXQ1EcghVPGbAnVu4haa/Keittpk8H\n6tevvL1+fVkfSBtbsXIHCMcrVM/drrugN1w99/z8fCYizsnJKd9+5MgRZmYuLS3lfv368aZNm5iZ\nuW/fvrxu3TouLS1lAJydnc3MzFOnTuUXX3yRmZmffPJJnjlzZnn7xx9/nJmZP/roIx42bBgzM7/4\n4ot8//33MzPz+vXruUaNGrxu3boqdpp2lJWV8dixY8vPl5GRwYsXL2Zm5jNnzvDp06d58eLF3K9f\nPy4pKam0bzCo566ECyuesl2xcrvamO38RRLsiDYg3j33SMevOnbsiMzMzPLP8+bNQ0ZGBjIyMpCX\nl4fNm6uO6apXrx6GDx8OAOjVq1e59+zO6NGjq7RZuXIlxo4dCwBIT09H165dPe77xRdfoHfv3khP\nT8fy5cuxadMmHDt2DIcPH8aNN94IQAYd1a9fH59//jnuvvtu1KtXDwDQrFmzwC+EUm2xK9XPDm/a\nrlj5hAnA7NlAUhJAJMvZs2V9IG3Mdrt3A2VlsnTfbrWNXcTsNHuJiRKK8bQ+HDRo0KD8fX5+Pv78\n5z9j9erVaNKkCe644w6Ped+1a9cuf5+QkIDz5897PHadOnX8tvFESUkJpkyZgrVr16JNmzZ46qmn\nNP9cCQtmGNQUSzMMClQIlF1tpk+v3AaoGr6wswNzwgT/ImuljdOIWc894vErF06cOIFGjRrhoosu\nwoEDB7B06VLbz9G3b18sWLAAAPDjjz96fDI4c+YMatSogRYtWuDkyZP48EOZaL5p06Zo2bIlPv74\nYwAyOKykpARDhw7Fm2++iTNnzgAAjh49arvdSmziz5u2y1O2y5u2K1Yez8SsuFt9VAoHGRkZSE1N\nRefOnfHLX/4Sffv2tf0cDz74IPbt24fU1FQ899xzSE1NRePGjSu1ad68Oe68806kpqZi+PDh6NOn\noirE3Llz8fLLL6N79+7o168fioqKcMMNNyArKwuZmZno0aMHZs6cabvdSuxhJTnBLk85EG/aV/jC\nkR2YTsNKYD4cLx3E5JvS0lI+c+YMMzNv27aNk5OTubS0NMpWVaC/VezgrxPPrjS+SKYDWvleVtvE\nGrDYoari7lCOHTvGGRkZ3L17d+7WrRsvXbo02iZVQn8rZ+BPvKxkejgtO0XxjYq7Elb0t4o+kR4W\nb5enHI/edCSxKu4kbSNPZmYmu8+hmpeXhy5dukTFHiUw9LeKPsnJnjPGkpIkTg1IB6mnf3EiiWcD\nVTNYAIlNR6oPSwkMIlrDzJn+2sVsh6qixDJ21E6x0jlpJWMkmskJSvhQcVcUm7GjPKwV7BoWD0R2\ncI0SGVTcFcVGrAi3XbVTrAi3euXVGCuB+XC8nNihOnDgQF6yZEmldTNnzuTJkyf73K9BgwbMzLxv\n3z6+5ZZbPLYZMGBApdo0npg5cyafPn26/PPw4cP52LFjVkyPONH+raKFHWmFdmWnWLFHiT+g2TKB\n8/rrr/PEiRMrrevTpw8vX77c536muPvCirgnJSVxUVGRf0MdQLR/q2AIVQjtSiuMdD64El9YFXcN\ny7gwZswYfPLJJ+UTc+zevRv79+/HNddcg1OnTmHw4MHIyMhAt27d8NFHH1XZf/fu3UhLSwMgpQHG\njh2LLl264Oabby4f8g8A9913X3m54GeeeQYA8Morr2D//v249tprce211wIAkpOTcfjwYQDAjBkz\nkJaWhrS0tPJywbt370aXLl3w61//Gl27dsV1111X6TwmH3/8Mfr06YOePXtiyJAhOHjwIADg1KlT\nuOuuu9CtWzd07969vHzBkiVLkJGRgfT0dAwePNiWaxttrE6DFuoQfLvi4BGf2EGJP6zcAcLx8ue5\nP/ww84AB9r4eftj/XXHEiBG8aNEiZpayu4899hgzy4jR4uJiZmYuKirijh07cllZGTNXeO67du3i\nrl27MjPzyy+/zHfddRczM2/YsIETEhLKPXez1O758+d5wIABvGHDBmau6rmbn3NzczktLY1PnTrF\nJ0+e5NTUVF67di3v2rWLExISyksB33rrrfzOO+9U+U5Hjx4tt/Vvf/sbP/roo8zM/Pjjj/PDLhfl\n6NGjfOjQIW7bti3v3Lmzkq3uxJrn7s8TjuRgH7NdqOEdpXoC9dyDY9y4cZg/fz4AYP78+Rg3bhwA\nuQk+8cQT6N69O4YMGYJ9+/aVe8Ce+Prrr3HHHXcAALp3747u3buXb1uwYAEyMjLQs2dPbNq0yWNR\nMFdWrlyJm2++GQ0aNEDDhg0xevRorFixAgDQvn179OjRA4D3ssKFhYUYNmwYunXrhj/+8Y/YtGkT\nAODzzz/HAw88UN6uadOm+O6779C/f3+0b98eQPyUBfbnCdvlldtVHrba10VRQsaxJX+NyEPEGTly\nJKZOnYq1a9eipKQEvXr1AiCFuIqKirBmzRrUqlULycnJQZXX3bVrF1566SXk5OSgadOmmDhxYkhl\nes1ywYCUDPYUlnnwwQfx6KOP4qabbsJXX32FZ599NujzOZW5c0WI9+wRsZ0+vWoVQV8loq2EQayU\nogXsKQ9r7u/rOymKL9Rzd6Nhw4a49tprcffdd5d77QBQXFyMiy++GLVq1cKyZctQ4EkpXOjfvz/e\nffddAMDGjRvxww8/AJBywQ0aNEDjxo1x8OBBfPrpp+X7NGrUCCdPnqxyrGuuuQaLFi1CSUkJTp8+\njYULF+Kaa66x/J2Ki4vRpk0bAMBbb71Vvn7o0KGYNWtW+edjx47hyiuvxNdff41du3YBiI2ywFbi\n6f48YScO9tHccyUUVNw9MG7cOGzYsKGSuE+YMAG5ubno1q0b3n77bXTu3NnnMe677z6cOnUKXbp0\nwdNPP13+BJCeno6ePXuic+fOGD9+fKVywZMmTUJWVlZ5h6pJRkYGJk6ciN69e6NPnz6455570LNn\nT8vf59lnn8Wtt96KXr16oUWLFuXrn3rqKRw7dgxpaWlIT0/HsmXL0LJlS8yePRujR49Geno6br/9\ndsvniRZ21AjXwT5KvKG1ZZSgcNJvZaV+ihX8hXYUxQlYrS3j2Ji7oljFrikXY3EqNUXxhoZlFMdj\nxzB8RaluOE7coxUmUqxj529kR5EtrZ+ihJvSUmDQIMAY5xcTOErc69atiyNHjqjAOxhmxpEjR1C3\nbt2Qj2VnkS3t6Iwvjh4FHn4YmDPHnuOFKikffQQsWwa8/bY99kQCR3WolpaWorCwMKS8byX81K1b\nF23btkWtWrVCOo5dk00o8cVHHwGTJwM//QT06wcY4/WC5pprgP79QwvTDRkCfPEF0LQpcPiw/F1G\nC6sdqo4SdyW+8Jd9YkW4rdwAlPjg8GHgoYeAefOA9HSgUSP52/EzpMQnpaVA3bryKiwUcQ6UbduA\nyy8HuncHfvgBWLcOMAaFRwWdiUmJKlZCLnZONqGEzo4dwPLl0Tn3hx8CXbsC778PPPccsHo1MHAg\nsG8fcP588Mfdv18chZIS4G9/C+4Yr78O1KwJ/N//yeevvgrenohipQBNOF6eCocp8YOVwldas9xZ\njBol17+kJHLnPHSI+bbb5LfPyGA2augxM/Ps2bK+oCD44y9fLsdo0oS5TRvmc+cC27+khLlZM+Zb\nb5XPHTsyjxwZvD12AC0cpkQTK7Va7CqypYQOM7BypXi4n38emfMtWACkpgILFwK//z3w3XcS+jDx\nV/fHCua+Tz8tTwHvvx/Y/h98IJ27kyfL54EDga+/jo3+HhV3JSxYCbkAKtxOYds2iXkDwKJF4T/f\n448Dt98ufSpr10rfjHv/vPm3EkrM3dz33nslbj5jRmCZM6+9BnTqBJgVQQYMAI4dA378MXibIoUl\ncSeiLCLaSkTbiWialza3EdFmItpERO/aa6biNHRgUXyxapUsMzKAxYuBCxfCd67iYuCvfwXGjgW+\n/RYw5repgl2e+8UXy9/e1KnAmjXyhGKFH34AvvlGbgxEsm7AAFnGRNzdX9wGQAKAHQA6AKgNYAOA\nVLc2KQDWAWhqfL7Y33E15h67aKw8/rjrLubmzZnnz5ff8+uvw3euN9+Uc3z7rf+2zZsz+5nC2CfD\nhjFnZsr706fleKNGWdv3/vuZ69RhPny48voOHawfIxzAxph7bwDbmXknM58DMB/ASLc2vwYwi5mP\nGTeMQ6HedBTnogOL4o+VK4G+fYHhw4HatcMbmpk7F+jYEejTx3/bxMTQPPeCAunLAcR7v+8+yaPf\nvt33fqdOAe+8A9x2G9C8eeVtsRJ3tyLubQDsdflcaKxzpROATkS0ioi+I6IsTwcioklElEtEuUVF\nRcFZrEQdnd8zuhQXi7i88gpw990SSunfP3ixOXQIyM+XAUMXXQQMHiziHo4hMPv2AV9+CdxxR0Wo\nwxfeisJZgblijIXJ/fdLWuOf/+x733nzgJMnKzpSXRkwQDpZN24Mzq5IYVeHak1IaGYggHEA/kZE\nTdwbMfNsZs5k5syWLVvadGol0ljtLFVC59w5iYE/9xwwejTQoQPQpIkIzMMPA//6l8THV6yQjslg\nMOPt5tQCo0YBO3eGR7zmzRPRtfoUl5RUMVYiUI4ckSdK03MHgNatgfHjgb//XTpGPcEMvPoq0K0b\ncNVVVbfHStzdirjvA9DO5XNbY50rhQAWM3MpM+8CsA0i9kocop2lkeOvfwVGjhRx37QJuOIK4IUX\ngOxsGaBz8KCkLhIBS5YEd46VK4E6dQBjPhncdJMcLxyhmTlzJByTYlEdEhMlRFJcHPi5zCdJV3EH\npGP19Gnvg5pyc2UU6uTJnp8ukpKA9u3jQ9xzAKQQUXsiqg1gLIDFbm0WQbx2EFELSJhmp412Kg5C\nqzBGjq1bJeZ74oS8f+894Le/ldh469Zy/Vu2FGFeujS4c6xaBfTuLQIPAK1aAVdeab+4b9wIbNgQ\n2N9JKOmQ5j7uT5Tp6RJ6euUVKU/gzmuvAQ0aSOjIGwMGOD/u7lfcmfk8gCkAlgLIA7CAmTcR0fNE\ndJPRbCmAI0S0GcAyAP/BzEfCZbQSfbSzNDIUFIiX2LCh73ZZWZJWePx4YMcvKZH0QJfZHgEAN98s\nYR47+1HmzgUSEiS/3Sqm1x2MHaa4u3vuAPDoo54HNR0/LqGj8eOl/8EbAwdK2GfTpsDtihSWYu7M\nnM3MnZi5IzNPN9Y9zcyLjffMzI8ycyozd2Pm+eE0WlGqC67ZHr7IypLY+xdfBHb81auldku/fpXX\njxoly48+Cux43igrE3EfNkzyzq0SSq77nj0SLmzWrOq2rCwZ1DRzZuV4/jvvAGfOSFaNL2Ih7q4j\nVJUq+BugFMvEUjVps+CaFXHv0wdo3DjwuLvZmXr11ZXXp6RIaQC7QjMrVgB79/oOdXji4oslNTNY\nz90MHbpTo4bE3nNzKwY1MUtIpndvwN/888nJ8lJxV2IGK9UcY5U335Twxn33SYea0ykqEi/SirjX\nrCk1x5csCSyzZOVKqcboqRTuqFFSJfLoUevH88acOXLtR7qPkPFDjRpAu3bBxdz37PF97X7xC+nP\nmDFDPq9cCWze7Dn90RNOj7uruCuVsDpAKdaYOxe45x7gssukhGvPnsD330fbKt/4ihl7IitLapZv\n3myt/YULMrzePSRjMmqUtPnkE2vH88bZsxLbHj26apaVFZKSgvfcfaXnug9qevVVefqx2icwcKDU\n47F6vSONirtSiXgcoPThh8Cdd4qntXatDKL5+WfpRHz6ac8ZE07AFPfkZGvthw2TpdXQzKZNkoXj\nTdx79QLatAk9NJOdLamMwXa6BzNK9cwZefLxd2N84AEpWPa730kFyDvvtH4DGjhQlk4Nzai4K5WI\ntwFK//qXFKjq0wf4+GP5xx04UIpCTZgA/Nd/yUCVLVuibWlVAvXc27WTEItVcTdjze6ZMiY1akgY\nZckSEctgmTNH0isHDQpu/8REyek/d876PubNwN/fbatWwLhxwPz5cpO/917r50hOluOruCsxQTwN\nUPrsM+CWW2RKtOzsyumEjRsDb70l3tru3RKm+ctf/MdPjx+XlMMPP5RH+XDOUllQIOl4TaqM9fZO\nVpbEga30KaxcCVx6qe8ng1GjQqvxfvSohHXGjZN+gWBISpLrvM996KQPArkxTp0qy/79pRM5EAYO\nlH6JKM1W6hMVd6US8TJAafly8Tq7dJHBPY0be253yy1Sm3vQIJm/MytLROToURG/2bOBRx4Bhg6V\nEEXTppJZMmaMZJS0aiU54S+9JPHrn3+27ztYzZRxJStLPFwr0+WtWiVeu68aLwMGyLULNjTzwQdi\nT6BZMq4Ekw5p1XMHZFDTrFmSFhkoTo67B3kvVeKZCRNiT8xd+fZbYMQI8Ug/+8xznrMrrVtL+Gb2\nbBnckpRUuZ55gwZykxgyRDy71FQR9bVrRdBXraoQv9q1gcxMuQEMHQpcd13w32P37sDFvV8/edJa\nsgS4/nrv7fbskddjj/k+Xu3aci3NGu8JCYHZM2eOXDt/qYW+CEbcCwrE1jbuJQ69cP/9gdsFVI67\nd+0a3DHChYq7ElesWSPea+vWMqDHan06Iom3Dhokuc6tW8s/a2qqxLJreHjGveKKihjtwYNyU1m1\nSgT/lVfEm9+xQ4p9BUNBAXDNNYHtU7euzBrkL+5u5rd760x1ZdQo4N135XsFYk9BgeS3T59urQKk\nN9q1qzieVfbsEWEPNhRkleRkse+rr6Rz1kmouCtxww8/iKfcrJlkxLRuHfgxUlKAl18OfL9LLhER\nNEd2rlolwpmXF5y4FxfLy2qmjCtZWRLn3rFD6qZ7YtUqeSJxnbPU1/HMGu+BiPu7xnxs48db38cT\n9erJYKZAPfdIJAEQifduji8I5SZmNxpzV+KCY8ckDFKvngh7u3b+9wknnTrJMj8/uP0DzZRxxUyJ\n9FVIbOVKyRKy4tk2aiQhqUBqvDPLUP5+/YK7QbkTaDqkvwFMdjJwoKRd5uVF5nxWUXFX4oKcHJl0\n4s03pdBWtGnRQjoioyHul10mTwveQjPFxfKUYyUkYxJojff160XsQulIdSWQSTsuXJBSB5FK33Vq\nnRkVdyUu2LZNlt26RdcOEyLx3oMV9927ZRmMuBNJKMUcrOXOd9+JZ+0tv90TN94YWI33OXNkcNCt\nt1o/hy/MUapWnhwOHJBiaJHy3Dt0ANq2VXFXlLCQny957K1aRduSClJSKm46gVJQIJ2jgVRQdCUr\nS3LdzY5TV1aulEwSK3OYmrRqJWEcK+J+4YKUzR0xwn+mklUSEyXf3kqdG2+TdIQLM+7utHx3FXcl\nLsjPFzF1UodWSooITTCVKM0OwWC/z7XXiufsKe6+cqUM7GrUKLBjjhplrcb7smXiPdsVkgECS4f0\nNklHOBk4UMKCThrprOKuxAXbtlmfui1SpKSIJ7cziDnJCgpC64hs2FAyW9zj7qWlUjAtkJCMiWuN\ndzOuvWqVeOn//d+SKz5ihNTr+fWvAAAgAElEQVRnadxY3ttFIDMyBTKAyS6cGHdXcVfCRlkZcPfd\nMvJz2TKJg4aD0lKJUTtN3EPJmAlmdKo7WVnScbp/f8W6deukTkwgnakmZo33xx+XKfkSE+U448fL\n1H/vvSce+xVXyIjPunVDs9+VQGZkKiiQcJC/2avspGNHyat3krhrnrsSNgoKZJZ5QOq2NGsmHXM3\n3yxpi8GUf/XErl3iSZpi6hTMm02g4n7mjDzi2yHujz8uoZm77pJ1/oqF+ePFF6WkQLt2Yl9ioizb\ntQuvmLZoITcLq+IeqXi7iRl3//xz5+S7q7grYcOMP376qXSGLVokj/RvvSX56FlZ8qh/ww2hdbyZ\nnZZO89ybNBFRCrRTNZQ0SFfS0qQw2JIlFeK+apWkil56aXDHvOkmeUUaIuvpkHv2eB+8FU4GDpR5\nA378UUZGHz0q86yaS9f3d9xREcoJFyruStjYulWWvXrJH/vo0RJCWb5chH7RImDhQhlIs3gxMHx4\ncOcxPWOniTsgNgXqudsl7mZK5MKFEhJLSBDP3RzkFGtYmbTDnD0s2PLCoWCKdXq69za1asnsT4GW\nlQgGFXclbGzZIlUUW7SoWFerlox2HDJE6q+sWSOlVr/4IjRxb9JE/mmcRkpK4JNWBzpJhy+GDZOB\nXTk58jscOhRcvN0JJCb6nxWquBg4eTLyYRlABo/NmiXeefPm8jTavHnl9w0aRC5ko+KuhI0tW4DO\nnb3/MdeoIZ1vXbpYH/noifx8ibc7Ic7pTqdOwNtvS855gwbW9ikokKeZYEMnrgwZItd5yZKKkbvB\nxtujTWIi8NNPMjCrTh3PbaKRBmlCFHx1yXCg2TJK2Ni6VcTdH127ypRvweLENEgT067t263vU1Ag\nIx4DLa/riWbNZLDS0qUSkmnaVG6msYgp2Hv3em8T6QFMTkbFvZoxd6487teoIcu5c8NznuJi8bIu\nv9x/27Q0mdj5+PHAz3P2rPyzO13cA4m7B1PH3RdZWcDq1eK9X3215/LFsYCVdMhoeu5OI0Z/ZiUY\n5s4FJk2SfwCz42nSpPAIvNmZasVzT0uTZTDe+44d8l2clgZpctllsgxE3O1O5cvKqpimLlbj7YC1\nUaqhlm2IJ1TcqxFPPikpia6UlMh6uzHTIK147uYMNsGIu1PTIE0aNZK6LFbFvbRUBh3ZKe69elV0\nNseyuLdtK0tf6ZB79oRWtiGeUHGvRnjzeAKpk22VLVukU9BKvnFiogyACaZT1clpkCaBVIcsLJSR\nvXZkypgkJMgkJnXqyBSAsUqdOjIBiz/PXUMygop7NcLbH304/hm2bhVhr1XLf9saNWRYezCee36+\nPIJ7mwDbCQRSHdKuHHd3/vAHibnbWRIgGvibtCOSk3Q4HRX3asT06VWH/NevL+vtZssWayEZk7S0\n4Dx3J2fKmKSkSH75iRP+24ZL3Nu2rZjMOZbxJe4//yy1bdRzF1TcqxETJgCzZ4twEMly9mxZbyfn\nz0vqn5XOVJO0NBHAoqLAzmWW+nUygWTMmJN0RHuaQKdiirunuumFhbJUz11Qca9mTJggAlJWJku7\nhR2Q4547F5i4B9OpeuqUeGpOF/dAqkMWFEhc2dsgnepOUpKkv3pyAjQNsjIq7ortmGmQgYZlgMBC\nM6ZYOjUN0sTsVLYq7nZ2psYbvtIhwxXSilUsiTsRZRHRViLaTkTTPGyfSERFRLTeeN1jv6mKPyI1\nQMkfgaRBmrRuLfVhghF3p3vu9epJmMVKp2o0ytXGEr4m7dizR8KNZspkdcdvbRkiSgAwC8BQAIUA\ncohoMTNvdmv6HjNPCYONigXMAUpmHrs5QAkIT+jFF1u2SJGqQAp5EYn3HkhYxhR3c6CQk7FSHbKs\nTEbbjhkTGZtiEV+jVM2QVu3akbXJqVjx3HsD2M7MO5n5HID5AEaG1ywlUCI5QMkfVmvKuGNmzFid\nZDg/X2a/sVqQK5pYEfeffpK+CvXcvdO0qfzensRd0yArY0Xc2wBwLdVTaKxz5xYi+oGIPiAij339\nRDSJiHKJKLco0LQIxSeRHKDkD7MaZKB07Sr1ZQ4csNY+FtIgTTp1qpiowRtmpowKlHfMSTu8ee7a\nmVqBXR2qHwNIZubuAD4D8JanRsw8m5kzmTmzZcuWNp1aAQIboPTll0B2tgx1t5ujRyWTIZB4u0mg\nnaqxkAZpYiUdUjsEreFpRqayMvXc3bEi7vsAuHribY115TDzEWb+2fj4BoBe9pinWMXqAKWzZ2Ue\n0xEjpF74Aw8A33xjPRTij0AKhrljpkNaEfdjx4DDh52fKWOi4m4fnmZkOnRIQ1ruWBH3HAApRNSe\niGoDGAtgsWsDImrt8vEmAHn2mahYweoApeXLK2LxgwbJLD19+0qn5O9+V5HpEizBZMqYtGwppQSs\ndKrGSqaMSYcOksXkT9ybNw/vRNPxQGKiPB2eOVOxTnPcq+JX3Jn5PIApAJZCRHsBM28ioueJyJwq\n9yEi2kREGwA8BGBiuAxWvGNlgFJ2ttQXefJJ4L33gIMHgX/8Q3KxX3hBJnLo1QuYMUNmDwqUrVul\nnow560+gWC1DEGviXru2pKf6E3f1PP3jKdddJ+moiqWYOzNnM3MnZu7IzNONdU8z82Lj/W+ZuSsz\npzPztcwcov+nhIvsbPHY69WTzxddBNx5J/Dvf8vw7ZkzxfN/7DFgWpURDf7ZskUEt2aQEziaszKV\nlflul58vdkZjlvtg8ZcxY/ckHfGKJ3FXz70qOkI1RrBjgFJ+vtR8uf56z9tbtwYeeQTIzZW4/JIl\ngZ9j69bgQjImaWnyxOAvy2fbNhHCWBqmb1aH9NS/YU6eouLuH0+57nv2SGVQJ1cHjTQq7jGAXTMo\nZWfL0pu4uzJ4sNwIAkmlLC0NvGCYO1YzZmIpU8YkJQU4eVI6/9w5ckT6QrT0gH/atJGnNnfPXb32\nyqi4xwB2DVD65BOJqVuJhw8aJMsvvrB+/J07pSJkKOKemipLX52qzLEr7oDn0IxmylinVi3J9HJN\nh9SnnqqouMcAdgxQOnVKMmWseO2AeNAXXyw58VYJpmCYO02aSG0QX557UZFMwB0raZAmvqpDqrgH\nhns6pOa4V0XFPQawYwalL7+UPGCr4k4k3vsXX1jPgQ8lDdKVrl19i3usZcqYJCVJR7OnAmIq7oHh\nOkr15EkZ96BhmcqouMcAdsyglJ0t+dOBTJA8aJCUArCa+751K3DJJeJ9h0JaGpCXB1y44Hl7rIp7\nzZqS7+7Jc9+9W36fpk0jblZMkpgoRdbMkamA3hjdUXGPAUKdQYlZxH3o0MAq5g0eLEurcfdga8q4\nk5YmU6bt2OF5e36+CGUsdj56S4c0Y8ZEkbcpFklMlCfRgwc1DdIbKu4xQigzKG3cKF7OiBGBnbND\nBxFQq3F3u8TdXxmCbdukU9jK5NtOo1MnyShyD3XpJB2B4ZoOqSEtz6i4VwPMFMjhwwPfd9AgYNky\n7yESk8OHpWhYqPF2wH/GTCxmypikpEim0/79lddrtkdguA5k2rNHbvStWkXXJqeh4l4NyM4GevSQ\n9LFAGTxYyvCuW+e7nRmXt8Nzb9BAPHNPnruZBhlrmTIm5k3JtVPV7BBUcbeO64xMBQUy01UNVbNK\n6OWIc44fB1atsp4l446Z7+4vNGNHGqQr3mZl2r9fPN9Y9tyBynF3DSsETpMmUjrD9Nz12lVFxT3O\n+ewzCakEK+6tWkmYxF+n6pYtUgrArn+ytDS5YZw7V3l9rGbKmLRrJ9fJVdx1ko7gMNMhdXSqZ1Tc\n45xPPgGaNQOuvDL4YwweDKxYIRks3ti6VUIlCQnBn8eVrl1ltKt7Tnisi3uNGlJe2ZPnrh2qgZGY\nKBlV+/frjdETKu5xTFkZ8OmnwLBhoYnu4MFSO/v777232bLFvpAMUFFjxj00s22beL7tPE7kGBuY\nBcRMCgrkO118cfRsikUSEysqiKrnXhUV9zhm7VopUhVsSMZkwADxOL2FZs6dk7oydnSmmlx+uZzT\nvVM1P1/K/Nr1hBANUlLE4zQzkMywgnYIBkZSUkVKqXruVdE/pzgmO1sGxQwbFtpxmjSRCTy8ibsp\nVHaKe926IoLunnssp0GapKTIDXGvMe28pkEGh6u3rtevKirucUx2NtC7t0xfFyqDB0tY5tSpqtvs\nqinjjvusTBcuyACgWE2DNHEvIKaTdASHq7jHcpguXKi4OwA7JuJwp6gIWL069JCMyeDB0sG5YkXV\nbXanQZp07Spibs6VuXeveLzx4LkDIu5nz8oQehX3wDGv2SWXyJOeUhkV9yhj10Qc7ixZIscLtOSA\nN66+WurSeArNbNkiA6QaNbLnXCZpafIdzCeDWM+UMWndWgZqbdtWUfRKM2UCp3Vr6XvRzlTPqLhH\nGbsm4nAnO1s8mp49QzuOSf36IvCeBjPZVVPGHfdZmcwMk1gPyxBVpEPqAKbgMYvHxdI8upFExT3K\n2DERhzvnzwNLl0otGTszMAYPBtavlynhTJglLBMOcb/sMqkZYop7fr54vK1b23+uSGNWh1RxD41/\n/hP4wx+ibYUzUXGPMnZMxOHO999LrRK74u0mgwaJmC9bVrHu0CEpcWB3vB0QYe/cuSJjJj9fBD8e\nyuJ26gTs2iWZRgkJMi+oEjjdu2tnqjdU3KOMHRNxuPPJJyIYQ4eGZps7V1whE0q4xt3NztRweO5A\n5VmZ4iEN0iQlRZ6wvvpKhL1mzWhbpMQbKu5RJtSJODyRnS0zLoU6I5I7tWrJgCbXuHu40iBN0tIk\ndHHsmAyUivV4u4l5k1q9WjtTlfCg4h5mrKQ5hjIRhzv79gEbNtgfkjEZNEg6NgsL5fOWLUC9euF7\nNDY7VT/5RPLc48lzB+Q313i7Eg5U3MNIuNIcffHpp7IMl7i7T723dWtFqYBwYM7KtHChLONF3Fu2\nlJK1gIq7Eh5U3MNIuNIcvcEMfPiheNGmKNpNt25AixYVoRm7C4a50769PBksWSKf4yUsQ1TxXVTc\nlXCg4h5GwpHm6I0TJ4BbbhERvOuu8GWU1KgBXHuteO5nz0oYKVydqYB0DHfpIjfFxo3lxhIvmE8h\nKu5KOFBxDyPhSHP0xObNUkNm8WJgxgzg2WftPb47gwdLbD87W2LG4RR3oCLunpISH2mQJiruSjhR\ncQ8j4UhzdOf990XYjx0Tb3rq1PALoBl3/9//lWU4wzJAhbjHS0jG5LbbpA+mQ4doW6LEIyruYSQc\naY4m588D//EfIhDduknt9gEDQj+uFTp2lLi+2akabtE1+w/ipTPVpGtX4PXXNcddCQ8q7mHGzjRH\nk0OHgOuuA156Cbj/fmD58siOcCSq8N7btZOSAOEkM1MGT/XtG97zKEo8YUnciSiLiLYS0XYimuaj\n3S1ExESUaZ+Jiivffy8TZ3z7LfDWW8CsWVKtMdKY4h7ueDsg08+dOGH/iFtFiWf8ijsRJQCYBWA4\ngFQA44go1UO7RgAeBuBjpk0lFObNA/r3l8f4b74BfvnL6NkyaJAswx1vN4mnjlRFiQRWPPfeALYz\n805mPgdgPoCRHtr9F4D/AXDWRvsUF55+WuK0a9bYV8o3WC69FHjjDeChh6Jrh6IonrEi7m0A7HX5\nXGisK4eIMgC0Y+ZPfB2IiCYRUS4R5RYVFQVsbHXm7FmprXLDDUCzZtG2RvjVr+Kvk1NR4oWQO1SJ\nqAaAGQAe89eWmWczcyYzZ7a0Y2LPasT27ZHJKVcUJT6wIu77ALiWhWprrDNpBCANwFdEtBvAlQAW\na6eqveTlybJLl+jaoShKbGBF3HMApBBReyKqDWAsgMXmRmYuZuYWzJzMzMkAvgNwEzPnhsXiaoop\n7pHqwFQUJbbxK+7MfB7AFABLAeQBWMDMm4joeSK6KdwGKkJengyCch/xqiiK4glLY+OYORtAttu6\np720HRi6WYo7W7ZoSEZRFOvoCNUQsDIRhx2UlUnddBV3RVGsolUtgsSciMOs125OxAHYU2LAlYIC\n4MwZzZRRFMU66rkHSSQn4jDnKVXPXVEUq6i4B0kkJ+LQNEhFUQJFxT1IIjURByDi3qJFfM1CpChK\neFFxD5JITMRhsmWLxtsVRQkMFfcgCedEHO7k5WlIRlGUwNBsmRCYMCE8Yu5KURFw5IiKu6IogaGe\nu8MxO1M1LKMoSiCouDscTYNUFCUYVNy9EKnRp/7Iy5OO2nBk4SiKEr9ozN0DkRx96o+8PKkEWUNv\nw4qiBIBKhgciOfrUH5oGqShKMKi4eyCSo099cfq0PDVovF1RlEBRcfdAJEef+mLrVlmquCuKEigq\n7h6I5OhTX5iZMhqWURQlUFTcPRDJ0ae+yMuTjtSUlMieV1GU2EezZbwQidGn/sjLAzp2BOrUia4d\niqLEHuq5O5i8PA3JKIoSHCruDuX8eSA/XztTFUUJDhV3h7JzJ1BaquKuKEpwqLg7FJ19SVGUUFBx\ndyiaBqkoSihUS3F3SlEwX+TlAa1bA40bR9sSRVFikWqXCumkomC+0NmXFEUJhWrnuTupKJg3mDUN\nUlGU0Kh24u6UomC+OHAAOHlSPXdFUYKn2om7U4qC+UIzZRRFCZVqJ+7RKAq2cyewaZP19jpvqqIo\noVLtxD0aRcEmTQIGDQLOnrXWfssWoFEj4NJLw2eToijxTbUTd0CEfPduoKxMluEU9rIyYPVq4NAh\nYN48a/uYmTJE4bNLUZT4plqKeyTZtk06R2vUAGbMkEwYf2gapKIooWJJ3Ikoi4i2EtF2IprmYftk\nIvqRiNYT0UoiSrXf1NgkJ0eWU6cCGzcCn3/uu31xsWTLaLxdUZRQ8CvuRJQAYBaA4QBSAYzzIN7v\nMnM3Zu4B4A8AZthuqUWcNvo0J0c6bJ9/HrjkEmDmTN/tzbID6rkrihIKVjz33gC2M/NOZj4HYD6A\nka4NmPmEy8cGACwEH+zHHH1aUCDhD3P0aTQFPjcXyMgQgZ8yBfj0U2DzZu/tNQ1SURQ7sCLubQDs\ndflcaKyrBBE9QEQ7IJ77Q54ORESTiCiXiHKLioqCsdcnTht9WloKrFsHXHGFfJ48GahbF/jTn7zv\nk5cH1KoFdOgQGRsVRYlPbOtQZeZZzNwRwH8CeMpLm9nMnMnMmS1btrTr1OU4bfTp5s2S/piZKZ9b\ntAB++Uvg7bcBb/e2LVtkztSa1a7qj6IodmJF3PcBaOfyua2xzhvzAYwKxahgcdroU7Mz1fTcAeCR\nR4CffwZee83zPpopoyiKHVgR9xwAKUTUnohqAxgLYLFrAyJKcfk4AkC+fSZaJxqjT32RkyMley+7\nrGJdly7A9dcDf/1r1UFNP/8M7NihmTKKooSOX3Fn5vMApgBYCiAPwAJm3kREzxPRTUazKUS0iYjW\nA3gUwJ1hs9gH0Rh96ovcXAnJuA9GmjrV86Cm7dtl0JN67oqihAqxlVE1YSAzM5Nzc3Ojcu5IcPas\nlBD4zW+AF1+svI0ZSE+X9xs2VIj/Bx8At94KrFkjGTaKoijuENEaZs70105HqIaJH34Azp+v6Ex1\nhQh49FHgxx+BL76oWG+mQV5+eWRsVBQlflFxDxOeOlNdGTdOBjXNcBnutWWLdP42aBB++xRFiW9U\n3MNETg5w8cVAu3aet9epU3VQk2bKKIpiFyruYcJbZ6orroOaysrEc1dxVxTFDlTcw8CpU+KFewvJ\nmJiDmt55B1i7FjhzRtMgFUWxBxX3MLBunXji/sQdkEFNZ8/KElDPXVEUe6i24n72rBQWCwdmZ6qn\nTBl3zEFNq1ZVfFYURQmVainuZWXA6NFSEvj664Fly6xNomGVnBzpSL3kEmvtp06VZfPmQBhK7iiK\nUg2pluI+c6Zkqdx2mwwYGjRIQijvvSe56aFidqZaZfBgoGdPeSmKothBtRP3nBxg2jTx3OfPl9DM\n66/LVHhjx0pFxldeAU6fDu74x45JGQEr8XYTIuCzz8QeRVEUO6hW4n7ihAj4pZcCb7wholq3rkzo\nkZcHLFwo2x5+WMIqTz0lNWACYc0aWQYi7oCEZJo3D2wfRVEUb1QbcWcG7r1XPPV33wWaNq28vUYN\nYNQo6dhctQoYOBB44QWgd2/g3Dnr5zE7U3v1ss10RVGUgKk24v73v0vY47nngL59fbe9+mrgn/8E\nFi2Sm8F771k/T06OlPh1v3koiqJEkmoh7nl5wIMPSsfptGnW97vxRklNnDnTejZNoJ2piqIo4SDu\nxf3sWYmz168vI0ETEqzvSySDi9atA1as8N/+4EFg797A4+2Koih2E/fi/pvfSPndt96SztJA+cUv\npKNz5kz/bc3y9CruiqJEm7gW94ULgVmzgMcek8FKwVCvnnTEfvSRTIHni5wc6ZjVfHVFUaJN3Ir7\nnj3Ar34l8e8XXgjtWA88ANSsKfnvvsjNlRh9w4ahnU9RFCVU4lLcz58Hxo+X5bx5QO3aoR3v0kuB\n228H3nwTKC723IZZPHftTFUUxQnElbgfOwa8+ipw1VWSq/7aa5KWaAdTp0op3zfe8Lx9714Z8KTx\ndkVRnEDMi/v581In5vbbgdatgfvvB37+GZg9W7x3u8jIAPr3B/7yF8/1Z7QzVVEUJxGz4r55M/D4\n4zLn6PXXy0TT994rk15s2AD8+tf2n3PqVBnUtGhR1W05ORKX797d/vMqiqIESs1oGxAo//oX8Pzz\nFWI6YgQwcaIIfKixdX/ceCPQoYOkRY4ZU3lbbq4Ie9264bVBURTFCjHnuRcXS62XmTOBffvEix41\nKvzCDsgAqIceAr75Bli9umI9s4i7hmQURXEKMSXuc+cCTzwhg5L+9Ccpkxtp7r4buOiiyoOatm8H\njh/XTBlFUZxDzIj73LlSmnfPHvGUCwrk89y5kbWjUSPgnnuA99+XDBlAO1MVRXEeMSPuTz4JlJRU\nXldSIusjzYMPyg1m1iz5nJMjsfbU1MjboiiK4omYEfc9ewJbH06Sk2Ump9mzZcam3FwpOVCrVuRt\nURRF8UTMiHtiYmDrw80jj8igqTfflPRLDckoiuIkYkbcp0+Xsr2u1K8v66PB1VeLoD/9tHjv2pmq\nKIqTiBlxnzBBwiBJSVJnPSlJPk+YEB17iGRQ0/Hj8lk9d0VRnASxhSmGiCgLwJ8BJAB4g5n/2237\nowDuAXAeQBGAu5m5wNcxMzMzOddMM4lRSktlUFNxsYh8jZi5VSqKEqsQ0Rpm9hsr8DtClYgSAMwC\nMBRAIYAcIlrMzJtdmq0DkMnMJUR0H4A/ALg9ONNjh1q1gNdfBwoLVdgVRXEWVsoP9AawnZl3AgAR\nzQcwEkC5uDPzMpf23wG4w04jnUywk4AoiqKEEyv+ZhsAe10+FxrrvPErAJ+GYpSiKIoSGrYWDiOi\nOwBkAhjgZfskAJMAIDFaOYyKoijVACue+z4A7Vw+tzXWVYKIhgB4EsBNzPyzpwMx82xmzmTmzJYt\nWwZjr6IoimIBK+KeAyCFiNoTUW0AYwEsdm1ARD0BvA4R9kP2m6koiqIEgl9xZ+bzAKYAWAogD8AC\nZt5ERM8T0U1Gsz8CaAjgfSJaT0SLvRxOURRFiQCWYu7MnA0g223d0y7vh9hsl6IoihICmp2tKIoS\nh6i4K4qixCGWyg+E5cRERQA8lShoAeBwhM0JFbU5MsSazbFmL6A2R4pQbE5iZr/phlETd28QUa6V\nuglOQm2ODLFmc6zZC6jNkSISNmtYRlEUJQ5RcVcURYlDnCjus6NtQBCozZEh1myONXsBtTlShN1m\nx8XcFUVRlNBxoueuKIqihIiKu6IoShziKHEnoiwi2kpE24loWrTtsQIR7SaiH42aOo6cN5CI3iSi\nQ0S00WVdMyL6jIjyjWXTaNroihd7nyWifcZ1Xk9EjpomhYjaEdEyItpMRJuI6GFjvZOvszebHXmt\niaguEa0mog2Gvc8Z69sT0feGbrxnFDh0BD5s/gcR7XK5xj1sPzkzO+IFmZ91B4AOAGoD2AAgNdp2\nWbB7N4AW0bbDj439AWQA2Oiy7g8AphnvpwH4n2jb6cfeZwH8Jtq2+bC5NYAM430jANsApDr8Onuz\n2ZHXGgABaGi8rwXgewBXAlgAYKyx/jUA90XbVgs2/wPAmHCe20mee/l0fsx8DoA5nZ8SIsz8NYCj\nbqtHAnjLeP8WgFERNcoHXux1NMx8gJnXGu9PQiqotoGzr7M3mx0JC6eMj7WMFwMYBOADY73TrrE3\nm8OOk8Q90On8nAID+DcRrTFmmooVLmHmA8b7nwBcEk1jLDKFiH4wwjaOCW+4Q0TJAHpCvLSYuM5u\nNgMOvdZElEBE6wEcAvAZ5Gn/OEtpcsCBuuFuMzOb13i6cY1nElEdu8/rJHGPVfoxcwaA4QAeIKL+\n0TYoUFieGZ2eE/sqgI4AegA4AODl6JrjGSJqCOBDAI8w8wnXbU69zh5sduy1ZuYLzNwDMiNcbwCd\no2ySX9xtJqI0AL+F2H4FgGYA/tPu8zpJ3C1N5+c0mHmfsTwEYCHkDy4WOEhErQHAWDp6Bi1mPmj8\nk5QB+BsceJ2JqBZEJOcy8z+N1Y6+zp5sjoVrzczHASwDcBWAJkRkzk3hWN1wsTnLCIkxy5Skf0cY\nrrGTxN3vdH5Og4gaEFEj8z2A6wBs9L2XY1gM4E7j/Z0APoqiLX4xBdLgZjjsOhMRAfg/AHnMPMNl\nk2OvszebnXqtiaglETUx3tcDMBTST7AMwBijmdOusSebt7jc8AnSR2D7NXbUCFUj5epPkMyZN5l5\nepRN8gkRdYB464DMavWuE20monkABkLKjB4E8AyARZAsg0RI6eXbmNkRnZhe7B0ICRMwJEPpXpdY\ndtQhon4AVgD4EUCZsfoJSAzbqdfZm83j4MBrTUTdIR2mCRDHdAEzP2/8H86HhDfWAbjD8Iijjg+b\nvwTQEpJNsx7AZJeOV3vO7SRxVxRFUezBSWEZRVEUxSZU3BVFUeIQFXdFUZQ4RMVdURQlDlFxVxRF\niUNU3BVFUeIQFXdFUQLkSXQAAAAJSURBVJQ45P8BNBrsrA158KIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAEICAYAAAB/Dx7IAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8VNX5+PHPA4QlgOyK7LhUCHtI\nERqRVQtuFEtRDG5fNepXq9b6q9SVYmnVUkWoX1vqLhGqUtwQl6+iyFeLBEQQEXEBDCCEKMiqJHl+\nf5yZEEKWO5OZuXeS5/16zWtm7py595k7yXPPnHPuuaKqGGOMSR51/A7AGGNMZCxxG2NMkrHEbYwx\nScYStzHGJBlL3MYYk2QscRtjTJKxxF0LiUhdEdkjIp1iWdZPInKCiMR8bKuIjBSRDaWerxORwV7K\nRrGth0XklmjfX8l6/ygij8d6vcY/9fwOwFRNRPaUepoK/AAUhZ5fqao5kaxPVYuAJrEuWxuo6kmx\nWI+IXA5MVNWhpdZ9eSzWbWo+S9xJQFVLEmeoRne5qv5vReVFpJ6qFiYiNmNM4llTSQ0Q+in8LxGZ\nIyK7gYkiMkhE/iMiO0Vkq4jMEJGUUPl6IqIi0iX0fHbo9YUisltE3heRrpGWDb0+WkQ+E5FdIjJT\nRP5PRC6pIG4vMV4pIp+LyHciMqPUe+uKyP0iUiAiXwKjKtk/t4rI3DLLHhSR+0KPLxeRtaHP80Wo\nNlzRuvJEZGjocaqIPBWKbQ3Qv0zZ20Tky9B614jIOaHlvYC/AYNDzVA7Su3byaXef1XosxeIyPMi\ncqyXfVMVERkbimeniLwlIieVeu0WEdkiIt+LyKelPutAEVkRWr5NRP7idXsmDlTVbkl0AzYAI8ss\n+yPwI3A27mDcCPgpcDLuV9VxwGfAtaHy9QAFuoSezwZ2ABlACvAvYHYUZY8GdgNjQq/dCBwELqng\ns3iJ8QWgGdAF+Db82YFrgTVAB6AVsNj9OZe7neOAPUDjUuveDmSEnp8dKiPAcGA/0Dv02khgQ6l1\n5QFDQ4+nAW8DLYDOwCdlyo4Hjg19JxeEYjgm9NrlwNtl4pwNTA49Pj0UY1+gIfA/wFte9k05n/+P\nwOOhx91DcQwPfUe3AOtCj3sAG4G2obJdgeNCj5cBE0KPmwIn+/2/UJtvVuOuOZao6kuqWqyq+1V1\nmaouVdVCVf0SmAUMqeT9z6lqrqoeBHJwCSPSsmcBK1X1hdBr9+OSfLk8xvhnVd2lqhtwSTK8rfHA\n/aqap6oFwN2VbOdL4GPcAQXgNOA7Vc0Nvf6Sqn6pzlvAm0C5HZBljAf+qKrfqepGXC269HafUdWt\noe/kadxBN8PDegGygIdVdaWqHgAmAUNEpEOpMhXtm8qcD7yoqm+FvqO7ccn/ZKAQd5DoEWpu+yq0\n78AdgE8UkVaqultVl3r8HCYOLHHXHF+XfiIi3URkgYh8IyLfA1OA1pW8/5tSj/dReYdkRWXblY5D\nVRVXQy2Xxxg9bQtXU6zM08CE0OMLQs/DcZwlIktF5FsR2Ymr7Va2r8KOrSwGEblERD4KNUnsBLp5\nXC+4z1eyPlX9HvgOaF+qTCTfWUXrLcZ9R+1VdR3wW9z3sD3U9NY2VPRSIA1YJyIfiMgZHj+HiQNL\n3DVH2aFw/8DVMk9Q1aOAO3BNAfG0Fdd0AYCICIcnmrKqE+NWoGOp51UNV3wGGCki7XE176dDMTYC\nngP+jGvGaA687jGObyqKQUSOAx4CrgZahdb7aan1VjV0cQuu+SW8vqa4JpnNHuKKZL11cN/ZZgBV\nna2qmbhmkrq4/YKqrlPV83HNYX8F5olIw2rGYqJkibvmagrsAvaKSHfgygRs82UgXUTOFpF6wPVA\nmzjF+Axwg4i0F5FWwM2VFVbVb4AlwOPAOlVdH3qpAVAfyAeKROQsYEQEMdwiIs3FjXO/ttRrTXDJ\nOR93DLsCV+MO2wZ0CHfGlmMOcJmI9BaRBrgE+q6qVvgLJoKYzxGRoaFt/z9cv8RSEekuIsNC29sf\nuhXjPsCFItI6VEPfFfpsxdWMxUTJEnfN9VvgYtw/5T9wnYhxparbgPOA+4AC4HjgQ9y481jH+BCu\nLXo1ruPsOQ/veRrX2VjSTKKqO4HfAPNxHXzjcAcgL+7E1fw3AAuBJ0utdxUwE/ggVOYkoHS78BvA\nemCbiJRu8gi//1Vck8X80Ps74dq9q0VV1+D2+UO4g8oo4JxQe3cD4F5cv8Q3uBr+raG3ngGsFTdq\naRpwnqr+WN14THTENUMaE3siUhf303ycqr7rdzzG1BRW4zYxJSKjQk0HDYDbcaMRPvA5LGNqFE+J\nW0SuF5GPQ4P2b4h3UCapnQJ8ifsZ/nNgrKpW1FRijIlClU0lItITmAsMwJ3k8Spwlap+Hv/wjDHG\nlOWlxt0dWKqq+9TNf/EOcG58wzLGGFMRL5NMfQxMDQ252o/rXc4tW0hEsoFsgMaNG/fv1q1b2SLG\nGGMqsHz58h2qWtnw2RKeRpWIyGXAfwN7cfND/KCqFbZ1Z2RkaG7uEbndGGNMBURkuap6mhLBU+ek\nqj6iqv1V9VTcabefVSdAY4wx0fM0H7eIHK2q20Nnh50LDIxvWMYYYyri9UIK80Jt3AeBa0Jnmxlj\njPGBp8Stql6muDTG+OTgwYPk5eVx4MABv0MxVWjYsCEdOnQgJaWiaWqqZpcuM6YGyMvLo2nTpnTp\n0gU3KaMJIlWloKCAvLw8unbtWvUbKhCYU95zcqBLF6hTx93nRHT5W2NqtwMHDtCqVStL2gEnIrRq\n1arav4wCUePOyYHsbNi3zz3fuNE9B8iq9nxoxtQOlrSTQyy+p0DUuG+99VDSDtu3zy03xhhzuEAk\n7k2bIltujAmOgoIC+vbtS9++fWnbti3t27cvef7jj96m7L700ktZt25dpWUefPBBcmLUhnrKKaew\ncuXKmKzLD4FoKunUyTWPlLfcGBN7OTnuF+2mTe7/bOrU6JslW7VqVZIEJ0+eTJMmTbjpppsOK1Ny\ndfI65dcVH3vssSq3c80110QXYA0UiBr31KmQmnr4stRUt9wYE1vhPqWNG0H1UJ9SrAcEfP7556Sl\npZGVlUWPHj3YunUr2dnZZGRk0KNHD6ZMmVJSNlwDLiwspHnz5kyaNIk+ffowaNAgtm/fDsBtt93G\n9OnTS8pPmjSJAQMGcNJJJ/Hee+8BsHfvXn75y1+SlpbGuHHjyMjIqLJmPXv2bHr16kXPnj255ZZb\nACgsLOTCCy8sWT5jxgwA7r//ftLS0ujduzcTJ06M7Q6LQCBq3OEjfaxqAMaYilXWpxTr/7lPP/2U\nJ598kowMNwXH3XffTcuWLSksLGTYsGGMGzeOtLS0w96za9cuhgwZwt13382NN97Io48+yqRJk45Y\nt6rywQcf8OKLLzJlyhReffVVZs6cSdu2bZk3bx4fffQR6enplcaXl5fHbbfdRm5uLs2aNWPkyJG8\n/PLLtGnThh07drB69WoAdu505xzee++9bNy4kfr165cs80Mgatzg/mA2bIDiYndvSduY+Ehkn9Lx\nxx9fkrQB5syZQ3p6Ounp6axdu5ZPPvnkiPc0atSI0aNHA9C/f382bNhQ7rrPPffcI8osWbKE888/\nH4A+ffrQo0ePSuNbunQpw4cPp3Xr1qSkpHDBBRewePFiTjjhBNatW8d1113Ha6+9RrNmzQDo0aMH\nEydOJCcnp1on0FRXYBK3MSYxKuo7ikefUuPGjUser1+/ngceeIC33nqLVatWMWrUqHLHM9evX7/k\ncd26dSksLCx33Q0aNKiyTLRatWrFqlWrGDx4MA8++CBXXnklAK+99hpXXXUVy5YtY8CAARQVFcV0\nu15Z4jamlvGrT+n777+nadOmHHXUUWzdupXXXnst5tvIzMzkmWeeAWD16tXl1uhLO/nkk1m0aBEF\nBQUUFhYyd+5chgwZQn5+PqrKr371K6ZMmcKKFSsoKioiLy+P4cOHc++997Jjxw72lW1zSpBAtHEb\nYxLHrz6l9PR00tLS6NatG507dyYzMzPm2/j1r3/NRRddRFpaWskt3MxRng4dOnDXXXcxdOhQVJWz\nzz6bM888kxUrVnDZZZehqogI99xzD4WFhVxwwQXs3r2b4uJibrrpJpo2bRrzz+CFpwspRMoupGBM\nYq1du5bu3bv7HYbvCgsLKSwspGHDhqxfv57TTz+d9evXU69esOqo5X1fkVxIIVifxhhjqmHPnj2M\nGDGCwsJCVJV//OMfgUvasVDzPpExptZq3rw5y5cv9zuMuLPOSWOMSTKWuI0xJsl4Stwi8hsRWSMi\nH4vIHBFpGO/AjDHGlK/KxC0i7YHrgAxV7QnUBc6Pd2DGGGPK57WppB7QSETqAanAlviFZIxJNsOG\nDTvihJrp06dz9dVXV/q+Jk2aALBlyxbGjRtXbpmhQ4dS1fDi6dOnH3YyzBlnnBGTuUQmT57MtGnT\nqr2eWKsycavqZmAasAnYCuxS1dfLlhORbBHJFZHc/Pz82EdqjAmsCRMmMHfu3MOWzZ07lwkTJnh6\nf7t27Xjuueei3n7ZxP3KK6/QvHnzqNcXdF6aSloAY4CuQDugsYgcMZ+hqs5S1QxVzWjTpk3sIzXG\nBNa4ceNYsGBByYUTNmzYwJYtWxg8eHDJ2Or09HR69erFCy+8cMT7N2zYQM+ePQHYv38/559/Pt27\nd2fs2LHs37+/pNzVV19dMi3snXfeCcCMGTPYsmULw4YNY9iwYQB06dKFHTt2AHDffffRs2dPevbs\nWTIt7IYNG+jevTtXXHEFPXr04PTTTz9sO+VZuXIlAwcOpHfv3owdO5bvvvuuZPvhqV7DE1y98847\nJReT6NevH7t3745635bHyzjukcBXqpoPICL/Bn4GzI5pJMaYmLjhBoj1xV369oVQzitXy5YtGTBg\nAAsXLmTMmDHMnTuX8ePHIyI0bNiQ+fPnc9RRR7Fjxw4GDhzIOeecU+G1Fx966CFSU1NZu3Ytq1at\nOmxq1qlTp9KyZUuKiooYMWIEq1at4rrrruO+++5j0aJFtG7d+rB1LV++nMcee4ylS5eiqpx88skM\nGTKEFi1asH79eubMmcM///lPxo8fz7x58yqdY/uiiy5i5syZDBkyhDvuuIM//OEPTJ8+nbvvvpuv\nvvqKBg0alDTPTJs2jQcffJDMzEz27NlDw4axHc/hpY17EzBQRFLF7ekRwNqYRmGMSXqlm0tKN5Oo\nKrfccgu9e/dm5MiRbN68mW3btlW4nsWLF5ck0N69e9O7d++S15555hnS09Pp168fa9asqXISqSVL\nljB27FgaN25MkyZNOPfcc3n33XcB6Nq1K3379gUqnz4W3BzhO3fuZMiQIQBcfPHFLF68uCTGrKws\nZs+eXXKWZmZmJjfeeCMzZsxg586dMT97s8q1qepSEXkOWAEUAh8Cs2IahTEmZiqrGcfTmDFj+M1v\nfsOKFSvYt28f/fv3ByAnJ4f8/HyWL19OSkoKXbp0KXc616p89dVXTJs2jWXLltGiRQsuueSSqNYT\nFp4WFtzUsFU1lVRkwYIFLF68mJdeeompU6eyevVqJk2axJlnnskrr7xCZmYmr732Gt26dYs61rI8\njSpR1TtVtZuq9lTVC1X1h5hFYIypEZo0acKwYcP4r//6r8M6JXft2sXRRx9NSkoKixYtYmN5F5gt\n5dRTT+Xpp58G4OOPP2bVqlWAmxa2cePGNGvWjG3btrFw4cKS9zRt2rTcduTBgwfz/PPPs2/fPvbu\n3cv8+fMZPHhwxJ+tWbNmtGjRoqS2/tRTTzFkyBCKi4v5+uuvGTZsGPfccw+7du1iz549fPHFF/Tq\n1Yubb76Zn/70p3z66acRb7MygZqr5Pvv4aij/I7CGBOtCRMmMHbs2MNGmGRlZXH22WfTq1cvMjIy\nqqx5Xn311Vx66aV0796d7t27l9Tc+/TpQ79+/ejWrRsdO3Y8bFrY7OxsRo0aRbt27Vi0aFHJ8vT0\ndC655BIGDBgAwOWXX06/fv0qbRapyBNPPMFVV13Fvn37OO6443jssccoKipi4sSJ7Nq1C1Xluuuu\no3nz5tx+++0sWrSIOnXq0KNHj5Ir+sRKYKZ1LS6GY46Bli1h6NBDt2OPjXl4xtQ4Nq1rcqnutK6B\nmavk4EH4/e/hJz+BuXPhggugXTvo1g2uusot27rV7yiNMcZ/gUncDRrAjTfCSy/Bt99Cbi5MmwYn\nnghz5sCECYcS+T//CXH4oXCYPXvg2WfhB2vNN8YETGASd2l160L//vDb37pEXlAAy5bBX/7imlKy\ns+Hss+Gbb2K/7f374f774bjjYPx4eOyx2G/DmHiIR7Onib1YfE+BTNxl1asHGRlw002wZAk88AC8\n+Sb07AnVOEv2MD/+CA89BCec4Gr+ffpA69YQ6kQ2JtAaNmxIQUGBJe+AU1UKCgqqfUJOoEaVeFGn\nDlx3HZx+Olx4IfzqVzBxIsycCdFMTVBYCE89BVOmwIYNcMopkJPjOkbHjYP/+79YfwJjYq9Dhw7k\n5eVh8wQFX8OGDenQoUO11pF0iTusWzd47z3405/grrvg7bdds8bIkd7eX1wM//oXTJ4Mn33mavR/\n/7s7IITPxM3MhHnzYPNmaN8+Xp/EmOpLSUmha9eufodhEiQpmkoqkpICd94J778PjRvDaae52nip\nScIoLoavv3aJ/ZFH3MiV8ePhpJPcyJUGDeD55+GDD+DnPz+UtMElbrBatzEmWJIqcefkQJcurrmk\nSxf3HOCnP4UVK1zSnjkT+vWDs86C7t0hNRU6dYJhw+Dyy91IlQ8/dG3Zc+e6yXjGjDk8YYf16weN\nGlniNsYES9I0leTkuNEk4dr0xo3uOUBWlkvQDzzg7v/6V9f80agRjBjhRqCccAIcfzx07Og6O71I\nSYEBAyxxG2OCJWlq3LfeengTCLjnt9566HlODsyY4U7mATe07+23oWlT1/bdtav3pB12yimuVr5n\nT7XCN8aYmEmaxL1pU9XLvST3SGVmQlGRawM3xpggSJrE3alT1cu9JPdIDRrk2r+tucQYExRJk7in\nTnXt16WlprrlYV6SO1TcyVme5s2hRw9L3MaY4EiaxJ2VBbNmQefOrgbcubN7npV1qIyX5B7u5Ny4\n0c13Eu7krCx5Z2a6IYdFRbH9TMYYEw0vFws+SURWlrp9LyI3JCK4srKy3NmNxcXuvnTSDr9eVXKP\nph08M9PNFb5mTaw+iTHGRM/LpcvWAX0BRKQusBmYH+e4opaVdWRCLy2advDSJ+KUuvydMcb4ItKm\nkhHAF6pa+bWHAsxrO3hpXbtC27bWzm2MCYZIE/f5wJx4BJIoXtrByxJxtW5L3MaYIPCcuEWkPnAO\n8GwFr2eLSK6I5AZ5hjIv7eDlycx07epbtiQkTGOMqVAkNe7RwApV3Vbei6o6S1UzVDWjTZs2sYku\nTqrq5CyPTThljAmKSBL3BJK8maQ6whNOLVnidyTGmNrOU+IWkcbAacC/4xtOcNmEU8aYoPCUuFV1\nr6q2UtVd8Q4oyDIzbcIpY4z/kubMySCwCaeMMUFgiTsCgwa5e2suMcb4yRJ3BFq0cFeWt8RtjPGT\nJe4I2YRTxhi/WeKOkE04ZYzxmyXuCNmJOMYYv1nijpBNOGWM8Zsl7gjZhFPGGL9Z4o6CTThljPGT\nJe4oWDu3McZPlrgrUNkFhcMTTlniNsb4ocpLl9VG4QsKh69NGb6gMLgpYG3CKWOMn6zGXQ4vFxTO\nzIQPP4S9exMbmzHGWOIuh5cLCtuEU8YYv1jiLoeXCwqHJ5yyCysYYxLNEnc5vFxQuEUL6NHD2rmN\nMYlnibscXi8obBNOGWP84PXSZc1F5DkR+VRE1orIoHgH5jcvFxS2CaeMMX7wWuN+AHhVVbsBfYC1\n8QspeQwf7mrkzz7rdyTGmNqkysQtIs2AU4FHAFT1R1XdGe/AkkGHDjB6NDzyCBQW+h2NMaa28FLj\n7grkA4+JyIci8nDoqu+HEZFsEckVkdz8/PyYBxpU2dmwdSssWOB3JMaY2sJL4q4HpAMPqWo/YC8w\nqWwhVZ2lqhmqmtGmTZsYhxlcZ54J7drBP/7hdyTGmNrCS+LOA/JUdWno+XO4RG6AevXgssvg1Vfd\nqfHGGBNvVSZuVf0G+FpETgotGgF8Eteokszll7v7hx/2Nw5jTO3gdVTJr4EcEVkF9AX+FL+Qkk+n\nTtZJaYxJHE+JW1VXhtqve6vqL1T1u3gHlmysk9IYkyh25mQ1lJ6z+7rr3Gnw1klpjIk3S9xRCs/Z\nvXEjqLqZA3fvtk5KY0z8WeKOUnlzdhcWuiT+yCP+xGSMqR0scUepojm7wTopjTHxZYk7ShXN2d2m\njbv6u3VSGmPixRJ3lCqas3vaNDuT0hgTX5a4o1TRnN0XXWRnUhpj4ssSdzVUNGd3+ExK66Q0xsSD\nJe44sDMpjTHxZIk7TrKzrZPSGBMflrjjJDzd66xZfkdijKlpLHHHSXi614ULrZPSGBNblrjjyDop\njTHxYIk7jqyT0hgTD5a44yzcSfnKK35HYoypKSxxx9kZZ8DRR8MTT/gdiTGmprDEHWcpKe7EnJde\ngoICv6MxxtQEnhK3iGwQkdUislJEcuMdVE2SkwNz58LBg3Diie65McZUR70Iyg5T1R1xi6QGCl9s\nITxv93ffuedw6PR4Y4yJlDWVxFF5F1vYt88tN8aYaHlN3Aq8LiLLRSS7vAIiki0iuSKSm5+fH7sI\nk1hFF1uo7CIMxhhTFa+J+xRVTQdGA9eIyKllC6jqrNCV4DPatGkT0yCTVUUXW+jYMbFxGGNqFk+J\nW1U3h+63A/OBAfEMqqYo72ILAOPHJz4WY0zNUWXiFpHGItI0/Bg4Hfg43oHVBGUvttCpEzRtanOX\nGGOqx0uN+xhgiYh8BHwALFDVV+MbVs1R+mILGzfCpZfCCy+4ESbGGBONKhO3qn6pqn1Ctx6qOjUR\ngdVUl1wCP/7oxnYbY0w0bDhggvXtC7162SnwxpjoWeJOMBFX6166FNau9TsaY0wyssTtg6wsqFvX\nat3GmOhY4vbBMce4ebqfegqKivyOxhiTbCxx++SSS9w83f/7v35HYoxJNpa4fXLWWdCihTWXGGMi\nZ4nbJw0awAUXwPz5sHOn39EYY5KJJW4fXXIJHDgAzzzjdyTGmGRiidtH/ftDWpo1lxhjImOJ20fh\nMd3vvQeffeZ3NMaYZGGJ22cTJ0KdOvDkk35HYoxJFpa4fXbssfDzn7vmEhvTbYzxwhK3z3JyYNky\nyMuD9u3tYsLGmKpFcrFgE2NlLya8bZtdTNgYUzWrcfvILiZsjImGJW4f2cWEjTHR8Jy4RaSuiHwo\nIi/HM6DapKKLCVe03BhjILIa9/WAzSAdQ+VdTLhuXbfcGGMq4ilxi0gH4Ezg4fiGU7uUvZhwkyZu\nDpPzzvM7MmNMkHmtcU8HfgcUV1RARLJFJFdEcvPz82MSXG1Q+mLCjz/uOifff9/vqIwxQVZl4haR\ns4Dtqrq8snKqOktVM1Q1o02bNjELsDYZORLq1YOFC/2OxBgTZF5q3JnAOSKyAZgLDBeR2XGNqpZq\n1gx+9jNL3MaYylWZuFX196raQVW7AOcDb6nqxLhHVkuNHg0rV8LWrX5HYowJKhvHHTCjR7v7V1/1\nNw5jTHBFlLhV9W1VPStewRjo3RvatbPmEmNMxazGHTAiMGoUvPEGFBb6HY0xJogscQfQ6NHuOpT/\n+Y/fkRhjgsgSdwCNHOnOoLTmEmNMeSxxB1Dz5jYs0BhTMUvcATV6NHz4oQ0LNMYcyRJ3QNmwQGNM\nRSxxB1SfPu56lNZcYowpyxJ3QNmwQGNMRSxxB5gNCzTGlMcSd4CddpoNCzTGHMkSd4A1bw6DBlni\nNsYczhJ3wIWHBX7zjd+RGGOCwhJ3wIWHBfbsCXXqQJcukJPja0jGGJ/V8zsAU7k1a9x9QYG737gR\nsrPd46wsf2IyxvjLatwBd9ttRy7btw9uvTXxsRhjgsESd8Bt2hTZcmNMzWeJO+A6dYpsuTGm5vNy\nlfeGIvKBiHwkImtE5A+JCMw4U6dCaurhy1JT3XJjTO3kpcb9AzBcVfsAfYFRIjIwvmGZsKwsmDXL\njekG6NDBPbeOSWNqLy9XeVdV3RN6mhK6aVyjMofJyoJXXnGPZ8ywpG1MbeepjVtE6orISmA78Iaq\nLi2nTLaI5IpIbn5+fqzjrPXS06F+fXj/fb8jMcb4zVPiVtUiVe0LdAAGiEjPcsrMUtUMVc1o06ZN\nrOOs9Ro0gH79LHEbYyIcVaKqO4FFwKj4hGMqM2gQ5ObCwYN+R2KM8ZOXUSVtRKR56HEj4DTg03gH\nZo40cCAcOAAffeR3JMYYP3mpcR8LLBKRVcAyXBv3y/ENy5Rn0CB3b/NzG1O7VTlXiaquAvolIBZT\nhY4doV0718597bV+R2OM8YudOZlERFxziXVQGlO7WeJOMoMGwVdfwbZtfkdijPGLJe4kY+3cxhhL\n3EkmPR3q1bPmEmNqM0vcSaZRI3cijtW4jam9LHEnoUGDYNkyKCz0OxJjjB8scSehgQPdVXBWrfI7\nEmOMHyxxJyHroDSmdrPEnYQ6d4a2ba2D0pjayhJ3EgqfiGM1bmNqJ0vcSWrQIPj8c7Cpz42pfSxx\nJylr5zam9rLEnaT693cn4ljiNqb2scSdpFJToU8f66A0pjayxJ3EBg2CDz6wE3GMqW0scSexgQNh\n715Ys8bvSIwxieTl0mUdRWSRiHwiImtE5PpEBGaqFu6gtOYSY2oXLzXuQuC3qpoGDASuEZG0+IZl\nvOjaFY4+2hK3MbVNlYlbVbeq6orQ493AWqB9vAMzVbMTcYypnSJq4xaRLrjrTy6NRzAmcoMGwWef\nQUGB35EYYxLFc+IWkSbAPOAGVf2+nNezRSRXRHLz7XS+hAm3c//5z9ClC9Sp4+5zcvyMyhgTT6Kq\nVRcSSQFeBl5T1fuqKp+RkaG5ubkxCM9UZe9eOOool7BLDwtMTYVZsyAry7/YjDHeichyVc3wUtbL\nqBIBHgHWeknaJrEaN3ZnUJYdy71vH9x6qz8xGWPiy0tTSSZwITBcRFaGbmfEOS4TgR9/LH/5pk2J\njcMYkxj1qiqgqksASUAsJkqPnu2DAAAOXUlEQVStWpXfOdmpU+JjMcbEn505WQP8/vdHLktNhalT\nEx+LMSb+LHHXADfeCE2buvZuEXeFHOuY9Nfrr8NXX/kdhampLHHXACIwZAh07AjFxbBhgyVtPy1Y\nAD//ubvt3et3NMlp506YPdsmUKuIJe4aYtAg+PRT+Pbb6q3nk0/gl7+E44+HpQk4zWr9eliyJP7b\nSZSvvoKJE+G449xnu/lmvyNKTtnZcOGFMH2635EEkyXuGiJ8Is4HH0T3/o0b4dJLoVcveOMNN1Jl\n6FCYOzdmIR5hwQJIT4dTT4UZM+K3nUQ5cMAd9MDtw9/8Bh580DWbGO/+/W949lk45hi4/XZ3ZrAp\nQ1Vjfuvfv7+axNq9W7VOHdU77ojsfdu3q95wg2r9+qoNGqjeeKNqfr5bfsopqqB6552qxcWxi7W4\nWPW++1y86emqY8a47fzud6pFRbHbTqJdfrn7HC+95J7v36+alqbarp3qt9/6G1uyKChQPeYY1X79\nVDdtUm3e3P0dJvPfhVdArnrMsZa4a5A+fVRPO81b2V27XEJu0sQl0Msuc/8opR04oHrxxe6v5Lzz\nVPftq36MP/6oesUVbp3nnqu6Z49qYaHq1Ve7ZRde6Mokm0cecfHfcsvhy3NzVevVU50wwZ+4ks1F\nF7n99eGH7vnjj7v9OnOmv3ElgiXuWuqqq1SbNlX9n/9Rffhh1SeeUJ0zR/W551RffFF14ULVN99U\n/etfVVu3dt/+uHGqa9dWvM7iYtW771YVUR0wQHXLlujjKyhQHT78UIIrXYsqLlb94x/da6efrvr9\n99FvJ9FWrFBt2FB1xAh3ECpryhT3uebOTXxsyWTBArefbrvt0LLiYtVRo1QbN1b98kv/YksES9y1\n1Pz57hv1cjvtNNVlyyJbd2qqaocOh2pDkVi3TvXEE12TzJNPVlzukUdU69ZV7d9f9ZtvIt9Oon37\nrepxx6m2b6+6bVv5ZQ4edAe9Fi1UN29ObHzJYudO97eVluZ+6ZW2caOrkIwYEdsmu6CxxF2L7dyp\nunWra/b4/HNXm161yv1kf/991XfeUV25Mrp1r1jhElTjxqrPP+/9fW++6ZJW69aq775bdfmXX1Zt\n1Ej1+ONV16+PLtZEKCpSPfts99P+vfcqL7tunftMo0bV7OQTrexs12T3n/+U//rf/+6y1axZiY0r\nUtX5biNJ3J5mB4yUzQ5Yc23ZAmPGwPLlcMMN0KMHNG8OLVocfmva1M1YOGsWXHMN/OQn8NJLbpic\nF0uXwplnunW88gpkeJozLbHuvtudtTpjBvz611WXf/BBuPZa+Pvf4corYxODKnz+uRvD37Bh9OvZ\ntw/mz4cffnDradgQGjQ48vFRR8V+KoW33oIRI+Cmm+Avfym/jCqMHAnLlrlrrHbsGNsYqmPzZjdC\nasEC2L49+itSRTI7oNW4a5HZs1U7d3bt1Z07u+fR2LvXdbZV1hRTp44bEQCulrlzZ+Tb+fRT1S5d\nXA3/iSdUd+yILt54ePNN9xnPP997LauoyDVRpaZW/5dEfr4bmdOjh9vH7durzpjhRrJE4sAB1/HX\ntq33ZrZhw7z9cvJizx7Vrl1dM1pVnd9ffOH23Rln+PurpbDQ/Xq99VbVvn0P7ZfOnVWvucY1jUUD\nq3GbsnJy3EkN+/YdWlbdObt374bvvqv81rmzOyW/XpXTmZVv61Y44wxYudI9T0uDwYMP3bzW/oqL\n3URcu3dDUZE7Iy98X/pxUZGLNTXVTSGQmnroVr++O0t182bo1w9at3bj5ps08f558vLcWPnu3eHd\nd6FuXe/vLS6GN9+Ehx+G5593Y+1PPhl+9St44QW3vnbt3Ek/V1wBjRpVvK6DB+GJJ2DKFPj6azeW\nfvJk94vowAF3++GHIx9/+SXcfz9s2wannw5/+IO7fF60brgBHngAFi9232dVZsyA66+HJ590J+gk\nyvffw6uvulr1K6/Ajh3u12BmpvtleOaZ7tenVGM6PqtxmyN07lx+7alz58PLxapWHks//ODa5qdO\ndbX3pk0Pxd+pk2pWlmsDfeEFd3/nna7N9OyzVTMyXG20Xj3vNcqKbnXrum03buyGUVY2GqcyOTlu\nfX/6k7fymza5kSldurj3tWypev31ru8irLhY9a23VIcMcWXatlW9//4ja7GFhapPPeX6D0D15JNV\n33gjshrs3r2qf/nLoZFJZ5wRWUd32JIl7u/smmu8v6eoSDUz0/WZbN1adflt21y7+Z49kce3Y4fr\nLD/zTNepHt73WVmqTz/tRknFElbjNmXVqePST1kiriYH3mvlOTnuIg2bNrka79SpiZ0bpagIVq1y\nNczwbdu2Q6+LQJs20LYtHHvsoVvbtq6NNiXF1XTr1Tt0H35ct66ree/bd+Rt7153v38/TJjgaqnR\nUIXzznO15t/+1u3/cI227P2uXe5i0MXFro33ssvgF7+ovD377bddTfjtt93Zh7/7nWtTX7gQ7rgD\n1q6Fvn3hrrtcTTHaWuKePfC3v7l26W+/dX0fkye7dVdl/35X7ocfYPVq1yfi1bp10KeP+yU2b97h\n8e/f7/4e3njD3T76yC2vW9f90hk40P1KGTjQ9bvUKXPu+JYt7nuZNw/eecf9rXXu7M6IHTvWnaEc\nya+kSFiN2xzBS43bS5nZs107Y+nXU1OPrJl7qbnHqkxxsepnn7laX15ecpzAs2OHardubv81aKB6\n1FGqRx+t2rGja+/t2dP9WjjlFNeW+sUXkW/jnXcOjZsP1xi7d1d99tnYnom4a5fqXXcd6tM45xx3\nBu/MmW7s+ptvqq5e7YZ3htt/b77ZlX399ei2ec897v1z5rgRU3/+s/usDRq45SkpqkOHul9p8+a5\nseEjR7r9HP67bdbM9Tncfrvqvfeq/uxn7u8MVE86yZ1rsHx54trTseGApiwvCTf8R1v2JnKoTKyS\ne6zKhMvF4gCQaMXFiUkK777rzox96qnyTxCKle++cwm7bdvK/5Zatjx0tm60Dh50B7bS6+7Vy03Z\nsHBhxU0jRUWqn3yi+uijqlde6c42rlPHvb9vX3cAWrMm+riqI6aJG3gU2A587HWllriDqark5SUp\nxyq5J/IXQBB/JQTxQBJLhYWuffnjj1UXLVJ95hnVv/3N9T/893+7+XGiGWlU2mefufbx2bO9tXdX\nZM+eYJwYFevEfSqQbom75vOS4GKV3GNVJmgHADuQmGjFvKkE6GKJu3ao6h84Vsk9VmWCdgCozQcS\nO9BUjy+JG8gGcoHcTp06JeqzGh/EIrnHqkzQDgC19UCSjAeaRJepitW4je8S9Q8RtANAbT2QBG3/\nBfFAUhVL3KZWCdIBoLYeSJLtQJPoMl5Y4jYmCkH6aZ1sB5KgJcqgHUi8iPWokjnAVuAgkAdcVtV7\nLHEbU33JdCBJtgNNost4YSfgGGNiLplGlQTtQOKFJW5jTK0XpAOJF5EkbptkyhhjAiCSSabqVF3E\nGGNMkFjiNsaYJGOJ2xhjkowlbmOMSTKWuI0xJsnEZVSJiOQDG8t5qTWwI+YbjK9kiznZ4gWLOVGS\nLeZkixeqF3NnVW3jpWBcEneFGxPJ9TrcJSiSLeZkixcs5kRJtpiTLV5IXMzWVGKMMUnGErcxxiSZ\nRCfuWQneXiwkW8zJFi9YzImSbDEnW7yQoJgT2sZtjDGm+qypxBhjkowlbmOMSTIJSdwiMkpE1onI\n5yIyKRHbrC4R2SAiq0VkpYgEcqpDEXlURLaLyMellrUUkTdEZH3ovoWfMZZVQcyTRWRzaF+vFJEz\n/IyxNBHpKCKLROQTEVkjIteHlgd2P1cSc5D3c0MR+UBEPgrF/IfQ8q4isjSUO/4lIvX9jjWskpgf\nF5GvSu3nvjHfuNf5X6O9AXWBL4DjgPrAR0BavLcbg7g3AK39jqOKGE8F0il1WTngXmBS6PEk4B6/\n4/QQ82TgJr9jqyDeY4H00OOmwGdAWpD3cyUxB3k/C9Ak9DgFWAoMBJ4Bzg8t/ztwtd+xeoj5cWBc\nPLediBr3AOBzVf1SVX8E5gJjErDdGk9VFwPfllk8Bngi9PgJ4BcJDaoKFcQcWKq6VVVXhB7vBtYC\n7Qnwfq4k5sBSZ0/oaUropsBw4LnQ8qDt54pijrtEJO72wNelnucR8D+iEAVeF5HlIpLtdzAROEZV\nt4YefwMc42cwEbhWRFaFmlIC0+xQmoh0AfrhalZJsZ/LxAwB3s8iUldEVgLbgTdwv9R3qmphqEjg\nckfZmFU1vJ+nhvbz/SLSINbbtc7Jip2iqunAaOAaETnV74Aipe43XDKM93wIOB7oi7sw9V/9DedI\nItIEmAfcoKrfl34tqPu5nJgDvZ9VtUhV+wIdcL/Uu/kcUpXKxiwiPYHf42L/KdASuDnW201E4t4M\ndCz1vENoWaCp6ubQ/XZgPu4PKRlsE5FjAUL3232Op0qqui30D1AM/JOA7WsRScElwBxV/XdocaD3\nc3kxB30/h6nqTmARMAhoLiL1Qi8FNneUinlUqKlKVfUH4DHisJ8TkbiXASeGeofrA+cDLyZgu1ET\nkcYi0jT8GDgd+LjydwXGi8DFoccXAy/4GIsn4QQYMpYA7WsREeARYK2q3lfqpcDu54piDvh+biMi\nzUOPGwGn4drmFwHjQsWCtp/Li/nTUgd0wbXJx3w/J+TMydCwo+m4ESaPqurUuG+0GkTkOFwtG6Ae\n8HQQYxaROcBQ3FSS24A7gedxPfGdcFPrjlfVwHQGVhDzUNzPd8WN5rmyVPuxr0TkFOBdYDVQHFp8\nC67NOJD7uZKYJxDc/dwb1/lYF1ehfEZVp4T+F+fimhw+BCaGarK+qyTmt4A2uFEnK4GrSnVixmbb\niUjcxhhjYsc6J40xJslY4jbGmCRjidsYY5KMJW5jjEkylriNMSbJWOI2xpgkY4nbGGOSzP8HNuOU\n5TQqBu4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#A simple single stage convolutional neural network\n",
    "\n",
    "#The neural network architectures will be built using combinations of the following parameters.\n",
    "params = {'classes' : [classes],\n",
    "          'activation' : ['relu', 'sigmoid', 'tanh'],\n",
    "          'batch_size': [1024],\n",
    "          'epochs': [200],\n",
    "          'optimizer': ['adam', 'rmsprop'],\n",
    "          'loss': ['sparse_categorical_crossentropy'],\n",
    "          'reg_param': [0],\n",
    "          'callbacks' : [[keras.callbacks.EarlyStopping(monitor='val_loss',\n",
    "                              min_delta=0,\n",
    "                              patience=6,\n",
    "                              verbose=0, mode='auto')]]}\n",
    "\n",
    "#Hyperopt will optimize the following parameters in the given ranges.\n",
    "search_space = {'lr': hp.loguniform('lr', -6, -4), 'dropout_prob' : hp.uniform('dropout_prob', 0.0, 0.6)}\n",
    "\n",
    "#The call to process_model will compile, train and evaulate the above models and optimize the hyper parameters.\n",
    "#The output will be saved to a file and the output details will be recorded in the model_res_file. \n",
    "model_name = \"simpleCNN\"\n",
    "meta_model.max_evals = 5\n",
    "model_results = meta_model.process_model(params, search_space, simple_cnn_model, model_name, [X_train, X_val, X_test, y_train, y_val, y_test], \n",
    "              [model_name, 0, 0, 0, np.NaN, 'epochs', 'activation', 'optimizer', 'lr','reg_param', 'dropout_prob', 'batchNorm' , np.NaN])\n",
    "\n",
    "#Now lets plot the results.\n",
    "history = model_results[0][0]\n",
    "plot_train_val(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6955582,
     "status": "error",
     "timestamp": 1565909626892,
     "user": {
      "displayName": "Krishan Rajaratnam",
      "photoUrl": "https://lh6.googleusercontent.com/-Za30nR0ehyQ/AAAAAAAAAAI/AAAAAAAAGMw/jV8ldTp8928/s64/photo.jpg",
      "userId": "09748671397076320798"
     },
     "user_tz": 240
    },
    "id": "viWvPYiqQ0GD",
    "outputId": "8afdee18-0832-4f17-f71f-5becca180905"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0815 20:57:49.869309 139920832087936 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:68: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "W0815 20:57:49.884992 139920832087936 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:508: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "W0815 20:57:49.893497 139920832087936 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3837: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "W0815 20:57:49.924626 139920832087936 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:168: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
      "\n",
      "W0815 20:57:49.925660 139920832087936 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:175: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [512, 128], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.3084970434581393, 'lr': 0.0008312092311295237}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0815 20:57:50.438393 139920832087936 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1801: The name tf.nn.fused_batch_norm is deprecated. Please use tf.compat.v1.nn.fused_batch_norm instead.\n",
      "\n",
      "W0815 20:57:50.528940 139920832087936 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3661: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n",
      "\n",
      "W0815 20:57:50.537891 139920832087936 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3144: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "W0815 20:57:51.116436 139920832087936 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/optimizers.py:757: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n",
      "W0815 20:57:51.232132 139920832087936 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 8s 203us/step - loss: 2.1401 - acc: 0.3145 - val_loss: 1.7843 - val_acc: 0.3897\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.6567 - acc: 0.4205 - val_loss: 2.1014 - val_acc: 0.2921\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.4641 - acc: 0.4784 - val_loss: 3.0784 - val_acc: 0.2195\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.3313 - acc: 0.5256 - val_loss: 2.8513 - val_acc: 0.2612\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.2442 - acc: 0.5575 - val_loss: 3.8964 - val_acc: 0.1612\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.1722 - acc: 0.5839 - val_loss: 1.5858 - val_acc: 0.4739\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.1175 - acc: 0.6028 - val_loss: 2.0368 - val_acc: 0.4092\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.0728 - acc: 0.6190 - val_loss: 1.6500 - val_acc: 0.4893\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.0413 - acc: 0.6293 - val_loss: 1.4792 - val_acc: 0.5092\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.0214 - acc: 0.6397 - val_loss: 1.3487 - val_acc: 0.5276\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.9829 - acc: 0.6517 - val_loss: 1.2050 - val_acc: 0.5957\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.9598 - acc: 0.6615 - val_loss: 1.1889 - val_acc: 0.6061\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.9403 - acc: 0.6687 - val_loss: 1.1336 - val_acc: 0.6030\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.9256 - acc: 0.6722 - val_loss: 1.6335 - val_acc: 0.5024\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.9019 - acc: 0.6834 - val_loss: 1.6332 - val_acc: 0.4852\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.8821 - acc: 0.6865 - val_loss: 1.4399 - val_acc: 0.5447\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.8729 - acc: 0.6929 - val_loss: 0.9587 - val_acc: 0.6596\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.8492 - acc: 0.7009 - val_loss: 1.1923 - val_acc: 0.5932\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.8337 - acc: 0.7072 - val_loss: 1.0033 - val_acc: 0.6624\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.8315 - acc: 0.7076 - val_loss: 0.9239 - val_acc: 0.6729\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.8025 - acc: 0.7187 - val_loss: 1.6597 - val_acc: 0.4985\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.8046 - acc: 0.7161 - val_loss: 0.9906 - val_acc: 0.6604\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.7859 - acc: 0.7228 - val_loss: 0.9819 - val_acc: 0.6671\n",
      "Epoch 24/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.7679 - acc: 0.7295 - val_loss: 1.2887 - val_acc: 0.5977\n",
      "Epoch 25/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.7674 - acc: 0.7297 - val_loss: 1.0211 - val_acc: 0.6526\n",
      "Epoch 26/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.7456 - acc: 0.7353 - val_loss: 0.9380 - val_acc: 0.6896\n",
      "10000/10000 [==============================] - 1s 141us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [512, 128], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.3973617779950215, 'lr': 0.0019778786743402756}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 7s 176us/step - loss: 2.0819 - acc: 0.3184 - val_loss: 2.1416 - val_acc: 0.3373\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.5708 - acc: 0.4372 - val_loss: 5.1176 - val_acc: 0.1278\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.4040 - acc: 0.4948 - val_loss: 5.0536 - val_acc: 0.1310\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.2912 - acc: 0.5349 - val_loss: 2.8597 - val_acc: 0.2820\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.2274 - acc: 0.5621 - val_loss: 3.5998 - val_acc: 0.2129\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.1634 - acc: 0.5853 - val_loss: 2.0324 - val_acc: 0.4156\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.1115 - acc: 0.6038 - val_loss: 2.0833 - val_acc: 0.4502\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.0779 - acc: 0.6157 - val_loss: 2.2285 - val_acc: 0.4401\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.0393 - acc: 0.6319 - val_loss: 1.9213 - val_acc: 0.4429\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.0100 - acc: 0.6425 - val_loss: 1.3974 - val_acc: 0.5378\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.9857 - acc: 0.6529 - val_loss: 1.8624 - val_acc: 0.4529\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.9720 - acc: 0.6585 - val_loss: 1.1696 - val_acc: 0.5957\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.9479 - acc: 0.6645 - val_loss: 1.2457 - val_acc: 0.5664\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.9203 - acc: 0.6773 - val_loss: 1.2912 - val_acc: 0.5697\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.9027 - acc: 0.6821 - val_loss: 1.4283 - val_acc: 0.5468\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.8969 - acc: 0.6838 - val_loss: 1.1354 - val_acc: 0.6034\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.8768 - acc: 0.6896 - val_loss: 1.5794 - val_acc: 0.5316\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.8590 - acc: 0.6992 - val_loss: 1.1752 - val_acc: 0.6128\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.8476 - acc: 0.7004 - val_loss: 0.9285 - val_acc: 0.6727\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.8252 - acc: 0.7097 - val_loss: 1.0631 - val_acc: 0.6471\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.8106 - acc: 0.7139 - val_loss: 1.4496 - val_acc: 0.5520\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.8009 - acc: 0.7188 - val_loss: 0.9238 - val_acc: 0.6766\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.7813 - acc: 0.7261 - val_loss: 1.3217 - val_acc: 0.5745\n",
      "Epoch 24/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.7788 - acc: 0.7265 - val_loss: 1.6543 - val_acc: 0.5165\n",
      "Epoch 25/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.7649 - acc: 0.7314 - val_loss: 1.4378 - val_acc: 0.5497\n",
      "Epoch 26/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.7746 - acc: 0.7306 - val_loss: 1.2444 - val_acc: 0.5953\n",
      "Epoch 27/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.7383 - acc: 0.7402 - val_loss: 0.8341 - val_acc: 0.7067\n",
      "Epoch 28/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.7430 - acc: 0.7394 - val_loss: 1.2986 - val_acc: 0.5832\n",
      "Epoch 29/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.7216 - acc: 0.7467 - val_loss: 0.9699 - val_acc: 0.6718\n",
      "Epoch 30/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.7181 - acc: 0.7480 - val_loss: 0.8551 - val_acc: 0.7031\n",
      "Epoch 31/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.7042 - acc: 0.7512 - val_loss: 1.2722 - val_acc: 0.6011\n",
      "Epoch 32/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.6971 - acc: 0.7548 - val_loss: 1.1171 - val_acc: 0.6371\n",
      "Epoch 33/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.6901 - acc: 0.7571 - val_loss: 1.0160 - val_acc: 0.6567\n",
      "10000/10000 [==============================] - 1s 143us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [512, 128], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.20870512266765648, 'lr': 0.0007478340593250693}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 7s 181us/step - loss: 1.8880 - acc: 0.3738 - val_loss: 1.5956 - val_acc: 0.4491\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.4411 - acc: 0.4941 - val_loss: 2.1026 - val_acc: 0.3644\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.2817 - acc: 0.5448 - val_loss: 2.0166 - val_acc: 0.3972\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.1617 - acc: 0.5899 - val_loss: 2.0234 - val_acc: 0.4426\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.0893 - acc: 0.6131 - val_loss: 1.4179 - val_acc: 0.5443\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.0360 - acc: 0.6327 - val_loss: 1.3954 - val_acc: 0.5312\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.9919 - acc: 0.6485 - val_loss: 1.6207 - val_acc: 0.5031\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.9595 - acc: 0.6588 - val_loss: 1.0980 - val_acc: 0.6261\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.9238 - acc: 0.6745 - val_loss: 1.7665 - val_acc: 0.4758\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.9032 - acc: 0.6797 - val_loss: 1.0628 - val_acc: 0.6373\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.8764 - acc: 0.6912 - val_loss: 1.3123 - val_acc: 0.5761\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.8494 - acc: 0.7006 - val_loss: 1.6498 - val_acc: 0.5066\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.8279 - acc: 0.7070 - val_loss: 1.0739 - val_acc: 0.6422\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.8031 - acc: 0.7186 - val_loss: 1.0921 - val_acc: 0.6279\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.7883 - acc: 0.7206 - val_loss: 1.3268 - val_acc: 0.5829\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.7714 - acc: 0.7283 - val_loss: 1.2175 - val_acc: 0.6111\n",
      "10000/10000 [==============================] - 1s 141us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [512, 128], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.4332078930599433, 'lr': 0.0024358556938475612}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 7s 187us/step - loss: 2.1242 - acc: 0.3110 - val_loss: 2.6400 - val_acc: 0.2429\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.6005 - acc: 0.4207 - val_loss: 3.6535 - val_acc: 0.1801\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 1.4370 - acc: 0.4789 - val_loss: 4.3475 - val_acc: 0.1681\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.3460 - acc: 0.5118 - val_loss: 3.3017 - val_acc: 0.2352\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.2879 - acc: 0.5385 - val_loss: 2.7861 - val_acc: 0.2878\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.2299 - acc: 0.5595 - val_loss: 3.1693 - val_acc: 0.2609\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.1651 - acc: 0.5853 - val_loss: 2.9992 - val_acc: 0.2923\n",
      "10000/10000 [==============================] - 1s 143us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [512, 128], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.4898324748221846, 'lr': 0.0004532852551100587}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 8s 194us/step - loss: 2.7509 - acc: 0.2103 - val_loss: 1.8688 - val_acc: 0.3235\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 2.2169 - acc: 0.2917 - val_loss: 1.8076 - val_acc: 0.3396\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 2.0074 - acc: 0.3324 - val_loss: 1.8795 - val_acc: 0.3251\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.8765 - acc: 0.3548 - val_loss: 1.8677 - val_acc: 0.3253\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.7681 - acc: 0.3792 - val_loss: 1.8734 - val_acc: 0.3202\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.6957 - acc: 0.3995 - val_loss: 1.9616 - val_acc: 0.3126\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.6339 - acc: 0.4150 - val_loss: 1.9102 - val_acc: 0.3236\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.5801 - acc: 0.4358 - val_loss: 2.4054 - val_acc: 0.2352\n",
      "10000/10000 [==============================] - 1s 142us/step\n",
      "40000/40000 [==============================] - 7s 163us/step\n",
      "10000/10000 [==============================] - 2s 181us/step\n",
      "Loss = 0.9450749446868897\n",
      "Test Accuracy = 0.6862\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [1024, 512], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.4939041258186643, 'lr': 0.0008251888964779829}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 9s 214us/step - loss: 2.5716 - acc: 0.2564 - val_loss: 2.0609 - val_acc: 0.2964\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.9608 - acc: 0.3496 - val_loss: 2.8062 - val_acc: 0.2045\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.7469 - acc: 0.3926 - val_loss: 4.0090 - val_acc: 0.1615\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.5946 - acc: 0.4353 - val_loss: 3.9874 - val_acc: 0.1474\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.5093 - acc: 0.4555 - val_loss: 4.2355 - val_acc: 0.1494\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.4290 - acc: 0.4837 - val_loss: 4.9170 - val_acc: 0.1188\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.3726 - acc: 0.5038 - val_loss: 4.7376 - val_acc: 0.1355\n",
      "10000/10000 [==============================] - 2s 154us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [1024, 512], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.43186566518255953, 'lr': 0.0014797849615483425}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 9s 222us/step - loss: 2.2387 - acc: 0.3095 - val_loss: 2.6463 - val_acc: 0.2559\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.6750 - acc: 0.4149 - val_loss: 5.5797 - val_acc: 0.1149\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.4762 - acc: 0.4708 - val_loss: 8.9218 - val_acc: 0.1001\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.3541 - acc: 0.5137 - val_loss: 5.1749 - val_acc: 0.1388\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.2687 - acc: 0.5460 - val_loss: 6.5504 - val_acc: 0.1138\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.1981 - acc: 0.5718 - val_loss: 3.7673 - val_acc: 0.2249\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.1561 - acc: 0.5863 - val_loss: 4.0184 - val_acc: 0.1945\n",
      "10000/10000 [==============================] - 2s 158us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [1024, 512], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.2472232601824096, 'lr': 0.0005799412924579146}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 2.0076 - acc: 0.3573 - val_loss: 1.5083 - val_acc: 0.4753\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.5484 - acc: 0.4661 - val_loss: 2.2578 - val_acc: 0.3269\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.3855 - acc: 0.5181 - val_loss: 3.7999 - val_acc: 0.1894\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.2681 - acc: 0.5541 - val_loss: 3.1152 - val_acc: 0.2411\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.1748 - acc: 0.5850 - val_loss: 4.0980 - val_acc: 0.2056\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.1141 - acc: 0.6079 - val_loss: 3.2353 - val_acc: 0.2699\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.0490 - acc: 0.6291 - val_loss: 2.7453 - val_acc: 0.3182\n",
      "10000/10000 [==============================] - 2s 157us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [1024, 512], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.2658802933411199, 'lr': 0.0004993081245989567}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 2.0956 - acc: 0.3412 - val_loss: 1.6579 - val_acc: 0.4311\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.6118 - acc: 0.4428 - val_loss: 2.8396 - val_acc: 0.2605\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.4520 - acc: 0.4925 - val_loss: 3.6629 - val_acc: 0.2088\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.3355 - acc: 0.5294 - val_loss: 3.2872 - val_acc: 0.2382\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.2630 - acc: 0.5544 - val_loss: 3.2543 - val_acc: 0.2547\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.1894 - acc: 0.5813 - val_loss: 3.0043 - val_acc: 0.2804\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.1302 - acc: 0.6012 - val_loss: 2.2949 - val_acc: 0.3846\n",
      "10000/10000 [==============================] - 2s 157us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [1024, 512], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.31767113328501445, 'lr': 0.0022067524636316125}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 10s 240us/step - loss: 1.8895 - acc: 0.3808 - val_loss: 5.5954 - val_acc: 0.1813\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.4119 - acc: 0.5026 - val_loss: 8.9893 - val_acc: 0.1101\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.2577 - acc: 0.5522 - val_loss: 7.4016 - val_acc: 0.1176\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.1343 - acc: 0.5953 - val_loss: 4.2795 - val_acc: 0.2033\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.0571 - acc: 0.6238 - val_loss: 3.3884 - val_acc: 0.2791\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.0055 - acc: 0.6448 - val_loss: 2.4958 - val_acc: 0.3637\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.9570 - acc: 0.6625 - val_loss: 2.8607 - val_acc: 0.3347\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.9079 - acc: 0.6778 - val_loss: 2.5772 - val_acc: 0.3883\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.8806 - acc: 0.6897 - val_loss: 3.2221 - val_acc: 0.2975\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.8404 - acc: 0.7015 - val_loss: 2.8523 - val_acc: 0.3436\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.8224 - acc: 0.7090 - val_loss: 2.7228 - val_acc: 0.3854\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.7943 - acc: 0.7179 - val_loss: 2.1278 - val_acc: 0.4442\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.7747 - acc: 0.7281 - val_loss: 1.6706 - val_acc: 0.5279\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.7548 - acc: 0.7320 - val_loss: 2.6820 - val_acc: 0.3724\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.7330 - acc: 0.7410 - val_loss: 1.4970 - val_acc: 0.5811\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.7250 - acc: 0.7440 - val_loss: 1.6445 - val_acc: 0.5687\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.6977 - acc: 0.7547 - val_loss: 1.3010 - val_acc: 0.6230\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.6810 - acc: 0.7607 - val_loss: 2.4535 - val_acc: 0.4355\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.6499 - acc: 0.7702 - val_loss: 1.6149 - val_acc: 0.5560\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.6460 - acc: 0.7700 - val_loss: 1.8213 - val_acc: 0.5181\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.6303 - acc: 0.7750 - val_loss: 1.3639 - val_acc: 0.6221\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.6198 - acc: 0.7817 - val_loss: 3.1323 - val_acc: 0.3876\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.5995 - acc: 0.7878 - val_loss: 1.6770 - val_acc: 0.5609\n",
      "10000/10000 [==============================] - 2s 160us/step\n",
      "40000/40000 [==============================] - 8s 195us/step\n",
      "10000/10000 [==============================] - 2s 203us/step\n",
      "Loss = 1.687518045425415\n",
      "Test Accuracy = 0.5578\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [4096, 2048], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.37298523735123934, 'lr': 0.0007079410598510764}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 14s 339us/step - loss: 2.3480 - acc: 0.3386 - val_loss: 2.2463 - val_acc: 0.4020\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 1.7142 - acc: 0.4328 - val_loss: 2.7559 - val_acc: 0.3042\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 1.5080 - acc: 0.4826 - val_loss: 2.6363 - val_acc: 0.2829\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.4059 - acc: 0.5177 - val_loss: 4.3479 - val_acc: 0.1805\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.3099 - acc: 0.5436 - val_loss: 4.1060 - val_acc: 0.1857\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.2477 - acc: 0.5630 - val_loss: 6.0207 - val_acc: 0.1753\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.1757 - acc: 0.5880 - val_loss: 4.6694 - val_acc: 0.2171\n",
      "10000/10000 [==============================] - 2s 210us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [4096, 2048], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.34454891347218175, 'lr': 0.0006374069312911223}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 14s 345us/step - loss: 2.2805 - acc: 0.3463 - val_loss: 2.0693 - val_acc: 0.4129\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 9s 231us/step - loss: 1.6534 - acc: 0.4476 - val_loss: 2.4388 - val_acc: 0.3371\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 9s 231us/step - loss: 1.4791 - acc: 0.4949 - val_loss: 2.4086 - val_acc: 0.3439\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.3671 - acc: 0.5327 - val_loss: 4.3424 - val_acc: 0.2154\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.2975 - acc: 0.5510 - val_loss: 3.3753 - val_acc: 0.2731\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 1.2310 - acc: 0.5714 - val_loss: 2.9359 - val_acc: 0.3036\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.1673 - acc: 0.5927 - val_loss: 2.3394 - val_acc: 0.3537\n",
      "10000/10000 [==============================] - 2s 208us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [4096, 2048], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.4372273664936155, 'lr': 0.0017082387191644998}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 14s 349us/step - loss: 2.5522 - acc: 0.3289 - val_loss: 2.5230 - val_acc: 0.3985\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 9s 229us/step - loss: 1.5994 - acc: 0.4422 - val_loss: 3.7132 - val_acc: 0.2249\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 9s 229us/step - loss: 1.4563 - acc: 0.4915 - val_loss: 6.3648 - val_acc: 0.1365\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 9s 229us/step - loss: 1.3560 - acc: 0.5244 - val_loss: 3.6204 - val_acc: 0.2490\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 9s 231us/step - loss: 1.2668 - acc: 0.5562 - val_loss: 7.4299 - val_acc: 0.1150\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.2009 - acc: 0.5777 - val_loss: 5.2109 - val_acc: 0.1591\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.1443 - acc: 0.5968 - val_loss: 6.0869 - val_acc: 0.1434\n",
      "10000/10000 [==============================] - 2s 213us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [4096, 2048], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.2802462537870997, 'lr': 0.002220671511508301}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 14s 358us/step - loss: 2.2906 - acc: 0.3663 - val_loss: 2.5816 - val_acc: 0.3195\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.4706 - acc: 0.4891 - val_loss: 5.7406 - val_acc: 0.1278\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.3324 - acc: 0.5384 - val_loss: 5.2738 - val_acc: 0.1509\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.1796 - acc: 0.5860 - val_loss: 4.1388 - val_acc: 0.1885\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.1057 - acc: 0.6156 - val_loss: 4.0195 - val_acc: 0.2333\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.0437 - acc: 0.6401 - val_loss: 2.8404 - val_acc: 0.2733\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 0.9708 - acc: 0.6623 - val_loss: 3.4000 - val_acc: 0.2864\n",
      "10000/10000 [==============================] - 2s 216us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [4096, 2048], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.3772372212217549, 'lr': 0.0015449075509119904}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 15s 368us/step - loss: 2.4264 - acc: 0.3456 - val_loss: 3.3544 - val_acc: 0.2717\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 9s 231us/step - loss: 1.5681 - acc: 0.4608 - val_loss: 6.4225 - val_acc: 0.1308\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 9s 230us/step - loss: 1.4107 - acc: 0.5119 - val_loss: 8.2602 - val_acc: 0.1136\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 9s 231us/step - loss: 1.2979 - acc: 0.5485 - val_loss: 8.1710 - val_acc: 0.1089\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.2350 - acc: 0.5746 - val_loss: 7.7381 - val_acc: 0.1111\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.1691 - acc: 0.5978 - val_loss: 4.2728 - val_acc: 0.1941\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.0851 - acc: 0.6235 - val_loss: 4.0865 - val_acc: 0.1982\n",
      "10000/10000 [==============================] - 2s 212us/step\n",
      "40000/40000 [==============================] - 10s 242us/step\n",
      "10000/10000 [==============================] - 3s 265us/step\n",
      "Loss = 2.346066194152832\n",
      "Test Accuracy = 0.3596\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [512, 128], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.27571760719184557, 'lr': 0.00037451413432830965}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 11s 272us/step - loss: 2.2529 - acc: 0.2922 - val_loss: 1.8288 - val_acc: 0.3628\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.7738 - acc: 0.3954 - val_loss: 1.6749 - val_acc: 0.3999\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 1.6130 - acc: 0.4419 - val_loss: 1.6737 - val_acc: 0.3984\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.4971 - acc: 0.4720 - val_loss: 1.7805 - val_acc: 0.3710\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.4183 - acc: 0.4968 - val_loss: 1.6826 - val_acc: 0.4072\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.3500 - acc: 0.5213 - val_loss: 2.1613 - val_acc: 0.3180\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.2989 - acc: 0.5389 - val_loss: 1.9803 - val_acc: 0.3611\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.2530 - acc: 0.5554 - val_loss: 1.7650 - val_acc: 0.4319\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.2011 - acc: 0.5720 - val_loss: 1.7030 - val_acc: 0.4494\n",
      "10000/10000 [==============================] - 2s 157us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [512, 128], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.3358355308031997, 'lr': 0.002181030836693047}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 11s 281us/step - loss: 1.9290 - acc: 0.3533 - val_loss: 5.4815 - val_acc: 0.1189\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.4399 - acc: 0.4819 - val_loss: 6.3938 - val_acc: 0.1076\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.2728 - acc: 0.5430 - val_loss: 5.5925 - val_acc: 0.1340\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.1699 - acc: 0.5804 - val_loss: 4.1179 - val_acc: 0.1900\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.1002 - acc: 0.6079 - val_loss: 3.0101 - val_acc: 0.2884\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.0442 - acc: 0.6291 - val_loss: 1.7590 - val_acc: 0.4749\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.0126 - acc: 0.6376 - val_loss: 2.4035 - val_acc: 0.3829\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.9696 - acc: 0.6588 - val_loss: 1.7570 - val_acc: 0.4841\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.9485 - acc: 0.6646 - val_loss: 1.7840 - val_acc: 0.4652\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.9157 - acc: 0.6770 - val_loss: 1.8006 - val_acc: 0.4824\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.8926 - acc: 0.6857 - val_loss: 1.8666 - val_acc: 0.4909\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.8723 - acc: 0.6946 - val_loss: 1.5887 - val_acc: 0.5147\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.8509 - acc: 0.7023 - val_loss: 1.9645 - val_acc: 0.4456\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.8342 - acc: 0.7069 - val_loss: 1.7399 - val_acc: 0.4982\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.8146 - acc: 0.7137 - val_loss: 2.0543 - val_acc: 0.4275\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.7898 - acc: 0.7221 - val_loss: 1.7541 - val_acc: 0.4776\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.7877 - acc: 0.7224 - val_loss: 1.3182 - val_acc: 0.5859\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.7572 - acc: 0.7329 - val_loss: 1.4390 - val_acc: 0.5505\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.7390 - acc: 0.7413 - val_loss: 1.8799 - val_acc: 0.4626\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.7344 - acc: 0.7394 - val_loss: 1.6002 - val_acc: 0.5123\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.7212 - acc: 0.7452 - val_loss: 1.4675 - val_acc: 0.5528\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.7027 - acc: 0.7518 - val_loss: 1.4575 - val_acc: 0.5753\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.6872 - acc: 0.7569 - val_loss: 1.1661 - val_acc: 0.6148\n",
      "Epoch 24/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.6828 - acc: 0.7603 - val_loss: 1.1924 - val_acc: 0.6171\n",
      "Epoch 25/200\n",
      "40000/40000 [==============================] - 6s 140us/step - loss: 0.6734 - acc: 0.7646 - val_loss: 1.1869 - val_acc: 0.6264\n",
      "Epoch 26/200\n",
      "40000/40000 [==============================] - 6s 140us/step - loss: 0.6528 - acc: 0.7691 - val_loss: 1.4871 - val_acc: 0.5485\n",
      "Epoch 27/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.6472 - acc: 0.7730 - val_loss: 1.0501 - val_acc: 0.6522\n",
      "Epoch 28/200\n",
      "40000/40000 [==============================] - 6s 140us/step - loss: 0.6441 - acc: 0.7725 - val_loss: 1.2150 - val_acc: 0.6157\n",
      "Epoch 29/200\n",
      "40000/40000 [==============================] - 6s 140us/step - loss: 0.6279 - acc: 0.7799 - val_loss: 1.0916 - val_acc: 0.6492\n",
      "Epoch 30/200\n",
      "40000/40000 [==============================] - 6s 140us/step - loss: 0.6221 - acc: 0.7813 - val_loss: 0.8616 - val_acc: 0.7091\n",
      "Epoch 31/200\n",
      "40000/40000 [==============================] - 6s 140us/step - loss: 0.6096 - acc: 0.7822 - val_loss: 0.9470 - val_acc: 0.6855\n",
      "Epoch 32/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.6193 - acc: 0.7827 - val_loss: 0.7630 - val_acc: 0.7406\n",
      "Epoch 33/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.5850 - acc: 0.7930 - val_loss: 0.8027 - val_acc: 0.7324\n",
      "Epoch 34/200\n",
      "40000/40000 [==============================] - 6s 140us/step - loss: 0.5781 - acc: 0.7946 - val_loss: 1.0961 - val_acc: 0.6517\n",
      "Epoch 35/200\n",
      "40000/40000 [==============================] - 6s 140us/step - loss: 0.5923 - acc: 0.7896 - val_loss: 0.8774 - val_acc: 0.7138\n",
      "Epoch 36/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.5698 - acc: 0.7999 - val_loss: 0.7205 - val_acc: 0.7533\n",
      "Epoch 37/200\n",
      "40000/40000 [==============================] - 6s 140us/step - loss: 0.5608 - acc: 0.7998 - val_loss: 1.3636 - val_acc: 0.6052\n",
      "Epoch 38/200\n",
      "40000/40000 [==============================] - 6s 140us/step - loss: 0.5670 - acc: 0.8001 - val_loss: 0.8752 - val_acc: 0.7107\n",
      "Epoch 39/200\n",
      "40000/40000 [==============================] - 6s 140us/step - loss: 0.5416 - acc: 0.8104 - val_loss: 1.1320 - val_acc: 0.6498\n",
      "Epoch 40/200\n",
      "40000/40000 [==============================] - 6s 140us/step - loss: 0.5470 - acc: 0.8080 - val_loss: 0.8526 - val_acc: 0.7205\n",
      "Epoch 41/200\n",
      "40000/40000 [==============================] - 6s 140us/step - loss: 0.5312 - acc: 0.8126 - val_loss: 0.7861 - val_acc: 0.7376\n",
      "Epoch 42/200\n",
      "40000/40000 [==============================] - 6s 140us/step - loss: 0.5311 - acc: 0.8147 - val_loss: 0.8111 - val_acc: 0.7288\n",
      "10000/10000 [==============================] - 2s 155us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [512, 128], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.3360652541282154, 'lr': 0.0003446970195180407}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 11s 285us/step - loss: 2.4263 - acc: 0.2616 - val_loss: 1.6280 - val_acc: 0.4264\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.9301 - acc: 0.3627 - val_loss: 1.5230 - val_acc: 0.4630\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.7628 - acc: 0.3983 - val_loss: 1.4794 - val_acc: 0.4722\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.6461 - acc: 0.4297 - val_loss: 1.6144 - val_acc: 0.4390\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.5523 - acc: 0.4558 - val_loss: 1.7755 - val_acc: 0.4061\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.4853 - acc: 0.4726 - val_loss: 1.6868 - val_acc: 0.4257\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.4300 - acc: 0.4932 - val_loss: 1.8856 - val_acc: 0.3847\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.3824 - acc: 0.5069 - val_loss: 1.7449 - val_acc: 0.4311\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.3498 - acc: 0.5189 - val_loss: 1.7353 - val_acc: 0.4354\n",
      "10000/10000 [==============================] - 2s 161us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [512, 128], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.4400050224284663, 'lr': 0.0016639402121471965}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 12s 291us/step - loss: 2.2424 - acc: 0.2899 - val_loss: 2.1836 - val_acc: 0.2902\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.6833 - acc: 0.3997 - val_loss: 2.3805 - val_acc: 0.2734\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.5036 - acc: 0.4573 - val_loss: 2.9473 - val_acc: 0.2426\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.4042 - acc: 0.4921 - val_loss: 2.3187 - val_acc: 0.3252\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.3256 - acc: 0.5237 - val_loss: 3.0319 - val_acc: 0.2445\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.2621 - acc: 0.5443 - val_loss: 2.4528 - val_acc: 0.3285\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.2228 - acc: 0.5629 - val_loss: 2.7829 - val_acc: 0.2863\n",
      "10000/10000 [==============================] - 2s 157us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [512, 128], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.4446573511039948, 'lr': 0.00037127026085325164}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 12s 297us/step - loss: 2.7356 - acc: 0.2158 - val_loss: 1.8025 - val_acc: 0.3528\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 2.2134 - acc: 0.3010 - val_loss: 1.6652 - val_acc: 0.3994\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 2.0022 - acc: 0.3387 - val_loss: 1.6465 - val_acc: 0.4028\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.8727 - acc: 0.3622 - val_loss: 1.7617 - val_acc: 0.3671\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.7641 - acc: 0.3901 - val_loss: 1.7993 - val_acc: 0.3683\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.6914 - acc: 0.4099 - val_loss: 1.7484 - val_acc: 0.3744\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.6197 - acc: 0.4282 - val_loss: 1.7403 - val_acc: 0.3802\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.5579 - acc: 0.4475 - val_loss: 1.7987 - val_acc: 0.3684\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.5177 - acc: 0.4583 - val_loss: 1.8033 - val_acc: 0.3772\n",
      "10000/10000 [==============================] - 2s 157us/step\n",
      "40000/40000 [==============================] - 7s 185us/step\n",
      "10000/10000 [==============================] - 2s 199us/step\n",
      "Loss = 0.8490020796775818\n",
      "Test Accuracy = 0.727\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [1024, 512], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.30384883931835144, 'lr': 0.001724445770930159}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 13s 319us/step - loss: 1.9286 - acc: 0.3735 - val_loss: 4.2455 - val_acc: 0.1693\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.4235 - acc: 0.4977 - val_loss: 3.8064 - val_acc: 0.2155\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 152us/step - loss: 1.2573 - acc: 0.5534 - val_loss: 6.1672 - val_acc: 0.1385\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 152us/step - loss: 1.1450 - acc: 0.5950 - val_loss: 4.1637 - val_acc: 0.2104\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 152us/step - loss: 1.0583 - acc: 0.6212 - val_loss: 5.0364 - val_acc: 0.1738\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.0068 - acc: 0.6427 - val_loss: 2.8505 - val_acc: 0.3307\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 152us/step - loss: 0.9469 - acc: 0.6667 - val_loss: 2.2287 - val_acc: 0.4317\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.9109 - acc: 0.6796 - val_loss: 1.8413 - val_acc: 0.4679\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.8845 - acc: 0.6870 - val_loss: 2.8856 - val_acc: 0.3706\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 6s 152us/step - loss: 0.8508 - acc: 0.6993 - val_loss: 1.6498 - val_acc: 0.5076\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 6s 152us/step - loss: 0.8260 - acc: 0.7094 - val_loss: 1.1083 - val_acc: 0.6387\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 6s 152us/step - loss: 0.7828 - acc: 0.7242 - val_loss: 1.4726 - val_acc: 0.5498\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 6s 152us/step - loss: 0.7684 - acc: 0.7290 - val_loss: 1.2670 - val_acc: 0.6114\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 6s 152us/step - loss: 0.7453 - acc: 0.7368 - val_loss: 3.0368 - val_acc: 0.3587\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 6s 152us/step - loss: 0.7348 - acc: 0.7400 - val_loss: 1.1374 - val_acc: 0.6561\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 6s 152us/step - loss: 0.7026 - acc: 0.7509 - val_loss: 1.2855 - val_acc: 0.6014\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 6s 152us/step - loss: 0.7001 - acc: 0.7514 - val_loss: 1.6328 - val_acc: 0.5500\n",
      "10000/10000 [==============================] - 2s 167us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [1024, 512], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.32099496933893135, 'lr': 0.0008208271000907531}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 13s 328us/step - loss: 2.0857 - acc: 0.3426 - val_loss: 1.5847 - val_acc: 0.4514\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.5684 - acc: 0.4558 - val_loss: 2.6197 - val_acc: 0.2910\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 152us/step - loss: 1.4053 - acc: 0.5050 - val_loss: 3.8428 - val_acc: 0.1935\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.2869 - acc: 0.5436 - val_loss: 3.0581 - val_acc: 0.2547\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 152us/step - loss: 1.2020 - acc: 0.5764 - val_loss: 3.2470 - val_acc: 0.2584\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.1270 - acc: 0.6048 - val_loss: 2.8265 - val_acc: 0.2924\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 152us/step - loss: 1.0742 - acc: 0.6193 - val_loss: 2.0963 - val_acc: 0.4071\n",
      "10000/10000 [==============================] - 2s 167us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [1024, 512], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.39215507003988975, 'lr': 0.0009011895453836491}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 13s 336us/step - loss: 2.2493 - acc: 0.3068 - val_loss: 1.9419 - val_acc: 0.3533\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 152us/step - loss: 1.7168 - acc: 0.4097 - val_loss: 2.6200 - val_acc: 0.2682\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.5251 - acc: 0.4637 - val_loss: 3.9504 - val_acc: 0.1910\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.4177 - acc: 0.4948 - val_loss: 2.9273 - val_acc: 0.2565\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.3215 - acc: 0.5259 - val_loss: 3.8322 - val_acc: 0.1910\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 152us/step - loss: 1.2573 - acc: 0.5503 - val_loss: 2.9442 - val_acc: 0.2707\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.1858 - acc: 0.5758 - val_loss: 4.3138 - val_acc: 0.1835\n",
      "10000/10000 [==============================] - 2s 167us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [1024, 512], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.4822043402637011, 'lr': 0.00039140507856517774}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 14s 345us/step - loss: 2.7962 - acc: 0.2203 - val_loss: 1.7000 - val_acc: 0.3870\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 2.2459 - acc: 0.3087 - val_loss: 1.7582 - val_acc: 0.3596\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 152us/step - loss: 2.0026 - acc: 0.3459 - val_loss: 1.9283 - val_acc: 0.3247\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.8669 - acc: 0.3740 - val_loss: 1.8598 - val_acc: 0.3578\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.7519 - acc: 0.3999 - val_loss: 2.0410 - val_acc: 0.3200\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 152us/step - loss: 1.6722 - acc: 0.4191 - val_loss: 2.1227 - val_acc: 0.3237\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.6141 - acc: 0.4304 - val_loss: 2.4570 - val_acc: 0.2606\n",
      "10000/10000 [==============================] - 2s 170us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [1024, 512], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.40124888640805767, 'lr': 0.0008089265511129759}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 14s 354us/step - loss: 2.3361 - acc: 0.2928 - val_loss: 2.1636 - val_acc: 0.3092\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.7942 - acc: 0.3914 - val_loss: 3.4380 - val_acc: 0.1564\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.6033 - acc: 0.4408 - val_loss: 4.2037 - val_acc: 0.1523\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.4785 - acc: 0.4785 - val_loss: 4.4480 - val_acc: 0.1542\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.3895 - acc: 0.5086 - val_loss: 3.9832 - val_acc: 0.1806\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.3107 - acc: 0.5321 - val_loss: 4.3394 - val_acc: 0.1527\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.2459 - acc: 0.5575 - val_loss: 4.5862 - val_acc: 0.1468\n",
      "10000/10000 [==============================] - 2s 177us/step\n",
      "40000/40000 [==============================] - 8s 204us/step\n",
      "10000/10000 [==============================] - 2s 231us/step\n",
      "Loss = 1.6726067478179931\n",
      "Test Accuracy = 0.5463\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [4096, 2048], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.4829091693968846, 'lr': 0.0004487591308800182}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 18s 452us/step - loss: 2.7262 - acc: 0.2677 - val_loss: 2.4869 - val_acc: 0.2475\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 9s 231us/step - loss: 2.1577 - acc: 0.3542 - val_loss: 2.0709 - val_acc: 0.3502\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.8891 - acc: 0.3926 - val_loss: 2.0717 - val_acc: 0.3531\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 9s 231us/step - loss: 1.7405 - acc: 0.4206 - val_loss: 2.6661 - val_acc: 0.2783\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 9s 231us/step - loss: 1.6273 - acc: 0.4467 - val_loss: 2.5876 - val_acc: 0.2798\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 1.5435 - acc: 0.4689 - val_loss: 3.7641 - val_acc: 0.1944\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 1.4695 - acc: 0.4880 - val_loss: 3.8911 - val_acc: 0.1808\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.4089 - acc: 0.5092 - val_loss: 4.1231 - val_acc: 0.1694\n",
      "10000/10000 [==============================] - 2s 219us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [4096, 2048], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.2645741183769343, 'lr': 0.000609564987918484}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 18s 458us/step - loss: 2.1394 - acc: 0.3770 - val_loss: 2.1824 - val_acc: 0.4041\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.5481 - acc: 0.4812 - val_loss: 3.5281 - val_acc: 0.2542\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 9s 231us/step - loss: 1.3537 - acc: 0.5351 - val_loss: 3.5923 - val_acc: 0.2767\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 9s 231us/step - loss: 1.2567 - acc: 0.5652 - val_loss: 4.2582 - val_acc: 0.2138\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 9s 230us/step - loss: 1.1660 - acc: 0.5936 - val_loss: 2.6285 - val_acc: 0.3480\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 9s 231us/step - loss: 1.1050 - acc: 0.6148 - val_loss: 3.1094 - val_acc: 0.3307\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 9s 230us/step - loss: 1.0397 - acc: 0.6361 - val_loss: 2.4090 - val_acc: 0.3943\n",
      "10000/10000 [==============================] - 2s 225us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [4096, 2048], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.4213631111189404, 'lr': 0.0012295932212156217}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 19s 468us/step - loss: 2.3974 - acc: 0.3322 - val_loss: 3.3709 - val_acc: 0.2912\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 1.6515 - acc: 0.4414 - val_loss: 3.5741 - val_acc: 0.1969\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 1.4781 - acc: 0.4886 - val_loss: 5.1976 - val_acc: 0.1273\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 9s 231us/step - loss: 1.3613 - acc: 0.5283 - val_loss: 7.6521 - val_acc: 0.1080\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.2709 - acc: 0.5534 - val_loss: 6.2278 - val_acc: 0.1170\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 1.2027 - acc: 0.5825 - val_loss: 5.5416 - val_acc: 0.1283\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 1.1329 - acc: 0.6015 - val_loss: 3.5981 - val_acc: 0.2354\n",
      "10000/10000 [==============================] - 2s 227us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [4096, 2048], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.43170098770292853, 'lr': 0.0008131518681112914}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 19s 476us/step - loss: 2.5015 - acc: 0.3085 - val_loss: 2.7254 - val_acc: 0.3241\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.7828 - acc: 0.4092 - val_loss: 4.9599 - val_acc: 0.1806\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.5822 - acc: 0.4576 - val_loss: 6.3875 - val_acc: 0.1291\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.4787 - acc: 0.4943 - val_loss: 7.5868 - val_acc: 0.1132\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.3851 - acc: 0.5192 - val_loss: 7.3270 - val_acc: 0.1140\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.3232 - acc: 0.5389 - val_loss: 7.4412 - val_acc: 0.1065\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.2453 - acc: 0.5659 - val_loss: 5.6869 - val_acc: 0.1234\n",
      "10000/10000 [==============================] - 2s 225us/step\n",
      "{'classes': 10, 'activation': 'relu', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [4096, 2048], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.3730121546701246, 'lr': 0.0005956127065924172}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 19s 482us/step - loss: 2.3949 - acc: 0.3194 - val_loss: 2.1888 - val_acc: 0.3247\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.8000 - acc: 0.4096 - val_loss: 2.6078 - val_acc: 0.3120\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.6033 - acc: 0.4585 - val_loss: 3.2153 - val_acc: 0.2587\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.4818 - acc: 0.4918 - val_loss: 3.8426 - val_acc: 0.1942\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.3925 - acc: 0.5168 - val_loss: 4.5255 - val_acc: 0.1849\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.3076 - acc: 0.5430 - val_loss: 4.7307 - val_acc: 0.1804\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.2355 - acc: 0.5669 - val_loss: 4.0437 - val_acc: 0.1984\n",
      "10000/10000 [==============================] - 2s 224us/step\n",
      "40000/40000 [==============================] - 11s 279us/step\n",
      "10000/10000 [==============================] - 3s 325us/step\n",
      "Loss = 2.386035181427002\n",
      "Test Accuracy = 0.3934\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [512, 128], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.20496809648271097, 'lr': 0.001284025107280471}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 16s 390us/step - loss: 1.8266 - acc: 0.3853 - val_loss: 2.0190 - val_acc: 0.4167\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 1.4112 - acc: 0.5000 - val_loss: 2.0546 - val_acc: 0.3961\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 1.2533 - acc: 0.5540 - val_loss: 3.0567 - val_acc: 0.2960\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.1497 - acc: 0.5871 - val_loss: 2.7827 - val_acc: 0.3096\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 1.0752 - acc: 0.6190 - val_loss: 1.5588 - val_acc: 0.5059\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 1.0088 - acc: 0.6429 - val_loss: 1.8737 - val_acc: 0.4725\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.9599 - acc: 0.6616 - val_loss: 1.8196 - val_acc: 0.4963\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.9156 - acc: 0.6786 - val_loss: 1.9227 - val_acc: 0.4612\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.8775 - acc: 0.6907 - val_loss: 1.7967 - val_acc: 0.5078\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.8576 - acc: 0.6945 - val_loss: 2.1795 - val_acc: 0.4262\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.8188 - acc: 0.7101 - val_loss: 2.4864 - val_acc: 0.3776\n",
      "10000/10000 [==============================] - 2s 165us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [512, 128], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.3194351924753132, 'lr': 0.0015362829079122705}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 16s 396us/step - loss: 1.9928 - acc: 0.3417 - val_loss: 2.0886 - val_acc: 0.3782\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.5371 - acc: 0.4519 - val_loss: 2.2526 - val_acc: 0.3635\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.3508 - acc: 0.5167 - val_loss: 2.5018 - val_acc: 0.3272\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 1.2577 - acc: 0.5507 - val_loss: 1.8022 - val_acc: 0.4448\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.1721 - acc: 0.5816 - val_loss: 3.5421 - val_acc: 0.2356\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.1232 - acc: 0.5987 - val_loss: 1.7797 - val_acc: 0.4807\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.0784 - acc: 0.6159 - val_loss: 1.2405 - val_acc: 0.5717\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.0473 - acc: 0.6280 - val_loss: 3.0603 - val_acc: 0.2978\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.0127 - acc: 0.6431 - val_loss: 2.1226 - val_acc: 0.4391\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.9768 - acc: 0.6537 - val_loss: 1.3621 - val_acc: 0.5695\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.9625 - acc: 0.6603 - val_loss: 1.7738 - val_acc: 0.4787\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.9260 - acc: 0.6741 - val_loss: 1.0501 - val_acc: 0.6410\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.8984 - acc: 0.6818 - val_loss: 1.8374 - val_acc: 0.4645\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.8824 - acc: 0.6866 - val_loss: 1.4423 - val_acc: 0.5646\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.8643 - acc: 0.6949 - val_loss: 1.0302 - val_acc: 0.6433\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.8382 - acc: 0.7029 - val_loss: 1.4621 - val_acc: 0.5474\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.8285 - acc: 0.7065 - val_loss: 0.9764 - val_acc: 0.6768\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.8031 - acc: 0.7162 - val_loss: 1.1327 - val_acc: 0.6346\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.7974 - acc: 0.7185 - val_loss: 1.0978 - val_acc: 0.6490\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.7754 - acc: 0.7256 - val_loss: 0.8509 - val_acc: 0.7085\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.7463 - acc: 0.7370 - val_loss: 0.9618 - val_acc: 0.6835\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.7319 - acc: 0.7429 - val_loss: 1.2941 - val_acc: 0.5894\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.7201 - acc: 0.7454 - val_loss: 1.0228 - val_acc: 0.6556\n",
      "Epoch 24/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.7051 - acc: 0.7533 - val_loss: 1.0086 - val_acc: 0.6734\n",
      "Epoch 25/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.6915 - acc: 0.7530 - val_loss: 1.0333 - val_acc: 0.6574\n",
      "Epoch 26/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.6766 - acc: 0.7614 - val_loss: 0.9651 - val_acc: 0.6916\n",
      "10000/10000 [==============================] - 2s 171us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [512, 128], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.22701095469997057, 'lr': 0.0010729397475353014}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 16s 400us/step - loss: 1.9085 - acc: 0.3705 - val_loss: 2.4216 - val_acc: 0.3568\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 1.4758 - acc: 0.4789 - val_loss: 1.6956 - val_acc: 0.4329\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.3082 - acc: 0.5326 - val_loss: 2.5947 - val_acc: 0.3178\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.2015 - acc: 0.5733 - val_loss: 1.5040 - val_acc: 0.5211\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.1401 - acc: 0.5951 - val_loss: 1.3824 - val_acc: 0.5418\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.0717 - acc: 0.6188 - val_loss: 1.8758 - val_acc: 0.4502\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.0332 - acc: 0.6311 - val_loss: 1.6029 - val_acc: 0.5151\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.9882 - acc: 0.6495 - val_loss: 1.5454 - val_acc: 0.5059\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.9407 - acc: 0.6691 - val_loss: 1.3068 - val_acc: 0.5696\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.9140 - acc: 0.6749 - val_loss: 1.1313 - val_acc: 0.6233\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.8884 - acc: 0.6881 - val_loss: 3.3833 - val_acc: 0.2765\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.8859 - acc: 0.6851 - val_loss: 1.8591 - val_acc: 0.4902\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.8510 - acc: 0.6982 - val_loss: 1.3630 - val_acc: 0.5618\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.8239 - acc: 0.7082 - val_loss: 1.0831 - val_acc: 0.6210\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.7927 - acc: 0.7194 - val_loss: 1.6102 - val_acc: 0.5035\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.7828 - acc: 0.7239 - val_loss: 1.3127 - val_acc: 0.5835\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.7586 - acc: 0.7328 - val_loss: 1.3819 - val_acc: 0.5661\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.7428 - acc: 0.7353 - val_loss: 0.9364 - val_acc: 0.6778\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.7178 - acc: 0.7463 - val_loss: 1.1400 - val_acc: 0.6108\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.7032 - acc: 0.7514 - val_loss: 1.4714 - val_acc: 0.5341\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.6892 - acc: 0.7565 - val_loss: 1.3407 - val_acc: 0.5903\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.6733 - acc: 0.7604 - val_loss: 0.9090 - val_acc: 0.6870\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.6501 - acc: 0.7702 - val_loss: 1.2054 - val_acc: 0.6155\n",
      "Epoch 24/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.6419 - acc: 0.7738 - val_loss: 0.8639 - val_acc: 0.7091\n",
      "Epoch 25/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.6182 - acc: 0.7815 - val_loss: 0.8985 - val_acc: 0.6977\n",
      "Epoch 26/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.6230 - acc: 0.7791 - val_loss: 0.8647 - val_acc: 0.7171\n",
      "Epoch 27/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.5950 - acc: 0.7891 - val_loss: 1.7063 - val_acc: 0.5975\n",
      "Epoch 28/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.5828 - acc: 0.7931 - val_loss: 1.0484 - val_acc: 0.6628\n",
      "Epoch 29/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.5744 - acc: 0.7956 - val_loss: 1.0560 - val_acc: 0.6814\n",
      "Epoch 30/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.5511 - acc: 0.8069 - val_loss: 0.9002 - val_acc: 0.7244\n",
      "10000/10000 [==============================] - 2s 170us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [512, 128], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.30881185951463624, 'lr': 0.0023357953400588347}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 16s 406us/step - loss: 1.8768 - acc: 0.3710 - val_loss: 3.5979 - val_acc: 0.2131\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.4472 - acc: 0.4794 - val_loss: 8.8023 - val_acc: 0.1102\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.2934 - acc: 0.5368 - val_loss: 2.5246 - val_acc: 0.3497\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.1824 - acc: 0.5796 - val_loss: 2.9021 - val_acc: 0.3226\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 1.1219 - acc: 0.6008 - val_loss: 3.7857 - val_acc: 0.2344\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.0738 - acc: 0.6197 - val_loss: 1.5127 - val_acc: 0.5063\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.0190 - acc: 0.6380 - val_loss: 1.7095 - val_acc: 0.4783\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.9854 - acc: 0.6508 - val_loss: 2.2143 - val_acc: 0.4452\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.9583 - acc: 0.6585 - val_loss: 1.5394 - val_acc: 0.5210\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.9152 - acc: 0.6776 - val_loss: 1.3944 - val_acc: 0.5248\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.8944 - acc: 0.6845 - val_loss: 1.0022 - val_acc: 0.6465\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.8596 - acc: 0.6972 - val_loss: 1.4570 - val_acc: 0.5500\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.8452 - acc: 0.7008 - val_loss: 1.1280 - val_acc: 0.6289\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.8139 - acc: 0.7118 - val_loss: 1.1498 - val_acc: 0.6198\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.7946 - acc: 0.7198 - val_loss: 0.9797 - val_acc: 0.6658\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.7763 - acc: 0.7267 - val_loss: 1.1578 - val_acc: 0.6363\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.7511 - acc: 0.7371 - val_loss: 0.9245 - val_acc: 0.6975\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.7397 - acc: 0.7394 - val_loss: 2.0453 - val_acc: 0.4486\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.7455 - acc: 0.7351 - val_loss: 2.1117 - val_acc: 0.4810\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.7250 - acc: 0.7436 - val_loss: 0.8123 - val_acc: 0.7192\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.6710 - acc: 0.7613 - val_loss: 0.9102 - val_acc: 0.6928\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.6827 - acc: 0.7603 - val_loss: 1.1211 - val_acc: 0.6663\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 6s 141us/step - loss: 0.6586 - acc: 0.7678 - val_loss: 0.8866 - val_acc: 0.7101\n",
      "Epoch 24/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.6410 - acc: 0.7723 - val_loss: 0.8859 - val_acc: 0.7146\n",
      "Epoch 25/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.6164 - acc: 0.7810 - val_loss: 0.9241 - val_acc: 0.6952\n",
      "Epoch 26/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 0.6104 - acc: 0.7851 - val_loss: 1.0773 - val_acc: 0.6765\n",
      "10000/10000 [==============================] - 2s 167us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [512, 128], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.3071330347349348, 'lr': 0.0007602116233028295}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 17s 415us/step - loss: 2.1660 - acc: 0.3072 - val_loss: 2.2835 - val_acc: 0.3315\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.6773 - acc: 0.4189 - val_loss: 1.9216 - val_acc: 0.4177\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.5070 - acc: 0.4691 - val_loss: 2.1237 - val_acc: 0.3772\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.3798 - acc: 0.5108 - val_loss: 2.2195 - val_acc: 0.3921\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.2991 - acc: 0.5378 - val_loss: 3.5051 - val_acc: 0.2481\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.2345 - acc: 0.5605 - val_loss: 2.5409 - val_acc: 0.3366\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.1878 - acc: 0.5784 - val_loss: 2.6016 - val_acc: 0.3688\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.1319 - acc: 0.5960 - val_loss: 3.7719 - val_acc: 0.2395\n",
      "10000/10000 [==============================] - 2s 169us/step\n",
      "40000/40000 [==============================] - 8s 197us/step\n",
      "10000/10000 [==============================] - 2s 210us/step\n",
      "Loss = 0.9056935463905335\n",
      "Test Accuracy = 0.7236\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [1024, 512], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.3485743876173286, 'lr': 0.0020666512566460946}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 17s 435us/step - loss: 2.0119 - acc: 0.3561 - val_loss: 3.9585 - val_acc: 0.2634\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.4764 - acc: 0.4757 - val_loss: 2.6984 - val_acc: 0.3630\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.3065 - acc: 0.5317 - val_loss: 5.2264 - val_acc: 0.2304\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.2067 - acc: 0.5696 - val_loss: 11.7992 - val_acc: 0.1011\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.1329 - acc: 0.5948 - val_loss: 4.1499 - val_acc: 0.2515\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.0589 - acc: 0.6232 - val_loss: 3.1453 - val_acc: 0.3270\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.0090 - acc: 0.6393 - val_loss: 6.1016 - val_acc: 0.1672\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.9886 - acc: 0.6504 - val_loss: 3.1142 - val_acc: 0.3518\n",
      "10000/10000 [==============================] - 2s 179us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [1024, 512], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.2325002392090742, 'lr': 0.0010902740612750083}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 18s 446us/step - loss: 1.9119 - acc: 0.3814 - val_loss: 2.3220 - val_acc: 0.4015\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.4314 - acc: 0.4997 - val_loss: 3.8569 - val_acc: 0.2490\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.2670 - acc: 0.5515 - val_loss: 3.5089 - val_acc: 0.3236\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.1489 - acc: 0.5938 - val_loss: 4.0143 - val_acc: 0.2440\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.0734 - acc: 0.6194 - val_loss: 2.9417 - val_acc: 0.3214\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 1.0035 - acc: 0.6455 - val_loss: 1.5447 - val_acc: 0.5292\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 0.9395 - acc: 0.6675 - val_loss: 3.2420 - val_acc: 0.2914\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.8954 - acc: 0.6837 - val_loss: 1.5679 - val_acc: 0.5252\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.8584 - acc: 0.6985 - val_loss: 1.1225 - val_acc: 0.6301\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.8175 - acc: 0.7101 - val_loss: 1.0445 - val_acc: 0.6557\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.7799 - acc: 0.7234 - val_loss: 1.3663 - val_acc: 0.5787\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.7539 - acc: 0.7348 - val_loss: 1.3695 - val_acc: 0.5941\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.7233 - acc: 0.7452 - val_loss: 1.3050 - val_acc: 0.5841\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.7013 - acc: 0.7525 - val_loss: 0.8166 - val_acc: 0.7183\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.6626 - acc: 0.7639 - val_loss: 2.3902 - val_acc: 0.4228\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.6767 - acc: 0.7597 - val_loss: 0.9566 - val_acc: 0.6802\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.6252 - acc: 0.7787 - val_loss: 0.8316 - val_acc: 0.7200\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.6049 - acc: 0.7873 - val_loss: 1.0204 - val_acc: 0.6836\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.5840 - acc: 0.7930 - val_loss: 0.8934 - val_acc: 0.7092\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.5671 - acc: 0.7999 - val_loss: 0.9440 - val_acc: 0.7075\n",
      "10000/10000 [==============================] - 2s 184us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [1024, 512], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.2496481954617537, 'lr': 0.0006604330122112726}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 18s 454us/step - loss: 2.0169 - acc: 0.3598 - val_loss: 2.8875 - val_acc: 0.3073\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.5394 - acc: 0.4701 - val_loss: 3.5401 - val_acc: 0.2460\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 1.3754 - acc: 0.5174 - val_loss: 5.5190 - val_acc: 0.1526\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.2700 - acc: 0.5509 - val_loss: 3.7937 - val_acc: 0.2363\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.1876 - acc: 0.5776 - val_loss: 3.5671 - val_acc: 0.2728\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.1190 - acc: 0.5986 - val_loss: 4.6008 - val_acc: 0.1991\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.0682 - acc: 0.6192 - val_loss: 3.1588 - val_acc: 0.3057\n",
      "10000/10000 [==============================] - 2s 182us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [1024, 512], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.47681004725320975, 'lr': 0.0011511062654590803}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 18s 457us/step - loss: 2.4720 - acc: 0.2789 - val_loss: 3.6919 - val_acc: 0.2464\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.8230 - acc: 0.3841 - val_loss: 3.3921 - val_acc: 0.2706\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.6126 - acc: 0.4341 - val_loss: 3.5455 - val_acc: 0.2265\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.4780 - acc: 0.4704 - val_loss: 2.5975 - val_acc: 0.3262\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.3951 - acc: 0.4994 - val_loss: 4.7451 - val_acc: 0.1645\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.3188 - acc: 0.5219 - val_loss: 4.6049 - val_acc: 0.2068\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.2635 - acc: 0.5466 - val_loss: 2.9425 - val_acc: 0.2745\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.2237 - acc: 0.5602 - val_loss: 1.7748 - val_acc: 0.4443\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.1856 - acc: 0.5749 - val_loss: 2.4961 - val_acc: 0.3447\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.1488 - acc: 0.5873 - val_loss: 3.5611 - val_acc: 0.2396\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.1108 - acc: 0.6026 - val_loss: 1.7473 - val_acc: 0.4606\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.0875 - acc: 0.6103 - val_loss: 1.4830 - val_acc: 0.5032\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.0623 - acc: 0.6204 - val_loss: 1.5545 - val_acc: 0.5134\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.0357 - acc: 0.6319 - val_loss: 1.5478 - val_acc: 0.4848\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.0100 - acc: 0.6401 - val_loss: 1.4766 - val_acc: 0.5293\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.9903 - acc: 0.6448 - val_loss: 1.4107 - val_acc: 0.5409\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.9808 - acc: 0.6518 - val_loss: 1.4002 - val_acc: 0.5291\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.9540 - acc: 0.6623 - val_loss: 1.4547 - val_acc: 0.5416\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.9376 - acc: 0.6665 - val_loss: 1.6114 - val_acc: 0.5101\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.9196 - acc: 0.6722 - val_loss: 1.4773 - val_acc: 0.5313\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.9040 - acc: 0.6776 - val_loss: 1.0251 - val_acc: 0.6394\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.8897 - acc: 0.6814 - val_loss: 1.6054 - val_acc: 0.5219\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.8715 - acc: 0.6887 - val_loss: 1.2923 - val_acc: 0.5682\n",
      "Epoch 24/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.8579 - acc: 0.6967 - val_loss: 1.1161 - val_acc: 0.6223\n",
      "Epoch 25/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.8430 - acc: 0.7020 - val_loss: 3.3575 - val_acc: 0.2953\n",
      "Epoch 26/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.8442 - acc: 0.6988 - val_loss: 0.9302 - val_acc: 0.6725\n",
      "Epoch 27/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.8128 - acc: 0.7133 - val_loss: 1.3995 - val_acc: 0.5632\n",
      "Epoch 28/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.8111 - acc: 0.7131 - val_loss: 1.0977 - val_acc: 0.6250\n",
      "Epoch 29/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.7926 - acc: 0.7190 - val_loss: 1.4284 - val_acc: 0.5575\n",
      "Epoch 30/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.7880 - acc: 0.7213 - val_loss: 0.7885 - val_acc: 0.7228\n",
      "Epoch 31/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.7553 - acc: 0.7311 - val_loss: 0.8362 - val_acc: 0.7138\n",
      "Epoch 32/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.7473 - acc: 0.7370 - val_loss: 0.8578 - val_acc: 0.6983\n",
      "Epoch 33/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.7328 - acc: 0.7372 - val_loss: 1.0925 - val_acc: 0.6431\n",
      "Epoch 34/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.7182 - acc: 0.7450 - val_loss: 1.9935 - val_acc: 0.4749\n",
      "Epoch 35/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.7269 - acc: 0.7422 - val_loss: 0.9631 - val_acc: 0.6817\n",
      "Epoch 36/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 0.7047 - acc: 0.7463 - val_loss: 1.3911 - val_acc: 0.6171\n",
      "10000/10000 [==============================] - 2s 183us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [1024, 512], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.44345150333710526, 'lr': 0.00036625274486872406}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 19s 464us/step - loss: 2.7356 - acc: 0.2318 - val_loss: 1.9576 - val_acc: 0.3937\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 2.1822 - acc: 0.3180 - val_loss: 2.2442 - val_acc: 0.3623\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.9651 - acc: 0.3601 - val_loss: 2.6207 - val_acc: 0.3140\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.8335 - acc: 0.3824 - val_loss: 3.0191 - val_acc: 0.2914\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.7190 - acc: 0.4086 - val_loss: 3.1039 - val_acc: 0.2708\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.6414 - acc: 0.4284 - val_loss: 3.1985 - val_acc: 0.2682\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.5789 - acc: 0.4455 - val_loss: 2.6497 - val_acc: 0.3349\n",
      "10000/10000 [==============================] - 2s 185us/step\n",
      "40000/40000 [==============================] - 9s 224us/step\n",
      "10000/10000 [==============================] - 2s 229us/step\n",
      "Loss = 0.9439798512458801\n",
      "Test Accuracy = 0.7054\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [4096, 2048], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.3390219526766358, 'lr': 0.0003907755781806953}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 23s 569us/step - loss: 2.3758 - acc: 0.3277 - val_loss: 3.1297 - val_acc: 0.3402\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.8606 - acc: 0.4128 - val_loss: 3.6305 - val_acc: 0.2830\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 9s 231us/step - loss: 1.6300 - acc: 0.4572 - val_loss: 4.5334 - val_acc: 0.2502\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 9s 231us/step - loss: 1.4897 - acc: 0.4908 - val_loss: 4.4079 - val_acc: 0.2737\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.3916 - acc: 0.5204 - val_loss: 6.1531 - val_acc: 0.1564\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 1.3188 - acc: 0.5410 - val_loss: 4.4954 - val_acc: 0.2630\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.2516 - acc: 0.5635 - val_loss: 2.8991 - val_acc: 0.3633\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.1985 - acc: 0.5797 - val_loss: 4.8797 - val_acc: 0.2266\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.1595 - acc: 0.5931 - val_loss: 3.3689 - val_acc: 0.3160\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.1151 - acc: 0.6073 - val_loss: 3.5200 - val_acc: 0.3015\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 1.0791 - acc: 0.6214 - val_loss: 2.9658 - val_acc: 0.3371\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.0511 - acc: 0.6275 - val_loss: 2.7653 - val_acc: 0.3646\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.0095 - acc: 0.6432 - val_loss: 3.0203 - val_acc: 0.3299\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 9s 231us/step - loss: 0.9835 - acc: 0.6524 - val_loss: 2.8015 - val_acc: 0.3667\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.9618 - acc: 0.6601 - val_loss: 1.8819 - val_acc: 0.4898\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 0.9285 - acc: 0.6731 - val_loss: 1.5049 - val_acc: 0.5376\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.9052 - acc: 0.6781 - val_loss: 1.7973 - val_acc: 0.4994\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.8824 - acc: 0.6893 - val_loss: 1.5988 - val_acc: 0.5318\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 0.8557 - acc: 0.6966 - val_loss: 2.0841 - val_acc: 0.4592\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 9s 231us/step - loss: 0.8438 - acc: 0.7031 - val_loss: 1.4534 - val_acc: 0.5682\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 9s 231us/step - loss: 0.8133 - acc: 0.7140 - val_loss: 1.3308 - val_acc: 0.5913\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 9s 231us/step - loss: 0.8016 - acc: 0.7178 - val_loss: 1.9485 - val_acc: 0.4664\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.7798 - acc: 0.7233 - val_loss: 1.2067 - val_acc: 0.6168\n",
      "Epoch 24/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.7544 - acc: 0.7308 - val_loss: 1.2773 - val_acc: 0.5968\n",
      "Epoch 25/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.7325 - acc: 0.7404 - val_loss: 1.1120 - val_acc: 0.6415\n",
      "Epoch 26/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.7158 - acc: 0.7452 - val_loss: 1.0476 - val_acc: 0.6616\n",
      "Epoch 27/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 0.6933 - acc: 0.7525 - val_loss: 1.0336 - val_acc: 0.6629\n",
      "Epoch 28/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 0.6809 - acc: 0.7579 - val_loss: 1.1790 - val_acc: 0.6315\n",
      "Epoch 29/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.6637 - acc: 0.7651 - val_loss: 0.9098 - val_acc: 0.6984\n",
      "Epoch 30/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.6376 - acc: 0.7729 - val_loss: 0.9431 - val_acc: 0.6906\n",
      "Epoch 31/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.6253 - acc: 0.7779 - val_loss: 0.9690 - val_acc: 0.6848\n",
      "Epoch 32/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.6022 - acc: 0.7839 - val_loss: 0.8908 - val_acc: 0.7044\n",
      "Epoch 33/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 0.5788 - acc: 0.7911 - val_loss: 1.1740 - val_acc: 0.6417\n",
      "Epoch 34/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 0.5586 - acc: 0.8015 - val_loss: 1.1872 - val_acc: 0.6501\n",
      "Epoch 35/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.5482 - acc: 0.8063 - val_loss: 1.8462 - val_acc: 0.5347\n",
      "Epoch 36/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.5299 - acc: 0.8103 - val_loss: 1.8196 - val_acc: 0.5376\n",
      "Epoch 37/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.5173 - acc: 0.8141 - val_loss: 1.3900 - val_acc: 0.6147\n",
      "Epoch 38/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.4911 - acc: 0.8243 - val_loss: 1.2498 - val_acc: 0.6326\n",
      "10000/10000 [==============================] - 2s 238us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [4096, 2048], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.2179302936435531, 'lr': 0.0010674332489462855}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 23s 575us/step - loss: 2.1652 - acc: 0.3859 - val_loss: 3.4144 - val_acc: 0.3917\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 9s 237us/step - loss: 1.3927 - acc: 0.5123 - val_loss: 3.3545 - val_acc: 0.3705\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 9s 236us/step - loss: 1.2774 - acc: 0.5578 - val_loss: 4.2625 - val_acc: 0.2783\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 1.1777 - acc: 0.5913 - val_loss: 3.1072 - val_acc: 0.3610\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.0826 - acc: 0.6220 - val_loss: 4.4874 - val_acc: 0.2518\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.0276 - acc: 0.6438 - val_loss: 2.4574 - val_acc: 0.4278\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.9612 - acc: 0.6606 - val_loss: 3.1364 - val_acc: 0.3199\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.9175 - acc: 0.6751 - val_loss: 1.7392 - val_acc: 0.5054\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 9s 236us/step - loss: 0.8567 - acc: 0.6968 - val_loss: 1.3891 - val_acc: 0.5704\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 9s 236us/step - loss: 0.8182 - acc: 0.7123 - val_loss: 1.9241 - val_acc: 0.4758\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 9s 236us/step - loss: 0.7766 - acc: 0.7281 - val_loss: 1.0823 - val_acc: 0.6377\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 9s 236us/step - loss: 0.7312 - acc: 0.7413 - val_loss: 2.3016 - val_acc: 0.4504\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 9s 236us/step - loss: 0.6903 - acc: 0.7565 - val_loss: 1.4865 - val_acc: 0.5657\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 9s 236us/step - loss: 0.6434 - acc: 0.7724 - val_loss: 0.9689 - val_acc: 0.6839\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 9s 236us/step - loss: 0.6053 - acc: 0.7858 - val_loss: 1.8142 - val_acc: 0.5105\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 0.5691 - acc: 0.7988 - val_loss: 0.9987 - val_acc: 0.6880\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.5318 - acc: 0.8106 - val_loss: 1.1945 - val_acc: 0.6438\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 0.5176 - acc: 0.8154 - val_loss: 1.1882 - val_acc: 0.6331\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 9s 236us/step - loss: 0.4654 - acc: 0.8339 - val_loss: 0.9466 - val_acc: 0.7020\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 9s 236us/step - loss: 0.4548 - acc: 0.8383 - val_loss: 1.1969 - val_acc: 0.6547\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 9s 236us/step - loss: 0.4518 - acc: 0.8401 - val_loss: 1.2525 - val_acc: 0.6696\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 0.3836 - acc: 0.8638 - val_loss: 1.1230 - val_acc: 0.6962\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 0.3413 - acc: 0.8785 - val_loss: 1.0174 - val_acc: 0.7069\n",
      "Epoch 24/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.3167 - acc: 0.8891 - val_loss: 1.1620 - val_acc: 0.6922\n",
      "Epoch 25/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 0.2937 - acc: 0.8951 - val_loss: 1.6325 - val_acc: 0.6340\n",
      "10000/10000 [==============================] - 2s 238us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [4096, 2048], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.45173380675120367, 'lr': 0.00035691206743074136}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 23s 579us/step - loss: 2.7125 - acc: 0.2655 - val_loss: 3.1035 - val_acc: 0.3035\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 2.2257 - acc: 0.3396 - val_loss: 3.3100 - val_acc: 0.3015\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.9726 - acc: 0.3770 - val_loss: 3.3836 - val_acc: 0.2867\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.8097 - acc: 0.4041 - val_loss: 4.3994 - val_acc: 0.2388\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 1.6935 - acc: 0.4292 - val_loss: 4.0964 - val_acc: 0.2343\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 9s 236us/step - loss: 1.5926 - acc: 0.4542 - val_loss: 4.4378 - val_acc: 0.2306\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 9s 236us/step - loss: 1.5221 - acc: 0.4745 - val_loss: 4.8792 - val_acc: 0.1996\n",
      "10000/10000 [==============================] - 2s 238us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [4096, 2048], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.27878391398468894, 'lr': 0.0006432338664489708}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 24s 588us/step - loss: 2.2616 - acc: 0.3590 - val_loss: 3.3094 - val_acc: 0.3502\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 1.6135 - acc: 0.4633 - val_loss: 7.6533 - val_acc: 0.1724\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 1.3986 - acc: 0.5199 - val_loss: 7.7554 - val_acc: 0.1667\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.3015 - acc: 0.5484 - val_loss: 4.4957 - val_acc: 0.2337\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 1.2015 - acc: 0.5801 - val_loss: 4.8064 - val_acc: 0.2247\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 9s 237us/step - loss: 1.1278 - acc: 0.6026 - val_loss: 3.8412 - val_acc: 0.2711\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 9s 236us/step - loss: 1.0682 - acc: 0.6240 - val_loss: 5.6655 - val_acc: 0.1737\n",
      "10000/10000 [==============================] - 2s 239us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'layers': [4096, 2048], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.3350159688194425, 'lr': 0.0019030136918495495}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 24s 598us/step - loss: 2.4359 - acc: 0.3466 - val_loss: 3.6529 - val_acc: 0.3536\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 1.5494 - acc: 0.4602 - val_loss: 4.8477 - val_acc: 0.2423\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 9s 231us/step - loss: 1.3760 - acc: 0.5183 - val_loss: 3.8161 - val_acc: 0.3132\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 9s 231us/step - loss: 1.2583 - acc: 0.5559 - val_loss: 4.7297 - val_acc: 0.2768\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 9s 231us/step - loss: 1.2100 - acc: 0.5732 - val_loss: 6.4239 - val_acc: 0.1745\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 9s 231us/step - loss: 1.1134 - acc: 0.6067 - val_loss: 3.0847 - val_acc: 0.3653\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 9s 231us/step - loss: 1.0489 - acc: 0.6301 - val_loss: 2.4408 - val_acc: 0.4300\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.9934 - acc: 0.6476 - val_loss: 2.6456 - val_acc: 0.3732\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.9465 - acc: 0.6654 - val_loss: 2.5718 - val_acc: 0.3722\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.9227 - acc: 0.6739 - val_loss: 1.6259 - val_acc: 0.5201\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.8688 - acc: 0.6959 - val_loss: 2.3894 - val_acc: 0.4156\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 0.8372 - acc: 0.7026 - val_loss: 1.5643 - val_acc: 0.5560\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.7911 - acc: 0.7208 - val_loss: 1.3455 - val_acc: 0.5836\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.7547 - acc: 0.7335 - val_loss: 1.8457 - val_acc: 0.4948\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.7490 - acc: 0.7347 - val_loss: 2.6759 - val_acc: 0.4540\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 0.7069 - acc: 0.7492 - val_loss: 1.3416 - val_acc: 0.6087\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 9s 231us/step - loss: 0.6760 - acc: 0.7615 - val_loss: 1.7002 - val_acc: 0.5636\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.6353 - acc: 0.7740 - val_loss: 1.2769 - val_acc: 0.6334\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.6175 - acc: 0.7807 - val_loss: 1.2403 - val_acc: 0.6518\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.5909 - acc: 0.7899 - val_loss: 2.0019 - val_acc: 0.6083\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.5509 - acc: 0.8042 - val_loss: 2.7034 - val_acc: 0.5253\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.5502 - acc: 0.8033 - val_loss: 2.1962 - val_acc: 0.5605\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.5344 - acc: 0.8105 - val_loss: 2.3938 - val_acc: 0.5334\n",
      "Epoch 24/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.4908 - acc: 0.8260 - val_loss: 3.6466 - val_acc: 0.4687\n",
      "Epoch 25/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.5205 - acc: 0.8148 - val_loss: 2.5705 - val_acc: 0.5240\n",
      "10000/10000 [==============================] - 2s 240us/step\n",
      "40000/40000 [==============================] - 12s 304us/step\n",
      "10000/10000 [==============================] - 3s 316us/step\n",
      "Loss = 1.257085157394409\n",
      "Test Accuracy = 0.6367\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [512, 128], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.20710070821028637, 'lr': 0.000776381420874868}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 20s 504us/step - loss: 1.8964 - acc: 0.3690 - val_loss: 2.0995 - val_acc: 0.4020\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 1.4900 - acc: 0.4753 - val_loss: 1.8653 - val_acc: 0.4147\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 1.3515 - acc: 0.5203 - val_loss: 1.7080 - val_acc: 0.4702\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 1.2403 - acc: 0.5611 - val_loss: 2.1310 - val_acc: 0.4205\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 1.1590 - acc: 0.5884 - val_loss: 2.2360 - val_acc: 0.3916\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 1.0947 - acc: 0.6135 - val_loss: 1.7141 - val_acc: 0.4731\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 1.0403 - acc: 0.6301 - val_loss: 1.4228 - val_acc: 0.5409\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 1.0006 - acc: 0.6477 - val_loss: 2.6769 - val_acc: 0.3104\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 0.9662 - acc: 0.6606 - val_loss: 2.3242 - val_acc: 0.3910\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 0.9290 - acc: 0.6709 - val_loss: 1.9840 - val_acc: 0.4814\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 0.8973 - acc: 0.6836 - val_loss: 1.8134 - val_acc: 0.4887\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 0.8724 - acc: 0.6921 - val_loss: 1.1455 - val_acc: 0.6171\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 0.8488 - acc: 0.6995 - val_loss: 1.1844 - val_acc: 0.6216\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 0.8428 - acc: 0.7038 - val_loss: 1.1782 - val_acc: 0.6182\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 0.8103 - acc: 0.7154 - val_loss: 1.2202 - val_acc: 0.6094\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 0.7922 - acc: 0.7214 - val_loss: 0.9491 - val_acc: 0.6773\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 0.7665 - acc: 0.7297 - val_loss: 0.9441 - val_acc: 0.6700\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 0.7509 - acc: 0.7383 - val_loss: 1.4068 - val_acc: 0.5626\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.7271 - acc: 0.7438 - val_loss: 0.9186 - val_acc: 0.6936\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.7217 - acc: 0.7431 - val_loss: 0.9147 - val_acc: 0.6893\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 0.7040 - acc: 0.7529 - val_loss: 1.0878 - val_acc: 0.6483\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.6852 - acc: 0.7572 - val_loss: 0.7884 - val_acc: 0.7294\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.6656 - acc: 0.7634 - val_loss: 0.8501 - val_acc: 0.7137\n",
      "Epoch 24/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.6502 - acc: 0.7695 - val_loss: 0.9072 - val_acc: 0.7016\n",
      "Epoch 25/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.6379 - acc: 0.7738 - val_loss: 0.9482 - val_acc: 0.6885\n",
      "Epoch 26/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 0.6361 - acc: 0.7732 - val_loss: 0.8733 - val_acc: 0.7062\n",
      "Epoch 27/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.5972 - acc: 0.7899 - val_loss: 0.9464 - val_acc: 0.6823\n",
      "Epoch 28/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.5969 - acc: 0.7886 - val_loss: 1.2637 - val_acc: 0.6202\n",
      "10000/10000 [==============================] - 2s 182us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [512, 128], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.2714152860038964, 'lr': 0.0019773444725281076}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 20s 512us/step - loss: 1.8641 - acc: 0.3730 - val_loss: 2.2838 - val_acc: 0.3773\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 1.4177 - acc: 0.4932 - val_loss: 2.8076 - val_acc: 0.3270\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 1.2512 - acc: 0.5504 - val_loss: 2.6684 - val_acc: 0.3417\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 1.1442 - acc: 0.5920 - val_loss: 5.0887 - val_acc: 0.1684\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 1.0712 - acc: 0.6188 - val_loss: 1.6133 - val_acc: 0.5052\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 1.0204 - acc: 0.6358 - val_loss: 1.8945 - val_acc: 0.4555\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 0.9687 - acc: 0.6553 - val_loss: 2.9510 - val_acc: 0.3558\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.9348 - acc: 0.6676 - val_loss: 2.1986 - val_acc: 0.4052\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 0.9022 - acc: 0.6768 - val_loss: 1.5550 - val_acc: 0.5364\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.8772 - acc: 0.6903 - val_loss: 2.0462 - val_acc: 0.4664\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.8541 - acc: 0.6970 - val_loss: 1.0127 - val_acc: 0.6441\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.8340 - acc: 0.7049 - val_loss: 1.8964 - val_acc: 0.4618\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.8047 - acc: 0.7149 - val_loss: 1.1924 - val_acc: 0.6067\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.7823 - acc: 0.7241 - val_loss: 1.8235 - val_acc: 0.5197\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.7724 - acc: 0.7280 - val_loss: 1.0244 - val_acc: 0.6535\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.7469 - acc: 0.7355 - val_loss: 1.2343 - val_acc: 0.6076\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 0.7253 - acc: 0.7444 - val_loss: 0.9156 - val_acc: 0.6906\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.6963 - acc: 0.7513 - val_loss: 1.2674 - val_acc: 0.6143\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 0.6806 - acc: 0.7586 - val_loss: 1.6571 - val_acc: 0.5733\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 0.6653 - acc: 0.7647 - val_loss: 3.1463 - val_acc: 0.3360\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.6622 - acc: 0.7658 - val_loss: 1.5603 - val_acc: 0.5597\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.6416 - acc: 0.7702 - val_loss: 0.7520 - val_acc: 0.7503\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.6047 - acc: 0.7846 - val_loss: 1.2153 - val_acc: 0.6388\n",
      "Epoch 24/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.6006 - acc: 0.7874 - val_loss: 1.0896 - val_acc: 0.6649\n",
      "Epoch 25/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.5930 - acc: 0.7889 - val_loss: 1.2135 - val_acc: 0.6678\n",
      "Epoch 26/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.5646 - acc: 0.7999 - val_loss: 0.8790 - val_acc: 0.7178\n",
      "Epoch 27/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.5501 - acc: 0.8041 - val_loss: 1.6577 - val_acc: 0.5708\n",
      "Epoch 28/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.5512 - acc: 0.8044 - val_loss: 0.8509 - val_acc: 0.7370\n",
      "10000/10000 [==============================] - 2s 183us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [512, 128], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.2994317243757456, 'lr': 0.0009292584109944156}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 21s 517us/step - loss: 2.0509 - acc: 0.3330 - val_loss: 1.9673 - val_acc: 0.4263\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.6117 - acc: 0.4343 - val_loss: 1.7978 - val_acc: 0.4358\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 1.4516 - acc: 0.4884 - val_loss: 2.4717 - val_acc: 0.3748\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.3356 - acc: 0.5243 - val_loss: 2.8426 - val_acc: 0.3266\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 1.2519 - acc: 0.5546 - val_loss: 2.8837 - val_acc: 0.3163\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 1.1923 - acc: 0.5740 - val_loss: 2.6995 - val_acc: 0.3618\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 1.1379 - acc: 0.5936 - val_loss: 4.2123 - val_acc: 0.2433\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 1.0964 - acc: 0.6100 - val_loss: 1.6171 - val_acc: 0.5161\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 1.0569 - acc: 0.6249 - val_loss: 2.3925 - val_acc: 0.4263\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 1.0375 - acc: 0.6318 - val_loss: 1.2813 - val_acc: 0.5819\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.9838 - acc: 0.6489 - val_loss: 1.2772 - val_acc: 0.5826\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.9620 - acc: 0.6606 - val_loss: 1.6920 - val_acc: 0.5205\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.9338 - acc: 0.6710 - val_loss: 1.5288 - val_acc: 0.5121\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.9052 - acc: 0.6812 - val_loss: 1.0433 - val_acc: 0.6481\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.8918 - acc: 0.6857 - val_loss: 1.1317 - val_acc: 0.6373\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 0.8755 - acc: 0.6896 - val_loss: 1.8606 - val_acc: 0.4640\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.8558 - acc: 0.6988 - val_loss: 1.2113 - val_acc: 0.6124\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.8446 - acc: 0.7020 - val_loss: 1.2434 - val_acc: 0.6065\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.8212 - acc: 0.7092 - val_loss: 1.3429 - val_acc: 0.5748\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 0.8192 - acc: 0.7098 - val_loss: 1.3490 - val_acc: 0.5707\n",
      "10000/10000 [==============================] - 2s 187us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [512, 128], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.46307480119881467, 'lr': 0.000346168363504218}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 21s 528us/step - loss: 2.8589 - acc: 0.1883 - val_loss: 2.1399 - val_acc: 0.3353\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 2.2943 - acc: 0.2783 - val_loss: 2.1580 - val_acc: 0.3681\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 2.0751 - acc: 0.3155 - val_loss: 2.6163 - val_acc: 0.3147\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 1.9484 - acc: 0.3387 - val_loss: 2.6915 - val_acc: 0.3038\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 142us/step - loss: 1.8424 - acc: 0.3652 - val_loss: 2.8968 - val_acc: 0.2913\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 1.7676 - acc: 0.3848 - val_loss: 3.2147 - val_acc: 0.2622\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 1.7062 - acc: 0.3987 - val_loss: 2.7736 - val_acc: 0.3082\n",
      "10000/10000 [==============================] - 2s 189us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [512, 128], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.4575275017616776, 'lr': 0.000693515801528763}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 22s 538us/step - loss: 2.5660 - acc: 0.2447 - val_loss: 2.3465 - val_acc: 0.3326\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 2.0030 - acc: 0.3381 - val_loss: 2.2895 - val_acc: 0.3266\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 143us/step - loss: 1.8069 - acc: 0.3726 - val_loss: 2.4005 - val_acc: 0.3078\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 1.6680 - acc: 0.4094 - val_loss: 2.8432 - val_acc: 0.2791\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 1.5743 - acc: 0.4360 - val_loss: 2.6364 - val_acc: 0.3030\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 1.4993 - acc: 0.4593 - val_loss: 4.1693 - val_acc: 0.1847\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 1.4482 - acc: 0.4760 - val_loss: 2.4212 - val_acc: 0.3314\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 144us/step - loss: 1.3899 - acc: 0.4974 - val_loss: 2.4487 - val_acc: 0.3139\n",
      "10000/10000 [==============================] - 2s 195us/step\n",
      "40000/40000 [==============================] - 9s 216us/step\n",
      "10000/10000 [==============================] - 2s 222us/step\n",
      "Loss = 0.8962637498855591\n",
      "Test Accuracy = 0.725\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [1024, 512], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.2252932709682651, 'lr': 0.001986181353617481}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 22s 559us/step - loss: 1.7994 - acc: 0.4075 - val_loss: 1.8227 - val_acc: 0.4456\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 1.3332 - acc: 0.5283 - val_loss: 2.5852 - val_acc: 0.3620\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 1.1801 - acc: 0.5821 - val_loss: 4.1122 - val_acc: 0.2253\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 1.0879 - acc: 0.6140 - val_loss: 1.4663 - val_acc: 0.5315\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 0.9921 - acc: 0.6477 - val_loss: 1.9936 - val_acc: 0.4349\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 0.9407 - acc: 0.6667 - val_loss: 1.5068 - val_acc: 0.5331\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 156us/step - loss: 0.8776 - acc: 0.6918 - val_loss: 1.2732 - val_acc: 0.5754\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 0.8334 - acc: 0.7068 - val_loss: 2.0639 - val_acc: 0.4487\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.7899 - acc: 0.7205 - val_loss: 1.7705 - val_acc: 0.4940\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 0.7508 - acc: 0.7342 - val_loss: 1.0876 - val_acc: 0.6281\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 0.7073 - acc: 0.7484 - val_loss: 1.0297 - val_acc: 0.6474\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.6803 - acc: 0.7592 - val_loss: 0.9712 - val_acc: 0.6661\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.6575 - acc: 0.7668 - val_loss: 1.2728 - val_acc: 0.6101\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.6165 - acc: 0.7820 - val_loss: 1.2070 - val_acc: 0.6305\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.6053 - acc: 0.7846 - val_loss: 0.9996 - val_acc: 0.6764\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.5728 - acc: 0.7978 - val_loss: 1.0781 - val_acc: 0.6687\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.5388 - acc: 0.8062 - val_loss: 1.2129 - val_acc: 0.6398\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.5095 - acc: 0.8193 - val_loss: 1.8032 - val_acc: 0.5943\n",
      "10000/10000 [==============================] - 2s 196us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [1024, 512], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.4825310732718501, 'lr': 0.000406339880431268}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 22s 554us/step - loss: 2.8589 - acc: 0.2141 - val_loss: 2.8096 - val_acc: 0.2801\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 2.2653 - acc: 0.3049 - val_loss: 3.6813 - val_acc: 0.2223\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 2.0165 - acc: 0.3466 - val_loss: 3.7556 - val_acc: 0.2260\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 153us/step - loss: 1.8742 - acc: 0.3720 - val_loss: 3.9574 - val_acc: 0.2202\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.7672 - acc: 0.3940 - val_loss: 3.3555 - val_acc: 0.2498\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.6814 - acc: 0.4164 - val_loss: 4.3875 - val_acc: 0.1956\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.6103 - acc: 0.4359 - val_loss: 3.9550 - val_acc: 0.2136\n",
      "10000/10000 [==============================] - 2s 195us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [1024, 512], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.2887860787581707, 'lr': 0.0004298458809873036}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 23s 567us/step - loss: 2.2206 - acc: 0.3175 - val_loss: 1.9372 - val_acc: 0.4119\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.7425 - acc: 0.4122 - val_loss: 2.3930 - val_acc: 0.3703\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.5753 - acc: 0.4560 - val_loss: 2.6149 - val_acc: 0.3302\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 1.4684 - acc: 0.4859 - val_loss: 2.9911 - val_acc: 0.2973\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 1.3693 - acc: 0.5195 - val_loss: 4.0074 - val_acc: 0.2145\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.3146 - acc: 0.5336 - val_loss: 2.9994 - val_acc: 0.2783\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 1.2467 - acc: 0.5581 - val_loss: 2.4373 - val_acc: 0.3629\n",
      "10000/10000 [==============================] - 2s 195us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [1024, 512], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.35194084746538024, 'lr': 0.0011179252453542047}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 23s 578us/step - loss: 2.1217 - acc: 0.3310 - val_loss: 4.3759 - val_acc: 0.2470\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 1.6115 - acc: 0.4366 - val_loss: 4.6259 - val_acc: 0.1851\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 1.4342 - acc: 0.4932 - val_loss: 3.9542 - val_acc: 0.2385\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 1.3130 - acc: 0.5351 - val_loss: 4.7795 - val_acc: 0.2097\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 1.2324 - acc: 0.5588 - val_loss: 6.5544 - val_acc: 0.1518\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 1.1776 - acc: 0.5792 - val_loss: 3.3009 - val_acc: 0.2699\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 1.1111 - acc: 0.6014 - val_loss: 2.5578 - val_acc: 0.3500\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.0611 - acc: 0.6226 - val_loss: 1.7043 - val_acc: 0.4851\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 1.0235 - acc: 0.6361 - val_loss: 1.3866 - val_acc: 0.5381\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 0.9918 - acc: 0.6468 - val_loss: 2.1047 - val_acc: 0.4236\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 0.9602 - acc: 0.6585 - val_loss: 1.5125 - val_acc: 0.5068\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.9167 - acc: 0.6752 - val_loss: 3.4860 - val_acc: 0.2632\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 0.8868 - acc: 0.6830 - val_loss: 4.0629 - val_acc: 0.2181\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 0.8794 - acc: 0.6868 - val_loss: 1.8546 - val_acc: 0.4565\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 0.8384 - acc: 0.7016 - val_loss: 1.5267 - val_acc: 0.5226\n",
      "10000/10000 [==============================] - 2s 202us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [1024, 512], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.49752239616666627, 'lr': 0.0022959751340924}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 23s 582us/step - loss: 2.2857 - acc: 0.2967 - val_loss: 3.9298 - val_acc: 0.2223\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.6420 - acc: 0.4158 - val_loss: 2.3125 - val_acc: 0.3409\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 1.4624 - acc: 0.4730 - val_loss: 4.5657 - val_acc: 0.1981\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 6s 156us/step - loss: 1.3505 - acc: 0.5117 - val_loss: 3.2762 - val_acc: 0.2860\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 1.2790 - acc: 0.5400 - val_loss: 6.7402 - val_acc: 0.1631\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 6s 156us/step - loss: 1.2395 - acc: 0.5546 - val_loss: 3.0418 - val_acc: 0.3146\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 6s 156us/step - loss: 1.1686 - acc: 0.5799 - val_loss: 1.4744 - val_acc: 0.4988\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 1.1444 - acc: 0.5904 - val_loss: 1.9433 - val_acc: 0.4421\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 1.0975 - acc: 0.6091 - val_loss: 1.3219 - val_acc: 0.5622\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 1.0570 - acc: 0.6242 - val_loss: 2.5326 - val_acc: 0.3511\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 6s 156us/step - loss: 1.0294 - acc: 0.6325 - val_loss: 1.6227 - val_acc: 0.4989\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 1.0165 - acc: 0.6394 - val_loss: 1.6134 - val_acc: 0.5134\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 0.9867 - acc: 0.6485 - val_loss: 1.7176 - val_acc: 0.5150\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 0.9538 - acc: 0.6620 - val_loss: 1.0050 - val_acc: 0.6491\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 0.9404 - acc: 0.6653 - val_loss: 2.1844 - val_acc: 0.4737\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 0.9237 - acc: 0.6717 - val_loss: 2.0659 - val_acc: 0.4562\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 0.9008 - acc: 0.6812 - val_loss: 1.9580 - val_acc: 0.4356\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 6s 154us/step - loss: 0.8857 - acc: 0.6883 - val_loss: 1.2683 - val_acc: 0.5901\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 6s 155us/step - loss: 0.8595 - acc: 0.6947 - val_loss: 1.9138 - val_acc: 0.4553\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 6s 156us/step - loss: 0.8550 - acc: 0.6965 - val_loss: 1.2456 - val_acc: 0.5919\n",
      "10000/10000 [==============================] - 2s 193us/step\n",
      "40000/40000 [==============================] - 9s 227us/step\n",
      "10000/10000 [==============================] - 2s 236us/step\n",
      "Loss = 1.250253914833069\n",
      "Test Accuracy = 0.5844\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [4096, 2048], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.3280388193077062, 'lr': 0.0010083698803930697}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 27s 673us/step - loss: 2.3714 - acc: 0.3456 - val_loss: 5.3779 - val_acc: 0.2586\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.5841 - acc: 0.4590 - val_loss: 7.5786 - val_acc: 0.1487\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.3988 - acc: 0.5089 - val_loss: 6.5237 - val_acc: 0.1662\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.2841 - acc: 0.5448 - val_loss: 6.3453 - val_acc: 0.2139\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.2102 - acc: 0.5737 - val_loss: 4.1394 - val_acc: 0.2606\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.1294 - acc: 0.6021 - val_loss: 4.5799 - val_acc: 0.2294\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.0712 - acc: 0.6193 - val_loss: 3.4187 - val_acc: 0.2718\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.0345 - acc: 0.6332 - val_loss: 3.8924 - val_acc: 0.2749\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.9849 - acc: 0.6518 - val_loss: 4.4650 - val_acc: 0.2171\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.9367 - acc: 0.6694 - val_loss: 2.4840 - val_acc: 0.3887\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.9252 - acc: 0.6739 - val_loss: 2.7675 - val_acc: 0.3614\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.8603 - acc: 0.6990 - val_loss: 1.9641 - val_acc: 0.4648\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 0.8244 - acc: 0.7079 - val_loss: 3.5823 - val_acc: 0.2767\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 0.7923 - acc: 0.7179 - val_loss: 2.2892 - val_acc: 0.4037\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.7612 - acc: 0.7295 - val_loss: 1.1774 - val_acc: 0.6144\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 9s 232us/step - loss: 0.7223 - acc: 0.7436 - val_loss: 2.9779 - val_acc: 0.3461\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.6963 - acc: 0.7511 - val_loss: 2.5677 - val_acc: 0.4253\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.6739 - acc: 0.7583 - val_loss: 1.2924 - val_acc: 0.6161\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 0.6265 - acc: 0.7770 - val_loss: 1.3414 - val_acc: 0.6078\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 9s 236us/step - loss: 0.6011 - acc: 0.7875 - val_loss: 1.3899 - val_acc: 0.6029\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 9s 236us/step - loss: 0.5791 - acc: 0.7922 - val_loss: 1.0859 - val_acc: 0.6493\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 0.5524 - acc: 0.8015 - val_loss: 1.1281 - val_acc: 0.6739\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.5320 - acc: 0.8100 - val_loss: 1.8875 - val_acc: 0.5670\n",
      "Epoch 24/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.4907 - acc: 0.8262 - val_loss: 1.2685 - val_acc: 0.6618\n",
      "Epoch 25/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 0.4531 - acc: 0.8406 - val_loss: 2.2087 - val_acc: 0.5607\n",
      "Epoch 26/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.4532 - acc: 0.8375 - val_loss: 1.5678 - val_acc: 0.6035\n",
      "Epoch 27/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.4019 - acc: 0.8573 - val_loss: 1.4556 - val_acc: 0.6689\n",
      "10000/10000 [==============================] - 2s 249us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [4096, 2048], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.24540475304903048, 'lr': 0.0020421573639950934}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 28s 697us/step - loss: 2.4630 - acc: 0.3622 - val_loss: 2.9632 - val_acc: 0.3499\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.5191 - acc: 0.4706 - val_loss: 4.9129 - val_acc: 0.1866\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.3341 - acc: 0.5297 - val_loss: 2.5439 - val_acc: 0.3617\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.2478 - acc: 0.5594 - val_loss: 2.7922 - val_acc: 0.3394\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.1673 - acc: 0.5929 - val_loss: 1.9729 - val_acc: 0.4354\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 1.0903 - acc: 0.6157 - val_loss: 4.7403 - val_acc: 0.2132\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 1.0213 - acc: 0.6434 - val_loss: 2.7757 - val_acc: 0.3624\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.9775 - acc: 0.6549 - val_loss: 2.3295 - val_acc: 0.4112\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 9s 233us/step - loss: 0.9370 - acc: 0.6718 - val_loss: 1.4247 - val_acc: 0.5439\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.8767 - acc: 0.6895 - val_loss: 2.1754 - val_acc: 0.4247\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.8288 - acc: 0.7089 - val_loss: 4.9929 - val_acc: 0.2246\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.7838 - acc: 0.7224 - val_loss: 1.7266 - val_acc: 0.5228\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.7577 - acc: 0.7382 - val_loss: 1.4078 - val_acc: 0.5772\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 9s 236us/step - loss: 0.7230 - acc: 0.7415 - val_loss: 1.7057 - val_acc: 0.5285\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 10s 238us/step - loss: 0.6759 - acc: 0.7603 - val_loss: 1.3130 - val_acc: 0.5922\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 10s 238us/step - loss: 0.6595 - acc: 0.7678 - val_loss: 1.7311 - val_acc: 0.5230\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 10s 238us/step - loss: 0.6287 - acc: 0.7772 - val_loss: 1.3243 - val_acc: 0.6052\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 9s 237us/step - loss: 0.6105 - acc: 0.7840 - val_loss: 1.7239 - val_acc: 0.5682\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 0.5670 - acc: 0.7990 - val_loss: 1.4950 - val_acc: 0.6047\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.5214 - acc: 0.8150 - val_loss: 1.8308 - val_acc: 0.5789\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.4977 - acc: 0.8222 - val_loss: 1.4065 - val_acc: 0.6282\n",
      "10000/10000 [==============================] - 3s 252us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [4096, 2048], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.4379044869377562, 'lr': 0.000962091959148995}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "40000/40000 [==============================] - 28s 697us/step - loss: 2.5585 - acc: 0.3116 - val_loss: 4.6896 - val_acc: 0.2762\n",
      "Epoch 2/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 1.7992 - acc: 0.4073 - val_loss: 6.5807 - val_acc: 0.1786\n",
      "Epoch 3/200\n",
      "40000/40000 [==============================] - 9s 236us/step - loss: 1.5664 - acc: 0.4555 - val_loss: 3.3890 - val_acc: 0.2815\n",
      "Epoch 4/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 1.4316 - acc: 0.4986 - val_loss: 5.7649 - val_acc: 0.1709\n",
      "Epoch 5/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 1.3245 - acc: 0.5298 - val_loss: 5.7413 - val_acc: 0.2076\n",
      "Epoch 6/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 1.2627 - acc: 0.5494 - val_loss: 3.4904 - val_acc: 0.3070\n",
      "Epoch 7/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 1.2117 - acc: 0.5702 - val_loss: 7.1694 - val_acc: 0.1386\n",
      "Epoch 8/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 1.1445 - acc: 0.5923 - val_loss: 3.1651 - val_acc: 0.2963\n",
      "Epoch 9/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 1.0875 - acc: 0.6126 - val_loss: 2.6972 - val_acc: 0.3382\n",
      "Epoch 10/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 1.0452 - acc: 0.6284 - val_loss: 3.4652 - val_acc: 0.2819\n",
      "Epoch 11/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 1.0094 - acc: 0.6419 - val_loss: 2.4447 - val_acc: 0.3734\n",
      "Epoch 12/200\n",
      "40000/40000 [==============================] - 9s 237us/step - loss: 0.9835 - acc: 0.6497 - val_loss: 2.9762 - val_acc: 0.3247\n",
      "Epoch 13/200\n",
      "40000/40000 [==============================] - 9s 237us/step - loss: 0.9531 - acc: 0.6619 - val_loss: 2.2104 - val_acc: 0.4249\n",
      "Epoch 14/200\n",
      "40000/40000 [==============================] - 9s 236us/step - loss: 0.9154 - acc: 0.6761 - val_loss: 1.7867 - val_acc: 0.5065\n",
      "Epoch 15/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 0.8904 - acc: 0.6817 - val_loss: 1.8082 - val_acc: 0.4744\n",
      "Epoch 16/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 0.8572 - acc: 0.6939 - val_loss: 3.5458 - val_acc: 0.2934\n",
      "Epoch 17/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.8477 - acc: 0.6982 - val_loss: 2.0116 - val_acc: 0.4816\n",
      "Epoch 18/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.8205 - acc: 0.7064 - val_loss: 1.3640 - val_acc: 0.5785\n",
      "Epoch 19/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 0.8102 - acc: 0.7115 - val_loss: 1.5972 - val_acc: 0.5392\n",
      "Epoch 20/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.7905 - acc: 0.7171 - val_loss: 2.0198 - val_acc: 0.4812\n",
      "Epoch 21/200\n",
      "40000/40000 [==============================] - 9s 234us/step - loss: 0.7649 - acc: 0.7283 - val_loss: 1.9738 - val_acc: 0.4961\n",
      "Epoch 22/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 0.7364 - acc: 0.7370 - val_loss: 2.1009 - val_acc: 0.4904\n",
      "Epoch 23/200\n",
      "40000/40000 [==============================] - 10s 238us/step - loss: 0.7156 - acc: 0.7451 - val_loss: 1.1057 - val_acc: 0.6570\n",
      "Epoch 24/200\n",
      "40000/40000 [==============================] - 9s 236us/step - loss: 0.7006 - acc: 0.7507 - val_loss: 1.2882 - val_acc: 0.6177\n",
      "Epoch 25/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 0.6760 - acc: 0.7587 - val_loss: 1.5289 - val_acc: 0.5677\n",
      "Epoch 26/200\n",
      "40000/40000 [==============================] - 9s 236us/step - loss: 0.6537 - acc: 0.7669 - val_loss: 3.1719 - val_acc: 0.4087\n",
      "Epoch 27/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 0.6298 - acc: 0.7745 - val_loss: 1.4260 - val_acc: 0.6029\n",
      "Epoch 28/200\n",
      "40000/40000 [==============================] - 9s 235us/step - loss: 0.6062 - acc: 0.7830 - val_loss: 1.4771 - val_acc: 0.6084\n",
      "Epoch 29/200\n",
      "40000/40000 [==============================] - 9s 237us/step - loss: 0.5951 - acc: 0.7875 - val_loss: 1.4076 - val_acc: 0.6189\n",
      "10000/10000 [==============================] - 3s 251us/step\n",
      "{'classes': 10, 'activation': 'tanh', 'batch_size': 1024, 'epochs': 200, 'optimizer': 'adam', 'loss': 'sparse_categorical_crossentropy', 'layers': [4096, 2048], 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f4188966668>], 'dropout_prob': 0.3518063082637903, 'lr': 0.0003672047061416017}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n"
     ]
    },
    {
     "ename": "ResourceExhaustedError",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mResourceExhaustedError\u001b[0m                    Traceback (most recent call last)",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1355\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1356\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1357\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1340\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1341\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1342\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1428\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1429\u001b[0;31m         run_metadata)\n\u001b[0m\u001b[1;32m   1430\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mResourceExhaustedError\u001b[0m: 2 root error(s) found.\n  (0) Resource exhausted: OOM when allocating tensor with shape[1024,2048] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[{{node training_58/Adam/gradients/zeros}}]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n\t [[metrics_58/acc/Mean/_19077]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n  (1) Resource exhausted: OOM when allocating tensor with shape[1024,2048] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[{{node training_58/Adam/gradients/zeros}}]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n0 successful operations.\n0 derived errors ignored.",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mResourceExhaustedError\u001b[0m                    Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-ad0bbd8e2ec0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0mmeta_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_evals\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m5\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m model_results = meta_model.process_model(params, search_space, deep_cnn_model, model_name, [X_train, X_val, X_test, y_train, y_val, y_test], \n\u001b[0;32m---> 21\u001b[0;31m               [model_name, 0, 0, 0, 'layers', 'epochs', 'activation', 'optimizer', 'lr','reg_param', 'dropout_prob', 'batchNorm' , np.NaN])\n\u001b[0m\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0;31m#Now lets plot the results.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/content/drive/My Drive/Colab Notebooks/Cifar10/../models/processModel.py\u001b[0m in \u001b[0;36mprocess_model\u001b[0;34m(self, params, hyper_params, model_generator, model_name, train_test, attributes)\u001b[0m\n\u001b[1;32m    155\u001b[0m         \"\"\"\n\u001b[1;32m    156\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_results\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 157\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecurse_params\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhyper_params\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_generator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattributes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    158\u001b[0m         \u001b[0;31m#Return a list where each entry is of the form [out, model] from earlier calculations.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    159\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_results\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/content/drive/My Drive/Colab Notebooks/Cifar10/../models/processModel.py\u001b[0m in \u001b[0;36mrecurse_params\u001b[0;34m(self, path, params, hyper_params, model_generator, model_name, train_test, attributes)\u001b[0m\n\u001b[1;32m    175\u001b[0m                 \u001b[0;32mdel\u001b[0m \u001b[0mnew_params\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    176\u001b[0m                 self.recurse_params(new_path, new_params, hyper_params,\n\u001b[0;32m--> 177\u001b[0;31m                                     model_generator, model_name, train_test, attributes)\n\u001b[0m",
      "\u001b[0;32m/content/drive/My Drive/Colab Notebooks/Cifar10/../models/processModel.py\u001b[0m in \u001b[0;36mrecurse_params\u001b[0;34m(self, path, params, hyper_params, model_generator, model_name, train_test, attributes)\u001b[0m\n\u001b[1;32m    175\u001b[0m                 \u001b[0;32mdel\u001b[0m \u001b[0mnew_params\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    176\u001b[0m                 self.recurse_params(new_path, new_params, hyper_params,\n\u001b[0;32m--> 177\u001b[0;31m                                     model_generator, model_name, train_test, attributes)\n\u001b[0m",
      "\u001b[0;32m/content/drive/My Drive/Colab Notebooks/Cifar10/../models/processModel.py\u001b[0m in \u001b[0;36mrecurse_params\u001b[0;34m(self, path, params, hyper_params, model_generator, model_name, train_test, attributes)\u001b[0m\n\u001b[1;32m    175\u001b[0m                 \u001b[0;32mdel\u001b[0m \u001b[0mnew_params\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    176\u001b[0m                 self.recurse_params(new_path, new_params, hyper_params,\n\u001b[0;32m--> 177\u001b[0;31m                                     model_generator, model_name, train_test, attributes)\n\u001b[0m",
      "\u001b[0;32m/content/drive/My Drive/Colab Notebooks/Cifar10/../models/processModel.py\u001b[0m in \u001b[0;36mrecurse_params\u001b[0;34m(self, path, params, hyper_params, model_generator, model_name, train_test, attributes)\u001b[0m\n\u001b[1;32m    175\u001b[0m                 \u001b[0;32mdel\u001b[0m \u001b[0mnew_params\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    176\u001b[0m                 self.recurse_params(new_path, new_params, hyper_params,\n\u001b[0;32m--> 177\u001b[0;31m                                     model_generator, model_name, train_test, attributes)\n\u001b[0m",
      "\u001b[0;32m/content/drive/My Drive/Colab Notebooks/Cifar10/../models/processModel.py\u001b[0m in \u001b[0;36mrecurse_params\u001b[0;34m(self, path, params, hyper_params, model_generator, model_name, train_test, attributes)\u001b[0m\n\u001b[1;32m    175\u001b[0m                 \u001b[0;32mdel\u001b[0m \u001b[0mnew_params\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    176\u001b[0m                 self.recurse_params(new_path, new_params, hyper_params,\n\u001b[0;32m--> 177\u001b[0;31m                                     model_generator, model_name, train_test, attributes)\n\u001b[0m",
      "\u001b[0;32m/content/drive/My Drive/Colab Notebooks/Cifar10/../models/processModel.py\u001b[0m in \u001b[0;36mrecurse_params\u001b[0;34m(self, path, params, hyper_params, model_generator, model_name, train_test, attributes)\u001b[0m\n\u001b[1;32m    175\u001b[0m                 \u001b[0;32mdel\u001b[0m \u001b[0mnew_params\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    176\u001b[0m                 self.recurse_params(new_path, new_params, hyper_params,\n\u001b[0;32m--> 177\u001b[0;31m                                     model_generator, model_name, train_test, attributes)\n\u001b[0m",
      "\u001b[0;32m/content/drive/My Drive/Colab Notebooks/Cifar10/../models/processModel.py\u001b[0m in \u001b[0;36mrecurse_params\u001b[0;34m(self, path, params, hyper_params, model_generator, model_name, train_test, attributes)\u001b[0m\n\u001b[1;32m    175\u001b[0m                 \u001b[0;32mdel\u001b[0m \u001b[0mnew_params\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    176\u001b[0m                 self.recurse_params(new_path, new_params, hyper_params,\n\u001b[0;32m--> 177\u001b[0;31m                                     model_generator, model_name, train_test, attributes)\n\u001b[0m",
      "\u001b[0;32m/content/drive/My Drive/Colab Notebooks/Cifar10/../models/processModel.py\u001b[0m in \u001b[0;36mrecurse_params\u001b[0;34m(self, path, params, hyper_params, model_generator, model_name, train_test, attributes)\u001b[0m\n\u001b[1;32m    175\u001b[0m                 \u001b[0;32mdel\u001b[0m \u001b[0mnew_params\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    176\u001b[0m                 self.recurse_params(new_path, new_params, hyper_params,\n\u001b[0;32m--> 177\u001b[0;31m                                     model_generator, model_name, train_test, attributes)\n\u001b[0m",
      "\u001b[0;32m/content/drive/My Drive/Colab Notebooks/Cifar10/../models/processModel.py\u001b[0m in \u001b[0;36mrecurse_params\u001b[0;34m(self, path, params, hyper_params, model_generator, model_name, train_test, attributes)\u001b[0m\n\u001b[1;32m    166\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    167\u001b[0m             \u001b[0;31m#End of the parameters, so evaluate these parameters.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 168\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval_model2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhyper_params\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_generator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattributes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    169\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    170\u001b[0m             \u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvals\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/content/drive/My Drive/Colab Notebooks/Cifar10/../models/processModel.py\u001b[0m in \u001b[0;36meval_model2\u001b[0;34m(self, params, hyper_params, model_generator, model_name, train_test, attributes)\u001b[0m\n\u001b[1;32m     90\u001b[0m         \u001b[0mtrials\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTrials\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m         best = fmin(hyperopt_objective, hyper_params, \n\u001b[0;32m---> 92\u001b[0;31m                     algo=tpe.suggest, max_evals=self.max_evals, trials= trials)\n\u001b[0m\u001b[1;32m     93\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m         \u001b[0mbest_trial\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mfmin\u001b[0;34m(fn, space, algo, max_evals, trials, rstate, allow_trials_fmin, pass_expr_memo_ctrl, catch_eval_exceptions, verbose, return_argmin)\u001b[0m\n\u001b[1;32m    305\u001b[0m             \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    306\u001b[0m             \u001b[0mcatch_eval_exceptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcatch_eval_exceptions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 307\u001b[0;31m             \u001b[0mreturn_argmin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreturn_argmin\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    308\u001b[0m         )\n\u001b[1;32m    309\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/hyperopt/base.py\u001b[0m in \u001b[0;36mfmin\u001b[0;34m(self, fn, space, algo, max_evals, rstate, verbose, pass_expr_memo_ctrl, catch_eval_exceptions, return_argmin)\u001b[0m\n\u001b[1;32m    633\u001b[0m             \u001b[0mpass_expr_memo_ctrl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpass_expr_memo_ctrl\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    634\u001b[0m             \u001b[0mcatch_eval_exceptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcatch_eval_exceptions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 635\u001b[0;31m             return_argmin=return_argmin)\n\u001b[0m\u001b[1;32m    636\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    637\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mfmin\u001b[0;34m(fn, space, algo, max_evals, trials, rstate, allow_trials_fmin, pass_expr_memo_ctrl, catch_eval_exceptions, verbose, return_argmin)\u001b[0m\n\u001b[1;32m    318\u001b[0m                     verbose=verbose)\n\u001b[1;32m    319\u001b[0m     \u001b[0mrval\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcatch_eval_exceptions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcatch_eval_exceptions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 320\u001b[0;31m     \u001b[0mrval\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexhaust\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    321\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mreturn_argmin\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    322\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mtrials\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmin\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mexhaust\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    197\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mexhaust\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m         \u001b[0mn_done\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 199\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_evals\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mn_done\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mblock_until_done\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    200\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrefresh\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    201\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, N, block_until_done)\u001b[0m\n\u001b[1;32m    171\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    172\u001b[0m                 \u001b[0;31m# -- loop over trials and do the jobs directly\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 173\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mserial_evaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    174\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    175\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mstopped\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mserial_evaluate\u001b[0;34m(self, N)\u001b[0m\n\u001b[1;32m     90\u001b[0m                 \u001b[0mctrl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbase\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCtrl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcurrent_trial\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m                     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdomain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspec\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mctrl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     93\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m                     \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'job exception: %s'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/hyperopt/base.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(self, config, ctrl, attach_attachments)\u001b[0m\n\u001b[1;32m    838\u001b[0m                 \u001b[0mmemo\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmemo\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    839\u001b[0m                 print_node_on_error=self.rec_eval_print_node_on_error)\n\u001b[0;32m--> 840\u001b[0;31m             \u001b[0mrval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpyll_rval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    841\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    842\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumber\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/content/drive/My Drive/Colab Notebooks/Cifar10/../models/processModel.py\u001b[0m in \u001b[0;36mhyperopt_objective\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m     79\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     80\u001b[0m                 \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 81\u001b[0;31m             \u001b[0m__out\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmy_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m__fit_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_generator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     82\u001b[0m             \u001b[0;31m#Store these variables for later use.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     83\u001b[0m             \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m__out\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/content/drive/My Drive/Colab Notebooks/Cifar10/../models/deepCNN.py\u001b[0m in \u001b[0;36mdeep_cnn_model\u001b[0;34m(X_train, y_train, X_val, y_val, params, verbose)\u001b[0m\n\u001b[1;32m    109\u001b[0m                     \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    110\u001b[0m                     \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mX_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_val\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 111\u001b[0;31m                     callbacks = callbacks)\n\u001b[0m\u001b[1;32m    112\u001b[0m     \u001b[0mt_end\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    113\u001b[0m     \u001b[0mt_diff\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mt_end\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mt_start\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1703\u001b[0m                               \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1704\u001b[0m                               \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1705\u001b[0;31m                               validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1706\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1707\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m   1234\u001b[0m                         \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1236\u001b[0;31m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1237\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1238\u001b[0m                         \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2480\u001b[0m         \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2481\u001b[0m         updated = session.run(fetches=fetches, feed_dict=feed_dict,\n\u001b[0;32m-> 2482\u001b[0;31m                               **self.session_kwargs)\n\u001b[0m\u001b[1;32m   2483\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2484\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    948\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    949\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 950\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    951\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    952\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1171\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1172\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1173\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1174\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1175\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1348\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1349\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1350\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1351\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1352\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1368\u001b[0m           \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1369\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0merror_interpolation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minterpolate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1370\u001b[0;31m       \u001b[0;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1371\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1372\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mResourceExhaustedError\u001b[0m: 2 root error(s) found.\n  (0) Resource exhausted: OOM when allocating tensor with shape[1024,2048] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[node training_58/Adam/gradients/zeros (defined at /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:2519) ]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n\t [[metrics_58/acc/Mean/_19077]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n  (1) Resource exhausted: OOM when allocating tensor with shape[1024,2048] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[node training_58/Adam/gradients/zeros (defined at /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:2519) ]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n0 successful operations.\n0 derived errors ignored.\n\nOriginal stack trace for 'training_58/Adam/gradients/zeros':\n  File \"/usr/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"/usr/lib/python3.6/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"/usr/local/lib/python3.6/dist-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelapp.py\", line 477, in start\n    ioloop.IOLoop.instance().start()\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/ioloop.py\", line 888, in start\n    handler_func(fd_obj, events)\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/zmq/eventloop/zmqstream.py\", line 450, in _handle_events\n    self._handle_recv()\n  File \"/usr/local/lib/python3.6/dist-packages/zmq/eventloop/zmqstream.py\", line 480, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/usr/local/lib/python3.6/dist-packages/zmq/eventloop/zmqstream.py\", line 432, in _run_callback\n    callback(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py\", line 235, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/zmqshell.py\", line 533, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\", line 2718, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\", line 2822, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\", line 2882, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-4-ad0bbd8e2ec0>\", line 21, in <module>\n    [model_name, 0, 0, 0, 'layers', 'epochs', 'activation', 'optimizer', 'lr','reg_param', 'dropout_prob', 'batchNorm' , np.NaN])\n  File \"/content/drive/My Drive/Colab Notebooks/Cifar10/../models/processModel.py\", line 157, in process_model\n    self.recurse_params({}, params, hyper_params, model_generator, model_name, train_test, attributes)\n  File \"/content/drive/My Drive/Colab Notebooks/Cifar10/../models/processModel.py\", line 177, in recurse_params\n    model_generator, model_name, train_test, attributes)\n  File \"/content/drive/My Drive/Colab Notebooks/Cifar10/../models/processModel.py\", line 177, in recurse_params\n    model_generator, model_name, train_test, attributes)\n  File \"/content/drive/My Drive/Colab Notebooks/Cifar10/../models/processModel.py\", line 177, in recurse_params\n    model_generator, model_name, train_test, attributes)\n  [Previous line repeated 5 more times]\n  File \"/content/drive/My Drive/Colab Notebooks/Cifar10/../models/processModel.py\", line 168, in recurse_params\n    self.eval_model2(path, hyper_params, model_generator, model_name, train_test, attributes)\n  File \"/content/drive/My Drive/Colab Notebooks/Cifar10/../models/processModel.py\", line 92, in eval_model2\n    algo=tpe.suggest, max_evals=self.max_evals, trials= trials)\n  File \"/usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py\", line 307, in fmin\n    return_argmin=return_argmin,\n  File \"/usr/local/lib/python3.6/dist-packages/hyperopt/base.py\", line 635, in fmin\n    return_argmin=return_argmin)\n  File \"/usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py\", line 320, in fmin\n    rval.exhaust()\n  File \"/usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py\", line 199, in exhaust\n    self.run(self.max_evals - n_done, block_until_done=self.async)\n  File \"/usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py\", line 173, in run\n    self.serial_evaluate()\n  File \"/usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py\", line 92, in serial_evaluate\n    result = self.domain.evaluate(spec, ctrl)\n  File \"/usr/local/lib/python3.6/dist-packages/hyperopt/base.py\", line 840, in evaluate\n    rval = self.fn(pyll_rval)\n  File \"/content/drive/My Drive/Colab Notebooks/Cifar10/../models/processModel.py\", line 81, in hyperopt_objective\n    __out, my_model, __fit_time = model_generator(X_train, y_train, X_val, y_val, {**params, **x}, verbose = self.verbose)\n  File \"/content/drive/My Drive/Colab Notebooks/Cifar10/../models/deepCNN.py\", line 111, in deep_cnn_model\n    callbacks = callbacks)\n  File \"/usr/local/lib/python3.6/dist-packages/keras/engine/training.py\", line 1682, in fit\n    self._make_train_function()\n  File \"/usr/local/lib/python3.6/dist-packages/keras/engine/training.py\", line 992, in _make_train_function\n    loss=self.total_loss)\n  File \"/usr/local/lib/python3.6/dist-packages/keras/legacy/interfaces.py\", line 91, in wrapper\n    return func(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/keras/optimizers.py\", line 445, in get_updates\n    grads = self.get_gradients(loss, params)\n  File \"/usr/local/lib/python3.6/dist-packages/keras/optimizers.py\", line 78, in get_gradients\n    grads = K.gradients(loss, params)\n  File \"/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py\", line 2519, in gradients\n    return tf.gradients(loss, variables, colocate_gradients_with_ops=True)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/gradients_impl.py\", line 158, in gradients\n    unconnected_gradients)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/gradients_util.py\", line 722, in _GradientsHelper\n    out_grads[i] = control_flow_ops.ZerosLikeOutsideLoop(op, i)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/control_flow_ops.py\", line 1357, in ZerosLikeOutsideLoop\n    return array_ops.zeros(zeros_shape, dtype=val.dtype)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/array_ops.py\", line 1883, in zeros\n    output = fill(shape, constant(zero, dtype=dtype), name=name)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/gen_array_ops.py\", line 3613, in fill\n    \"Fill\", dims=dims, value=value, name=name)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py\", line 788, in _apply_op_helper\n    op_def=op_def)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/deprecation.py\", line 507, in new_func\n    return func(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py\", line 3616, in create_op\n    op_def=op_def)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py\", line 2005, in __init__\n    self._traceback = tf_stack.extract_stack()\n"
     ]
    }
   ],
   "source": [
    "#A multiple stage convolutional neural network\n",
    "\n",
    "#The neural network architectures will be built using combinations of the following parameters.\n",
    "params = {'classes' : [classes],\n",
    "          'activation' : ['relu', 'tanh'],\n",
    "          'batch_size': [1024],\n",
    "          'epochs': [200],\n",
    "          'optimizer': ['rmsprop', 'adam'],\n",
    "          'loss': ['sparse_categorical_crossentropy'],\n",
    "          'layers' : [[512, 128], [1024, 512], [4096, 2048]],\n",
    "          'callbacks' : [[keras.callbacks.EarlyStopping(monitor='val_loss',\n",
    "                              min_delta=0,\n",
    "                              patience=6,\n",
    "                              verbose=0, mode='auto')]]}\n",
    "\n",
    "#Hyperopt will optimize the following parameters in the given ranges.\n",
    "search_space = {'lr': hp.loguniform('lr', -8, -6), 'dropout_prob' : hp.uniform('dropout_prob', 0.2, 0.5)}\n",
    "\n",
    "#The call to process_model will compile, train and evaulate the above models and optimize the hyper parameters.\n",
    "#The output will be saved to a file and the output details will be recorded in the model_res_file. \n",
    "model_name = \"deepCNN\"\n",
    "meta_model.max_evals = 5\n",
    "model_results = meta_model.process_model(params, search_space, deep_cnn_model, model_name, [X_train, X_val, X_test, y_train, y_val, y_test], \n",
    "              [model_name, 0, 0, 0, 'layers', 'epochs', 'activation', 'optimizer', 'lr','reg_param', 'dropout_prob', 'batchNorm' , np.NaN])\n",
    "\n",
    "#Now lets plot the results.\n",
    "history = model_results[0][0]\n",
    "plot_train_val(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 7585151,
     "status": "ok",
     "timestamp": 1566179817629,
     "user": {
      "displayName": "Krishan Rajaratnam",
      "photoUrl": "https://lh6.googleusercontent.com/-Za30nR0ehyQ/AAAAAAAAAAI/AAAAAAAAGMw/jV8ldTp8928/s64/photo.jpg",
      "userId": "09748671397076320798"
     },
     "user_tz": 240
    },
    "id": "Zo6gNO62nGIU",
    "outputId": "2d14fc39-3868-413c-c0d2-6931a5a12e0c"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0818 23:50:34.182018 140156112357248 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:508: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "W0818 23:50:34.187039 140156112357248 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3837: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "W0818 23:50:34.223945 140156112357248 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:168: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
      "\n",
      "W0818 23:50:34.225319 140156112357248 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:175: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'classes': 10, 'layers': [3, 1024], 'conv_layer_start': 64, 'activation': 'relu', 'batch_size': 512, 'epochs': 50, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f784f2f69b0>], 'dropout_prob': 0.2262910809311608, 'lr': 0.0005109002007452063}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0818 23:50:34.713379 140156112357248 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1801: The name tf.nn.fused_batch_norm is deprecated. Please use tf.compat.v1.nn.fused_batch_norm instead.\n",
      "\n",
      "W0818 23:50:34.871857 140156112357248 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3661: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n",
      "\n",
      "W0818 23:50:34.878453 140156112357248 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3144: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "W0818 23:50:35.545478 140156112357248 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/optimizers.py:757: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n",
      "W0818 23:50:35.665121 140156112357248 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/50\n",
      "40000/40000 [==============================] - 21s 530us/step - loss: 1.9608 - acc: 0.3543 - val_loss: 1.5723 - val_acc: 0.4441\n",
      "Epoch 2/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 1.4376 - acc: 0.4953 - val_loss: 1.3244 - val_acc: 0.5272\n",
      "Epoch 3/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 1.2019 - acc: 0.5740 - val_loss: 1.1426 - val_acc: 0.5955\n",
      "Epoch 4/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 1.0621 - acc: 0.6230 - val_loss: 1.0465 - val_acc: 0.6281\n",
      "Epoch 5/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 0.9601 - acc: 0.6616 - val_loss: 0.9290 - val_acc: 0.6678\n",
      "Epoch 6/50\n",
      "40000/40000 [==============================] - 17s 417us/step - loss: 0.8628 - acc: 0.6964 - val_loss: 0.8918 - val_acc: 0.6850\n",
      "Epoch 7/50\n",
      "40000/40000 [==============================] - 17s 418us/step - loss: 0.7999 - acc: 0.7175 - val_loss: 0.8184 - val_acc: 0.7140\n",
      "Epoch 8/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 0.7390 - acc: 0.7383 - val_loss: 0.8090 - val_acc: 0.7173\n",
      "Epoch 9/50\n",
      "40000/40000 [==============================] - 17s 420us/step - loss: 0.6872 - acc: 0.7563 - val_loss: 0.7630 - val_acc: 0.7355\n",
      "Epoch 10/50\n",
      "40000/40000 [==============================] - 17s 420us/step - loss: 0.6356 - acc: 0.7769 - val_loss: 0.7630 - val_acc: 0.7361\n",
      "Epoch 11/50\n",
      "40000/40000 [==============================] - 17s 420us/step - loss: 0.6018 - acc: 0.7878 - val_loss: 0.7343 - val_acc: 0.7458\n",
      "Epoch 12/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 0.5717 - acc: 0.7964 - val_loss: 0.7070 - val_acc: 0.7612\n",
      "Epoch 13/50\n",
      "40000/40000 [==============================] - 17s 417us/step - loss: 0.5303 - acc: 0.8114 - val_loss: 0.6750 - val_acc: 0.7728\n",
      "Epoch 14/50\n",
      "40000/40000 [==============================] - 17s 416us/step - loss: 0.5080 - acc: 0.8201 - val_loss: 0.6970 - val_acc: 0.7633\n",
      "Epoch 15/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 0.4818 - acc: 0.8301 - val_loss: 0.6455 - val_acc: 0.7838\n",
      "Epoch 16/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 0.4573 - acc: 0.8378 - val_loss: 0.6456 - val_acc: 0.7828\n",
      "Epoch 17/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 0.4325 - acc: 0.8448 - val_loss: 0.6370 - val_acc: 0.7907\n",
      "Epoch 18/50\n",
      "40000/40000 [==============================] - 17s 420us/step - loss: 0.4037 - acc: 0.8578 - val_loss: 0.6384 - val_acc: 0.7887\n",
      "Epoch 19/50\n",
      "40000/40000 [==============================] - 17s 418us/step - loss: 0.3871 - acc: 0.8618 - val_loss: 0.6154 - val_acc: 0.8010\n",
      "Epoch 20/50\n",
      "40000/40000 [==============================] - 17s 418us/step - loss: 0.3662 - acc: 0.8696 - val_loss: 0.6611 - val_acc: 0.7898\n",
      "Epoch 21/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 0.3473 - acc: 0.8752 - val_loss: 0.6283 - val_acc: 0.7999\n",
      "10000/10000 [==============================] - 3s 280us/step\n",
      "{'classes': 10, 'layers': [3, 1024], 'conv_layer_start': 64, 'activation': 'relu', 'batch_size': 512, 'epochs': 50, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f784f2f69b0>], 'dropout_prob': 0.31902594686816155, 'lr': 0.0010273409623721955}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/50\n",
      "40000/40000 [==============================] - 19s 464us/step - loss: 1.9890 - acc: 0.3461 - val_loss: 1.5747 - val_acc: 0.4466\n",
      "Epoch 2/50\n",
      "40000/40000 [==============================] - 17s 420us/step - loss: 1.4152 - acc: 0.4996 - val_loss: 1.2812 - val_acc: 0.5499\n",
      "Epoch 3/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 1.1750 - acc: 0.5848 - val_loss: 1.1328 - val_acc: 0.6009\n",
      "Epoch 4/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 1.0342 - acc: 0.6361 - val_loss: 0.9816 - val_acc: 0.6571\n",
      "Epoch 5/50\n",
      "40000/40000 [==============================] - 17s 418us/step - loss: 0.9228 - acc: 0.6732 - val_loss: 0.9298 - val_acc: 0.6743\n",
      "Epoch 6/50\n",
      "40000/40000 [==============================] - 17s 418us/step - loss: 0.8258 - acc: 0.7100 - val_loss: 0.8463 - val_acc: 0.6987\n",
      "Epoch 7/50\n",
      "40000/40000 [==============================] - 17s 415us/step - loss: 0.7613 - acc: 0.7316 - val_loss: 0.7950 - val_acc: 0.7253\n",
      "Epoch 8/50\n",
      "40000/40000 [==============================] - 17s 417us/step - loss: 0.6995 - acc: 0.7533 - val_loss: 0.7566 - val_acc: 0.7345\n",
      "Epoch 9/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 0.6492 - acc: 0.7723 - val_loss: 0.7359 - val_acc: 0.7421\n",
      "Epoch 10/50\n",
      "40000/40000 [==============================] - 17s 420us/step - loss: 0.6111 - acc: 0.7838 - val_loss: 0.6878 - val_acc: 0.7599\n",
      "Epoch 11/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 0.5744 - acc: 0.7994 - val_loss: 0.6712 - val_acc: 0.7690\n",
      "Epoch 12/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 0.5513 - acc: 0.8041 - val_loss: 0.6577 - val_acc: 0.7753\n",
      "Epoch 13/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 0.5165 - acc: 0.8167 - val_loss: 0.6544 - val_acc: 0.7821\n",
      "Epoch 14/50\n",
      "40000/40000 [==============================] - 17s 418us/step - loss: 0.4902 - acc: 0.8283 - val_loss: 0.6438 - val_acc: 0.7814\n",
      "Epoch 15/50\n",
      "40000/40000 [==============================] - 17s 416us/step - loss: 0.4658 - acc: 0.8338 - val_loss: 0.6357 - val_acc: 0.7875\n",
      "Epoch 16/50\n",
      "40000/40000 [==============================] - 17s 417us/step - loss: 0.4514 - acc: 0.8423 - val_loss: 0.6352 - val_acc: 0.7878\n",
      "Epoch 17/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 0.4218 - acc: 0.8508 - val_loss: 0.6008 - val_acc: 0.8000\n",
      "Epoch 18/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 0.3935 - acc: 0.8610 - val_loss: 0.6061 - val_acc: 0.8024\n",
      "Epoch 19/50\n",
      "40000/40000 [==============================] - 17s 418us/step - loss: 0.3886 - acc: 0.8626 - val_loss: 0.6021 - val_acc: 0.8003\n",
      "10000/10000 [==============================] - 3s 275us/step\n",
      "{'classes': 10, 'layers': [3, 1024], 'conv_layer_start': 64, 'activation': 'relu', 'batch_size': 512, 'epochs': 50, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f784f2f69b0>], 'dropout_prob': 0.434497109569947, 'lr': 0.0016573876575835507}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/50\n",
      "40000/40000 [==============================] - 19s 473us/step - loss: 2.1156 - acc: 0.3135 - val_loss: 1.6584 - val_acc: 0.4107\n",
      "Epoch 2/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 1.5238 - acc: 0.4569 - val_loss: 1.3724 - val_acc: 0.5095\n",
      "Epoch 3/50\n",
      "40000/40000 [==============================] - 17s 418us/step - loss: 1.2917 - acc: 0.5391 - val_loss: 1.2062 - val_acc: 0.5691\n",
      "Epoch 4/50\n",
      "40000/40000 [==============================] - 17s 416us/step - loss: 1.1145 - acc: 0.6015 - val_loss: 1.0515 - val_acc: 0.6240\n",
      "Epoch 5/50\n",
      "40000/40000 [==============================] - 17s 420us/step - loss: 0.9758 - acc: 0.6516 - val_loss: 0.9513 - val_acc: 0.6609\n",
      "Epoch 6/50\n",
      "40000/40000 [==============================] - 17s 420us/step - loss: 0.8720 - acc: 0.6908 - val_loss: 0.8663 - val_acc: 0.6887\n",
      "Epoch 7/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 0.7966 - acc: 0.7164 - val_loss: 0.8173 - val_acc: 0.7089\n",
      "Epoch 8/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 0.7437 - acc: 0.7387 - val_loss: 0.7854 - val_acc: 0.7227\n",
      "Epoch 9/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 0.7098 - acc: 0.7498 - val_loss: 0.7571 - val_acc: 0.7356\n",
      "Epoch 10/50\n",
      "40000/40000 [==============================] - 17s 420us/step - loss: 0.6622 - acc: 0.7679 - val_loss: 0.7094 - val_acc: 0.7533\n",
      "Epoch 11/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 0.6209 - acc: 0.7809 - val_loss: 0.6954 - val_acc: 0.7609\n",
      "Epoch 12/50\n",
      "40000/40000 [==============================] - 17s 416us/step - loss: 0.5973 - acc: 0.7913 - val_loss: 0.6731 - val_acc: 0.7674\n",
      "Epoch 13/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 0.5620 - acc: 0.8032 - val_loss: 0.6624 - val_acc: 0.7681\n",
      "Epoch 14/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 0.5429 - acc: 0.8068 - val_loss: 0.6672 - val_acc: 0.7675\n",
      "Epoch 15/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 0.5257 - acc: 0.8133 - val_loss: 0.6429 - val_acc: 0.7815\n",
      "Epoch 16/50\n",
      "40000/40000 [==============================] - 17s 418us/step - loss: 0.4989 - acc: 0.8252 - val_loss: 0.6378 - val_acc: 0.7838\n",
      "Epoch 17/50\n",
      "40000/40000 [==============================] - 17s 420us/step - loss: 0.4892 - acc: 0.8277 - val_loss: 0.6473 - val_acc: 0.7860\n",
      "Epoch 18/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 0.4643 - acc: 0.8366 - val_loss: 0.6249 - val_acc: 0.7899\n",
      "Epoch 19/50\n",
      "40000/40000 [==============================] - 17s 419us/step - loss: 0.4387 - acc: 0.8476 - val_loss: 0.6160 - val_acc: 0.7909\n",
      "Epoch 20/50\n",
      "40000/40000 [==============================] - 17s 416us/step - loss: 0.4319 - acc: 0.8488 - val_loss: 0.6148 - val_acc: 0.8007\n",
      "Epoch 21/50\n",
      "40000/40000 [==============================] - 17s 420us/step - loss: 0.4141 - acc: 0.8549 - val_loss: 0.6052 - val_acc: 0.8037\n",
      "Epoch 22/50\n",
      "40000/40000 [==============================] - 17s 420us/step - loss: 0.4073 - acc: 0.8582 - val_loss: 0.5934 - val_acc: 0.8055\n",
      "Epoch 23/50\n",
      "40000/40000 [==============================] - 17s 420us/step - loss: 0.3917 - acc: 0.8628 - val_loss: 0.6062 - val_acc: 0.8074\n",
      "Epoch 24/50\n",
      "40000/40000 [==============================] - 17s 420us/step - loss: 0.3748 - acc: 0.8683 - val_loss: 0.5991 - val_acc: 0.8051\n",
      "10000/10000 [==============================] - 3s 278us/step\n",
      "40000/40000 [==============================] - 11s 277us/step\n",
      "10000/10000 [==============================] - 3s 292us/step\n",
      "Loss = 0.6895639985084534\n",
      "Test Accuracy = 0.7793\n",
      "{'classes': 10, 'layers': [3, 1024], 'conv_layer_start': 64, 'activation': 'relu', 'batch_size': 512, 'epochs': 50, 'optimizer': 'adadelta', 'loss': 'sparse_categorical_crossentropy', 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f784f2f69b0>], 'dropout_prob': 0.3758335428030174, 'lr': 0.0005294967203913549}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/50\n",
      "40000/40000 [==============================] - 19s 486us/step - loss: 3.4023 - acc: 0.1041 - val_loss: 3.4103 - val_acc: 0.1025\n",
      "Epoch 2/50\n",
      "40000/40000 [==============================] - 17s 427us/step - loss: 3.3850 - acc: 0.1060 - val_loss: 3.3986 - val_acc: 0.1120\n",
      "Epoch 3/50\n",
      "40000/40000 [==============================] - 17s 425us/step - loss: 3.3884 - acc: 0.1075 - val_loss: 3.3309 - val_acc: 0.1103\n",
      "Epoch 4/50\n",
      "40000/40000 [==============================] - 17s 428us/step - loss: 3.3514 - acc: 0.1131 - val_loss: 3.3670 - val_acc: 0.1089\n",
      "Epoch 5/50\n",
      "40000/40000 [==============================] - 17s 427us/step - loss: 3.3627 - acc: 0.1111 - val_loss: 3.3451 - val_acc: 0.1103\n",
      "10000/10000 [==============================] - 3s 278us/step\n",
      "{'classes': 10, 'layers': [3, 1024], 'conv_layer_start': 64, 'activation': 'relu', 'batch_size': 512, 'epochs': 50, 'optimizer': 'adadelta', 'loss': 'sparse_categorical_crossentropy', 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f784f2f69b0>], 'dropout_prob': 0.26204778597378675, 'lr': 0.000407147775442425}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/50\n",
      "40000/40000 [==============================] - 20s 496us/step - loss: 3.2917 - acc: 0.1004 - val_loss: 3.3058 - val_acc: 0.1003\n",
      "Epoch 2/50\n",
      "40000/40000 [==============================] - 17s 426us/step - loss: 3.2751 - acc: 0.1025 - val_loss: 3.2348 - val_acc: 0.1053\n",
      "Epoch 3/50\n",
      "40000/40000 [==============================] - 17s 427us/step - loss: 3.2558 - acc: 0.1050 - val_loss: 3.2233 - val_acc: 0.1058\n",
      "Epoch 4/50\n",
      "40000/40000 [==============================] - 17s 427us/step - loss: 3.2392 - acc: 0.1081 - val_loss: 3.1834 - val_acc: 0.1164\n",
      "Epoch 5/50\n",
      "40000/40000 [==============================] - 17s 424us/step - loss: 3.2026 - acc: 0.1085 - val_loss: 3.1674 - val_acc: 0.1118\n",
      "Epoch 6/50\n",
      "40000/40000 [==============================] - 17s 425us/step - loss: 3.1700 - acc: 0.1143 - val_loss: 3.1437 - val_acc: 0.1144\n",
      "Epoch 7/50\n",
      "40000/40000 [==============================] - 17s 427us/step - loss: 3.1556 - acc: 0.1149 - val_loss: 3.1312 - val_acc: 0.1197\n",
      "Epoch 8/50\n",
      "40000/40000 [==============================] - 17s 426us/step - loss: 3.1245 - acc: 0.1190 - val_loss: 3.1303 - val_acc: 0.1174\n",
      "Epoch 9/50\n",
      "40000/40000 [==============================] - 17s 427us/step - loss: 3.1086 - acc: 0.1212 - val_loss: 3.0887 - val_acc: 0.1245\n",
      "Epoch 10/50\n",
      "40000/40000 [==============================] - 17s 426us/step - loss: 3.0911 - acc: 0.1241 - val_loss: 3.0860 - val_acc: 0.1242\n",
      "Epoch 11/50\n",
      "40000/40000 [==============================] - 17s 426us/step - loss: 3.0746 - acc: 0.1256 - val_loss: 3.0844 - val_acc: 0.1270\n",
      "Epoch 12/50\n",
      "40000/40000 [==============================] - 17s 426us/step - loss: 3.0672 - acc: 0.1278 - val_loss: 3.0880 - val_acc: 0.1219\n",
      "Epoch 13/50\n",
      "40000/40000 [==============================] - 17s 424us/step - loss: 3.0532 - acc: 0.1280 - val_loss: 3.0463 - val_acc: 0.1272\n",
      "Epoch 14/50\n",
      "40000/40000 [==============================] - 17s 425us/step - loss: 3.0288 - acc: 0.1346 - val_loss: 3.0383 - val_acc: 0.1292\n",
      "Epoch 15/50\n",
      "40000/40000 [==============================] - 17s 426us/step - loss: 3.0096 - acc: 0.1340 - val_loss: 3.0102 - val_acc: 0.1345\n",
      "Epoch 16/50\n",
      "40000/40000 [==============================] - 17s 427us/step - loss: 3.0143 - acc: 0.1368 - val_loss: 3.0052 - val_acc: 0.1392\n",
      "Epoch 17/50\n",
      "40000/40000 [==============================] - 17s 426us/step - loss: 2.9945 - acc: 0.1391 - val_loss: 2.9871 - val_acc: 0.1346\n",
      "Epoch 18/50\n",
      "40000/40000 [==============================] - 17s 426us/step - loss: 2.9727 - acc: 0.1429 - val_loss: 2.9728 - val_acc: 0.1423\n",
      "Epoch 19/50\n",
      "40000/40000 [==============================] - 17s 426us/step - loss: 2.9696 - acc: 0.1431 - val_loss: 2.9645 - val_acc: 0.1422\n",
      "Epoch 20/50\n",
      "40000/40000 [==============================] - 17s 425us/step - loss: 2.9526 - acc: 0.1446 - val_loss: 2.9488 - val_acc: 0.1414\n",
      "Epoch 21/50\n",
      "40000/40000 [==============================] - 17s 424us/step - loss: 2.9258 - acc: 0.1476 - val_loss: 2.9626 - val_acc: 0.1450\n",
      "Epoch 22/50\n",
      "40000/40000 [==============================] - 17s 425us/step - loss: 2.9306 - acc: 0.1492 - val_loss: 2.9228 - val_acc: 0.1460\n",
      "Epoch 23/50\n",
      "40000/40000 [==============================] - 17s 425us/step - loss: 2.9125 - acc: 0.1541 - val_loss: 2.9324 - val_acc: 0.1432\n",
      "Epoch 24/50\n",
      "40000/40000 [==============================] - 17s 426us/step - loss: 2.9006 - acc: 0.1538 - val_loss: 2.8879 - val_acc: 0.1606\n",
      "Epoch 25/50\n",
      "40000/40000 [==============================] - 17s 426us/step - loss: 2.8905 - acc: 0.1577 - val_loss: 2.8807 - val_acc: 0.1587\n",
      "Epoch 26/50\n",
      "40000/40000 [==============================] - 17s 425us/step - loss: 2.8799 - acc: 0.1549 - val_loss: 2.8626 - val_acc: 0.1589\n",
      "Epoch 27/50\n",
      "40000/40000 [==============================] - 17s 426us/step - loss: 2.8602 - acc: 0.1592 - val_loss: 2.8809 - val_acc: 0.1586\n",
      "Epoch 28/50\n",
      "40000/40000 [==============================] - 17s 425us/step - loss: 2.8676 - acc: 0.1603 - val_loss: 2.8707 - val_acc: 0.1657\n",
      "10000/10000 [==============================] - 3s 284us/step\n",
      "{'classes': 10, 'layers': [3, 1024], 'conv_layer_start': 64, 'activation': 'relu', 'batch_size': 512, 'epochs': 50, 'optimizer': 'adadelta', 'loss': 'sparse_categorical_crossentropy', 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f784f2f69b0>], 'dropout_prob': 0.38162782150887564, 'lr': 0.0016550752180412571}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/50\n",
      "40000/40000 [==============================] - 20s 494us/step - loss: 3.4913 - acc: 0.0999 - val_loss: 3.4377 - val_acc: 0.1048\n",
      "Epoch 2/50\n",
      "40000/40000 [==============================] - 17s 427us/step - loss: 3.4526 - acc: 0.1042 - val_loss: 3.3924 - val_acc: 0.1094\n",
      "Epoch 3/50\n",
      "40000/40000 [==============================] - 17s 426us/step - loss: 3.4013 - acc: 0.1095 - val_loss: 3.3513 - val_acc: 0.1127\n",
      "Epoch 4/50\n",
      "40000/40000 [==============================] - 17s 424us/step - loss: 3.3361 - acc: 0.1164 - val_loss: 3.3535 - val_acc: 0.1177\n",
      "Epoch 5/50\n",
      "40000/40000 [==============================] - 17s 426us/step - loss: 3.3299 - acc: 0.1180 - val_loss: 3.3613 - val_acc: 0.1136\n",
      "10000/10000 [==============================] - 3s 284us/step\n",
      "40000/40000 [==============================] - 12s 295us/step\n",
      "10000/10000 [==============================] - 3s 300us/step\n",
      "Loss = 2.8454482166290282\n",
      "Test Accuracy = 0.1638\n",
      "{'classes': 10, 'layers': [3, 1024], 'conv_layer_start': 128, 'activation': 'relu', 'batch_size': 512, 'epochs': 50, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f784f2f69b0>], 'dropout_prob': 0.2538355114456819, 'lr': 0.001762649117034368}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/50\n",
      "40000/40000 [==============================] - 46s 1ms/step - loss: 1.7077 - acc: 0.4198 - val_loss: 1.3447 - val_acc: 0.5228\n",
      "Epoch 2/50\n",
      "40000/40000 [==============================] - 37s 922us/step - loss: 1.1600 - acc: 0.5933 - val_loss: 1.0220 - val_acc: 0.6333\n",
      "Epoch 3/50\n",
      "40000/40000 [==============================] - 37s 922us/step - loss: 0.9083 - acc: 0.6845 - val_loss: 0.8634 - val_acc: 0.6992\n",
      "Epoch 4/50\n",
      "40000/40000 [==============================] - 37s 920us/step - loss: 0.7334 - acc: 0.7411 - val_loss: 0.7456 - val_acc: 0.7416\n",
      "Epoch 5/50\n",
      "40000/40000 [==============================] - 37s 922us/step - loss: 0.6451 - acc: 0.7750 - val_loss: 0.6858 - val_acc: 0.7636\n",
      "Epoch 6/50\n",
      "40000/40000 [==============================] - 37s 921us/step - loss: 0.5557 - acc: 0.8051 - val_loss: 0.6353 - val_acc: 0.7817\n",
      "Epoch 7/50\n",
      "40000/40000 [==============================] - 37s 921us/step - loss: 0.4915 - acc: 0.8252 - val_loss: 0.6151 - val_acc: 0.7903\n",
      "Epoch 8/50\n",
      "40000/40000 [==============================] - 37s 919us/step - loss: 0.4421 - acc: 0.8434 - val_loss: 0.5937 - val_acc: 0.7987\n",
      "Epoch 9/50\n",
      "40000/40000 [==============================] - 37s 922us/step - loss: 0.3883 - acc: 0.8614 - val_loss: 0.5764 - val_acc: 0.8089\n",
      "Epoch 10/50\n",
      "40000/40000 [==============================] - 37s 922us/step - loss: 0.3345 - acc: 0.8805 - val_loss: 0.6298 - val_acc: 0.8006\n",
      "Epoch 11/50\n",
      "40000/40000 [==============================] - 37s 921us/step - loss: 0.2939 - acc: 0.8954 - val_loss: 0.5730 - val_acc: 0.8163\n",
      "Epoch 12/50\n",
      "40000/40000 [==============================] - 37s 921us/step - loss: 0.2575 - acc: 0.9084 - val_loss: 0.6114 - val_acc: 0.8142\n",
      "Epoch 13/50\n",
      "40000/40000 [==============================] - 37s 921us/step - loss: 0.2349 - acc: 0.9152 - val_loss: 0.6145 - val_acc: 0.8192\n",
      "10000/10000 [==============================] - 5s 527us/step\n",
      "{'classes': 10, 'layers': [3, 1024], 'conv_layer_start': 128, 'activation': 'relu', 'batch_size': 512, 'epochs': 50, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f784f2f69b0>], 'dropout_prob': 0.4432276624530292, 'lr': 0.0004666347577574277}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/50\n",
      "40000/40000 [==============================] - 41s 1ms/step - loss: 2.3159 - acc: 0.2941 - val_loss: 1.8084 - val_acc: 0.3783\n",
      "Epoch 2/50\n",
      "40000/40000 [==============================] - 37s 920us/step - loss: 1.6527 - acc: 0.4243 - val_loss: 1.5086 - val_acc: 0.4755\n",
      "Epoch 3/50\n",
      "40000/40000 [==============================] - 37s 922us/step - loss: 1.4170 - acc: 0.5082 - val_loss: 1.3182 - val_acc: 0.5378\n",
      "Epoch 4/50\n",
      "40000/40000 [==============================] - 37s 922us/step - loss: 1.2480 - acc: 0.5657 - val_loss: 1.2130 - val_acc: 0.5788\n",
      "Epoch 5/50\n",
      "40000/40000 [==============================] - 37s 923us/step - loss: 1.1318 - acc: 0.6075 - val_loss: 1.1039 - val_acc: 0.6156\n",
      "Epoch 6/50\n",
      "40000/40000 [==============================] - 37s 920us/step - loss: 1.0420 - acc: 0.6408 - val_loss: 0.9983 - val_acc: 0.6527\n",
      "Epoch 7/50\n",
      "40000/40000 [==============================] - 37s 925us/step - loss: 0.9435 - acc: 0.6713 - val_loss: 0.9457 - val_acc: 0.6707\n",
      "Epoch 8/50\n",
      "40000/40000 [==============================] - 37s 923us/step - loss: 0.8746 - acc: 0.6961 - val_loss: 0.8782 - val_acc: 0.6916\n",
      "Epoch 9/50\n",
      "40000/40000 [==============================] - 37s 919us/step - loss: 0.8082 - acc: 0.7190 - val_loss: 0.8545 - val_acc: 0.7065\n",
      "Epoch 10/50\n",
      "40000/40000 [==============================] - 37s 922us/step - loss: 0.7503 - acc: 0.7392 - val_loss: 0.7939 - val_acc: 0.7354\n",
      "Epoch 11/50\n",
      "40000/40000 [==============================] - 37s 924us/step - loss: 0.7064 - acc: 0.7531 - val_loss: 0.7651 - val_acc: 0.7371\n",
      "Epoch 12/50\n",
      "40000/40000 [==============================] - 37s 924us/step - loss: 0.6603 - acc: 0.7698 - val_loss: 0.7272 - val_acc: 0.7502\n",
      "Epoch 13/50\n",
      "40000/40000 [==============================] - 37s 920us/step - loss: 0.6233 - acc: 0.7825 - val_loss: 0.6991 - val_acc: 0.7619\n",
      "Epoch 14/50\n",
      "40000/40000 [==============================] - 37s 923us/step - loss: 0.5822 - acc: 0.7968 - val_loss: 0.6955 - val_acc: 0.7645\n",
      "Epoch 15/50\n",
      "40000/40000 [==============================] - 37s 923us/step - loss: 0.5660 - acc: 0.7996 - val_loss: 0.6681 - val_acc: 0.7723\n",
      "Epoch 16/50\n",
      "40000/40000 [==============================] - 37s 923us/step - loss: 0.5275 - acc: 0.8153 - val_loss: 0.6493 - val_acc: 0.7774\n",
      "Epoch 17/50\n",
      "40000/40000 [==============================] - 37s 922us/step - loss: 0.5079 - acc: 0.8224 - val_loss: 0.6259 - val_acc: 0.7919\n",
      "Epoch 18/50\n",
      "40000/40000 [==============================] - 37s 923us/step - loss: 0.4842 - acc: 0.8302 - val_loss: 0.6331 - val_acc: 0.7846\n",
      "Epoch 19/50\n",
      "40000/40000 [==============================] - 37s 923us/step - loss: 0.4572 - acc: 0.8389 - val_loss: 0.6028 - val_acc: 0.7997\n",
      "Epoch 20/50\n",
      "40000/40000 [==============================] - 37s 923us/step - loss: 0.4350 - acc: 0.8459 - val_loss: 0.6015 - val_acc: 0.8000\n",
      "Epoch 21/50\n",
      "40000/40000 [==============================] - 37s 923us/step - loss: 0.4244 - acc: 0.8504 - val_loss: 0.5972 - val_acc: 0.8025\n",
      "Epoch 22/50\n",
      "40000/40000 [==============================] - 37s 923us/step - loss: 0.4043 - acc: 0.8578 - val_loss: 0.5751 - val_acc: 0.8099\n",
      "Epoch 23/50\n",
      "40000/40000 [==============================] - 37s 923us/step - loss: 0.3775 - acc: 0.8681 - val_loss: 0.5903 - val_acc: 0.8077\n",
      "Epoch 24/50\n",
      "40000/40000 [==============================] - 37s 921us/step - loss: 0.3625 - acc: 0.8694 - val_loss: 0.5877 - val_acc: 0.8129\n",
      "10000/10000 [==============================] - 5s 505us/step\n",
      "{'classes': 10, 'layers': [3, 1024], 'conv_layer_start': 128, 'activation': 'relu', 'batch_size': 512, 'epochs': 50, 'optimizer': 'rmsprop', 'loss': 'sparse_categorical_crossentropy', 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f784f2f69b0>], 'dropout_prob': 0.2634613130218564, 'lr': 0.00035957923687231826}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/50\n",
      "40000/40000 [==============================] - 41s 1ms/step - loss: 1.9335 - acc: 0.3703 - val_loss: 1.5467 - val_acc: 0.4642\n",
      "Epoch 2/50\n",
      "40000/40000 [==============================] - 37s 924us/step - loss: 1.3844 - acc: 0.5180 - val_loss: 1.2646 - val_acc: 0.5626\n",
      "Epoch 3/50\n",
      "40000/40000 [==============================] - 37s 923us/step - loss: 1.1641 - acc: 0.5945 - val_loss: 1.0978 - val_acc: 0.6171\n",
      "Epoch 4/50\n",
      "40000/40000 [==============================] - 37s 924us/step - loss: 1.0007 - acc: 0.6517 - val_loss: 0.9862 - val_acc: 0.6557\n",
      "Epoch 5/50\n",
      "40000/40000 [==============================] - 37s 923us/step - loss: 0.8839 - acc: 0.6933 - val_loss: 0.8861 - val_acc: 0.6931\n",
      "Epoch 6/50\n",
      "40000/40000 [==============================] - 37s 921us/step - loss: 0.7902 - acc: 0.7245 - val_loss: 0.8383 - val_acc: 0.7105\n",
      "Epoch 7/50\n",
      "40000/40000 [==============================] - 37s 923us/step - loss: 0.7196 - acc: 0.7464 - val_loss: 0.7717 - val_acc: 0.7333\n",
      "Epoch 8/50\n",
      "40000/40000 [==============================] - 37s 924us/step - loss: 0.6490 - acc: 0.7725 - val_loss: 0.7331 - val_acc: 0.7484\n",
      "Epoch 9/50\n",
      "40000/40000 [==============================] - 37s 924us/step - loss: 0.5966 - acc: 0.7896 - val_loss: 0.7040 - val_acc: 0.7555\n",
      "Epoch 10/50\n",
      "40000/40000 [==============================] - 37s 921us/step - loss: 0.5509 - acc: 0.8084 - val_loss: 0.6768 - val_acc: 0.7662\n",
      "Epoch 11/50\n",
      "40000/40000 [==============================] - 37s 924us/step - loss: 0.5156 - acc: 0.8194 - val_loss: 0.6754 - val_acc: 0.7752\n",
      "Epoch 12/50\n",
      "40000/40000 [==============================] - 37s 923us/step - loss: 0.4686 - acc: 0.8344 - val_loss: 0.6575 - val_acc: 0.7805\n",
      "Epoch 13/50\n",
      "40000/40000 [==============================] - 37s 924us/step - loss: 0.4500 - acc: 0.8406 - val_loss: 0.6355 - val_acc: 0.7878\n",
      "Epoch 14/50\n",
      "40000/40000 [==============================] - 37s 921us/step - loss: 0.3999 - acc: 0.8582 - val_loss: 0.6270 - val_acc: 0.7957\n",
      "Epoch 15/50\n",
      "40000/40000 [==============================] - 37s 922us/step - loss: 0.3658 - acc: 0.8715 - val_loss: 0.6137 - val_acc: 0.7988\n",
      "Epoch 16/50\n",
      "40000/40000 [==============================] - 37s 923us/step - loss: 0.3418 - acc: 0.8787 - val_loss: 0.6126 - val_acc: 0.7953\n",
      "Epoch 17/50\n",
      "40000/40000 [==============================] - 37s 921us/step - loss: 0.3154 - acc: 0.8892 - val_loss: 0.6332 - val_acc: 0.8003\n",
      "Epoch 18/50\n",
      "40000/40000 [==============================] - 37s 923us/step - loss: 0.2891 - acc: 0.8969 - val_loss: 0.6127 - val_acc: 0.8114\n",
      "10000/10000 [==============================] - 5s 499us/step\n",
      "40000/40000 [==============================] - 20s 505us/step\n",
      "10000/10000 [==============================] - 5s 513us/step\n",
      "Loss = 0.6713437209367752\n",
      "Test Accuracy = 0.7888\n",
      "{'classes': 10, 'layers': [3, 1024], 'conv_layer_start': 128, 'activation': 'relu', 'batch_size': 512, 'epochs': 50, 'optimizer': 'adadelta', 'loss': 'sparse_categorical_crossentropy', 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f784f2f69b0>], 'dropout_prob': 0.41706030063298793, 'lr': 0.0004902336360626134}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/50\n",
      "40000/40000 [==============================] - 42s 1ms/step - loss: 3.4760 - acc: 0.1064 - val_loss: 3.4568 - val_acc: 0.1114\n",
      "Epoch 2/50\n",
      "40000/40000 [==============================] - 37s 929us/step - loss: 3.4473 - acc: 0.1061 - val_loss: 3.4544 - val_acc: 0.1086\n",
      "Epoch 3/50\n",
      "40000/40000 [==============================] - 37s 932us/step - loss: 3.4071 - acc: 0.1125 - val_loss: 3.4047 - val_acc: 0.1149\n",
      "Epoch 4/50\n",
      "40000/40000 [==============================] - 37s 933us/step - loss: 3.3898 - acc: 0.1127 - val_loss: 3.3829 - val_acc: 0.1154\n",
      "Epoch 5/50\n",
      "40000/40000 [==============================] - 37s 931us/step - loss: 3.3694 - acc: 0.1159 - val_loss: 3.3653 - val_acc: 0.1188\n",
      "Epoch 6/50\n",
      "40000/40000 [==============================] - 37s 931us/step - loss: 3.3580 - acc: 0.1203 - val_loss: 3.3664 - val_acc: 0.1174\n",
      "Epoch 7/50\n",
      "40000/40000 [==============================] - 37s 931us/step - loss: 3.3249 - acc: 0.1221 - val_loss: 3.3410 - val_acc: 0.1174\n",
      "Epoch 8/50\n",
      "40000/40000 [==============================] - 37s 931us/step - loss: 3.3054 - acc: 0.1236 - val_loss: 3.2927 - val_acc: 0.1254\n",
      "Epoch 9/50\n",
      "40000/40000 [==============================] - 37s 928us/step - loss: 3.3040 - acc: 0.1248 - val_loss: 3.2976 - val_acc: 0.1289\n",
      "Epoch 10/50\n",
      "40000/40000 [==============================] - 37s 930us/step - loss: 3.2567 - acc: 0.1323 - val_loss: 3.2658 - val_acc: 0.1292\n",
      "Epoch 11/50\n",
      "40000/40000 [==============================] - 37s 930us/step - loss: 3.2630 - acc: 0.1297 - val_loss: 3.2307 - val_acc: 0.1324\n",
      "Epoch 12/50\n",
      "40000/40000 [==============================] - 37s 931us/step - loss: 3.2479 - acc: 0.1323 - val_loss: 3.2229 - val_acc: 0.1346\n",
      "Epoch 13/50\n",
      "40000/40000 [==============================] - 37s 929us/step - loss: 3.2266 - acc: 0.1353 - val_loss: 3.2546 - val_acc: 0.1308\n",
      "Epoch 14/50\n",
      "40000/40000 [==============================] - 37s 931us/step - loss: 3.2203 - acc: 0.1347 - val_loss: 3.1890 - val_acc: 0.1370\n",
      "Epoch 15/50\n",
      "40000/40000 [==============================] - 37s 930us/step - loss: 3.1918 - acc: 0.1404 - val_loss: 3.1826 - val_acc: 0.1350\n",
      "Epoch 16/50\n",
      "40000/40000 [==============================] - 37s 930us/step - loss: 3.2050 - acc: 0.1390 - val_loss: 3.1852 - val_acc: 0.1445\n",
      "Epoch 17/50\n",
      "40000/40000 [==============================] - 37s 930us/step - loss: 3.1906 - acc: 0.1422 - val_loss: 3.1971 - val_acc: 0.1390\n",
      "10000/10000 [==============================] - 5s 500us/step\n",
      "{'classes': 10, 'layers': [3, 1024], 'conv_layer_start': 128, 'activation': 'relu', 'batch_size': 512, 'epochs': 50, 'optimizer': 'adadelta', 'loss': 'sparse_categorical_crossentropy', 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f784f2f69b0>], 'dropout_prob': 0.4401168168092132, 'lr': 0.0018712420190298163}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/50\n",
      "40000/40000 [==============================] - 42s 1ms/step - loss: 3.5513 - acc: 0.1042 - val_loss: 3.4874 - val_acc: 0.1118\n",
      "Epoch 2/50\n",
      "40000/40000 [==============================] - 37s 931us/step - loss: 3.4271 - acc: 0.1167 - val_loss: 3.3965 - val_acc: 0.1210\n",
      "Epoch 3/50\n",
      "40000/40000 [==============================] - 37s 928us/step - loss: 3.3254 - acc: 0.1286 - val_loss: 3.2557 - val_acc: 0.1369\n",
      "Epoch 4/50\n",
      "40000/40000 [==============================] - 37s 930us/step - loss: 3.2563 - acc: 0.1390 - val_loss: 3.2527 - val_acc: 0.1391\n",
      "Epoch 5/50\n",
      "40000/40000 [==============================] - 37s 932us/step - loss: 3.2088 - acc: 0.1432 - val_loss: 3.1594 - val_acc: 0.1489\n",
      "Epoch 6/50\n",
      "40000/40000 [==============================] - 37s 932us/step - loss: 3.1542 - acc: 0.1521 - val_loss: 3.1251 - val_acc: 0.1551\n",
      "Epoch 7/50\n",
      "40000/40000 [==============================] - 37s 932us/step - loss: 3.1071 - acc: 0.1584 - val_loss: 3.1101 - val_acc: 0.1568\n",
      "Epoch 8/50\n",
      "40000/40000 [==============================] - 37s 932us/step - loss: 3.0480 - acc: 0.1671 - val_loss: 3.0371 - val_acc: 0.1728\n",
      "Epoch 9/50\n",
      "40000/40000 [==============================] - 37s 930us/step - loss: 3.0337 - acc: 0.1717 - val_loss: 3.0143 - val_acc: 0.1740\n",
      "Epoch 10/50\n",
      "40000/40000 [==============================] - 37s 932us/step - loss: 2.9730 - acc: 0.1813 - val_loss: 3.0012 - val_acc: 0.1760\n",
      "Epoch 11/50\n",
      "40000/40000 [==============================] - 37s 932us/step - loss: 2.9552 - acc: 0.1802 - val_loss: 2.9173 - val_acc: 0.1900\n",
      "Epoch 12/50\n",
      "40000/40000 [==============================] - 37s 932us/step - loss: 2.9345 - acc: 0.1870 - val_loss: 2.9353 - val_acc: 0.1826\n",
      "Epoch 13/50\n",
      "40000/40000 [==============================] - 37s 929us/step - loss: 2.9002 - acc: 0.1915 - val_loss: 2.8931 - val_acc: 0.1920\n",
      "Epoch 14/50\n",
      "40000/40000 [==============================] - 37s 933us/step - loss: 2.8880 - acc: 0.1922 - val_loss: 2.8495 - val_acc: 0.1997\n",
      "Epoch 15/50\n",
      "40000/40000 [==============================] - 37s 932us/step - loss: 2.8500 - acc: 0.2041 - val_loss: 2.8657 - val_acc: 0.1963\n",
      "Epoch 16/50\n",
      "40000/40000 [==============================] - 37s 932us/step - loss: 2.8411 - acc: 0.2040 - val_loss: 2.8234 - val_acc: 0.2091\n",
      "Epoch 17/50\n",
      "40000/40000 [==============================] - 37s 929us/step - loss: 2.8152 - acc: 0.2105 - val_loss: 2.7771 - val_acc: 0.2145\n",
      "Epoch 18/50\n",
      "40000/40000 [==============================] - 37s 932us/step - loss: 2.7887 - acc: 0.2098 - val_loss: 2.7618 - val_acc: 0.2153\n",
      "Epoch 19/50\n",
      "40000/40000 [==============================] - 37s 931us/step - loss: 2.7654 - acc: 0.2186 - val_loss: 2.7969 - val_acc: 0.2088\n",
      "Epoch 20/50\n",
      "40000/40000 [==============================] - 37s 929us/step - loss: 2.7494 - acc: 0.2165 - val_loss: 2.7556 - val_acc: 0.2173\n",
      "Epoch 21/50\n",
      "40000/40000 [==============================] - 37s 929us/step - loss: 2.7312 - acc: 0.2225 - val_loss: 2.7492 - val_acc: 0.2230\n",
      "Epoch 22/50\n",
      "40000/40000 [==============================] - 37s 931us/step - loss: 2.7089 - acc: 0.2260 - val_loss: 2.7124 - val_acc: 0.2234\n",
      "Epoch 23/50\n",
      "40000/40000 [==============================] - 37s 932us/step - loss: 2.6919 - acc: 0.2285 - val_loss: 2.7117 - val_acc: 0.2260\n",
      "Epoch 24/50\n",
      "40000/40000 [==============================] - 37s 930us/step - loss: 2.6805 - acc: 0.2307 - val_loss: 2.6867 - val_acc: 0.2312\n",
      "Epoch 25/50\n",
      "40000/40000 [==============================] - 37s 931us/step - loss: 2.6758 - acc: 0.2336 - val_loss: 2.6631 - val_acc: 0.2314\n",
      "Epoch 26/50\n",
      "40000/40000 [==============================] - 37s 930us/step - loss: 2.6555 - acc: 0.2360 - val_loss: 2.6589 - val_acc: 0.2393\n",
      "Epoch 27/50\n",
      "40000/40000 [==============================] - 37s 930us/step - loss: 2.6312 - acc: 0.2373 - val_loss: 2.6384 - val_acc: 0.2429\n",
      "Epoch 28/50\n",
      "40000/40000 [==============================] - 37s 928us/step - loss: 2.6254 - acc: 0.2411 - val_loss: 2.6410 - val_acc: 0.2372\n",
      "Epoch 29/50\n",
      "40000/40000 [==============================] - 37s 931us/step - loss: 2.6133 - acc: 0.2454 - val_loss: 2.5976 - val_acc: 0.2456\n",
      "Epoch 30/50\n",
      "40000/40000 [==============================] - 37s 931us/step - loss: 2.6035 - acc: 0.2470 - val_loss: 2.5906 - val_acc: 0.2434\n",
      "Epoch 31/50\n",
      "40000/40000 [==============================] - 37s 931us/step - loss: 2.5814 - acc: 0.2478 - val_loss: 2.5942 - val_acc: 0.2470\n",
      "Epoch 32/50\n",
      "40000/40000 [==============================] - 37s 929us/step - loss: 2.5823 - acc: 0.2495 - val_loss: 2.5804 - val_acc: 0.2550\n",
      "Epoch 33/50\n",
      "40000/40000 [==============================] - 37s 930us/step - loss: 2.5413 - acc: 0.2534 - val_loss: 2.5908 - val_acc: 0.2432\n",
      "Epoch 34/50\n",
      "40000/40000 [==============================] - 37s 931us/step - loss: 2.5568 - acc: 0.2547 - val_loss: 2.5761 - val_acc: 0.2464\n",
      "Epoch 35/50\n",
      "40000/40000 [==============================] - 37s 928us/step - loss: 2.5408 - acc: 0.2546 - val_loss: 2.5300 - val_acc: 0.2572\n",
      "Epoch 36/50\n",
      "40000/40000 [==============================] - 37s 933us/step - loss: 2.5276 - acc: 0.2575 - val_loss: 2.4810 - val_acc: 0.2711\n",
      "Epoch 37/50\n",
      "40000/40000 [==============================] - 37s 932us/step - loss: 2.5042 - acc: 0.2627 - val_loss: 2.5219 - val_acc: 0.2596\n",
      "Epoch 38/50\n",
      "40000/40000 [==============================] - 37s 930us/step - loss: 2.5054 - acc: 0.2611 - val_loss: 2.5021 - val_acc: 0.2626\n",
      "10000/10000 [==============================] - 5s 502us/step\n",
      "{'classes': 10, 'layers': [3, 1024], 'conv_layer_start': 128, 'activation': 'relu', 'batch_size': 512, 'epochs': 50, 'optimizer': 'adadelta', 'loss': 'sparse_categorical_crossentropy', 'callbacks': [<keras.callbacks.EarlyStopping object at 0x7f784f2f69b0>], 'dropout_prob': 0.41203062024193915, 'lr': 0.0016233296507025482}\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/50\n",
      "40000/40000 [==============================] - 42s 1ms/step - loss: 3.4809 - acc: 0.1060 - val_loss: 3.3863 - val_acc: 0.1160\n",
      "Epoch 2/50\n",
      "40000/40000 [==============================] - 37s 930us/step - loss: 3.3678 - acc: 0.1175 - val_loss: 3.2977 - val_acc: 0.1231\n",
      "Epoch 3/50\n",
      "40000/40000 [==============================] - 37s 932us/step - loss: 3.2786 - acc: 0.1293 - val_loss: 3.2602 - val_acc: 0.1307\n",
      "Epoch 4/50\n",
      "40000/40000 [==============================] - 37s 927us/step - loss: 3.2069 - acc: 0.1372 - val_loss: 3.1706 - val_acc: 0.1533\n",
      "Epoch 5/50\n",
      "40000/40000 [==============================] - 37s 932us/step - loss: 3.1610 - acc: 0.1477 - val_loss: 3.1393 - val_acc: 0.1474\n",
      "Epoch 6/50\n",
      "40000/40000 [==============================] - 37s 930us/step - loss: 3.1002 - acc: 0.1571 - val_loss: 3.0776 - val_acc: 0.1555\n",
      "Epoch 7/50\n",
      "40000/40000 [==============================] - 37s 929us/step - loss: 3.0551 - acc: 0.1614 - val_loss: 3.0267 - val_acc: 0.1677\n",
      "Epoch 8/50\n",
      "40000/40000 [==============================] - 37s 930us/step - loss: 3.0168 - acc: 0.1702 - val_loss: 3.0000 - val_acc: 0.1735\n",
      "Epoch 9/50\n",
      "40000/40000 [==============================] - 37s 932us/step - loss: 2.9790 - acc: 0.1764 - val_loss: 2.9656 - val_acc: 0.1781\n",
      "Epoch 10/50\n",
      "40000/40000 [==============================] - 37s 930us/step - loss: 2.9515 - acc: 0.1806 - val_loss: 2.9412 - val_acc: 0.1898\n",
      "Epoch 11/50\n",
      "40000/40000 [==============================] - 37s 929us/step - loss: 2.9046 - acc: 0.1916 - val_loss: 2.8866 - val_acc: 0.1920\n",
      "Epoch 12/50\n",
      "40000/40000 [==============================] - 37s 930us/step - loss: 2.8883 - acc: 0.1925 - val_loss: 2.9153 - val_acc: 0.1871\n",
      "Epoch 13/50\n",
      "40000/40000 [==============================] - 37s 932us/step - loss: 2.8619 - acc: 0.1981 - val_loss: 2.8749 - val_acc: 0.1928\n",
      "Epoch 14/50\n",
      "40000/40000 [==============================] - 37s 928us/step - loss: 2.8512 - acc: 0.1988 - val_loss: 2.8734 - val_acc: 0.1949\n",
      "Epoch 15/50\n",
      "40000/40000 [==============================] - 37s 930us/step - loss: 2.8134 - acc: 0.2041 - val_loss: 2.8072 - val_acc: 0.2052\n",
      "Epoch 16/50\n",
      "40000/40000 [==============================] - 37s 931us/step - loss: 2.7906 - acc: 0.2072 - val_loss: 2.7734 - val_acc: 0.2089\n",
      "Epoch 17/50\n",
      "40000/40000 [==============================] - 37s 931us/step - loss: 2.7850 - acc: 0.2100 - val_loss: 2.7927 - val_acc: 0.2078\n",
      "Epoch 18/50\n",
      "40000/40000 [==============================] - 37s 928us/step - loss: 2.7537 - acc: 0.2160 - val_loss: 2.7400 - val_acc: 0.2164\n",
      "Epoch 19/50\n",
      "40000/40000 [==============================] - 37s 931us/step - loss: 2.7366 - acc: 0.2187 - val_loss: 2.7448 - val_acc: 0.2164\n",
      "Epoch 20/50\n",
      "40000/40000 [==============================] - 37s 930us/step - loss: 2.7266 - acc: 0.2190 - val_loss: 2.7078 - val_acc: 0.2266\n",
      "Epoch 21/50\n",
      "40000/40000 [==============================] - 37s 930us/step - loss: 2.6903 - acc: 0.2259 - val_loss: 2.7058 - val_acc: 0.2277\n",
      "Epoch 22/50\n",
      "40000/40000 [==============================] - 37s 928us/step - loss: 2.6820 - acc: 0.2273 - val_loss: 2.6970 - val_acc: 0.2275\n",
      "Epoch 23/50\n",
      "40000/40000 [==============================] - 37s 930us/step - loss: 2.6734 - acc: 0.2314 - val_loss: 2.6891 - val_acc: 0.2309\n",
      "Epoch 24/50\n",
      "40000/40000 [==============================] - 37s 931us/step - loss: 2.6664 - acc: 0.2309 - val_loss: 2.6497 - val_acc: 0.2301\n",
      "Epoch 25/50\n",
      "40000/40000 [==============================] - 37s 928us/step - loss: 2.6343 - acc: 0.2360 - val_loss: 2.6635 - val_acc: 0.2276\n",
      "Epoch 26/50\n",
      "40000/40000 [==============================] - 37s 930us/step - loss: 2.6345 - acc: 0.2393 - val_loss: 2.6244 - val_acc: 0.2347\n",
      "Epoch 27/50\n",
      "40000/40000 [==============================] - 37s 931us/step - loss: 2.6218 - acc: 0.2406 - val_loss: 2.6218 - val_acc: 0.2374\n",
      "Epoch 28/50\n",
      "40000/40000 [==============================] - 37s 931us/step - loss: 2.5875 - acc: 0.2438 - val_loss: 2.5703 - val_acc: 0.2451\n",
      "Epoch 29/50\n",
      "40000/40000 [==============================] - 37s 927us/step - loss: 2.6057 - acc: 0.2382 - val_loss: 2.5671 - val_acc: 0.2473\n",
      "Epoch 30/50\n",
      "40000/40000 [==============================] - 37s 930us/step - loss: 2.5832 - acc: 0.2449 - val_loss: 2.5659 - val_acc: 0.2513\n",
      "Epoch 31/50\n",
      "40000/40000 [==============================] - 37s 931us/step - loss: 2.5743 - acc: 0.2473 - val_loss: 2.5628 - val_acc: 0.2552\n",
      "Epoch 32/50\n",
      "40000/40000 [==============================] - 37s 931us/step - loss: 2.5529 - acc: 0.2525 - val_loss: 2.5655 - val_acc: 0.2552\n",
      "Epoch 33/50\n",
      "40000/40000 [==============================] - 37s 929us/step - loss: 2.5422 - acc: 0.2543 - val_loss: 2.5431 - val_acc: 0.2522\n",
      "Epoch 34/50\n",
      "40000/40000 [==============================] - 37s 930us/step - loss: 2.5347 - acc: 0.2566 - val_loss: 2.5339 - val_acc: 0.2628\n",
      "Epoch 35/50\n",
      "40000/40000 [==============================] - 37s 932us/step - loss: 2.5343 - acc: 0.2545 - val_loss: 2.5239 - val_acc: 0.2595\n",
      "Epoch 36/50\n",
      "40000/40000 [==============================] - 37s 928us/step - loss: 2.4983 - acc: 0.2637 - val_loss: 2.5323 - val_acc: 0.2515\n",
      "Epoch 37/50\n",
      "40000/40000 [==============================] - 37s 929us/step - loss: 2.4930 - acc: 0.2596 - val_loss: 2.5221 - val_acc: 0.2550\n",
      "Epoch 38/50\n",
      "40000/40000 [==============================] - 37s 931us/step - loss: 2.4881 - acc: 0.2625 - val_loss: 2.4781 - val_acc: 0.2719\n",
      "Epoch 39/50\n",
      "40000/40000 [==============================] - 37s 929us/step - loss: 2.4875 - acc: 0.2603 - val_loss: 2.4725 - val_acc: 0.2685\n",
      "Epoch 40/50\n",
      "40000/40000 [==============================] - 37s 928us/step - loss: 2.4645 - acc: 0.2653 - val_loss: 2.4352 - val_acc: 0.2703\n",
      "Epoch 41/50\n",
      "40000/40000 [==============================] - 37s 932us/step - loss: 2.4296 - acc: 0.2735 - val_loss: 2.4472 - val_acc: 0.2648\n",
      "Epoch 42/50\n",
      "40000/40000 [==============================] - 37s 931us/step - loss: 2.4468 - acc: 0.2701 - val_loss: 2.4400 - val_acc: 0.2752\n",
      "10000/10000 [==============================] - 5s 509us/step\n",
      "40000/40000 [==============================] - 20s 509us/step\n",
      "10000/10000 [==============================] - 5s 511us/step\n",
      "Loss = 2.476077160644531\n",
      "Test Accuracy = 0.2707\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmYVNW19/HvYhIZREYHUJqgkRls\nWtBXNBpF0ShecAjYeeMQRY2YON28JHiFqGiMcYiGRNGIxqDINVExYpxigppoAEUUEQEFbUBERKbG\nMK33j32qqW6ququ7q7uG/n2ep56qc86uOqtOV6/atc8+e5u7IyIi+aVRpgMQEZH0U3IXEclDSu4i\nInlIyV1EJA8puYuI5CEldxGRPKTknsfMrLGZbTazg9NZNpPM7BAzS3v/XTM70cyWxy0vNrNjUilb\ng309YGY/q+nzRVLRJNMByG5mtjlusQXwH2BntHyJu0+rzuu5+06gVbrLNgTuflg6XsfMLgK+5+7H\nxb32Rel4bZHKKLlnEXcvS65RzfAid38pWXkza+LuO+ojNpGq6POYXdQsk0PM7CYze9zMHjOzTcD3\nzOwoM3vDzL4ys9VmdreZNY3KNzEzN7OCaPmP0fbnzGyTmf3LzLpVt2y0/RQz+9DMNpjZPWb2upmd\nnyTuVGK8xMyWmtl6M7s77rmNzexOM1tnZh8Bwyo5PuPNbHqFdZPN7I7o8UVmtih6P8uiWnWy1yox\ns+Oixy3M7JEotoXAwAplrzOzj6LXXWhmw6P1fYHfAMdETV5fxB3biXHPvzR67+vM7CkzOyCVY1Od\n4xyLx8xeMrMvzewzM/tJ3H7+JzomG81srpkdmKgJzMxei/2do+M5O9rPl8B1Znaomb0S7eOL6Li1\niXt+1+g9ro22/9rMmkcx94wrd4CZlZpZ+2TvV6rg7rpl4Q1YDpxYYd1NwDbgdMIX897AEcBgwq+w\nbwAfAmOj8k0ABwqi5T8CXwBFQFPgceCPNSjbCdgEnBFtuxrYDpyf5L2kEuPTQBugAPgy9t6BscBC\noAvQHpgdPrYJ9/MNYDPQMu61PweKouXTozIGfBvYCvSLtp0ILI97rRLguOjxr4C/A22BrsD7Fcqe\nAxwQ/U3OjWLYL9p2EfD3CnH+EZgYPT4pinEA0Bz4LfC3VI5NNY9zG2AN8GNgL2AfYFC07afAO8Ch\n0XsYALQDDql4rIHXYn/n6L3tAC4DGhM+j98ETgCaRZ+T14Ffxb2f96Lj2TIqf3S0bQowKW4/1wBP\nZvr/MJdvGQ9AtyR/mOTJ/W9VPO9a4H+jx4kS9r1xZYcD79Wg7IXAq3HbDFhNkuSeYoxHxm3/M3Bt\n9Hg2oXkqtu3Uigmnwmu/AZwbPT4FWFxJ2b8Al0ePK0vun8T/LYAfxpdN8LrvAd+JHleV3B8Gbo7b\ntg/hPEuXqo5NNY/z/wXmJCm3LBZvhfWpJPePqojhrNh+gWOAz4DGCcodDXwMWLQ8HxiZ7v+rhnRT\ns0zu+TR+wcx6mNmz0c/sjcANQIdKnv9Z3ONSKj+JmqzsgfFxePhvLEn2IinGmNK+gBWVxAvwKDA6\nenxutByL4zQzezNqMviKUGuu7FjFHFBZDGZ2vpm9EzUtfAX0SPF1Iby/stdz943AeqBzXJmU/mZV\nHOeDCEk8kcq2VaXi53F/M5thZiujGB6qEMNyDyfvy3H31wm/AoaYWR/gYODZGsYkqM09F1XsBngf\noaZ4iLvvA1xPqEnXpdWEmiUAZmaUT0YV1SbG1YSkEFNVV80ZwIlm1pnQbPRoFOPewBPALYQmk32B\nF1KM47NkMZjZN4DfEZom2kev+0Hc61bVbXMVoakn9nqtCc0/K1OIq6LKjvOnQPckz0u2bUsUU4u4\ndftXKFPx/d1K6OXVN4rh/AoxdDWzxkni+APwPcKvjBnu/p8k5SQFSu65rzWwAdgSnZC6pB72+Reg\n0MxON7MmhHbcjnUU4wzgSjPrHJ1c+3+VFXb3zwhNBw8RmmSWRJv2IrQDrwV2mtlphLbhVGP4mZnt\na+E6gLFx21oREtxawvfcxYSae8waoEv8ic0KHgN+YGb9zGwvwpfPq+6e9JdQJSo7zjOBg81srJnt\nZWb7mNmgaNsDwE1m1t2CAWbWjvCl9hnhxH1jMxtD3BdRJTFsATaY2UGEpqGYfwHrgJstnKTe28yO\njtv+CKEZ51xCopdaUHLPfdcA5xFOcN5HOPFZp9x9DfBd4A7CP2t34G1CjS3dMf4OeBl4F5hDqH1X\n5VFCG3pZk4y7fwVcBTxJOCl5FuFLKhUTCL8glgPPEZd43H0BcA/w76jMYcCbcc99EVgCrDGz+OaV\n2PP/Smg+eTJ6/sFAcYpxVZT0OLv7BmAocCbhC+dD4FvR5tuApwjHeSPh5GbzqLntYuBnhJPrh1R4\nb4lMAAYRvmRmAn+Ki2EHcBrQk1CL/4Twd4htX074O//H3f9ZzfcuFcROXojUWPQzexVwlru/mul4\nJHeZ2R8IJ2knZjqWXKeLmKRGzGwYoWfKVkJXuu2E2qtIjUTnL84A+mY6lnygZhmpqSHAR4S25pOB\nEToBJjVlZrcQ+trf7O6fZDqefKBmGRGRPKSau4hIHspYm3uHDh28oKAgU7sXEclJ8+bN+8LdK+t6\nDGQwuRcUFDB37txM7V5EJCeZWVVXaQNqlhERyUtK7iIieUjJXUQkD2XVRUzbt2+npKSEr7/+OtOh\nSCWaN29Oly5daNo02XApIpJpWZXcS0pKaN26NQUFBYSBBiXbuDvr1q2jpKSEbt26Vf0EEcmIrGqW\n+frrr2nfvr0SexYzM9q3b69fVyI1MG0aFBRAo0bhflq1pryvnqxK7oASew7Q30gaotom5mnTYMwY\nWLEC3MP9mDF1l+CzLrmLiGSbdCTm8eOhtLT8utLSsL4uKLnHWbduHQMGDGDAgAHsv//+dO7cuWx5\n27ZtKb3GBRdcwOLFiystM3nyZKbV5e8xEUmrdCTmT5IMh5ZsfW1l1QnV6po2LRzcTz6Bgw+GSZOg\nuKbTHADt27dn/vz5AEycOJFWrVpx7bXXlitTNvlso8Tfi1OnTq1yP5dffnnNgxSRepeOxHzwwaHG\nn2h9XcjZmnt9tl8tXbqUXr16UVxcTO/evVm9ejVjxoyhqKiI3r17c8MNN5SVHTJkCPPnz2fHjh3s\nu+++jBs3jv79+3PUUUfx+eefA3Dddddx1113lZUfN24cgwYN4rDDDuOf/wwT0GzZsoUzzzyTXr16\ncdZZZ1FUVFT2xRNvwoQJHHHEEfTp04dLL700NpM8H374Id/+9rfp378/hYWFLF++HICbb76Zvn37\n0r9/f8bX1e9BkSxT2/byZAm4Ool50iRo0aL8uhYtwvo6EauJ1vdt4MCBXtH777+/x7pkunZ1D2m9\n/K1r15RfolITJkzw2267zd3dlyxZ4mbmc+bMKdu+bt06d3ffvn27DxkyxBcuXOju7kcffbS//fbb\nvn37dgd81qxZ7u5+1VVX+S233OLu7uPHj/c777yzrPxPfvITd3d/+umn/eSTT3Z391tuucV/+MMf\nurv7/PnzvVGjRv7222/vEWcsjl27dvmoUaPK9ldYWOgzZ850d/etW7f6li1bfObMmT5kyBAvLS0t\n99yaqM7fSiST/vhH9xYtyueJFi3C+vp8jdjrdO3qbhbuq/t8d3dgrqeQY3O25l7f7Vfdu3enqKio\nbPmxxx6jsLCQwsJCFi1axPvvv7/Hc/bee29OOeUUAAYOHFhWe65o5MiRe5R57bXXGDVqFAD9+/en\nd+/eCZ/78ssvM2jQIPr3788//vEPFi5cyPr16/niiy84/fTTgXDRUYsWLXjppZe48MIL2XvvvQFo\n165d9Q+ESI5JR3t5cTFMmQJdu4JZuJ8ypfrNwMXFsHw57NoV7mvTjFyVnE3u6fiZVB0tW7Yse7xk\nyRJ+/etf87e//Y0FCxYwbNiwhP2+mzVrVva4cePG7NixI+Fr77XXXlWWSaS0tJSxY8fy5JNPsmDB\nAi688EL1P5e8U9smlXRVBOszMadDzib3em+/irNx40Zat27NPvvsw+rVq3n++efTvo+jjz6aGTNm\nAPDuu+8m/GWwdetWGjVqRIcOHdi0aRN/+lOYaL5t27Z07NiRZ555BggXh5WWljJ06FAefPBBtm7d\nCsCXX36Z9rhF0ikd59bquyKYLXI2uafrZ1JNFBYW0qtXL3r06MH3v/99jj766LTv44orrmDlypX0\n6tWLn//85/Tq1Ys2bdqUK9O+fXvOO+88evXqxSmnnMLgwYPLtk2bNo3bb7+dfv36MWTIENauXctp\np53GsGHDKCoqYsCAAdx5551pj1skndLRpJLJimBGpdIwXxe32p5QzXfbt2/3rVu3urv7hx9+6AUF\nBb59+/YMR7Wb/laSitqeQDRL3HHCrH7jyCakeEI1p/u557PNmzdzwgknsGPHDtyd++67jyZN9OeS\n3BFrUonVvGNNKpD6L+x09Q0vLs7+NvJ0y9lmmXy37777Mm/ePN555x0WLFjASSedlOmQpIGp7YlM\nNalklpK7iOwhHScy09FLJZPn1nKdkruI7CEdte509VLJtS6I2ULJXUT2kI5at5pUMkvJXUT2kI5a\nt5pUMkvJPc7xxx+/xwVJd911F5dddlmlz2vVqhUAq1at4qyzzkpY5rjjjmPu3LmVvs5dd91Fadxv\n4VNPPZWvvvoqldBFyqntydB01brVpJI5Su5xRo8ezfTp08utmz59OqNHj07p+QceeCBPPPFEjfdf\nMbnPmjWLfffdt8avJw1TOk6Gqtad+5Tc45x11lk8++yzZRNzLF++nFWrVnHMMceU9TsvLCykb9++\nPP3003s8f/ny5fTp0wcIQwOMGjWKnj17MmLEiLJL/gEuu+yysuGCJ0yYAMDdd9/NqlWrOP744zn+\n+OMBKCgo4IsvvgDgjjvuoE+fPvTp06dsuODly5fTs2dPLr74Ynr37s1JJ51Ubj8xzzzzDIMHD+bw\nww/nxBNPZM2aNUDoS3/BBRfQt29f+vXrVzZ8wV//+lcKCwvp378/J5xwQlqOrdSfdM34o1p3bsva\nq2KuvBISDF9eKwMGQJQXE2rXrh2DBg3iueee44wzzmD69Omcc845mBnNmzfnySefZJ999uGLL77g\nyCOPZPjw4UnnE/3d735HixYtWLRoEQsWLKCwsLBs26RJk2jXrh07d+7khBNOYMGCBfzoRz/ijjvu\n4JVXXqFDhw7lXmvevHlMnTqVN998E3dn8ODBfOtb36Jt27YsWbKExx57jPvvv59zzjmHP/3pT3zv\ne98r9/whQ4bwxhtvYGY88MAD/PKXv+T222/nxhtvpE2bNrz77rsArF+/nrVr13LxxRcze/ZsunXr\npvFnclB9j5gq2Uk19wrim2bim2TcnZ/97Gf069ePE088kZUrV5bVgBOZPXt2WZLt168f/fr1K9s2\nY8YMCgsLOfzww1m4cGHCQcHivfbaa4wYMYKWLVvSqlUrRo4cyauvvgpAt27dGDBgAJB8WOGSkhJO\nPvlk+vbty2233cbChQsBeOmll8rNCtW2bVveeOMNjj32WLp16wZoWOBc1FAHypLysrbmXlkNuy6d\nccYZXHXVVbz11luUlpYycOBAIAzEtXbtWubNm0fTpk0pKCio0fC6H3/8Mb/61a+YM2cObdu25fzz\nz6/VML2x4YIhDBmcqFnmiiuu4Oqrr2b48OH8/e9/Z+LEiTXen9S92k4fOWlS+cv+QV0QG6KUau5m\nNszMFpvZUjMbl2D7wWb2ipm9bWYLzOzU9IdaP1q1asXxxx/PhRdeWO5E6oYNG+jUqRNNmzbllVde\nYUWiAS/iHHvssTz66KMAvPfeeyxYsAAIwwW3bNmSNm3asGbNGp577rmy57Ru3ZpNmzbt8VrHHHMM\nTz31FKWlpWzZsoUnn3ySY445JuX3tGHDBjp37gzAww8/XLZ+6NChTJ48uWx5/fr1HHnkkcyePZuP\nP/4Y0LDA1VXbXio6GSrpUmXN3cwaA5OBoUAJMMfMZrp7fFvCdcAMd/+dmfUCZgEFdRBvvRg9ejQj\nRowo13OmuLiY008/nb59+1JUVESPHj0qfY3LLruMCy64gJ49e9KzZ8+yXwD9+/fn8MMPp0ePHhx0\n0EHlhgseM2YMw4YN48ADD+SVV14pW19YWMj555/PoEGDALjooos4/PDDk87sVNHEiRM5++yzadu2\nLd/+9rfLEvd1113H5ZdfTp8+fWjcuDETJkxg5MiRTJkyhZEjR7Jr1y46derEiy++mNJ+Grp0DJRV\n2cnQ6iTnhjhQVm25w9dfw6ZNsHFj4tumTdC0KbRqFW6tW+9+HL+uZctQLpPMowmVkxYwOwqY6O4n\nR8s/BXD3W+LK3Ad85O63RuVvd/f/U9nrFhUVecV+34sWLaJnz541eiNSv/S32lNBQeIRDLt2Db1N\nUtGoUUgyFZmFXitSfTt3wpw58NJL8OmnyRP3xo1QjYnQqrTXXnsm/djjSy+Fk0+u2eua2Tx3L6qq\nXCpt7p2BT+OWS4DBFcpMBF4wsyuAlsCJSYIaA4wBOFhndyTPpKOXSrqGuG3oPvkEXngBnn8+JPWv\nvgpfkJ06wT777L5161Z+uXXr8ssVb61ahS+AzZt33zZtKr9c1bo1a2DDhro/Buk6oToaeMjdb49q\n7o+YWR93L1fXcPcpwBQINfc07VskK6QjMetkaM1s2QKzZ4dk/vzz8MEHYf2BB8KIEaGWfOKJ0L59\nevZXYVK0rJRKcl8JHBS33CVaF+8HwDAAd/+XmTUHOgCfVzcgd0/ad1yyQ1VNeQ1VOhJzrJ28Nr1l\nGgJ3WLBgd+381Vdh2zZo3hyOPRYuvjgk9F69Qo29IUqlt8wc4FAz62ZmzYBRwMwKZT4BTgAws55A\nc2BtdYNp3rw569atU/LIYu7OunXraN68eaZDSbva9nRJVy8VXRma2Oefw6OPwnnnhRr5gAHwk5+E\nZo6xY0OS//LLcH/11dC7d8NN7JDCCVWAqGvjXUBj4EF3n2RmNxDm8psZ9ZC5H2gFOPATd3+hstdM\ndEJ1+/btlJSU1Krft9S95s2b06VLF5pmujtAGlXs6QKh1q0uhNXz9dfw8cewbBksXRruly2D9evD\nl1Xs5p58OdG2nTthZdRe0L49DB0KJ50UblEv3wYj1ROqKSX3upAouYtkSjp6ujQUGzbsTtoVk3hJ\nSfnePq1bwyGHQIcO0Lhx+FVkFu7jb1WtMwsnP08+GQoLw2s1VOnsLSOS9/JhPJbt20Ot+YMPYPHi\ncL9mDTRpUrvbpk3lk3g0ll2ZTp2ge3c47rhw3717SOjdu4ek3pCbRjJJyV2E3OqCuG5dSN6xBB67\nX7asfD/t/fYLbdO7doX1qd527iy/P7NwHLp3Dz1PYok7dmvdun7fv6RGyV3yRm3GZMm2LogVa+Hx\niTy+5tysWUi2vXvDyJFw2GHQo0e4r+lUALE27liyb9Ys3CS3KLlLXqjtpf+Z6IIYS+BLl8KSJeXv\nly8vX4Pu1Ckk7REjdifvww4L5wqapPm/2Gx3k4zkLp1QlbyQrSdEt20L+6+YvJcsCfHGJ/DWreHQ\nQ8PtkEPCfSyJt22bsbcgWUYnVKVByfQJUfewr7lzwzgm8+cnTuD77BOS9hFHwLnn7k7ihxwCHTvq\n5KOkj5K75IX6PiG6Zk1I4rFkPmcOrI0u22vaNLSBV0zghx6q3iNSf5TcJS/U5QnRr77ancRj959G\nQ+mZhUvcv/MdKCoKCb1fv3AZvEgmKblLXkjXCdFdu+CNN+DNN3fXyJcu3b29e3c4+uiQxIuKwgU1\nrVql732IpItOqEpWqO3UcrX12Wfw4INw//27T8B26bK7Nn7EETBwIGhKWck0nVCVnJGOGYxqYtcu\nePlluO8+ePrp0Kf7+OPh5pvD1ZYHHFB3+xapa6q5S8bVdzfGzz+HqVNDLX3ZsjAQ1fnnhy+Ub34z\n/fsTSSfV3CVn1Ec3Rnf4+9/h3nvhySfDBUTHHgs33BCu7NQJUMk3Su6ScXXZjfGLL+Dhh8PQvR9+\nGC7Jv/zyUEvXFLCSz1KZrEOkTk2aFLotxqtNN0b3MOVacXEY6/vaa0P/8ocfhlWr4M47ldgl/6nm\nLhmXjm6MX30FCxeGboy//z0sWhTmuRwzBi65BPr0qZvYRbKVkrtkheLi1JL51q0hcb/3Xri9+264\nLynZXWbw4NCt8ZxzoGXLuotZJJspuUut1UUf9e3bw9gssSQeuy1dunumn732Cs0rxx0XauZ9+kDf\nvtk5BrtIfVNyl1pJRx/19evDZf1z5+6uiX/wQUjwEKZZ++Y3oX//8JqxRN69u4alFUlG/dylVqrb\nR33rVnj7bfj3v3df3r9kSfnnxWrgsSR+2GHqqigSo37uUi8q66O+Y0eohceS+L//HZZjQ+B27gyD\nBsEFF+weq6WmsweJSHlK7lIryfqoN2sWxi7fujUst20bEvhpp4WEfsQRurxfpC4puUut3HQT/OAH\nYcaheF27hmFwjzgiJPNvfEPjmIvUJyV3qbH33w99yrdtCzX1bdtCbfwXv4Dvfz/T0Yk0bLpCVapt\n0yb47/8OvVfeeSeM11JaGroorlqlxC6SDVRzl5S5w4wZcM01sHJlaI75xS/Cpf0ikl1Uc2/gpk0L\n3RkbNQr306YlLvfBBzB0KIwaBZ06wb/+BQ88oMQukq2U3Buw2AVIK1aEWnnsAqT4BL95M4wbF+YF\nnTcPJk8O3RqPPDJzcYtI1ZTcG7Dx48tPKA1hefz4kOyfeCJc3n/rrfC978HixfDDH0LjxpmJV0RS\nl1JyN7NhZrbYzJaa2bgE2+80s/nR7UMz+yr9oUq6JbsAacUKGDYMzj47zFL0+uthIK5Oneo3PhGp\nuSpPqJpZY2AyMBQoAeaY2Ux3fz9Wxt2viit/BXB4HcQqaZbsAiQIQ+fefTdcdpnGbxHJRanU3AcB\nS939I3ffBkwHzqik/GjgsXQEJ3Ur0SQZAEOGhFmLrrhCiV0kV6WS3DsDn8Ytl0Tr9mBmXYFuwN+S\nbB9jZnPNbO7atWurG6ukWXEx/M//7B6Uq2lTuO46ePVV2G+/zMYmIrWT7hOqo4An3H1noo3uPsXd\ni9y9qGPHjmnetVTH6tVw8cXh5Gnz5mHqudJSuPHGTEcmIumQyo/ulcBBcctdonWJjAIur21QUnc2\nb4bbboNf/SqMl/7jH4faert2mY5MRNIpleQ+BzjUzLoRkvoo4NyKhcysB9AW+FdaI5S02LEj9Hi5\n/npYsyZMQXfzzWHCCxHJP1U2y7j7DmAs8DywCJjh7gvN7AYzGx5XdBQw3TM1+4ck5A7PPhsuQrrk\nEjjkkHB16eOPK7GL5LOU+kK4+yxgVoV111dYnpi+sCQd3noLrr0WXnkFDj0U/vxn+K//0tC7Ig2B\nrlDNQytWhCtKBw4Mc5Lecw8sXAgjRiixizQU6sWcR776Cm65BX7965DEx40LtzZtMh2ZiNQ3Jfcc\nNm1a6Mq4YkWYxm77dtiyJdTab7opXIEqIg2TknuOmjYt9FOPzVG6fn0YtvfGG0PCF5GGTW3uOejd\nd8OYL7HEHrNrF9x/f2ZiEpHsouSeIzZuDIl78ODQrXHTpsTlko30KCINi5J7FnOHf/4TLrwwTDw9\nZkxoU7/jDujSJfFz1M4uIqA296y0di384Q9hGrsPPoCWLeHcc8OcpYMHh54wnTqFZB8/2UaLFmGk\nRxERJfcssXMnvPgi/P738PTToefLUUeFBH/OOdC6dfnyxcXhfvz40BRz8MEhscfWi0jDpuSeYStW\nwNSpYdyXTz8NMx+NHRtq6b17V/7c4mIlcxFJTMk9Q9zDZBi//W1YHjoUbr8dhg+HvfbKbGwikvuU\n3DPkN7+ByZN3j6netWumIxKRfKLkngGvvQZXXw2nnw733hsuPhIRSSellXq2ejWcfTYUFIQeMUrs\nIlIXVHOvR9u2hcS+cSO88ALsu2+mIxKRfKXkXo+uvRZefx0efRT69s10NCKSz9QoUE/++McwrvqV\nV8Lo0WHgr4KC0CxTUBCWRUTSRTX3evDOO+Fq0mOPhV/+MiTy+KtLV6wIy6B+6yKSHqq517H162Hk\nyDDe+uOPQ9Omoetj/LABEJY1VK+IpItq7nVo164wccann8I//gH77x/WJxu5USM6iki6qOZeh264\nAWbNgrvuCuPExCQbuVEjOopIuii515Fnn4Wf/xy+//0wsUa8SZPCCI7xNKKjiKSTknsdWLo0NMcM\nGBCuQDUrv724GKZMCUMOmIX7KVN0MlVE0kdt7mm2ZUs4gWoGf/4z7L134nIa0VFE6pKSexq5hy6N\n770Hzz0H3bplOiIRaaiU3NPonnvC1ac33ggnn5zpaESkIVObe5q8+ipcc00Yj/1nP8t0NCLS0Cm5\np8GqVWFAsG7dNNKjiGQHNcvUUmykx02b4KWXoE2bTEckIqLkXmvXXAP//CdMnw59+mQ6GhGRIKUG\nBDMbZmaLzWypmY1LUuYcM3vfzBaa2aPpDTM7PfJImC7v6qvhu9/NdDQiIrtVWXM3s8bAZGAoUALM\nMbOZ7v5+XJlDgZ8CR7v7ejPrVFcBZ4v580O3x299C269NdPRiIiUl0rNfRCw1N0/cvdtwHTgjApl\nLgYmu/t6AHf/PL1hZpdVq0KvmPbtw0iPTdS4JSJZJpXk3hn4NG65JFoX75vAN83sdTN7w8yGJXoh\nMxtjZnPNbO7atWtrFnGGbd4cJrb+8kt45hnYb79MRyQisqd0ddprAhwKHAeMBu43sz1mCHX3Ke5e\n5O5FHTt2TNOu68+OHTBqVGiSmTEDDj880xGJiCSWSnJfCRwUt9wlWhevBJjp7tvd/WPgQ0Kyzxvu\n8OMfh9Ee77kHTj010xGJiCSXSnKfAxxqZt3MrBkwCphZocxThFo7ZtaB0EzzURrjzLg774Tf/jZM\nct2mjeY/FZHsVuWpQHffYWZjgeeBxsCD7r7QzG4A5rr7zGjbSWb2PrAT+G93X1eXgdenP/85JPUz\nz4R+/TT/qYhkP3P3jOy4qKjI586dm5F9V8ebb8Jxx4Wx2f/2N+jZMyT0irp2heXL6zs6EWlozGye\nuxdVVU6joFTio49Cz5gDD4SU+lJMAAAL6klEQVSnnw5js2v+UxHJBUruSXz5ZThpumNHmAe1U3RZ\nluY/FZFcoOSewH/+E2ZT+vhjeOopOOyw3ds0/6mI5AIl9wrc4Qc/gH/8A6ZOhWOPLb9d85+KSC7Q\nhfMVTJgQujbedBOce27iMpr/VESynWrucaZODVPkXXihZlMSkdym5B55+eXQX/3EE+Hee0OTi4hI\nrlJyB957L5xA7dEDnngCmjbNdEQiIrXT4JP76tXwne+EHi/PPqtp8kQkPzToE6pbtoSLlNatg9mz\n1VddRPJHg03uO3fC6NHw9tvh6tPCwkxHJCKSPg0yubvDlVeGyTZ+8xs47bRMRyQikl4Nss196tTd\nE1tffnmmoxERSb8Gl9x37gxDBQweDLfdluloRETqRoNL7s89F0Z7vOqqMNmGiEg+anDp7e67wxC+\nI0dmOhIRkbrToJL7okXw4otw2WW6UElE8luDSu6/+Q00a7Z7WjwRkXzVYJL7hg3w8MMwatTuiTdE\nRPJVg0nuU6eGK1J/9KNMRyIiUvcaRHLftSs0yRx1FAwcmOloRETqXoNI7s89B8uWqdYuIg1Hg0ju\n99wDBxwAZ56Z6UhEROpH3if3xYvh+efV/VFEGpa8T+7q/igiDVFeJ/eNG+Ghh+C734X99st0NCIi\n9Sevk/tDD8HmzXDFFZmORESkfuVtct+1K5xIPfJIOOKITEcjIlK/8ja5P/88LF2qWruINEwpJXcz\nG2Zmi81sqZmNS7D9fDNba2bzo9tF6Q+1eu6+G/bfH846q/z6adOgoCAM91tQEJZFRPJNldPsmVlj\nYDIwFCgB5pjZTHd/v0LRx919bB3EWG0ffgh//StMnBh6ysRMmxZ6zZSWhuUVK3b3oikurvcwRUTq\nTCo190HAUnf/yN23AdOBM+o2rNqZPDn0ab/kkvLrx4/fndhjSkvDehGRfJJKcu8MfBq3XBKtq+hM\nM1tgZk+Y2UGJXsjMxpjZXDObu3bt2hqEW7VNm8IgYeecE5pl4n3ySeLnJFsvIpKr0nVC9RmgwN37\nAS8CDycq5O5T3L3I3Ys6duyYpl2X9/DDIcEnGkfm4IMTPyfZehGRXJVKcl8JxNfEu0Tryrj7Onf/\nT7T4AJCRsRdj3R8HDQq3iiZNghYtyq9r0SKsFxHJJ6kk9znAoWbWzcyaAaOAmfEFzOyAuMXhwKL0\nhZi6F18MJ1OTjf5YXAxTpkDXrmAW7qdM0clUEck/VfaWcfcdZjYWeB5oDDzo7gvN7AZgrrvPBH5k\nZsOBHcCXwPl1GHNSd98dhhk4++zkZYqLlcxFJP9VmdwB3H0WMKvCuuvjHv8U+Gl6Q6ueJUtg1iyY\nMKF890cRkYYob65QnTwZmjTZs/ujiEhDlBfJPb774wEHVF1eRCTf5UVy/8MfwvC+GkdGRCTI+eQe\nm/z6iCNg8OBMRyMikh1SOqGazV56CT74INTezTIdjYhIdsj5mvs990CnTqG9XUREgpxO7suWwbPP\nhh4ye+2V6WhERLJHTif3yZOhcWO49NJMRyIikl1yNrlv3gy//32YjOPAAzMdjYhIdsnZ5P7II6H7\nY7JxZEREGrKcTO7u4UTqwIFhAmwRESkvJ7tCvvwyLFoUxm5X90cRkT3lZM397ruhY0f47nczHYmI\nSHbKueT+0Ufwl7+o+6OISGVyLrk/+KC6P4qIVCXnkvv118Mrr0DnRFN0i4gIkIPJvVkzGDIk01GI\niGS3nEvuIiJSNSV3EZE8pOQuIpKHlNxFRPKQkruISB5SchcRyUNK7iIieUjJXUQkDym5i4jkISV3\nEZE8pOQuIpKHlNxFRPJQSsndzIaZ2WIzW2pm4yopd6aZuZkVpS9EERGpriqTu5k1BiYDpwC9gNFm\n1itBudbAj4E30x2kiIhUTyo190HAUnf/yN23AdOBMxKUuxG4Ffg6jfGJiEgNpJLcOwOfxi2XROvK\nmFkhcJC7P1vZC5nZGDOba2Zz165dW+1gRUQkNbU+oWpmjYA7gGuqKuvuU9y9yN2LOnbsWNtdi4hI\nEqkk95XAQXHLXaJ1Ma2BPsDfzWw5cCQwUydVRUQyJ5XkPgc41My6mVkzYBQwM7bR3Te4ewd3L3D3\nAuANYLi7z62TiEVEpEpVJnd33wGMBZ4HFgEz3H2hmd1gZsPrOkAREam+JqkUcvdZwKwK665PUva4\n2oclIiK1oStURUTykJK7iEgeUnIXEclDSu4iInlIyV1EJA8puYuI5CEldxGRPKTkLiKSh5TcRUTy\nkJK7iEgeUnIXEclDSu4iInkop5L7tGlQUACNGoX7adMyHZGISHZKaVTIbDBtGowZA6WlYXnFirAM\nUFycubhERLJRztTcx4/fndhjSkvDehERKS9nkvsnn1RvvYhIQ5Yzyf3gg6u3XkSkIcuZ5D5pErRo\nUX5dixZhvYiIlJczyb24GKZMga5dwSzcT5mik6kiIonkTG8ZCIlcyVxEpGo5U3MXEZHUKbmLiOQh\nJXcRkTyk5C4ikoeU3EVE8pC5e2Z2bLYWWJGRnaeuA/BFpoNIgeJMr1yJE3InVsWZPl3dvWNVhTKW\n3HOBmc1196JMx1EVxZleuRIn5E6sirP+qVlGRCQPKbmLiOQhJffKTcl0AClSnOmVK3FC7sSqOOuZ\n2txFRPKQau4iInlIyV1EJA816ORuZgeZ2Stm9r6ZLTSzHycoc5yZbTCz+dHt+kzEGsWy3MzejeKY\nm2C7mdndZrbUzBaYWWEGYjws7ljNN7ONZnZlhTIZO6Zm9qCZfW5m78Wta2dmL5rZkui+bZLnnheV\nWWJm52UgztvM7IPob/ukme2b5LmVfk7qIc6JZrYy7u97apLnDjOzxdHndVwG4nw8LsblZjY/yXPr\n7Ximlbs32BtwAFAYPW4NfAj0qlDmOOAvmY41imU50KGS7acCzwEGHAm8meF4GwOfES66yIpjChwL\nFALvxa37JTAuejwOuDXB89oBH0X3baPHbes5zpOAJtHjWxPFmcrnpB7inAhcm8JnYxnwDaAZ8E7F\n/726jrPC9tuB6zN9PNN5a9A1d3df7e5vRY83AYuAzpmNqlbOAP7gwRvAvmZ2QAbjOQFY5u5ZcyWy\nu88Gvqyw+gzg4ejxw8B/JXjqycCL7v6lu68HXgSG1Wec7v6Cu++IFt8AutTV/lOV5HimYhCw1N0/\ncvdtwHTC36FOVBanmRlwDvBYXe0/Exp0co9nZgXA4cCbCTYfZWbvmNlzZta7XgMrz4EXzGyemY1J\nsL0z8GnccgmZ/bIaRfJ/mGw5pgD7ufvq6PFnwH4JymTbsb2Q8Cstkao+J/VhbNR89GCSZq5sOp7H\nAGvcfUmS7dlwPKtNyR0ws1bAn4Ar3X1jhc1vEZoV+gP3AE/Vd3xxhrh7IXAKcLmZHZvBWCplZs2A\n4cD/JticTce0HA+/w7O6f7CZjQd2ANOSFMn05+R3QHdgALCa0OSRzUZTea0908ezRhp8cjezpoTE\nPs3d/1xxu7tvdPfN0eNZQFMz61DPYcZiWRndfw48SfhpG28lcFDccpdoXSacArzl7msqbsimYxpZ\nE2u+iu4/T1AmK46tmZ0PnAYUR19Ee0jhc1Kn3H2Nu+90913A/Un2ny3HswkwEng8WZlMH8+aatDJ\nPWpr+z2wyN3vSFJm/6gcZjaIcMzW1V+UZXG0NLPWsceEk2vvVSg2E/h+1GvmSGBDXHNDfUtaG8qW\nYxpnJhDr/XIe8HSCMs8DJ5lZ26iZ4aRoXb0xs2HAT4Dh7l6apEwqn5M6VeE8z4gk+58DHGpm3aJf\neaMIf4f6diLwgbuXJNqYDcezxjJ9RjeTN2AI4Sf4AmB+dDsVuBS4NCozFlhIOJv/BvB/MhTrN6IY\n3oniGR+tj4/VgMmEXgjvAkUZirUlIVm3iVuXFceU8IWzGthOaOf9AdAeeBlYArwEtIvKFgEPxD33\nQmBpdLsgA3EuJbRTxz6r90ZlDwRmVfY5qec4H4k+fwsICfuAinFGy6cSeqgty0Sc0fqHYp/LuLIZ\nO57pvGn4ARGRPNSgm2VERPKVkruISB5SchcRyUNK7iIieUjJXUQkDym5i4jkISV3EZE89P8BTAze\nGzqmSS8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8VOXZ//HPhewS2d0IELQqu4AR\nsIiAWkVUKJQqi3VFqj+XtrZP4XEvlqdqraI+al2KVglSq3WrC20fsWhVFKiCiggiYARZIqCIVgPX\n74/7BCdhkkySk0wy+b5fr3ll5px7zrlmklxz5j73uW5zd0REJLM0SHcAIiISPyV3EZEMpOQuIpKB\nlNxFRDKQkruISAZSchcRyUBK7pKUme1lZtvNrFOcbdPJzL5jZrGP/TWz481sdcLj5WY2OJW2ldjX\nfWZ2eWWfX8Z2f21mD8S9XUmfhukOQOJhZtsTHjYH/gPsjB7/2N3zKrI9d98JtIi7bX3g7ofFsR0z\nmwSc4e5DE7Y9KY5tS+ZTcs8Q7r47uUZHhpPc/R+ltTezhu5eWBOxiUjNU7dMPRF97f6TmT1sZp8D\nZ5jZUWb2mpltNbP1ZnabmTWK2jc0MzeznOjxrGj9c2b2uZm9amZdKto2Wn+Smb1vZtvM7HYz+5eZ\nnV1K3KnE+GMzW2lmW8zstoTn7mVmt5hZgZmtAoaX8f5cYWZzSiy7w8xuju5PMrNl0ev5IDqqLm1b\n+WY2NLrf3MweimJ7BziiRNsrzWxVtN13zGxktLwX8L/A4KjLa3PCe3ttwvMviF57gZk9YWYHpPLe\nlMfMRkfxbDWzF8zssIR1l5vZOjP7zMzeS3itA81scbR8g5n9NtX9STVwd90y7AasBo4vsezXwNfA\nqYQP9WbAkcAAwje4g4D3gYuj9g0BB3Kix7OAzUAu0Aj4EzCrEm33BT4HRkXrLgO+Ac4u5bWkEuOT\nQEsgB/i06LUDFwPvANlAW2B++JNPup+DgO3A3gnb3gjkRo9PjdoYcCzwJdA7Wnc8sDphW/nA0Oj+\nTcCLQGugM/BuibanAQdEv5MJUQz7ResmAS+WiHMWcG10/4Qoxj5AU+BO4IVU3pskr//XwAPR/W5R\nHMdGv6PLgeXR/R7AGmD/qG0X4KDo/hvA+Oh+FjAg3f8L9fmmI/f65WV3f9rdd7n7l+7+hrsvcPdC\nd18F3AMMKeP5j7r7Qnf/BsgjJJWKtj0FeNPdn4zW3UL4IEgqxRh/4+7b3H01IZEW7es04BZ3z3f3\nAuD6MvazCnib8KED8D1gi7svjNY/7e6rPHgB+D8g6UnTEk4Dfu3uW9x9DeFoPHG/j7j7+uh3Mpvw\nwZybwnYBJgL3ufub7v4VMBUYYmbZCW1Ke2/KMg54yt1fiH5H1xM+IAYAhYQPkh5R196H0XsH4UP6\nEDNr6+6fu/uCFF+HVAMl9/rlo8QHZtbVzJ4xs0/M7DNgGtCujOd/knB/B2WfRC2t7YGJcbi7E450\nk0oxxpT2RTjiLMtsYHx0f0L0uCiOU8xsgZl9amZbCUfNZb1XRQ4oKwYzO9vM3oq6P7YCXVPcLoTX\nt3t77v4ZsAXokNCmIr+z0ra7i/A76uDuy4GfE34PG6Nuvv2jpucA3YHlZva6mY1I8XVINVByr19K\nDgO8m3C0+h133we4mtDtUJ3WE7pJADAzo3gyKqkqMa4HOiY8Lm+o5iPA8WbWgXAEPzuKsRnwKPAb\nQpdJK+BvKcbxSWkxmNlBwF3AhUDbaLvvJWy3vGGb6whdPUXbyyJ0/3ycQlwV2W4Dwu/sYwB3n+Xu\ngwhdMnsR3hfcfbm7jyN0vf0OeMzMmlYxFqkkJff6LQvYBnxhZt2AH9fAPv8K9DOzU82sIfAToH01\nxfgI8FMz62BmbYEpZTV290+Al4EHgOXuviJa1QRoDGwCdprZKcBxFYjhcjNrZeE6gIsT1rUgJPBN\nhM+58wlH7kU2ANlFJ5CTeBg4z8x6m1kTQpJ9yd1L/SZUgZhHmtnQaN//RThPssDMupnZsGh/X0a3\nXYQX8CMzaxcd6W+LXtuuKsYilaTkXr/9HDiL8I97N+HEZ7Vy9w3A6cDNQAFwMPBvwrj8uGO8i9A3\nvpRwsu/RFJ4zm3CCdHeXjLtvBX4GPE44KTmW8CGVimsI3yBWA88BDyZsdwlwO/B61OYwILGf+u/A\nCmCDmSV2rxQ9/3lC98jj0fM7Efrhq8Td3yG853cRPniGAyOj/vcmwI2E8ySfEL4pXBE9dQSwzMJo\nrJuA093966rGI5VjoctTJD3MbC9CN8BYd38p3fGIZAoduUuNM7PhUTdFE+AqwiiL19MclkhGUXKX\ndDgaWEX4yn8iMNrdS+uWEZFKULeMiEgG0pG7iEgGSlvhsHbt2nlOTk66di8iUictWrRos7uXNXwY\nSGNyz8nJYeHChenavYhInWRm5V1pDahbRkQkIym5i4hkICV3EZEMpJmYROqJb775hvz8fL766qt0\nhyIpaNq0KdnZ2TRqVFppobKVm9zNrCOhHsZ+hEJA97j7rSXaGHArobbEDsLEC4srFZGIVIv8/Hyy\nsrLIyckh/MtKbeXuFBQUkJ+fT5cuXcp/QhKpdMsUAj939+7AQOAiM+teos1JwCHRbTKh4FDs8vIg\nJwcaNAg/8yo05bNI/fbVV1/Rtm1bJfY6wMxo27Ztlb5llZvco1liFkf3PweWsWf97VHAg9EsNa8B\nrYrmcoxLXh5Mngxr1oB7+Dl5shK8SEUosdcdVf1dVeiEqoUJkPtSvCwphGSfONtMPkkmYDCzyWa2\n0MwWbtq0qUKBXnEF7NhRfNmOHWG5iIgUl3JyN7MWwGPAT6PpvCrM3e9x91x3z23fvtwLrIpZu7Zi\ny0WkdikoKKBPnz706dOH/fffnw4dOux+/PXXqZV9P+ecc1i+fHmZbe644w7yYvpKf/TRR/Pmm2/G\nsq2altJomWg2lseAPHf/S5ImH1N8KrHdU3LFpVOn0BWTbLmIxC8vL3wzXrs2/J9Nnw4TqzAVSNu2\nbXcnymuvvZYWLVrwi1/8olgbd8fdadAg+XHn/fffX+5+LrroosoHmUHKPXKPRsL8AVjm7jeX0uwp\n4EwLBgLb3H19jHEyfTo0b158WfPmYbmIxKsmz3GtXLmS7t27M3HiRHr06MH69euZPHkyubm59OjR\ng2nTpu1uW3QkXVhYSKtWrZg6dSqHH344Rx11FBs3bgTgyiuvZMaMGbvbT506lf79+3PYYYfxyiuv\nAPDFF1/wgx/8gO7duzN27Fhyc3PLPUKfNWsWvXr1omfPnlx++eUAFBYW8qMf/Wj38ttuuw2AW265\nhe7du9O7d2/OOOOM2N+zVKRy5D4I+BGw1MyKXv3lRBP9uvvvgWcJwyBXEoZCnhN3oEVHDHEeSYhI\ncmWd46qO/7n33nuPBx98kNzcXACuv/562rRpQ2FhIcOGDWPs2LF07158kN62bdsYMmQI119/PZdd\ndhkzZ85k6tSpe2zb3Xn99dd56qmnmDZtGs8//zy33347+++/P4899hhvvfUW/fr1KzO+/Px8rrzy\nShYuXEjLli05/vjj+etf/0r79u3ZvHkzS5cuBWDr1q0A3HjjjaxZs4bGjRvvXlbTUhkt87K7m7v3\ndvc+0e1Zd/99lNiJRslc5O4Hu3svd6+WimATJ8Lq1bBrV/ipxC5SPWr6HNfBBx+8O7EDPPzww/Tr\n149+/fqxbNky3n333T2e06xZM0466SQAjjjiCFavXp1022PGjNmjzcsvv8y4ceMAOPzww+nRo0eZ\n8S1YsIBjjz2Wdu3a0ahRIyZMmMD8+fP5zne+w/Lly7n00kuZO3cuLVu2BKBHjx6cccYZ5OXlVfoi\npKpS+QER2UNp57Kq6xzX3nvvvfv+ihUruPXWW3nhhRdYsmQJw4cPTzreu3Hjxrvv77XXXhQWFibd\ndpMmTcptU1lt27ZlyZIlDB48mDvuuIMf//jHAMydO5cLLriAN954g/79+7Nz585Y95sKJXcR2UM6\nz3F99tlnZGVlsc8++7B+/Xrmzp0b+z4GDRrEI488AsDSpUuTfjNINGDAAObNm0dBQQGFhYXMmTOH\nIUOGsGnTJtydH/7wh0ybNo3Fixezc+dO8vPzOfbYY7nxxhvZvHkzO0r2cdUA1ZYRkT2k8xxXv379\n6N69O127dqVz584MGjQo9n1ccsklnHnmmXTv3n33rahLJZns7Gyuu+46hg4dirtz6qmncvLJJ7N4\n8WLOO+883B0z44YbbqCwsJAJEybw+eefs2vXLn7xi1+QlZUV+2soT9rmUM3NzXVN1iFSc5YtW0a3\nbt3SHUatUFhYSGFhIU2bNmXFihWccMIJrFixgoYNa9fxbrLfmZktcvfcUp6yW+16JSIiNWD79u0c\nd9xxFBYW4u7cfffdtS6xV1VmvRoRkRS0atWKRYsWpTuMaqUTqiIiGUjJXUQkAym5i4hkICV3EZEM\npOQuIjVi2LBhe1yQNGPGDC688MIyn9eiRQsA1q1bx9ixY5O2GTp0KOUNrZ4xY0axi4lGjBgRS92X\na6+9lptuuqnK24mbkruI1Ijx48czZ86cYsvmzJnD+PHjU3r+gQceyKOPPlrp/ZdM7s8++yytWrWq\n9PZqOyV3EakRY8eO5Zlnntk9Mcfq1atZt24dgwcP3j3uvF+/fvTq1Ysnn3xyj+evXr2anj17AvDl\nl18ybtw4unXrxujRo/nyyy93t7vwwgt3lwu+5pprALjttttYt24dw4YNY9iwYQDk5OSwefNmAG6+\n+WZ69uxJz549d5cLXr16Nd26deP888+nR48enHDCCcX2k8ybb77JwIED6d27N6NHj2bLli27919U\nArioYNk///nP3ZOV9O3bl88//7zS720yGucuUg/99KcQ9wRDffpAlBeTatOmDf379+e5555j1KhR\nzJkzh9NOOw0zo2nTpjz++OPss88+bN68mYEDBzJy5MhS5xG96667aN68OcuWLWPJkiXFSvZOnz6d\nNm3asHPnTo477jiWLFnCpZdeys0338y8efNo165dsW0tWrSI+++/nwULFuDuDBgwgCFDhtC6dWtW\nrFjBww8/zL333stpp53GY489VmZ99jPPPJPbb7+dIUOGcPXVV/OrX/2KGTNmcP311/Phhx/SpEmT\n3V1BN910E3fccQeDBg1i+/btNG3atALvdvl05C4iNSaxayaxS8bdufzyy+nduzfHH388H3/8MRs2\nbCh1O/Pnz9+dZHv37k3v3r13r3vkkUfo168fffv25Z133im3KNjLL7/M6NGj2XvvvWnRogVjxozh\npZdeAqBLly706dMHKLusMIT68lu3bmXIkCEAnHXWWcyfP393jBMnTmTWrFm7r4QdNGgQl112Gbfd\ndhtbt26N/QrZcrdmZjOBU4CN7t4zyfqWwCzC5B0NgZvcvfy5sEQkbco6wq5Oo0aN4mc/+xmLFy9m\nx44dHHHEEQDk5eWxadMmFi1aRKNGjcjJyUla5rc8H374ITfddBNvvPEGrVu35uyzz67UdooUlQuG\nUDK4vG6Z0jzzzDPMnz+fp59+munTp7N06VKmTp3KySefzLPPPsugQYOYO3cuXbt2rXSsJaVy5P4A\nMLyM9RcB77r74cBQ4Hdm1riM9iJST7Vo0YJhw4Zx7rnnFjuRum3bNvbdd18aNWrEvHnzWJNswuQE\nxxxzDLNnzwbg7bffZsmSJUAoF7z33nvTsmVLNmzYwHPPPbf7OVlZWUn7tQcPHswTTzzBjh07+OKL\nL3j88ccZPHhwhV9by5Ytad269e6j/oceeoghQ4awa9cuPvroI4YNG8YNN9zAtm3b2L59Ox988AG9\nevViypQpHHnkkbz33nsV3mdZyj1yd/f5ZpZTVhMgK5prtQXwKRBvRXwRyRjjx49n9OjRxUbOTJw4\nkVNPPZVevXqRm5tb7hHshRdeyDnnnEO3bt3o1q3b7m8Ahx9+OH379qVr16507NixWLngyZMnM3z4\ncA488EDmzZu3e3m/fv04++yz6d+/PwCTJk2ib9++ZXbBlOaPf/wjF1xwATt27OCggw7i/vvvZ+fO\nnZxxxhls27YNd+fSSy+lVatWXHXVVcybN48GDRrQo0eP3bNKxSWlkr9Rcv9rKd0yWYQJsrsCWcDp\n7v5MedtUyV+RmqWSv3VPVUr+xnFC9UTgTeBAoA/wv2a2T7KGZjbZzBaa2cJNmzbFsGsREUkmjuR+\nDvCXaJLslcCHhKP4Pbj7Pe6e6+657du3j2HXIiKSTBzJfS1wHICZ7QccBqyKYbsiErN0zbwmFVfV\n31UqQyEfJoyCaWdm+cA1QKNo578HrgMeMLOlgAFT3H1zlaISkdg1bdqUgoIC2rZtW+rFQVI7uDsF\nBQVVurApldEyZRZ+cPd1wAmVjkBEakR2djb5+fnofFfd0LRpU7Kzsyv9fJUfEKknGjVqRJcuXdId\nhtQQlR8QEclASu4iIhlIyV1EJAMpuYuIZCAldxGRDKTkLiKSgZTcRUQykJK7iEgGUnIXEclASu4i\nIhlIyV1EJAMpuYuIZCAldxGRDKTkLiKSgZTcRUQyULnJ3cxmmtlGM3u7jDZDzexNM3vHzP4Zb4gi\nIlJRqRy5PwAML22lmbUC7gRGunsP4IfxhCYiIpVVbnJ39/nAp2U0mQD8xd3XRu03xhSbiIhUUhx9\n7ocCrc3sRTNbZGZnltbQzCab2UIzW6h5HEVEqk8cyb0hcARwMnAicJWZHZqsobvf4+657p7bvn37\nGHYtIiLJxJHc84G57v6Fu28G5gOHx7DdpD77DO67D9yraw8iInVfHMn9SeBoM2toZs2BAcCyGLab\n1BNPwPnnw/PPV9ceRETqvlSGQj4MvAocZmb5ZnaemV1gZhcAuPsy4HlgCfA6cJ+7lzpssqrGjYPs\nbLjhhurag4hI3dewvAbuPj6FNr8FfhtLROVo3Bh+/nP42c/gtddg4MCa2KuISN1SJ69QnTQJWrfW\n0buISGnqZHJv0QIuvjj0vy+rtt59EZG6q04md4BLLoFmzeC3NdIZJCJSt9TZ5N6+PZx3HsyaBfn5\n6Y5GRKR2qbPJHcKJ1V274JZb0h2JiEjtUqeTe05OGBp5zz3waVnVb0RE6pk6ndwBfvlL2L4d7rwz\n3ZGIiNQedT659+4NI0bAbbfBl1+mOxoRkdqhzid3gClTYNMmuP/+dEciIlI7ZERyHzwYjjoqDIss\nLEx3NCIi6ZcRyd0sHL2vXg1//nO6oxERSb+MSO4Ap54K3bqFkgQqBywi9V3GJPcGDcLImbfegrlz\n0x2NiEh6ZUxyB5gwQeWARUQgw5J748Zw2WXw4ouwYEHyNnl54eKnBg3Cz7y8GgxQRKSGZFRyhzBL\nU2nlgPPyYPJkWLMm9MuvWRMeK8GLSKZJZSammWa20czKnF3JzI40s0IzGxtfeBWXWA74vfeKr7vi\nCtixo/iyHTvCchGRTJLKkfsDwPCyGpjZXsANwN9iiKnKLrkEmjbdsxzw2rXJ25e2XESkrio3ubv7\nfKC8slyXAI8BG+MIqqrat4dzz4WHHipeDrhTp+TtS1suIlJXVbnP3cw6AKOBu1JoO9nMFprZwk2b\nNlV112UqKgc8Y8a3y6ZPh+bNi7dr3jwsFxHJJHGcUJ0BTHH3XeU1dPd73D3X3XPbt28fw65L16UL\nnH463H03bNkSlk2cGMoDd+4crmrt3Dk8njixWkMREalxcST3XGCOma0GxgJ3mtn3Y9hulU2Zsmc5\n4IkTQ5mCXbvCTyV2EclEVU7u7t7F3XPcPQd4FPh/7v5ElSOLQe/ecNJJcOutKgcsIvVLKkMhHwZe\nBQ4zs3wzO8/MLjCzC6o/vKpTOWARqY/M01RlKzc31xcuXFjt+3GH734XNmyA99+Hhg2rfZciItXG\nzBa5e2557TLuCtWSisoBf/ghPPpouqMREakZGZ/cAUaOhK5d4frrVQ5YROqHepHcE8sB/61WXEMr\nIlK96kVyhzDksUOHcPQuIpLp6k1yTywH/Prr6Y5GRKR61ZvkDqEccKtWmsxDRDJfvUruWVmhHPDj\nj8Py5emORkSk+tSr5A6hHHCTJnuWAxYRyST1Lrnvu28oB/zgg/Dyy+mORkSketS75A7wq1/BQQfB\nKaeE4ZEiIpmmXib3du3CePesLDjxRPjgg3RHJCISr3qZ3CHMvvS3v0FhIXzve7B+fbojEhGJT71N\n7gDdusGzz8LGjeEIvmhSDxGRuq5eJ3eA/v3hiSfC0MhTToEdO9IdkYhI1dX75A5w/PEweza89hqM\nHQvffJPuiEREqkbJPfKDH8Dvfw/PPQdnnx2m4RMRqatSmYlpppltNLO3S1k/0cyWmNlSM3vFzA6P\nP8yacf758JvfhKP4n/xE5YFFpO5KZV6iB4D/BR4sZf2HwBB332JmJwH3AAPiCa/mTZkCmzfD734H\n7dvD1VenOyIRkYorN7m7+3wzyylj/SsJD18DsqseVvqYhdIEBQVwzTXQpk2oRyMiUpfE3ed+HvBc\naSvNbLKZLTSzhZs2bYp51/Exg3vvhVGjQi2a2bOLr8/Lg5ycMAlITk54LCJSm8SW3M1sGCG5Tymt\njbvf4+657p7bvn37uHZdLRo2hDlzYMgQOOuscKIVQiKfPBnWrAl98mvWhMdK8CJSm8SS3M2sN3Af\nMMrdC+LYZm3QtCk89RT07h1G07zyClxxxZ5j4XfsCMtFRGqLKid3M+sE/AX4kbu/X/WQapd99glH\n7dnZcPLJ4Ug9mbVrazYuEZGylHtC1cweBoYC7cwsH7gGaATg7r8HrgbaAneaGUChu+dWV8DpsO++\n8Pe/w6BB8PnnsHPnnm06dar5uERESpPKaJnx5ayfBEyKLaJaqnPnUGhswAD44oviY+CbN4fp09MX\nm4hISbpCtQK6d4d//CNMtt2oUVjWuTPccw9MnJje2EREEim5V9CAAfD00+H+0UfDsmVK7CJS+yi5\nV8L3vgezZsG//gXjxoWa8CIitYmSeyWddhrcdlsYKnnhhapDIyK1Syq1ZaQUF18MGzbAr38N++8P\n112X7ohERAIl9yqaNg0++eTbBH/RRemOSEREyb3KzOCuu8JUfZdcEsbE//CH6Y5KROo79bnHoKgO\nzaBBcMYZ8MIL6Y5IROo7JfeYNGsWTq4ecgh8//vw73+nOyIRqc+U3GPUujXMnRt+nnQSrFqV7ohE\npL5Sco9Zhw4hwX/zDZxwQhhNIyJS05Tcq0HXrvDMM7BuHYwYEYqNlaQJP0SkOim5V5OBA+HRR+Gt\nt2DMGPj662/XacIPEaluSu7VaMQImDkzFBs76yzYtSss14QfIlLdNM69mp15ZrjIacqUMAZ+xozS\nJ/bQhB8iEpdUJuuYCZwCbHT3nknWG3ArMALYAZzt7ovjDrQu+6//Cgn+llvggAPCxB7JZnTShB8i\nEpdUumUeAIaXsf4k4JDoNhm4q+phZRYzuOkmmDAB/vu/wyia5s2Lt9GEHyISp3KTu7vPBz4to8ko\n4EEPXgNamdkBcQWYKRo0gPvvD4l95sxQSbJz55D4NeGHiMQtjj73DsBHCY/zo2XrSzY0s8mEo3s6\n1cM+iMaN4bHHYNgwuPPOcKL1u99Nd1QikolqdLSMu9/j7rnuntu+ffua3HWt0aJFGAOfnQ2nnALv\nvpvuiEQkE8WR3D8GOiY8zo6WSSn23TdcxdqkCfTvD+eeCy+9pAk/RCQ+cST3p4AzLRgIbHP3Pbpk\npLguXWD+/DBN35//DMccA4ceGk6qfvRR+c8XESlLucndzB4GXgUOM7N8MzvPzC4wswuiJs8Cq4CV\nwL3A/6u2aDPMIYfAffeFYZJ//GPoqrnyynCC9YQT4OGH4csv0x2liNRF5mnqC8jNzfWFCxemZd+1\n2apV8OCD8MADYSx8y5YwfjycfXbowjFLd4Qikk5mtsjdc8trp/IDtcxBB8G114Yk/3//B6eeGo7q\nBw6Enj3ht78NR/oiImVRcq+lGjSAY4+Fhx6C9evh3nuhVSv45S+/HWnzk5+ELhxVlhSRktQtU8cs\nXx66bO6+G7ZsKb6ueXNdDCWS6dQtk6EOOwx+8xvIytpz3Y4dcPnlNR+TiNQ+Su51VGnDJdeu1fR+\nIqLkXmeVVr3BDHr0gOuug//8p2ZjEpHaQ8m9jpo+PXllyVtvhZEj4eqroVcv+Pvf0xOfiKSXknsd\nNXFiOHlasrLkJZfAn/4Uyhu4h4uhxo0L87mKSP2h0TIZ7Kuv4MYb4X/+J1SknDYNLr4YGmr+LZE6\nS6NlhKZNQ/fMO+/AoEHws5/BEUfAK6+kOzIRqW5K7vXAwQfDs8+GWvKffhoS/fnnQ0FBuiMTkeqi\n5F5PmMGYMbBsWZjT9YEHwpj5P/wBdu1Kd3QiEjcl93qmRYvQD//vf0O3bjBpEhx9NPzrX6onL5JJ\nlNzrqZ49Qz35yZNhwYKQ4Bs3htNPh4811YpInafkXo/Nng2zZn3bLVNYCI88Ah07wvDhqicvUpcp\nuddjV1wR6tGUtM8+oW9+wgQ44AD48Y/h1VfVbSNSl6SU3M1suJktN7OVZjY1yfpOZjbPzP5tZkvM\nbET8oUrc1q5Nvvyzz+DDD0M9+ZEjw9H9d78LXbuGomX5+TUbp4hUXCrT7O0F3AGcBHQHxptZ9xLN\nrgQecfe+wDjgzrgDlfiVVp+mU6dv68k/+GCYHGTmTNh//1B1slOncOXr7NnJj/xFJP1SOXLvD6x0\n91Xu/jUwBxhVoo0D+0T3WwK62L0OKK0+zfTpxZdlZcE558A//wkrV8JVV8H774cSCAccEE7KarSN\nSO2SSnLvACQWmM2PliW6FjjDzPIJE2ZfkmxDZjbZzBaa2cJNmzZVIlyJU2n1acqa7OPgg+FXvwpl\nhV94Ab7//TAD1NFHhxOxkybBo4/C1q019zpEZE/l1pYxs7HAcHefFD3+ETDA3S9OaHNZtK3fmdlR\nwB+Anu5e6uUxqi2TOT7/HP7yF/jrX0MVym3bYK+9wryvw4fDiSeGsgcNdPpepMpSrS2TSgmpj4GO\nCY+zo2WJzgOGA7j7q2bWFGgHbEwtXKnLnnoKrrkmnKDt2DHM7bprFzz/fOjCueoqaNcu9NMPHx5+\n7rdfuqMWyWypHEu9ARxiZl2GZ4rPAAANZklEQVTMrDHhhOlTJdqsBY4DMLNuQFNA/S71QF5e6HNf\nsyb0ua9dCzfdFEbWvPEGbNgQRtsMHx6O6s88M5yYPeKIMBTzpZfgm2/S/SpEMk9KJX+joY0zgL2A\nme4+3cymAQvd/alo9My9QAvCydVfuvvfytqmumUyQ05OSOwlde4Mq1cXX7ZrVyh78Pzz4fbqq7Bz\nZxhXf/zx4QPgyCPhkENg771rInqRuifVbhnVc5cqadAg+SgZs/ILkm3dGk7KFiX7xHlhs7NDYbPD\nDoNDD/32fqdOoT9fpL6Ks89dpFSdOiU/ci9tDH2iVq1CpcoxY8IHxPLlsHRp+Pn+++FnXl44QVuk\nSRP4znf2TPqHHgpt28b3ukTqOiV3qZLp00Ofe+LFTMnGypfHLPTTd+1afLk7bNoUEn1i0n/nnXAi\nt7Dw27Zt24ZEf+KJMH586N4Rqa/ULSNVlpcXTo6uXRuO2KdPL3usfFwKC0OZhMSkv3QpvPZa+FA4\n8shQH+f008PFViKZQH3uUm/l54dJwmfPhsWLw3mBYcNCoh8zJnQHidRVmkNV6pS8vDDypkGD8DMv\nr/Lbys6Gn/8cFi0K1S2vvDKcFzjvvDC+fsyYcBWtyhlLJtORu6Rd0Vj5kv325ZVCqAh3WLgwHM3P\nmROKoWVlhUQ/YUIoktZQZ6CkDlC3jNQZFRkrH4edO+HFF0Oif+yxMBpn331D3/yECTBgQDjBK1Ib\nKblLnVGVsfJV9dVX8NxzIdE//TT85z/hw6ZPn/DhUvLWtq0Sv6SXxrlLnVGVsfJV1bQpjB4dbtu2\nwRNPwOOPw4oV8I9/wPbtxds3bx7iSpb4O3eGAw/URVZSOyi5S9rFNVa+qlq2hLPOCjcI3ya2bAkf\nPMluixbB5s3Ft9GwYTihW5ToDzjg25+J97Oy9A1AqpeSu6Rd0UnTqo6Vj3u8vRm0aRNuffsmb/PF\nF2F/yZL/ggWwfn3yUTnNmydP+omP99svfLNo3Dh8aOjDQCpCfe6SEWpixE1luIfunvXrv72tW5f8\ncckuoJIaNw63Jk2+vV/yVnJds2bhPEHbtqHscrt2e94vORuX1G46oSr1Sk2PuKkO27cXT/YbN4YT\nvF9/nfyWyrodO6CgIHQvlaZZs+RJv+h+mzbhwq+WLcOt6H6LFvo2kQ46oSr1ytq1FVteG7VoEerh\nVEdNnMLCkOA3bw7JfvPmb2+JjwsKwofk5s1lfyBAGOW0zz7JE3/Jx82bhw+C0m5Q9nqz0E3Vt6+u\nR0iV3ibJCHGNuElXnZzq1rAhtG8fbqkq+kAoKAhdS9u2hTLNRfdLPt66NfwOiu5/9ln8Q1mzsmDw\nYBg6NJSU6NNHyb40elskI8Qx4qZkv/2aNeExZEaCr6jKfCAkcg9dTdu2hffUvfRbUfuybqtWwbx5\n4QK0Z58Nz9lnHzjmmJDshw4NyV5DUYNUZ2IaDtxKmInpPne/Pkmb04BrCTMxveXuE8rapvrcJW5V\nPerOhH77+mLdOvjnP0Oif/HFUBUUQhfQMceEo/qhQ6F378xL9rGdUDWzvYD3ge8B+YQ5Vce7+7sJ\nbQ4BHgGOdfctZravu5c5ObaSu9Q26bxSVqrm44+/Tfbz5sHKlWF569bFj+w7dw7fIkq7ffll2et3\n7Ah/D02ahGGqFfmZeD87O9wqI84Tqv2Ble6+KtrwHGAU8G5Cm/OBO9x9C0B5iV2kNkrnlbJSNR06\nhLpAE6L+gvz8kOyLunGefLJy223SJHTvFd2aNQvLv/oqjEgq+TPVwYdTpsD1e/R/xCuV5N4BSJjd\nknxgQIk2hwKY2b8IXTfXuvvzJTdkZpOByQCd9B8jtUxcV8pm6knZuiQ7O7znRe/7Rx+FZF9QEBJ0\nYsIu7dasWcW6dNzhm2+SJ/2SP3NyquVlFxPXCdWGwCHAUCAbmG9mvdx9a2Ijd78HuAdCt0xM+xaJ\nRRxXyuqkbO3UsSOccUb17sPs24vHsrKqd1+pSGWyjo+BjgmPs6NlifKBp9z9G3f/kNBHrxkspc6Z\nODGcPN21K/ysaEK+4oriR/4QHl9xRVwRiqQmleT+BnCImXUxs8bAOOCpEm2eIBy1Y2btCN00q2KM\nU6ROiONiqjhnpZL6q9zk7u6FwMXAXGAZ8Ii7v2Nm08xsZNRsLlBgZu8C84D/cveC6gpapLYq7VRS\nqqeYirp11qwJfbhF3TpK8FJRqi0jEqOqFjDTWHspjybIFkmDiRNDIu/cOZxg69y5YpUpM6FGjtQO\nSu4iMavKSdmqdusUUb+9KLmL1CLTp+9ZX72yNXLUb1+/KbmL1CJV7dYBDceUQMldpJap6lj7uPrt\n1bVTtym5i2SYOPrt1bVT9ym5i2SYOPrt1bVT9ym5i2SYOPrt1bVT92kmJpEMlFgRsTLiKH+sImrp\npSN3EdlDbera0dF/5Si5i8geakvXjk7sVp5qy4hItYijTo5q7exJtWVEJK3i6NpRrZ3KU3IXkWoR\nR9eOau1UnpK7iFSbql5tq1o7lZdScjez4Wa23MxWmtnUMtr9wMzczMrtDxIRKY9q7VReucndzPYC\n7gBOAroD482se5J2WcBPgAVxByki9Zdq7VROKkfu/YGV7r7K3b8G5gCjkrS7DrgB+CrG+EREqqQ2\n1dqpyQ+IVJJ7B+CjhMf50bLdzKwf0NHdn4kxNhGRKqstF2TVdN9/lU+omlkD4Gbg5ym0nWxmC81s\n4aZNm6q6axGRctWWC7Jquu+/3IuYzOwo4Fp3PzF6/N8A7v6b6HFL4ANge/SU/YFPgZHuXupVSrqI\nSUTqijgupmrQIByxl2QWziekKs6LmN4ADjGzLmbWGBgHPFW00t23uXs7d89x9xzgNcpJ7CIidUkc\nXTtxjdlPVbnJ3d0LgYuBucAy4BF3f8fMppnZyOoJS0Sk9oijayeOD4iKUG0ZEZEakpcX+tjXrg1H\n7NOnV3xoZ6rdMqrnLiJSQ6paZ78iVH5ARCQDKbmLiGQgJXcRkQyk5C4ikoGU3EVEMlDahkKa2SYg\nyTVftUo7YHO6g0iB4oxfXYlVccarLsTZ2d3bl9cobcm9LjCzhamMJ003xRm/uhKr4oxXXYkzFeqW\nERHJQEruIiIZSMm9bPekO4AUKc741ZVYFWe86kqc5VKfu4hIBtKRu4hIBlJyFxHJQPU+uZtZRzOb\nZ2bvmtk7ZvaTJG2Gmtk2M3szul2dplhXm9nSKIY96iVbcJuZrTSzJdHctjUd42EJ79ObZvaZmf20\nRJu0vZ9mNtPMNprZ2wnL2pjZ381sRfSzdSnPPStqs8LMzkpDnL81s/ei3+3jZtaqlOeW+XdSA3Fe\na2YfJ/x+R5Ty3OFmtjz6e52ahjj/lBDjajN7s5Tn1tj7GSt3r9c34ACgX3Q/C3gf6F6izVDgr7Ug\n1tVAuzLWjwCeAwwYCCxIc7x7AZ8QLrqoFe8ncAzQD3g7YdmNwNTo/lTghiTPawOsin62ju63ruE4\nTwAaRvdvSBZnKn8nNRDntcAvUvjb+AA4CGgMvFXy/6664yyx/nfA1el+P+O81fsjd3df7+6Lo/uf\nE2ab6pDeqCptFPCgB68BrczsgDTGcxzwgbvXmiuR3X0+YY7fRKOAP0b3/wh8P8lTTwT+7u6fuvsW\n4O/A8JqM093/5mFmNAjTWWZX1/5TVcr7mYr+wEp3X+XuXwNzCL+HalFWnGZmwGnAw9W1/3So98k9\nkZnlAH2BBUlWH2Vmb5nZc2bWo0YD+5YDfzOzRWY2Ocn6DsBHCY/zSe8H1ThK/4epDe9nkf3cfX10\n/xNgvyRtatt7ey7hW1oy5f2d1ISLo+6jmaV0c9Wm93MwsMHdV5Syvja8nxWm5B4xsxbAY8BP3f2z\nEqsXE7oWDgduB56o6fgiR7t7P+Ak4CIzOyZNcZQrmkx9JPDnJKtry/u5Bw/fw2v1+GAzuwIoBPJK\naZLuv5O7gIOBPsB6QpdHbTaeso/a0/1+VoqSO2BmjQiJPc/d/1Jyvbt/5u7bo/vPAo3MrF0Nh4m7\nfxz93Ag8Tvhqm+hjoGPC4+xoWTqcBCx29w0lV9SW9zPBhqLuq+jnxiRtasV7a2ZnA6cAE6MPoj2k\n8HdSrdx9g7vvdPddwL2l7L+2vJ8NgTHAn0prk+73s7LqfXKP+tv+ACxz95tLabN/1A4z60943wpq\nLkows73NLKvoPuHk2tslmj0FnBmNmhkIbEvobqhppR4N1Yb3s4SngKLRL2cBTyZpMxc4wcxaR90M\nJ0TLaoyZDQd+CYx09x2ltEnl76RalTjPM7qU/b8BHGJmXaJveeMIv4eadjzwnrvnJ1tZG97PSkv3\nGd1034CjCV/DlwBvRrcRwAXABVGbi4F3CGf0XwO+m4Y4D4r2/1YUyxXR8sQ4DbiDMAphKZCbpvd0\nb0KybpmwrFa8n4QPnPXAN4R+3vOAtsD/ASuAfwBtora5wH0Jzz0XWBndzklDnCsJ/dRFf6e/j9oe\nCDxb1t9JDcf5UPT3t4SQsA8oGWf0eARhdNoH6YgzWv5A0d9lQtu0vZ9x3lR+QEQkA9X7bhkRkUyk\n5C4ikoGU3EVEMpCSu4hIBlJyFxHJQEruIiIZSMldRCQD/X/GAGn/K46wtgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#A VGG network\n",
    "\n",
    "#The neural network architectures will be built using combinations of the following parameters.\n",
    "params = {'classes' : [classes],\n",
    "          'layers' : [[3, 1024]],\n",
    "          'conv_layer_start' : [64],\n",
    "          'activation' : ['relu'],\n",
    "          'batch_size': [512],\n",
    "          'epochs': [50],\n",
    "          'optimizer': ['rmsprop'],\n",
    "          'loss': ['sparse_categorical_crossentropy'],\n",
    "          'callbacks' : [[keras.callbacks.EarlyStopping(monitor='val_loss',\n",
    "                              min_delta=0,\n",
    "                              patience=2,\n",
    "                              verbose=0, mode='auto')]]}\n",
    "\n",
    "#Hyperopt will optimize the following parameters in the given ranges.\n",
    "search_space = {'lr': hp.loguniform('lr', -8, -6), 'dropout_prob' : hp.uniform('dropout_prob', 0.2, 0.5)}\n",
    "\n",
    "#The call to process_model will compile, train and evaulate the above models and optimize the hyper parameters.\n",
    "#The output will be saved to a file and the output details will be recorded in the model_res_file. \n",
    "model_name = \"VGG\"\n",
    "meta_model.max_evals = 3\n",
    "model_results = meta_model.process_model(params, search_space, vgg_model, model_name, [X_train, X_val, X_test, y_train, y_val, y_test], \n",
    "              [model_name, 0, 0, 0, 'layers', 'epochs', 'activation', 'optimizer', 'lr','reg_param', 'dropout_prob', 'batchNorm' , 'conv_layer_start'])\n",
    "\n",
    "#Now lets plot the results.\n",
    "history = model_results[0][0]\n",
    "plot_train_val(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1468,
     "status": "ok",
     "timestamp": 1566502824537,
     "user": {
      "displayName": "Krishan Rajaratnam",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mAto1HIjJ0u4TNrl13hFRazE_mhv2-W5G8WFYoicQ=s64",
      "userId": "09748671397076320798"
     },
     "user_tz": 240
    },
    "id": "PVLiAyC-1NVM",
    "outputId": "6d18afb7-bb5a-47a6-81e7-7cbcf1e819bb"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>networkType</th>\n",
       "      <th>trainAcc</th>\n",
       "      <th>trainTime (s)</th>\n",
       "      <th>testAcc</th>\n",
       "      <th>layers</th>\n",
       "      <th>epochs</th>\n",
       "      <th>activations</th>\n",
       "      <th>optim</th>\n",
       "      <th>learning_rate</th>\n",
       "      <th>reg_param</th>\n",
       "      <th>dropout_prob</th>\n",
       "      <th>normTech</th>\n",
       "      <th>misc</th>\n",
       "      <th>testAccPerT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>VGG</td>\n",
       "      <td>0.855225</td>\n",
       "      <td>634.297548</td>\n",
       "      <td>0.7888</td>\n",
       "      <td>3-1024</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>rmsprop</td>\n",
       "      <td>0.000467</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.443228</td>\n",
       "      <td>NaN</td>\n",
       "      <td>128.0</td>\n",
       "      <td>0.124358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>VGG</td>\n",
       "      <td>0.864375</td>\n",
       "      <td>196.658036</td>\n",
       "      <td>0.7868</td>\n",
       "      <td>NaN</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>rmsprop</td>\n",
       "      <td>0.001616</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.271490</td>\n",
       "      <td>NaN</td>\n",
       "      <td>64.0</td>\n",
       "      <td>0.400085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>VGG</td>\n",
       "      <td>0.847875</td>\n",
       "      <td>244.679290</td>\n",
       "      <td>0.7793</td>\n",
       "      <td>3-1024</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>rmsprop</td>\n",
       "      <td>0.001027</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.319026</td>\n",
       "      <td>NaN</td>\n",
       "      <td>64.0</td>\n",
       "      <td>0.318499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>VGG</td>\n",
       "      <td>0.876775</td>\n",
       "      <td>166.077780</td>\n",
       "      <td>0.7691</td>\n",
       "      <td>NaN</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>rmsprop</td>\n",
       "      <td>0.001969</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.252096</td>\n",
       "      <td>NaN</td>\n",
       "      <td>64.0</td>\n",
       "      <td>0.463096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>VGG</td>\n",
       "      <td>0.832350</td>\n",
       "      <td>203.996223</td>\n",
       "      <td>0.7642</td>\n",
       "      <td>NaN</td>\n",
       "      <td>200</td>\n",
       "      <td>relu</td>\n",
       "      <td>rmsprop</td>\n",
       "      <td>0.001992</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.311089</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.374615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>VGG</td>\n",
       "      <td>0.822100</td>\n",
       "      <td>119.169290</td>\n",
       "      <td>0.7623</td>\n",
       "      <td>NaN</td>\n",
       "      <td>25</td>\n",
       "      <td>relu</td>\n",
       "      <td>rmsprop</td>\n",
       "      <td>0.001591</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.205474</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.639678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>VGG</td>\n",
       "      <td>0.823950</td>\n",
       "      <td>230.599190</td>\n",
       "      <td>0.7552</td>\n",
       "      <td>NaN</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>rmsprop</td>\n",
       "      <td>0.000893</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.425419</td>\n",
       "      <td>NaN</td>\n",
       "      <td>64.0</td>\n",
       "      <td>0.327495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>VGG</td>\n",
       "      <td>0.800700</td>\n",
       "      <td>233.410083</td>\n",
       "      <td>0.7474</td>\n",
       "      <td>NaN</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>rmsprop</td>\n",
       "      <td>0.000702</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.290495</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32.0</td>\n",
       "      <td>0.320209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>VGG</td>\n",
       "      <td>0.808500</td>\n",
       "      <td>283.771374</td>\n",
       "      <td>0.7368</td>\n",
       "      <td>NaN</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>rmsprop</td>\n",
       "      <td>0.000525</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.416822</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32.0</td>\n",
       "      <td>0.259646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>VGG</td>\n",
       "      <td>0.804025</td>\n",
       "      <td>222.012115</td>\n",
       "      <td>0.7351</td>\n",
       "      <td>NaN</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>rmsprop</td>\n",
       "      <td>0.001014</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.465692</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32.0</td>\n",
       "      <td>0.331108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>deepCNN</td>\n",
       "      <td>0.823450</td>\n",
       "      <td>174.392470</td>\n",
       "      <td>0.7270</td>\n",
       "      <td>512-128</td>\n",
       "      <td>200</td>\n",
       "      <td>relu</td>\n",
       "      <td>adam</td>\n",
       "      <td>0.002181</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.335836</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.416876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>deepCNN</td>\n",
       "      <td>0.830675</td>\n",
       "      <td>129.214671</td>\n",
       "      <td>0.7250</td>\n",
       "      <td>512-128</td>\n",
       "      <td>200</td>\n",
       "      <td>tanh</td>\n",
       "      <td>adam</td>\n",
       "      <td>0.001977</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.271415</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.561082</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>deepCNN</td>\n",
       "      <td>0.821825</td>\n",
       "      <td>130.668939</td>\n",
       "      <td>0.7236</td>\n",
       "      <td>512-128</td>\n",
       "      <td>200</td>\n",
       "      <td>tanh</td>\n",
       "      <td>rmsprop</td>\n",
       "      <td>0.001073</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.227011</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.553766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>deepCNN</td>\n",
       "      <td>0.787900</td>\n",
       "      <td>100.808334</td>\n",
       "      <td>0.7054</td>\n",
       "      <td>1024-512</td>\n",
       "      <td>200</td>\n",
       "      <td>tanh</td>\n",
       "      <td>rmsprop</td>\n",
       "      <td>0.001090</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.232500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.699744</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>deepCNN</td>\n",
       "      <td>0.729350</td>\n",
       "      <td>108.255273</td>\n",
       "      <td>0.6862</td>\n",
       "      <td>512-128</td>\n",
       "      <td>200</td>\n",
       "      <td>relu</td>\n",
       "      <td>rmsprop</td>\n",
       "      <td>0.000831</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.308497</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.633872</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>deepCNN</td>\n",
       "      <td>0.731925</td>\n",
       "      <td>277.092485</td>\n",
       "      <td>0.6367</td>\n",
       "      <td>4096-2048</td>\n",
       "      <td>200</td>\n",
       "      <td>tanh</td>\n",
       "      <td>rmsprop</td>\n",
       "      <td>0.000391</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.339022</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.229779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>simpleCNN</td>\n",
       "      <td>0.676175</td>\n",
       "      <td>59.332155</td>\n",
       "      <td>0.6038</td>\n",
       "      <td>NaN</td>\n",
       "      <td>200</td>\n",
       "      <td>relu</td>\n",
       "      <td>rmsprop</td>\n",
       "      <td>0.002978</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.432042</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.017661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>deepCNN</td>\n",
       "      <td>0.615675</td>\n",
       "      <td>107.094035</td>\n",
       "      <td>0.5844</td>\n",
       "      <td>1024-512</td>\n",
       "      <td>200</td>\n",
       "      <td>tanh</td>\n",
       "      <td>adam</td>\n",
       "      <td>0.002296</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.497522</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.545689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>deepCNN</td>\n",
       "      <td>0.605000</td>\n",
       "      <td>106.321745</td>\n",
       "      <td>0.5578</td>\n",
       "      <td>1024-512</td>\n",
       "      <td>200</td>\n",
       "      <td>relu</td>\n",
       "      <td>rmsprop</td>\n",
       "      <td>0.002207</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.317671</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.524634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>deepCNN</td>\n",
       "      <td>0.572825</td>\n",
       "      <td>81.613067</td>\n",
       "      <td>0.5463</td>\n",
       "      <td>1024-512</td>\n",
       "      <td>200</td>\n",
       "      <td>relu</td>\n",
       "      <td>adam</td>\n",
       "      <td>0.001724</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.303849</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.669378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>simpleCNN</td>\n",
       "      <td>0.590625</td>\n",
       "      <td>66.808673</td>\n",
       "      <td>0.5411</td>\n",
       "      <td>NaN</td>\n",
       "      <td>200</td>\n",
       "      <td>relu</td>\n",
       "      <td>adam</td>\n",
       "      <td>0.004196</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.541364</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.809925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>simpleCNN</td>\n",
       "      <td>0.467150</td>\n",
       "      <td>70.789448</td>\n",
       "      <td>0.4277</td>\n",
       "      <td>NaN</td>\n",
       "      <td>200</td>\n",
       "      <td>tanh</td>\n",
       "      <td>adam</td>\n",
       "      <td>0.007222</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.233961</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.604186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>deepCNN</td>\n",
       "      <td>0.394725</td>\n",
       "      <td>57.906200</td>\n",
       "      <td>0.3934</td>\n",
       "      <td>4096-2048</td>\n",
       "      <td>200</td>\n",
       "      <td>relu</td>\n",
       "      <td>adam</td>\n",
       "      <td>0.000610</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.264574</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.679375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>deepCNN</td>\n",
       "      <td>0.360150</td>\n",
       "      <td>53.930150</td>\n",
       "      <td>0.3596</td>\n",
       "      <td>4096-2048</td>\n",
       "      <td>200</td>\n",
       "      <td>relu</td>\n",
       "      <td>rmsprop</td>\n",
       "      <td>0.000637</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.344549</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.666788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>logisticRegression</td>\n",
       "      <td>0.362500</td>\n",
       "      <td>4.221944</td>\n",
       "      <td>0.3549</td>\n",
       "      <td>NaN</td>\n",
       "      <td>50</td>\n",
       "      <td>softmax</td>\n",
       "      <td>adam</td>\n",
       "      <td>0.004552</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.406080</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>simpleCNN</td>\n",
       "      <td>0.338875</td>\n",
       "      <td>17.212508</td>\n",
       "      <td>0.3236</td>\n",
       "      <td>NaN</td>\n",
       "      <td>200</td>\n",
       "      <td>tanh</td>\n",
       "      <td>rmsprop</td>\n",
       "      <td>0.004778</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008589</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.880028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>logisticRegression</td>\n",
       "      <td>0.318650</td>\n",
       "      <td>30.498545</td>\n",
       "      <td>0.3147</td>\n",
       "      <td>NaN</td>\n",
       "      <td>50</td>\n",
       "      <td>softmax</td>\n",
       "      <td>adadelta</td>\n",
       "      <td>0.007105</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.031853</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>logisticRegression</td>\n",
       "      <td>0.314150</td>\n",
       "      <td>8.412281</td>\n",
       "      <td>0.3077</td>\n",
       "      <td>NaN</td>\n",
       "      <td>50</td>\n",
       "      <td>softmax</td>\n",
       "      <td>adagrad</td>\n",
       "      <td>0.012521</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.657748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>VGG</td>\n",
       "      <td>0.267400</td>\n",
       "      <td>1122.177484</td>\n",
       "      <td>0.2707</td>\n",
       "      <td>3-1024</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>adadelta</td>\n",
       "      <td>0.001623</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.412031</td>\n",
       "      <td>NaN</td>\n",
       "      <td>128.0</td>\n",
       "      <td>0.024123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>logisticRegression</td>\n",
       "      <td>0.258025</td>\n",
       "      <td>2.448996</td>\n",
       "      <td>0.2499</td>\n",
       "      <td>NaN</td>\n",
       "      <td>50</td>\n",
       "      <td>softmax</td>\n",
       "      <td>nadam</td>\n",
       "      <td>0.005123</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.204182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>logisticRegression</td>\n",
       "      <td>0.218925</td>\n",
       "      <td>5.114744</td>\n",
       "      <td>0.2151</td>\n",
       "      <td>NaN</td>\n",
       "      <td>50</td>\n",
       "      <td>softmax</td>\n",
       "      <td>rmsprop</td>\n",
       "      <td>0.003709</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.205489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>VGG</td>\n",
       "      <td>0.162700</td>\n",
       "      <td>367.537869</td>\n",
       "      <td>0.1638</td>\n",
       "      <td>3-1024</td>\n",
       "      <td>50</td>\n",
       "      <td>relu</td>\n",
       "      <td>adadelta</td>\n",
       "      <td>0.000407</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.262048</td>\n",
       "      <td>NaN</td>\n",
       "      <td>64.0</td>\n",
       "      <td>0.044567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>simpleCNN</td>\n",
       "      <td>0.151500</td>\n",
       "      <td>28.660981</td>\n",
       "      <td>0.1532</td>\n",
       "      <td>NaN</td>\n",
       "      <td>200</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>rmsprop</td>\n",
       "      <td>0.003098</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.404623</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.534525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>simpleCNN</td>\n",
       "      <td>0.129850</td>\n",
       "      <td>22.753640</td>\n",
       "      <td>0.1305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>200</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>adam</td>\n",
       "      <td>0.004474</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.038564</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.573535</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           networkType  trainAcc  trainTime (s)  ...  normTech   misc  testAccPerT\n",
       "32                 VGG  0.855225     634.297548  ...       NaN  128.0     0.124358\n",
       "25                 VGG  0.864375     196.658036  ...       NaN   64.0     0.400085\n",
       "30                 VGG  0.847875     244.679290  ...       NaN   64.0     0.318499\n",
       "27                 VGG  0.876775     166.077780  ...       NaN   64.0     0.463096\n",
       "23                 VGG  0.832350     203.996223  ...       NaN    NaN     0.374615\n",
       "22                 VGG  0.822100     119.169290  ...       NaN    NaN     0.639678\n",
       "29                 VGG  0.823950     230.599190  ...       NaN   64.0     0.327495\n",
       "24                 VGG  0.800700     233.410083  ...       NaN   32.0     0.320209\n",
       "28                 VGG  0.808500     283.771374  ...       NaN   32.0     0.259646\n",
       "26                 VGG  0.804025     222.012115  ...       NaN   32.0     0.331108\n",
       "14             deepCNN  0.823450     174.392470  ...       NaN    NaN     0.416876\n",
       "20             deepCNN  0.830675     129.214671  ...       NaN    NaN     0.561082\n",
       "17             deepCNN  0.821825     130.668939  ...       NaN    NaN     0.553766\n",
       "18             deepCNN  0.787900     100.808334  ...       NaN    NaN     0.699744\n",
       "11             deepCNN  0.729350     108.255273  ...       NaN    NaN     0.633872\n",
       "19             deepCNN  0.731925     277.092485  ...       NaN    NaN     0.229779\n",
       "6            simpleCNN  0.676175      59.332155  ...       NaN    NaN     1.017661\n",
       "21             deepCNN  0.615675     107.094035  ...       NaN    NaN     0.545689\n",
       "12             deepCNN  0.605000     106.321745  ...       NaN    NaN     0.524634\n",
       "15             deepCNN  0.572825      81.613067  ...       NaN    NaN     0.669378\n",
       "5            simpleCNN  0.590625      66.808673  ...       NaN    NaN     0.809925\n",
       "9            simpleCNN  0.467150      70.789448  ...       NaN    NaN     0.604186\n",
       "16             deepCNN  0.394725      57.906200  ...       NaN    NaN     0.679375\n",
       "13             deepCNN  0.360150      53.930150  ...       NaN    NaN     0.666788\n",
       "3   logisticRegression  0.362500       4.221944  ...       NaN    NaN     8.406080\n",
       "10           simpleCNN  0.338875      17.212508  ...       NaN    NaN     1.880028\n",
       "1   logisticRegression  0.318650      30.498545  ...       NaN    NaN     1.031853\n",
       "0   logisticRegression  0.314150       8.412281  ...       NaN    NaN     3.657748\n",
       "33                 VGG  0.267400    1122.177484  ...       NaN  128.0     0.024123\n",
       "4   logisticRegression  0.258025       2.448996  ...       NaN    NaN    10.204182\n",
       "2   logisticRegression  0.218925       5.114744  ...       NaN    NaN     4.205489\n",
       "31                 VGG  0.162700     367.537869  ...       NaN   64.0     0.044567\n",
       "8            simpleCNN  0.151500      28.660981  ...       NaN    NaN     0.534525\n",
       "7            simpleCNN  0.129850      22.753640  ...       NaN    NaN     0.573535\n",
       "\n",
       "[34 rows x 14 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Display the model results after sorting by test accuracy.\n",
    "import pandas as pd\n",
    "data = pd.read_csv(os.path.join(local_dir, model_res_file))\n",
    "data['testAccPerT'] = 100 * data['testAcc'] / data['trainTime (s)']\n",
    "data = data.sort_values('testAcc', ascending = False)\n",
    "data.head(200)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "EvaluateModels_cifar10.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}