2.2 ConvNet - MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Convolution Nets for MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "#Import the required libraries\n",
    "import numpy as np\n",
    "# np.random.seed(1338)\n",
    "\n",
    "from keras.datasets import mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "from keras.models import Sequential"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "from keras.layers.core import Dense, Dropout, Activation, Flatten"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "from keras.layers.convolutional import Convolution2D\n",
    "from keras.layers.pooling import MaxPooling2D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "from keras.utils import np_utils\n",
    "from keras.optimizers import SGD"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "## Loading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "path_to_dataset = \"/home/ubuntu/deep-learning-keras-tensorflow/data/mnist.pkl.gz\"\n",
    "\n",
    "#Load the training and testing data\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data(path_to_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 28, 28)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "X_test_orig = X_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "## Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "img_rows, img_cols = 28, 28\n",
    "nb_classes = 10\n",
    "\n",
    "X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols,1)\n",
    "X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols,1)\n",
    "\n",
    "X_train = X_train.astype('float32')\n",
    "X_test = X_test.astype('float32')\n",
    "\n",
    "X_train /= 255\n",
    "X_test /= 255\n",
    "\n",
    "Y_train = np_utils.to_categorical(y_train,nb_classes)\n",
    "Y_test = np_utils.to_categorical(y_test,nb_classes)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5, 0, 4, ..., 5, 6, 8], dtype=uint8)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 1.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       ..., \n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  1.,  0.]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 1.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       ..., \n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  1.,  0.]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 28, 28, 1) (60000, 10) (10000, 28, 28, 1) (10000, 10)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape, Y_train.shape, X_test.shape, Y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# A simple CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "#Initializing the values for the convolution neural network\n",
    "nb_epoch = 2\n",
    "batch_size = 128\n",
    "# number of convolutional filters to use\n",
    "nb_filters = 32\n",
    "# size of pooling area for max pooling\n",
    "nb_pool = 2\n",
    "# convolution kernel size\n",
    "nb_conv = 3\n",
    "\n",
    "sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Step 1: Model Definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda2/envs/deep-learning/lib/python3.5/site-packages/ipykernel/__main__.py:5: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(32, (3, 3), padding=\"valid\", input_shape=(28, 28, 1...)`\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "\n",
    "model.add(Convolution2D(nb_filters, nb_conv, nb_conv,\n",
    "                        border_mode='valid',\n",
    "                        input_shape=(img_rows, img_cols,1)))\n",
    "model.add(Activation('relu'))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(nb_classes))\n",
    "model.add(Activation('softmax'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Step 2: Compile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='sgd',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Step 3: Fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda2/envs/deep-learning/lib/python3.5/site-packages/keras/models.py:939: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n",
      "  warnings.warn('The `nb_epoch` argument in `fit` '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/2\n",
      "60000/60000 [==============================] - 26s 438us/step - loss: 0.7051 - acc: 0.8289 - val_loss: 0.3601 - val_acc: 0.8971\n",
      "Epoch 2/2\n",
      "60000/60000 [==============================] - 24s 397us/step - loss: 0.3490 - acc: 0.9006 - val_loss: 0.3092 - val_acc: 0.9124\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7faaee3eafd0>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, Y_train, batch_size=batch_size, \n",
    "          nb_epoch=2,verbose=1,\n",
    "          validation_data=(X_test, Y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Step 4: Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test score: 0.309175731695\n",
      "Test accuracy: 0.9124\n"
     ]
    }
   ],
   "source": [
    "# Evaluating the model on the test data    \n",
    "score, accuracy = model.evaluate(X_test, Y_test, verbose=0)\n",
    "print('Test score:', score)\n",
    "print('Test accuracy:', accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Let's plot our model Predictions!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA50AAABVCAYAAADKf4AOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztnXmYFcW1wH8loJKMgoKAAmog8GQLixhJxA9ClKAPNFFQ\n2SIorhhB8DmjM8bGMMigEMEFRFQSURQXkBghihEFoqIiPjEkxgURn6hoJKAoi/X+OKdnYwZmuX37\nLuf3ffNVd3V131PT1VudzXnvMQzDMAzDMAzDMIwoOCBuAQzDMAzDMAzDMIzMxT46DcMwDMMwDMMw\njMiI5KOzfv36m51zvrp/9evX3xyFPFFj/c3sPmdbf8HGtPXX+pvOfc62/oKNaeuv9Tcb+5uufc62\n/oa4KHw6nXO+Jsd1zuG9dwkXKGKsv9XaN+36nG39BRvT1djP+psGZNs1nG39BRvT1djP+psGWH+r\ntW/a9Tnb+hti5rWGYRiGYRiGYRhGZNhHp2EYhmEYhmEYhhEZSfvofPvtt+natSvdunWja9euNGjQ\ngBkzZiTr55POpk2b6NOnD+3bt6dTp04Z3VeACy+8kKZNm/KjH/0oblGSxtKlSznuuONo27YtRUVF\ncYuTFL777ju6devGGWecEbcokZONYzqbzi/A9OnT6dSpU1bcowG2bt3KoEGDaNeuHR06dODll1+O\nW6RIybbzm43PJBvTmU22jemMf+/w3if8Tw5bOXv27PFHHnmk37hxY5l63S8SmaL8q6i/H3/8sX/9\n9de9995v27bNt23b1q9fvz5j+7tixQr/+uuv+06dOu21Ld37XFF/9+zZ41u3bu03bNjgd+7c6Tt3\n7rzX+U3X/lbWZ++9nzZtmh86dKgfMGBAhdszqb9VGdOZ1F/vs+v8rlu3znfq1Ml/8803fvfu3f6U\nU07x77zzTkb0ubLze/755/t7773Xe+/9rl27/NatWzOivxX1OZPPb0X9zdZn0v7GdCb1typjOpP6\nm8ljurLxnMnvHd77eMxrly1bRuvWrWnZsmUcP58UmjVrRpcuXQDIycmhXbt2fPTRRzFLFR09e/bk\nsMMOi1uMpLF69WratGnDMcccQ7169TjvvPN44okn4hYrUjZt2sRTTz3FqFGj4hYlKWTbmM6287t+\n/Xp69OjBQQcdRJ06dejVqxcLFy6MW6zI2LZtGytWrGDkyJEA1K1bl0MPPTRmqaIj285vNj6TbEzb\nmM40Mv29I5aPzocffpjBgwfH8dOxsGHDBtauXcuJJ54YtyhGgvjoo4/KTJq0aNEioycVAK666ipu\nvvlmnEvLoGnGfsi289uxY0deeOEF/v3vf/P111/z1FNP8eGHH8YtVmS89957NG7cmJEjR9KtWzcu\nvvhiduzYEbdYkZFt5zcbn0k2pm1MG+lF0j86d+3axeLFixk0aFCyfzoWtm/fzsCBA5k+fTo5OTlx\ni2MkCDWPKEMmv6z/+c9/pmnTpnTp0qW0eYiRIWTj+T3uuOPIzc3llFNO4fTTT6dLly7UrVs3brEi\nY/fu3axZs4bRo0ezZs0avve97zF58uS4xYqMbDu/2fZMAhvTNqaNdCPpH51Llizh+OOP54gjjkj2\nTyed3bt3M3DgQIYPH86ZZ54ZtzhGAmnRogUbN24sXt+0aRNHHXVUjBJFy6pVq1i8eDGtWrVi8ODB\nPPfcc/z617+OWywjQWTr+R05ciSvvfYay5cv57DDDqNNmzZxixQZLVq0oGXLlnTv3h2AgQMHsmbN\nmpilipZsO7/Z9EwCG9M2po10I+kfnfPnz88a09oLLriA9u3bM2bMmLhFSQrZoiEBOOGEE3jnnXf4\n4IMP2LlzJw899FBGR/ycNGkSGzdu5L333uOhhx6iT58+/PGPf4xbrMjJljGdref3s88+A2Djxo0s\nXLgwo59NTZs2pWXLlrz99tsAPPvss7Rv3z5mqaIlm85vtj2TwMa0jenMJJPfO5Kql9+xYwfLli1j\n9uzZlbZxLkjZ/7T3QZX1+qtWreKBBx6gU6dOdO3aFecckyZNol+/fmXaZUp/hwwZwvLly/n88885\n+uijmTBhQrFzf3lStc/V6W+dOnW4/fbb6du3L9999x0XXngh7dq1q7BtqvYXqtfnqpIp/a3qmM6U\n/laVTOrv2WefzRdffEG9evW48847adCgQYXtUrXP1e3vjBkzGDp0KLt27aJVq1bcd999FbZL1f5C\n9fqc7ucXqt7fbH0mVWVMZ1J/qzKmM6W/2TimM/29I6kfnfXr1y+epcl0TjrpJPbs2RO3GEnjwQcf\njFuEpNOvXz/++c9/xi1G0unVqxe9evWKW4zIycYxDdlzfgFeeOGFuEVIKp07d+aVV16JW4ykkW3n\nNxufSTamM5tsG9OZ/t4RS/RawzAMwzAMwzAMIzuwj07DMAzDMAzDMAwjMjI31nKac47/AQAtkRxM\nt3x4PQDB0SVtTvHdADjtq6UAbM+5I4kSGoZhGIZhpCvHAuAvEZ+5orukNvd1cF1viEkmw8hcTNNp\nGIZhGIZhGIZhRIZpOlMM/8kEAIrKxX4q0rJ+qbpVTvJRbeveBABHGMwqiEy+lKBZAMDEzfJPesYv\nAeB591JcEiUI0Vyv8BOKz22uZttx07Pk3BqGkWEMgJ7HV7xp5csABP5xKd/SJ92lwMogcsmMSrg6\nkPKWaVIeN45h6+8G4P7pFwMwfEzZLATz/uci3SdIgoC15VgAVvjbAJim71v1dOvVXX4H7E66VIYR\nFQV+J3XcTQAEs6TOXZp8bX4kH50HH3zwJ865ptXd76CDGvHtt1FIFC3W36qTjn3Otv6CjemqYv1N\nD7LtGs62/oKN6api/U0PrL9VJx37nG39DXFRJyBN5Vwy1aUquXZq01//yQSKKhmCuX2lfOQv/Rm0\n9kkAirqWbfOBnwrATPefmopQVp6I+1tj3pHZmak/FOvw8S+pCD2CWh22qrmUIutz4wCA6VtKxNih\n5RP+WQBedIkNl57Uc9wvkN98S37SfViDWbZZgWhBgJpofVN2TD8ZAHBDfxHv6eLzvRao+fWcnP6O\nkt+6vSUA00eLJmSsUwepGmvnT5HiyZ5S9p+p9Z9UukfKnt+ISMn+PhoA8NrZ7QHo1mM9RS9X3DRX\nM/M8/LyUG0pty6Pi+0NK9jlCktPf3vJbK38GwGN6yYUZET8CVu/nCMdp+SOg7VoVp0tQbUmS0d/B\nXoJjdHcXAjBugR73KT3s3KA2h68WNp73Jrr+DpBislhe3JM7BICNbj5QYkmY+wnc22QwABe632tt\n+PypPqlwjv2sCQSXlq2bEP5kECT2t/bRX/PpNAzDMAzDMAzDMCLDfDpTgV8GAMxsOqG4KvdXUrqF\nMhOR9/QSrXgZOAMAP0I0CkVzZdNP+Zsch46Rihs3t7UW35HiXtZSwxk/5wPgfyOTQzMyNGjexUum\nA/B5Le46z17yU3426EUADmiUCf+ocwDYOVTOfaHW9nU/B+BFHqI2ms7oGcDbiIZz5hVSM2aQ+HqN\n5cgaHlM0nL7jyQAs7y+1P2uss7JbghoeNyp6A7Dc5wLQa/pq3Ng7dVvlWtm0JycAYME20R4c5MQC\n552B8hx7Rps9U36/UhQ9H5FsRrV40Y8FoLCcfuIfWub3hVPP0JXu5Xb+RorC3iX7PNdFDvSzHnrN\nvhQkTtgEMIDFAHys68P1nsU5QSzyGFHSlAv9HgBmNpTBe1uebPlSy1DDGfr0TmsKINrPd/1zALR2\nl0QvaoSU13ICFNxwHQATgwOTJodpOg3DMAzDMAzDMIzISC1N59oAAP+ezJJ98Ksj+JijAPjJVWul\nzVxt+2WQVNEiRVyi2LWotIZzq24M9mp+md8GwNxys5LDCx+tdJ+MoHsAwNFOOl5X85RSJU/M1KOP\nl1m3Z+e0AmDGRZW3/dunovnqHUbqvbafbJgcRCZf4jgcgLumSCjez2txpPN4iE9HHaNrev5ZU4sj\nxsxK8Xsr7Fm2Om+7lBNy3kmyQFVlKAD+vrYUSYo7unnphFMtbU2ZqxElwwjes/06WXBBrY6baOpt\nGQfAzq4NgBJ5JYa2OiuyINliJQ9RjvGeajirQxijgPsqb5PXvPoiJY2GAfxSFmffNxyAi66eJxUH\nSdHoxk0AfNGseQpq50vRMaCRm1CmKl8C4uMuF01lQeDh6aW6tbyDruiJCry8sxzmbmK5bvF/VP/9\ntq9qzZ8SJ3ctaM/fAdij6/NOCx++QRzixMcpAQC+wFHUW6rCOBKl/RsB3H+p1vrLxbBI1d6/fFhb\nrY9UzNrQ0/+Q9k5uOLdV0iZXAjMzrYJ3sEVuczSCpQATXfI0nCGm6TQMwzAMwzAMwzAiI6U0nX6Q\n+jX9K6z5TP9goqqzTtItvX9bwx9Rs2w3WmdtFgU1PFAC6R8AMAbPmIX/q5XTKm1+5znjgZLcnVnD\nHCne7CJlwfWv6YYgDmlqTV8nvlAzqtB2hkY1PovTAFiuDq2uvy+OfJqy3H4lANOuEE1nHS++yDXR\nUP+UVUxbGK510jIdNZ3HAuDvl39CUG5r/UUpnpd1VhuAYi0nQF/3R136Q82P2zFgk1oy5L8tVXnu\n/2p+vEgQX+K3GsucbeiHW/phumN7BwDq57yhNY8nR7SEIxrtIi93qdw/3AYjAtmkCgCNMUw3MWhg\nzRe6riYN4w6fxK13XCsrV8jDPe9p1Yw0T5NrV61sVr8i95zGbgIPzpVNn2pZWG6XcZNaAGLncZkG\nzXxpcWcAfuJGa6uPopC2enSHBWpMEI7hn34i0bOrZl0gurFQa3Krv5imTnwkC9tqk0c1T+vAVNB0\nXsNS1ezm6kkbnh/EJ05S0ReJWy8DYOcNcq+d1rvEnzGkrH8jbPKNAGg+9wuKVMsffCkX+TcNU1DT\n2TEAYL5zldqbvOznAuDc1wDc5uWdcqe7J2LhspeU+uh0l8iL1pnjxYH3iT8M5szzZfleLgDgsEbi\ntV54o+yTfxQUVvJOos9Azv++lL//CtD97vcDARjuUinoTrDfFoP90cwr97Keqx/geQWPJV6kFMLf\npx3/gRQFExfHJ0wt8H+RB17wi/23PUfLH8g7PkU6IVOkLwmT17lKUwukBN0D3rpCzlv7E6XKOX1i\n7WWmtX8W3TFkH9MxacTAEQAEd40sUx3esxgWJFOaaiC+AK9dImbBz1wKU/0HssndW/PD6gvClvUO\nzVtN/SahIXZVpmWSxyIvpu2P72PSZHqOlK8iHxk3+fsBeMyFxmsp8LGxT8R03Y+Qr4bQdJilpQzU\n5gQAdEcnSL4IdIOWjcJp0R2k7ORJZWhwutdelHH+tX6kPF3qnF+nF+u3G6W88/vyIl/XSWqFK6+R\n+sIp8PVf9bDNZRKis5eP1zdcCoyDFiWLg3wzAPJqkZprrDsSr/lTCjUS0dtnS7CxtlxY4+MmihX+\nZFYlwiVnVCDl2FJ14QzM5iABPxABw2SMTh0r/4Dwar7qcDhggV7H/yi7S6vRbwHwP58cC8CbI3MI\nLd+/mXM4KYc+S/wc6WNRj5IP6HE6Ll1d7asrfY+C3zidHDlvDpsfkv0f0Bb+V3IPcAtT+H1rH+zx\n11LH3VSmrsDvBJJrZmvmtYZhGIZhGIZhGEZkpJSmk6sDAJ64OqwIeGKELDUiV+vKJnctyFsOt/Su\n+HgbpLj89vcBeItWhJZ5t/EbXXqulkInCTUDPt254jnyUXWkdN+luClerQkAKJwuM035oY31+2li\nmlWaggDGSD9CR/36FTRrp0FZWv15hVSs1N1vkhDXR5SasermZXp1jVuWaGlrzXd/cTwgVjl0aKnj\n9OWgBkc6C4DgigkV/r/SjdseEY3hlnIz7leqVduY2sXiiYzPvajcZzixOAkugTy3ttbH7fymhN+Z\n5WCwalsKXGppOIWABq7sScsfrAuDpCg6q2RbeEXeVVeCzTzGFq2pLKRF3IgOw79yJgBFJ0it85oP\nx1VkTRPsZz29eM0v4DDVbD5Y7vrMf1DKRwb35wB3rqyUC/blVVXvJsv9zv/W8bBaWJ2rAXoGu58A\n8Abv6V61MEmvJb6hKzYNPpPQeujPtTrmyvVhkD95Rrf5xaZaHS+R9Gy+hlW63OQ6tdLI37+Vxot+\nEQA9moq2eskcGSPr55S0CcOHzfe/A2Cq211reROCav8+VDO50Nx0nFr9u9m+OKhQed67QlT2frSY\nDE4DDvSqsU6x4G4AOS+JS16RWpvsoMSCaOB6sTipXG6NmvRQwONeUtnVdXJtBvrxsNbLTaCLG5I4\noZNAeS1nXJim0zAMwzAMwzAMw4iM1NJ0Vgl1RM8r5ZB+9fJ97/JqIHt2h3ydjS5whyVcsigpOFM0\nXKU9QFbu1rjzKTjblEgO3629DkdrmKR6VUWtU5UAgC03OWbsqbjFlepefPebw+hb7PcYlGkzcfIk\nALYeJLNWc7+Fie5kAO70Mn/5pAu1KTEmptf0R8saTWCYGicMfzSo8eEe9OK4vNFBrqZZyBu5dB97\npDZX3CiBCgJdD31k3O9S22phj16ExdrmkcBdNfFLawfAJC8azv9Wzc8SUjwJ9zJYpX5b+bdK6caq\nj8/8HwLQwp/Ih13FF7JQlcC36TXvxzeWfaamWoAhcbie6uW5GvpwDtWtLUOtHqlnTVF75Lxd498F\n4Bm3nlA/db6Wd3kJhORcD6kYsgaoJJ3RX7RUTYs7yhcHQix8P3FS15qDAynnTih2RSy4TFXbtdR0\nnvyGBGSZmFL5zMSP9uutkKsa57xK/dAPh/MkAN5Xf1L/R+1K+NQJx8ZpbwIanyHQ+CI3NbwegKmk\nxv38u+dF+ClqdZQrt10JRAj7Tukz63sATLu0pOo314fq3X3sFxOfISf3zlJ1F+lt9mJX9RRkl7tj\nAfChBcsjUnYe+a8K2xtVwzSdhmEYhmEYhmEYRmSkoaazOojf1IfdZZZnDuBydWbnkSAekaqJf138\nBh7WWbYNwHn+CACOdWHYtOpHAU0npteRVBvqfYHrkBqzh9VCFBw8sGXvTVeGbgbDNUeEewB4s5ID\nBQCc8c0SAM5ypxEm2TnVSazbJ3P0/7M9qI3EtcIfqaHYgYLFy6XS1cR/OgCgj/rRPQ64T9Lw/Jfm\n0oDghrLJ2LuFC8uDZEtTK4p6gJ91uayor7Vb5SttP/USaTvuRul/qFFbottzP4G8plFImiBalDw0\nXZfy41Bm0Te5d4q1XXXLR6huEy6kQNTSUhz8pWg6d7nTAcjVCOHu/e+0xYQK9soQ5g4DoLfeY9YA\n1zSQTQeeFka5DLRxRXEEDtVyHADnPi7/PN9cjvfMpbBaW+zScrQXNWiBi89a45odck4LHeRrBF6O\nDmKTJ2r6+IMA8Rv/8fbQ57R8K/FfLPATOUzHQxhN+xp1DJzxuaT8auV0S6eSa8O/JMvTVCHOCC3n\n1lr8GrPO388U7Wf4nHEjqqDh1PQqXv9JYZzX3Lth/EX72i8u5P3n4B5736vq96l5JPRzF8wFoJsb\nUUO5jNKYptMwDMMwDMMwDMOIjIzWdAZeIgT+QWd5BgATRsUnT/WQWdP3u8qszQatHQq0PORTXQuS\nLFMMjAr4kc44DtNzV9B/XnzyJJDc8VK64dXX3D3vJFfg8mugaEpi5aod6h3UtWS2cXWNNJzCrV78\n+x7Qa3jcALg8L6jx8VKBvjOfKJk+V573kj8tpVygKqDJgG0A7GygmuytUKS+PqEf3OR9dGKHti0q\nV5/bRUrX2pPK9zW/yxXLvq5XawA6Mnyvdu9XkoP39ks06uOlqWWd8kyDU4FSbvJPadkugzWcIep7\neUipqnqq1eo1XzSRv54vVigXvDy/pJFqg9ExPWOh3NCH6ltVYQW++3qV0+TIbboU1FTqWlP0c/nt\nlwE3Lc2tR6pAmOv9MaCIPK19q0ybFV7iI6xyJRrtcRrq1d1YXuu997Ux/8QzdemJ2gucIDoMf6/Y\nzfjEg3ThH8E+9hAN551e7lHT9HYe5rp0H6XoPXqW5NQt7XsKkHs75DVMoDP18+HCOC0zInN40jBN\np2EYhmEYhmEYhhEZmanpvCIA4CjVkIV6we55HiYHsYhUXfxvxamk6May9Sv8mRkfrbY0Y+++icc1\nUFrr23WOZE7VI5ClGjtKLbupGvWyNrOG35Y9JsDWLw8EoEHd62p+3BpzLADTNIrfqN31GF+Lu8yY\nT2fL8XR91/1Aw5ofLxV4ZM+g4v6cpmUPd4cupbhm6ckAgAN5VdYnHs/sfNH0ferEAmGENl3sh+21\n+8VdxYF58tqy2tDNr6sTXYrf21w9X6zJ7TBEcyweG0j5pBQ7jnXMVu1ZeH7DWOt9nEQt5jjVmuxT\n45A8mruyvooPS3BhrvXi03nTqfogWhYkUaokMfBhAHqqG/0XnaBIFSO9nZzBj7VpmM/yEGAbFTOr\nnIazPjBOfSbd0VtlYXNQO5kTyInXAZOCuMWInGMWflb5xmYBAIe6kvvvT31nAJwbrTVBlX9rnISh\nYPz0u6shYdwEANzjJf/kl25+ma1XqfZ/fJBaVhohl12iT9Vymk63ObGa2Wmh0vR29eW+ImGHTjoT\n3YFJ/82M/OicepsEq/j0dlkPEzoXDJkZk0TVYGkAwNx+ZV8+wzQRzl1CpgcOKs3vP7yu+EGf82r4\nNA9ikqbm3P/ZQAC+SJD5ZC8vZl8zXEn6irBs0HCnLgWJ+bFqIQZ648KsF4N3AWfpSnXSQ4gtdVHT\nstdBrwbPAi/URsD4GBsAMLNuSZ9ObKkLH6b4x+Ze6GdUwZ+4uKC11skkSkHYpKKxfrAUoSlu/ght\n6l7Xmj8kUsjEc9z/FqeXKNJ3sjAtRF1NeTQdyA2Trl8gH5e+gwbjUPvV1eslfcOP3dlRS1wlwv96\neA/ZoGUDJ0nupyNlO9+T//5c0mmMbCQPpdlv6Rv2wHIHfVTLjp7UnkxZD4DrFE4CnsadXt4Vcqfo\nf0azky3t0guAJnxKt1zZr3A/7g0/9t1wLkzgkAqmeBpO5ot4pUg6mr5oF1Cn+A4kPP2xmNUu0XvW\nRf5gGhWnLat60K/2/F0WDg9r4g8Y9tWcA2CeTB49/K3U+WF7X4+7/qRBkMqZ04b83+faKbeEVORY\n5GtwV/kNE4OEHL/85L5RM8y81jAMwzAMwzAMw4iMDNN0ik3QuItkxmau1roX0sVJ/hz8X8uGpw6Z\nP0Id1Edmi5bzNwB8cvQE8mVymYKeQXzi1JJhPR4DahKwO0RTURdI9IrH6so4eaBUixEaJGDM9hr/\nSAKQ2X/UGm9aU/DXqZnSkMpTaQQdcgG4as/vATh0qs665pZtt4c6CZQ1yfSXYsetJVVvbWwlCyke\nQChRbNoh2cnnaX9ddx0Tc4N4BKo2j9P9VpF5xVjpxLJyLTr7Pjg3UtcCAKavlDQLu52Yi5/Qap1s\nznkz1tRGIT/wkm5gs1tQ4fZwln+NW8nvaFBm22TG7r0DgGp+RwBNNGPBAY1uqLhtSrGkODH85aq9\nJ3fvVp/6qbIwpewNVzMHMcLLvbDAPUVJUqD4aeHlPalQr8H80xP/G592lrBMs8OKBpU2TR766KgH\n7Cn36nvq+yuBkkRlh8/7ppoHl1QrS9U8t3OYgiYFhnvOwQ/iV54HQFFPqZu2j1iMeY+qdcbJGixO\nU1i1GBmmHQkikLL2XLNYTBujsiUIrUDKa4CN6mGaTsMwDMMwDMMwDCMyMkrTeY3/B1BqBk/jRI/8\nRRCPQNXkVj+9OFl6SK5EaldfTsgWf84LvThMzXEwbPkRUpkl2qCKWOTFO/4D9zRQVsOZe5yUrr9q\njW4JkidYJbiTdLb0I8djzaVu6qTKT+BQLXfpbPSMCtINAKx2mxMkYfLZ2luc9qdREmCm46B3dSmI\nQaIksi4AYK5qAkJtUBh8J61Q39yTVeP57pgjAXiN7gAMcJdT/j491vUBwL8pup8icenkY9+QI10l\nmsIkMsKdKAsaVMdv0GtV497M+JeUNfFrmgvUEwU3Y72YY9zqvq2RnKnCMN+cR13FJiU/3yz137mb\nkylS6tA/YKMr6zPoztVn0yNB8uWpAq6VpE7ZUqcDAPOGw2VeAsXM1PRkULEVAIC/5vsALFb/Xvfb\nVLKuW4/rqSrXIADgnBvK+s4vePh8OE+2LT9b7gXhu2iuWpqNn5vAtCNpzI/DhXR8dqUApuk0DMMw\nDMMwDMMwIiMzNJ2PBgA01xQpGpsN90gqzTbtnwZuNuU9CVxbnWpOiah3yWMUkiflWWA1J8YrTIz4\nv+iMsaYwqNAndJaWvYPoBaoq7wQAuOY3wCmyzC2VNx/fRduohvNjL46PD7qt5VquT5CAyURmmctE\nrR2gC3rvynTe7SDawDAI/wnf6HznwUEs8iQE1Xi2HntJuQ0VWaNohNRDJLL0ZETr3azpVmoW3TnR\n/EcKHY+uvDNaaD3RGPzfNe7AfqK2liaMKPn7dZLG6dZUcHarAa38IAB+7Tqyuty2fL2mC5o9klyh\nUoX+AQDbH63DrRqlOl+DUxQMrHk0g9oj/pbM21cEZdFiNt4jzxs/ugGfuvGyfKuUbuzT2lZDUC8K\nZPvfHHfrtfBPL1GeUzb1k2o6FwR7bSheesnJyA59Fz9brvmfXIpHFo+Ih88aAZTEWZnkJT5Hyp7j\nSghmQVAunUyBl+dRMlOnmKbTMAzDMAzDMAzDiIwM0HSegy+Qmdcwn2P30NUiJ4hDoMTSURPQrhu6\nj0Zh/MRPtNRwY8UZ5YDGbQBY8dnxFR5hB9+jb7Fv0ZsVtkkmxzd8AxBN5zlFYWr1IC5xao9q8Mr4\nRM0LyjTZOVqjxZVS7gW/kLI+leN6p7jWIEwo36Xqu8zgSgAaa37AYroH8GqQCKmSxz/kvO44rqRq\nymLNKJ0lfsoN6oovrroykvPL9M25WyuOFYuVMMJ0UQ5c4xcCMCWVx8LVQfHiKH8bAG2mSITxMCVh\nFy/han88QJ4fWxfJ7PnMuntlzks/1N/tHrWmWlVqU+jj5Y5JD8uqTUXyLnDSftpVHXlH2akhqacc\nDNeFKR3fTIX/iaohVVW16yz40yqJ1uyKk8mG7zxyfbo7bijWZI4fVvZoa/1ooCRS7WJKNJxTXdn8\nn+lFoKXHk9YEAAAJS0lEQVT0KxzXTU7bVm57auJ+L2NtM2Uj+7/r76K1K2+NUnX+85SUI3Q9z31d\n42MZaf3RKTc6P7oDhXdITb6+1LpmqXCjSwyT1+3/TeQajWS98HAJT3L6VxKefXpOqUZbpFi1j8Nd\n4cU+6PY4X3705WbmLamcTLz6DHhFzHdOdecU100eJv/o8INyppYVfWBW9tHZzvfMyA+XMHn3Xq+r\n6fbBCXT7r5Vl1ocCbd2VuvbAXu0zjc6+H7P0BW1UWLk0iEucmJFpp/oa9OtRHO+EZlrhc2tzkHyx\nqsE9p8mEyWRNa/WF1v/VSRoYf79OnlXwdvHXjj+JXL4omDT/KgBWPVRSF06g9A30vKnpYsqTFwDQ\nu4+uvwolodyqcD/qIfuvflH+AyeMlfNe2LCkyQGX6/9kYlBzOROMO+s7APygA4pTh/jCgbLt5yrv\nnJL2k86Uc37t/4irxzSd210aBtjR/5+70aedqWVFvO1bAvC49u9Hu9XAtm4QhzjVZ3kAwA+3fwpA\nQU4TANa5zdBPtlXnubPO3w/ATP1/hO/ZNHqnloImGb0vBZemxju1mdcahmEYhmEYhmEYkZG+ms5m\n4wAovGN8cZWbqLNV6tCeboxYD0Xtqr/flEbhkmo4S20LTZ/Ka8qGqSXukCb3FNfNv+gCXQqqL0SC\nePDmXwLwngadyR8FBXnpH0TpSScBKB44CObWIFvACMk0wKEaNMj9Rse6e4xUMIdONGHy7kxIxPza\n2pOBkquqzShgzrLKmmcca4f8pDgIw398C1ko1s6HRn6nahkkS6x40Vn5+f5+urrhgAQkAXCtQnVa\nigbNWippVcZ9KavTGpbdXDS87PqhwGV6i3IuT2vXRCZeYpHx2dj1BeDTUluOVHPidNdyFT4N/pK2\nsjJ4/+3/1Vs0Jgv0Gg7D6wzR8lt/DAXusYTKmBhEbvfIVvxJDQCYli9bpuaXNRfaBdRTrWf5t48z\n/TFyHKd2l38NIpA1yXQM+KRcmpum28P0ZHEGgao+2xtLir3LVFM7s+4u+i+R4F5PVtUqbHKAC61z\nNI3bASelpwWlbyadDuIVoxjTdBqGYRiGYRiGYRiRkYaazgAAX68keNDranuNK6x4lzTBtbuBYb45\nAA35d4Vtbv4qt6yvZin6+1YAzKLEaTqvjYYL1xQWIcPDWENsLFVbtk1yEf+QwTqLGp7JS+6+FeZU\n/L9ILwIAGlzlWXCT+M6+56qeXbjfN88C8KJ7oczxMpWj+D8Adur6D+MTpRboRZZfrrollAT9yi6+\nVpuLelvEUmVnvmgcNs+S8sjiYGbZwWPuJB7tLsuFrbSyx7lSvhTEIVIVEA3sgQ1FE+KXNwPg85/L\n1jkaIypXA2ddv/5aXHFI/nTRcEqarp0NRMM5pVzmpvzB4FyurqWZj5fibhTNjW/lKLxLK++qvH15\nQuupX2lZqAGm7nFbSG3rm2m4VS8B0NGL3uVqzefV2knshQ3+bPZQp8xeI34gGv7xxZrtBdGLmiRy\nXvqMVfpeGVoXfbPh8ErbpzTfBAA0qCtvD76N4ySNpXGKvxiAse7Isvs0k33qrZPUUf9s7Fiomw7Z\nrZGk0s2iYR++nBNCP/QY+mSaTsMwDMMwDMMwDCMy0k7TOdVfDkBhKdvsx34VxrQOki5PopnnPtrn\n9tv3kVQ7r/h/Ujqkc1BbkZKEesv0kiJfAsbhXAuoROublkwOOGeypq2ZK6lg/Ily4maoP++Vb4N7\nQWeiwiu0+NyGms7MZpATX+MwYNxVfpEsuNfjEahGSKTSD/4sPia4zwDY/NsG7OMyzmiedP8C4HeI\nZvMNL+kburi3tUUQg1Rx8gfcq3KtT9SLfPty0bDkHHx9bFJVDYm3HaZsytkuqSSe/H5/qQ/9+tyc\nvXdNdZZKJPjCfrJa3rfcdfYwP0iqSAmnZwCA4yv8r74PQOHCfbRXTvPyoLqAewEY75bKBrcl4SJG\nh8S/CJMDjCAMpqE35gp9/4KIZYqPn39/WfEYHyfhJxjfJYhLnAQRAOD+tRX/ijxvcLMB8PrJ0On+\n1QAs0VRICxpL/UJgnL5quEPuL3O8dGWPv5aJocVJjJG2TdNpGIZhGIZhGIZhREb6aDo1f2MnnZFY\nHaMoRhSIj5u7UWcabwzrU9k3pJaMCABw5dReY9pCus+q1Zammtqy2cr4fA9qj/iHHOs2APCubw3A\nWTxAtmisAVxzj79b7tvXj7oWgIluDAB5brG2CmKQLFUIAMhXF7HFB0s+QY6Tev4RJFugGrE9RxJm\n9+Y0rUlDDafi7ymJGVGaJj5N/bv2yRTcwqqbXhQUawGXRiKNkXwWDR9SHKX33AVzZcG9H5c4CWYa\n7oQVsnieJGj1l8ogHul+DMASva4P0T0uHnw/47uGUXuDJMmZYFSbOWGf2vvkY5pOwzAMwzAMwzAM\nIzLSRtN58c2SfXL1LWXr87tAwdoYBDIMIzJcz3DmPYhTjAQxBYDWLowqnT1aTgBuCUq0+ReFlTPj\nkiZlcec8B4D/wc+kQv3a6RePPNmMl2Dh7NL1UVq2dHfoUvrnjjaMkK/mHADzvotbjAjRvNgPSeke\nKqfZ30sLGEQtUNaSNh+d5ckPrVzmvYENEMMwDCO9WQ6Ae19fiPoFsUmS7Zzxudg6H6+pFgb7p2WD\ns49NI/Notnsz265pAoC7/nytDWKTx8hczLzWMAzDMAzDMAzDiIy00XTOdmHaDJkFLpgXbnk8DnEM\nwzAMw8hAnnRvSVkchGNVjNIYRrRsz7mjVEDDIE5RjAzHNJ2GYRiGYRiGYRhGZDjvfdwyGIZhGIZh\nGIZhGBmKaToNwzAMwzAMwzCMyLCPTsMwDMMwDMMwDCMy7KPTMAzDMAzDMAzDiAz76DQMwzAMwzAM\nwzAiwz46DcMwDMMwDMMwjMiwj07DMAzDMAzDMAwjMuyj0zAMwzAMwzAMw4gM++g0DMMwDMMwDMMw\nIsM+Og3DMAzDMAzDMIzIsI9OwzAMwzAMwzAMIzLso9MwDMMwDMMwDMOIDPvoNAzDMAzDMAzDMCLD\nPjoNwzAMwzAMwzCMyLCPTsMwDMMwDMMwDCMy7KPTMAzDMAzDMAzDiAz76DQMwzAMwzAMwzAiwz46\nDcMwDMMwDMMwjMiwj07DMAzDMAzDMAwjMuyj0zAMwzAMwzAMw4gM++g0DMMwDMMwDMMwIuP/AdcL\nx92kDiEbAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7faad80bf128>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "slice = 15\n",
    "predicted = model.predict(X_test[:slice]).argmax(-1)\n",
    "\n",
    "plt.figure(figsize=(16,8))\n",
    "for i in range(slice):\n",
    "    plt.subplot(1, slice, i+1)\n",
    "    plt.imshow(X_test_orig[i], interpolation='nearest')\n",
    "    plt.text(0, 0, predicted[i], color='black', \n",
    "             bbox=dict(facecolor='white', alpha=1))\n",
    "    plt.axis('off')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Adding more Dense Layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda2/envs/deep-learning/lib/python3.5/site-packages/ipykernel/__main__.py:4: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(32, (3, 3), padding=\"valid\", input_shape=(28, 28, 1...)`\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Convolution2D(nb_filters, nb_conv, nb_conv,\n",
    "                        border_mode='valid',\n",
    "                        input_shape=(img_rows, img_cols,1)))\n",
    "model.add(Activation('relu'))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(128))\n",
    "model.add(Activation('relu'))\n",
    "\n",
    "model.add(Dense(nb_classes))\n",
    "model.add(Activation('softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda2/envs/deep-learning/lib/python3.5/site-packages/keras/models.py:939: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n",
      "  warnings.warn('The `nb_epoch` argument in `fit` '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/2\n",
      "60000/60000 [==============================] - 55s 915us/step - loss: 0.6324 - acc: 0.8480 - val_loss: 0.3163 - val_acc: 0.9109\n",
      "Epoch 2/2\n",
      "60000/60000 [==============================] - 56s 935us/step - loss: 0.2940 - acc: 0.9156 - val_loss: 0.2535 - val_acc: 0.9288\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7faad80bca58>"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='sgd',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model.fit(X_train, Y_train, batch_size=batch_size, \n",
    "          nb_epoch=nb_epoch,verbose=1,\n",
    "          validation_data=(X_test, Y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test score: 0.253540886793\n",
      "Test accuracy: 0.9288\n"
     ]
    }
   ],
   "source": [
    "#Evaluating the model on the test data    \n",
    "score, accuracy = model.evaluate(X_test, Y_test, verbose=0)\n",
    "print('Test score:', score)\n",
    "print('Test accuracy:', accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Adding Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda2/envs/deep-learning/lib/python3.5/site-packages/ipykernel/__main__.py:8: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(32, (3, 3), padding=\"valid\", input_shape=(28, 28, 1...)`\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "# tf.python.control_flow_ops = tf\n",
    "\n",
    "model = Sequential()\n",
    "\n",
    "model.add(Convolution2D(nb_filters, nb_conv, nb_conv,\n",
    "                        border_mode='valid',\n",
    "                        input_shape=(img_rows, img_cols,1)))\n",
    "model.add(Activation('relu'))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(128))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(nb_classes))\n",
    "model.add(Activation('softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda2/envs/deep-learning/lib/python3.5/site-packages/keras/models.py:939: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n",
      "  warnings.warn('The `nb_epoch` argument in `fit` '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/2\n",
      "60000/60000 [==============================] - 75s 1ms/step - loss: 0.8048 - acc: 0.7740 - val_loss: 0.3411 - val_acc: 0.9048\n",
      "Epoch 2/2\n",
      "60000/60000 [==============================] - 90s 2ms/step - loss: 0.3783 - acc: 0.8898 - val_loss: 0.2402 - val_acc: 0.9327\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7faaec194ba8>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='sgd',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model.fit(X_train, Y_train, batch_size=batch_size, \n",
    "          nb_epoch=nb_epoch,verbose=1,\n",
    "          validation_data=(X_test, Y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test score: 0.240223283434\n",
      "Test accuracy: 0.9327\n"
     ]
    }
   ],
   "source": [
    "#Evaluating the model on the test data    \n",
    "score, accuracy = model.evaluate(X_test, Y_test, verbose=0)\n",
    "print('Test score:', score)\n",
    "print('Test accuracy:', accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Adding more Convolution Layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda2/envs/deep-learning/lib/python3.5/site-packages/ipykernel/__main__.py:4: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(32, (3, 3), padding=\"valid\", input_shape=(28, 28, 1...)`\n",
      "/home/ubuntu/anaconda2/envs/deep-learning/lib/python3.5/site-packages/ipykernel/__main__.py:6: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(32, (3, 3))`\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Convolution2D(nb_filters, nb_conv, nb_conv,\n",
    "                        border_mode='valid',\n",
    "                        input_shape=(img_rows, img_cols,1)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Convolution2D(nb_filters, nb_conv, nb_conv))\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))\n",
    "model.add(Dropout(0.25))\n",
    "    \n",
    "model.add(Flatten())\n",
    "model.add(Dense(128))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(nb_classes))\n",
    "#10 numbers randing between whatever to whateve\n",
    "model.add(Activation('softmax'))\n",
    "#10 numbers randing from 0 to 1 that sum up to 1 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda2/envs/deep-learning/lib/python3.5/site-packages/keras/models.py:939: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n",
      "  warnings.warn('The `nb_epoch` argument in `fit` '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/2\n",
      "60000/60000 [==============================] - 196s 3ms/step - loss: 1.0403 - acc: 0.6683 - val_loss: 0.3277 - val_acc: 0.9064\n",
      "Epoch 2/2\n",
      "60000/60000 [==============================] - 202s 3ms/step - loss: 0.4797 - acc: 0.8521 - val_loss: 0.2472 - val_acc: 0.9299\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7faab36233c8>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='sgd',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model.fit(X_train, Y_train, batch_size=batch_size, \n",
    "          nb_epoch=nb_epoch,verbose=1,\n",
    "          validation_data=(X_test, Y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test score: 0.24722215724\n",
      "Test accuracy: 0.9299\n"
     ]
    }
   ],
   "source": [
    "#Evaluating the model on the test data    \n",
    "score, accuracy = model.evaluate(X_test, Y_test, verbose=0)\n",
    "print('Test score:', score)\n",
    "print('Test accuracy:', accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercise\n",
    "\n",
    "Define and train a convnet with the following architecture:\n",
    "* ConvLayer with 16 filters of 3 by 3\n",
    "* relu activation\n",
    "* Pooling layer of 2x2\n",
    "* ConvLayer with 32 filters of 3 by 3\n",
    "* relu activation\n",
    "* Pooling layer of 2x2\n",
    "* drop out of 0.5\n",
    "* fully connected layer with 128 neurons\n",
    "* relu activation\n",
    "* drop out of 0.5\n",
    "* fully connected layer with 128 neurons\n",
    "* relu activation\n",
    "* fully connected layer with neuron for every class\n",
    "* softmax activation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda2/envs/deep-learning/lib/python3.5/site-packages/ipykernel/__main__.py:7: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(16, (3, 3), padding=\"valid\", input_shape=(28, 28, 1...)`\n",
      "/home/ubuntu/anaconda2/envs/deep-learning/lib/python3.5/site-packages/ipykernel/__main__.py:10: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(32, (3, 3))`\n",
      "/home/ubuntu/anaconda2/envs/deep-learning/lib/python3.5/site-packages/keras/models.py:939: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n",
      "  warnings.warn('The `nb_epoch` argument in `fit` '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/5\n",
      "60000/60000 [==============================] - 24s 396us/step - loss: 2.2462 - acc: 0.1914 - val_loss: 1.9571 - val_acc: 0.5226\n",
      "Epoch 2/5\n",
      "60000/60000 [==============================] - 21s 346us/step - loss: 1.5416 - acc: 0.4801 - val_loss: 0.6093 - val_acc: 0.8559\n",
      "Epoch 3/5\n",
      "60000/60000 [==============================] - 25s 417us/step - loss: 0.8284 - acc: 0.7268 - val_loss: 0.3286 - val_acc: 0.9156\n",
      "Epoch 4/5\n",
      "60000/60000 [==============================] - 29s 491us/step - loss: 0.5799 - acc: 0.8126 - val_loss: 0.2385 - val_acc: 0.9348\n",
      "Epoch 5/5\n",
      "60000/60000 [==============================] - 25s 416us/step - loss: 0.4773 - acc: 0.8487 - val_loss: 0.1955 - val_acc: 0.9444\n",
      "Test score: 0.195522360513\n",
      "Test accuracy: 0.9444\n"
     ]
    }
   ],
   "source": [
    "# Function for constructing the convolution neural network\n",
    "# Feel free to add parameters, if you want\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Convolution2D(16, 3, 3,\n",
    "                    border_mode='valid',\n",
    "                    input_shape=(img_rows, img_cols,1)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Convolution2D(32, 3, 3))\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.5))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(128))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(128))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dense(nb_classes))\n",
    "model.add(Activation('softmax'))\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "          optimizer='sgd',\n",
    "          metrics=['accuracy'])\n",
    "\n",
    "model.fit(X_train, Y_train, batch_size=batch_size, \n",
    "          nb_epoch=5,verbose=1,\n",
    "          validation_data=(X_test, Y_test))\n",
    "\n",
    "\n",
    "#Evaluating the model on the test data    \n",
    "score, accuracy = model.evaluate(X_test, Y_test, verbose=0)\n",
    "print('Test score:', score)\n",
    "print('Test accuracy:', accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}