2.5 HyperParameter Tuning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# HyperParameter Tuning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `keras.wrappers.scikit_learn`\n",
    "\n",
    "Example adapted from: [https://github.com/fchollet/keras/blob/master/examples/mnist_sklearn_wrapper.py]()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Problem: \n",
    "Builds simple CNN models on MNIST and uses sklearn's GridSearchCV to find best model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "np.random.seed(1337)  # for reproducibility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras.utils import np_utils\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras import backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nb_classes = 10\n",
    "\n",
    "# input image dimensions\n",
    "img_rows, img_cols = 28, 28"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load training data and do basic data normalization\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "\n",
    "if K.image_dim_ordering() == 'th':\n",
    "    X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)\n",
    "    X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols)\n",
    "    input_shape = (1, img_rows, img_cols)\n",
    "else:\n",
    "    X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1)\n",
    "    X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1)\n",
    "    input_shape = (img_rows, img_cols, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = X_train.astype('float32')\n",
    "X_test = X_test.astype('float32')\n",
    "X_train /= 255\n",
    "X_test /= 255\n",
    "\n",
    "# convert class vectors to binary class matrices\n",
    "y_train = np_utils.to_categorical(y_train, nb_classes)\n",
    "y_test = np_utils.to_categorical(y_test, nb_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def make_model(dense_layer_sizes, nb_filters, nb_conv, nb_pool):\n",
    "    '''Creates model comprised of 2 convolutional layers followed by dense layers\n",
    "\n",
    "    dense_layer_sizes: List of layer sizes. This list has one number for each layer\n",
    "    nb_filters: Number of convolutional filters in each convolutional layer\n",
    "    nb_conv: Convolutional kernel size\n",
    "    nb_pool: Size of pooling area for max pooling\n",
    "    '''\n",
    "\n",
    "    model = Sequential()\n",
    "\n",
    "    model.add(Conv2D(nb_filters, (nb_conv, nb_conv),\n",
    "                     padding='valid', input_shape=input_shape))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Conv2D(nb_filters, (nb_conv, nb_conv)))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))\n",
    "    model.add(Dropout(0.25))\n",
    "\n",
    "    model.add(Flatten())\n",
    "    for layer_size in dense_layer_sizes:\n",
    "        model.add(Dense(layer_size))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(nb_classes))\n",
    "    model.add(Activation('softmax'))\n",
    "\n",
    "    model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer='adadelta',\n",
    "                  metrics=['accuracy'])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dense_size_candidates = [[32], [64], [32, 32], [64, 64]]\n",
    "my_classifier = KerasClassifier(make_model, batch_size=32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GridSearch HyperParameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "validator = GridSearchCV(my_classifier,\n",
    "                         param_grid={'dense_layer_sizes': dense_size_candidates,\n",
    "                                     # nb_epoch is avail for tuning even when not\n",
    "                                     # an argument to model building function\n",
    "                                     'nb_epoch': [3, 6],\n",
    "                                     'nb_filters': [8],\n",
    "                                     'nb_conv': [3],\n",
    "                                     'nb_pool': [2]},\n",
    "                         scoring='neg_log_loss',\n",
    "                         n_jobs=1)\n",
    "validator.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print('The parameters of the best model are: ')\n",
    "print(validator.best_params_)\n",
    "\n",
    "# validator.best_estimator_ returns sklearn-wrapped version of best model.\n",
    "# validator.best_estimator_.model returns the (unwrapped) keras model\n",
    "best_model = validator.best_estimator_.model\n",
    "metric_names = best_model.metrics_names\n",
    "metric_values = best_model.evaluate(X_test, y_test)\n",
    "for metric, value in zip(metric_names, metric_values):\n",
    "    print(metric, ': ', value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}