09_organizing_code.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![book](https://raw.githubusercontent.com/ageron/tensorflow-safari-course/master/images/intro_to_tf_course.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Try not to peek at the solutions when you go through the exercises. ;-)**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First let's make sure this notebook works well in both Python 2 and Python 3:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import, division, print_function, unicode_literals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"tmp/data/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Organizing Your Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def neural_net_layer(inputs, n_neurons, activation=None, seed=None):\n",
    "    n_inputs = int(inputs.get_shape()[1])\n",
    "    b = tf.Variable(tf.zeros([n_neurons]), name=\"b\")\n",
    "    W = tf.Variable(tf.random_uniform([n_inputs, n_neurons], -1.0, 1.0, seed=seed), name=\"W\")\n",
    "    logits = tf.matmul(inputs, W) + b\n",
    "    if activation:\n",
    "        return activation(logits)\n",
    "    else:\n",
    "        return logits"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's simplify our code by using `neural_net_layer()`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 100\n",
    "n_outputs = 10\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    with tf.name_scope(\"inputs\"):\n",
    "        X = tf.placeholder(tf.float32, shape=[None, n_inputs], name=\"X\")\n",
    "        y = tf.placeholder(tf.int32, shape=[None], name=\"y\")\n",
    "\n",
    "#########################################################################\n",
    "# This section is simplified (the rest is unchanged)\n",
    "#\n",
    "    with tf.name_scope(\"hidden1\"):\n",
    "        hidden1 = neural_net_layer(X, n_hidden1, activation=tf.nn.relu) # <= CHANGED\n",
    "\n",
    "    with tf.name_scope(\"output\"):\n",
    "        logits = neural_net_layer(hidden1, n_outputs)                   # <= CHANGED\n",
    "        Y_proba = tf.nn.softmax(logits, name=\"Y_proba\")\n",
    "#\n",
    "#\n",
    "#########################################################################\n",
    "    \n",
    "    with tf.name_scope(\"train\"):\n",
    "        xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y)\n",
    "        loss = tf.reduce_mean(xentropy)\n",
    "        optimizer = tf.train.AdamOptimizer()\n",
    "        training_op = optimizer.minimize(loss)\n",
    "\n",
    "    with tf.name_scope(\"eval\"):\n",
    "        correct = tf.nn.in_top_k(logits, y, 1)\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "    with tf.name_scope(\"init_and_save\"):\n",
    "        init = tf.global_variables_initializer()\n",
    "        saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[var.op.name for var in graph.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's check that training still works:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 50\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_val = accuracy.eval(feed_dict={X: mnist.validation.images, y: mnist.validation.labels})\n",
    "        print(epoch, \"Train accuracy:\", acc_train, \"Validation accuracy:\", acc_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_mnist_model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's use `tf.layers.dense()` instead:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 100\n",
    "n_outputs = 10\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    with tf.name_scope(\"inputs\"):\n",
    "        X = tf.placeholder(tf.float32, shape=[None, n_inputs], name=\"X\")\n",
    "        y = tf.placeholder(tf.int32, shape=[None], name=\"y\")\n",
    "\n",
    "    with tf.name_scope(\"hidden1\"):\n",
    "        hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\") # <= CHANGED\n",
    "\n",
    "    with tf.name_scope(\"output\"):\n",
    "        logits = tf.layers.dense(hidden1, n_outputs, name=\"output\")                    # <= CHANGED\n",
    "        Y_proba = tf.nn.softmax(logits)\n",
    "    \n",
    "    with tf.name_scope(\"train\"):\n",
    "        xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y)\n",
    "        loss = tf.reduce_mean(xentropy)\n",
    "        optimizer = tf.train.AdamOptimizer()\n",
    "        training_op = optimizer.minimize(loss)\n",
    "\n",
    "    with tf.name_scope(\"eval\"):\n",
    "        correct = tf.nn.in_top_k(logits, y, 1)\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "    with tf.name_scope(\"init_and_save\"):\n",
    "        init = tf.global_variables_initializer()\n",
    "        saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[var.op.name for var in graph.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's check that training still works:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 50\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_val = accuracy.eval(feed_dict={X: mnist.validation.images, y: mnist.validation.labels})\n",
    "        print(epoch, \"Train accuracy:\", acc_train, \"Validation accuracy:\", acc_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_mnist_model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now suppose you want two more hidden layers with shared weights & biases. Let's use variable scopes for this:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_inputs = 28 * 28\n",
    "n_hidden = 100\n",
    "n_outputs = 10\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    with tf.name_scope(\"inputs\"):\n",
    "        X = tf.placeholder(tf.float32, shape=[None, n_inputs], name=\"X\")\n",
    "        y = tf.placeholder(tf.int32, shape=[None], name=\"y\")\n",
    "\n",
    "    hidden1 = tf.layers.dense(X, n_hidden, activation=tf.nn.relu, name=\"hidden1\")                    # <= CHANGED\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden, activation=tf.nn.relu, name=\"hidden23\")             # <= CHANGED\n",
    "    hidden3 = tf.layers.dense(hidden2, n_hidden, activation=tf.nn.relu, name=\"hidden23\", reuse=True) # <= CHANGED\n",
    "\n",
    "    with tf.name_scope(\"output\"):\n",
    "        logits = tf.layers.dense(hidden3, n_outputs, name=\"output\")\n",
    "        Y_proba = tf.nn.softmax(logits, name=\"Y_proba\")\n",
    "    \n",
    "    with tf.name_scope(\"train\"):\n",
    "        xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y)\n",
    "        loss = tf.reduce_mean(xentropy)\n",
    "        optimizer = tf.train.AdamOptimizer()\n",
    "        training_op = optimizer.minimize(loss)\n",
    "\n",
    "    with tf.name_scope(\"eval\"):\n",
    "        correct = tf.nn.in_top_k(logits, y, 1)\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "    with tf.name_scope(\"init_and_save\"):\n",
    "        init = tf.global_variables_initializer()\n",
    "        saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[var.op.name for var in graph.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check that training works well:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 50\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_val = accuracy.eval(feed_dict={X: mnist.validation.images, y: mnist.validation.labels})\n",
    "        print(epoch, \"Train accuracy:\", acc_train, \"Validation accuracy:\", acc_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_mnist_model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "How would we implement variable sharing in `neural_net_layer()`?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def neural_net_layer(inputs, n_neurons, activation=None, name=None, reuse=None, seed=None):\n",
    "    with tf.variable_scope(name, default_name=\"layer\", reuse=reuse):\n",
    "        n_inputs = int(inputs.get_shape()[1])\n",
    "        rnd_init = lambda shape, dtype, partition_info: tf.random_uniform(shape, -1.0, 1.0, dtype=dtype, seed=seed)\n",
    "        b = tf.get_variable(\"biases\", shape=[n_neurons], initializer=rnd_init)\n",
    "        W = tf.get_variable(\"weights\", shape=[n_inputs, n_neurons], initializer=rnd_init)\n",
    "        logits = tf.matmul(inputs, W) + b\n",
    "        if activation:\n",
    "            return activation(logits)\n",
    "        else:\n",
    "            return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    with tf.variable_scope(\"foo\"): \n",
    "        a = tf.constant(1., name=\"a\")\n",
    "        with tf.name_scope(\"bar\"): \n",
    "            b = tf.constant(2., name=\"b\")\n",
    "            with tf.name_scope(\"baz\"):\n",
    "                c = tf.get_variable(\"c\", shape=[], initializer=tf.constant_initializer(2))\n",
    "                s = tf.add_n([a,b,c], name=\"s\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a.name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "b.name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "c.name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s.name"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}