MNIST_LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "mnist = input_data.read_data_sets('MNIST_data/', one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "LSTM_UNITS = 256\n",
    "BATCH_SIZE = 100\n",
    "LEARNING_RATE = 1e-3\n",
    "N_CLASSES = 10\n",
    "TRAINING_ITER = 1000\n",
    "\n",
    "def LSTM(x):\n",
    "    \n",
    "    lstm_cell = tf.contrib.rnn.BasicLSTMCell(num_units=LSTM_UNITS)\n",
    "    \n",
    "    init_state = lstm_cell.zero_state(batch_size=BATCH_SIZE, dtype=tf.float32)\n",
    "    \n",
    "    outputs, final_state = tf.nn.dynamic_rnn(cell=lstm_cell,inputs=x,initial_state=init_state,time_major=False) \n",
    "    \n",
    "    return tf.unstack(tf.transpose(outputs, [1, 0, 2]))\n",
    "\n",
    "def init_weights(shape):\n",
    "    return tf.Variable(tf.truncated_normal(shape, stddev=0.1))\n",
    "\n",
    "def init_biases(shape):\n",
    "    return tf.Variable(tf.zeros(shape) + 0.1)\n",
    "\n",
    "def add_layer(x, weights, biases, activation_function=None):\n",
    "    results = tf.matmul(x, weights) + biases\n",
    "    if activation_function:\n",
    "        return activation_function(results)\n",
    "    else:\n",
    "        return results\n",
    "\n",
    "weights = {'output': init_weights([LSTM_UNITS, N_CLASSES])}\n",
    "biases = {'output': init_biases([N_CLASSES])}\n",
    "\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "y = tf.placeholder(tf.float32, [None, N_CLASSES])\n",
    "\n",
    "x_2d = tf.reshape(x, [-1, 28, 28])\n",
    "\n",
    "lstm_outputs = LSTM(x_2d)\n",
    "\n",
    "predictions = add_layer(lstm_outputs[-1], weights['output'], biases['output'], None)\n",
    "\n",
    "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=predictions))\n",
    "\n",
    "train_step = tf.train.AdamOptimizer(LEARNING_RATE).minimize(loss)\n",
    "\n",
    "accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.arg_max(y, 1), tf.arg_max(predictions, 1)), tf.float32))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TRAIN_ACC@0: 0.27\n",
      "VALID_ACC@0:"
     ]
    },
    {
     "ename": "InvalidArgumentError",
     "evalue": "ConcatOp : Dimensions of inputs should match: shape[0] = [5000,28] vs. shape[1] = [100,256]\n\t [[Node: rnn/while/basic_lstm_cell/basic_lstm_cell/concat = ConcatV2[N=2, T=DT_FLOAT, Tidx=DT_INT32, _device=\"/job:localhost/replica:0/task:0/cpu:0\"](rnn/while/TensorArrayReadV3, rnn/while/Identity_3, rnn/while/basic_lstm_cell/basic_lstm_cell/concat/axis)]]\n\nCaused by op u'rnn/while/basic_lstm_cell/basic_lstm_cell/concat', defined at:\n  File \"/Users/miaofan/anaconda/lib/python2.7/runpy.py\", line 162, in _run_module_as_main\n    \"__main__\", fname, loader, pkg_name)\n  File \"/Users/miaofan/anaconda/lib/python2.7/runpy.py\", line 72, in _run_code\n    exec code in run_globals\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/ipykernel/__main__.py\", line 3, in <module>\n    app.launch_new_instance()\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/ipykernel/kernelapp.py\", line 405, in start\n    ioloop.IOLoop.instance().start()\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/zmq/eventloop/ioloop.py\", line 162, in start\n    super(ZMQIOLoop, self).start()\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tornado/ioloop.py\", line 883, in start\n    handler_func(fd_obj, events)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tornado/stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tornado/stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/ipykernel/kernelbase.py\", line 260, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/ipykernel/kernelbase.py\", line 212, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/ipykernel/kernelbase.py\", line 370, in execute_request\n    user_expressions, allow_stdin)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/ipykernel/ipkernel.py\", line 175, in do_execute\n    shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2717, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2821, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-3-97985244513f>\", line 38, in <module>\n    lstm_outputs = LSTM(x_2d)\n  File \"<ipython-input-3-97985244513f>\", line 13, in LSTM\n    outputs, final_state = tf.nn.dynamic_rnn(cell=lstm_cell,inputs=x,initial_state=init_state,time_major=False)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/ops/rnn.py\", line 546, in dynamic_rnn\n    dtype=dtype)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/ops/rnn.py\", line 713, in _dynamic_rnn_loop\n    swap_memory=swap_memory)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py\", line 2605, in while_loop\n    result = context.BuildLoop(cond, body, loop_vars, shape_invariants)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py\", line 2438, in BuildLoop\n    pred, body, original_loop_vars, loop_vars, shape_invariants)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py\", line 2388, in _BuildLoop\n    body_result = body(*packed_vars_for_body)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/ops/rnn.py\", line 698, in _time_step\n    (output, new_state) = call_cell()\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/ops/rnn.py\", line 684, in <lambda>\n    call_cell = lambda: cell(input_t, state)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/contrib/rnn/python/ops/core_rnn_cell_impl.py\", line 179, in __call__\n    concat = _linear([inputs, h], 4 * self._num_units, True, scope=scope)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/contrib/rnn/python/ops/core_rnn_cell_impl.py\", line 751, in _linear\n    res = math_ops.matmul(array_ops.concat(args, 1), weights)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/ops/array_ops.py\", line 1034, in concat\n    name=name)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/ops/gen_array_ops.py\", line 519, in _concat_v2\n    name=name)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py\", line 763, in apply_op\n    op_def=op_def)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/framework/ops.py\", line 2327, in create_op\n    original_op=self._default_original_op, op_def=op_def)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/framework/ops.py\", line 1226, in __init__\n    self._traceback = _extract_stack()\n\nInvalidArgumentError (see above for traceback): ConcatOp : Dimensions of inputs should match: shape[0] = [5000,28] vs. shape[1] = [100,256]\n\t [[Node: rnn/while/basic_lstm_cell/basic_lstm_cell/concat = ConcatV2[N=2, T=DT_FLOAT, Tidx=DT_INT32, _device=\"/job:localhost/replica:0/task:0/cpu:0\"](rnn/while/TensorArrayReadV3, rnn/while/Identity_3, rnn/while/basic_lstm_cell/basic_lstm_cell/concat/axis)]]\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-50c696363f46>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m50\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m             \u001b[0;32mprint\u001b[0m \u001b[0;34m'TRAIN_ACC@%d:'\u001b[0m \u001b[0;34m%\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maccuracy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch_xs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mbatch_ys\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m             \u001b[0;32mprint\u001b[0m \u001b[0;34m'VALID_ACC@%d:'\u001b[0m \u001b[0;34m%\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maccuracy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mmnist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalidation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimages\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mmnist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalidation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    765\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    766\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 767\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    768\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    769\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    963\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    964\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 965\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    966\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    967\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1013\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1014\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1015\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m   1016\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1017\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1033\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1034\u001b[0m           \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1035\u001b[0;31m       \u001b[0;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1036\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1037\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mInvalidArgumentError\u001b[0m: ConcatOp : Dimensions of inputs should match: shape[0] = [5000,28] vs. shape[1] = [100,256]\n\t [[Node: rnn/while/basic_lstm_cell/basic_lstm_cell/concat = ConcatV2[N=2, T=DT_FLOAT, Tidx=DT_INT32, _device=\"/job:localhost/replica:0/task:0/cpu:0\"](rnn/while/TensorArrayReadV3, rnn/while/Identity_3, rnn/while/basic_lstm_cell/basic_lstm_cell/concat/axis)]]\n\nCaused by op u'rnn/while/basic_lstm_cell/basic_lstm_cell/concat', defined at:\n  File \"/Users/miaofan/anaconda/lib/python2.7/runpy.py\", line 162, in _run_module_as_main\n    \"__main__\", fname, loader, pkg_name)\n  File \"/Users/miaofan/anaconda/lib/python2.7/runpy.py\", line 72, in _run_code\n    exec code in run_globals\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/ipykernel/__main__.py\", line 3, in <module>\n    app.launch_new_instance()\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/ipykernel/kernelapp.py\", line 405, in start\n    ioloop.IOLoop.instance().start()\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/zmq/eventloop/ioloop.py\", line 162, in start\n    super(ZMQIOLoop, self).start()\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tornado/ioloop.py\", line 883, in start\n    handler_func(fd_obj, events)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tornado/stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tornado/stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/ipykernel/kernelbase.py\", line 260, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/ipykernel/kernelbase.py\", line 212, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/ipykernel/kernelbase.py\", line 370, in execute_request\n    user_expressions, allow_stdin)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/ipykernel/ipkernel.py\", line 175, in do_execute\n    shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2717, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2821, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-3-97985244513f>\", line 38, in <module>\n    lstm_outputs = LSTM(x_2d)\n  File \"<ipython-input-3-97985244513f>\", line 13, in LSTM\n    outputs, final_state = tf.nn.dynamic_rnn(cell=lstm_cell,inputs=x,initial_state=init_state,time_major=False)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/ops/rnn.py\", line 546, in dynamic_rnn\n    dtype=dtype)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/ops/rnn.py\", line 713, in _dynamic_rnn_loop\n    swap_memory=swap_memory)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py\", line 2605, in while_loop\n    result = context.BuildLoop(cond, body, loop_vars, shape_invariants)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py\", line 2438, in BuildLoop\n    pred, body, original_loop_vars, loop_vars, shape_invariants)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py\", line 2388, in _BuildLoop\n    body_result = body(*packed_vars_for_body)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/ops/rnn.py\", line 698, in _time_step\n    (output, new_state) = call_cell()\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/ops/rnn.py\", line 684, in <lambda>\n    call_cell = lambda: cell(input_t, state)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/contrib/rnn/python/ops/core_rnn_cell_impl.py\", line 179, in __call__\n    concat = _linear([inputs, h], 4 * self._num_units, True, scope=scope)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/contrib/rnn/python/ops/core_rnn_cell_impl.py\", line 751, in _linear\n    res = math_ops.matmul(array_ops.concat(args, 1), weights)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/ops/array_ops.py\", line 1034, in concat\n    name=name)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/ops/gen_array_ops.py\", line 519, in _concat_v2\n    name=name)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py\", line 763, in apply_op\n    op_def=op_def)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/framework/ops.py\", line 2327, in create_op\n    original_op=self._default_original_op, op_def=op_def)\n  File \"/Users/miaofan/anaconda/lib/python2.7/site-packages/tensorflow/python/framework/ops.py\", line 1226, in __init__\n    self._traceback = _extract_stack()\n\nInvalidArgumentError (see above for traceback): ConcatOp : Dimensions of inputs should match: shape[0] = [5000,28] vs. shape[1] = [100,256]\n\t [[Node: rnn/while/basic_lstm_cell/basic_lstm_cell/concat = ConcatV2[N=2, T=DT_FLOAT, Tidx=DT_INT32, _device=\"/job:localhost/replica:0/task:0/cpu:0\"](rnn/while/TensorArrayReadV3, rnn/while/Identity_3, rnn/while/basic_lstm_cell/basic_lstm_cell/concat/axis)]]\n"
     ]
    }
   ],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    \n",
    "    for i in range(TRAINING_ITER):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(BATCH_SIZE)\n",
    "        sess.run(train_step, feed_dict= {x: batch_xs, y: batch_ys})\n",
    "        if i % 50 == 0:\n",
    "            print 'TRAIN_ACC@%d:' %i, sess.run(accuracy, feed_dict = {x: batch_xs, y:batch_ys})\n",
    "            \n",
    "            \n",
    "            #print 'VALID_ACC@%d:' %i, sess.run(accuracy, feed_dict = {x: mnist.validation.images, y: mnist.validation.labels})"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}