question-answer/3.end-to-end-lstm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from copy import deepcopy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "epoch = 20\n",
    "batch_size = 64\n",
    "size_layer = 64\n",
    "dropout_rate = 0.5\n",
    "n_hops = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BaseDataLoader():\n",
    "    def __init__(self):\n",
    "        self.data = {\n",
    "            'size': None,\n",
    "            'val':{\n",
    "                'inputs': None,\n",
    "                'questions': None,\n",
    "                'answers': None,},\n",
    "            'len':{\n",
    "                'inputs_len': None,\n",
    "                'inputs_sent_len': None,\n",
    "                'questions_len': None,\n",
    "                'answers_len': None}\n",
    "        }\n",
    "        self.vocab = {\n",
    "            'size': None,\n",
    "            'word2idx': None,\n",
    "            'idx2word': None,\n",
    "        }\n",
    "        self.params = {\n",
    "            'vocab_size': None,\n",
    "            '<start>': None,\n",
    "            '<end>': None,\n",
    "            'max_input_len': None,\n",
    "            'max_sent_len': None,\n",
    "            'max_quest_len': None,\n",
    "            'max_answer_len': None,\n",
    "        }\n",
    "        \n",
    "class DataLoader(BaseDataLoader):\n",
    "    def __init__(self, path, is_training, vocab=None, params=None):\n",
    "        super().__init__()\n",
    "        data, lens = self.load_data(path)\n",
    "        if is_training:\n",
    "            self.build_vocab(data)\n",
    "        else:\n",
    "            self.demo = data\n",
    "            self.vocab = vocab\n",
    "            self.params = deepcopy(params)\n",
    "        self.is_training = is_training\n",
    "        self.padding(data, lens)\n",
    "\n",
    "\n",
    "    def load_data(self, path):\n",
    "        data, lens = bAbI_data_load(path)\n",
    "        self.data['size'] = len(data[0])\n",
    "        return data, lens\n",
    "\n",
    "\n",
    "    def build_vocab(self, data):\n",
    "        signals = ['<pad>', '<unk>', '<start>', '<end>']\n",
    "        inputs, questions, answers = data\n",
    "        i_words = [w for facts in inputs for fact in facts for w in fact if w != '<end>']\n",
    "        q_words = [w for question in questions for w in question]\n",
    "        a_words = [w for answer in answers for w in answer if w != '<end>']\n",
    "        words = list(set(i_words + q_words + a_words))\n",
    "        self.params['vocab_size'] = len(words) + 4\n",
    "        self.params['<start>'] = 2\n",
    "        self.params['<end>'] = 3\n",
    "        self.vocab['word2idx'] = {word: idx for idx, word in enumerate(signals + words)}\n",
    "        self.vocab['idx2word'] = {idx: word for word, idx in self.vocab['word2idx'].items()}\n",
    "        \n",
    "\n",
    "    def padding(self, data, lens):\n",
    "        inputs_len, inputs_sent_len, questions_len, answers_len = lens\n",
    "\n",
    "        self.params['max_input_len'] = max(inputs_len)\n",
    "        self.params['max_sent_len'] = max([fact_len for batch in inputs_sent_len for fact_len in batch])\n",
    "        self.params['max_quest_len'] = max(questions_len)\n",
    "        self.params['max_answer_len'] = max(answers_len)\n",
    "\n",
    "        self.data['len']['inputs_len'] = np.array(inputs_len)\n",
    "        for batch in inputs_sent_len:\n",
    "            batch += [0] * (self.params['max_input_len'] - len(batch))\n",
    "        self.data['len']['inputs_sent_len'] = np.array(inputs_sent_len)\n",
    "        self.data['len']['questions_len'] = np.array(questions_len)\n",
    "        self.data['len']['answers_len'] = np.array(answers_len)\n",
    "        \n",
    "        inputs, questions, answers = deepcopy(data)\n",
    "        for facts in inputs:\n",
    "            for sentence in facts:\n",
    "                for i in range(len(sentence)):\n",
    "                    sentence[i] = self.vocab['word2idx'].get(sentence[i], self.vocab['word2idx']['<unk>'])\n",
    "                sentence += [0] * (self.params['max_sent_len'] - len(sentence))\n",
    "            paddings = [0] * self.params['max_sent_len']\n",
    "            facts += [paddings] * (self.params['max_input_len'] - len(facts))\n",
    "        for question in questions:\n",
    "            for i in range(len(question)):\n",
    "                question[i] = self.vocab['word2idx'].get(question[i], self.vocab['word2idx']['<unk>'])\n",
    "            question += [0] * (self.params['max_quest_len'] - len(question))\n",
    "        for answer in answers:\n",
    "            for i in range(len(answer)):\n",
    "                answer[i] = self.vocab['word2idx'].get(answer[i], self.vocab['word2idx']['<unk>'])\n",
    "\n",
    "        self.data['val']['inputs'] = np.array(inputs)\n",
    "        self.data['val']['questions'] = np.array(questions)\n",
    "        self.data['val']['answers'] = np.array(answers)\n",
    "        \n",
    "def bAbI_data_load(path, END=['<end>']):\n",
    "    inputs = []\n",
    "    questions = []\n",
    "    answers = []\n",
    "\n",
    "    inputs_len = []\n",
    "    inputs_sent_len = []\n",
    "    questions_len = []\n",
    "    answers_len = []\n",
    "\n",
    "    for d in open(path):\n",
    "        index = d.split(' ')[0]\n",
    "        if index == '1':\n",
    "            fact = []\n",
    "        if '?' in d:\n",
    "            temp = d.split('\\t')\n",
    "            q = temp[0].strip().replace('?', '').split(' ')[1:] + ['?']\n",
    "            a = temp[1].split() + END\n",
    "            fact_copied = deepcopy(fact)\n",
    "            inputs.append(fact_copied)\n",
    "            questions.append(q)\n",
    "            answers.append(a)\n",
    "\n",
    "            inputs_len.append(len(fact_copied))\n",
    "            inputs_sent_len.append([len(s) for s in fact_copied])\n",
    "            questions_len.append(len(q))\n",
    "            answers_len.append(len(a))\n",
    "        else:\n",
    "            tokens = d.replace('.', '').replace('\\n', '').split(' ')[1:] + END\n",
    "            fact.append(tokens)\n",
    "    return [inputs, questions, answers], [inputs_len, inputs_sent_len, questions_len, answers_len]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = DataLoader(path='qa5_three-arg-relations_train.txt',is_training=True)\n",
    "test_data = DataLoader(path='qa5_three-arg-relations_test.txt',is_training=False,\n",
    "                       vocab=train_data.vocab, params=train_data.params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "START = train_data.params['<start>']\n",
    "END = train_data.params['<end>']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hop_forward(question, memory_o, memory_i, response_proj, \n",
    "                inputs_len, questions_len, is_training):\n",
    "    match = tf.matmul(question, memory_i, transpose_b=True)\n",
    "    match = pre_softmax_masking(match, inputs_len)\n",
    "    match = tf.nn.softmax(match)\n",
    "    match = post_softmax_masking(match, questions_len)\n",
    "    response = tf.matmul(match, memory_o)\n",
    "    return response_proj(tf.concat([response, question], -1))\n",
    "\n",
    "def pre_softmax_masking(x, seq_len):\n",
    "    paddings = tf.fill(tf.shape(x), float('-inf'))\n",
    "    T = tf.shape(x)[1]\n",
    "    max_seq_len = tf.shape(x)[2]\n",
    "    masks = tf.sequence_mask(seq_len, max_seq_len, dtype=tf.float32)\n",
    "    masks = tf.tile(tf.expand_dims(masks, 1), [1, T, 1])\n",
    "    return tf.where(tf.equal(masks, 0), paddings, x)\n",
    "\n",
    "\n",
    "def post_softmax_masking(x, seq_len):\n",
    "    T = tf.shape(x)[2]\n",
    "    max_seq_len = tf.shape(x)[1]\n",
    "    masks = tf.sequence_mask(seq_len, max_seq_len, dtype=tf.float32)\n",
    "    masks = tf.tile(tf.expand_dims(masks, -1), [1, 1, T])\n",
    "    return (x * masks)\n",
    "\n",
    "\n",
    "def shift_right(x):\n",
    "    batch_size = tf.shape(x)[0]\n",
    "    start = tf.to_int32(tf.fill([batch_size, 1], START))\n",
    "    return tf.concat([start, x[:, :-1]], 1)\n",
    "\n",
    "\n",
    "def embed_seq(x, vocab_size, zero_pad=True):\n",
    "    lookup_table = tf.get_variable('lookup_table', [vocab_size, size_layer], tf.float32)\n",
    "    if zero_pad:\n",
    "        lookup_table = tf.concat((tf.zeros([1, size_layer]), lookup_table[1:, :]), axis=0)\n",
    "    return tf.nn.embedding_lookup(lookup_table, x)\n",
    "\n",
    "\n",
    "def position_encoding(sentence_size, embedding_size):\n",
    "    encoding = np.ones((embedding_size, sentence_size), dtype=np.float32)\n",
    "    ls = sentence_size + 1\n",
    "    le = embedding_size + 1\n",
    "    for i in range(1, le):\n",
    "        for j in range(1, ls):\n",
    "            encoding[i-1, j-1] = (i - (le-1)/2) * (j - (ls-1)/2)\n",
    "    encoding = 1 + 4 * encoding / embedding_size / sentence_size\n",
    "    return tf.convert_to_tensor(np.transpose(encoding))\n",
    "\n",
    "def input_mem(x, vocab_size, max_sent_len, is_training):\n",
    "    x = embed_seq(x, vocab_size)\n",
    "    x = tf.layers.dropout(x, dropout_rate, training=is_training)\n",
    "    pos = position_encoding(max_sent_len, size_layer)\n",
    "    x = tf.reduce_sum(x * pos, 2)\n",
    "    return x\n",
    "\n",
    "def quest_mem(x, vocab_size, max_quest_len, is_training):\n",
    "    x = embed_seq(x, vocab_size)\n",
    "    x = tf.layers.dropout(x, dropout_rate, training=is_training)\n",
    "    pos = position_encoding(max_quest_len, size_layer)\n",
    "    return (x * pos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class QA:\n",
    "    def __init__(self, vocab_size):\n",
    "        self.questions = tf.placeholder(tf.int32,[None,None])\n",
    "        self.inputs = tf.placeholder(tf.int32,[None,None,None])\n",
    "        self.questions_len = tf.placeholder(tf.int32,[None])\n",
    "        self.inputs_len = tf.placeholder(tf.int32,[None])\n",
    "        self.answers_len = tf.placeholder(tf.int32,[None])\n",
    "        self.answers = tf.placeholder(tf.int32,[None,None])\n",
    "        self.training = tf.placeholder(tf.bool)\n",
    "        max_sent_len = train_data.params['max_sent_len']\n",
    "        max_quest_len = train_data.params['max_quest_len']\n",
    "        max_answer_len = train_data.params['max_answer_len']\n",
    "        \n",
    "        lookup_table = tf.get_variable('lookup_table', [vocab_size, size_layer], tf.float32)\n",
    "        lookup_table = tf.concat((tf.zeros([1, size_layer]), lookup_table[1:, :]), axis=0)\n",
    "        \n",
    "        with tf.variable_scope('questions'):\n",
    "            question = quest_mem(self.questions, vocab_size, max_quest_len, self.training)\n",
    "        \n",
    "        with tf.variable_scope('memory_o'):\n",
    "            memory_o = input_mem(self.inputs, vocab_size, max_sent_len, self.training)\n",
    "        \n",
    "        with tf.variable_scope('memory_i'):\n",
    "            memory_i = input_mem(self.inputs, vocab_size, max_sent_len, self.training)\n",
    "            \n",
    "        with tf.variable_scope('interaction'):\n",
    "            response_proj = tf.layers.Dense(size_layer)\n",
    "            for _ in range(n_hops):\n",
    "                answer = hop_forward(question,\n",
    "                                     memory_o,\n",
    "                                     memory_i,\n",
    "                                     response_proj,\n",
    "                                     self.inputs_len,\n",
    "                                     self.questions_len,\n",
    "                                     self.training)\n",
    "                question = answer\n",
    "                \n",
    "        with tf.variable_scope('memory_o', reuse=True):\n",
    "            embedding = tf.get_variable('lookup_table')\n",
    "        cell = tf.nn.rnn_cell.LSTMCell(size_layer)\n",
    "        vocab_proj = tf.layers.Dense(vocab_size)\n",
    "        state_proj = tf.layers.Dense(size_layer)\n",
    "        init_state = state_proj(tf.layers.flatten(answer))\n",
    "        init_state = tf.layers.dropout(init_state, dropout_rate, training=self.training)\n",
    "        helper = tf.contrib.seq2seq.TrainingHelper(\n",
    "            inputs = tf.nn.embedding_lookup(embedding, shift_right(self.answers)),\n",
    "            sequence_length = tf.to_int32(self.answers_len))\n",
    "        encoder_state = tf.nn.rnn_cell.LSTMStateTuple(c=init_state, h=init_state)\n",
    "        decoder = tf.contrib.seq2seq.BasicDecoder(cell = cell,\n",
    "                                                  helper = helper,\n",
    "                                                  initial_state = encoder_state,\n",
    "                                                  output_layer = vocab_proj)\n",
    "        decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(decoder = decoder,\n",
    "                                                                maximum_iterations = tf.shape(self.inputs)[1])\n",
    "        self.outputs = decoder_output.rnn_output\n",
    "        \n",
    "        helper = tf.contrib.seq2seq.GreedyEmbeddingHelper(embedding = embedding,\n",
    "                                                          start_tokens = tf.tile(\n",
    "                                                              tf.constant([START], \n",
    "                                                                          dtype=tf.int32), \n",
    "                                                              [tf.shape(self.inputs)[0]]),\n",
    "                                                          end_token = END)\n",
    "        decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "            cell = cell,\n",
    "            helper = helper,\n",
    "            initial_state = encoder_state,\n",
    "            output_layer = vocab_proj)\n",
    "        decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "            decoder = decoder,\n",
    "            maximum_iterations = max_answer_len)\n",
    "        self.logits = decoder_output.sample_id\n",
    "        correct_pred = tf.equal(self.logits, self.answers)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "        self.cost = tf.reduce_mean(tf.contrib.seq2seq.sequence_loss(logits = self.outputs,\n",
    "                                                        targets = self.answers,\n",
    "                                                        weights = tf.ones_like(self.answers, tf.float32)))\n",
    "        self.optimizer = tf.train.AdamOptimizer().minimize(self.cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = QA(train_data.params['vocab_size'])\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "batching = (train_data.data['val']['inputs'].shape[0] // batch_size) * batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1, avg cost 1.293238, avg acc 0.579077\n",
      "epoch 2, avg cost 0.701424, avg acc 0.658403\n",
      "epoch 3, avg cost 0.604266, avg acc 0.731721\n",
      "epoch 4, avg cost 0.501699, avg acc 0.789914\n",
      "epoch 5, avg cost 0.448811, avg acc 0.816256\n",
      "epoch 6, avg cost 0.391464, avg acc 0.842198\n",
      "epoch 7, avg cost 0.344856, avg acc 0.863682\n",
      "epoch 8, avg cost 0.315198, avg acc 0.873147\n",
      "epoch 9, avg cost 0.296241, avg acc 0.877103\n",
      "epoch 10, avg cost 0.279457, avg acc 0.879006\n",
      "epoch 11, avg cost 0.273381, avg acc 0.884315\n",
      "epoch 12, avg cost 0.259506, avg acc 0.886569\n",
      "epoch 13, avg cost 0.257100, avg acc 0.885266\n",
      "epoch 14, avg cost 0.248125, avg acc 0.886218\n",
      "epoch 15, avg cost 0.246457, avg acc 0.890124\n",
      "epoch 16, avg cost 0.238909, avg acc 0.892879\n",
      "epoch 17, avg cost 0.237348, avg acc 0.888922\n",
      "epoch 18, avg cost 0.231390, avg acc 0.891076\n",
      "epoch 19, avg cost 0.224595, avg acc 0.894732\n",
      "epoch 20, avg cost 0.223235, avg acc 0.896735\n"
     ]
    }
   ],
   "source": [
    "for i in range(epoch):\n",
    "    total_cost, total_acc = 0, 0\n",
    "    for k in range(0, batching, batch_size):\n",
    "        batch_questions = train_data.data['val']['questions'][k:k+batch_size]\n",
    "        batch_inputs = train_data.data['val']['inputs'][k:k+batch_size]\n",
    "        batch_inputs_len = train_data.data['len']['inputs_len'][k:k+batch_size]\n",
    "        batch_questions_len = train_data.data['len']['questions_len'][k:k+batch_size]\n",
    "        batch_answers_len = train_data.data['len']['answers_len'][k:k+batch_size]\n",
    "        batch_answers = train_data.data['val']['answers'][k:k+batch_size]\n",
    "        acc, cost, _ = sess.run([model.accuracy,model.cost,model.optimizer],\n",
    "                                  feed_dict={model.questions:batch_questions,\n",
    "                                            model.inputs:batch_inputs,\n",
    "                                            model.inputs_len:batch_inputs_len,\n",
    "                                            model.questions_len:batch_questions_len,\n",
    "                                            model.answers_len:batch_answers_len,\n",
    "                                            model.answers:batch_answers,\n",
    "                                            model.training:True})\n",
    "        total_cost += cost\n",
    "        total_acc += acc\n",
    "    total_cost /= (train_data.data['val']['inputs'].shape[0] // batch_size)\n",
    "    total_acc /= (train_data.data['val']['inputs'].shape[0] // batch_size)\n",
    "    print('epoch %d, avg cost %f, avg acc %f'%(i+1,total_cost,total_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "testing_size = 32\n",
    "batch_questions = test_data.data['val']['questions'][:testing_size]\n",
    "batch_inputs = test_data.data['val']['inputs'][:testing_size]\n",
    "batch_inputs_len = test_data.data['len']['inputs_len'][:testing_size]\n",
    "batch_questions_len = test_data.data['len']['questions_len'][:testing_size]\n",
    "batch_answers_len = test_data.data['len']['answers_len'][:testing_size]\n",
    "batch_answers = test_data.data['val']['answers'][:testing_size]\n",
    "logits = sess.run(model.logits,\n",
    "                        feed_dict={model.questions:batch_questions,\n",
    "                                   model.inputs:batch_inputs,\n",
    "                                   model.inputs_len:batch_inputs_len,\n",
    "                                   model.questions_len:batch_questions_len,\n",
    "                                   model.answers_len:batch_answers_len,\n",
    "                                   model.training:False})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "QUESTION: What did Fred give to Jeff ? <pad>\n",
      "REAL: football\n",
      "PREDICT: football \n",
      "\n",
      "QUESTION: Who gave the football to Jeff ? <pad>\n",
      "REAL: Fred\n",
      "PREDICT: Fred \n",
      "\n",
      "QUESTION: What did Fred give to Jeff ? <pad>\n",
      "REAL: football\n",
      "PREDICT: football \n",
      "\n",
      "QUESTION: Who did Fred give the football to ?\n",
      "REAL: Jeff\n",
      "PREDICT: Jeff \n",
      "\n",
      "QUESTION: Who did Jeff give the football to ?\n",
      "REAL: Fred\n",
      "PREDICT: Fred \n",
      "\n",
      "QUESTION: Who gave the apple ? <pad> <pad> <pad>\n",
      "REAL: Jeff\n",
      "PREDICT: Jeff \n",
      "\n",
      "QUESTION: Who received the apple ? <pad> <pad> <pad>\n",
      "REAL: Bill\n",
      "PREDICT: Fred \n",
      "\n",
      "QUESTION: What did Bill give to Fred ? <pad>\n",
      "REAL: apple\n",
      "PREDICT: apple \n",
      "\n",
      "QUESTION: What did Bill give to Fred ? <pad>\n",
      "REAL: apple\n",
      "PREDICT: apple \n",
      "\n",
      "QUESTION: Who received the apple ? <pad> <pad> <pad>\n",
      "REAL: Fred\n",
      "PREDICT: Fred \n",
      "\n",
      "QUESTION: Who received the milk ? <pad> <pad> <pad>\n",
      "REAL: Mary\n",
      "PREDICT: Fred \n",
      "\n",
      "QUESTION: What did Mary give to Bill ? <pad>\n",
      "REAL: milk\n",
      "PREDICT: milk \n",
      "\n",
      "QUESTION: Who received the milk ? <pad> <pad> <pad>\n",
      "REAL: Bill\n",
      "PREDICT: Mary \n",
      "\n",
      "QUESTION: Who did Mary give the milk to ?\n",
      "REAL: Bill\n",
      "PREDICT: Bill \n",
      "\n",
      "QUESTION: What did Mary give to Bill ? <pad>\n",
      "REAL: milk\n",
      "PREDICT: milk \n",
      "\n",
      "QUESTION: Who received the football ? <pad> <pad> <pad>\n",
      "REAL: Jeff\n",
      "PREDICT: Fred \n",
      "\n",
      "QUESTION: Who gave the football to Jeff ? <pad>\n",
      "REAL: Fred\n",
      "PREDICT: Fred \n",
      "\n",
      "QUESTION: What did Fred give to Jeff ? <pad>\n",
      "REAL: football\n",
      "PREDICT: football \n",
      "\n",
      "QUESTION: Who received the football ? <pad> <pad> <pad>\n",
      "REAL: Jeff\n",
      "PREDICT: Fred \n",
      "\n",
      "QUESTION: Who gave the milk ? <pad> <pad> <pad>\n",
      "REAL: Mary\n",
      "PREDICT: Jeff \n",
      "\n",
      "QUESTION: What did Jeff give to Mary ? <pad>\n",
      "REAL: milk\n",
      "PREDICT: milk \n",
      "\n",
      "QUESTION: Who gave the milk to Jeff ? <pad>\n",
      "REAL: Mary\n",
      "PREDICT: Mary \n",
      "\n",
      "QUESTION: Who gave the milk to Jeff ? <pad>\n",
      "REAL: Mary\n",
      "PREDICT: Mary \n",
      "\n",
      "QUESTION: Who gave the apple to Jeff ? <pad>\n",
      "REAL: Bill\n",
      "PREDICT: Bill \n",
      "\n",
      "QUESTION: Who gave the apple ? <pad> <pad> <pad>\n",
      "REAL: Jeff\n",
      "PREDICT: Jeff \n",
      "\n",
      "QUESTION: What did Jeff give to Fred ? <pad>\n",
      "REAL: apple\n",
      "PREDICT: apple \n",
      "\n",
      "QUESTION: Who received the apple ? <pad> <pad> <pad>\n",
      "REAL: Fred\n",
      "PREDICT: Jeff \n",
      "\n",
      "QUESTION: Who received the apple ? <pad> <pad> <pad>\n",
      "REAL: Jeff\n",
      "PREDICT: Jeff \n",
      "\n",
      "QUESTION: Who gave the apple ? <pad> <pad> <pad>\n",
      "REAL: Jeff\n",
      "PREDICT: Jeff \n",
      "\n",
      "QUESTION: Who did Jeff give the apple to ?\n",
      "REAL: Fred\n",
      "PREDICT: Fred \n",
      "\n",
      "QUESTION: Who gave the milk ? <pad> <pad> <pad>\n",
      "REAL: Mary\n",
      "PREDICT: Mary \n",
      "\n",
      "QUESTION: Who gave the milk ? <pad> <pad> <pad>\n",
      "REAL: Bill\n",
      "PREDICT: Mary \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(testing_size):\n",
    "    print('QUESTION:',' '.join([train_data.vocab['idx2word'][k] for k in batch_questions[i]]))\n",
    "    print('REAL:',train_data.vocab['idx2word'][batch_answers[i,0]])\n",
    "    print('PREDICT:',train_data.vocab['idx2word'][logits[i,0]],'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}