Attention_Onestep_Mod.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "import tensorflow as tf\n",
    "import cv2\n",
    "import pickle\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from sklearn.model_selection import train_test_split\n",
    "import time\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.layers import Dense, LSTM, Input, Embedding, Conv2D,Concatenate,Flatten,Add,Dropout,GRU\n",
    "import random\n",
    "import datetime\n",
    "from nltk.translate.bleu_score import sentence_bleu\n",
    "from math import log\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = pd.read_csv('Train_Data.csv')\n",
    "cv_dataset = pd.read_csv('CV_Data.csv')\n",
    "test_dataset = pd.read_csv('Test_Data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, X_cv = train_dataset['Person_id'], test_dataset['Person_id'], cv_dataset['Person_id'][:546]\n",
    "y_train, y_test, y_cv = train_dataset['Report'], test_dataset['Report'], cv_dataset['Report'][:546]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_capt_len = 153\n",
    "pad_size = max_capt_len "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(filters='!\"#$%&()*+,-/:;<=>?@[\\\\]^_`{|}~\\t\\n')\n",
    "tokenizer.fit_on_texts(y_train.values)\n",
    "\n",
    "train_rep_tok = tokenizer.texts_to_sequences(y_train)\n",
    "cv_rep_tok = tokenizer.texts_to_sequences(y_cv)\n",
    "test_rep_tok = tokenizer.texts_to_sequences(y_test)\n",
    "\n",
    "train_rep_padded = pad_sequences(train_rep_tok, maxlen=153, padding='post')\n",
    "cv_rep_padded = pad_sequences(cv_rep_tok, maxlen=153, padding='post')\n",
    "test_rep_padded = pad_sequences(test_rep_tok, maxlen=153, padding='post')\n",
    "\n",
    "tokenizer.word_index['<pad>'] = 0\n",
    "tokenizer.index_word[0] = '<pad>'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('Image_features_attention.pickle','rb') # contains the features from chexNet\n",
    "Xnet_Features = pickle.load(f)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('GLOVE_VECTORS.pickle','rb') # 300d glove vectors  \n",
    "glove_vectors = pickle.load(f)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([98, 1024])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = Xnet_Features['Scanned Images/CXR1_1_IM-0001_0'][0]\n",
    "a.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 14\n",
    "BUFFER_SIZE = 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_image(id_, report):\n",
    "    '''Loads the Image Features with their corresponding Ids'''\n",
    "    img_feature = Xnet_Features[id_.decode('utf-8')][0]\n",
    "    return img_feature, report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_dataset(img_name_train,reps):\n",
    "  \n",
    "    dataset = tf.data.Dataset.from_tensor_slices((img_name_train, reps))\n",
    "\n",
    "  # Use map to load the numpy files in parallel\n",
    "    dataset = dataset.map(lambda item1, item2: tf.numpy_function(load_image, [item1, item2], [tf.float32, tf.int32]),\n",
    "                          num_parallel_calls=tf.data.experimental.AUTOTUNE)\n",
    "\n",
    "  # Shuffle and batch\n",
    "    dataset = dataset.shuffle(500).batch(BATCH_SIZE).prefetch(buffer_size=tf.data.experimental.AUTOTUNE)\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_generator = create_dataset(X_train.values, train_rep_padded)\n",
    "cv_generator = create_dataset(X_cv.values, cv_rep_padded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_size = len(tokenizer.word_index.keys()) + 1\n",
    "\n",
    "embedding_matrix = np.zeros((vocab_size,300))\n",
    "for word, i in tokenizer.word_index.items():\n",
    "    if word in glove_vectors.keys():\n",
    "        vec = glove_vectors[word]\n",
    "        embedding_matrix[i] = vec\n",
    "    else:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(tf.keras.layers.Layer):\n",
    "    def __init__(self, units):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.units = units\n",
    "       # self.bs = batch_size\n",
    "        \n",
    "    def build(self, input_shape):\n",
    "        self.maxpool = tf.keras.layers.MaxPool1D()\n",
    "        self.dense = Dense(self.units, kernel_initializer=tf.keras.initializers.glorot_uniform(seed = 56), name='dense_encoder')\n",
    "        \n",
    "    def call(self, input_, training=True):\n",
    "        \n",
    "        x = self.maxpool(input_)\n",
    "        x = self.dense(x)\n",
    "        \n",
    "        return x\n",
    "    \n",
    "    def get_states(self, bs):\n",
    "        \n",
    "        return tf.zeros((bs, self.units))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OneStepDecoder(tf.keras.layers.Layer):\n",
    "    def __init__(self, vocab_size, att_units, dec_units):\n",
    "        super(OneStepDecoder, self).__init__()\n",
    "        self.vocab_size = vocab_size\n",
    "       # self.emb_dim = emb_dim\n",
    "        self.att_units = att_units\n",
    "        self.dec_units = dec_units\n",
    "        \n",
    "    def build(self, input_shape):\n",
    "        self.embedding = Embedding(self.vocab_size, output_dim=300, input_length=max_capt_len, mask_zero=True,\n",
    "                                   weights = [embedding_matrix],\n",
    "                                   name=\"embedding_layer_decoder\")\n",
    "        self.gru = GRU(self.dec_units, return_sequences=True, return_state=True, name=\"Decoder_GRU\")\n",
    "        self.fc = Dense(self.vocab_size)\n",
    "        \n",
    "        self.V = Dense(1)\n",
    "        self.W = Dense(self.att_units)\n",
    "        self.U = Dense(self.att_units)\n",
    "        \n",
    "    def call(self, dec_input, hidden_state, enc_output):\n",
    "       \n",
    "\n",
    "        hidden_with_time = tf.expand_dims(hidden_state, 1)\n",
    "        attention_weights = self.V(tf.nn.tanh(self.U(enc_output) + self.W(hidden_with_time)))        \n",
    "        attention_weights = tf.nn.softmax(attention_weights, 1)\n",
    "        context_vector = attention_weights * enc_output\n",
    "        context_vector = tf.reduce_sum(context_vector, axis=1)\n",
    "       \n",
    "\n",
    "        x = self.embedding(dec_input)\n",
    "        x = tf.concat([tf.expand_dims(context_vector, axis=1),x], axis=-1)\n",
    "        output, h_state = self.gru(x, initial_state = hidden_state)\n",
    "        output = tf.reshape(output, (-1, output.shape[2]))\n",
    "        x = self.fc(output)\n",
    "        \n",
    "        return x, h_state, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(tf.keras.layers.Layer):\n",
    "    \n",
    "    def __init__(self, vocab_size, input_length, dec_units, att_units):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.vocab_size = vocab_size\n",
    "    #    self.embedding_dim = embedding_dim\n",
    "        self.input_length = input_length\n",
    "        self.dec_units = dec_units\n",
    "        self.att_units = att_units\n",
    "        self.onestep_decoder = OneStepDecoder(self.vocab_size, self.att_units, self.dec_units)\n",
    "    @tf.function    \n",
    "    def call(self, dec_input, hidden_state, enc_output):\n",
    "        all_outputs = tf.TensorArray(tf.float32, dec_input.shape[1], name='output_arrays')\n",
    "        \n",
    "        for timestep in range(dec_input.shape[1]):\n",
    "            \n",
    "            output, hidden_state, attention_weights = self.onestep_decoder(dec_input[:, timestep:timestep+1], \n",
    "                                                                           hidden_state, enc_output)\n",
    "            \n",
    "            all_outputs = all_outputs.write(timestep, output)\n",
    "            \n",
    "        all_outputs = tf.transpose(all_outputs.stack(), [1,0,2])\n",
    "        return all_outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Attention_Model(tf.keras.Model):\n",
    "    def __init__(self, vocab, units, max_capt_len, att_units, batch_size):\n",
    "        super(Attention_Model, self).__init__()\n",
    "        self.batch_size = batch_size\n",
    "        self.encoder = Encoder(units)\n",
    "        self.decoder = Decoder(vocab_size, max_capt_len, units, att_units)\n",
    "        \n",
    "    def call(self, data):\n",
    "        enc_input, dec_input = data[0], data[1]\n",
    "    \n",
    "        enc_output = self.encoder(enc_input)\n",
    "        enc_state = self.encoder.get_states(self.batch_size)\n",
    "        dec_output = self.decoder(dec_input, enc_state, enc_output)\n",
    "\n",
    "        return dec_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "units = 256\n",
    "att_units = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "model1 = Attention_Model(vocab_size, units, max_capt_len, att_units, BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)\n",
    "loss_function = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction='auto')\n",
    "\n",
    "def maskedLoss(y_true, y_pred):\n",
    "    #getting mask value\n",
    "    mask = tf.math.logical_not(tf.math.equal(y_true, 0))\n",
    "    \n",
    "    #calculating the loss\n",
    "    loss_ = loss_function(y_true, y_pred)\n",
    "    \n",
    "    #converting mask dtype to loss_ dtype\n",
    "    mask = tf.cast(mask, dtype=loss_.dtype)\n",
    "    \n",
    "    #applying the mask to loss\n",
    "    loss_ = loss_*mask\n",
    "    \n",
    "    #getting mean over all the values\n",
    "    loss_ = tf.reduce_mean(loss_)\n",
    "    return loss_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "model1.compile(optimizer=optimizer, loss=maskedLoss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "EPOCHS = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "current_time = datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n",
    "train_log_dir = 'Tensorboard/attention_OneStep/fit2/' + current_time + '/train'\n",
    "val_log_dir = 'Tensorboard/attention_OneStep/fit2/' + current_time + '/test'\n",
    "train_summary_writer = tf.summary.create_file_writer(train_log_dir)\n",
    "val_summary_writer = tf.summary.create_file_writer(val_log_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:  1\n",
      "Training Loss: 0.2559060056500023,  Validation Loss: 0.21404053385441119\n",
      "Time Taken for this Epoch : 198.18446278572083 sec\n",
      "EPOCH:  2\n",
      "Training Loss: 0.17927777952500407,  Validation Loss: 0.16760547440021467\n",
      "Time Taken for this Epoch : 47.06950402259827 sec\n",
      "EPOCH:  3\n",
      "Training Loss: 0.1466302885886679,  Validation Loss: 0.1463114470243454\n",
      "Time Taken for this Epoch : 47.24710178375244 sec\n",
      "EPOCH:  4\n",
      "Training Loss: 0.1299016094828015,  Validation Loss: 0.1320826938519111\n",
      "Time Taken for this Epoch : 47.12834644317627 sec\n",
      "EPOCH:  5\n",
      "Training Loss: 0.1190420383124182,  Validation Loss: 0.12441700582320873\n",
      "Time Taken for this Epoch : 47.12254285812378 sec\n",
      "EPOCH:  6\n",
      "Training Loss: 0.11052309273538856,  Validation Loss: 0.11708339504324473\n",
      "Time Taken for this Epoch : 47.20883822441101 sec\n",
      "EPOCH:  7\n",
      "Training Loss: 0.10242496251227892,  Validation Loss: 0.11009295705037239\n",
      "Time Taken for this Epoch : 49.40894174575806 sec\n",
      "EPOCH:  8\n",
      "Training Loss: 0.09512821724874719,  Validation Loss: 0.10467810756885089\n",
      "Time Taken for this Epoch : 49.492817640304565 sec\n",
      "EPOCH:  9\n",
      "Training Loss: 0.08811999560537072,  Validation Loss: 0.10016657029971099\n",
      "Time Taken for this Epoch : 48.19546294212341 sec\n",
      "EPOCH:  10\n",
      "Training Loss: 0.08201246507238918,  Validation Loss: 0.0967164372977538\n",
      "Time Taken for this Epoch : 50.21811628341675 sec\n"
     ]
    }
   ],
   "source": [
    "epoch_train_loss = []\n",
    "epoch_val_loss = []\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "    start = time.time()\n",
    "    print(\"EPOCH: \", epoch+1)\n",
    "    batch_loss_tr = 0\n",
    "    batch_loss_val = 0\n",
    "#    print('Training...')\n",
    "    for img, rep in train_generator:\n",
    "        res = model1.train_on_batch([img, rep[:,:-1]], rep[:,1:])\n",
    "        batch_loss_tr += res\n",
    "        \n",
    "    train_loss = batch_loss_tr/(X_train.shape[0]/BATCH_SIZE)\n",
    "\n",
    "    with train_summary_writer.as_default():\n",
    "        tf.summary.scalar('loss', train_loss, step = epoch)\n",
    "    \n",
    "#    print(\"VALIDATING..\")\n",
    "    for img, rep in cv_generator:\n",
    "        res = model1.test_on_batch([img, rep[:,:-1]], rep[:,1:])\n",
    "        batch_loss_val += res\n",
    "        \n",
    "    val_loss = batch_loss_val/(X_cv.shape[0]/BATCH_SIZE)\n",
    "\n",
    "    with val_summary_writer.as_default():\n",
    "        tf.summary.scalar('loss', val_loss, step = epoch)    \n",
    "        \n",
    "    epoch_train_loss.append(train_loss)\n",
    "\n",
    "    epoch_val_loss.append(val_loss)\n",
    "    \n",
    "    print('Training Loss: {},  Validation Loss: {}'.format(train_loss, val_loss))\n",
    "    print('Time Taken for this Epoch : {} sec'.format(time.time()-start))   \n",
    "    model1.save_weights('Weights/Attention/OneStep/epoch_'+ str(epoch+1) + '.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def inference_concat(inputs):\n",
    "    \n",
    "    in_ = len(inputs.split()) - 1\n",
    "    inputs = Xnet_Features[inputs]\n",
    "    enc_state = tf.zeros((1, 256))\n",
    "    enc_output = model1.layers[0](inputs)\n",
    "    input_state = enc_state\n",
    "    pred = []\n",
    "    cur_vec = np.array([tokenizer.word_index['startseq']]).reshape(-1,1)\n",
    "\n",
    "    for i in range(153):\n",
    "\n",
    "        inf_output, input_state, attention_weights = model1.layers[1].onestep_decoder(cur_vec, input_state, enc_output)\n",
    "\n",
    "        cur_vec = np.reshape(np.argmax(inf_output), (1, 1))\n",
    "        if cur_vec[0][0] != 0:\n",
    "            pred.append(cur_vec)\n",
    "        else:\n",
    "            break\n",
    "\n",
    "    final = ' '.join([tokenizer.index_word[e[0][0]] for e in pred if e[0][0] != 0 and e[0][0] != 7])\n",
    "    return final#, att_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = inference_concat(X_cv.values[72])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'startseq the heart size and pulmonary vascularity appear within normal limits .  the lungs are free focal airspace disease .  no pleural effusion pneumothora seen .  no noncalcified nodules are identified .  endseq'"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_cv.values[72]  # original"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'the heart size and pulmonary vascularity appear within normal limits . the lungs are clear . no pleural effusion pneumothora . no acute bony abnormality .'"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a  # predicted"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The attention model is already giving decent outputs within just 10 epochs of training!!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### You can try with other examples."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:py3-TF2.0] *",
   "language": "python",
   "name": "conda-env-py3-TF2.0-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}