lstm_siamese_text_similarity.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "C:\\Users\\spider-online\\Anaconda3\\envs\\few_shot_learning\\lib\\site-packages\\gensim\\utils.py:1197: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from gensim.models import Word2Vec\n",
    "import numpy as np\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "# keras imports\n",
    "from keras.layers import Dense, Input, LSTM, Dropout, Bidirectional\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers.merge import concatenate\n",
    "from keras.callbacks import TensorBoard\n",
    "from keras.models import load_model\n",
    "from keras.models import Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from operator import itemgetter\n",
    "import time\n",
    "import gc\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "EMBEDDING_DIM = 50\n",
    "\n",
    "MAX_SEQUENCE_LENGTH = 10\n",
    "VALIDATION_SPLIT = 0.1\n",
    "\n",
    "\n",
    "RATE_DROP_LSTM = 0.17\n",
    "RATE_DROP_DENSE = 0.25\n",
    "NUMBER_LSTM = 50\n",
    "NUMBER_DENSE_UNITS = 50\n",
    "ACTIVATION_FUNCTION = 'relu'\n",
    "\n",
    "\n",
    "siamese_config = {\n",
    "    'EMBEDDING_DIM': EMBEDDING_DIM,\n",
    "    'MAX_SEQUENCE_LENGTH' : MAX_SEQUENCE_LENGTH,\n",
    "    'VALIDATION_SPLIT': VALIDATION_SPLIT,\n",
    "    'RATE_DROP_LSTM': RATE_DROP_LSTM,\n",
    "    'RATE_DROP_DENSE': RATE_DROP_DENSE,\n",
    "    'NUMBER_LSTM': NUMBER_LSTM,\n",
    "    'NUMBER_DENSE_UNITS': NUMBER_DENSE_UNITS,\n",
    "    'ACTIVATION_FUNCTION': ACTIVATION_FUNCTION\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_word2vec(documents, embedding_dim):\n",
    "    \"\"\"\n",
    "    train word2vector over traning documents\n",
    "    Args:\n",
    "        documents (list): list of document\n",
    "        embedding_dim (int): outpu wordvector size\n",
    "    Returns:\n",
    "        word_vectors(dict): dict containing words and their respective vectors\n",
    "    \"\"\"\n",
    "    model = Word2Vec(documents, min_count=1, size=embedding_dim)\n",
    "    word_vectors = model.wv\n",
    "    del model\n",
    "    return word_vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_embedding_matrix(tokenizer, word_vectors, embedding_dim):\n",
    "    \"\"\"\n",
    "    Create embedding matrix containing word indexes and respective vectors from word vectors\n",
    "    Args:\n",
    "        tokenizer (keras.preprocessing.text.Tokenizer): keras tokenizer object containing word indexes\n",
    "        word_vectors (dict): dict containing word and their respective vectors\n",
    "        embedding_dim (int): dimention of word vector\n",
    "    Returns:\n",
    "    \"\"\"\n",
    "    nb_words = len(tokenizer.word_index) + 1\n",
    "    word_index = tokenizer.word_index\n",
    "    embedding_matrix = np.zeros((nb_words, embedding_dim))\n",
    "    print(\"Embedding matrix shape: %s\" % str(embedding_matrix.shape))\n",
    "    for word, i in word_index.items():\n",
    "        try:\n",
    "            embedding_vector = word_vectors[word]\n",
    "            if embedding_vector is not None:\n",
    "                embedding_matrix[i] = embedding_vector\n",
    "        except KeyError:\n",
    "            print(\"vector not found for word - %s\" % word)\n",
    "    print('Null word embeddings: %d' % np.sum(np.sum(embedding_matrix, axis=1) == 0))\n",
    "    return embedding_matrix\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def word_embed_meta_data(documents, embedding_dim):\n",
    "    \"\"\"\n",
    "    Load tokenizer object for given vocabs list\n",
    "    Args:\n",
    "        documents (list): list of document\n",
    "        embedding_dim (int): embedding dimension\n",
    "    Returns:\n",
    "        tokenizer (keras.preprocessing.text.Tokenizer): keras tokenizer object\n",
    "        embedding_matrix (dict): dict with word_index and vector mapping\n",
    "    \"\"\"\n",
    "    documents = [x.lower().split() for x in documents]\n",
    "    tokenizer = Tokenizer()\n",
    "    tokenizer.fit_on_texts(documents)\n",
    "    word_vector = train_word2vec(documents, embedding_dim)\n",
    "    embedding_matrix = create_embedding_matrix(tokenizer, word_vector, embedding_dim)\n",
    "    del word_vector\n",
    "    gc.collect()\n",
    "    return tokenizer, embedding_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_train_dev_set(tokenizer, sentences_pair, is_similar, max_sequence_length, validation_split_ratio):\n",
    "    \"\"\"\n",
    "    Create training and validation dataset\n",
    "    Args:\n",
    "        tokenizer (keras.preprocessing.text.Tokenizer): keras tokenizer object\n",
    "        sentences_pair (list): list of tuple of sentences pairs\n",
    "        is_similar (list): list containing labels if respective sentences in sentence1 and sentence2\n",
    "                           are same or not (1 if same else 0)\n",
    "        max_sequence_length (int): max sequence length of sentences to apply padding\n",
    "        validation_split_ratio (float): contain ratio to split training data into validation data\n",
    "    Returns:\n",
    "        train_data_1 (list): list of input features for training set from sentences1\n",
    "        train_data_2 (list): list of input features for training set from sentences2\n",
    "        labels_train (np.array): array containing similarity score for training data\n",
    "        leaks_train(np.array): array of training leaks features\n",
    "        val_data_1 (list): list of input features for validation set from sentences1\n",
    "        val_data_2 (list): list of input features for validation set from sentences1\n",
    "        labels_val (np.array): array containing similarity score for validation data\n",
    "        leaks_val (np.array): array of validation leaks features\n",
    "    \"\"\"\n",
    "    sentences1 = [x[0].lower() for x in sentences_pair]\n",
    "    sentences2 = [x[1].lower() for x in sentences_pair]\n",
    "    train_sequences_1 = tokenizer.texts_to_sequences(sentences1)\n",
    "    train_sequences_2 = tokenizer.texts_to_sequences(sentences2)\n",
    "    leaks = [[len(set(x1)), len(set(x2)), len(set(x1).intersection(x2))]\n",
    "             for x1, x2 in zip(train_sequences_1, train_sequences_2)]\n",
    "\n",
    "    train_padded_data_1 = pad_sequences(train_sequences_1, maxlen=max_sequence_length)\n",
    "    train_padded_data_2 = pad_sequences(train_sequences_2, maxlen=max_sequence_length)\n",
    "    train_labels = np.array(is_similar)\n",
    "    leaks = np.array(leaks)\n",
    "\n",
    "    shuffle_indices = np.random.permutation(np.arange(len(train_labels)))\n",
    "    train_data_1_shuffled = train_padded_data_1[shuffle_indices]\n",
    "    train_data_2_shuffled = train_padded_data_2[shuffle_indices]\n",
    "    train_labels_shuffled = train_labels[shuffle_indices]\n",
    "    leaks_shuffled = leaks[shuffle_indices]\n",
    "\n",
    "    dev_idx = max(1, int(len(train_labels_shuffled) * validation_split_ratio))\n",
    "\n",
    "    del train_padded_data_1\n",
    "    del train_padded_data_2\n",
    "    gc.collect()\n",
    "\n",
    "    train_data_1, val_data_1 = train_data_1_shuffled[:-dev_idx], train_data_1_shuffled[-dev_idx:]\n",
    "    train_data_2, val_data_2 = train_data_2_shuffled[:-dev_idx], train_data_2_shuffled[-dev_idx:]\n",
    "    labels_train, labels_val = train_labels_shuffled[:-dev_idx], train_labels_shuffled[-dev_idx:]\n",
    "    leaks_train, leaks_val = leaks_shuffled[:-dev_idx], leaks_shuffled[-dev_idx:]\n",
    "\n",
    "    return train_data_1, train_data_2, labels_train, leaks_train, val_data_1, val_data_2, labels_val, leaks_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_test_data(tokenizer, test_sentences_pair, max_sequence_length):\n",
    "    \"\"\"\n",
    "    Create training and validation dataset\n",
    "    Args:\n",
    "        tokenizer (keras.preprocessing.text.Tokenizer): keras tokenizer object\n",
    "        test_sentences_pair (list): list of tuple of sentences pairs\n",
    "        max_sequence_length (int): max sequence length of sentences to apply padding\n",
    "    Returns:\n",
    "        test_data_1 (list): list of input features for training set from sentences1\n",
    "        test_data_2 (list): list of input features for training set from sentences2\n",
    "    \"\"\"\n",
    "    test_sentences1 = [x[0].lower() for x in test_sentences_pair]\n",
    "    test_sentences2 = [x[1].lower() for x in test_sentences_pair]\n",
    "\n",
    "    test_sequences_1 = tokenizer.texts_to_sequences(test_sentences1)\n",
    "    test_sequences_2 = tokenizer.texts_to_sequences(test_sentences2)\n",
    "    leaks_test = [[len(set(x1)), len(set(x2)), len(set(x1).intersection(x2))]\n",
    "                  for x1, x2 in zip(test_sequences_1, test_sequences_2)]\n",
    "\n",
    "    leaks_test = np.array(leaks_test)\n",
    "    test_data_1 = pad_sequences(test_sequences_1, maxlen=max_sequence_length)\n",
    "    test_data_2 = pad_sequences(test_sequences_2, maxlen=max_sequence_length)\n",
    "\n",
    "    return test_data_1, test_data_2, leaks_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SiameseBiLSTM:\n",
    "    def __init__(self, embedding_dim, max_sequence_length, number_lstm, number_dense, rate_drop_lstm, \n",
    "                 rate_drop_dense, hidden_activation, validation_split_ratio):\n",
    "        self.embedding_dim = embedding_dim\n",
    "        self.max_sequence_length = max_sequence_length\n",
    "        self.number_lstm_units = number_lstm\n",
    "        self.rate_drop_lstm = rate_drop_lstm\n",
    "        self.number_dense_units = number_dense\n",
    "        self.activation_function = hidden_activation\n",
    "        self.rate_drop_dense = rate_drop_dense\n",
    "        self.validation_split_ratio = validation_split_ratio\n",
    "\n",
    "    def train_model(self, sentences_pair, is_similar, embedding_meta_data, model_save_directory='./'):\n",
    "        \"\"\"\n",
    "        Train Siamese network to find similarity between sentences in `sentences_pair`\n",
    "            Steps Involved:\n",
    "                1. Pass the each from sentences_pairs  to bidirectional LSTM encoder.\n",
    "                2. Merge the vectors from LSTM encodes and passed to dense layer.\n",
    "                3. Pass the  dense layer vectors to sigmoid output layer.\n",
    "                4. Use cross entropy loss to train weights\n",
    "        Args:\n",
    "            sentences_pair (list): list of tuple of sentence pairs\n",
    "            is_similar (list): target value 1 if same sentences pair are similar otherwise 0\n",
    "            embedding_meta_data (dict): dict containing tokenizer and word embedding matrix\n",
    "            model_save_directory (str): working directory for where to save models\n",
    "        Returns:\n",
    "            return (best_model_path):  path of best model\n",
    "        \"\"\"\n",
    "        tokenizer, embedding_matrix = embedding_meta_data['tokenizer'], embedding_meta_data['embedding_matrix']\n",
    "\n",
    "        train_data_x1, train_data_x2, train_labels, leaks_train, \\\n",
    "        val_data_x1, val_data_x2, val_labels, leaks_val = create_train_dev_set(tokenizer, sentences_pair,\n",
    "                                                                               is_similar, self.max_sequence_length,\n",
    "                                                                               self.validation_split_ratio)\n",
    "\n",
    "        if train_data_x1 is None:\n",
    "            print(\"++++ !! Failure: Unable to train model ++++\")\n",
    "            return None\n",
    "\n",
    "        nb_words = len(tokenizer.word_index) + 1\n",
    "\n",
    "        # Creating word embedding layer\n",
    "        embedding_layer = Embedding(nb_words, self.embedding_dim, weights=[embedding_matrix],\n",
    "                                    input_length=self.max_sequence_length, trainable=False)\n",
    "\n",
    "        # Creating LSTM Encoder\n",
    "        lstm_layer = Bidirectional(LSTM(self.number_lstm_units, dropout=self.rate_drop_lstm, recurrent_dropout=self.rate_drop_lstm))\n",
    "\n",
    "        # Creating LSTM Encoder layer for First Sentence\n",
    "        sequence_1_input = Input(shape=(self.max_sequence_length,), dtype='int32')\n",
    "        embedded_sequences_1 = embedding_layer(sequence_1_input)\n",
    "        x1 = lstm_layer(embedded_sequences_1)\n",
    "\n",
    "        # Creating LSTM Encoder layer for Second Sentence\n",
    "        sequence_2_input = Input(shape=(self.max_sequence_length,), dtype='int32')\n",
    "        embedded_sequences_2 = embedding_layer(sequence_2_input)\n",
    "        x2 = lstm_layer(embedded_sequences_2)\n",
    "\n",
    "        # Creating leaks input\n",
    "        leaks_input = Input(shape=(leaks_train.shape[1],))\n",
    "        leaks_dense = Dense(int(self.number_dense_units/2), activation=self.activation_function)(leaks_input)\n",
    "\n",
    "        # Merging two LSTM encodes vectors from sentences to\n",
    "        # pass it to dense layer applying dropout and batch normalisation\n",
    "        merged = concatenate([x1, x2, leaks_dense])\n",
    "        merged = BatchNormalization()(merged)\n",
    "        merged = Dropout(self.rate_drop_dense)(merged)\n",
    "        merged = Dense(self.number_dense_units, activation=self.activation_function)(merged)\n",
    "        merged = BatchNormalization()(merged)\n",
    "        merged = Dropout(self.rate_drop_dense)(merged)\n",
    "        preds = Dense(1, activation='sigmoid')(merged)\n",
    "\n",
    "        model = Model(inputs=[sequence_1_input, sequence_2_input, leaks_input], outputs=preds)\n",
    "        model.compile(loss='binary_crossentropy', optimizer='nadam', metrics=['acc'])\n",
    "\n",
    "        early_stopping = EarlyStopping(monitor='val_loss', patience=3)\n",
    "\n",
    "        STAMP = 'lstm_%d_%d_%.2f_%.2f' % (self.number_lstm_units, self.number_dense_units, self.rate_drop_lstm, self.rate_drop_dense)\n",
    "\n",
    "        checkpoint_dir = model_save_directory + 'checkpoints/' + str(int(time.time())) + '/'\n",
    "\n",
    "        if not os.path.exists(checkpoint_dir):\n",
    "            os.makedirs(checkpoint_dir)\n",
    "\n",
    "        bst_model_path = checkpoint_dir + STAMP + '.h5'\n",
    "\n",
    "        model_checkpoint = ModelCheckpoint(bst_model_path, save_best_only=True, save_weights_only=False)\n",
    "\n",
    "        tensorboard = TensorBoard(log_dir=checkpoint_dir + \"logs/{}\".format(time.time()))\n",
    "\n",
    "        model.fit([train_data_x1, train_data_x2, leaks_train], train_labels,\n",
    "                  validation_data=([val_data_x1, val_data_x2, leaks_val], val_labels),\n",
    "                  epochs=50, batch_size=64, shuffle=True,\n",
    "                  callbacks=[early_stopping, model_checkpoint, tensorboard])\n",
    "\n",
    "        return bst_model_path\n",
    "\n",
    "\n",
    "    def update_model(self, saved_model_path, new_sentences_pair, is_similar, embedding_meta_data):\n",
    "        \"\"\"\n",
    "        Update trained siamese model for given new sentences pairs \n",
    "            Steps Involved:\n",
    "                1. Pass the each from sentences from new_sentences_pair to bidirectional LSTM encoder.\n",
    "                2. Merge the vectors from LSTM encodes and passed to dense layer.\n",
    "                3. Pass the  dense layer vectors to sigmoid output layer.\n",
    "                4. Use cross entropy loss to train weights\n",
    "        Args:\n",
    "            model_path (str): model path of already trained siamese model\n",
    "            new_sentences_pair (list): list of tuple of new sentences pairs\n",
    "            is_similar (list): target value 1 if same sentences pair are similar otherwise 0\n",
    "            embedding_meta_data (dict): dict containing tokenizer and word embedding matrix\n",
    "        Returns:\n",
    "            return (best_model_path):  path of best model\n",
    "        \"\"\"\n",
    "        tokenizer = embedding_meta_data['tokenizer']\n",
    "        train_data_x1, train_data_x2, train_labels, leaks_train, \\\n",
    "        val_data_x1, val_data_x2, val_labels, leaks_val = create_train_dev_set(tokenizer, new_sentences_pair,\n",
    "                                                                               is_similar, self.max_sequence_length,\n",
    "                                                                               self.validation_split_ratio)\n",
    "        model = load_model(saved_model_path)\n",
    "        model_file_name = saved_model_path.split('/')[-1]\n",
    "        new_model_checkpoint_path  = saved_model_path.split('/')[:-2] + str(int(time.time())) + '/' \n",
    "\n",
    "        new_model_path = new_model_checkpoint_path + model_file_name\n",
    "        model_checkpoint = ModelCheckpoint(new_model_checkpoint_path + model_file_name,\n",
    "                                           save_best_only=True, save_weights_only=False)\n",
    "\n",
    "        early_stopping = EarlyStopping(monitor='val_loss', patience=3)\n",
    "\n",
    "        tensorboard = TensorBoard(log_dir=new_model_checkpoint_path + \"logs/{}\".format(time.time()))\n",
    "\n",
    "        model.fit([train_data_x1, train_data_x2, leaks_train], train_labels,\n",
    "                  validation_data=([val_data_x1, val_data_x2, leaks_val], val_labels),\n",
    "                  epochs=50, batch_size=3, shuffle=True,\n",
    "                  callbacks=[early_stopping, model_checkpoint, tensorboard])\n",
    "\n",
    "        return new_model_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>What is the story of Kohinoor (Koh-i-Noor) Dia...</td>\n",
       "      <td>What would happen if the Indian government sto...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>How can I increase the speed of my internet co...</td>\n",
       "      <td>How can Internet speed be increased by hacking...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>Why am I mentally very lonely? How can I solve...</td>\n",
       "      <td>Find the remainder when [math]23^{24}[/math] i...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>Which one dissolve in water quikly sugar, salt...</td>\n",
       "      <td>Which fish would survive in salt water?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  qid1  qid2                                          question1  \\\n",
       "0   0     1     2  What is the step by step guide to invest in sh...   \n",
       "1   1     3     4  What is the story of Kohinoor (Koh-i-Noor) Dia...   \n",
       "2   2     5     6  How can I increase the speed of my internet co...   \n",
       "3   3     7     8  Why am I mentally very lonely? How can I solve...   \n",
       "4   4     9    10  Which one dissolve in water quikly sugar, salt...   \n",
       "\n",
       "                                           question2  is_duplicate  \n",
       "0  What is the step by step guide to invest in sh...             0  \n",
       "1  What would happen if the Indian government sto...             0  \n",
       "2  How can Internet speed be increased by hacking...             0  \n",
       "3  Find the remainder when [math]23^{24}[/math] i...             0  \n",
       "4            Which fish would survive in salt water?             0  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('train.csv')\n",
    "df.dropna(axis=0, inplace=True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sentences1 = list(df['sentences1'])\n",
    "# sentences2 = list(df['sentences2'])\n",
    "# is_similar = list(df['is_similar'])\n",
    "question1 = list(df['question1'])\n",
    "question2 = list(df['question2'])\n",
    "is_duplicate = list(df['is_duplicate'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Embedding matrix shape: (201100, 50)\n",
      "Null word embeddings: 1\n"
     ]
    }
   ],
   "source": [
    "# creating word embedding meta data for word embedding \n",
    "tokenizer, embedding_matrix = word_embed_meta_data(question1 + question2,  siamese_config['EMBEDDING_DIM'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_meta_data = {\n",
    "    'tokenizer': tokenizer,\n",
    "    'embedding_matrix': embedding_matrix\n",
    "}\n",
    "## creating sentence pairs\n",
    "sentences_pair = [(x1, x2) for x1, x2 in zip(question1, question2)]\n",
    "del question1\n",
    "del question2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Configuration(object):\n",
    "    \"\"\"Dump stuff here\"\"\"\n",
    "\n",
    "CONFIG = Configuration()\n",
    "CONFIG.embedding_dim = siamese_config['EMBEDDING_DIM']\n",
    "CONFIG.max_sequence_length = siamese_config['MAX_SEQUENCE_LENGTH']\n",
    "CONFIG.number_lstm_units = siamese_config['NUMBER_LSTM']\n",
    "CONFIG.rate_drop_lstm = siamese_config['RATE_DROP_LSTM']\n",
    "CONFIG.number_dense_units = siamese_config['NUMBER_DENSE_UNITS']\n",
    "CONFIG.activation_function = siamese_config['ACTIVATION_FUNCTION']\n",
    "CONFIG.rate_drop_dense = siamese_config['RATE_DROP_DENSE']\n",
    "CONFIG.validation_split_ratio = siamese_config['VALIDATION_SPLIT']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 363859 samples, validate on 40428 samples\n",
      "Epoch 1/50\n",
      "363859/363859 [==============================] - 535s 1ms/step - loss: 0.4755 - acc: 0.7490 - val_loss: 0.4395 - val_acc: 0.7707\n",
      "Epoch 2/50\n",
      "363859/363859 [==============================] - 415s 1ms/step - loss: 0.4478 - acc: 0.7684 - val_loss: 0.4272 - val_acc: 0.7804\n",
      "Epoch 3/50\n",
      "363859/363859 [==============================] - 465s 1ms/step - loss: 0.4410 - acc: 0.7744 - val_loss: 0.4228 - val_acc: 0.7834\n",
      "Epoch 4/50\n",
      "363859/363859 [==============================] - 514s 1ms/step - loss: 0.4363 - acc: 0.7773 - val_loss: 0.4156 - val_acc: 0.7905\n",
      "Epoch 5/50\n",
      "363859/363859 [==============================] - 500s 1ms/step - loss: 0.4337 - acc: 0.7790 - val_loss: 0.4134 - val_acc: 0.7897\n",
      "Epoch 6/50\n",
      "363859/363859 [==============================] - 482s 1ms/step - loss: 0.4310 - acc: 0.7815 - val_loss: 0.4106 - val_acc: 0.7952\n",
      "Epoch 7/50\n",
      "363859/363859 [==============================] - 420s 1ms/step - loss: 0.4291 - acc: 0.7825 - val_loss: 0.4096 - val_acc: 0.7971\n",
      "Epoch 8/50\n",
      "363859/363859 [==============================] - 427s 1ms/step - loss: 0.4284 - acc: 0.7822 - val_loss: 0.4058 - val_acc: 0.7970\n",
      "Epoch 9/50\n",
      "363859/363859 [==============================] - 428s 1ms/step - loss: 0.4275 - acc: 0.7839 - val_loss: 0.4076 - val_acc: 0.7993\n",
      "Epoch 10/50\n",
      "363859/363859 [==============================] - 388s 1ms/step - loss: 0.4261 - acc: 0.7857 - val_loss: 0.4058 - val_acc: 0.7982\n",
      "Epoch 11/50\n",
      "363859/363859 [==============================] - 380s 1ms/step - loss: 0.4257 - acc: 0.7857 - val_loss: 0.4054 - val_acc: 0.7981\n",
      "Epoch 12/50\n",
      "363859/363859 [==============================] - 379s 1ms/step - loss: 0.4244 - acc: 0.7865 - val_loss: 0.4066 - val_acc: 0.7955\n",
      "Epoch 13/50\n",
      "363859/363859 [==============================] - 376s 1ms/step - loss: 0.4245 - acc: 0.7863 - val_loss: 0.4054 - val_acc: 0.7987\n",
      "Epoch 14/50\n",
      "363859/363859 [==============================] - 385s 1ms/step - loss: 0.4240 - acc: 0.7864 - val_loss: 0.4026 - val_acc: 0.7997\n",
      "Epoch 15/50\n",
      "363859/363859 [==============================] - 381s 1ms/step - loss: 0.4238 - acc: 0.7866 - val_loss: 0.4051 - val_acc: 0.7988\n",
      "Epoch 16/50\n",
      "363859/363859 [==============================] - 491s 1ms/step - loss: 0.4228 - acc: 0.7876 - val_loss: 0.4023 - val_acc: 0.7985\n",
      "Epoch 17/50\n",
      "363859/363859 [==============================] - 507s 1ms/step - loss: 0.4221 - acc: 0.7886 - val_loss: 0.4018 - val_acc: 0.8007\n",
      "Epoch 18/50\n",
      "363859/363859 [==============================] - 478s 1ms/step - loss: 0.4219 - acc: 0.7881 - val_loss: 0.4015 - val_acc: 0.8017\n",
      "Epoch 19/50\n",
      "363859/363859 [==============================] - 537s 1ms/step - loss: 0.4212 - acc: 0.7888 - val_loss: 0.4023 - val_acc: 0.7985\n",
      "Epoch 20/50\n",
      "363859/363859 [==============================] - 524s 1ms/step - loss: 0.4217 - acc: 0.7886 - val_loss: 0.4035 - val_acc: 0.8010\n",
      "Epoch 21/50\n",
      "363859/363859 [==============================] - 561s 2ms/step - loss: 0.4214 - acc: 0.7897 - val_loss: 0.4013 - val_acc: 0.8010\n",
      "Epoch 22/50\n",
      "363859/363859 [==============================] - 514s 1ms/step - loss: 0.4212 - acc: 0.7886 - val_loss: 0.4011 - val_acc: 0.8016\n",
      "Epoch 23/50\n",
      "363859/363859 [==============================] - 499s 1ms/step - loss: 0.4208 - acc: 0.7887 - val_loss: 0.4015 - val_acc: 0.8005\n",
      "Epoch 24/50\n",
      "363859/363859 [==============================] - 489s 1ms/step - loss: 0.4211 - acc: 0.7890 - val_loss: 0.3995 - val_acc: 0.8035\n",
      "Epoch 25/50\n",
      "363859/363859 [==============================] - 470s 1ms/step - loss: 0.4210 - acc: 0.7895 - val_loss: 0.4002 - val_acc: 0.8028\n",
      "Epoch 26/50\n",
      "363859/363859 [==============================] - 448s 1ms/step - loss: 0.4202 - acc: 0.7889 - val_loss: 0.4018 - val_acc: 0.8018\n",
      "Epoch 27/50\n",
      "363859/363859 [==============================] - 423s 1ms/step - loss: 0.4202 - acc: 0.7901 - val_loss: 0.3996 - val_acc: 0.8045\n"
     ]
    }
   ],
   "source": [
    "siamese = SiameseBiLSTM(CONFIG.embedding_dim , CONFIG.max_sequence_length, CONFIG.number_lstm_units , CONFIG.number_dense_units, CONFIG.rate_drop_lstm, CONFIG.rate_drop_dense, CONFIG.activation_function, CONFIG.validation_split_ratio)\n",
    "\n",
    "best_model_path = siamese.train_model(sentences_pair, is_duplicate, embedding_meta_data, model_save_directory='./')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3/3 [==============================] - 47s 16s/step\n",
      "[('What can make Physics easy to learn?', 'How can you make physics easy to learn?', 0.6940281), ('I am software engineering', 'the boy in french', 0.005807623), ('How many times a day do a clocks hands overlap?', 'What does it mean that every time I look at the clock the numbers are the same?', 0.00017503016)]\n"
     ]
    }
   ],
   "source": [
    "model = load_model(best_model_path)\n",
    "\n",
    "test_sentence_pairs = [('What can make Physics easy to learn?','How can you make physics easy to learn?'),('How many times a day do a clocks hands overlap?','What does it mean that every time I look at the clock the numbers are the same?'),('I am software engineering','the boy in french')]\n",
    "\n",
    "test_data_x1, test_data_x2, leaks_test = create_test_data(tokenizer,test_sentence_pairs,  siamese_config['MAX_SEQUENCE_LENGTH'])\n",
    "\n",
    "preds = list(model.predict([test_data_x1, test_data_x2, leaks_test], verbose=1).ravel())\n",
    "results = [(x, y, z) for (x, y), z in zip(test_sentence_pairs, preds)]\n",
    "results.sort(key=itemgetter(2), reverse=True)\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}