5.TEXT_SIM/5.3.2.Quora_CNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Parameter & Directory setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_IN_PATH = './data_in/'\n",
    "DATA_OUT_PATH = './data_out/'\n",
    "\n",
    "TRAIN_Q1_DATA_FILE = 'train_q1.npy'\n",
    "TRAIN_Q2_DATA_FILE = 'train_q2.npy'\n",
    "TRAIN_LABEL_DATA_FILE = 'train_label.npy'\n",
    "DATA_CONFIGS = 'data_configs.json'\n",
    "\n",
    "TEST_SPLIT = 0.1\n",
    "RNG_SEED = 13371447"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "EPOCH=1\n",
    "BATCH_SIZE=1024\n",
    "\n",
    "MAX_SEQUENCE_LENGTH = 31\n",
    "\n",
    "WORD_EMBEDDING_DIM = 100\n",
    "CONV_FEATURE_DIM = 300\n",
    "CONV_OUTPUT_DIM = 128\n",
    "CONV_WINDOW_SIZE = 3\n",
    "SIMILARITY_DENSE_FEATURE_DIM = 200\n",
    "\n",
    "prepro_configs = None\n",
    "\n",
    "with open(DATA_IN_PATH + DATA_CONFIGS, 'r') as f:\n",
    "    prepro_configs = json.load(f)\n",
    "    \n",
    "VOCAB_SIZE = prepro_configs['vocab_size']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "q1_data = np.load(open(DATA_IN_PATH + TRAIN_Q1_DATA_FILE, 'rb'))\n",
    "q2_data = np.load(open(DATA_IN_PATH + TRAIN_Q2_DATA_FILE, 'rb'))\n",
    "labels = np.load(open(DATA_IN_PATH + TRAIN_LABEL_DATA_FILE, 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.stack((q1_data, q2_data), axis=1)\n",
    "y = labels\n",
    "train_X, eval_X, train_y, eval_y = train_test_split(X, y, test_size=TEST_SPLIT, random_state=RNG_SEED)\n",
    "\n",
    "train_Q1 = train_X[:,0]\n",
    "train_Q2 = train_X[:,1]\n",
    "eval_Q1 = eval_X[:,0]\n",
    "eval_Q2 = eval_X[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rearrange(base, hypothesis, label):\n",
    "    features = {\"x1\": base, \"x2\": hypothesis}\n",
    "    return features, label\n",
    "\n",
    "def train_input_fn():\n",
    "    dataset = tf.data.Dataset.from_tensor_slices((train_Q1, train_Q2, train_y))\n",
    "    dataset = dataset.shuffle(buffer_size=10000)\n",
    "    dataset = dataset.batch(BATCH_SIZE)\n",
    "    dataset = dataset.map(rearrange)\n",
    "    dataset = dataset.repeat(EPOCH)\n",
    "    iterator = dataset.make_one_shot_iterator()\n",
    "    \n",
    "    return iterator.get_next()\n",
    "\n",
    "def eval_input_fn():\n",
    "    dataset = tf.data.Dataset.from_tensor_slices((eval_Q1, eval_Q2, eval_y))\n",
    "    dataset = dataset.shuffle(buffer_size=10000)\n",
    "    dataset = dataset.batch(BATCH_SIZE)\n",
    "    dataset = dataset.map(rearrange)\n",
    "    iterator = dataset.make_one_shot_iterator()\n",
    "    \n",
    "    return iterator.get_next()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def basic_conv_sementic_network(inputs, name):\n",
    "    conv_layer = tf.keras.layers.Conv1D(CONV_FEATURE_DIM, \n",
    "                                        CONV_WINDOW_SIZE, \n",
    "                                        activation=tf.nn.relu, \n",
    "                                        name=name + 'conv_1d',\n",
    "                                        padding='same')(inputs)\n",
    "    \n",
    "    max_pool_layer = tf.keras.layers.MaxPool1D(MAX_SEQUENCE_LENGTH, \n",
    "                                               1)(conv_layer)\n",
    "\n",
    "    output_layer = tf.keras.layers.Dense(CONV_OUTPUT_DIM, \n",
    "                                         activation=tf.nn.relu,\n",
    "                                         name=name + 'dense')(max_pool_layer)\n",
    "    output_layer = tf.squeeze(output_layer, 1)\n",
    "    \n",
    "    return output_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_fn(features, labels, mode):\n",
    "    TRAIN = mode == tf.estimator.ModeKeys.TRAIN\n",
    "    EVAL = mode == tf.estimator.ModeKeys.EVAL\n",
    "    PREDICT = mode == tf.estimator.ModeKeys.PREDICT\n",
    "    \n",
    "    embedding = tf.keras.layers.Embedding(VOCAB_SIZE,\n",
    "                                          WORD_EMBEDDING_DIM)\n",
    "    \n",
    "    base_embedded_matrix = embedding(features['x1'])\n",
    "    hypothesis_embedded_matrix = embedding(features['x2'])\n",
    "    \n",
    "    base_embedded_matrix = tf.keras.layers.Dropout(0.2)(base_embedded_matrix)\n",
    "    hypothesis_embedded_matrix = tf.keras.layers.Dropout(0.2)(hypothesis_embedded_matrix)  \n",
    "    \n",
    "    base_sementic_matrix = basic_conv_sementic_network(base_embedded_matrix, 'base')\n",
    "    hypothesis_sementic_matrix = basic_conv_sementic_network(hypothesis_embedded_matrix, 'hypothesis')  \n",
    "    \n",
    "    merged_matrix = tf.concat([base_sementic_matrix, hypothesis_sementic_matrix], -1)\n",
    "\n",
    "    similarity_dense_layer = tf.keras.layers.Dense(SIMILARITY_DENSE_FEATURE_DIM,\n",
    "                                             activation=tf.nn.relu)(merged_matrix)\n",
    "    \n",
    "    similarity_dense_layer = tf.keras.layers.Dropout(0.2)(similarity_dense_layer)    \n",
    "    logit_layer = tf.keras.layers.Dense(1)(similarity_dense_layer)\n",
    "    logit_layer = tf.squeeze(logit_layer, 1)\n",
    "    similarity = tf.nn.sigmoid(logit_layer)\n",
    "    \n",
    "    if PREDICT:\n",
    "        return tf.estimator.EstimatorSpec(\n",
    "                  mode=mode,\n",
    "                  predictions={\n",
    "                      'is_duplicate':similarity\n",
    "                  })\n",
    "    \n",
    "    loss = tf.losses.sigmoid_cross_entropy(labels, logit_layer)\n",
    "\n",
    "    if EVAL:\n",
    "        accuracy = tf.metrics.accuracy(labels, tf.round(similarity))\n",
    "        return tf.estimator.EstimatorSpec(\n",
    "                  mode=mode,\n",
    "                  eval_metric_ops= {'acc': accuracy},\n",
    "                  loss=loss)\n",
    "    \n",
    "    if TRAIN:\n",
    "        global_step = tf.train.get_global_step()\n",
    "        train_op = tf.train.AdamOptimizer(1e-3).minimize(loss, global_step)\n",
    "\n",
    "        return tf.estimator.EstimatorSpec(\n",
    "                  mode=mode,\n",
    "                  train_op=train_op,\n",
    "                  loss=loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Start training & Eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_model_dir': '/Users/sinseongjin/github/tensorflow-ml-nlp/5.TEXT_SIM/./data_out/checkpoint/cnn/', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x10f70fd30>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n"
     ]
    }
   ],
   "source": [
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"6\" #For TEST\n",
    "\n",
    "model_dir = os.path.join(os.getcwd(), DATA_OUT_PATH + \"checkpoint/cnn/\")\n",
    "os.makedirs(model_dir, exist_ok=True)\n",
    "\n",
    "est = tf.estimator.Estimator(model_fn, model_dir=model_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into /Users/sinseongjin/github/tensorflow-ml-nlp/5.TEXT_SIM/./data_out/checkpoint/cnn/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.69333047, step = 1\n",
      "INFO:tensorflow:global_step/sec: 1.60758\n",
      "INFO:tensorflow:loss = 0.54390585, step = 101 (62.208 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.52619\n",
      "INFO:tensorflow:loss = 0.52071404, step = 201 (65.519 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 263 into /Users/sinseongjin/github/tensorflow-ml-nlp/5.TEXT_SIM/./data_out/checkpoint/cnn/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.47354826.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.estimator.estimator.Estimator at 0x10f70f7f0>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "est.train(train_input_fn) #train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-01-20-02:25:27\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /Users/sinseongjin/github/tensorflow-ml-nlp/5.TEXT_SIM/./data_out/checkpoint/cnn/model.ckpt-263\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-01-20-02:25:33\n",
      "INFO:tensorflow:Saving dict for global step 263: acc = 0.75828224, global_step = 263, loss = 0.495392\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 263: /Users/sinseongjin/github/tensorflow-ml-nlp/5.TEXT_SIM/./data_out/checkpoint/cnn/model.ckpt-263\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'acc': 0.75828224, 'loss': 0.495392, 'global_step': 263}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "est.evaluate(eval_input_fn) #eval"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load test dataset & create submit dataset to kaggle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEST_Q1_DATA_FILE = 'test_q1.npy'\n",
    "TEST_Q2_DATA_FILE = 'test_q2.npy'\n",
    "TEST_ID_DATA_FILE = 'test_id.npy'\n",
    "\n",
    "test_q1_data = np.load(open(DATA_IN_PATH + TEST_Q1_DATA_FILE, 'rb'))\n",
    "test_q2_data = np.load(open(DATA_IN_PATH + TEST_Q2_DATA_FILE, 'rb'))\n",
    "test_id_data = np.load(open(DATA_IN_PATH + TEST_ID_DATA_FILE, 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /Users/sinseongjin/github/tensorflow-ml-nlp/5.TEXT_SIM/./data_out/checkpoint/cnn/model.ckpt-263\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "2345796\n"
     ]
    }
   ],
   "source": [
    "predict_input_fn = tf.estimator.inputs.numpy_input_fn(x={\"x1\":test_q1_data,\n",
    "                                                         \"x2\":test_q2_data},\n",
    "                                                      shuffle=False)\n",
    "\n",
    "predictions = np.array([p['is_duplicate'] for p in est.predict(input_fn=predict_input_fn)])\n",
    "\n",
    "output = pd.DataFrame( data={\"test_id\":test_id_data, \"is_duplicate\": list(predictions)} )\n",
    "output.to_csv(\"cnn_predict.csv\", index=False, quoting=3)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}