pp_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Colorful Extended Cleanup World (CECW)\n",
    "The Colorful Extended Cleanup World (CECW) dataset is a color-extended version of the Cleanup World (CW) borrowed from the mobile-manipulation robot domain [(MacGlashan et al., 2015)](http://cs.brown.edu/~jmacglashan/pubpdfs/rss_commands.pdf). CW refers to a world equipped with a movable object as well as four rooms in four colors, including \"blue,\" \"green,\" \"red,\" and \"yellow,\" which is designed as a simulation environment where the agent can act based on the instructions received [(Gopalan et al., 2018)](http://roboticsproceedings.org/rss14/p67.pdf)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Preprocessing \n",
    "This notebook shows how we make use of CECW to investigate synonymous generalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dependency\n",
    "import os\n",
    "# import json\n",
    "import random\n",
    "# import numpy as np\n",
    "from collections import Counter\n",
    "from helper_functions import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# path definition\n",
    "RAW_TRAIN_SRC_PATH = 'train_src.txt' # souce data for training\n",
    "RAW_TRAIN_TAR_PATH = 'train_tar.txt' # target data for training\n",
    "RAW_TEST_SRC_PATH = 'test_src.txt' # source data for testing\n",
    "RAW_TEST_TAR_PATH = 'test_tar.txt' # target data for testing\n",
    "\n",
    "PP_DATA_PATH = 'data_dict.json' # to save output data dictionary\n",
    "VOCAB_PATH = 'vocab_dict.json' # to save output vocabulary dictionary\n",
    "\n",
    "B_DATA_PATH = os.path.join('B', PP_DATA_PATH) # with entity for B\n",
    "B_VOCAB_PATH = os.path.join('B', VOCAB_PATH)\n",
    "BC_DATA_PATH = os.path.join('BC', PP_DATA_PATH) # with entity for B and C\n",
    "BC_VOCAB_PATH = os.path.join('BC', VOCAB_PATH)\n",
    "BCR_DATA_PATH = os.path.join('BCR', PP_DATA_PATH) # with entity for B, C and R\n",
    "BCR_VOCAB_PATH = os.path.join('BCR', VOCAB_PATH)\n",
    "BCRY_DATA_PATH = os.path.join('BCRY', PP_DATA_PATH) # with entity for B, C, R and Y\n",
    "BCRY_VOCAB_PATH = os.path.join('BCRY', VOCAB_PATH)\n",
    "Colorless_DATA_PATH = os.path.join('Colorless', PP_DATA_PATH) # with no entity\n",
    "Colorless_VOCAB_PATH = os.path.join('Colorless', VOCAB_PATH)\n",
    "ALL_DATA_PATH = os.path.join('ALL', PP_DATA_PATH) # original dataset\n",
    "ALL_VOCAB_PATH = os.path.join('ALL', VOCAB_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "raw training size: 8922\n",
      "raw test size: 2231\n"
     ]
    }
   ],
   "source": [
    "raw_train_src_list = read_txt(RAW_TRAIN_SRC_PATH) # training source\n",
    "raw_train_tar_list = read_txt(RAW_TRAIN_TAR_PATH) # training target\n",
    "raw_test_src_list = read_txt(RAW_TEST_SRC_PATH) # test source\n",
    "raw_test_tar_list = read_txt(RAW_TEST_TAR_PATH) # test target\n",
    "\n",
    "raw_train_size = len(raw_train_src_list)\n",
    "raw_test_size = len(raw_test_src_list)\n",
    "print('raw training size:', raw_train_size)\n",
    "print('raw test size:', raw_test_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# white space tokenization\n",
    "tk_train_src_list = [seq.split() for seq in raw_train_src_list]\n",
    "tk_train_tar_list = [seq.split() for seq in raw_train_tar_list]\n",
    "tk_test_src_list = [seq.split() for seq in raw_test_src_list]\n",
    "tk_test_tar_list = [seq.split() for seq in raw_test_tar_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CECW training command: ['go', 'to', 'orange', 'room', 'push', 'yellow', 'thing', 'into', 'green', 'room']\n",
      "CECW training expression: ['F', '&', 'R', 'F', 'Z']\n",
      "CECW test command: ['go', 'to', 'the', 'yellow', 'room', 'but', 'avoid', 'the', 'blue', 'room']\n",
      "CECW test expression: ['&', 'F', 'Y', 'G', '!', 'B']\n"
     ]
    }
   ],
   "source": [
    "# take a look\n",
    "index = random.choice(range(raw_train_size))\n",
    "print('CECW training command:', tk_train_src_list[index])\n",
    "print('CECW training expression:', tk_train_tar_list[index])\n",
    "index = random.choice(range(raw_test_size))\n",
    "print('CECW test command:', tk_test_src_list[index])\n",
    "print('CECW test expression:', tk_test_tar_list[index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# token frequency dictionary\n",
    "# for source side\n",
    "src_c = Counter()\n",
    "for train_src in tk_train_src_list:\n",
    "    src_c.update(train_src)\n",
    "for test_src in tk_test_src_list:\n",
    "    src_c.update(test_src)\n",
    "# for target side\n",
    "tar_c = Counter()\n",
    "for train_tar in tk_train_tar_list:\n",
    "    tar_c.update(train_tar)\n",
    "for test_tar in tk_test_tar_list:\n",
    "    tar_c.update(test_tar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CECW souce vocab size: 193\n",
      "CECW souce vocab frequency\n",
      "[('the', 19623), ('room', 18760), ('to', 11655), ('go', 6898), ('through', 5429), ('blue', 4226), ('green', 4148), ('red', 3920), ('yellow', 3502), ('move', 2565), ('and', 2309), ('or', 2004), ('into', 1934), ('get', 1752), ('enter', 1632), ('then', 1447), ('purple', 1284), ('pink', 1265), ('navy', 1254), ('olive', 1213), ('lime', 1213), ('orange', 1157), ('not', 1138), ('brown', 976), ('tan', 976), ('avoiding', 963), ('but', 881), ('that', 820), ('by', 724), ('reach', 681), ('without', 675), ('going', 609), ('area', 575), ('avoid', 565), ('chair', 540), ('robot', 535), ('while', 527), ('t', 515), ('box', 514), ('only', 505), ('rooms', 452), ('is', 445), ('do', 431), ('from', 427), ('pass', 384), ('a', 368), ('push', 358), ('isn', 335), ('large', 316), ('either', 310), ('travel', 305), ('via', 277), ('way', 272), ('first', 268), ('towards', 265), ('entering', 262), ('in', 254), ('you', 239), ('square', 235), ('small', 216), ('object', 216), ('using', 208), ('proceed', 206), ('up', 205), ('passing', 200), ('on', 199), ('it', 198), ('don', 180), ('of', 175), ('walk', 175), ('path', 162), ('which', 162), ('one', 162), ('take', 158), ('door', 155), ('rectangle', 155), ('thru', 150), ('any', 150), ('back', 141), ('rectangular', 140), ('colour', 136), ('goes', 125), ('until', 115), ('your', 112), ('toward', 106), ('straight', 104), ('continue', 103), ('are', 102), ('avoids', 100), ('navigate', 100), ('follow', 100), ('has', 93), ('head', 92), ('non', 90), ('exit', 87), ('there', 87), ('must', 78), ('end', 77), ('other', 76), ('make', 75), ('drive', 75), ('crossing', 75), ('away', 67), ('bring', 65), ('place', 62), ('taking', 62), ('black', 62), ('stop', 62), ('if', 53), ('sure', 50), ('use', 50), ('moves', 50), ('will', 50), ('lead', 50), ('when', 50), ('except', 50), ('big', 45), ('at', 40), ('find', 38), ('white', 37), ('color', 32), ('item', 31), ('instead', 31), ('over', 30), ('thing', 28), ('took', 28), ('passage', 28), ('them', 28), ('behind', 27), ('leave', 26), ('pick', 25), ('each', 25), ('time', 25), ('before', 25), ('never', 25), ('progress', 25), ('totally', 25), ('out', 25), ('throught', 25), ('isnt', 25), ('keep', 25), ('cross', 25), ('leads', 25), ('side', 25), ('all', 25), ('got', 25), ('stepping', 25), ('spaces', 25), ('once', 25), ('opposite', 25), ('c', 25), ('shortest', 25), ('always', 25), ('proceeding', 25), ('rectanglular', 25), ('being', 25), ('route', 25), ('gain', 25), ('access', 25), ('hallway', 25), ('takes', 25), ('though', 25), ('rom', 25), ('ends', 25), ('moving', 25), ('please', 25), ('this', 19), ('stay', 12), ('after', 12), ('retrieve', 12), ('i', 12), ('want', 12), ('middle', 12), ('right', 12), ('with', 12), ('return', 10), ('grab', 10), ('far', 10), ('wall', 10), ('come', 8), ('under', 7), ('locate', 7), ('goto', 5), ('down', 5), ('left', 5), ('dog', 5), ('backpack', 5), ('backto', 5), ('carry', 5), ('across', 5), ('front', 5), ('below', 5), ('violet', 3)]\n"
     ]
    }
   ],
   "source": [
    "# vocab frequency dict for source side\n",
    "src_freq_dict = dict(src_c)\n",
    "print('CECW souce vocab size:', len(src_c))\n",
    "print('CECW souce vocab frequency')\n",
    "print(src_c.most_common())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CECW target vocab size: 11\n",
      "CECW target vocab frequency\n",
      "[('F', 18054), ('&', 10888), ('C', 6223), ('B', 6101), ('R', 5918), ('Y', 5030), ('G', 3987), ('!', 3987), ('|', 1970), ('X', 428), ('Z', 311)]\n"
     ]
    }
   ],
   "source": [
    "# vocab frequency dict for target side\n",
    "tar_freq_dict = dict(tar_c)\n",
    "print('CECW target vocab size:', len(tar_c))\n",
    "print('CECW target vocab frequency')\n",
    "print(tar_c.most_common())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CECW souce vocab size: 197\n"
     ]
    }
   ],
   "source": [
    "# generate source vocabulary2index dictionary\n",
    "src_vocab_dict = dict()\n",
    "src_vocab_dict['<s>'] = 0\n",
    "src_vocab_dict['</s>'] = 1\n",
    "src_vocab_dict['<pad>'] = 2\n",
    "src_vocab_dict['<unk>'] = 3\n",
    "\n",
    "i = len(src_vocab_dict)\n",
    "\n",
    "for token in src_freq_dict:\n",
    "    src_vocab_dict[token] = i\n",
    "    i += 1\n",
    "\n",
    "print('CECW souce vocab size:', len(src_vocab_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# source index2vocabulary dictionary\n",
    "src_index_dict = {v:k for (k,v) in src_vocab_dict.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CECW target vocab size: 14\n"
     ]
    }
   ],
   "source": [
    "# generate target vocabulary2index dictionary\n",
    "tar_vocab_dict = dict()\n",
    "tar_vocab_dict['<s>'] = 0\n",
    "tar_vocab_dict['</s>'] = 1\n",
    "tar_vocab_dict['<pad>'] = 2\n",
    "\n",
    "i = len(tar_vocab_dict)\n",
    "\n",
    "for token in tar_freq_dict:\n",
    "    tar_vocab_dict[token] = i\n",
    "    i += 1\n",
    "\n",
    "print('CECW target vocab size:', len(tar_vocab_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# target index2vocabulary dictionary\n",
    "tar_index_dict = {v:k for (k,v) in tar_vocab_dict.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save data list to a dictionary\n",
    "# x for source input command\n",
    "# y for target output expression\n",
    "train_dict = dict()\n",
    "test_dict = dict()\n",
    "\n",
    "train_dict['x'] = tk_train_src_list\n",
    "train_dict['y'] = tk_train_tar_list\n",
    "\n",
    "test_dict['x'] = tk_test_src_list\n",
    "test_dict['y'] = tk_test_tar_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ALL\n",
    "As a \"subset\" of CECW, ALL actually is the original CECW dataset without any modification."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ALL train size: 8922\n",
      "ALL test size: 2231\n"
     ]
    }
   ],
   "source": [
    "# train size and test size\n",
    "print('ALL train size:', len(train_dict['x']))\n",
    "print('ALL test size:', len(test_dict['x']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make everything a dictionary\n",
    "data_dict = dict()\n",
    "data_dict['train_dict'] = train_dict\n",
    "data_dict['test_dict'] = test_dict\n",
    "\n",
    "vocab_dict = dict()\n",
    "vocab_dict['src_vocab2index_dict'] = src_vocab_dict\n",
    "vocab_dict['src_index2vocab_dict'] = src_index_dict\n",
    "vocab_dict['tar_vocab2index_dict'] = tar_vocab_dict\n",
    "vocab_dict['tar_index2vocab_dict'] = tar_index_dict\n",
    "\n",
    "# output data dict and vocab dict as two json files\n",
    "save_json(ALL_DATA_PATH, data_dict)\n",
    "save_json(ALL_VOCAB_PATH, vocab_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Colorless\n",
    "As a subset of CECW, Colorless only contains data related to original four color words, that is, \"blue,\" \"green,\" \"red,\" and \"yellow.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fresh color words for each entity in CECW\n",
    "B_list = ['purple', 'navy']\n",
    "C_list = ['olive', 'lime']\n",
    "R_list = ['pink', 'orange']\n",
    "Y_list = ['brown', 'tan']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove all sequence pairs having new color words from the training set\n",
    "for i in range(len(tk_train_src_list)):\n",
    "    # replace new primitive with original color words\n",
    "    train_command_list = tk_train_src_list[i]\n",
    "    # blue for purple and navy\n",
    "    for b in B_list:\n",
    "        index_list = [index for index, token in enumerate(train_command_list) if token == b]\n",
    "        for j in index_list:\n",
    "            tk_train_src_list[i][j] = 'blue'\n",
    "    for c in C_list:\n",
    "        index_list = [index for index, token in enumerate(train_command_list) if token == c]\n",
    "        for j in index_list:\n",
    "            tk_train_src_list[i][j] = 'green'\n",
    "    for r in R_list:\n",
    "        index_list = [index for index, token in enumerate(train_command_list) if token == r]\n",
    "        for j in index_list:\n",
    "            tk_train_src_list[i][j] = 'red'\n",
    "    for y in Y_list:\n",
    "        index_list = [index for index, token in enumerate(train_command_list) if token == y]\n",
    "        for j in index_list:\n",
    "            tk_train_src_list[i][j] = 'yellow'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Colorless train size: 2118\n"
     ]
    }
   ],
   "source": [
    "# remove duplicates\n",
    "src_list = []\n",
    "tar_list = []\n",
    "for i in range(len(tk_train_src_list)):\n",
    "    if tk_train_src_list[i] not in src_list:\n",
    "        src_list.append(tk_train_src_list[i])\n",
    "        tar_list.append(tk_train_tar_list[i])\n",
    "        \n",
    "print('Colorless train size:', len(src_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save data list to a dictionary\n",
    "# x for source input command\n",
    "# y for target output expression\n",
    "train_dict = dict()\n",
    "train_dict['x'] = src_list\n",
    "train_dict['y'] = tar_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# there is no change in test dict and vocab dict\n",
    "# make everything a dictionary\n",
    "data_dict['train_dict'] = train_dict\n",
    "\n",
    "# output data dict and vocab dict as two json files\n",
    "save_json(Colorless_DATA_PATH, data_dict)\n",
    "save_json(Colorless_VOCAB_PATH, vocab_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## B\n",
    "As a subset of CECW, B is Colorless in addition to two primitive rules including \"purple\" $\\rightarrow$ \"B\" and \"navy\" $\\rightarrow$ \"B.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add two primitive rules\n",
    "for b in B_list:\n",
    "    src_list, tar_list = add_primitive_rule([b], ['B'], src_list, tar_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "B train size: 2120\n"
     ]
    }
   ],
   "source": [
    "print('B train size:', len(src_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save data list to a dictionary\n",
    "# x for source input command\n",
    "# y for target output expression\n",
    "train_dict = dict()\n",
    "train_dict['x'] = src_list\n",
    "train_dict['y'] = tar_list\n",
    "\n",
    "# there is no change in test dict and vocab dict\n",
    "# make everything a dictionary\n",
    "data_dict['train_dict'] = train_dict\n",
    "\n",
    "# output data dict and vocab dict as two json files\n",
    "save_json(B_DATA_PATH, data_dict)\n",
    "save_json(B_VOCAB_PATH, vocab_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BC\n",
    "As a subset of CECW, BC is B in addition to two primitive rules including \"olive\" $\\rightarrow$ \"C\" and \"lime\" $\\rightarrow$ \"C.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add two primitive rules\n",
    "for c in C_list:\n",
    "    src_list, tar_list = add_primitive_rule([c], ['C'], src_list, tar_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BC train size: 2122\n"
     ]
    }
   ],
   "source": [
    "print('BC train size:', len(src_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save data list to a dictionary\n",
    "# x for source input command\n",
    "# y for target output expression\n",
    "train_dict = dict()\n",
    "train_dict['x'] = src_list\n",
    "train_dict['y'] = tar_list\n",
    "\n",
    "# there is no change in test dict and vocab dict\n",
    "# make everything a dictionary\n",
    "data_dict['train_dict'] = train_dict\n",
    "\n",
    "# output data dict and vocab dict as two json files\n",
    "save_json(BC_DATA_PATH, data_dict)\n",
    "save_json(BC_VOCAB_PATH, vocab_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BCR\n",
    "As a \"subset\" of CECW, BCR is BC in addition to two primitive rules including \"pink\" $\\rightarrow$ \"R\" and \"orange\" $\\rightarrow$ \"R.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add two primitive rules\n",
    "for r in R_list:\n",
    "    src_list, tar_list = add_primitive_rule([r], ['R'], src_list, tar_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BCR train size: 2124\n"
     ]
    }
   ],
   "source": [
    "print('BCR train size:', len(src_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save data list to a dictionary\n",
    "# x for source input command\n",
    "# y for target output expression\n",
    "train_dict = dict()\n",
    "train_dict['x'] = src_list\n",
    "train_dict['y'] = tar_list\n",
    "\n",
    "# there is no change in test dict and vocab dict\n",
    "# make everything a dictionary\n",
    "data_dict['train_dict'] = train_dict\n",
    "\n",
    "# output data dict and vocab dict as two json files\n",
    "save_json(BCR_DATA_PATH, data_dict)\n",
    "save_json(BCR_VOCAB_PATH, vocab_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BCRY\n",
    "As a subset of CECW, BCRY is BCR in addition to two primitive rules including \"brown\" $\\rightarrow$ \"Y\" and \"tan\" $\\rightarrow$ \"Y.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add two primitive rules\n",
    "for y in Y_list:\n",
    "    src_list, tar_list = add_primitive_rule([y], ['Y'], src_list, tar_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BCRY train size: 2126\n"
     ]
    }
   ],
   "source": [
    "print('BCRY train size:', len(src_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save data list to a dictionary\n",
    "# x for source input command\n",
    "# y for target output expression\n",
    "train_dict = dict()\n",
    "train_dict['x'] = src_list\n",
    "train_dict['y'] = tar_list\n",
    "\n",
    "# there is no change in test dict and vocab dict\n",
    "# make everything a dictionary\n",
    "data_dict['train_dict'] = train_dict\n",
    "\n",
    "# output data dict and vocab dict as two json files\n",
    "save_json(BCRY_DATA_PATH, data_dict)\n",
    "save_json(BCRY_VOCAB_PATH, vocab_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reference"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Squire, S., Tellex, S., Arumugam, D., & Yang, L. Grounding English Commands to Reward Functions.\n",
    "2. Gopalan, N., Arumugam, D., Wong, L. L., & Tellex, S. (2018). Sequence-to-Sequence Language Grounding of Non-Markovian Task Specifications. In Robotics: Science and Systems."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}