table-to-text-genration-inference-scoring.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "160a0c29",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:38:31.685172Z",
     "iopub.status.busy": "2021-08-23T15:38:31.684680Z",
     "iopub.status.idle": "2021-08-23T15:38:31.808906Z",
     "shell.execute_reply": "2021-08-23T15:38:31.808171Z",
     "shell.execute_reply.started": "2021-08-23T15:23:58.465024Z"
    },
    "executionInfo": {
     "elapsed": 71447,
     "status": "ok",
     "timestamp": 1619773218560,
     "user": {
      "displayName": "Aishik Rakshit",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj51jIPTS5SPRR4YiqMU00JQ0UuzUavXBl9dsaRDw=s64",
      "userId": "15122983906308089605"
     },
     "user_tz": -330
    },
    "id": "V-ZMFQpvp96r",
    "papermill": {
     "duration": 0.154306,
     "end_time": "2021-08-23T15:38:31.809052",
     "exception": false,
     "start_time": "2021-08-23T15:38:31.654746",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df=pd.read_csv(\"/kaggle/input/table-to-text-generation-dataset-google-totto/totto_data/tablesWithTagDev.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a55ea823",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:38:31.925700Z",
     "iopub.status.busy": "2021-08-23T15:38:31.924922Z",
     "iopub.status.idle": "2021-08-23T15:38:31.929038Z",
     "shell.execute_reply": "2021-08-23T15:38:31.929466Z",
     "shell.execute_reply.started": "2021-08-23T15:23:58.582562Z"
    },
    "papermill": {
     "duration": 0.10132,
     "end_time": "2021-08-23T15:38:31.929609",
     "exception": false,
     "start_time": "2021-08-23T15:38:31.828289",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7700"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8f2dadf8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:38:31.972154Z",
     "iopub.status.busy": "2021-08-23T15:38:31.971470Z",
     "iopub.status.idle": "2021-08-23T15:38:31.973758Z",
     "shell.execute_reply": "2021-08-23T15:38:31.974116Z",
     "shell.execute_reply.started": "2021-08-23T15:23:58.591505Z"
    },
    "executionInfo": {
     "elapsed": 71444,
     "status": "ok",
     "timestamp": 1619773218561,
     "user": {
      "displayName": "Aishik Rakshit",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj51jIPTS5SPRR4YiqMU00JQ0UuzUavXBl9dsaRDw=s64",
      "userId": "15122983906308089605"
     },
     "user_tz": -330
    },
    "id": "-iNCX7X0R3j3",
    "papermill": {
     "duration": 0.025148,
     "end_time": "2021-08-23T15:38:31.974281",
     "exception": false,
     "start_time": "2021-08-23T15:38:31.949133",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "MAXLENI=400\n",
    "MAXLENO=200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3f3038cf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:38:32.019386Z",
     "iopub.status.busy": "2021-08-23T15:38:32.018768Z",
     "iopub.status.idle": "2021-08-23T15:38:37.819048Z",
     "shell.execute_reply": "2021-08-23T15:38:37.818141Z",
     "shell.execute_reply.started": "2021-08-23T15:23:58.601863Z"
    },
    "executionInfo": {
     "elapsed": 78366,
     "status": "ok",
     "timestamp": 1619773225487,
     "user": {
      "displayName": "Aishik Rakshit",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj51jIPTS5SPRR4YiqMU00JQ0UuzUavXBl9dsaRDw=s64",
      "userId": "15122983906308089605"
     },
     "user_tz": -330
    },
    "id": "S3g3I7iaqh8-",
    "papermill": {
     "duration": 5.825391,
     "end_time": "2021-08-23T15:38:37.819193",
     "exception": false,
     "start_time": "2021-08-23T15:38:31.993802",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from transformers import T5Tokenizer\n",
    "from transformers import T5ForConditionalGeneration\n",
    "from transformers import AdamW, WarmUp, get_linear_schedule_with_warmup\n",
    "from tqdm.notebook import tqdm\n",
    "import pandas as pd\n",
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import os\n",
    "import time\n",
    "import copy\n",
    "import numpy\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2301edc8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:38:37.912056Z",
     "iopub.status.busy": "2021-08-23T15:38:37.911460Z",
     "iopub.status.idle": "2021-08-23T15:38:37.915781Z",
     "shell.execute_reply": "2021-08-23T15:38:37.916323Z",
     "shell.execute_reply.started": "2021-08-23T15:23:58.610899Z"
    },
    "papermill": {
     "duration": 0.077504,
     "end_time": "2021-08-23T15:38:37.916500",
     "exception": false,
     "start_time": "2021-08-23T15:38:37.838996",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "device=torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "dd756b89",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:38:37.961802Z",
     "iopub.status.busy": "2021-08-23T15:38:37.961063Z",
     "iopub.status.idle": "2021-08-23T15:38:58.153587Z",
     "shell.execute_reply": "2021-08-23T15:38:58.153068Z",
     "shell.execute_reply.started": "2021-08-23T15:23:58.624149Z"
    },
    "executionInfo": {
     "elapsed": 120508,
     "status": "ok",
     "timestamp": 1619773267634,
     "user": {
      "displayName": "Aishik Rakshit",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj51jIPTS5SPRR4YiqMU00JQ0UuzUavXBl9dsaRDw=s64",
      "userId": "15122983906308089605"
     },
     "user_tz": -330
    },
    "id": "CjalEnb2ptQd",
    "outputId": "777408b1-3095-4292-b13e-7d98f968c082",
    "papermill": {
     "duration": 20.216659,
     "end_time": "2021-08-23T15:38:58.153727",
     "exception": false,
     "start_time": "2021-08-23T15:38:37.937068",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ca409f6ad58e4c8f86ad191163039be1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/792k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00e872414a27455c9d38003410db1dc6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/1.39M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenizer = T5Tokenizer.from_pretrained(\"t5-base\")\n",
    "model=torch.load(\"/kaggle/input/table-to-text-generation-utils/T5Epoch 7\", map_location=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "aa1f84aa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:38:58.201774Z",
     "iopub.status.busy": "2021-08-23T15:38:58.201197Z",
     "iopub.status.idle": "2021-08-23T15:38:58.208716Z",
     "shell.execute_reply": "2021-08-23T15:38:58.208274Z",
     "shell.execute_reply.started": "2021-08-23T15:24:01.104694Z"
    },
    "executionInfo": {
     "elapsed": 121253,
     "status": "ok",
     "timestamp": 1619773268387,
     "user": {
      "displayName": "Aishik Rakshit",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj51jIPTS5SPRR4YiqMU00JQ0UuzUavXBl9dsaRDw=s64",
      "userId": "15122983906308089605"
     },
     "user_tz": -330
    },
    "id": "mn0t_4NFOftQ",
    "outputId": "3bc1daf2-fef0-46f0-e46c-260920bece14",
    "papermill": {
     "duration": 0.034353,
     "end_time": "2021-08-23T15:38:58.208834",
     "exception": false,
     "start_time": "2021-08-23T15:38:58.174481",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We have added 14 tokens\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Embedding(32114, 768)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "special_tokens_dict = {'pad_token': '<pad>', 'bos_token': '<bos>', 'eos_token': '<eos>', \n",
    "                       'additional_special_tokens': ['<PAGESTART>', '<PAGEEND>', '<SECTIONSTART>', '<SECTIONEND>',\n",
    "                                                     '<TABLESTART>','<TABLEEND>','<CELLSTART>','<CELLEND>','<COLHEADERSTART>',\n",
    "                                                     '<COLHEADEREND>','<ROWHEADERSTART>','<ROWHEADEREND>']}\n",
    "\n",
    "num_added_toks = tokenizer.add_special_tokens(special_tokens_dict)\n",
    "\n",
    "print('We have added', num_added_toks, 'tokens')\n",
    "model.encoder.resize_token_embeddings(len(tokenizer))\n",
    "model.decoder.resize_token_embeddings(len(tokenizer))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f1b3c158",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:38:58.261867Z",
     "iopub.status.busy": "2021-08-23T15:38:58.260617Z",
     "iopub.status.idle": "2021-08-23T15:38:58.263314Z",
     "shell.execute_reply": "2021-08-23T15:38:58.262897Z",
     "shell.execute_reply.started": "2021-08-23T15:24:01.118249Z"
    },
    "executionInfo": {
     "elapsed": 121248,
     "status": "ok",
     "timestamp": 1619773268387,
     "user": {
      "displayName": "Aishik Rakshit",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj51jIPTS5SPRR4YiqMU00JQ0UuzUavXBl9dsaRDw=s64",
      "userId": "15122983906308089605"
     },
     "user_tz": -330
    },
    "id": "WxHdZMxluF0I",
    "papermill": {
     "duration": 0.033816,
     "end_time": "2021-08-23T15:38:58.263418",
     "exception": false,
     "start_time": "2021-08-23T15:38:58.229602",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class tottodataset(Dataset):\n",
    "  def __init__(self,df,tokenizer):\n",
    "    self.sentence=df['sentence']\n",
    "    self.table=df['table']\n",
    "    self.tokenizer=tokenizer\n",
    "\n",
    "  def __len__(self):\n",
    "    return len(self.sentence)\n",
    "  \n",
    "  def __getitem__(self,idx):\n",
    "    inp=(self.table[idx]+'</s>').replace(\"<page_title>\", \"<PAGESTART>\").replace(\"</page_title>\", \"<PAGEEND>\") \\\n",
    "                                    .replace(\"<section_title>\", \"<SECTIONSTART>\").replace(\"</section_title>\", \"<SECTIONEND>\") \\\n",
    "                                    .replace(\"<table>\", \"<TABLESTART>\").replace(\"</table>\", \"<TABLEEND>\") \\\n",
    "                                    .replace(\"<cell>\", \"<CELLSTART>\").replace(\"</cell>\", \"<CELLEND>\") \\\n",
    "                                    .replace(\"<col_header>\", \"<COLHEADERSTART>\").replace(\"</col_header>\", \"<COLHEADEREND>\") \\\n",
    "                                    .replace(\"<row_header>\", \"<ROWHEADERSTART>\").replace(\"</row_header>\", \"<ROWHEADEREND>\")\n",
    "    out=self.sentence[idx]+'</s>'\n",
    "    inp_tokens=self.tokenizer.encode_plus(inp, padding=\"max_length\", max_length=MAXLENI, truncation=True)\n",
    "    out_tokens=self.tokenizer.encode_plus(out, padding=\"max_length\", max_length=MAXLENO, truncation=True)\n",
    "    inp_id=inp_tokens.input_ids\n",
    "    out_id=out_tokens.input_ids\n",
    "    inp_mask=inp_tokens.attention_mask\n",
    "    out_mask=out_tokens.attention_mask\n",
    "    labels=out_tokens.input_ids.copy()\n",
    "    labels=[-100  if x==self.tokenizer.pad_token_id else x for x in labels]\n",
    "\n",
    "    return {\n",
    "        \"table_text\":inp,\n",
    "        \"sentence\":out,\n",
    "        \"input_ids\":torch.tensor(inp_id, dtype=torch.long),\n",
    "        \"input_attention_mask\":torch.tensor(inp_mask, dtype=torch.long),\n",
    "        \"decoder_input_ids\":torch.tensor(out_id, dtype=torch.long),\n",
    "        \"decoder_attention_mask\":torch.tensor(out_mask, dtype=torch.long),\n",
    "        \"labels\":torch.tensor(labels, dtype=torch.long)\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f6ce7939",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:38:58.316614Z",
     "iopub.status.busy": "2021-08-23T15:38:58.315393Z",
     "iopub.status.idle": "2021-08-23T15:38:58.317682Z",
     "shell.execute_reply": "2021-08-23T15:38:58.318078Z",
     "shell.execute_reply.started": "2021-08-23T15:24:01.134191Z"
    },
    "executionInfo": {
     "elapsed": 121240,
     "status": "ok",
     "timestamp": 1619773268388,
     "user": {
      "displayName": "Aishik Rakshit",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj51jIPTS5SPRR4YiqMU00JQ0UuzUavXBl9dsaRDw=s64",
      "userId": "15122983906308089605"
     },
     "user_tz": -330
    },
    "id": "S7YQPF5joFU2",
    "papermill": {
     "duration": 0.034191,
     "end_time": "2021-08-23T15:38:58.318191",
     "exception": false,
     "start_time": "2021-08-23T15:38:58.284000",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "test_dataset=tottodataset(df,tokenizer)\n",
    "\n",
    "test_dataloader=DataLoader(test_dataset,\n",
    "                            batch_size=64,\n",
    "                            num_workers=2,\n",
    "                            shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e64ca284",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:38:58.364905Z",
     "iopub.status.busy": "2021-08-23T15:38:58.363612Z",
     "iopub.status.idle": "2021-08-23T15:38:58.384050Z",
     "shell.execute_reply": "2021-08-23T15:38:58.384459Z",
     "shell.execute_reply.started": "2021-08-23T15:24:01.148249Z"
    },
    "executionInfo": {
     "elapsed": 121226,
     "status": "ok",
     "timestamp": 1619773268390,
     "user": {
      "displayName": "Aishik Rakshit",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj51jIPTS5SPRR4YiqMU00JQ0UuzUavXBl9dsaRDw=s64",
      "userId": "15122983906308089605"
     },
     "user_tz": -330
    },
    "id": "tJYgCj14gX8E",
    "outputId": "e1888448-0338-4c3b-d2a2-5aa02509e84c",
    "papermill": {
     "duration": 0.045653,
     "end_time": "2021-08-23T15:38:58.384574",
     "exception": false,
     "start_time": "2021-08-23T15:38:58.338921",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "T5ForConditionalGeneration(\n",
       "  (shared): Embedding(32128, 768)\n",
       "  (encoder): T5Stack(\n",
       "    (embed_tokens): Embedding(32114, 768)\n",
       "    (block): ModuleList(\n",
       "      (0): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (relative_attention_bias): Embedding(32, 12)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
       "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (1): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
       "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (2): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
       "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (3): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
       "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (4): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
       "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (5): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
       "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (6): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
       "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (7): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
       "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (8): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
       "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (9): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
       "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (10): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
       "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (11): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
       "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (final_layer_norm): T5LayerNorm()\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       "  (decoder): T5Stack(\n",
       "    (embed_tokens): Embedding(32114, 768)\n",
       "    (block): ModuleList(\n",
       "      (0): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (relative_attention_bias): Embedding(32, 12)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
       "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (1): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
       "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (2): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
       "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (3): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
       "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (4): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
       "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (5): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
       "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (6): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
       "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (7): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
       "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (8): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
       "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (9): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
       "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (10): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
       "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (11): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
       "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (layer_norm): T5LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (final_layer_norm): T5LayerNorm()\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       "  (lm_head): Linear(in_features=768, out_features=32128, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8d922283",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:38:58.432033Z",
     "iopub.status.busy": "2021-08-23T15:38:58.431542Z",
     "iopub.status.idle": "2021-08-23T15:38:59.137652Z",
     "shell.execute_reply": "2021-08-23T15:38:59.138426Z",
     "shell.execute_reply.started": "2021-08-23T15:24:01.173503Z"
    },
    "executionInfo": {
     "elapsed": 121217,
     "status": "ok",
     "timestamp": 1619773268390,
     "user": {
      "displayName": "Aishik Rakshit",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj51jIPTS5SPRR4YiqMU00JQ0UuzUavXBl9dsaRDw=s64",
      "userId": "15122983906308089605"
     },
     "user_tz": -330
    },
    "id": "MBTbrmY5H1fV",
    "outputId": "6b936e74-730e-4762-ff1f-134ae050b1d1",
    "papermill": {
     "duration": 0.731866,
     "end_time": "2021-08-23T15:38:59.138593",
     "exception": false,
     "start_time": "2021-08-23T15:38:58.406727",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mon Aug 23 15:38:59 2021       \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| NVIDIA-SMI 450.119.04   Driver Version: 450.119.04   CUDA Version: 11.0     |\r\n",
      "|-------------------------------+----------------------+----------------------+\r\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\r\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\r\n",
      "|                               |                      |               MIG M. |\r\n",
      "|===============================+======================+======================|\r\n",
      "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\r\n",
      "| N/A   34C    P0    32W / 250W |   1981MiB / 16280MiB |      0%      Default |\r\n",
      "|                               |                      |                  N/A |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "                                                                               \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| Processes:                                                                  |\r\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\r\n",
      "|        ID   ID                                                   Usage      |\r\n",
      "|=============================================================================|\r\n",
      "+-----------------------------------------------------------------------------+\r\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8b4f4ae2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:38:59.190860Z",
     "iopub.status.busy": "2021-08-23T15:38:59.190242Z",
     "iopub.status.idle": "2021-08-23T15:38:59.243424Z",
     "shell.execute_reply": "2021-08-23T15:38:59.244363Z",
     "shell.execute_reply.started": "2021-08-23T15:24:02.008852Z"
    },
    "papermill": {
     "duration": 0.082944,
     "end_time": "2021-08-23T15:38:59.244555",
     "exception": false,
     "start_time": "2021-08-23T15:38:59.161611",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "test_iterator=iter(test_dataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a7388f59",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:38:59.391635Z",
     "iopub.status.busy": "2021-08-23T15:38:59.390878Z",
     "iopub.status.idle": "2021-08-23T15:39:00.300319Z",
     "shell.execute_reply": "2021-08-23T15:39:00.298965Z",
     "shell.execute_reply.started": "2021-08-23T15:24:02.222298Z"
    },
    "papermill": {
     "duration": 0.987227,
     "end_time": "2021-08-23T15:39:00.300508",
     "exception": false,
     "start_time": "2021-08-23T15:38:59.313281",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "test_batch=next(test_iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "18a2c639",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:39:00.448740Z",
     "iopub.status.busy": "2021-08-23T15:39:00.447494Z",
     "iopub.status.idle": "2021-08-23T15:39:03.051146Z",
     "shell.execute_reply": "2021-08-23T15:39:03.050689Z",
     "shell.execute_reply.started": "2021-08-23T15:24:03.102317Z"
    },
    "executionInfo": {
     "elapsed": 157340,
     "status": "ok",
     "timestamp": 1619773304540,
     "user": {
      "displayName": "Aishik Rakshit",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj51jIPTS5SPRR4YiqMU00JQ0UuzUavXBl9dsaRDw=s64",
      "userId": "15122983906308089605"
     },
     "user_tz": -330
    },
    "id": "WN9DuhwxkrSN",
    "papermill": {
     "duration": 2.682172,
     "end_time": "2021-08-23T15:39:03.051404",
     "exception": false,
     "start_time": "2021-08-23T15:39:00.369232",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "generation_output = model.generate(test_batch['input_ids'].to(device), return_dict_in_generate=True, output_scores=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "436fd2df",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:39:03.103043Z",
     "iopub.status.busy": "2021-08-23T15:39:03.102267Z",
     "iopub.status.idle": "2021-08-23T15:39:03.108735Z",
     "shell.execute_reply": "2021-08-23T15:39:03.108243Z",
     "shell.execute_reply.started": "2021-08-23T15:24:04.673941Z"
    },
    "papermill": {
     "duration": 0.034008,
     "end_time": "2021-08-23T15:39:03.108835",
     "exception": false,
     "start_time": "2021-08-23T15:39:03.074827",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[    0,  4173,  7780,  ...,   552,   507,  4581],\n",
       "        [    0,    86,  5123,  ...,  2129,     6,    11],\n",
       "        [    0, 12833,   141,  ...,     0,     0,     0],\n",
       "        ...,\n",
       "        [    0,  1184,    51,  ...,    52,     7,    10],\n",
       "        [    0,  3037,  3233,  ...,     6, 12035,     6],\n",
       "        [    0,    37,  1348,  ...,    16,  6156,     9]], device='cuda:0')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generation_output[\"sequences\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a0879e97",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:39:03.161634Z",
     "iopub.status.busy": "2021-08-23T15:39:03.160391Z",
     "iopub.status.idle": "2021-08-23T15:39:03.163108Z",
     "shell.execute_reply": "2021-08-23T15:39:03.162726Z",
     "shell.execute_reply.started": "2021-08-23T15:24:04.683756Z"
    },
    "executionInfo": {
     "elapsed": 157336,
     "status": "ok",
     "timestamp": 1619773304541,
     "user": {
      "displayName": "Aishik Rakshit",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj51jIPTS5SPRR4YiqMU00JQ0UuzUavXBl9dsaRDw=s64",
      "userId": "15122983906308089605"
     },
     "user_tz": -330
    },
    "id": "RPRk1up_mlzW",
    "papermill": {
     "duration": 0.031251,
     "end_time": "2021-08-23T15:39:03.163260",
     "exception": false,
     "start_time": "2021-08-23T15:39:03.132009",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "expected=test_batch[\"sentence\"][0]\n",
    "predicted=tokenizer.decode(generation_output[\"sequences\"][0],skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "06832f4b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:39:03.215112Z",
     "iopub.status.busy": "2021-08-23T15:39:03.214576Z",
     "iopub.status.idle": "2021-08-23T15:39:03.219172Z",
     "shell.execute_reply": "2021-08-23T15:39:03.219715Z",
     "shell.execute_reply.started": "2021-08-23T15:24:04.697104Z"
    },
    "executionInfo": {
     "elapsed": 157332,
     "status": "ok",
     "timestamp": 1619773304541,
     "user": {
      "displayName": "Aishik Rakshit",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj51jIPTS5SPRR4YiqMU00JQ0UuzUavXBl9dsaRDw=s64",
      "userId": "15122983906308089605"
     },
     "user_tz": -330
    },
    "id": "4ymtj_OJwgKh",
    "outputId": "a4f35011-15cc-4275-b15e-50504dc7dab9",
    "papermill": {
     "duration": 0.032539,
     "end_time": "2021-08-23T15:39:03.219880",
     "exception": false,
     "start_time": "2021-08-23T15:39:03.187341",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expected Sentence: Daniel Henry Chamberlain was the 76th Governor of South Carolina from 1874.</s>\n",
      "Predicted Sentence: Daniel Henry Chamberlain was the 76th Governor of South Carolina, serving until 1874\n"
     ]
    }
   ],
   "source": [
    "print(f\"Expected Sentence: {expected}\")\n",
    "print(f\"Predicted Sentence: {predicted}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7c5c1a59",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:39:03.287015Z",
     "iopub.status.busy": "2021-08-23T15:39:03.270502Z",
     "iopub.status.idle": "2021-08-23T15:39:03.964816Z",
     "shell.execute_reply": "2021-08-23T15:39:03.964334Z",
     "shell.execute_reply.started": "2021-08-23T15:24:04.707336Z"
    },
    "executionInfo": {
     "elapsed": 157327,
     "status": "ok",
     "timestamp": 1619773304542,
     "user": {
      "displayName": "Aishik Rakshit",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj51jIPTS5SPRR4YiqMU00JQ0UuzUavXBl9dsaRDw=s64",
      "userId": "15122983906308089605"
     },
     "user_tz": -330
    },
    "id": "HZJKQbhrxMh7",
    "outputId": "351d15b5-0647-4e77-9d69-6d3da1355615",
    "papermill": {
     "duration": 0.721344,
     "end_time": "2021-08-23T15:39:03.964936",
     "exception": false,
     "start_time": "2021-08-23T15:39:03.243592",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mon Aug 23 15:39:03 2021       \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| NVIDIA-SMI 450.119.04   Driver Version: 450.119.04   CUDA Version: 11.0     |\r\n",
      "|-------------------------------+----------------------+----------------------+\r\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\r\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\r\n",
      "|                               |                      |               MIG M. |\r\n",
      "|===============================+======================+======================|\r\n",
      "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\r\n",
      "| N/A   36C    P0    38W / 250W |   4419MiB / 16280MiB |      0%      Default |\r\n",
      "|                               |                      |                  N/A |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "                                                                               \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| Processes:                                                                  |\r\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\r\n",
      "|        ID   ID                                                   Usage      |\r\n",
      "|=============================================================================|\r\n",
      "+-----------------------------------------------------------------------------+\r\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "db491c25",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:39:04.018500Z",
     "iopub.status.busy": "2021-08-23T15:39:04.017799Z",
     "iopub.status.idle": "2021-08-23T15:39:04.020921Z",
     "shell.execute_reply": "2021-08-23T15:39:04.020497Z",
     "shell.execute_reply.started": "2021-08-23T15:24:05.411633Z"
    },
    "executionInfo": {
     "elapsed": 157319,
     "status": "ok",
     "timestamp": 1619773304543,
     "user": {
      "displayName": "Aishik Rakshit",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj51jIPTS5SPRR4YiqMU00JQ0UuzUavXBl9dsaRDw=s64",
      "userId": "15122983906308089605"
     },
     "user_tz": -330
    },
    "id": "TwXJV8SGPK1M",
    "papermill": {
     "duration": 0.030963,
     "end_time": "2021-08-23T15:39:04.021037",
     "exception": false,
     "start_time": "2021-08-23T15:39:03.990074",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# bleurt_dataloader=DataLoader(test_dataset,\n",
    "#                             batch_size=1,\n",
    "#                             num_workers=2,\n",
    "#                             shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "313d75fb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:39:04.079843Z",
     "iopub.status.busy": "2021-08-23T15:39:04.078881Z",
     "iopub.status.idle": "2021-08-23T15:42:18.195051Z",
     "shell.execute_reply": "2021-08-23T15:42:18.194144Z",
     "shell.execute_reply.started": "2021-08-23T15:24:05.419086Z"
    },
    "executionInfo": {
     "elapsed": 189299,
     "status": "ok",
     "timestamp": 1619773336534,
     "user": {
      "displayName": "Aishik Rakshit",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj51jIPTS5SPRR4YiqMU00JQ0UuzUavXBl9dsaRDw=s64",
      "userId": "15122983906308089605"
     },
     "user_tz": -330
    },
    "id": "xGiruKrwM1lB",
    "papermill": {
     "duration": 194.14852,
     "end_time": "2021-08-23T15:42:18.195189",
     "exception": false,
     "start_time": "2021-08-23T15:39:04.046669",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c9ceae5341845c79b5ecef0564d8f07",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/121 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from tqdm.notebook import tqdm\n",
    "\n",
    "fref = open('./corrects.txt', 'w+')\n",
    "fout = open('./outputs.txt', 'w+')\n",
    "\n",
    "output=[]\n",
    "corrects=[]\n",
    "\n",
    "for batch in tqdm(test_dataloader):\n",
    "    input_ids=batch[\"input_ids\"].to(device)\n",
    "    \n",
    "    outputs = model.generate(input_ids, return_dict_in_generate=True, output_scores=True)\n",
    "    for i in range(64):\n",
    "        try:\n",
    "            outputstring = tokenizer.decode(outputs[\"sequences\"][i], skip_special_tokens=True)\n",
    "\n",
    "            refstring=batch[\"sentence\"][i].replace('</s>',\"\")\n",
    "            output.append(outputstring)\n",
    "            corrects.append(refstring)\n",
    "\n",
    "            fref.write(refstring+'\\n')\n",
    "            fout.write(outputstring+'.\\n')\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "fref.close()\n",
    "fout.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e35aac0f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:42:18.265850Z",
     "iopub.status.busy": "2021-08-23T15:42:18.249102Z",
     "iopub.status.idle": "2021-08-23T15:42:27.145813Z",
     "shell.execute_reply": "2021-08-23T15:42:27.146419Z",
     "shell.execute_reply.started": "2021-08-23T15:27:19.600282Z"
    },
    "papermill": {
     "duration": 8.926278,
     "end_time": "2021-08-23T15:42:27.146600",
     "exception": false,
     "start_time": "2021-08-23T15:42:18.220322",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: absl-py in /opt/conda/lib/python3.7/site-packages (from -r language/totto/eval_requirements.txt (line 1)) (0.12.0)\n",
      "Requirement already satisfied: pytest in /opt/conda/lib/python3.7/site-packages (from -r language/totto/eval_requirements.txt (line 2)) (6.2.4)\n",
      "Collecting sacrebleu\n",
      "  Downloading sacrebleu-2.0.0-py3-none-any.whl (90 kB)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from -r language/totto/eval_requirements.txt (line 4)) (1.15.0)\n",
      "Requirement already satisfied: wheel in /opt/conda/lib/python3.7/site-packages (from -r language/totto/eval_requirements.txt (line 5)) (0.36.2)\n",
      "Requirement already satisfied: importlib-metadata>=0.12 in /opt/conda/lib/python3.7/site-packages (from pytest->-r language/totto/eval_requirements.txt (line 2)) (3.4.0)\n",
      "Requirement already satisfied: py>=1.8.2 in /opt/conda/lib/python3.7/site-packages (from pytest->-r language/totto/eval_requirements.txt (line 2)) (1.10.0)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.7/site-packages (from pytest->-r language/totto/eval_requirements.txt (line 2)) (20.9)\n",
      "Requirement already satisfied: toml in /opt/conda/lib/python3.7/site-packages (from pytest->-r language/totto/eval_requirements.txt (line 2)) (0.10.2)\n",
      "Requirement already satisfied: iniconfig in /opt/conda/lib/python3.7/site-packages (from pytest->-r language/totto/eval_requirements.txt (line 2)) (1.1.1)\n",
      "Requirement already satisfied: attrs>=19.2.0 in /opt/conda/lib/python3.7/site-packages (from pytest->-r language/totto/eval_requirements.txt (line 2)) (21.2.0)\n",
      "Requirement already satisfied: pluggy<1.0.0a1,>=0.12 in /opt/conda/lib/python3.7/site-packages (from pytest->-r language/totto/eval_requirements.txt (line 2)) (0.13.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata>=0.12->pytest->-r language/totto/eval_requirements.txt (line 2)) (3.4.1)\n",
      "Requirement already satisfied: typing-extensions>=3.6.4 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata>=0.12->pytest->-r language/totto/eval_requirements.txt (line 2)) (3.7.4.3)\n",
      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.7/site-packages (from sacrebleu->-r language/totto/eval_requirements.txt (line 3)) (1.19.5)\n",
      "Requirement already satisfied: portalocker in /opt/conda/lib/python3.7/site-packages (from sacrebleu->-r language/totto/eval_requirements.txt (line 3)) (2.3.0)\n",
      "Requirement already satisfied: regex in /opt/conda/lib/python3.7/site-packages (from sacrebleu->-r language/totto/eval_requirements.txt (line 3)) (2021.4.4)\n",
      "Requirement already satisfied: tabulate>=0.8.9 in /opt/conda/lib/python3.7/site-packages (from sacrebleu->-r language/totto/eval_requirements.txt (line 3)) (0.8.9)\n",
      "Requirement already satisfied: colorama in /opt/conda/lib/python3.7/site-packages (from sacrebleu->-r language/totto/eval_requirements.txt (line 3)) (0.4.4)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging->pytest->-r language/totto/eval_requirements.txt (line 2)) (2.4.7)\n",
      "Installing collected packages: sacrebleu\n",
      "Successfully installed sacrebleu-2.0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Cloning into 'language_repo'...\n",
      "WARNING: Running pip as root will break packages and permissions. You should install packages reliably by using venv: https://pip.pypa.io/warnings/venv\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "git clone https://github.com/google-research/language.git language_repo\n",
    "cd language_repo\n",
    "pip3 install -r language/totto/eval_requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "974d7fe6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:42:27.204783Z",
     "iopub.status.busy": "2021-08-23T15:42:27.203997Z",
     "iopub.status.idle": "2021-08-23T15:42:27.864790Z",
     "shell.execute_reply": "2021-08-23T15:42:27.864094Z",
     "shell.execute_reply.started": "2021-08-23T15:27:29.014218Z"
    },
    "papermill": {
     "duration": 0.693475,
     "end_time": "2021-08-23T15:42:27.864919",
     "exception": false,
     "start_time": "2021-08-23T15:42:27.171444",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__notebook__.ipynb  corrects.txt  language_repo  outputs.txt\r\n"
     ]
    }
   ],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "8de398e1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:42:27.921845Z",
     "iopub.status.busy": "2021-08-23T15:42:27.920638Z",
     "iopub.status.idle": "2021-08-23T15:42:42.432588Z",
     "shell.execute_reply": "2021-08-23T15:42:42.432057Z",
     "shell.execute_reply.started": "2021-08-23T15:27:29.697686Z"
    },
    "papermill": {
     "duration": 14.542943,
     "end_time": "2021-08-23T15:42:42.432727",
     "exception": false,
     "start_time": "2021-08-23T15:42:27.889784",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/working/language_repo\n",
      "Running with the following variables:\r\n",
      "PREDICTION_PATH   : language/totto/sample/output_sample.txt\r\n",
      "TARGET_PATH       : language/totto/sample/dev_sample.jsonl \r\n",
      "BLEURT_CKPT       : unset \r\n",
      "OUTPUT_DIR        : temp\r\n",
      "MODE              : test\r\n",
      "Creating Output directory.\r\n",
      "Cloning moses for BLEU script.\r\n",
      "Cloning into 'temp/mosesdecoder'...\r\n",
      "remote: Enumerating objects: 148070, done.\u001b[K\r\n",
      "remote: Counting objects: 100% (498/498), done.\u001b[K\r\n",
      "remote: Compressing objects: 100% (206/206), done.\u001b[K\r\n",
      "remote: Total 148070 (delta 315), reused 433 (delta 289), pack-reused 147572\u001b[K\r\n",
      "Receiving objects: 100% (148070/148070), 129.86 MiB | 19.84 MiB/s, done.\r\n",
      "Resolving deltas: 100% (114341/114341), done.\r\n",
      "Writing references.\r\n",
      "Writing tables in PARENT format.\r\n",
      "Preparing predictions.\r\n",
      "Writing predictions.\r\n",
      "Running detokenizers.\r\n",
      "======== EVALUATE OVERALL ========\r\n",
      "Computing BLEU (overall)\r\n",
      "{\r\n",
      " \"name\": \"BLEU\",\r\n",
      " \"score\": 45.5,\r\n",
      " \"signature\": \"nrefs:3|case:mixed|eff:no|tok:13a|smooth:exp|version:2.0.0\",\r\n",
      " \"verbose_score\": \"86.0/63.5/44.7/31.0 (BP = 0.869 ratio = 0.877 hyp_len = 57 ref_len = 65)\",\r\n",
      " \"nrefs\": \"3\",\r\n",
      " \"case\": \"mixed\",\r\n",
      " \"eff\": \"no\",\r\n",
      " \"tok\": \"13a\",\r\n",
      " \"smooth\": \"exp\",\r\n",
      " \"version\": \"2.0.0\"\r\n",
      "}\r\n",
      "\u001b[0mComputing PARENT (overall)\r\n",
      "Traceback (most recent call last):\r\n",
      "  File \"/opt/conda/lib/python3.7/runpy.py\", line 193, in _run_module_as_main\r\n",
      "    \"__main__\", mod_spec)\r\n",
      "  File \"/opt/conda/lib/python3.7/runpy.py\", line 85, in _run_code\r\n",
      "    exec(code, run_globals)\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 472, in <module>\r\n",
      "    app.run(main)\r\n",
      "  File \"/opt/conda/lib/python3.7/site-packages/absl/app.py\", line 303, in run\r\n",
      "    _run_main(main, args)\r\n",
      "  File \"/opt/conda/lib/python3.7/site-packages/absl/app.py\", line 251, in _run_main\r\n",
      "    sys.exit(main(argv))\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 444, in main\r\n",
      "    references = _text_reference_reader(FLAGS.reference_path)\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 120, in _text_reference_reader\r\n",
      "    references0 = _text_reader(multi_reference_paths[0])\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 89, in _text_reader\r\n",
      "    line = _normalize_text(line)\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 74, in _normalize_text\r\n",
      "    return tokenize_fn(s.strip().lower())\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 73, in <lambda>\r\n",
      "    tokenize_fn = lambda x: sacrebleu.tokenizers.Tokenizer13a()(x)\r\n",
      "AttributeError: module 'sacrebleu.tokenizers' has no attribute 'Tokenizer13a'\r\n",
      "======== EVALUATE OVERLAP SUBSET ========\r\n",
      "Computing BLEU (overlap subset)\r\n",
      "{\r\n",
      " \"name\": \"BLEU\",\r\n",
      " \"score\": 37.2,\r\n",
      " \"signature\": \"nrefs:3|case:mixed|eff:no|tok:13a|smooth:exp|version:2.0.0\",\r\n",
      " \"verbose_score\": \"84.8/56.7/37.0/25.0 (BP = 0.809 ratio = 0.825 hyp_len = 33 ref_len = 40)\",\r\n",
      " \"nrefs\": \"3\",\r\n",
      " \"case\": \"mixed\",\r\n",
      " \"eff\": \"no\",\r\n",
      " \"tok\": \"13a\",\r\n",
      " \"smooth\": \"exp\",\r\n",
      " \"version\": \"2.0.0\"\r\n",
      "}\r\n",
      "\u001b[0mComputing PARENT (overlap subset)\r\n",
      "Traceback (most recent call last):\r\n",
      "  File \"/opt/conda/lib/python3.7/runpy.py\", line 193, in _run_module_as_main\r\n",
      "    \"__main__\", mod_spec)\r\n",
      "  File \"/opt/conda/lib/python3.7/runpy.py\", line 85, in _run_code\r\n",
      "    exec(code, run_globals)\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 472, in <module>\r\n",
      "    app.run(main)\r\n",
      "  File \"/opt/conda/lib/python3.7/site-packages/absl/app.py\", line 303, in run\r\n",
      "    _run_main(main, args)\r\n",
      "  File \"/opt/conda/lib/python3.7/site-packages/absl/app.py\", line 251, in _run_main\r\n",
      "    sys.exit(main(argv))\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 444, in main\r\n",
      "    references = _text_reference_reader(FLAGS.reference_path)\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 120, in _text_reference_reader\r\n",
      "    references0 = _text_reader(multi_reference_paths[0])\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 89, in _text_reader\r\n",
      "    line = _normalize_text(line)\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 74, in _normalize_text\r\n",
      "    return tokenize_fn(s.strip().lower())\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 73, in <lambda>\r\n",
      "    tokenize_fn = lambda x: sacrebleu.tokenizers.Tokenizer13a()(x)\r\n",
      "AttributeError: module 'sacrebleu.tokenizers' has no attribute 'Tokenizer13a'\r\n",
      "======== EVALUATE NON-OVERLAP SUBSET ========\r\n",
      "Computing BLEU (non-overlap subset)\r\n",
      "{\r\n",
      " \"name\": \"BLEU\",\r\n",
      " \"score\": 58.3,\r\n",
      " \"signature\": \"nrefs:3|case:mixed|eff:no|tok:13a|smooth:exp|version:2.0.0\",\r\n",
      " \"verbose_score\": \"87.5/72.7/55.0/38.9 (BP = 0.959 ratio = 0.960 hyp_len = 24 ref_len = 25)\",\r\n",
      " \"nrefs\": \"3\",\r\n",
      " \"case\": \"mixed\",\r\n",
      " \"eff\": \"no\",\r\n",
      " \"tok\": \"13a\",\r\n",
      " \"smooth\": \"exp\",\r\n",
      " \"version\": \"2.0.0\"\r\n",
      "}\r\n",
      "\u001b[0mComputing PARENT (non-overlap subset)\r\n",
      "Traceback (most recent call last):\r\n",
      "  File \"/opt/conda/lib/python3.7/runpy.py\", line 193, in _run_module_as_main\r\n",
      "    \"__main__\", mod_spec)\r\n",
      "  File \"/opt/conda/lib/python3.7/runpy.py\", line 85, in _run_code\r\n",
      "    exec(code, run_globals)\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 472, in <module>\r\n",
      "    app.run(main)\r\n",
      "  File \"/opt/conda/lib/python3.7/site-packages/absl/app.py\", line 303, in run\r\n",
      "    _run_main(main, args)\r\n",
      "  File \"/opt/conda/lib/python3.7/site-packages/absl/app.py\", line 251, in _run_main\r\n",
      "    sys.exit(main(argv))\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 444, in main\r\n",
      "    references = _text_reference_reader(FLAGS.reference_path)\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 120, in _text_reference_reader\r\n",
      "    references0 = _text_reader(multi_reference_paths[0])\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 89, in _text_reader\r\n",
      "    line = _normalize_text(line)\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 74, in _normalize_text\r\n",
      "    return tokenize_fn(s.strip().lower())\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 73, in <lambda>\r\n",
      "    tokenize_fn = lambda x: sacrebleu.tokenizers.Tokenizer13a()(x)\r\n",
      "AttributeError: module 'sacrebleu.tokenizers' has no attribute 'Tokenizer13a'\r\n"
     ]
    }
   ],
   "source": [
    "%cd language_repo\n",
    "!bash language/totto/totto_eval.sh --prediction_path language/totto/sample/output_sample.txt --target_path language/totto/sample/dev_sample.jsonl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "6b173c33",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:42:42.561313Z",
     "iopub.status.busy": "2021-08-23T15:42:42.560717Z",
     "iopub.status.idle": "2021-08-23T15:42:42.565328Z",
     "shell.execute_reply": "2021-08-23T15:42:42.565698Z",
     "shell.execute_reply.started": "2021-08-23T15:27:43.454569Z"
    },
    "papermill": {
     "duration": 0.075835,
     "end_time": "2021-08-23T15:42:42.565882",
     "exception": false,
     "start_time": "2021-08-23T15:42:42.490047",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7700"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "7fb17535",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-23T15:42:42.703988Z",
     "iopub.status.busy": "2021-08-23T15:42:42.686006Z",
     "iopub.status.idle": "2021-08-23T15:43:55.152016Z",
     "shell.execute_reply": "2021-08-23T15:43:55.151522Z",
     "shell.execute_reply.started": "2021-08-23T15:27:43.461877Z"
    },
    "executionInfo": {
     "elapsed": 189295,
     "status": "ok",
     "timestamp": 1619773336534,
     "user": {
      "displayName": "Aishik Rakshit",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj51jIPTS5SPRR4YiqMU00JQ0UuzUavXBl9dsaRDw=s64",
      "userId": "15122983906308089605"
     },
     "user_tz": -330
    },
    "id": "xDQG6rMeNzX8",
    "papermill": {
     "duration": 72.529427,
     "end_time": "2021-08-23T15:43:55.152149",
     "exception": false,
     "start_time": "2021-08-23T15:42:42.622722",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running with the following variables:\r\n",
      "PREDICTION_PATH   : /kaggle/working/outputs.txt\r\n",
      "TARGET_PATH       : /kaggle/input/table-to-text-generation-dataset-google-totto/totto_data/totto_dev_data.jsonl \r\n",
      "BLEURT_CKPT       : unset \r\n",
      "OUTPUT_DIR        : /kaggle/working\r\n",
      "MODE              : test\r\n",
      "Cloning moses for BLEU script.\r\n",
      "Cloning into '/kaggle/working/mosesdecoder'...\r\n",
      "remote: Enumerating objects: 148070, done.\u001b[K\r\n",
      "remote: Counting objects: 100% (498/498), done.\u001b[K\r\n",
      "remote: Compressing objects: 100% (206/206), done.\u001b[K\r\n",
      "remote: Total 148070 (delta 315), reused 433 (delta 289), pack-reused 147572\u001b[K\r\n",
      "Receiving objects: 100% (148070/148070), 129.86 MiB | 22.12 MiB/s, done.\r\n",
      "Resolving deltas: 100% (114341/114341), done.\r\n",
      "Writing references.\r\n",
      "Writing tables in PARENT format.\r\n",
      "Preparing predictions.\r\n",
      "Writing predictions.\r\n",
      "Running detokenizers.\r\n",
      "======== EVALUATE OVERALL ========\r\n",
      "Computing BLEU (overall)\r\n",
      "{\r\n",
      " \"name\": \"BLEU\",\r\n",
      " \"score\": 32.6,\r\n",
      " \"signature\": \"nrefs:3|case:mixed|eff:no|tok:13a|smooth:exp|version:2.0.0\",\r\n",
      " \"verbose_score\": \"69.8/42.2/30.0/22.2 (BP = 0.870 ratio = 0.878 hyp_len = 111456 ref_len = 126951)\",\r\n",
      " \"nrefs\": \"3\",\r\n",
      " \"case\": \"mixed\",\r\n",
      " \"eff\": \"no\",\r\n",
      " \"tok\": \"13a\",\r\n",
      " \"smooth\": \"exp\",\r\n",
      " \"version\": \"2.0.0\"\r\n",
      "}\r\n",
      "\u001b[0mComputing PARENT (overall)\r\n",
      "Traceback (most recent call last):\r\n",
      "  File \"/opt/conda/lib/python3.7/runpy.py\", line 193, in _run_module_as_main\r\n",
      "    \"__main__\", mod_spec)\r\n",
      "  File \"/opt/conda/lib/python3.7/runpy.py\", line 85, in _run_code\r\n",
      "    exec(code, run_globals)\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 472, in <module>\r\n",
      "    app.run(main)\r\n",
      "  File \"/opt/conda/lib/python3.7/site-packages/absl/app.py\", line 303, in run\r\n",
      "    _run_main(main, args)\r\n",
      "  File \"/opt/conda/lib/python3.7/site-packages/absl/app.py\", line 251, in _run_main\r\n",
      "    sys.exit(main(argv))\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 444, in main\r\n",
      "    references = _text_reference_reader(FLAGS.reference_path)\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 120, in _text_reference_reader\r\n",
      "    references0 = _text_reader(multi_reference_paths[0])\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 89, in _text_reader\r\n",
      "    line = _normalize_text(line)\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 74, in _normalize_text\r\n",
      "    return tokenize_fn(s.strip().lower())\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 73, in <lambda>\r\n",
      "    tokenize_fn = lambda x: sacrebleu.tokenizers.Tokenizer13a()(x)\r\n",
      "AttributeError: module 'sacrebleu.tokenizers' has no attribute 'Tokenizer13a'\r\n",
      "======== EVALUATE OVERLAP SUBSET ========\r\n",
      "Computing BLEU (overlap subset)\r\n",
      "{\r\n",
      " \"name\": \"BLEU\",\r\n",
      " \"score\": 37.6,\r\n",
      " \"signature\": \"nrefs:3|case:mixed|eff:no|tok:13a|smooth:exp|version:2.0.0\",\r\n",
      " \"verbose_score\": \"72.7/47.2/35.6/27.5 (BP = 0.879 ratio = 0.886 hyp_len = 55163 ref_len = 62285)\",\r\n",
      " \"nrefs\": \"3\",\r\n",
      " \"case\": \"mixed\",\r\n",
      " \"eff\": \"no\",\r\n",
      " \"tok\": \"13a\",\r\n",
      " \"smooth\": \"exp\",\r\n",
      " \"version\": \"2.0.0\"\r\n",
      "}\r\n",
      "\u001b[0mComputing PARENT (overlap subset)\r\n",
      "Traceback (most recent call last):\r\n",
      "  File \"/opt/conda/lib/python3.7/runpy.py\", line 193, in _run_module_as_main\r\n",
      "    \"__main__\", mod_spec)\r\n",
      "  File \"/opt/conda/lib/python3.7/runpy.py\", line 85, in _run_code\r\n",
      "    exec(code, run_globals)\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 472, in <module>\r\n",
      "    app.run(main)\r\n",
      "  File \"/opt/conda/lib/python3.7/site-packages/absl/app.py\", line 303, in run\r\n",
      "    _run_main(main, args)\r\n",
      "  File \"/opt/conda/lib/python3.7/site-packages/absl/app.py\", line 251, in _run_main\r\n",
      "    sys.exit(main(argv))\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 444, in main\r\n",
      "    references = _text_reference_reader(FLAGS.reference_path)\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 120, in _text_reference_reader\r\n",
      "    references0 = _text_reader(multi_reference_paths[0])\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 89, in _text_reader\r\n",
      "    line = _normalize_text(line)\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 74, in _normalize_text\r\n",
      "    return tokenize_fn(s.strip().lower())\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 73, in <lambda>\r\n",
      "    tokenize_fn = lambda x: sacrebleu.tokenizers.Tokenizer13a()(x)\r\n",
      "AttributeError: module 'sacrebleu.tokenizers' has no attribute 'Tokenizer13a'\r\n",
      "======== EVALUATE NON-OVERLAP SUBSET ========\r\n",
      "Computing BLEU (non-overlap subset)\r\n",
      "{\r\n",
      " \"name\": \"BLEU\",\r\n",
      " \"score\": 27.5,\r\n",
      " \"signature\": \"nrefs:3|case:mixed|eff:no|tok:13a|smooth:exp|version:2.0.0\",\r\n",
      " \"verbose_score\": \"66.9/37.4/24.6/16.9 (BP = 0.862 ratio = 0.871 hyp_len = 56293 ref_len = 64666)\",\r\n",
      " \"nrefs\": \"3\",\r\n",
      " \"case\": \"mixed\",\r\n",
      " \"eff\": \"no\",\r\n",
      " \"tok\": \"13a\",\r\n",
      " \"smooth\": \"exp\",\r\n",
      " \"version\": \"2.0.0\"\r\n",
      "}\r\n",
      "\u001b[0mComputing PARENT (non-overlap subset)\r\n",
      "Traceback (most recent call last):\r\n",
      "  File \"/opt/conda/lib/python3.7/runpy.py\", line 193, in _run_module_as_main\r\n",
      "    \"__main__\", mod_spec)\r\n",
      "  File \"/opt/conda/lib/python3.7/runpy.py\", line 85, in _run_code\r\n",
      "    exec(code, run_globals)\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 472, in <module>\r\n",
      "    app.run(main)\r\n",
      "  File \"/opt/conda/lib/python3.7/site-packages/absl/app.py\", line 303, in run\r\n",
      "    _run_main(main, args)\r\n",
      "  File \"/opt/conda/lib/python3.7/site-packages/absl/app.py\", line 251, in _run_main\r\n",
      "    sys.exit(main(argv))\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 444, in main\r\n",
      "    references = _text_reference_reader(FLAGS.reference_path)\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 120, in _text_reference_reader\r\n",
      "    references0 = _text_reader(multi_reference_paths[0])\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 89, in _text_reader\r\n",
      "    line = _normalize_text(line)\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 74, in _normalize_text\r\n",
      "    return tokenize_fn(s.strip().lower())\r\n",
      "  File \"/kaggle/working/language_repo/language/totto/totto_parent_eval.py\", line 73, in <lambda>\r\n",
      "    tokenize_fn = lambda x: sacrebleu.tokenizers.Tokenizer13a()(x)\r\n",
      "AttributeError: module 'sacrebleu.tokenizers' has no attribute 'Tokenizer13a'\r\n"
     ]
    }
   ],
   "source": [
    "!bash language/totto/totto_eval.sh --prediction_path /kaggle/working/outputs.txt --target_path /kaggle/input/table-to-text-generation-dataset-google-totto/totto_data/totto_dev_data.jsonl --output_dir /kaggle/working/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a093b14e",
   "metadata": {
    "id": "YvT_1qkWGwn5",
    "papermill": {
     "duration": 0.088906,
     "end_time": "2021-08-23T15:43:55.330986",
     "exception": false,
     "start_time": "2021-08-23T15:43:55.242080",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 332.716807,
   "end_time": "2021-08-23T15:43:57.850648",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-08-23T15:38:25.133841",
   "version": "2.3.3"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "00a3030fc981420486c29e1f0a0c2a38": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_b104fc53be58422aa9eb06a213f66056",
       "placeholder": "​",
       "style": "IPY_MODEL_4a1a764da48d441b85b2d3f892e68d0f",
       "value": " 1.39M/1.39M [00:00&lt;00:00, 3.94MB/s]"
      }
     },
     "00e872414a27455c9d38003410db1dc6": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_bd2747b507a740e784b2b6ee566ac5b0",
        "IPY_MODEL_a3aa1ce3f53a42fb828713956364c05d",
        "IPY_MODEL_00a3030fc981420486c29e1f0a0c2a38"
       ],
       "layout": "IPY_MODEL_28243c4a90c5426f8cfed072fa3d24f7"
      }
     },
     "10c018c5ec734d7eabb7f7b255f02ec2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "10ce7adbc8d648fea6db8cc2cecfbb96": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "1214ae53d76a49bbb8d31ab343bb4b44": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "17580c5a94f64a8fb6716a749ede4c17": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_7b78ede7f4c84e98b9dd5c3e38c94746",
       "placeholder": "​",
       "style": "IPY_MODEL_4702e878b2a8451ab9f025c55778121c",
       "value": "100%"
      }
     },
     "28243c4a90c5426f8cfed072fa3d24f7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "351539aa363248f295f76567e6a12a74": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_76f4137dc80b4f42b50696f7439f829b",
       "placeholder": "​",
       "style": "IPY_MODEL_fb999217f46f46f6b27c7e64b302b3ac",
       "value": "Downloading: 100%"
      }
     },
     "39f572952a3b438fba5074188ea41140": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "3ef36506b4bb4e329151ae81d63b3f80": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_10ce7adbc8d648fea6db8cc2cecfbb96",
       "max": 121.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_b9d14e7bfc92425ab03509c381a43c93",
       "value": 121.0
      }
     },
     "4702e878b2a8451ab9f025c55778121c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "4a1a764da48d441b85b2d3f892e68d0f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "4a47ab3a57d94fbea12843321bfc984a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "4c9ceae5341845c79b5ecef0564d8f07": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_17580c5a94f64a8fb6716a749ede4c17",
        "IPY_MODEL_3ef36506b4bb4e329151ae81d63b3f80",
        "IPY_MODEL_9308af29794e45a9b829d6db49c0fcc7"
       ],
       "layout": "IPY_MODEL_f5a30e06ad5f43fa8e40a482e07cd215"
      }
     },
     "7062a32e4f37457f82a130704b577610": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "76f4137dc80b4f42b50696f7439f829b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "7b78ede7f4c84e98b9dd5c3e38c94746": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "7ce99b2fe40d441ebf49526f2ce22067": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_b1ae536fb9b44c54b310a0153663e387",
       "placeholder": "​",
       "style": "IPY_MODEL_fd69f7b25aa741f2be4e563ea8fe5935",
       "value": " 792k/792k [00:00&lt;00:00, 1.59MB/s]"
      }
     },
     "898d690676ac4a9da5a57a9421442d0f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "9308af29794e45a9b829d6db49c0fcc7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_f4001b55800042668f7bb973783506c0",
       "placeholder": "​",
       "style": "IPY_MODEL_db027fbc708d4a8cb74abce556962825",
       "value": " 121/121 [03:14&lt;00:00,  1.34s/it]"
      }
     },
     "a3aa1ce3f53a42fb828713956364c05d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_4a47ab3a57d94fbea12843321bfc984a",
       "max": 1389353.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_39f572952a3b438fba5074188ea41140",
       "value": 1389353.0
      }
     },
     "b104fc53be58422aa9eb06a213f66056": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "b1ae536fb9b44c54b310a0153663e387": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "b9d14e7bfc92425ab03509c381a43c93": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "bd2747b507a740e784b2b6ee566ac5b0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_10c018c5ec734d7eabb7f7b255f02ec2",
       "placeholder": "​",
       "style": "IPY_MODEL_1214ae53d76a49bbb8d31ab343bb4b44",
       "value": "Downloading: 100%"
      }
     },
     "ca409f6ad58e4c8f86ad191163039be1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_351539aa363248f295f76567e6a12a74",
        "IPY_MODEL_cf737e726fc64ae0bdbccfe54346c4be",
        "IPY_MODEL_7ce99b2fe40d441ebf49526f2ce22067"
       ],
       "layout": "IPY_MODEL_7062a32e4f37457f82a130704b577610"
      }
     },
     "cc83bba544594a7b9608444f180a6010": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "cf737e726fc64ae0bdbccfe54346c4be": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_cc83bba544594a7b9608444f180a6010",
       "max": 791656.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_898d690676ac4a9da5a57a9421442d0f",
       "value": 791656.0
      }
     },
     "db027fbc708d4a8cb74abce556962825": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "f4001b55800042668f7bb973783506c0": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "f5a30e06ad5f43fa8e40a482e07cd215": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "fb999217f46f46f6b27c7e64b302b3ac": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "fd69f7b25aa741f2be4e563ea8fe5935": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}