From b511e1dc9bdcc5cec3c24e4448a7c7f8044a6177 Mon Sep 17 00:00:00 2001
From: huseinzol05 <husein.zol05@gmail.com>
Date: Fri, 6 Dec 2024 20:59:20 +0800
Subject: [PATCH] initial evaluation 2.1

---
 .../end-to-end/evaluate/base-2.1-en-ms.ipynb  | 335 ++++++++++++++++++
 .../end-to-end/evaluate/base-2.1-ind-ms.ipynb | 239 +++++++++++++
 .../end-to-end/evaluate/small-2.1-en-ms.ipynb | 294 +++++++++++++++
 .../evaluate/small-2.1-ind-ms.ipynb           | 231 ++++++++++++
 .../nanot5-base-multipack-compile.sh          |  28 ++
 .../nanot5-small-multipack-compile.sh         |  13 +-
 6 files changed, 1134 insertions(+), 6 deletions(-)
 create mode 100644 session/translation/end-to-end/evaluate/base-2.1-en-ms.ipynb
 create mode 100644 session/translation/end-to-end/evaluate/base-2.1-ind-ms.ipynb
 create mode 100644 session/translation/end-to-end/evaluate/small-2.1-en-ms.ipynb
 create mode 100644 session/translation/end-to-end/evaluate/small-2.1-ind-ms.ipynb
 create mode 100644 session/translation/end-to-end/nanot5-base-multipack-compile.sh

diff --git a/session/translation/end-to-end/evaluate/base-2.1-en-ms.ipynb b/session/translation/end-to-end/evaluate/base-2.1-en-ms.ipynb
new file mode 100644
index 00000000..35d1735d
--- /dev/null
+++ b/session/translation/end-to-end/evaluate/base-2.1-en-ms.ipynb
@@ -0,0 +1,335 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "3560a64c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/bjn_Latn.dev\n",
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/eng_Latn.dev\n",
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/ind_Latn.dev\n",
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/jav_Latn.dev\n",
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/zsm_Latn.dev\n",
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/zho_Hans.dev\n",
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/tam_Taml.dev"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "019bd464",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "2d713f77",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open('eng_Latn.dev') as fopen:\n",
+    "    en = fopen.read().split('\\n')\n",
+    "    \n",
+    "with open('zsm_Latn.dev') as fopen:\n",
+    "    ms = fopen.read().split('\\n')\n",
+    "    \n",
+    "en_, ms_ = [], []\n",
+    "for i in range(len(en)):\n",
+    "    if len(en[i]) and len(ms[i]):\n",
+    "        en_.append(en[i])\n",
+    "        ms_.append(ms[i])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "076a2a37",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(997, 997)"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(en_), len(ms_)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "14d402ec",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from tqdm import tqdm\n",
+    "import requests\n",
+    "import os\n",
+    "import json"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "4795de0a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!rm -rf base-en-ms\n",
+    "!mkdir base-en-ms"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "37ac8ece",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "62383ba5f6d846a2b79873571a96b13a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "tokenizer_config.json:   0%|          | 0.00/21.0k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3146f7ae87a147f7ba53bebf1bfb2eb1",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "788d6a02976c4f10b2cbe23f36aaa3fb",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "special_tokens_map.json:   0%|          | 0.00/2.68k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "9fb7a59925d348379c921f885b726e1b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "config.json:   0%|          | 0.00/853 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "7da062dc345045daae694fa651a84b45",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "model.safetensors:   0%|          | 0.00/990M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "aeb21db8a9284b15ad9ebf2df181e745",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "generation_config.json:   0%|          | 0.00/147 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from transformers import AutoTokenizer, T5ForConditionalGeneration\n",
+    "\n",
+    "tokenizer = AutoTokenizer.from_pretrained('mesolitica/nanot5-base-malaysian-translation-v2.1')\n",
+    "model = T5ForConditionalGeneration.from_pretrained('mesolitica/nanot5-base-malaysian-translation-v2.1')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "bc6b27f8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "_ = model.cuda()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "3274b3a7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "all_special_ids = [0, 1, 2]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "4ed56e3f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|█████████████████████████████████████████| 997/997 [06:40<00:00,  2.49it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "for i in tqdm(range(len(ms_))):\n",
+    "    filename = os.path.join('base-en-ms', f'{i}.json')\n",
+    "    \n",
+    "    if os.path.exists(filename):\n",
+    "        continue\n",
+    "    \n",
+    "    headers = {\n",
+    "        'accept': 'application/json',\n",
+    "        'Content-Type': 'application/json',\n",
+    "    }\n",
+    "    \n",
+    "    input_ids = tokenizer.encode(f'terjemah ke Melayu: {en_[i]}{tokenizer.eos_token}', return_tensors = 'pt')\n",
+    "    outputs = model.generate(input_ids.cuda(), max_length = 1024, num_beams=5, early_stopping=True)\n",
+    "    outputs = [i for i in outputs[0] if i not in all_special_ids]\n",
+    "    r = tokenizer.decode(outputs, spaces_between_special_tokens = False).strip()\n",
+    "\n",
+    "    with open(filename, 'w') as fopen:\n",
+    "        json.dump({'text': ms_[i], 'r': r}, fopen)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "73e0fe41",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sacrebleu.metrics import BLEU, CHRF, TER\n",
+    "\n",
+    "chrf = CHRF(word_order = 2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "2a764b8d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "predicted = []\n",
+    "for i in range(len(ms_)):\n",
+    "    filename = os.path.join('base-en-ms', f'{i}.json')\n",
+    "    with open(filename) as fopen:\n",
+    "        d = json.load(fopen)\n",
+    "    predicted.append(d['r'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "f1772a2b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "chrF2++ = 66.57"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "score = chrf.corpus_score(predicted, [ms_])\n",
+    "score"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0e552296",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/session/translation/end-to-end/evaluate/base-2.1-ind-ms.ipynb b/session/translation/end-to-end/evaluate/base-2.1-ind-ms.ipynb
new file mode 100644
index 00000000..9d38407a
--- /dev/null
+++ b/session/translation/end-to-end/evaluate/base-2.1-ind-ms.ipynb
@@ -0,0 +1,239 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "3560a64c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/bjn_Latn.dev\n",
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/eng_Latn.dev\n",
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/ind_Latn.dev\n",
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/jav_Latn.dev\n",
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/zsm_Latn.dev\n",
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/zho_Hans.dev\n",
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/tam_Taml.dev"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "2d713f77",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open('ind_Latn.dev') as fopen:\n",
+    "    en = fopen.read().split('\\n')\n",
+    "    \n",
+    "with open('zsm_Latn.dev') as fopen:\n",
+    "    ms = fopen.read().split('\\n')\n",
+    "    \n",
+    "en_, ms_ = [], []\n",
+    "for i in range(len(en)):\n",
+    "    if len(en[i]) and len(ms[i]):\n",
+    "        en_.append(en[i])\n",
+    "        ms_.append(ms[i])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "076a2a37",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(997, 997)"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(en_), len(ms_)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "14d402ec",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from tqdm import tqdm\n",
+    "import requests\n",
+    "import os\n",
+    "import json"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "4795de0a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!rm -rf base-ind-ms\n",
+    "!mkdir base-ind-ms"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "37ac8ece",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from transformers import AutoTokenizer, T5ForConditionalGeneration\n",
+    "\n",
+    "tokenizer = AutoTokenizer.from_pretrained('mesolitica/nanot5-small-malaysian-translation-v2.1')\n",
+    "model = T5ForConditionalGeneration.from_pretrained('mesolitica/nanot5-small-malaysian-translation-v2.1')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "32c0829d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "_ = model.cuda()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "3274b3a7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "all_special_ids = [0, 1, 2]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "4ed56e3f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|█████████████████████████████████████████| 997/997 [04:22<00:00,  3.80it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "for i in tqdm(range(len(ms_))):\n",
+    "    filename = os.path.join('base-ind-ms', f'{i}.json')\n",
+    "    \n",
+    "    if os.path.exists(filename):\n",
+    "        continue\n",
+    "    \n",
+    "    headers = {\n",
+    "        'accept': 'application/json',\n",
+    "        'Content-Type': 'application/json',\n",
+    "    }\n",
+    "    \n",
+    "    input_ids = tokenizer.encode(f'terjemah ke Melayu: {en_[i]}{tokenizer.eos_token}', return_tensors = 'pt')\n",
+    "    outputs = model.generate(input_ids.cuda(), max_length = 1024, num_beams=5, early_stopping=True)\n",
+    "    outputs = [i for i in outputs[0] if i not in all_special_ids]\n",
+    "    r = tokenizer.decode(outputs, spaces_between_special_tokens = False).strip()\n",
+    "\n",
+    "    with open(filename, 'w') as fopen:\n",
+    "        json.dump({'text': ms_[i], 'r': r}, fopen)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "73e0fe41",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sacrebleu.metrics import BLEU, CHRF, TER\n",
+    "\n",
+    "chrf = CHRF(word_order = 2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "2a764b8d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "predicted = []\n",
+    "for i in range(len(ms_)):\n",
+    "    filename = os.path.join('base-ind-ms', f'{i}.json')\n",
+    "    with open(filename) as fopen:\n",
+    "        d = json.load(fopen)\n",
+    "    predicted.append(d['r'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "f1772a2b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "chrF2++ = 59.03"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "score = chrf.corpus_score(predicted, [ms_])\n",
+    "score"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "01f24ac6",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/session/translation/end-to-end/evaluate/small-2.1-en-ms.ipynb b/session/translation/end-to-end/evaluate/small-2.1-en-ms.ipynb
new file mode 100644
index 00000000..1e5918c3
--- /dev/null
+++ b/session/translation/end-to-end/evaluate/small-2.1-en-ms.ipynb
@@ -0,0 +1,294 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "3560a64c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/bjn_Latn.dev\n",
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/eng_Latn.dev\n",
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/ind_Latn.dev\n",
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/jav_Latn.dev\n",
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/zsm_Latn.dev\n",
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/zho_Hans.dev\n",
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/tam_Taml.dev"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "6e48019b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "os.environ['CUDA_VISIBLE_DEVICES'] = '1'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "2d713f77",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open('eng_Latn.dev') as fopen:\n",
+    "    en = fopen.read().split('\\n')\n",
+    "    \n",
+    "with open('zsm_Latn.dev') as fopen:\n",
+    "    ms = fopen.read().split('\\n')\n",
+    "    \n",
+    "en_, ms_ = [], []\n",
+    "for i in range(len(en)):\n",
+    "    if len(en[i]) and len(ms[i]):\n",
+    "        en_.append(en[i])\n",
+    "        ms_.append(ms[i])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "076a2a37",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(997, 997)"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(en_), len(ms_)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "14d402ec",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from tqdm import tqdm\n",
+    "import requests\n",
+    "import os\n",
+    "import json"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "4795de0a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
+      "To disable this warning, you can either:\n",
+      "\t- Avoid using `tokenizers` before the fork if possible\n",
+      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
+      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
+      "To disable this warning, you can either:\n",
+      "\t- Avoid using `tokenizers` before the fork if possible\n",
+      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
+     ]
+    }
+   ],
+   "source": [
+    "!rm -rf small-en-ms\n",
+    "!mkdir small-en-ms"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "f4fc3b40",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ba629460f07e42ef8e73f05ec1d94463",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "config.json:   0%|          | 0.00/851 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "457b0fbc3a2b4605b3aa41af13f7a4e2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "model.safetensors:   0%|          | 0.00/358M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from transformers import AutoTokenizer, T5ForConditionalGeneration\n",
+    "\n",
+    "tokenizer = AutoTokenizer.from_pretrained('mesolitica/nanot5-small-malaysian-translation-v2.1')\n",
+    "model = T5ForConditionalGeneration.from_pretrained('mesolitica/nanot5-small-malaysian-translation-v2.1')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "fbd6e7c4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "all_special_ids = [0, 1, 2]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "7c75afb7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "_ = model.cuda()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "4ed56e3f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|█████████████████████████████████████████| 997/997 [04:20<00:00,  3.83it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "for i in tqdm(range(len(ms_))):\n",
+    "    filename = os.path.join('small-en-ms', f'{i}.json')\n",
+    "    \n",
+    "    if os.path.exists(filename):\n",
+    "        continue\n",
+    "    \n",
+    "    headers = {\n",
+    "        'accept': 'application/json',\n",
+    "        'Content-Type': 'application/json',\n",
+    "    }\n",
+    "    \n",
+    "    input_ids = tokenizer.encode(f'terjemah ke Melayu: {en_[i]}{tokenizer.eos_token}', return_tensors = 'pt')\n",
+    "    outputs = model.generate(input_ids.cuda(), max_length = 1024, num_beams=5, early_stopping=True)\n",
+    "    outputs = [i for i in outputs[0] if i not in all_special_ids]\n",
+    "    r = tokenizer.decode(outputs, spaces_between_special_tokens = False).strip()\n",
+    "\n",
+    "    with open(filename, 'w') as fopen:\n",
+    "        json.dump({'text': en_[i], 'r': r}, fopen)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "2c0aedbd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sacrebleu.metrics import BLEU, CHRF, TER\n",
+    "\n",
+    "chrf = CHRF(word_order = 2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "75747378",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "predicted = []\n",
+    "for i in range(len(ms_)):\n",
+    "    filename = os.path.join('small-en-ms', f'{i}.json')\n",
+    "    with open(filename) as fopen:\n",
+    "        d = json.load(fopen)\n",
+    "    predicted.append(d['r'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "94e69780",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "chrF2++ = 66.38"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "score = chrf.corpus_score(predicted, [ms_])\n",
+    "score"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a9e1558a",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/session/translation/end-to-end/evaluate/small-2.1-ind-ms.ipynb b/session/translation/end-to-end/evaluate/small-2.1-ind-ms.ipynb
new file mode 100644
index 00000000..b36980ff
--- /dev/null
+++ b/session/translation/end-to-end/evaluate/small-2.1-ind-ms.ipynb
@@ -0,0 +1,231 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "3560a64c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/bjn_Latn.dev\n",
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/eng_Latn.dev\n",
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/ind_Latn.dev\n",
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/jav_Latn.dev\n",
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/zsm_Latn.dev\n",
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/zho_Hans.dev\n",
+    "# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/tam_Taml.dev"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "2d713f77",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open('ind_Latn.dev') as fopen:\n",
+    "    en = fopen.read().split('\\n')\n",
+    "    \n",
+    "with open('zsm_Latn.dev') as fopen:\n",
+    "    ms = fopen.read().split('\\n')\n",
+    "    \n",
+    "en_, ms_ = [], []\n",
+    "for i in range(len(en)):\n",
+    "    if len(en[i]) and len(ms[i]):\n",
+    "        en_.append(en[i])\n",
+    "        ms_.append(ms[i])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "076a2a37",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(997, 997)"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(en_), len(ms_)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "14d402ec",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from tqdm import tqdm\n",
+    "import requests\n",
+    "import os\n",
+    "import json"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "4795de0a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!rm -rf small-ind-ms\n",
+    "!mkdir small-ind-ms"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "37ac8ece",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from transformers import AutoTokenizer, T5ForConditionalGeneration\n",
+    "\n",
+    "tokenizer = AutoTokenizer.from_pretrained('mesolitica/nanot5-small-malaysian-translation-v2.1')\n",
+    "model = T5ForConditionalGeneration.from_pretrained('mesolitica/nanot5-small-malaysian-translation-v2.1')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "34d4e289",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "_ = model.cuda()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "3274b3a7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "all_special_ids = [0, 1, 2]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "4ed56e3f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|█████████████████████████████████████████| 997/997 [04:18<00:00,  3.86it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "for i in tqdm(range(len(ms_))):\n",
+    "    filename = os.path.join('small-ind-ms', f'{i}.json')\n",
+    "    \n",
+    "    if os.path.exists(filename):\n",
+    "        continue\n",
+    "    \n",
+    "    headers = {\n",
+    "        'accept': 'application/json',\n",
+    "        'Content-Type': 'application/json',\n",
+    "    }\n",
+    "    \n",
+    "    input_ids = tokenizer.encode(f'terjemah ke Melayu: {en_[i]}{tokenizer.eos_token}', return_tensors = 'pt')\n",
+    "    outputs = model.generate(input_ids.cuda(), max_length = 1024, num_beams=5, early_stopping=True)\n",
+    "    outputs = [i for i in outputs[0] if i not in all_special_ids]\n",
+    "    r = tokenizer.decode(outputs, spaces_between_special_tokens = False).strip()\n",
+    "\n",
+    "    with open(filename, 'w') as fopen:\n",
+    "        json.dump({'text': ms_[i], 'r': r}, fopen)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "73e0fe41",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sacrebleu.metrics import BLEU, CHRF, TER\n",
+    "\n",
+    "chrf = CHRF(word_order = 2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "2a764b8d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "predicted = []\n",
+    "for i in range(len(ms_)):\n",
+    "    filename = os.path.join('small-ind-ms', f'{i}.json')\n",
+    "    with open(filename) as fopen:\n",
+    "        d = json.load(fopen)\n",
+    "    predicted.append(d['r'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "f1772a2b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "chrF2++ = 59.03"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "score = chrf.corpus_score(predicted, [ms_])\n",
+    "score"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/session/translation/end-to-end/nanot5-base-multipack-compile.sh b/session/translation/end-to-end/nanot5-base-multipack-compile.sh
new file mode 100644
index 00000000..2832ab41
--- /dev/null
+++ b/session/translation/end-to-end/nanot5-base-multipack-compile.sh
@@ -0,0 +1,28 @@
+WANDB_PROJECT="nanot5-base-malaysian-cased-translation-v6-multipack-post" \
+torchrun \
+--nproc_per_node 4 \
+-m run_t5_multipack_compile \
+--model_name_or_path mesolitica/nanot5-base-malaysian-translation-v2 \
+--num_train_epochs 2 \
+--eval_steps 1000000000 \
+--logging_steps 2 \
+--save_steps 200 \
+--save_total_limit 3 \
+--do_train \
+--train_file /workspace/malaysian-translation-v2-multipack-2048-post \
+--output_dir nanot5-base-malaysian-cased-translation-v5-multipack-post \
+--dataloader_num_workers 5 \
+--dataloader_prefetch_factor 4 \
+--per_device_train_batch_size=6 \
+--per_device_eval_batch_size=3 \
+--gradient_accumulation_steps=5 \
+--max_source_length 2048 \
+--max_target_length 2048 \
+--learning_rate 2e-5 \
+--max_grad_norm 1.0 \
+--gradient_checkpointing false \
+--weight_decay 0.001 \
+--bf16 \
+--ddp_find_unused_parameters true \
+--model_revision 57cebd22c04e2c3ffe4697a1e9dbfa5839e8750c \
+--torch_compile
diff --git a/session/translation/end-to-end/nanot5-small-multipack-compile.sh b/session/translation/end-to-end/nanot5-small-multipack-compile.sh
index b23b78ce..229baa42 100644
--- a/session/translation/end-to-end/nanot5-small-multipack-compile.sh
+++ b/session/translation/end-to-end/nanot5-small-multipack-compile.sh
@@ -3,7 +3,7 @@ torchrun \
 --nproc_per_node 2 \
 -m run_t5_multipack_compile \
 --model_name_or_path mesolitica/nanot5-small-malaysian-translation-v2 \
---num_train_epochs 2 \
+--num_train_epochs 3 \
 --eval_steps 1000000000 \
 --logging_steps 2 \
 --save_steps 200 \
@@ -11,11 +11,11 @@ torchrun \
 --do_train \
 --train_file /home/husein/ssd3/t5-sdpa-multipack/packing-post \
 --output_dir nanot5-small-malaysian-cased-translation-v5-multipack-post \
---dataloader_num_workers 5 \
---dataloader_prefetch_factor 4 \
---per_device_train_batch_size=2 \
+--dataloader_num_workers 2 \
+--dataloader_prefetch_factor 1 \
+--per_device_train_batch_size=10 \
 --per_device_eval_batch_size=3 \
---gradient_accumulation_steps=8 \
+--gradient_accumulation_steps=3 \
 --max_source_length 2048 \
 --max_target_length 2048 \
 --learning_rate 2e-5 \
@@ -25,4 +25,5 @@ torchrun \
 --bf16 \
 --ddp_find_unused_parameters true \
 --model_revision d3d615448b7153d849116cf028a1290f86d7f744 \
---torch_compile
+--torch_compile \
+--torch_compile_backend inductor