notebooks/02_NER_Baselines.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3a11bef0-b53a-43b1-8115-bf2bd975b701",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../util')\n",
    "sys.path.append('../experiments')\n",
    "\n",
    "import os\n",
    "# Disable weights and biases (if installed)\n",
    "os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
    "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"2\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7a1cf53c-d388-4283-9cea-3ae11f4a4ca5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import transformers\n",
    "import datasets\n",
    "from transformers import AutoModelForTokenClassification, AutoTokenizer, Trainer, TrainingArguments, pipeline, DataCollatorForTokenClassification, EarlyStoppingCallback, trainer_utils\n",
    "from huggingface_utils import load_custom_dataset, LabelAligner, compute_metrics, eval_on_test_set\n",
    "from run_experiment import get_train_args\n",
    "from convert_annotations import entity_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "350ad2ba-67e9-4daa-b8c2-b415080ebd15",
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets.logging.set_verbosity_error()\n",
    "transformers.logging.disable_default_handler()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9777251b-f8c6-448d-9c24-9bdf541890fa",
   "metadata": {},
   "source": [
    "# Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "db7ff9c5-934c-45b2-91a6-01cf12db84e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "level = 'fine' # Change to 'coarse' to look at high-level entity classes only\n",
    "spans = 'long' # Change to 'short' to consider short spans ignoring specifications"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d0880b43-b6ee-4b3e-b46a-445ff74fb542",
   "metadata": {},
   "outputs": [],
   "source": [
    "config_files = {\n",
    "    ('coarse' , 'short') : '01_ggponc_coarse_short.yaml',\n",
    "    ('fine', 'short') : '02_ggponc_fine_short.yaml',\n",
    "    ('coarse' , 'long' ) : '03_ggponc_coarse_long.yaml',\n",
    "    ('fine', 'long' ) : '04_ggponc_fine_long.yaml'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "33ca90fa-6e16-41c2-8d1e-164ce4a4222c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import hydra\n",
    "from hydra import compose, initialize\n",
    "\n",
    "hydra.core.global_hydra.GlobalHydra.instance().clear()\n",
    "initialize(config_path=Path('..') / 'experiments', job_name='foo')\n",
    "config = compose(config_name=config_files[(level, spans)], overrides=['cuda=0', 'link=false'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0fba68f4-1216-4902-8d1c-c881b9bcacd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_file = config['train_dataset']\n",
    "dev_file = config['dev_dataset']\n",
    "test_file = config['test_dataset']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45224845-4d54-4877-8d46-5157eb290b7d",
   "metadata": {},
   "source": [
    "# Setup IOB-encoded dataset with train / dev / test splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "75372f49-e40d-4c98-9479-ea6d03b0b0b0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "07504f6091c842d08ed005fe23de119d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aca8d84287d241379a4a1daaaa17eba4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/59515 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f94445c7839540b8b6df6b83d4691a4f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12770 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3b9c048066064aefa917e97c508ff235",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13714 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d2855c63ac0843ce992c78a605101f1d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/6 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c4bdfee4bc8a4f67ba12151da1a9ed63",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b5cf7a206aa43a3a37ec07185ede1de",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataset, tags = load_custom_dataset(train=train_file, dev=dev_file, test=test_file, tag_strings=config['task'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "167acfe0-7690-4082-878c-cabc210eed75",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(config['base_model_checkpoint'])\n",
    "assert isinstance(tokenizer, transformers.PreTrainedTokenizerFast)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a0fbcf2e-5b4a-442b-a18f-d5992f0a5f52",
   "metadata": {},
   "outputs": [],
   "source": [
    "label_aligner = LabelAligner(tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f94d16e1-63c9-4024-992d-1141b9637b56",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1250ec80b1fa4cfa971e029f228c7883",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/60 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c49a612cc940481b858792356f8ea3cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99cb65857ef2427da3b5fa3a2900f6cb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/14 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataset = dataset.map(lambda e: label_aligner.tokenize_and_align_labels(e, config['label_all_tokens']), batched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b37f9a69-b199-4974-8a1f-d07c487fff64",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 'O',\n",
       " 1: 'B-Other_Finding',\n",
       " 2: 'I-Other_Finding',\n",
       " 3: 'B-Diagnosis_or_Pathology',\n",
       " 4: 'I-Diagnosis_or_Pathology',\n",
       " 5: 'B-Therapeutic',\n",
       " 6: 'I-Therapeutic',\n",
       " 7: 'B-Diagnostic',\n",
       " 8: 'I-Diagnostic',\n",
       " 9: 'B-Nutrient_or_Body_Substance',\n",
       " 10: 'I-Nutrient_or_Body_Substance',\n",
       " 11: 'B-External_Substance',\n",
       " 12: 'I-External_Substance',\n",
       " 13: 'B-Clinical_Drug',\n",
       " 14: 'I-Clinical_Drug'}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "id2label = dict(enumerate(tags))\n",
    "id2label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "08fcd242-41e0-420e-9fbf-2f9cdc125916",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['_tags', 'attention_mask', 'fname', 'input_ids', 'labels', 'offset_mapping', 'sentence_id', 'special_tokens_mask', 'tags', 'token_type_ids', 'tokens'],\n",
       "        num_rows: 59515\n",
       "    })\n",
       "    dev: Dataset({\n",
       "        features: ['_tags', 'attention_mask', 'fname', 'input_ids', 'labels', 'offset_mapping', 'sentence_id', 'special_tokens_mask', 'tags', 'token_type_ids', 'tokens'],\n",
       "        num_rows: 12770\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['_tags', 'attention_mask', 'fname', 'input_ids', 'labels', 'offset_mapping', 'sentence_id', 'special_tokens_mask', 'tags', 'token_type_ids', 'tokens'],\n",
       "        num_rows: 13714\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d7662b9-273a-4285-8ec0-1f0354153c1a",
   "metadata": {},
   "source": [
    "# Configure and train 🤗 token classification model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e89d59f4-5267-4023-88ab-e3ef66fa044f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from run_experiment import get_train_args"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7df4091b-00c3-43c3-8407-1363f1443881",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_train_epochs = 10 # Remove this line to train for default value of 100 epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c73ddba2-4fd5-46a2-8dcd-ff737f2a0fff",
   "metadata": {},
   "outputs": [],
   "source": [
    "config['num_train_epochs'] = num_train_epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ef69d25f-e0cb-424f-8f82-87fd3716b33c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:run_experiment:ner_baseline\n"
     ]
    }
   ],
   "source": [
    "training_args = get_train_args(cp_path='../ner_results', run_name='ner_baseline', report_to=[], **config, resume_from_checkpoint=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "37a4e392-7ac5-4276-aea0-09741f439246",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at deepset/gbert-base were not used when initializing BertForTokenClassification: ['cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight']\n",
      "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForTokenClassification were not initialized from the model checkpoint at deepset/gbert-base and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "def model_init():\n",
    "    return AutoModelForTokenClassification.from_pretrained(\n",
    "        config['base_model_checkpoint'],\n",
    "        num_labels=len(tags), \n",
    "        id2label=id2label,\n",
    "    )\n",
    "\n",
    "data_collator = DataCollatorForTokenClassification(tokenizer)\n",
    "tr = Trainer(\n",
    "    args=training_args,\n",
    "    model_init=model_init,\n",
    "    train_dataset=dataset[\"train\"],\n",
    "    eval_dataset=dataset[\"dev\"],\n",
    "    data_collator=data_collator,\n",
    "    tokenizer=tokenizer,\n",
    "    compute_metrics=compute_metrics(tags, True),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "29759ecf-e4b4-47d1-825c-7a1f3a0acd87",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at deepset/gbert-base were not used when initializing BertForTokenClassification: ['cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight']\n",
      "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForTokenClassification were not initialized from the model checkpoint at deepset/gbert-base and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='18600' max='18600' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [18600/18600 35:54, Epoch 10/10]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Clinical Drug Precision</th>\n",
       "      <th>Clinical Drug Recall</th>\n",
       "      <th>Clinical Drug F1</th>\n",
       "      <th>Clinical Drug Number</th>\n",
       "      <th>Diagnosis Or Pathology Precision</th>\n",
       "      <th>Diagnosis Or Pathology Recall</th>\n",
       "      <th>Diagnosis Or Pathology F1</th>\n",
       "      <th>Diagnosis Or Pathology Number</th>\n",
       "      <th>Diagnostic Precision</th>\n",
       "      <th>Diagnostic Recall</th>\n",
       "      <th>Diagnostic F1</th>\n",
       "      <th>Diagnostic Number</th>\n",
       "      <th>External Substance Precision</th>\n",
       "      <th>External Substance Recall</th>\n",
       "      <th>External Substance F1</th>\n",
       "      <th>External Substance Number</th>\n",
       "      <th>Nutrient Or Body Substance Precision</th>\n",
       "      <th>Nutrient Or Body Substance Recall</th>\n",
       "      <th>Nutrient Or Body Substance F1</th>\n",
       "      <th>Nutrient Or Body Substance Number</th>\n",
       "      <th>Other Finding Precision</th>\n",
       "      <th>Other Finding Recall</th>\n",
       "      <th>Other Finding F1</th>\n",
       "      <th>Other Finding Number</th>\n",
       "      <th>Therapeutic Precision</th>\n",
       "      <th>Therapeutic Recall</th>\n",
       "      <th>Therapeutic F1</th>\n",
       "      <th>Therapeutic Number</th>\n",
       "      <th>Overall Precision</th>\n",
       "      <th>Overall Recall</th>\n",
       "      <th>Overall F1</th>\n",
       "      <th>Overall Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.783700</td>\n",
       "      <td>0.771889</td>\n",
       "      <td>0.640820</td>\n",
       "      <td>0.706690</td>\n",
       "      <td>0.672145</td>\n",
       "      <td>2257</td>\n",
       "      <td>0.689686</td>\n",
       "      <td>0.758792</td>\n",
       "      <td>0.722591</td>\n",
       "      <td>9042</td>\n",
       "      <td>0.641399</td>\n",
       "      <td>0.693838</td>\n",
       "      <td>0.666589</td>\n",
       "      <td>4122</td>\n",
       "      <td>0.452830</td>\n",
       "      <td>0.279070</td>\n",
       "      <td>0.345324</td>\n",
       "      <td>172</td>\n",
       "      <td>0.557447</td>\n",
       "      <td>0.478976</td>\n",
       "      <td>0.515241</td>\n",
       "      <td>547</td>\n",
       "      <td>0.533783</td>\n",
       "      <td>0.562993</td>\n",
       "      <td>0.547999</td>\n",
       "      <td>6469</td>\n",
       "      <td>0.658216</td>\n",
       "      <td>0.749292</td>\n",
       "      <td>0.700807</td>\n",
       "      <td>7415</td>\n",
       "      <td>0.636119</td>\n",
       "      <td>0.693578</td>\n",
       "      <td>0.663607</td>\n",
       "      <td>0.907070</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.727600</td>\n",
       "      <td>0.777850</td>\n",
       "      <td>0.640922</td>\n",
       "      <td>0.702260</td>\n",
       "      <td>0.670190</td>\n",
       "      <td>2257</td>\n",
       "      <td>0.699415</td>\n",
       "      <td>0.766866</td>\n",
       "      <td>0.731589</td>\n",
       "      <td>9042</td>\n",
       "      <td>0.639650</td>\n",
       "      <td>0.708394</td>\n",
       "      <td>0.672269</td>\n",
       "      <td>4122</td>\n",
       "      <td>0.590909</td>\n",
       "      <td>0.226744</td>\n",
       "      <td>0.327731</td>\n",
       "      <td>172</td>\n",
       "      <td>0.494118</td>\n",
       "      <td>0.614260</td>\n",
       "      <td>0.547677</td>\n",
       "      <td>547</td>\n",
       "      <td>0.555911</td>\n",
       "      <td>0.590199</td>\n",
       "      <td>0.572543</td>\n",
       "      <td>6469</td>\n",
       "      <td>0.660948</td>\n",
       "      <td>0.763722</td>\n",
       "      <td>0.708628</td>\n",
       "      <td>7415</td>\n",
       "      <td>0.642693</td>\n",
       "      <td>0.709266</td>\n",
       "      <td>0.674341</td>\n",
       "      <td>0.907181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.682900</td>\n",
       "      <td>0.779998</td>\n",
       "      <td>0.667372</td>\n",
       "      <td>0.698715</td>\n",
       "      <td>0.682684</td>\n",
       "      <td>2257</td>\n",
       "      <td>0.739320</td>\n",
       "      <td>0.750276</td>\n",
       "      <td>0.744758</td>\n",
       "      <td>9042</td>\n",
       "      <td>0.646110</td>\n",
       "      <td>0.719311</td>\n",
       "      <td>0.680748</td>\n",
       "      <td>4122</td>\n",
       "      <td>0.518519</td>\n",
       "      <td>0.325581</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>172</td>\n",
       "      <td>0.528440</td>\n",
       "      <td>0.526508</td>\n",
       "      <td>0.527473</td>\n",
       "      <td>547</td>\n",
       "      <td>0.564177</td>\n",
       "      <td>0.604730</td>\n",
       "      <td>0.583750</td>\n",
       "      <td>6469</td>\n",
       "      <td>0.690718</td>\n",
       "      <td>0.752664</td>\n",
       "      <td>0.720361</td>\n",
       "      <td>7415</td>\n",
       "      <td>0.665608</td>\n",
       "      <td>0.704869</td>\n",
       "      <td>0.684676</td>\n",
       "      <td>0.909273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.647900</td>\n",
       "      <td>0.801537</td>\n",
       "      <td>0.654618</td>\n",
       "      <td>0.722198</td>\n",
       "      <td>0.686750</td>\n",
       "      <td>2257</td>\n",
       "      <td>0.730420</td>\n",
       "      <td>0.765317</td>\n",
       "      <td>0.747462</td>\n",
       "      <td>9042</td>\n",
       "      <td>0.667199</td>\n",
       "      <td>0.709607</td>\n",
       "      <td>0.687750</td>\n",
       "      <td>4122</td>\n",
       "      <td>0.465909</td>\n",
       "      <td>0.238372</td>\n",
       "      <td>0.315385</td>\n",
       "      <td>172</td>\n",
       "      <td>0.549729</td>\n",
       "      <td>0.555759</td>\n",
       "      <td>0.552727</td>\n",
       "      <td>547</td>\n",
       "      <td>0.569381</td>\n",
       "      <td>0.614005</td>\n",
       "      <td>0.590852</td>\n",
       "      <td>6469</td>\n",
       "      <td>0.685938</td>\n",
       "      <td>0.766419</td>\n",
       "      <td>0.723949</td>\n",
       "      <td>7415</td>\n",
       "      <td>0.665891</td>\n",
       "      <td>0.715261</td>\n",
       "      <td>0.689694</td>\n",
       "      <td>0.910361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.626800</td>\n",
       "      <td>0.821937</td>\n",
       "      <td>0.621163</td>\n",
       "      <td>0.762074</td>\n",
       "      <td>0.684441</td>\n",
       "      <td>2257</td>\n",
       "      <td>0.729642</td>\n",
       "      <td>0.770958</td>\n",
       "      <td>0.749731</td>\n",
       "      <td>9042</td>\n",
       "      <td>0.671638</td>\n",
       "      <td>0.721009</td>\n",
       "      <td>0.695449</td>\n",
       "      <td>4122</td>\n",
       "      <td>0.532609</td>\n",
       "      <td>0.284884</td>\n",
       "      <td>0.371212</td>\n",
       "      <td>172</td>\n",
       "      <td>0.471698</td>\n",
       "      <td>0.594150</td>\n",
       "      <td>0.525890</td>\n",
       "      <td>547</td>\n",
       "      <td>0.560083</td>\n",
       "      <td>0.626836</td>\n",
       "      <td>0.591582</td>\n",
       "      <td>6469</td>\n",
       "      <td>0.699393</td>\n",
       "      <td>0.745516</td>\n",
       "      <td>0.721718</td>\n",
       "      <td>7415</td>\n",
       "      <td>0.661708</td>\n",
       "      <td>0.720091</td>\n",
       "      <td>0.689666</td>\n",
       "      <td>0.907629</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.608700</td>\n",
       "      <td>0.828534</td>\n",
       "      <td>0.654184</td>\n",
       "      <td>0.758529</td>\n",
       "      <td>0.702503</td>\n",
       "      <td>2257</td>\n",
       "      <td>0.734447</td>\n",
       "      <td>0.778146</td>\n",
       "      <td>0.755665</td>\n",
       "      <td>9042</td>\n",
       "      <td>0.685411</td>\n",
       "      <td>0.721494</td>\n",
       "      <td>0.702990</td>\n",
       "      <td>4122</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>0.319767</td>\n",
       "      <td>0.375427</td>\n",
       "      <td>172</td>\n",
       "      <td>0.511811</td>\n",
       "      <td>0.594150</td>\n",
       "      <td>0.549915</td>\n",
       "      <td>547</td>\n",
       "      <td>0.565603</td>\n",
       "      <td>0.630391</td>\n",
       "      <td>0.596242</td>\n",
       "      <td>6469</td>\n",
       "      <td>0.705785</td>\n",
       "      <td>0.746999</td>\n",
       "      <td>0.725808</td>\n",
       "      <td>7415</td>\n",
       "      <td>0.671376</td>\n",
       "      <td>0.723388</td>\n",
       "      <td>0.696412</td>\n",
       "      <td>0.909424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.596800</td>\n",
       "      <td>0.841437</td>\n",
       "      <td>0.655376</td>\n",
       "      <td>0.737262</td>\n",
       "      <td>0.693912</td>\n",
       "      <td>2257</td>\n",
       "      <td>0.750751</td>\n",
       "      <td>0.773833</td>\n",
       "      <td>0.762117</td>\n",
       "      <td>9042</td>\n",
       "      <td>0.669423</td>\n",
       "      <td>0.726589</td>\n",
       "      <td>0.696836</td>\n",
       "      <td>4122</td>\n",
       "      <td>0.546296</td>\n",
       "      <td>0.343023</td>\n",
       "      <td>0.421429</td>\n",
       "      <td>172</td>\n",
       "      <td>0.547273</td>\n",
       "      <td>0.550274</td>\n",
       "      <td>0.548769</td>\n",
       "      <td>547</td>\n",
       "      <td>0.568194</td>\n",
       "      <td>0.636265</td>\n",
       "      <td>0.600306</td>\n",
       "      <td>6469</td>\n",
       "      <td>0.708418</td>\n",
       "      <td>0.751315</td>\n",
       "      <td>0.729236</td>\n",
       "      <td>7415</td>\n",
       "      <td>0.676127</td>\n",
       "      <td>0.722855</td>\n",
       "      <td>0.698711</td>\n",
       "      <td>0.909643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.589000</td>\n",
       "      <td>0.847882</td>\n",
       "      <td>0.660407</td>\n",
       "      <td>0.747895</td>\n",
       "      <td>0.701434</td>\n",
       "      <td>2257</td>\n",
       "      <td>0.731917</td>\n",
       "      <td>0.781132</td>\n",
       "      <td>0.755724</td>\n",
       "      <td>9042</td>\n",
       "      <td>0.676525</td>\n",
       "      <td>0.715915</td>\n",
       "      <td>0.695662</td>\n",
       "      <td>4122</td>\n",
       "      <td>0.484615</td>\n",
       "      <td>0.366279</td>\n",
       "      <td>0.417219</td>\n",
       "      <td>172</td>\n",
       "      <td>0.535336</td>\n",
       "      <td>0.553931</td>\n",
       "      <td>0.544474</td>\n",
       "      <td>547</td>\n",
       "      <td>0.578289</td>\n",
       "      <td>0.621734</td>\n",
       "      <td>0.599225</td>\n",
       "      <td>6469</td>\n",
       "      <td>0.710516</td>\n",
       "      <td>0.755361</td>\n",
       "      <td>0.732253</td>\n",
       "      <td>7415</td>\n",
       "      <td>0.675690</td>\n",
       "      <td>0.722455</td>\n",
       "      <td>0.698291</td>\n",
       "      <td>0.910024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.582400</td>\n",
       "      <td>0.855722</td>\n",
       "      <td>0.663909</td>\n",
       "      <td>0.746566</td>\n",
       "      <td>0.702815</td>\n",
       "      <td>2257</td>\n",
       "      <td>0.740437</td>\n",
       "      <td>0.779252</td>\n",
       "      <td>0.759349</td>\n",
       "      <td>9042</td>\n",
       "      <td>0.681549</td>\n",
       "      <td>0.725861</td>\n",
       "      <td>0.703008</td>\n",
       "      <td>4122</td>\n",
       "      <td>0.524194</td>\n",
       "      <td>0.377907</td>\n",
       "      <td>0.439189</td>\n",
       "      <td>172</td>\n",
       "      <td>0.537801</td>\n",
       "      <td>0.572212</td>\n",
       "      <td>0.554473</td>\n",
       "      <td>547</td>\n",
       "      <td>0.578171</td>\n",
       "      <td>0.636265</td>\n",
       "      <td>0.605829</td>\n",
       "      <td>6469</td>\n",
       "      <td>0.707249</td>\n",
       "      <td>0.755226</td>\n",
       "      <td>0.730451</td>\n",
       "      <td>7415</td>\n",
       "      <td>0.677820</td>\n",
       "      <td>0.726652</td>\n",
       "      <td>0.701387</td>\n",
       "      <td>0.910728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>0.576200</td>\n",
       "      <td>0.861115</td>\n",
       "      <td>0.660452</td>\n",
       "      <td>0.737705</td>\n",
       "      <td>0.696944</td>\n",
       "      <td>2257</td>\n",
       "      <td>0.744181</td>\n",
       "      <td>0.781464</td>\n",
       "      <td>0.762367</td>\n",
       "      <td>9042</td>\n",
       "      <td>0.681127</td>\n",
       "      <td>0.727559</td>\n",
       "      <td>0.703578</td>\n",
       "      <td>4122</td>\n",
       "      <td>0.533898</td>\n",
       "      <td>0.366279</td>\n",
       "      <td>0.434483</td>\n",
       "      <td>172</td>\n",
       "      <td>0.532986</td>\n",
       "      <td>0.561243</td>\n",
       "      <td>0.546750</td>\n",
       "      <td>547</td>\n",
       "      <td>0.578977</td>\n",
       "      <td>0.638584</td>\n",
       "      <td>0.607321</td>\n",
       "      <td>6469</td>\n",
       "      <td>0.706823</td>\n",
       "      <td>0.757249</td>\n",
       "      <td>0.731167</td>\n",
       "      <td>7415</td>\n",
       "      <td>0.678616</td>\n",
       "      <td>0.727618</td>\n",
       "      <td>0.702263</td>\n",
       "      <td>0.911163</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_result = tr.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "539fc922-1ab1-47e2-badb-d52f43999213",
   "metadata": {},
   "source": [
    "# Evaluate Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f9d4552d-4ac5-4e5f-9f2e-f794cd91ab5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tr.model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "a8d4df77-71ae-41f5-9833-ee7eb2e8ac13",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='429' max='429' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [429/429 00:09]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "13714it [00:05, 2561.41it/s]\n"
     ]
    }
   ],
   "source": [
    "test_metrics = eval_on_test_set(dataset[\"test\"], tr, tokenizer, \"test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "4065841a-2cf2-4bb0-876a-d1f6b2118cb8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "F1: 0.71\n",
      " P: 0.69\n",
      " R: 0.73\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(f\"\"\"\n",
    "F1: {test_metrics[\"test/overall_f1\"]:.2f}\n",
    " P: {test_metrics[\"test/overall_precision\"]:.2f}\n",
    " R: {test_metrics[\"test/overall_recall\"]:.2f}\n",
    "\"\"\")\n",
    "# Note: somewhat lower than the reported results when only trained for 10 epochs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca141156-6c0f-44f4-8f20-ae8cabe83089",
   "metadata": {},
   "source": [
    "### Detailed analysis of model performance\n",
    "\n",
    "See notebook: [03_NER_Analysis](03_NER_Analysis.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c12c8fbe-e707-4021-a3c6-245a2933e54f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:ggponc]",
   "language": "python",
   "name": "conda-env-ggponc-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}