ner_bert.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NER with Bert\n",
    "\n",
    "Source: https://medium.com/@andrewmarmon/fine-tuned-named-entity-recognition-with-hugging-face-bert-d51d4cb3d7b5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import itertools\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from datasets import Dataset\n",
    "from datasets import load_metric\n",
    "from transformers import AutoTokenizer\n",
    "from transformers import AutoModelForTokenClassification, TrainingArguments, Trainer\n",
    "from transformers import DataCollatorForTokenClassification\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_list = ['O','B-MISC','I-MISC','B-PER','I-PER','B-ORG','I-ORG','B-LOC','I-LOC']\n",
    "label_encoding_dict = {'I-PRG': 2,'I-I-MISC': 2, 'I-OR': 6,  'I-': 0, 'VMISC': 0, # line covers ann=tation erros in train file \n",
    "                       'O': 0, 'B-MISC': 1, 'I-MISC': 2, 'B-PER': 3, 'I-PER': 4, 'B-ORG': 5, 'I-ORG': 6, 'B-LOC': 7, 'I-LOC': 8,}\n",
    "\n",
    "task = \"ner\" \n",
    "model_checkpoint = \"distilbert-base-uncased\"\n",
    "batch_size = 16\n",
    "    \n",
    "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_all_tokens_and_ner_tags(directory):\n",
    "    return pd.concat([get_tokens_and_ner_tags(os.path.join(directory, filename)) for filename in os.listdir(directory)]).reset_index().drop('index', axis=1)\n",
    "    \n",
    "def get_tokens_and_ner_tags(filename):\n",
    "    with open(filename, 'r', encoding=\"utf8\") as f:\n",
    "        lines = f.readlines()\n",
    "        split_list = [list(y) for x, y in itertools.groupby(lines, lambda z: z == '\\n') if not x]\n",
    "        tokens = [[x.split('\\t')[0] for x in y] for y in split_list]\n",
    "        entities = [[x.split('\\t')[1][:-1] for x in y] for y in split_list] \n",
    "    return pd.DataFrame({'tokens': tokens, 'ner_tags': entities})\n",
    "  \n",
    "def get_un_token_dataset(train_directory, test_directory):\n",
    "    train_df = get_all_tokens_and_ner_tags(train_directory)\n",
    "    test_df = get_all_tokens_and_ner_tags(test_directory)\n",
    "    train_dataset = Dataset.from_pandas(train_df)\n",
    "    test_dataset = Dataset.from_pandas(test_df)\n",
    "\n",
    "    return (train_dataset, test_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_and_align_labels(examples):\n",
    "    label_all_tokens = True\n",
    "    tokenized_inputs = tokenizer(list(examples[\"tokens\"]), truncation=True, is_split_into_words=True)\n",
    "\n",
    "    labels = []\n",
    "    for i, label in enumerate(examples[f\"{task}_tags\"]):\n",
    "        word_ids = tokenized_inputs.word_ids(batch_index=i)\n",
    "        previous_word_idx = None\n",
    "        label_ids = []\n",
    "        for word_idx in word_ids:\n",
    "            if word_idx is None:\n",
    "                label_ids.append(-100)\n",
    "            elif label[word_idx] == '0':\n",
    "                label_ids.append(0)\n",
    "            elif word_idx != previous_word_idx:\n",
    "                label_ids.append(label_encoding_dict[label[word_idx]])\n",
    "            else:\n",
    "                label_ids.append(label_encoding_dict[label[word_idx]] if label_all_tokens else -100)\n",
    "            previous_word_idx = word_idx\n",
    "        labels.append(label_ids)\n",
    "        \n",
    "    tokenized_inputs[\"labels\"] = labels\n",
    "    return tokenized_inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_metrics(p):\n",
    "    predictions, labels = p\n",
    "    predictions = np.argmax(predictions, axis=2)\n",
    "\n",
    "    true_predictions = [[label_list[p] \n",
    "                         for (p, l) in zip(prediction, label) \n",
    "                         if l != -100] for prediction, label in zip(predictions, labels)]\n",
    "    true_labels = [[label_list[l] for (p, l) in zip(prediction, label) if l != -100] for prediction, label in zip(predictions, labels)]\n",
    "\n",
    "    results = metric.compute(predictions=true_predictions, references=true_labels)\n",
    "    return {\"precision\": results[\"overall_precision\"], \n",
    "            \"recall\": results[\"overall_recall\"], \"f1\": results[\"overall_f1\"], \n",
    "            \"accuracy\": results[\"overall_accuracy\"]}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read annotated data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset, test_dataset = get_un_token_dataset('../../data/UN-named-entity-recognition/tagged-training/', \n",
    "                                                   '../../data/UN-named-entity-recognition/tagged-test/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████| 4/4 [00:00<00:00, 11.76ba/s]\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:00<00:00, 19.07ba/s]\n"
     ]
    }
   ],
   "source": [
    "train_tokenized_datasets = train_dataset.map(tokenize_and_align_labels, batched=True)\n",
    "test_tokenized_datasets = test_dataset.map(tokenize_and_align_labels, batched=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForTokenClassification: ['vocab_layer_norm.weight', 'vocab_projector.bias', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_transform.weight', 'vocab_transform.bias']\n",
      "- This IS expected if you are initializing DistilBertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of DistilBertForTokenClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "/tmp/ipykernel_7980/951226195.py:14: FutureWarning: load_metric is deprecated and will be removed in the next major version of datasets. Use 'evaluate.load' instead, from the new library 🤗 Evaluate: https://huggingface.co/docs/evaluate\n",
      "  metric = load_metric(\"seqeval\")\n"
     ]
    }
   ],
   "source": [
    "model = AutoModelForTokenClassification.from_pretrained(model_checkpoint, num_labels=len(label_list))\n",
    "\n",
    "args = TrainingArguments(\n",
    "    f\"test-{task}\",\n",
    "    evaluation_strategy = \"epoch\",\n",
    "    learning_rate=1e-4,\n",
    "    per_device_train_batch_size=batch_size,\n",
    "    per_device_eval_batch_size=batch_size,\n",
    "    num_train_epochs=3,\n",
    "    weight_decay=1e-5,\n",
    ")\n",
    "\n",
    "data_collator = DataCollatorForTokenClassification(tokenizer)\n",
    "metric = load_metric(\"seqeval\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the training set don't have a corresponding argument in `DistilBertForTokenClassification.forward` and have been ignored: tokens, ner_tags. If tokens, ner_tags are not expected by `DistilBertForTokenClassification.forward`,  you can safely ignore this message.\n",
      "/home/erikt/projects/rel20/venv3/lib/python3.9/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "***** Running training *****\n",
      "  Num examples = 3657\n",
      "  Num Epochs = 3\n",
      "  Instantaneous batch size per device = 16\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 687\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='687' max='687' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [687/687 13:44, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.052608</td>\n",
       "      <td>0.789836</td>\n",
       "      <td>0.850255</td>\n",
       "      <td>0.818933</td>\n",
       "      <td>0.982015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.049980</td>\n",
       "      <td>0.830493</td>\n",
       "      <td>0.858600</td>\n",
       "      <td>0.844313</td>\n",
       "      <td>0.984916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.063400</td>\n",
       "      <td>0.057054</td>\n",
       "      <td>0.828901</td>\n",
       "      <td>0.866945</td>\n",
       "      <td>0.847496</td>\n",
       "      <td>0.985039</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForTokenClassification.forward` and have been ignored: tokens, ner_tags. If tokens, ner_tags are not expected by `DistilBertForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2074\n",
      "  Batch size = 16\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForTokenClassification.forward` and have been ignored: tokens, ner_tags. If tokens, ner_tags are not expected by `DistilBertForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2074\n",
      "  Batch size = 16\n",
      "Saving model checkpoint to test-ner/checkpoint-500\n",
      "Configuration saved in test-ner/checkpoint-500/config.json\n",
      "Model weights saved in test-ner/checkpoint-500/pytorch_model.bin\n",
      "tokenizer config file saved in test-ner/checkpoint-500/tokenizer_config.json\n",
      "Special tokens file saved in test-ner/checkpoint-500/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForTokenClassification.forward` and have been ignored: tokens, ner_tags. If tokens, ner_tags are not expected by `DistilBertForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2074\n",
      "  Batch size = 16\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForTokenClassification.forward` and have been ignored: tokens, ner_tags. If tokens, ner_tags are not expected by `DistilBertForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2074\n",
      "  Batch size = 16\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='130' max='130' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [130/130 00:38]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to un-ner.model\n",
      "Configuration saved in un-ner.model/config.json\n",
      "Model weights saved in un-ner.model/pytorch_model.bin\n",
      "tokenizer config file saved in un-ner.model/tokenizer_config.json\n",
      "Special tokens file saved in un-ner.model/special_tokens_map.json\n"
     ]
    }
   ],
   "source": [
    "trainer = Trainer(\n",
    "    model,\n",
    "    args,\n",
    "    train_dataset=train_tokenized_datasets,\n",
    "    eval_dataset=test_tokenized_datasets,\n",
    "    data_collator=data_collator,\n",
    "    tokenizer=tokenizer,\n",
    "    compute_metrics=compute_metrics\n",
    ")\n",
    "\n",
    "trainer.train()\n",
    "trainer.evaluate()\n",
    "trainer.save_model('un-ner.model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Didn't find file ./un-ner.model/added_tokens.json. We won't load it.\n",
      "loading file ./un-ner.model/vocab.txt\n",
      "loading file ./un-ner.model/tokenizer.json\n",
      "loading file None\n",
      "loading file ./un-ner.model/special_tokens_map.json\n",
      "loading file ./un-ner.model/tokenizer_config.json\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 103])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained('./un-ner.model/')\n",
    "\n",
    "paragraph = '''\n",
    "    Before proceeding further, I should like to inform members that action on draft \n",
    "    resolution iv, entitled situation of human rights of Rohingya Muslims and other \n",
    "    minorities in Myanmar is postponed to a later date to allow time for the review \n",
    "    of its programme budget implications by the fifth committee. The assembly will \n",
    "    take action on draft resolution iv as soon as the report of the fifth committee \n",
    "    on the programme budget implications is available. I now give the floor to \n",
    "    delegations wishing to deliver explanations of vote or position before voting or \n",
    "    adoption.'''\n",
    "tokens = tokenizer(paragraph)\n",
    "torch.tensor(tokens['input_ids']).unsqueeze(0).size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading configuration file ./un-ner.model/config.json\n",
      "Model config DistilBertConfig {\n",
      "  \"_name_or_path\": \"./un-ner.model/\",\n",
      "  \"activation\": \"gelu\",\n",
      "  \"architectures\": [\n",
      "    \"DistilBertForTokenClassification\"\n",
      "  ],\n",
      "  \"attention_dropout\": 0.1,\n",
      "  \"dim\": 768,\n",
      "  \"dropout\": 0.1,\n",
      "  \"hidden_dim\": 3072,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"LABEL_0\",\n",
      "    \"1\": \"LABEL_1\",\n",
      "    \"2\": \"LABEL_2\",\n",
      "    \"3\": \"LABEL_3\",\n",
      "    \"4\": \"LABEL_4\",\n",
      "    \"5\": \"LABEL_5\",\n",
      "    \"6\": \"LABEL_6\",\n",
      "    \"7\": \"LABEL_7\",\n",
      "    \"8\": \"LABEL_8\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1,\n",
      "    \"LABEL_2\": 2,\n",
      "    \"LABEL_3\": 3,\n",
      "    \"LABEL_4\": 4,\n",
      "    \"LABEL_5\": 5,\n",
      "    \"LABEL_6\": 6,\n",
      "    \"LABEL_7\": 7,\n",
      "    \"LABEL_8\": 8\n",
      "  },\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"distilbert\",\n",
      "  \"n_heads\": 12,\n",
      "  \"n_layers\": 6,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"qa_dropout\": 0.1,\n",
      "  \"seq_classif_dropout\": 0.2,\n",
      "  \"sinusoidal_pos_embds\": false,\n",
      "  \"tie_weights_\": true,\n",
      "  \"torch_dtype\": \"float32\",\n",
      "  \"transformers_version\": \"4.20.1\",\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n",
      "loading weights file ./un-ner.model/pytorch_model.bin\n",
      "All model checkpoint weights were used when initializing DistilBertForTokenClassification.\n",
      "\n",
      "All the weights of DistilBertForTokenClassification were initialized from the model checkpoint at ./un-ner.model/.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use DistilBertForTokenClassification for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "model = AutoModelForTokenClassification.from_pretrained('./un-ner.model/', num_labels=len(label_list))\n",
    "predictions = model.forward(input_ids=torch.tensor(tokens['input_ids']).unsqueeze(0), attention_mask=torch.tensor(tokens['attention_mask']).unsqueeze(0))\n",
    "predictions = torch.argmax(predictions.logits.squeeze(), axis=1)\n",
    "predictions = [label_list[i] for i in predictions]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = tokenizer.batch_decode(tokens['input_ids'])\n",
    "pd.DataFrame({'ner': predictions, 'words': words}).to_csv('un_ner.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# bert large NER\n",
    "\n",
    "Source: https://huggingface.co/dslim/bert-large-NER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from transformers import AutoTokenizer, AutoModelForTokenClassification\n",
    "from transformers import pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/erikt/projects/rel20/venv3/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'entity': 'B-PER', 'score': 0.9971501, 'index': 4, 'word': 'Wolfgang', 'start': 11, 'end': 19}, {'entity': 'B-LOC', 'score': 0.9986046, 'index': 9, 'word': 'Berlin', 'start': 34, 'end': 40}]\n"
     ]
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"dslim/bert-large-NER\")\n",
    "model = AutoModelForTokenClassification.from_pretrained(\"dslim/bert-large-NER\")\n",
    "\n",
    "nlp = pipeline(\"ner\", model=model, tokenizer=tokenizer)\n",
    "example = \"My name is Wolfgang and I live in Berlin\"\n",
    "\n",
    "ner_results = nlp(example)\n",
    "print(ner_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "example = \"SOCCER - GERMAN FIRST DIVISION SUMMARIES. BONN 1996-12-06 Summaries of matches played in the German first division on Friday : Bochum 2 (Stickroth 30th pen, Wosz 89th) Bayer Leverkusen 2 (Kirsten 18th, Ramelow 56th). Halftime 1-1. Attendance : 24,602 Werder Bremen 1 (Bode 85th) 1860 Munich 1 (Bormirow 8th). Halftime 0-1. Attendance 33,000 Karlsruhe 3 (Reich 29th, Carl 44th, Dundee 69th) Freiburg 0. Halftime 2-0. Attendance 33,000 Schalke 2 (Mulder 2nd and 27th) Hansa Rostock 0. Halftime 2-0. Attendance 29,300\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['S',\n",
       " '##OC',\n",
       " '##CE',\n",
       " '##R',\n",
       " '-',\n",
       " 'GE',\n",
       " '##R',\n",
       " '##MA',\n",
       " '##N',\n",
       " 'F',\n",
       " '##IR',\n",
       " '##ST',\n",
       " 'D',\n",
       " '##I',\n",
       " '##VI',\n",
       " '##SI',\n",
       " '##ON']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.tokenize(example)[:17]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 Soccer\n",
      "2 -\n",
      "3 Italian\n",
      "4 First\n",
      "5 Division\n",
      "6 Matches\n",
      "7 This\n",
      "8 Weekend\n",
      "9 .\n",
      "10 Rome\n",
      "11 1996\n",
      "12 -\n",
      "13 12\n",
      "14 -\n",
      "15 06\n",
      "16 Italian\n",
      "17 Serie\n",
      "18 A\n",
      "19 games\n",
      "20 to\n",
      "21 be\n",
      "22 played\n",
      "23 on\n",
      "24 Sunday\n",
      "25 (\n",
      "26 league\n",
      "27 positions\n",
      "28 in\n",
      "29 parent\n",
      "30 ##heses\n",
      "31 ,\n",
      "32 all\n",
      "33 kick\n",
      "34 -\n",
      "35 off\n",
      "36 times\n",
      "37 G\n",
      "38 ##m\n",
      "39 ##t\n",
      "40 )\n",
      "41 :\n",
      "42 Bologna\n",
      "43 (\n",
      "44 4\n",
      "45 )\n",
      "46 v\n",
      "47 Pi\n",
      "48 ##ace\n",
      "49 ##nza\n",
      "50 (\n",
      "51 13\n",
      "52 )\n",
      "53 133\n",
      "54 ##0\n",
      "55 Along\n",
      "56 with\n",
      "57 leaders\n",
      "58 Vice\n",
      "59 ##nza\n",
      "60 ,\n",
      "61 fourth\n",
      "62 -\n",
      "63 placed\n",
      "64 Bologna\n",
      "65 represent\n",
      "66 the\n",
      "67 biggest\n",
      "68 surprise\n",
      "69 of\n",
      "70 this\n",
      "71 Italian\n",
      "72 autumn\n",
      "73 .\n",
      "74 Led\n",
      "75 as\n",
      "76 usual\n",
      "77 by\n",
      "78 S\n",
      "79 ##wed\n",
      "80 ##e\n",
      "81 Ken\n",
      "82 ##net\n",
      "83 Anders\n",
      "84 ##son\n",
      "85 and\n",
      "86 Russian\n",
      "87 Igor\n",
      "88 Ko\n",
      "89 ##ly\n",
      "90 ##van\n",
      "91 ##ov\n",
      "92 in\n",
      "93 attack\n",
      "94 ,\n",
      "95 Bologna\n",
      "96 can\n",
      "97 expect\n",
      "98 a\n",
      "99 tough\n",
      "100 home\n",
      "101 match\n",
      "102 against\n",
      "103 a\n",
      "104 Pi\n",
      "105 ##ace\n",
      "106 ##nza\n",
      "107 side\n",
      "108 still\n",
      "109 ex\n",
      "110 ##ult\n",
      "111 ##ant\n",
      "112 after\n",
      "113 a\n",
      "114 3\n",
      "115 -\n",
      "116 2\n",
      "117 league\n",
      "118 win\n",
      "119 over\n",
      "120 A\n",
      "121 ##c\n",
      "122 Milan\n",
      "123 last\n",
      "124 Sunday\n",
      "125 .\n",
      "126 C\n",
      "127 ##ag\n",
      "128 ##lia\n",
      "129 ##ri\n",
      "130 (\n",
      "131 16\n",
      "132 )\n",
      "133 v\n",
      "134 Reg\n",
      "135 ##gia\n",
      "136 ##na\n",
      "137 (\n",
      "138 18\n",
      "139 )\n",
      "140 153\n",
      "141 ##0\n",
      "142 C\n",
      "143 ##ag\n",
      "144 ##lia\n",
      "145 ##ri\n",
      "146 start\n",
      "147 favourite\n",
      "148 in\n",
      "149 this\n",
      "150 relegation\n",
      "151 scrap\n",
      "152 following\n",
      "153 draws\n",
      "154 with\n",
      "155 Na\n",
      "156 ##poli\n",
      "157 and\n",
      "158 Inter\n",
      "159 in\n",
      "160 last\n",
      "161 two\n",
      "162 out\n",
      "163 ##ings\n",
      "164 but\n",
      "165 will\n",
      "166 be\n",
      "167 without\n",
      "168 suspended\n",
      "169 i\n",
      "170 ##S\n",
      "171 ##wi\n",
      "172 ##ss\n",
      "173 defender\n",
      "174 Ramon\n",
      "175 Vega\n",
      "176 .\n",
      "177 Bottom\n",
      "178 team\n",
      "179 Reg\n",
      "180 ##gia\n",
      "181 ##na\n",
      "182 are\n",
      "183 also\n",
      "184 without\n",
      "185 a\n",
      "186 suspended\n",
      "187 defender\n",
      "188 ,\n",
      "189 German\n",
      "190 Diet\n",
      "191 ##mar\n",
      "192 Be\n",
      "193 ##iers\n",
      "194 ##dorf\n",
      "195 ##er\n",
      "196 .\n",
      "197 Fi\n",
      "198 ##ore\n",
      "199 ##ntina\n",
      "200 (\n",
      "201 10\n",
      "202 )\n",
      "203 v\n",
      "204 Peru\n",
      "205 ##gia\n",
      "206 (\n",
      "207 8\n",
      "208 )\n",
      "209 133\n",
      "210 ##0\n",
      "211 Fi\n",
      "212 ##ore\n",
      "213 ##ntina\n",
      "214 will\n",
      "215 be\n",
      "216 without\n",
      "217 three\n",
      "218 suspended\n",
      "219 players\n",
      "220 –\n",
      "221 defenders\n",
      "222 Daniel\n",
      "223 ##e\n",
      "224 Car\n",
      "225 ##nas\n",
      "226 ##cial\n",
      "227 ##i\n",
      "228 and\n",
      "229 Lorenzo\n",
      "230 Amor\n",
      "231 ##us\n",
      "232 ##o\n",
      "233 and\n",
      "234 midfielder\n",
      "235 Emilia\n",
      "236 ##no\n",
      "237 Big\n",
      "238 ##ica\n",
      "239 –\n",
      "240 for\n",
      "241 a\n",
      "242 difficult\n",
      "243 home\n",
      "244 match\n",
      "245 against\n",
      "246 unpredictable\n",
      "247 ,\n",
      "248 attack\n",
      "249 -\n",
      "250 oriented\n",
      "251 Peru\n",
      "252 ##gia\n",
      "253 led\n",
      "254 by\n",
      "255 in\n",
      "256 -\n",
      "257 form\n",
      "258 C\n",
      "259 ##roa\n",
      "260 ##t\n",
      "261 striker\n",
      "262 Milan\n",
      "263 Rap\n",
      "264 ##aj\n",
      "265 ##ic\n",
      "266 and\n",
      "267 the\n",
      "268 experienced\n",
      "269 Faust\n",
      "270 ##o\n",
      "271 Pi\n",
      "272 ##zzi\n",
      "273 .\n",
      "274 La\n",
      "275 ##zio\n",
      "276 (\n",
      "277 12\n",
      "278 )\n",
      "279 v\n",
      "280 As\n",
      "281 Roma\n",
      "282 (\n",
      "283 7\n",
      "284 )\n",
      "285 1930\n",
      "286 Poor\n",
      "287 man\n",
      "288 ’\n",
      "289 s\n",
      "290 Roman\n",
      "291 derby\n",
      "292 in\n",
      "293 what\n",
      "294 has\n",
      "295 been\n",
      "296 a\n",
      "297 miserable\n",
      "298 season\n",
      "299 for\n",
      "300 both\n",
      "301 Rome\n",
      "302 teams\n",
      "303 ,\n",
      "304 already\n",
      "305 eliminated\n",
      "306 from\n",
      "307 the\n",
      "308 Italian\n",
      "309 and\n",
      "310 U\n",
      "311 ##ef\n",
      "312 ##a\n",
      "313 Cups\n",
      "314 .\n",
      "315 La\n",
      "316 ##zio\n",
      "317 have\n",
      "318 injury\n",
      "319 doubts\n",
      "320 about\n",
      "321 striker\n",
      "322 Pier\n",
      "323 ##lu\n",
      "324 ##igi\n",
      "325 C\n",
      "326 ##asi\n",
      "327 ##rag\n",
      "328 ##hi\n",
      "329 ,\n",
      "330 Czech\n",
      "331 midfielder\n",
      "332 Pavel\n",
      "333 Ned\n",
      "334 ##ved\n",
      "335 and\n",
      "336 defender\n",
      "337 Paolo\n",
      "338 Negro\n",
      "339 ,\n",
      "340 while\n",
      "341 Roma\n",
      "342 present\n",
      "343 a\n",
      "344 full\n",
      "345 strength\n",
      "346 side\n",
      "347 led\n",
      "348 by\n",
      "349 Argentine\n",
      "350 Abel\n",
      "351 Ba\n",
      "352 ##l\n",
      "353 ##bo\n",
      "354 ,\n",
      "355 Marco\n",
      "356 Del\n",
      "357 ##ve\n",
      "358 ##cchi\n",
      "359 ##o\n",
      "360 and\n",
      "361 Francesco\n",
      "362 To\n",
      "363 ##tti\n",
      "364 in\n",
      "365 attack\n",
      "366 .\n",
      "367 A\n",
      "368 ##c\n",
      "369 Milan\n",
      "370 (\n",
      "371 9\n",
      "372 )\n",
      "373 v\n",
      "374 U\n",
      "375 ##dine\n",
      "376 ##se\n",
      "377 (\n",
      "378 11\n",
      "379 )\n",
      "380 133\n",
      "381 ##0\n",
      "382 Can\n",
      "383 Milan\n",
      "384 sink\n",
      "385 any\n",
      "386 further\n",
      "387 ?\n",
      "388 Following\n",
      "389 their\n",
      "390 mid\n",
      "391 ##week\n",
      "392 Champions\n",
      "393 ’\n",
      "394 League\n",
      "395 elimination\n",
      "396 by\n",
      "397 Norwegian\n",
      "398 side\n",
      "399 Rosen\n",
      "400 ##borg\n",
      "401 ,\n",
      "402 a\n",
      "403 morale\n",
      "404 -\n",
      "405 boost\n",
      "406 ##ing\n",
      "407 win\n",
      "408 is\n",
      "409 badly\n",
      "410 needed\n",
      "411 .\n",
      "412 Liberia\n",
      "413 ##n\n",
      "414 striker\n",
      "415 George\n",
      "416 We\n",
      "417 ##ah\n",
      "418 makes\n",
      "419 a\n",
      "420 welcome\n",
      "421 return\n",
      "422 for\n",
      "423 Milan\n",
      "424 alongside\n",
      "425 Roberto\n",
      "426 Ba\n",
      "427 ##ggio\n",
      "428 ,\n",
      "429 with\n",
      "430 Montenegrin\n",
      "431 De\n",
      "432 ##jan\n",
      "433 Sa\n",
      "434 ##vice\n",
      "435 ##vic\n",
      "436 in\n",
      "437 midfield\n",
      "438 .\n",
      "439 Good\n",
      "440 news\n",
      "441 for\n",
      "442 Milan\n",
      "443 is\n",
      "444 that\n",
      "445 U\n",
      "446 ##dine\n",
      "447 ##se\n",
      "448 ’\n",
      "449 s\n",
      "450 German\n",
      "451 striker\n",
      "452 Oliver\n",
      "453 B\n",
      "454 ##ier\n",
      "455 ##hoff\n",
      "456 is\n",
      "457 out\n",
      "458 through\n",
      "459 injury\n",
      "460 .\n",
      "461 Na\n",
      "462 ##poli\n",
      "463 (\n",
      "464 5\n",
      "465 )\n",
      "466 v\n",
      "467 Verona\n",
      "468 (\n",
      "469 17\n",
      "470 )\n",
      "471 133\n",
      "472 ##0\n",
      "473 In\n",
      "474 -\n",
      "475 form\n",
      "476 Na\n",
      "477 ##poli\n",
      "478 should\n",
      "479 prove\n",
      "480 too\n",
      "481 strong\n",
      "482 for\n",
      "483 second\n",
      "484 from\n",
      "485 bottom\n",
      "486 Verona\n",
      "487 despite\n",
      "488 the\n",
      "489 absence\n",
      "490 of\n",
      "491 their\n",
      "492 suspended\n",
      "493 Argentine\n",
      "494 defender\n",
      "495 Roberto\n",
      "496 A\n",
      "497 ##yal\n",
      "498 ##a\n",
      "499 .\n",
      "500 Verona\n",
      "501 ’\n",
      "502 s\n",
      "503 slim\n",
      "504 chances\n",
      "505 have\n",
      "506 been\n",
      "507 further\n",
      "508 reduced\n",
      "509 by\n",
      "510 a\n",
      "511 knee\n",
      "512 injury\n",
      "513 to\n",
      "514 their\n",
      "515 experienced\n",
      "516 midfielder\n",
      "517 E\n",
      "518 ##ugen\n",
      "519 ##io\n"
     ]
    }
   ],
   "source": [
    "counter = 0\n",
    "for word in tokenizer.tokenize(\"Soccer - Italian First Division Matches This Weekend. Rome 1996-12-06 Italian Serie A games to be played on Sunday ( league positions in parentheses , all kick- off times Gmt ) : Bologna ( 4 ) v Piacenza ( 13 ) 1330 Along with leaders Vicenza , fourth-placed Bologna represent the biggest surprise of this Italian autumn. Led as usual by Swede Kennet Andersson and Russian Igor Kolyvanov in attack , Bologna can expect a tough home match against a Piacenza side still exultant after a 3-2 league win over Ac Milan last Sunday. Cagliari ( 16 ) v Reggiana ( 18 ) 1530 Cagliari start favourite in this relegation scrap following draws with Napoli and Inter in last two outings but will be without suspended iSwiss defender Ramon Vega. Bottom team Reggiana are also without a suspended defender , German Dietmar Beiersdorfer. Fiorentina ( 10 ) v Perugia ( 8 ) 1330 Fiorentina will be without three suspended players – defenders Daniele Carnasciali and Lorenzo Amoruso and midfielder Emiliano Bigica – for a difficult home match against unpredictable , attack-oriented Perugia led by in-form Croat striker Milan Rapajic and the experienced Fausto Pizzi. Lazio ( 12 ) v As Roma ( 7 ) 1930 Poor man ’s Roman derby in what has been a miserable season for both Rome teams , already eliminated from the Italian and Uefa Cups. Lazio have injury doubts about striker Pierluigi Casiraghi , Czech midfielder Pavel Nedved and defender Paolo Negro , while Roma present a full strength side led by Argentine Abel Balbo , Marco Delvecchio and Francesco Totti in attack. Ac Milan ( 9 ) v Udinese ( 11 ) 1330 Can Milan sink any further ? Following their midweek Champions ’ League elimination by Norwegian side Rosenborg , a morale-boosting win is badly needed. Liberian striker George Weah makes a welcome return for Milan alongside Roberto Baggio , with Montenegrin Dejan Savicevic in midfield. Good news for Milan is that Udinese ’s German striker Oliver Bierhoff is out through injury. Napoli ( 5 ) v Verona ( 17 ) 1330 In-form Napoli should prove too strong for second from bottom Verona despite the absence of their suspended Argentine defender Roberto Ayala. Verona ’s slim chances have been further reduced by a knee injury to their experienced midfielder Eugenio\"):\n",
    "    counter += 1\n",
    "    print(counter, word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'entity': 'B-MISC', 'score': 0.99924064, 'index': 6, 'word': 'GE', 'start': 9, 'end': 11}, {'entity': 'I-MISC', 'score': 0.8793608, 'index': 7, 'word': '##R', 'start': 11, 'end': 12}, {'entity': 'I-MISC', 'score': 0.884922, 'index': 8, 'word': '##MA', 'start': 12, 'end': 14}, {'entity': 'I-MISC', 'score': 0.9773442, 'index': 9, 'word': '##N', 'start': 14, 'end': 15}, {'entity': 'B-MISC', 'score': 0.9992536, 'index': 40, 'word': 'German', 'start': 93, 'end': 99}, {'entity': 'B-ORG', 'score': 0.9987205, 'index': 46, 'word': 'Bo', 'start': 127, 'end': 129}, {'entity': 'I-ORG', 'score': 0.99158394, 'index': 47, 'word': '##chu', 'start': 129, 'end': 132}, {'entity': 'I-ORG', 'score': 0.9947837, 'index': 48, 'word': '##m', 'start': 132, 'end': 133}, {'entity': 'B-PER', 'score': 0.998579, 'index': 51, 'word': 'Stick', 'start': 137, 'end': 142}, {'entity': 'B-PER', 'score': 0.44963565, 'index': 52, 'word': '##roth', 'start': 142, 'end': 146}, {'entity': 'B-PER', 'score': 0.9985172, 'index': 56, 'word': 'W', 'start': 157, 'end': 158}, {'entity': 'I-PER', 'score': 0.9197838, 'index': 57, 'word': '##os', 'start': 158, 'end': 160}, {'entity': 'I-PER', 'score': 0.739916, 'index': 58, 'word': '##z', 'start': 160, 'end': 161}, {'entity': 'B-ORG', 'score': 0.99935025, 'index': 62, 'word': 'Bay', 'start': 168, 'end': 171}, {'entity': 'I-ORG', 'score': 0.9981505, 'index': 63, 'word': '##er', 'start': 171, 'end': 173}, {'entity': 'I-ORG', 'score': 0.99845874, 'index': 64, 'word': 'Lev', 'start': 174, 'end': 177}, {'entity': 'I-ORG', 'score': 0.9981659, 'index': 65, 'word': '##er', 'start': 177, 'end': 179}, {'entity': 'I-ORG', 'score': 0.9964371, 'index': 66, 'word': '##kus', 'start': 179, 'end': 182}, {'entity': 'I-ORG', 'score': 0.99873143, 'index': 67, 'word': '##en', 'start': 182, 'end': 184}, {'entity': 'B-PER', 'score': 0.99836916, 'index': 70, 'word': 'Ki', 'start': 188, 'end': 190}, {'entity': 'B-PER', 'score': 0.98304445, 'index': 71, 'word': '##rsten', 'start': 190, 'end': 195}, {'entity': 'B-PER', 'score': 0.9985753, 'index': 74, 'word': 'Ram', 'start': 202, 'end': 205}, {'entity': 'I-PER', 'score': 0.9433469, 'index': 75, 'word': '##elo', 'start': 205, 'end': 208}, {'entity': 'I-PER', 'score': 0.59934485, 'index': 76, 'word': '##w', 'start': 208, 'end': 209}, {'entity': 'B-ORG', 'score': 0.99929786, 'index': 93, 'word': 'We', 'start': 251, 'end': 253}, {'entity': 'I-ORG', 'score': 0.99351996, 'index': 94, 'word': '##rde', 'start': 253, 'end': 256}, {'entity': 'I-ORG', 'score': 0.99213785, 'index': 95, 'word': '##r', 'start': 256, 'end': 257}, {'entity': 'I-ORG', 'score': 0.99871624, 'index': 96, 'word': 'Bremen', 'start': 258, 'end': 264}, {'entity': 'B-PER', 'score': 0.9979316, 'index': 99, 'word': 'Bo', 'start': 268, 'end': 270}, {'entity': 'I-PER', 'score': 0.59899634, 'index': 100, 'word': '##de', 'start': 270, 'end': 272}, {'entity': 'B-ORG', 'score': 0.99926776, 'index': 104, 'word': '1860', 'start': 279, 'end': 283}, {'entity': 'I-ORG', 'score': 0.99897707, 'index': 105, 'word': 'Munich', 'start': 284, 'end': 290}, {'entity': 'B-PER', 'score': 0.9982256, 'index': 108, 'word': 'Bo', 'start': 294, 'end': 296}, {'entity': 'B-PER', 'score': 0.64868224, 'index': 110, 'word': '##iro', 'start': 298, 'end': 301}, {'entity': 'I-PER', 'score': 0.65515053, 'index': 111, 'word': '##w', 'start': 301, 'end': 302}, {'entity': 'B-ORG', 'score': 0.9992824, 'index': 125, 'word': 'Karl', 'start': 341, 'end': 345}, {'entity': 'I-ORG', 'score': 0.9980318, 'index': 126, 'word': '##s', 'start': 345, 'end': 346}, {'entity': 'I-ORG', 'score': 0.99667144, 'index': 127, 'word': '##ru', 'start': 346, 'end': 348}, {'entity': 'I-ORG', 'score': 0.9923685, 'index': 128, 'word': '##he', 'start': 348, 'end': 350}, {'entity': 'B-PER', 'score': 0.9988121, 'index': 131, 'word': 'Reich', 'start': 354, 'end': 359}, {'entity': 'B-PER', 'score': 0.9981529, 'index': 134, 'word': 'Carl', 'start': 366, 'end': 370}, {'entity': 'B-PER', 'score': 0.99630994, 'index': 138, 'word': 'Dundee', 'start': 377, 'end': 383}, {'entity': 'B-ORG', 'score': 0.9992298, 'index': 142, 'word': 'Freiburg', 'start': 390, 'end': 398}, {'entity': 'B-ORG', 'score': 0.9993808, 'index': 155, 'word': 'Sc', 'start': 434, 'end': 436}, {'entity': 'I-ORG', 'score': 0.9970982, 'index': 156, 'word': '##hal', 'start': 436, 'end': 439}, {'entity': 'I-ORG', 'score': 0.9967765, 'index': 157, 'word': '##ke', 'start': 439, 'end': 441}, {'entity': 'B-PER', 'score': 0.99819237, 'index': 160, 'word': 'Mu', 'start': 445, 'end': 447}, {'entity': 'B-PER', 'score': 0.55266804, 'index': 161, 'word': '##lder', 'start': 447, 'end': 451}, {'entity': 'B-ORG', 'score': 0.9992756, 'index': 166, 'word': 'Hans', 'start': 466, 'end': 470}, {'entity': 'I-ORG', 'score': 0.9984034, 'index': 167, 'word': '##a', 'start': 470, 'end': 471}, {'entity': 'I-ORG', 'score': 0.99900514, 'index': 168, 'word': 'R', 'start': 472, 'end': 473}, {'entity': 'I-ORG', 'score': 0.9982064, 'index': 169, 'word': '##ost', 'start': 473, 'end': 476}, {'entity': 'I-ORG', 'score': 0.99908614, 'index': 170, 'word': '##ock', 'start': 476, 'end': 479}]\n"
     ]
    }
   ],
   "source": [
    "ner_results = nlp(example)\n",
    "print(ner_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def combine_entities(ner_results):\n",
    "    ner_results_out = []\n",
    "    i = 0\n",
    "    while i < len(ner_results)-1:\n",
    "        last_end = ner_results[i][\"end\"]\n",
    "        ner_results_out.append(dict(ner_results[i]))\n",
    "        j = 1\n",
    "        while i + j < len(ner_results) and (ner_results[i+j][\"start\"] == last_end or\n",
    "                                            (ner_results[i+j][\"start\"] == last_end + 1 and \n",
    "                                             re.search(\"^I\", ner_results[i+j][\"entity\"]) and\n",
    "                                             re.sub(\"^..\", \"\", ner_results[i+j][\"entity\"]) == re.sub(\"^..\", \"\", ner_results[i][\"entity\"]))):\n",
    "            if ner_results[i+j][\"start\"] == last_end:\n",
    "                ner_results_out[-1][\"word\"] += re.sub(\"^##\", \"\", ner_results[i+j][\"word\"])\n",
    "            else:\n",
    "                ner_results_out[-1][\"word\"] += \" \" + ner_results[i+j][\"word\"]\n",
    "            ner_results_out[-1][\"end\"] = ner_results[i+j][\"end\"]\n",
    "            last_end = ner_results[i+j][\"end\"]\n",
    "            j += 1\n",
    "        i += j\n",
    "    return ner_results_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'entity': 'B-MISC',\n",
       "  'score': 0.99924064,\n",
       "  'index': 6,\n",
       "  'word': 'GERMAN',\n",
       "  'start': 9,\n",
       "  'end': 15},\n",
       " {'entity': 'B-MISC',\n",
       "  'score': 0.9992536,\n",
       "  'index': 40,\n",
       "  'word': 'German',\n",
       "  'start': 93,\n",
       "  'end': 99},\n",
       " {'entity': 'B-ORG',\n",
       "  'score': 0.9987205,\n",
       "  'index': 46,\n",
       "  'word': 'Bochum',\n",
       "  'start': 127,\n",
       "  'end': 133},\n",
       " {'entity': 'B-PER',\n",
       "  'score': 0.998579,\n",
       "  'index': 51,\n",
       "  'word': 'Stickroth',\n",
       "  'start': 137,\n",
       "  'end': 146},\n",
       " {'entity': 'B-PER',\n",
       "  'score': 0.9985172,\n",
       "  'index': 56,\n",
       "  'word': 'Wosz',\n",
       "  'start': 157,\n",
       "  'end': 161},\n",
       " {'entity': 'B-ORG',\n",
       "  'score': 0.99935025,\n",
       "  'index': 62,\n",
       "  'word': 'Bayer Leverkusen',\n",
       "  'start': 168,\n",
       "  'end': 184},\n",
       " {'entity': 'B-PER',\n",
       "  'score': 0.99836916,\n",
       "  'index': 70,\n",
       "  'word': 'Kirsten',\n",
       "  'start': 188,\n",
       "  'end': 195},\n",
       " {'entity': 'B-PER',\n",
       "  'score': 0.9985753,\n",
       "  'index': 74,\n",
       "  'word': 'Ramelow',\n",
       "  'start': 202,\n",
       "  'end': 209},\n",
       " {'entity': 'B-ORG',\n",
       "  'score': 0.99929786,\n",
       "  'index': 93,\n",
       "  'word': 'Werder Bremen',\n",
       "  'start': 251,\n",
       "  'end': 264},\n",
       " {'entity': 'B-PER',\n",
       "  'score': 0.9979316,\n",
       "  'index': 99,\n",
       "  'word': 'Bode',\n",
       "  'start': 268,\n",
       "  'end': 272},\n",
       " {'entity': 'B-ORG',\n",
       "  'score': 0.99926776,\n",
       "  'index': 104,\n",
       "  'word': '1860 Munich',\n",
       "  'start': 279,\n",
       "  'end': 290},\n",
       " {'entity': 'B-PER',\n",
       "  'score': 0.9982256,\n",
       "  'index': 108,\n",
       "  'word': 'Bo',\n",
       "  'start': 294,\n",
       "  'end': 296},\n",
       " {'entity': 'B-PER',\n",
       "  'score': 0.64868224,\n",
       "  'index': 110,\n",
       "  'word': '##irow',\n",
       "  'start': 298,\n",
       "  'end': 302},\n",
       " {'entity': 'B-ORG',\n",
       "  'score': 0.9992824,\n",
       "  'index': 125,\n",
       "  'word': 'Karlsruhe',\n",
       "  'start': 341,\n",
       "  'end': 350},\n",
       " {'entity': 'B-PER',\n",
       "  'score': 0.9988121,\n",
       "  'index': 131,\n",
       "  'word': 'Reich',\n",
       "  'start': 354,\n",
       "  'end': 359},\n",
       " {'entity': 'B-PER',\n",
       "  'score': 0.9981529,\n",
       "  'index': 134,\n",
       "  'word': 'Carl',\n",
       "  'start': 366,\n",
       "  'end': 370},\n",
       " {'entity': 'B-PER',\n",
       "  'score': 0.99630994,\n",
       "  'index': 138,\n",
       "  'word': 'Dundee',\n",
       "  'start': 377,\n",
       "  'end': 383},\n",
       " {'entity': 'B-ORG',\n",
       "  'score': 0.9992298,\n",
       "  'index': 142,\n",
       "  'word': 'Freiburg',\n",
       "  'start': 390,\n",
       "  'end': 398},\n",
       " {'entity': 'B-ORG',\n",
       "  'score': 0.9993808,\n",
       "  'index': 155,\n",
       "  'word': 'Schalke',\n",
       "  'start': 434,\n",
       "  'end': 441},\n",
       " {'entity': 'B-PER',\n",
       "  'score': 0.99819237,\n",
       "  'index': 160,\n",
       "  'word': 'Mulder',\n",
       "  'start': 445,\n",
       "  'end': 451},\n",
       " {'entity': 'B-ORG',\n",
       "  'score': 0.9992756,\n",
       "  'index': 166,\n",
       "  'word': 'Hansa Rostock',\n",
       "  'start': 466,\n",
       "  'end': 479}]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combine_entities(ner_results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Performance graph for different document sizes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_base  = [  150,  200,  250,  300,  350,  400,  450,  500,  550,  600  ]\n",
    "y_base  = [  73.8, 73.9, 74.5, 74.4, 74.0, 74.1, 74.3, 74.3, 74.3, 74.1 ]\n",
    "x_large = [  150,  200,  250,  300,  350,  400,  450,  500,  550,  600  ]\n",
    "y_large = [  75.4, 76.0, 76.1, 76.1, 76.1, 76.2, 76.3, 76.4, 76.3, 76.1 ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEICAYAAABF82P+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAABBjElEQVR4nO3dd3iUZfbw8e9JKIEASYCAtBBQipQ0AkgRQUUEWWwoYMUCuru2dde+lnXdn73ra2exiw0suHZABaWFXhTBhCpNCAk9yXn/uJ8kQ0iZlMmknM91zZWZp555ksyZ566iqhhjjDEFhQQ7AGOMMVWTJQhjjDGFsgRhjDGmUJYgjDHGFMoShDHGmEJZgjDGGFOogCUIEekiIot9HntE5AZv3bUislpEVojIQ0Xsnyoiy7x9FwQqTmOMMYWTyugHISKhwCagL9ARuAM4Q1UPikgLVd1WyD6pQLKq7vD3PM2bN9fY2NiKCdoYY2qBhQsX7lDV6MLW1amkGE4B1qpqmog8DDygqgcBCksOZRUbG8uCBXazYYwx/hKRtKLWVVYdxFjgbe95Z+BEEZkrIrNEpHcR+yjwpYgsFJGJlRKlMcaYPAG/gxCResAo4DafczYFTgB6A++KSEc9uqxroKpuEpEWwFcislpVvyvk+BOBiQAxMTGBehvGGFPrVMYdxHAgRVW3eq83Ah+qMw/IAZoX3ElVN3k/twFTgT6FHVxVX1TVZFVNjo4utBjNGGNMGVRGghhHfvESwDRgCICIdAbqAUdURItIuIg0zn0OnAYsr4RYjTHGeAKaILwP96HAhz6LJwEdRWQ58A5wqaqqiLQWkc+8bVoCP4jIEmAeMF1VPw9krMYYY44U0DoIVd0LNCuw7BBwUSHbbgZGeM/XAfGBjM0YY0zxrCe1McaYQlVWPwhjjCm3P/YeIiVtF2u2ZXJsdDhJ7aNo3qh+sMOqsSxBGGOqpOwcZc22DFLSdrMwbRcp63fx2469R23XvllDesVEkdg+il4xUXQ5pjGhIRKEiGseSxDGmCohff9hFm/YTYqXDBav303GwSwAmoXXI6l9FOcnt6NX+yg6t2zEr9sy8xLHd2t28OGiTQCE1wslISYyL2kktYsiomHdYL61assShDGm0qkq63bszUsGC71iI1UIEehyTBNGJbSmV/soerWPIqZpQ0SOvCtIjm1KcmzTvONt3LWfhWm78pLGMzN+Jcfrfntci0b0iokiqX0kvdpH0bF5I0LsLqNElTJYX2VJTk5WG4vJmKpn78EslmzczaL1rrho0fpd7Np3GIAmYXVIah9FUoxLBvHtImlUv/zfXXPP6ZLQblLW72J3gM9ZHYnIQlVNLmxd7bwixpiAyf02n3tnsDBtF6t/zyDb+zp/XItGDO3Wkl7eB/Sx0YH5Nh9evw79j21O/2PdQA05Od5dy/pdeXcuM3/eDuTftfRqH5mXNAq7a6lt7A7CGFMuBw5ns3xTel5CSFm/m+0ZB4H8+oCkmCiS2keR2C6SyIb1ghxxvtx6j4VpLmks3rCbzAL1HrkJI65tBGF1Q4McccWzOwhjTIX5Pf1AXjn/wrRdrNiczuFs90WzfbOGDDyuOUnVpEVRRIO6nNQ5mpM6u3HcsnOUX7Zm5L23Ret389VKN4xcnRChe+smRySN1pENghl+wNkdhDGmSIezc1i5ec8RH5ibdu8HoH6dEOLbRpLY3msxFBNFdOOa1ydhZ+ZBV3fiFU0t2bibA4dzAGjXtAEXn9CesX1iaBJWPVtKFXcHYQnCGJNnZ+ZBUtbnF7ks3ZT/Ydg6Iiyvr0Gv9lEc36oJ9erUvsEYDmfnsGrLHlLSdvHFiq38uG4njerXYUzvdlw2IJa2UQ2DHWKpWIIwxhwlO0f5+feMIyptU3fuA6BuqNC9dUReUUpS+0haRdTs4pSyWr4pnZe/X8enS7egwPAexzDhxI7Et4sMdmh+sQRhjCF932FSNuxiUdouFnod0fYeygageaP6JMVEeskgip5tamaFbCBt3r2fV+ek8tbc9WQczKJPbFOuPLEDpx7fskr3ubAEYUwt45p0ZuYNU7Fw/S5+3ZYJuCadx7dqkn93EBNFu6YNan2TzoqSceAwU+Zv4L+zU9m0ez8dmodzxcAOnJvUlgb1ql7StQRhTA2XeTCLJd4wFQvXu8rk9P2uU1hEg7pH3B3Et40kvJZ2CqtMWdk5/G/577z8/TqWbEwnqmFdLj6hPRf3i61SlfmWIIypQVSV9X/s8+mItpuff9+TN6xE55aN8vodJMVE0bF5eJUu4qjpVJX5qbt46ft1fL1qK3VDQzg7oQ1XntiBTi0bBzs8SxDGVGcHDmezdGO6T1PTXezIPARAo/p1SGgX6fodtI8ioV0kEQ2qZ3PL2mDd9kwmzf6N9xdu5MDhHAZ3iWbCiR3pf2yzoBXxWYIwphrZvDt/mIqUtF2s2LyHLO/2oEPzcBJzi4tioujcsmp3RDOF+2PvId78KY1Xf0xjR+ZBurVqwpUndmBkXOtKbzpsCcKYKupQVg4rNqe7weS8pqZb0g8AEFY3hLi2+ckgKSaSZjY5To1y4HA2Hy/ezEvfr2PNtkyOaRLG+AGxjOsTU2l3gpYgjKkitmUcICVtN4u8O4Rlm9I5mOU6orWJbOANUeGKjI5v1YS6obWvI1ptpKrM/GU7L3+/jtm/7iS8Xijn927H5QM60K5pYDveWYIwJgiysnNY/XtGXjJIWb+b9X+4jmj1QkPo3qaJN0eBu0M4JiIsyBGbqmDF5nRe+f43Pl6ymRxVhvdoxZUndiAxJiog57MEYUwl2L3vUN58Bwu9MXv2eR3RohvXzxuiIql9JN1bW0c0U7zf0w8weU4qb85NI+NAFsnto7jyxI4M7dayQuudLEEYU8FycpRft2e6fgde3cHa7W6+5NAQ4fhWjY+4O2gbZR3RTNnsPZjFuws2MGn2b2z4Yz+xzRpy+cAOjO7Vlob1yt+fxRKEMeWUcSB3vuTdXke0XWQccPMGRDase0QyiG8XUSH/uMb4ysrO4cuVW3nxu3Us3rCbyIZ1ubBvDJf2i6VFk7IXT1qCMCXKHacnxZv9Kyen5vxdlIfimp3+vDUDVRCBzi0ae8nAtTDq0Dzc7g5MpVFVFqa5jndfrtxK3ZAQRiW05r6zepSp2LJcEwaJyADgHqC9t724GLVjqSMxVUJOjrJ2e+YRM4DljtMTGiJ0bB5O/brWeiZXdOP6DOt+jOuIFhNZbcf9NzWDiJAc25Tk2Kak7tjLpNm/sXZ7ZkDqtPy5D34F+BuwEMiu8AhMwOWO05Nbebpo/S72+BSPJMVEcXZiGyseMaaaiW0ezr1n9iBQJUH+fBKkq+r/AnJ2U+FUlbSd+464O8gdpye3eOSMuFZ5Y/V0tOIRY6q9QP0P+5MgZojIw8CHwMHchaqaEpCITKnsP5TN0o2782YBW7R+Fzv3unF6GtevQ0JMJKed3MmKR4wxpeZPgujr/fStxFDg5OJ2EpEuwBSfRR2Bu1T1CRG5FvgrrshquqreXMj+pwNPAqHAy6r6gB+x1miqyub0A0c0rVzpM05Px+bhDO7SIq+tfacWNk6PMabsSkwQqjqkLAdW1Z+BBAARCQU2AVNFZAhwJhCvqgdFpEXBfb3tnwWGAhuB+SLysaquLEss1dXBrGxWbN6TN0ZPStpuft/jxulpUDeU+HYRTBzUkV7t3YTxTcPrBTliY0xN4k8rpgjgbmCQt2gWcK+qppfiPKcAa1U1zSuuekBVDwKo6rZCtu8D/Kqq67wY3sEllRqdIHZkHmRB6q68+oNlm9I55I3T0zaqAX06NM0buK1rq8Y2To8xJqD8KWKaBCwHzvdeXwz8FzinFOcZC7ztPe8MnCgi/wEOAP9Q1fkFtm8DbPB5vZH8oq4aaUHqH1zw8lwOZeVQLzSEnm0juLRf+7yEUJ6OMMYYUxb+JIhjVfVcn9f/EpHF/p5AROoBo4DbfM7ZFDgB6A28KyIdtYzttERkIjARICYmpiyHCLpdew9x3duLOKZJGI+PiadHmwjq17FxeowxweVPGcV+ERmY+8LrOLe/FOcYDqSo6lbv9UbgQ3XmATlA8wL7bALa+bxu6y07iqq+qKrJqpocHR1dirCqBlXlpveXsD3zIM9ekESv9k0tORhjqgR/7iD+DLzq1UUI8AcwvhTnGEd+8RLANGAIrvlsZ6AesKPAPvOBTiLSAZcYxgIXlOKc1cak2al8vWobd43sRs+2EcEOxxhj8vjTimkxEC8iTbzXe/w9uIiE41oiXeWzeBIwSUSWA4eAS1VVRaQ1rjnrCFXNEpFrgC9wzVwnqeoKf89bXSzduJsH/reKU49vyWUDYoMdjjHGHKHIBCEiF6nqGyJyY4HlAKjqYyUdXFX3As0KLDsEXFTItpuBET6vPwM+K+kc1dWeA4e55q1FRDeqzyPnxVlvZmNMlVPcHUS497NxIetsqM9yUFVu+3AZm3bv592rTiCyofVfMMZUPUUmCFV9wXv6tarO9l3nVVSbMnpr3nqmL93Czad3oVf7psEOxxhjCuVPK6an/Vxm/LBqyx7+9clKTuzUnKsHHRvscIwxpkjF1UH0A/oD0QXqIZrgKo5NKe09mMU1b6UQ0aAuj49JIMTGSTLGVGHF1UHUAxp52/jWQ+wBRgcyqJrqro9WsG7HXt68si/NG9UPdjjGGFOs4uogZgGzRGSyqqZVYkw10vsLN/JBykauP6UT/Y8t2C/QGGOqHn/qIF4WkcjcFyISJSJfBC6kmufXbZncOW05fTs05bpTOgU7HGOM8Ys/CaK5qu7OfaGqu4Cjhug2hTtwOJtr3kqhQb1QnhybaPMzGGOqDX8SRI6I5I2CJyLtsX4Qfvv3pytZ/XsGj54fzzERNiKrMab68GcspjuAH0RkFm4sphPxRk81xft06WbenLueq07qyJAudtNljKle/BmL6XMRScINzw1wg6oWHFzPFJC2cy+3fbCMxJhI/nFal2CHY4wxpVZiEZO4QYJOB5JU9VOgoYj0CXhk1dihrByufXsRIvDU2ESb+c0YUy3588n1/4B+uGG7ATJw80WbIjz4+WqWbkznodHxtGvaMNjhGGNMmfhTB9FXVZNEZBG4VkzeLHGmEF+t3MorP/zG+P6xnN7jmGCHY4wxZebPHcRhEQnFa7kkItG4WeBMAZt27+cf7y2hR5sm3Daia7DDMcaYcvEnQTwFTAVaiMh/gB+A+wMaVTV0ODuH695eRFZ2Dk+PS7JpQ40x1Z4/rZjeFJGFwCm4Zq5nqeqqgEdWzTz+1S8sTNvFk2MT6NA8vOQdjDGmiisxQYjI66p6MbC6kGUG+O6X7Tw3ay1je7fjzIQ2wQ7HGGMqhD9FTN19X3j1Eb0CE071s23PAW58dzGdWjTi7j91L3kHY4ypJopMECJym4hkAHEiskdEMrzX24CPKi3CKiw7R7lhymIyD2bx7AVJNKhn9Q7GmJqjyAShqveramPgYVVtoqqNvUczVb2tEmOssp6d8Stz1u7k3lE96NSysKm7jTGm+vKniOkOEblIRO4EEJF21pMaflq3kye+/oWzElpzXnLbYIdjjDEVzp8E8SyuJ/UF3utManlP6p2ZB7n+nUW0bxbOfWf3xI1GYowxNYv1pC6lnBzl7+8tYde+w0wa35tG9f25hMYYU/1YT+pSevmHdcz8eTv/PON4ureOCHY4xhgTMGXtSf1/AY2qikpZv4uHPv+Z07sfw8UntA92OMYYE1DWk9pP6fsOc+1bizgmIowHR8dZvYMxpsYrMkGISFOfl9uAt33XqeofgQysKlFVbvlgKVv3HOC9q/sR0aBusEMyxpiAK+4OYiGu3kGAGGCX9zwSWA90CHRwVcXrP6Xx+YrfuX1EVxJjooIdjjHGVIoiE4SqdgAQkZeAqar6mfd6OHBWSQcWkS7AFJ9FHYG7cAlmArDdW3577rEL7J+Km5woG8hS1eQS300ALN+Uzn2frmJIl2iuHNgxGCEYY0xQ+NNG8wRVnZD7QlX/JyIPlbSTqv4MJEDe+E2bcJXdlwGPq+ojfpx7SDDnv848mMU1b6UQFV6XR89PICTE6h2MMbWHPwlis4j8E3jDe30hsLmU5zkFWKuqadWlcldVuWPqMtb/sY+3J5xA0/Ba2/XDGFNL+dPMdRwQjfv2/6H3fFyxexxtLD6V3MA1IrJURCaJSFGF+gp8KSILRWRiKc9Xbu8t2MhHizdzw6md6duxWWWf3hhjgk5UNbAncL2uNwPdVXWriLQEduASwL+BVqp6eSH7tVHVTSLSAvgKuFZVvytku4nARICYmJheaWlp5Y75l60ZjHrmB5Jionj9ir6EWtGSMaaGEpGFRdXx+nMHUV7DgRRV3QqgqltVNVtVc4CXgEIH/lPVTd7Pbbi7l6K2e1FVk1U1OTo6utzB7j+UzV/fTKFR/To8MSbBkoMxptaqjAQxjiP7ULTyWXc2sLzgDiISLiKNc58DpxW2XSDc8/EKft2eyeNjEmjRJKwyTmmMMVVSQEea8z7chwJX+Sx+SEQScEVMqbnrRKQ18LKqjgBaAlO9Cu06wFuq+nkgYwX4aPEmpizYwF+HHMuJncp/N2KMMdWZP3NSR+P6LcT6bl9YvUFBqroXaFZgWaFzWavqZmCE93wdEF/S8SvSbzv2cvuHy0huH8XfTu1cmac2xpgqyZ87iI+A74GvcZ3WapwDh129Q906ITw1LpE6oZVR8maMMVWbPwmioareEvBIguj+z1axcsseXr4kmdaRDYIdjjHGVAn+fFX+VERGBDySINm97xCfLf+dKwZ24NRuLYMdjjHGVBkl9oMQkQwgHDgEHPYWq6o2CXBspZacnKwLFiwo9X47Mg/SJKwu9epY0ZIxpnYprh+EP/NBNK74kKqW5o3qBzsEY4ypcvxq5ioio4BB3suZqvpp4EIyxhhTFZRYpiIiDwDXAyu9x/Uicn+gAzPGGBNc/txBjAASvKExEJFXgUXAbYEMzBhjTHD5Wysb6fM8IgBxGGOMqWL8uYO4H1gkIjNwU44OAm4NaFTG1GCHDx9m48aNHDhwINihmFokLCyMtm3bUrduXb/38acV09siMhPo7S26RVV/L1uIxpiNGzfSuHFjYmNjqS4TaJnqTVXZuXMnGzdupEOHDn7vV2QRk4h09X4mAa2Ajd6jtbfMGFMGBw4coFmzZpYcTKUREZo1a1bqu9bi7iBuxE3E82gh6xQ4uVRnMsbkseRgKltZ/uaKvINQ1dxpPoer6hDfB96oq8aY6ik0NJSEhATi4+NJSkpizpw5AKSmptKgQQMSEhLyHq+99hoAsbGx9OzZk7i4OE466STS0tI4++yzSUhI4LjjjiMiIiJvn9zj5Ro/fjzvv/9+pb9Pf91zzz088sgjANx11118/fXXADzxxBPs27cvmKGVie/7KQ9/KqnnAAWLlApbZoypJho0aMDixYsB+OKLL7jtttuYNWsWAMcee2zeuoJmzJhB8+bNufvuu7nvvvuYOnUqADNnzuSRRx7h008rpg9tVlYWdeoEdLqaIt177715z5944gkuuugiGjZsGJRYgq24OohjRKQX0EBEEkUkyXsMBmrn1TKmBtqzZw9RUVGl2qdfv35s2rSpTOe799576d27Nz169GDixInkjgc3ePBgbrjhBpKTk3nyySeZP38+cXFxJCQkcNNNN9GjRw8AsrOzuemmm+jduzdxcXG88MILR51j7969nHHGGcTHx9OjRw+mTJkCuLugm2++mZ49e9KnTx9+/fXXo/bNvdt56qmn2Lx5M0OGDGHIkCFHbRcbG8uOHTsAWLBgAYMHDwbct/fLL7+cwYMH07FjR5566qm8fV577TXi4uKIj4/n4ovd1DiffPIJffv2JTExkVNPPZWtW7cCMGvWrLw7ssTERDIyMgB4+OGH89773XffnXfs//znP3Tu3JmBAwfy888/l+6XUoTiUvQwYDzQFnjMZ3kGcHuFnN2YWu5fn6xg5eY9FXrMbq2bcPefuhe7zf79+0lISODAgQNs2bKFb7/9Nm/d2rVrSUhIyHv99NNPc+KJJx6x/+eff85ZZ51VpviuueYa7rrrLgAuvvhiPv30U/70pz8BcOjQIXIH3OzRowcvvfQS/fr149Zb81vWv/LKK0RERDB//nwOHjzIgAEDOO20045onfP555/TunVrpk+fDkB6enreuoiICJYtW8Zrr73GDTfcUORdz3XXXcdjjz2Wd9dUGqtXr2bGjBlkZGTQpUsX/vznP/PLL79w3333MWfOHJo3b84ff/wBwMCBA/npp58QEV5++WUeeughHn30UR555BGeffZZBgwYQGZmJmFhYXz55ZesWbOGefPmoaqMGjWK7777jvDwcN555x0WL15MVlYWSUlJ9OrVq1QxF6bIBKGqrwKvisi5qvpBuc9kjKkyfIuYfvzxRy655BKWL3fTvhdXxDRkyBD++OMPGjVqxL///e8ynXvGjBk89NBD7Nu3jz/++IPu3bvnJYgxY8YAsHv3bjIyMujXrx8AF1xwQd4H+ZdffsnSpUvz6jTS09NZs2bNEQmiZ8+e/P3vf+eWW25h5MiRRyS4cePG5f3829/+Vqb3UJIzzjiD+vXrU79+fVq0aMHWrVv59ttvOe+88/KSTdOmTQHX7HnMmDFs2bKFQ4cO5b2PAQMGcOONN3LhhRdyzjnn0LZtW7788ku+/PJLEhMTAcjMzGTNmjVkZGRw9tln5xWFjRo1qkLehz+FfJ+KyAUcPeXovUXuYYzxS0nf9CtDv3792LFjB9u3by9x2xkzZhAZGcmFF17I3XffzWOPPVbiPr4OHDjAX/7yFxYsWEC7du245557jmh6GR4eXuIxVJWnn36aYcOGFblN586dSUlJ4bPPPuOf//wnp5xySt5di29rnvK0JqtTpw45OTkARzUfrV8/f4To0NBQsrKyijzOtddey4033sioUaOYOXMm99xzDwC33norZ5xxBp999hkDBgzgiy++QFW57bbbuOqqq444xhNPPFHm91Ecf4ba+Ag4E8gC9vo8jDE1wOrVq8nOzqZZs2Ylb4z7YHziiSd47bXX8opJ/JX7Qdq8eXMyMzOLbNkUGRlJ48aNmTt3LgDvvPNO3rphw4bx3HPPcfiwm57ml19+Ye/eIz+SNm/eTMOGDbnooou46aabSElJyVuXWx8xZcqUvDuUojRu3Div7L+g2NhYFi5cCMAHH5RcyHLyySfz3nvvsXPnToC8a5eenk6bNm0AePXVV/O2X7t2LT179uSWW26hd+/erF69mmHDhjFp0iQyMzMB2LRpE9u2bWPQoEFMmzaN/fv3k5GRwSeffFJiPP7w5w6iraqeXiFnM8ZUCbl1EOC+kb/66quEhoYCR9dBXH755Vx33XVH7N+qVSvGjRvHs88+y5133un3eSMjI5kwYQI9evTgmGOOoXfv3kVu+8orrzBhwgRCQkI46aSTiIhww8BdeeWVpKamkpSUhKoSHR3NtGnTjth32bJl3HTTTYSEhFC3bl2ee+65vHW7du0iLi6O+vXr8/bbbxcb78SJEzn99NNp3bo1M2bMOGLd3XffzRVXXMGdd96ZV0FdnO7du3PHHXdw0kknERoaSmJiIpMnT+aee+7hvPPOIyoqipNPPpnffvsNcHcFM2bMICQkhO7duzN8+HDq16/PqlWr8hJbo0aNeOONN0hKSmLMmDHEx8fTokWLYq9rafgzo9yLwNOquqxCzhhAZZ1RzpjKtGrVKo4//vhgh1HlZWZm0qhRIwAeeOABtmzZwpNPPlmuY8bGxrJgwYJSVzrXFIX97ZVrRjlgIDBeRH4DDuIG7FNVjStvsMYYU5Tp06dz//33k5WVRfv27Zk8eXKwQ6p1/EkQwwMehTHGFDBmzJi8Vk0VJTU1tUKPV9OVWEmtqmlAO+Bk7/k+f/YzxhhTvfkz5ejdwC3kzyBXF3gjkEEZY4wJPn/uBM4GRuE1bVXVzUDjQAZljDEm+PxJEIfUNXVSABEpuSeLMcaYas+fBPGuiLwARIrIBOBr4KXAhmWMCaTKHu578ODBVMUm6DNnzmTkyJEAfPzxxzzwwAMATJs2jZUrVwYztDLxfT8VwZ8pRx8RkaHAHqALcJeqflXSfiLSBZjis6gjcBcQCUwAcvv1366qnxWy/+nAk0Ao8LKqPlDSOY0x/qnqw30Hw6hRo/LGMJo2bRojR46kW7duQY4quPxqjaSqX6nqTar6D3+Sg7fPz6qaoKoJQC9c66ep3urHc9cVkRxCgWdxTWy7AeNEpHb/powJkMoa7vv1118nISGBHj16MG/ePADmzZtHv379SExMpH///nnDVK9YsYI+ffqQkJBAXFwca9asAeCNN97IW37VVVeRnZ191HluvfVWunXrRlxcHP/4xz8AN4T31VdfTXJyMp07dy40kU2ePJlrrrmGOXPm8PHHH3PTTTeRkJDA2rVrj9iu4ORHuZ35Zs6cyeDBgxk9ejRdu3blwgsvzBvKfP78+fTv35/4+Hj69OlDRkYGqampnHjiiSQlJR1xF7dlyxYGDRqUd62+//57wA1S2K9fP5KSkjjvvPPyhtv4/PPP6dq1K0lJSXz44Yel+p2UpMg7CBHJwKt3KLgK11GuSSnOcwqwVlXT/Bwcqw/wq6qu82J5BzceVPW75zOmOP+7FX6v4EEKjukJw4u/4Q7GcN/79u1j8eLFfPfdd1x++eUsX76crl278v3331OnTh2+/vprbr/9dj744AOef/55rr/+ei688EIOHTpEdnY2q1atYsqUKcyePZu6devyl7/8hTfffJNLLrkk7xw7d+5k6tSprF69GhFh9+7deetSU1OZN28ea9euZciQIYXOBQHQv39/Ro0axciRIxk9enSp3uOiRYtYsWIFrVu3ZsCAAcyePZs+ffowZswYpkyZQu/evdmzZw8NGjSgRYsWfPXVV4SFhbFmzRrGjRvHggULeOuttxg2bBh33HEH2dnZ7Nu3jx07dnDffffx9ddfEx4ezoMPPshjjz3GzTffzIQJE/j222857rjjKrzfSHFFTN8AxwAfAu+o6vpynGcs4DvoyTUicgmwAPi7qu4qsH0bYIPP641A33Kc3xjjIxjDfecOsz1o0CD27NmTN6T3pZdeypo1axCRvAH4+vXrx3/+8x82btzIOeecQ6dOnfjmm29YuHBh3jhD+/fvp0WLFkecIyIigrCwMK644gpGjhx5RHn8+eefT0hICJ06daJjx46sXr26VPH7o0+fPrRt2xaAhIQEUlNTiYiIoFWrVnlxN2nivlvv3buXa665hsWLFxMaGsovv/wCQO/evbn88ss5fPgwZ511FgkJCcyaNYuVK1cyYMAAwM2b0a9fP1avXk2HDh3o1KkTABdddBEvvvhihb2f4uaDOEtEIoBzgJdEJAxXp/COqvo9hKOI1MM1k83tR/Ec8G/c3cm/gUeBy8sWPojIRGAiQExMTFkPY0xwlPBNvzJU1nDfBUsPRIQ777yTIUOGMHXqVFJTU/MGvbvgggvo27cv06dPZ8SIEbzwwguoKpdeein3339/keeoU6cO8+bN45tvvuH999/nmWeeybs7Kuz8ZeE7zHdOTg6HDh3KW1eaYb4ff/xxWrZsyZIlS8jJySEsLAxwCfS7775j+vTpjB8/nhtvvJGoqCiGDh161OCCRSXyilJsHYSqpqvqf3F1AS8A9+JmmSuN4UCKqm71jrlVVbNVNQfXGqpPIftswvXeztXWW1ZYjC+qarKqJkdHR5cyNGNMZQ33nTvM9g8//EBERAQRERFHDHXtO9bSunXr6NixI9dddx1nnnkmS5cu5ZRTTuH9999n27ZtgBsuOy0t7YhzZGZmkp6ezogRI3j88cdZsmRJ3rr33nuPnJwc1q5dy7p16+jSpUuRsfo7zPfHH3+cd9dTlC5durBlyxbmz58PQEZGBllZWaSnp9OqVStCQkJ4/fXX8+pT0tLSaNmyJRMmTODKK68kJSWFE044gdmzZ+cVi+3du5dffvmFrl27kpqamldPUtLotKVVbIIQkf4i8jSQAvQHzlbV0s0QAuPwKV4SkVY+684Glheyz3ygk4h08O5AxgIfl/K8xpgi5NZBJCQkMGbMmEKH+859+M6pnMt3uG9/hYWFkZiYyNVXX80rr7wCwM0338xtt91GYmLiEd+23333XXr06EFCQgLLly/nkksuoVu3btx3332cdtppxMXFMXToULZs2XLEOTIyMhg5ciRxcXEMHDjwiDucmJgY+vTpw/Dhw3n++efzvrEXZuzYsTz88MMkJiYeVUk9YcIEZs2aRXx8PD/++GOJkxzVq1ePKVOmcO211xIfH8/QoUPzJk569dVXiY+PZ/Xq1XnHmTlzJvHx8SQmJjJlyhSuv/56oqOjmTx5MuPGjSMuLi6veCksLIwXX3yRM844g6SkpKOK3MqryOG+RSQV2A28A3yLmzAoj6qmHL3XUccIB9YDHVU13Vv2OpCAK2JKBa5S1S0i0hrXnHWEt90I4AlcM9dJqvqfks5nw32b6sCG+65848ePL1Olc01TkcN9p+I+xIcBp+FaL+VS4OSSglHVvUCzAssuLmLbzcAIn9efAUc1gTXGGFM5iqukHlyJcRhjTMDYXBJlY8N2G2OMKZQlCGOCoKSpfo2paGX5myupFZOISLvitjHGlE5YWBg7d+60JGEqjaqyc+fOYltuFabYwfpUVUXkM6BneYIzxuRr27YtGzdu9KtjmjEVJSwsLK+Xt7/8mZM6RUR6q+r8soVljPFVt25dOnToEOwwjCmRPwmiL3ChiKThZpXLHawvLqCRGWOMCSp/EsSwgEdhjDGmyimxFZOqpuEm+fmT94j0lhljjKnBSkwQInI98CbQwnu8ISLXBjowY4wxweVPEdMVQF9v2AxE5EHgR+DpQAZmjDEmuPzpKCeA77x+2Rw5LpMxxpgayJ87iP8Cc0Ukdz7ps4BXAhaRMcaYKqG4Oak7qOpvqvqYiMwEBnqrLlPVRZUSnTHGmKAp7g7ifaCXiHyjqqfgJg0yxhhTSxSXIEJE5Hags4jcWHBlGWaWM8YYU40UV0k9FlchXQdoXMjDGGNMDVbchEE/Aw+KyFJV/V8lxmSMMaYK8KcntSUHY4yphWzCIGOMMYWyBGGMMaZQ/ozFdJ6INPae/1NEPhSRpMCHZowxJpj8uYO4U1UzRGQgcCquF/VzgQ3LGGNMsPmTIHLHYToDeFFVpwP1AheSMcaYqsCfBLFJRF4AxgCfiUh9P/czxhhTjfnzQX8+8AUwTFV3A02BmwIZlDHGmODzpx/EPmAb+YP1ZQFrAhmUMcaY4POnFdPdwC3Abd6iusAbgQzKGGNM8PlTxHQ2MArYC6Cqm7GxmIwxpsbzJ0EcUlUFFEBEwv05sIh0EZHFPo89InKDz/q/i4iKSPMi9s/22fdjf85pjDGm4vgzo9y7XiumSBGZAFwOvFTSTt5gfwkAIhIKbAKmeq/bAacB64s5xH5VTfAjPmOMMQFQbIIQEQGmAF2BPUAX4C5V/aqU5zkFWKuqad7rx4GbgY9KeRxTGbKzINSf7w7GmJqs2E8BVVUR+UxVewKlTQq+xgJvA4jImcAmVV3i8k+RwkRkAa7V1AOqOq0c5zf+WvoeTL0Kmh0H7fp4j77QrBOEWPcXY2oTf74mpohIb1WdX5YTiEg9XCX3bSLSELgdV7xUkvaquklEOgLfisgyVV1byPEnAhMBYmJiyhKiybVjDXxyPbTsBo1bw+pPYdHrbl1YJLTt7ZJFu97QphfUt7YKxtRk/iSIvsCFIpKGa8kkuJuLOD/PMRxIUdWtItIT6ADk3j20xSWgPqr6u+9OqrrJ+7lORGYCicBRCUJVXwReBEhOTlY/YzIFHT4A742HOvXhgnehSWvIyYGdv8LGebBhLmyYD796N5ISAi26599htOsNUR2g+LtCY0w14k+CGFbOc4zDK15S1WVAi9wVIpIKJKvqDt8dRCQK2KeqB71WTgOAh8oZhynOl3fA1uX5yQFckVJ0Z/dIvMgt278bNi2ADV7SWPouLHjFrQuPhrZ98oumWidC3QZBeTvGmPIrMUGoapqIxAMneou+V9Ul/hzcaxI7FLjKj22TgatV9UrgeOAFEcnBNcV9QFVX+nNOUwYrpsH8l6H/tdC5hO8DDSLhuFPdAyAnG7at8u4yvMfP0926kDpwTFz+HUa7vhDRNpDvxBhTgcR1cShmA5HrgQnAh96is3Gjuj4d4NhKLTk5WRcsWBDsMKqXP36DFwZB885w2f+gTgUM1Lt3B2ycn18stWkhZO136xq39imW6uMSSEWc0xhTJiKyUFWTC13nR4JYCvRT1b3e63Dgx1LUQVQaSxCllHUIJg2DnWvh6u8hqn1gzpN9GH5fdmTSSPe6wITWd0VRucVSbftA45aBicMYc5TiEoQ/dRBC/pwQeM+tJrIm+OZfsDkFzn89cMkBILQutElyj75eaeOeLT7FUnNh7vMw5ym3LrK9u8PoNBR6nmcV38YEiT8J4r/AXBGZ6r0+CzernKnOfv4cfnwGek+AbqMq//xNWkG3M90DXCuqLUvyW0z9NguWvQvrf4IRD0NIaOXHaEwtV2IRE4A3B3XucN/fq+qigEZVRlbE5Kf0TfD8AFdhfMXXUDcs2BEdLScHvrkHZj8Jx/8Jznm5asZpTDVXriImETkBWKGqKd7rJiLSV1XnVnCcpjJkZ8EHV7h6gdGTq+6HbkgIDL0XGh0DX9wGb5wDY99yraiMMZXCn7ETngMyfV5nestMdTTzflj/I4x8HJofF+xoStbvL3DuK66u4r/D3d2PMaZS+JMgRH3KoVQ1B//qLkxVs3YGfP+o6/QWd36wo/Ffz9Fw0fuwewO8chps/znYERlTK/iTINaJyHUiUtd7XA+sC3RgpoJlbIUPJ0J0FxheDTuldxwMl02H7EMuSay3Ek5jAs2fBHE10B83n8NG3NhMEwMZlKlgOdnw4QQ4mAHnTYZ6fs35VPW0iocrvoSGzeC1UbD6s2BHZEyN5s9QG9tww3Wb6uqHx1yz0VFPQ4vjgx1N+TTt4JLEm+fBlAth5BPQ69JgR1XzZR10426lvAr7dwU7mqqjfhNIugTix9bIccf8acX0EHAfsB/4HIgD/qaqbwQ4NlMR0ubAjP+DHqMh8eJgR1MxwpvDpZ/Ae5fCJ9dB5lYYdJN1qAuEfX/A/Fdg3ouwd5sbwbdVfLCjqjp2rIFPb4Bv74PeV7pHo+hgR1Vh/BlqY7GqJojI2cBI4EbgO1Wtcn8l1g+igL074fmBrinrxFkQ1iTYEVWs7MPw8bWw5G1IvhxGPGId6irKzrXw47Ow+C03jtZxp0K/a1xdkCXifKqQNhvmPAO//M8NHRM/1l2r6M7Bjs4v5R1qI3ebM4D3VDW9hJngTFWgCtP+DPt2wBVf1bzkAG4Ij7Oeg0YtYfYTkLkNzn25Rt7qVwpV13P9x2dg9XR3fePOdx921b1oMlBEIHage+xY45LqkrddUVynYdD/Gog9sdomVX8SxKcishpXxPRnEYkGDgQ2LFNuPz4La76A4Q9D64RgRxM4IjD0X9D4GPj8Nnj9bBj3NjSICnZk1Ud2Fqz62CWGTQvdtRv0DzcMiw2c6L/mneBPT8DJ/3TD5897CV79kyuS63ctdD/LJd1qxN+hNpoC6aqa7Y3m2rjgDHBVgRUxeTYuhEmnQefTYcwb1fbbS6kt/wCmXg1Nj3X9JmzuieIdzICU1+Gn59zouk07wgl/gYQLqm9Lt6rk8H5YOsV9WdvxCzRpA32vdo0qwiKCHV2ecg33XZ1YgsDN+PbCIFdccPV3te+b9LpZ8M6Frkjtog+saKQw6Zvc6LkLX4WD6RDTzxUjdRludTiBkJPjpuqd8zSkfg/1GruWTydcDZExwY7OEkStoepa9qyeDpd97mZxq422LIU3R0PWATeFaswJwY6oatiyxFWmrvgQNMeNpNvvWmjbK9iR1R6bF7s7ihUfuv/Xbme6eoo2wfsdWIKoLea/DNP/7ga5G3B9sKMJrl2p8Po5sGeTG8vp+JHBjig4cnLg16/hx6fht++gXiP37bXv1YGdA8QUL30jzH0BFk6Gg3sgpr9LFJ2Hu4EqK1GFJwgR6aqqq8sdWQWr1Qliy1J4+VToMMh9a67kP7Iqae8OeOt82LwIznjUNYWtLQ4f8Cn//tlN9XrC1ZB0qY2IW5Uc2AOLXoefnvfqgY6Ffn+F+HFQr2GlhBCIBLFeVYNfeFZArU0QBzPhxZPg0F64+gfXkcw4h/bCu5e6MuCTboXBt9bsSvu9O92d5PyXYO92OKan14LmbJv7uyrLzoJVH7kiwM0p0KCp63TXZwI0ahHQU5epH4SIPFXUKiCyAuIyFUEVpt8If6xzvYstORypXrhr9vrJ9TDrAcj8HUY8CqE1bEBi3zb4WQeg02mu4rnDoJqdEGuK0DrQ41zofo4bjn/OM/Ddw27CrLy+KF0rPazi/ksuA/4OHCxk3bjAhGNKbfFbrihh8O2us445WmhdOPNZ16Huh8cgczuMfqX6d6hTdUOp/PgM/Pw/CK0H8WPghL8G5cPEVAARaN/fPXb8Cj/9P/c/vuh1OG6oq6focFKlJf0ii5hE5Fvgn6o6p5B1v6lqh0AHV1q1rohp22p4aYhrAXHJR9ZE0R9zX4D/3QLt+sC4d6Bh02BHVHrZWbBymksMmxe54og+E7xxgAJbHGGCYO9OWDApfzysY3q6O4ru51RIsWGZ6iC8znEHVHVfuSOoJLUqQRzeDy+d7IaX+PNs15PY+GfFVDc3RtOOrq9EdelQd2APpLzm+jCkb4Bmx7mObZVYoWmC6PABWPae+2KwfbVreNB3IvS6rFwND8qaIGJUdX2ZzxoEtSpBfHK9ayJ30QduIDVTOr995zrU1WvkrmHLbsGO6GiqsHu9m251/RxY9r5rEtl+gPsG2fl0a61WG6m6pstznnbD+NcNh6SL4dR/lWmO+bIO1jcNSPIO8IGqnlvqM5vAWPa+Sw4D/2bJoaw6DILLPoM3RsN/T3fFTe37BzemwwdcZ7YNc2HjPJcYMre6dfUaQedhLjG0SQpunCa4RKDTUPfYstQ1Tti4AOrUr/hTFXMHsUhVEws+r8pqxR3EzrXwwknuG+/46dVu8K8qZ1cavHGu+6Y++hU4/k+Vd+49m10S2DDPJYQtS9yUqgBRHVw9Sbs+0LYPtOhW81pemYqTnVXmv4+y3kFoEc9NsGQdhPcvc5XR575iyaEiRLWHy79wHerevcTNKdH7ioo/T/Zh+H1ZfjLYMM/VIwDUCYPWSXDCn6FdX2jb2yqbTekE6MtDcUeNF5E9uH4PDbzneK9VVWvgBANV3Fd3uW+ZY9+GyHbBjqbmCG8Gl34M7413fUoyfocht5evKeHeHd7dwVzYOB82pbiJdwCatHXjZPX7q7tDaNnTOrGZKqnIBKGq5WozKSJdgCk+izoCd6nqE976vwOPANGquqOQ/S8F/um9vE9VXy1PPNXeqk9d65W+f4auI4IdTc1TLxzGvgWf3ADfPeQ61J3xuH/fzHKyYduq/GSwYa7ruAgQUtfNB5B8WX5xUUSbgL4VYypKwAo1VfVnIAFAREKBTcBU73U74DSg0FZSXhPbu4FkXPHWQhH5WFVr52zpu9fDR3+BVgluchwTGKF14cxnXJPh7x/xOtRNOroJ6f7drlIwtzJ540I4lOHWhbdwiaDXeJcMWidU/w55ptaqrFqvU4C1qprmvX4cuBn4qIjthwFfqeofACLyFXA68HagA61ysg/D+1e4UTnP+29AWioYHyJwyp0uSXx2E7x2Jgx/ELauyK872O6NUykh0LKH672cW3cQFWtDW5gao7ISxFi8D3cRORPYpKpLipnbug2wwef1Rm9Z7fPtfe6DafQk17HLVI4+EyA8Gj6c4Hqrg5t8qW0f6DnaJYTWSVC/UXDjNCaAAp4gRKQeMAq4TUQaArfjipcq6vgTgYkAMTFVboDZ8vn1a5j9hCuu6GHdUCpd97Og2bHw+3I3nEmz46xjmqlVKuOvfTiQoqpbgWOBDsASEUkF2gIpIlJwnIhNgG8znbbesqOo6ouqmqyqydHR0RUefNDs2QIfXuXav5/+QLCjqb2O6QkJ4yC6syUHU+tUxl/8OLziJVVdpqotVDVWVWNxRUdJqvp7gX2+AE4TkSgRicLdcXxRCbFWDTnZrmjj8D44b7JVchpjgiKgCUJEwoGhwId+bJssIi8DeJXT/wbme497cyusa4XvHnaTm5/xKER3CXY0xphaKqB1EKq6F2hWzPpYn+cLgCt9Xk8CJgUyvirpt+9h1oMQNxYSLgh2NMaYWswKVauSvTvggytda6UzHg12NMaYWs5G/6oKsg/D70vhm3th/y646H1rPmmMCTpLEMGQuT2/09WGeW5WsNxxev70pGs5Y4wxQWYJItBysmHbSjcswwZvnJ5dv7l1BcfpadcXmrQObrzGGOOxBFHR9u/KH6dnwzzYtBAOZbp1ueP0JF/mkkGreGvCaoypsixBlEdODuxck58MNsyDHT+7dXnj9Ix1yaBdH4hsb+P0GGOqDUsQpXEww90R+M4CdiDdrcsdpyfuPBunxxhTI1iCKIqqG9M/d3z/DfNh2wrQHEAguit0Oyu/7qDZcXZ3YIypUSxB5Dq837Um8q1M3ufNY1SvMbRNhkE3uYTQJhkaRAY1XGOMCTRLEFkH4b/D3VSeOVluWdNjodNpblrIdn3d3UJIuSbYM8aYascSRJ36rniow0nelJC9Ibx5sKMyxpigswQBcM6LwY7AGGOqHBuLyRhjTKEsQRhjjCmUJQhjjDGFsgRhjDGmUJYgjDHGFMoShDHGmEJZgjDGGFMoSxDGGGMKJaoa7BgqjIhsB9KCHUc5NQd2BDuIKsKuxZHsehzJrke+8lyL9qoaXdiKGpUgagIRWaCqycGOoyqwa3Ekux5HsuuRL1DXwoqYjDHGFMoShDHGmEJZgqh6bOTAfHYtjmTX40h2PfIF5FpYHYQxxphC2R2EMcaYQlmCqEQi0k5EZojIShFZISLXe8ubishXIrLG+xnlLRcReUpEfhWRpSKSFNx3ULFEJExE5onIEu96/Mtb3kFE5nrve4qI1POW1/de/+qtjw3qGwgAEQkVkUUi8qn3ujZfi1QRWSYii0Vkgbestv6vRIrI+yKyWkRWiUi/yrgWliAqVxbwd1XtBpwA/FVEugG3At+oaifgG+81wHCgk/eYCDxX+SEH1EHgZFWNBxKA00XkBOBB4HFVPQ7YBVzhbX8FsMtb/ri3XU1zPbDK53VtvhYAQ1Q1wacJZ239X3kS+FxVuwLxuL+RwF8LVbVHkB7AR8BQ4GeglbesFfCz9/wFYJzP9nnb1bQH0BBIAfriOvzU8Zb3A77wnn8B9POe1/G2k2DHXoHXoK33j34y8CkgtfVaeO8rFWheYFmt+18BIoDfCv5+K+Na2B1EkHhFAonAXKClqm7xVv0OtPSetwE2+Oy20VtWY3hFKouBbcBXwFpgt6pmeZv4vue86+GtTweaVWrAgfUEcDOQ471uRu29FgAKfCkiC0VkoresNv6vdAC2A//1ih9fFpFwKuFaWIIIAhFpBHwA3KCqe3zXqUv5taZpmapmq2oC7ttzH6BrcCMKDhEZCWxT1YXBjqUKGaiqSbgik7+KyCDflbXof6UOkAQ8p6qJwF7yi5OAwF0LSxCVTETq4pLDm6r6obd4q4i08ta3wn2bBtgEtPPZva23rMZR1d3ADFwxSqSI1PFW+b7nvOvhrY8AdlZupAEzABglIqnAO7hipiepndcCAFXd5P3cBkzFfYGojf8rG4GNqjrXe/0+LmEE/FpYgqhEIiLAK8AqVX3MZ9XHwKXe80txdRO5yy/xWiWcAKT73FJWeyISLSKR3vMGuPqYVbhEMdrbrOD1yL1Oo4FvvW9O1Z6q3qaqbVU1FhiLe28XUguvBYCIhItI49znwGnAcmrh/4qq/g5sEJEu3qJTgJVUxrUIdgVMbXoAA3G3gUuBxd5jBK7s+BtgDfA10NTbXoBnceXyy4DkYL+HCr4eccAi73osB+7ylncE5gG/Au8B9b3lYd7rX731HYP9HgJ0XQYDn9bma+G97yXeYwVwh7e8tv6vJAALvP+VaUBUZVwL60ltjDGmUFbEZIwxplCWIIwxxhTKEoQxxphCWYIwxhhTKEsQxhhjCmUJohYSkVEicmvJWwaeiAzOHbm0qvNi7V+K7VuLyPsVcN7PcvuLBFNl/a68/jFzvWElTiyw7gYRaejHMTIDF2HtYQmiFlLVj1X1gWDHUQ0NBvxOEKq6WVVHl7xliccZoa6nebUmIqF+bnoKsExVE1X1+wLrbsAN7GgqgSWIKkxEYr3x3yeLyC8i8qaInCois70x4Pt42/URkR+9b1xzcntcisjfRGSS97yniCwXkYYiMl5EnvGWTxaR50TkJxFZ531LnOSNOT/ZJ5ZMn+ejc9f5u3+B93W6975SgHN8ljcVkWneGPY/iUict7yRiPxX3NwAS0Xk3IqISURO865bioi8542RlTsPwb+85ctEpKu4wRWvBv4mbn6Cgt9sT/KWL/Z+D429399yb/3LPuu3i8jd3vKbRGS+977+VcT1ShWR5t7xVonIS+Lmz/hSXA/0gttPFjcfwBzv/Y/2lh9xByAiz4jIeJ9z3O/Ft0BEkkTkCxFZKyJX+xy+iYhMF5GfReR5EQnx41o+6P2uzysQZ6yIfOu9929EJEZEEoCHgDO9WBr4bH8d0BqYISIzvGXjvN/RchE5ashz77r9KCJniLsz+cC73vNFZIC3zT3e38dM73pd5y0P997rEu/4Ywr7/dRowe4haI9ie0/G4uaQ6IlL5guBSbiekmcC07ztmpA/JPSpwAfe8xDgO+BsXC/MAd7y8cAz3vPJuLF/co+5p8D5ErztMn3iGg1MLs3+PvuG4Uaa7OTt8y75vYafBu72np8MLPaePwg84XOMqPLGBDT3rk24t88t5PfkTgWu9Z7/BXjZe34P8I8iflef+FzfRrgB1mKB5QW2a48bTqQ9bviIF704Q3BDfA8q5NipXry5fw+5v5N3gYsK2X4yrpd1CNAN+NVbPjj3WnuvnwHG+5zjz97zx3E9dhsD0cBWn/0P4Ho5h+JG3x3tx7W8uZhrdqn3/HLy/57H4/19FnUtvOetgfVejHWAb4Gzcv82cKObzgWGesvewg0ACBCDG/Im9/c6B6jvvZedQF3gXOAln3NHBPszobIfuYOAmarrN1VdBiAiK3AThKiILMN9YIAbqO1VEemEG8qjLoCq5njfEJcCL6jq7CLO8YnPMbcWOF8sbkiQ4pRm/67ee1rjbfMGblITcEORnOvF/q2INBORJrikNzb3AKq6q4R4/ImpLe7Dc7aIANQDfvTZP3cgxYX43OUUYzbwmIi8CXyoqhu94+YRkdzhMa5V1TQRuRaXJBZ5mzTCJc7vijnPb6q62Ce22CK2m6aqOcBKEWlZxDYFfez9XAY0UtUMIENEDkp+Hcg8VV3nvZ+3cb+zAxR/LacUcb5+5F/b13F3DqXRG5ipqtu9eN4EBuGGoqiLG4bir6o6y9v+VKCbz++lSe6dDjBdVQ8CB0VkGy65LAMe9e5MPtWji7tqPEsQVd9Bn+c5Pq9zyP/9/RuYoapne0UhM3326YT7NtXaj3P4Hr/gOXzHZAkrw/6BUJ6YsoGvVHVcEcfO3ScbP96Dqj4gItNxY2vNFpFhuA9OX8/jksfX3msB7lfVF0o6fiFx5cZ2VBFTIdvlfiJmcWSxcll+jwXH5lHv+MVdy71FLA+kLFwCHQbkJogQ4ARVPeL34iWMgte1jqr+Im66zhHAfSLyjareG/DIqxCrg6gZIsgfznd87kIRiQCewn2rapZbFl1GW0XkeK/M+exyHGc1ECsix3qvfT9UvgcuBFdeDuxQN1/GV8BfczcSb+7dcsb0EzBARI7zjhkuIp1L2CcDV+xyFBE5VlWXqeqDwHwKzGshIn8FGuuRjQO+AC73Ka9vIyItSvk+SiMN9w26vndHcEoZjtFH3DzZIcAY4AfKdi3BFevk3hleiPv9l8T3dzAPOMmrZwjF/S3lJgPFFVt1FZFbvGVfAtfmHsir7yiSiLQG9qnqG8DDuCG2axVLEDXDQ8D9IrKII7/tPg48q6q/4OYwfqAcH0C34srI5wBlHkbZ+/Y2EZjuVVxu81l9D9BLRJYCD5A/lPF9QJRXUbgEGFLemLxiifHA2975fqTkyYo+Ac6WQiqpgRu8+JYCh4H/FVj/D6Cn5FdUX62qX+LKxX/0isLep4gEVBFUdQOu3mK593NR8XsUaj6u7mIVbhrMqWW8luA+rC/z9rkYNx93SV4EPheRGeqGsL4VNyT6EmChquYOeY2qZuOSxski8hfgOiDZqxRfiWt0UJyewDxxMx7ejfs7rFVsNFdjjDGFsjsIY4wxhbIEYYwxplCWIIwxxhTKEoQxxphCWYIwxhhTKEsQxhhjCmUJwhhjTKEsQRhjjCnU/wfrHIGCgLyeFQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(x_large, y_large, label=\"BERT large split uncased\")\n",
    "plt.plot( x_base,  y_base, label=\"BERT base split uncased\")\n",
    "plt.xlabel(\"maximum document size in number of tokens\")\n",
    "plt.ylabel(\"F1 scores for Mention detection\")\n",
    "plt.legend()\n",
    "plt.savefig(\"f1.png\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Accessing the BERT tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/erikt/projects/rel20/venv3/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer, AutoModelForTokenClassification\n",
    "from transformers import pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_or_url = \"dslim/bert-base-NER\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(path_or_url)\n",
    "model = AutoModelForTokenClassification.from_pretrained(path_or_url)\n",
    "ner_tagger = pipeline(\"ner\", model=model, tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CLS]\n",
      "John\n",
      "saw\n",
      "Mary\n",
      "in\n",
      "G\n",
      "##ron\n",
      "##ingen\n",
      ".\n",
      "[SEP]\n"
     ]
    }
   ],
   "source": [
    "for token_id in ner_tagger.tokenizer([\"John saw Mary in Groningen.\"])[\"input_ids\"][0]:\n",
    "    print(ner_tagger.tokenizer.decode(token_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}