week10_nlp&w2v/3. w2v_pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a90c041c",
   "metadata": {},
   "source": [
    "# Word2Vec training\n",
    "\n",
    "This notebook is based on the [word2vec-pytorch](https://github.com/OlgaChernytska/word2vec-pytorch/tree/18d4f24c4d82dde76de9f6e4d88bbcbcf68befc2) repo. Notebook from [Ildus course](https://github.com/isadrtdinov/intro-to-dl-hse/blob/2022-2023/seminars/202/seminar-06-w2v.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "61b9afb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%bash\n",
    "# wget https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-v1.zip\n",
    "# unzip wikitext-2-v1.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ca161130",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install torchtext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "282f0e5e",
   "metadata": {},
   "outputs": [
    {
     "ename": "OSError",
     "evalue": "dlopen(/opt/homebrew/lib/python3.10/site-packages/torchtext/lib/libtorchtext.so, 0x0006): Symbol not found: (__ZN3c106detail19maybe_wrap_dim_slowIxEET_S2_S2_b)\n  Referenced from: '/opt/homebrew/lib/python3.10/site-packages/torchtext/lib/libtorchtext.so'\n  Expected in: '/opt/homebrew/lib/python3.10/site-packages/torch/lib/libc10.dylib'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[7], line 6\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtorch\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m nn\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtorch\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdata\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Dataset, DataLoader\n\u001b[0;32m----> 6\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtorchtext\u001b[39;00m\n\u001b[1;32m      8\u001b[0m \u001b[38;5;66;03m# from torchtext.data.utils import get_tokenizer\u001b[39;00m\n\u001b[1;32m      9\u001b[0m \u001b[38;5;66;03m# from torchtext.vocab import build_vocab_from_iterator\u001b[39;00m\n\u001b[1;32m     11\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mmatplotlib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpyplot\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mplt\u001b[39;00m\n",
      "File \u001b[0;32m/opt/homebrew/lib/python3.10/site-packages/torchtext/__init__.py:6\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtorch\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mhub\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _get_torch_home\n\u001b[1;32m      5\u001b[0m \u001b[38;5;66;03m# the following import has to happen first in order to load the torchtext C++ library\u001b[39;00m\n\u001b[0;32m----> 6\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtorchtext\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _extension  \u001b[38;5;66;03m# noqa: F401\u001b[39;00m\n\u001b[1;32m      8\u001b[0m _TEXT_BUCKET \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttps://download.pytorch.org/models/text/\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     10\u001b[0m _CACHE_DIR \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mexpanduser(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(_get_torch_home(), \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m\"\u001b[39m))\n",
      "File \u001b[0;32m/opt/homebrew/lib/python3.10/site-packages/torchtext/_extension.py:64\u001b[0m\n\u001b[1;32m     59\u001b[0m     \u001b[38;5;66;03m# This import is for initializing the methods registered via PyBind11\u001b[39;00m\n\u001b[1;32m     60\u001b[0m     \u001b[38;5;66;03m# This has to happen after the base library is loaded\u001b[39;00m\n\u001b[1;32m     61\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtorchtext\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _torchtext  \u001b[38;5;66;03m# noqa\u001b[39;00m\n\u001b[0;32m---> 64\u001b[0m \u001b[43m_init_extension\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/homebrew/lib/python3.10/site-packages/torchtext/_extension.py:58\u001b[0m, in \u001b[0;36m_init_extension\u001b[0;34m()\u001b[0m\n\u001b[1;32m     55\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m _mod_utils\u001b[38;5;241m.\u001b[39mis_module_available(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtorchtext._torchtext\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[1;32m     56\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtorchtext C++ Extension is not found.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m---> 58\u001b[0m \u001b[43m_load_lib\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mlibtorchtext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m     59\u001b[0m \u001b[38;5;66;03m# This import is for initializing the methods registered via PyBind11\u001b[39;00m\n\u001b[1;32m     60\u001b[0m \u001b[38;5;66;03m# This has to happen after the base library is loaded\u001b[39;00m\n\u001b[1;32m     61\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtorchtext\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _torchtext\n",
      "File \u001b[0;32m/opt/homebrew/lib/python3.10/site-packages/torchtext/_extension.py:50\u001b[0m, in \u001b[0;36m_load_lib\u001b[0;34m(lib)\u001b[0m\n\u001b[1;32m     48\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m path\u001b[38;5;241m.\u001b[39mexists():\n\u001b[1;32m     49\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[0;32m---> 50\u001b[0m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mops\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload_library\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     51\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m\n",
      "File \u001b[0;32m/opt/homebrew/lib/python3.10/site-packages/torch/_ops.py:573\u001b[0m, in \u001b[0;36mload_library\u001b[0;34m(self, path)\u001b[0m\n\u001b[1;32m    566\u001b[0m except RuntimeError as e:\n\u001b[1;32m    567\u001b[0m     # Turn this into AttributeError so getattr(obj, key, default)\n\u001b[1;32m    568\u001b[0m     # works (this is called by TorchScript with __origin__)\n\u001b[1;32m    569\u001b[0m     raise AttributeError(\n\u001b[1;32m    570\u001b[0m         f\"'_OpNamespace' '{self.name}' object has no attribute '{op_name}'\"\n\u001b[1;32m    571\u001b[0m     ) from e\n\u001b[0;32m--> 573\u001b[0m # let the script frontend know that op is identical to the builtin op\n\u001b[1;32m    574\u001b[0m # with qualified_op_name\n\u001b[1;32m    575\u001b[0m torch.jit._builtins._register_builtin(op, qualified_op_name)\n\u001b[1;32m    576\u001b[0m op.__module__ = self.__module__ + \".\" + namespace_name\n",
      "File \u001b[0;32m/opt/homebrew/Cellar/python@3.10/3.10.9/Frameworks/Python.framework/Versions/3.10/lib/python3.10/ctypes/__init__.py:374\u001b[0m, in \u001b[0;36mCDLL.__init__\u001b[0;34m(self, name, mode, handle, use_errno, use_last_error, winmode)\u001b[0m\n\u001b[1;32m    371\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_FuncPtr \u001b[38;5;241m=\u001b[39m _FuncPtr\n\u001b[1;32m    373\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m handle \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 374\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_handle \u001b[38;5;241m=\u001b[39m \u001b[43m_dlopen\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    375\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    376\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_handle \u001b[38;5;241m=\u001b[39m handle\n",
      "\u001b[0;31mOSError\u001b[0m: dlopen(/opt/homebrew/lib/python3.10/site-packages/torchtext/lib/libtorchtext.so, 0x0006): Symbol not found: (__ZN3c106detail19maybe_wrap_dim_slowIxEET_S2_S2_b)\n  Referenced from: '/opt/homebrew/lib/python3.10/site-packages/torchtext/lib/libtorchtext.so'\n  Expected in: '/opt/homebrew/lib/python3.10/site-packages/torch/lib/libc10.dylib'"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "from torch import nn\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "import torchtext\n",
    "\n",
    "# from torchtext.data.utils import get_tokenizer\n",
    "# from torchtext.vocab import build_vocab_from_iterator\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "sns.set_style('whitegrid')\n",
    "plt.rcParams.update({'font.size': 14})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1b6b28f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_TOKENS = 'wikitext-2/wiki.train.tokens'\n",
    "VALID_TOKENS = 'wikitext-2/wiki.valid.tokens'\n",
    "\n",
    "SKIPGRAM_N_WORDS = 4\n",
    "MIN_WORD_FREQUENCY = 20\n",
    "MAX_SEQUENCE_LENGTH = 256\n",
    "\n",
    "EMBED_DIM = 256\n",
    "EMBED_MAX_NORM = 1\n",
    "BATCH_SIZE = 16\n",
    "NUM_EPOCHS = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "394ba4d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = get_tokenizer(\"basic_english\", language=\"en\")\n",
    "\n",
    "def load_and_preprocess_dataset(path):\n",
    "    with open(path, 'r') as file:\n",
    "        lines = file.readlines()\n",
    "\n",
    "    data = list(filter(\n",
    "        lambda sent: len(sent) > SKIPGRAM_N_WORDS * 2,\n",
    "        map(tokenizer, lines)\n",
    "    ))\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "278e49c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num sentences: train - 17715, valid - 1892)\n",
      "Total tokens: train - 2014338, valid - 211057)\n",
      "Average tokens in sentence: 113.71\n"
     ]
    }
   ],
   "source": [
    "train_data = load_and_preprocess_dataset(TRAIN_TOKENS)\n",
    "valid_data = load_and_preprocess_dataset(VALID_TOKENS)\n",
    "\n",
    "total_train_tokens = sum(len(sent) for sent in train_data)\n",
    "total_valid_tokens = sum(len(sent) for sent in valid_data)\n",
    "\n",
    "print(f'Num sentences: train - {len(train_data)}, valid - {len(valid_data)})')\n",
    "print(f'Total tokens: train - {total_train_tokens}, valid - {total_valid_tokens})')\n",
    "print(f'Average tokens in sentence: {total_train_tokens / len(train_data):.2f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "522172b6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd4AAAHdCAYAAACzChhtAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAwCklEQVR4nO3dfVyUdb7/8feAmg6QYsHqgkrazmaKayq4GmmnLMu1ElZdK0otq+PNZvbwmO1ma+Vv7dGdd5U3lVnhtspZyY7HWk+eo8mWYlJJScebAu8TW01xUJCZ3x+cmRhmkJkRvgzD6/l48Gjme33mmu/Ml3xzXd/rxuJ0Op0CAABGRDR2BwAAaE4IXgAADCJ4AQAwiOAFAMAgghcAAIMIXgAADCJ4AQAwiOBtBE6nU5WVleIUagBofgjeRuBwOPTFF1/I4XD4Xb9z506/69FwGIvQwViEDsYiMARvE+B0OlVRUcEWcghgLEIHYxE6GIvAELwAABhE8AIAYBDBCwCAQQQvAAAGEbwAABhE8AIAYBDBCwCAQQQvAAAGEbwAABhE8AIAYBDBCwCAQQQvAAAGEbwAABhE8AIAYBDBCwCAQQQvAAAGEbwAABhE8AIAYFCLxu4Agrd4014dPFEmSUqMbaOJ11/ZyD0CANSF4G3CDp4o076S0sbuBgAgAOxqBgDAIIIXAACDCF4AAAwieAEAMIjgBQDAIIIXAACDCF4AAAwieAEAMIjgBQDAIIIXAACDCF4AAAwieAEAMIjgBQDAIIIXAACDCF4AAAwieAEAMIjgBQDAIIIXAACDCF4AAAwieAEAMIjgBQDAIIIXAACDCF4AAAwieAEAMIjgBQDAIIIXAACDCF4AAAwieAEAMIjgBQDAoBaN3YGavv/+e33wwQf6+OOP9e233+r48eNq27at+vTpowkTJuhXv/qV12tKS0u1aNEibdiwQSUlJYqPj9fQoUM1ZcoURUVFedU7HA6tXLlSq1evVnFxsaxWqwYOHKhp06apU6dOPvu1ZcsWLV26VF9//bUsFot69OihSZMmacCAAfX+HQAAwlfIbfG+8847mjt3rg4cOKBrr71W48ePV9++fbVx40aNGTNG69ev96i32+3KzMzUihUr1LVrV40bN05XXHGFli9frrFjx+rcuXNe7/Hkk09qzpw5cjqduueee3Tddddpw4YNGjlypIqKirzq165dqwkTJmjfvn3KyMhQenq69u7dq/Hjx+vDDz9sqK8CABCGQm6Lt1evXnrnnXeUmprq0f7ZZ59p3Lhxmj17toYMGaJWrVpJkl5//XUVFhbqgQce0PTp0931L7zwgl577TWtWLFCDz30kLt969atys7OVkpKipYvX+5ez/Dhw/Xggw/qmWee0RtvvOGu//HHHzVnzhzFxsYqJydHHTp0kCQ98MADGjFihGbPnq20tDRFR0c32HcCAAgfIbfFe/PNN3uFriT169dP/fv3148//qj//d//lSQ5nU5lZ2fLarVq0qRJHvWTJk2S1WpVdna2R7vr+dSpU92hK0mDBw9WamqqcnNzdfjwYXf7hx9+qFOnTikzM9MdupLUoUMHZWZm6sSJE/roo48u/oMDAJqFkNvivZAWLVp4/LeoqEjHjh1TWlqarFarR63ValWfPn2Um5urI0eOqGPHjpKkbdu2uZfVdN111ykvL095eXkaMWKEJCkvL0+SlJaW5lWflpamRYsWedQHorKyMqC6mvVOZ9WP67G/60PwahsLmMdYhA7GokpkZKRfdU0meA8fPqxPPvlEcXFxstlskqTi4mJJUlJSks/XJCUlKTc3V0VFRerYsaPsdrtKSkpks9l8fkFdunTxWK8k95yva1ld9YEoKCgIut5qtaqszK7S0lJJUll0hHbv3i273R5UXxCYQMcODYexCB3NfSz69u3rV12TCN6KigrNmDFD5eXlmj59ujs0T58+LUm1zq+62l3h5G+9q676a2NiYvyqD0RycrJffyFVVlaqoKDAq75N4S5FRzuqHrexuv8gQcOpbSxgHmMROhiLwIR88DocDs2cOVPbt2/X6NGjg9qlG6oiIyMD+iWtWW+xVP24HvMLb06gY4eGw1iEDsbCPyF3cFV1DodDf/jDH7Ru3TrdfvvteuqppzyWu7ZCXVulNbnaXVum/tZX37q90FbthbaGAQDwJWSD1+Fw6PHHH1dOTo6GDx+uZ599VhERnt11zbH6Ove2ertrDthqtSouLk4HDx70eRCAa662+nyu67W+5nF91QMAcCEhGbyu0H3vvfc0bNgwPffccz53XyQlJSk+Pl75+fleBxXZ7Xbl5+crMTHRfUSzJKWmprqX1bRlyxZJUkpKirvN9Tg3N9er3tXm6/QnAAB8Cbngde1efu+993TLLbfo+eefr3XOwGKxaNSoUbLb7Xr11Vc9lr366quy2+0aPXq0R7vr+YIFC1ReXu5u37x5s/Ly8pSWlqaEhAR3+6233qqYmBhlZWXp6NGj7vajR48qKytLsbGxGjJkyEV/bgBA8xByB1e98sorysnJkdVqVVJSkhYvXuxVM2TIEHXv3l2SNGHCBG3cuFGvvfaaCgsLdfXVV2vXrl3Kzc1VcnKyxo4d6/HaX//61xo1apSys7OVkZGhwYMHq6SkROvXr1e7du30xBNPeNS3bdtWs2bN0owZM5Senq5hw4ZJktavX6+TJ09q3rx5XLUKAOC3kAveQ4cOSaraVbxkyRKfNQkJCe7gtVqtysrKct8kYdu2bYqLi9N9992nyZMnq3Xr1l6vf/rpp2Wz2bR69Wq9/fbbslqtuummmzRt2jR17tzZq/6OO+5QbGysli5dqjVr1kiSevbsqYkTJ2rgwIH19dEBAM2Axel0XfsIplRWVuqLL75Q7969/T6P11f9H3MKtK+k6sjqbnHR+n/pyQ3WZ1QJdOzQcBiL0MFYBCbk5ngBAAhnBC8AAAYRvAAAGETwAgBgEMELAIBBBC8AAAYRvAAAGETwAgBgEMELAIBBBC8AAAYRvAAAGETwAgBgEMELAIBBBC8AAAYRvAAAGETwAgBgEMELAIBBBC8AAAYRvAAAGETwAgBgEMELAIBBBC8AAAYRvAAAGETwAgBgEMELAIBBBC8AAAYRvAAAGETwAgBgEMELAIBBBC8AAAYRvAAAGETwAgBgEMELAIBBBC8AAAYRvAAAGETwAgBgEMELAIBBBC8AAAYRvAAAGETwAgBgEMELAIBBBC8AAAYRvAAAGETwAgBgEMELAIBBBC8AAAYRvAAAGETwAgBgEMELAIBBBC8AAAYRvAAAGETwAgBgEMELAIBBBC8AAAYRvAAAGETwAgBgEMELAIBBBC8AAAYRvAAAGETwAgBgEMELAIBBBC8AAAYRvAAAGETwAgBgEMELAIBBBC8AAAYRvAAAGETwAgBgEMELAIBBBC8AAAYRvAAAGETwAgBgEMELAIBBBC8AAAYRvAAAGETwAgBgEMELAIBBBC8AAAa1aOwO+LJ27Vrt2LFDX331lXbv3q2KigrNnTtXGRkZXrWLFi3Syy+/XOu6Nm7cqMTERK/2LVu2aOnSpfr6669lsVjUo0cPTZo0SQMGDPC5nu+++07z58/X1q1bVVZWpqSkJI0ZM0Z33nmnLBZL8B8WANCshGTwLliwQIcOHVJsbKzi4+N16NChOl+Tnp6uhIQEr/ZLL73Uq23t2rWaMWOG2rdv7w7z9evXa/z48Zo/f75uueUWj/q9e/dqzJgxOnv2rG699VbFx8dr8+bNeuqpp7Rv3z7NmjUryE8KAGhuQjJ458yZoy5duighIUHLli3Tiy++WOdr0tPT1b9//zrrfvzxR82ZM0exsbHKyclRhw4dJEkPPPCARowYodmzZystLU3R0dHu18yePVunT5/WsmXLNHjwYEnS1KlTNX78eGVlZWn48OG65pprgvy0AIDmJCTneAcOHOhz67U+fPjhhzp16pQyMzPdoStJHTp0UGZmpk6cOKGPPvrI3f7dd99p+/bt6t+/vzt0JalVq1aaOnWqJGn16tUN0lcAQPgJyS3eYGzfvl1ffvmlIiIilJSUpAEDBigqKsqrLi8vT5KUlpbmtSwtLU2LFi1SXl6eRowYUWd93759ZbVatX379qD6XFlZGVBdzXqns+rH9djf9SF4tY0FzGMsQgdjUSUyMtKvurAJ3kWLFnk8v/TSS/XHP/7RHaAuRUVFkqQuXbp4rcPVVlxc7Fd9ZGSkEhMTtXfvXp0/f14tWgT2dRYUFARdb7VaVVZmV2lpqSSpLDpCu3fvlt1uD2idCE6gY4eGw1iEjuY+Fn379vWrrskH71VXXaU///nPSk1NVXx8vEpKSrRp0yYtXLhQM2fOVExMjG688UZ3vSuoYmJivNblmtc9ffq0X/WSFBUVJYfDoTNnzqht27YB9T05Odmvv5AqKytVUFDgVd+mcJeiox1Vj9tYZbPZAnp/BK62sYB5jEXoYCwC0+SD96abbvJ4npiYqMzMTHXr1s19lHL14A0lkZGRAf2S1qy3WKp+XI/5hTcn0LFDw2EsQgdj4Z+QPLiqPgwYMECdO3fW7t273Vutku+tWhdfW7cXqpekM2fOyGKx+JxPBgCgprANXkmKjY2VJJWVlbnbkpKSJHnO47q42qrP516ovrKyUgcPHlRiYmLA87sAgOYpbIPXbrdrz549slqt7gCWpJSUFElSbm6u12tcbampqX7V79ixQ3a73V0DAEBdmnTwlpaW6rvvvvNqP3v2rGbNmqUzZ87olltu8dgavfXWWxUTE6OsrCwdPXrU3X706FFlZWUpNjZWQ4YMcbd37dpVKSkp2rZtmzZv3uxuLy8v14IFCyRJo0aNaoiPBwAIQyG5fzQ7O1s7duyQJO3evdvd5jqntm/fvho1apROnjypW2+9VcnJyerWrZsuv/xy/fDDD/rkk0909OhR2Ww2zZgxw2Pdbdu21axZszRjxgylp6dr2LBhkqouGXny5EnNmzfP46pVkvSnP/1Jd955pyZPnqxhw4YpLi5Omzdv1p49e5SZmak+ffo09FcCAAgTIRm8O3bsUE5Ojkdbfn6+8vPz3c9HjRqldu3a6a677tLOnTu1efNmnTp1Spdccom6deume+65R5mZmWrdurXX+u+44w7FxsZq6dKlWrNmjSSpZ8+emjhxogYOHOhV/4tf/EKrV6/W/PnztXnzZtntdiUlJenJJ5/UXXfdVc+fHgAQzkIyeJ999lk9++yzddZFR0frySefDOo9Bg0apEGDBvld37VrVy1cuDCo9wIAwKVJz/ECANDUELwAABhE8AIAYBDBCwCAQQQvAAAGEbwAABhE8AIAYBDBCwCAQQQvAAAGEbwAABhE8AIAYBDBCwCAQQQvAAAGEbwAABhE8AIAYBDBCwCAQQQvAAAGEbwAABhE8AIAYBDBCwCAQQQvAAAGEbwAABhE8AIAYBDBCwCAQQQvAAAGEbwAABhE8AIAYBDBCwCAQQQvAAAGEbwAABhE8AIAYFDQwbt9+3YdPnz4gjVHjhzR9u3bg30LAADCTtDBe++992rNmjUXrHnvvfd07733BvsWAACEnaCD1+l01lnjcDhksViCfQsAAMJOg87xFhcXKyYmpiHfAgCAJqVFIMWPP/64x/ONGzfq0KFDXnUOh0NHjhzRZ599pkGDBl1cDwEACCMBBW9OTo77scViUWFhoQoLC33WWiwWJScne4U1AADNWUDBu3HjRklV87tDhgzR2LFjfR48FRkZqUsvvVRWq7V+egkAQJgIKHgTEhLcj+fOnavu3bt7tAEAgAsLKHirS09Pr89+AADQLAQdvC47d+5UQUGBTp06pcrKSq/lFotFkydPvti3AQAgLAQdvCdPntTkyZOVn59/wXN6CV4AAH4SdPA+++yz2rFjh1JTU5Wenq4OHTooMjKyPvsGAEDYCTp4/+d//ke9evXSW2+9xdWpAADwU9BXrjp37pz69etH6AIAEICgg/eqq67yedUqAABQu6CDd8qUKfrv//5vffHFF/XYHQAAwlvQc7zHjx/X9ddfr8zMTN12223q0aOHoqOjfdaOGDEi2LcBACCsBB28M2fOlMVikdPpVE5OjnJycrzme51OpywWC8ELAMD/CTp4586dW5/9AACgWeCSkQAAGBT0wVUAACBwQW/xHj582O/an//858G+DQAAYSXo4L3hhhv8uniGxWLRrl27gn0bAADCStDBO2LECJ/Be/r0aX3zzTc6ePCgUlJSlJiYeFEdBAAgnFzUTRJq43Q6tXz5cr3++uv685//HOxbAAAQdhrk4CqLxaL7779fV155pZ577rmGeAsAAJqkBj2quWfPntq6dWtDvgUAAE1KgwbvgQMHdP78+YZ8CwAAmpSg53hr43A49P3332vNmjXauHGjBgwYUN9vAQBAkxV08F511VUXPJ3I6XSqbdu2euyxx4J9CwAAwk7QwZuSkuKzPSIiQm3btlXPnj3129/+VpdddlnQnQMAINwEHbzvvPNOffYDAIBmgWs1AwBgUL0cXLVjxw598803Ki0tVXR0tK666ir17du3PlYNAEBYuajgzc/P1+OPP679+/dL+unG95LUpUsXzZ07V9dcc83F9xIAgDARdPDu2bNH999/v8rKynTttdeqf//+iouLU0lJibZt26Z//OMfuv/++7V69WpdeeWV9dlnAACarKCD95VXXlFFRYWWLVumQYMGeSx78MEH9fHHH2vSpEl65ZVXNG/evIvuKAAA4SDog6vy8vI0dOhQr9B1GTRokIYOHapt27YF3TkAAMJN0MF7+vTpOm/5l5iYqNOnTwf7FgAAhJ2ggzc+Pl5ffPHFBWu+/PJLxcfHB/sWAACEnaCD94YbblBeXp7mz5+vc+fOeSw7d+6cFi5cqG3btunGG2+86E4CABAugj64atKkSdq0aZOWLl2qVatWqVevXrrsssv0ww8/qKCgQP/85z/VqVMnTZo0qT77CwBAkxZ08MbGxmrVqlV6/vnntX79em3evNm97JJLLlFGRoamT5+udu3a1Uc/AQAICxd1AY327dtr7ty5evrpp/Xtt9+6r1zVtWtXtWzZsr76CABA2Ag4eBcvXqyysjL9/ve/d4dry5Yt9ctf/tJdU15ernnz5ikqKkoPPvhg/fUWAIAmLqCDqz755BMtXLhQ7dq1u+AWbatWrdSuXTvNmzdPW7duvehOAgAQLgIK3vfee0+XXnqpMjMz66y9++671bZtW61ZsybozgEAEG4C2tX8+eefa+DAgWrVqlWdta1atdLAgQOVn58fcKfWrl2rHTt26KuvvtLu3btVUVGhuXPnKiMjw2d9aWmpFi1apA0bNqikpETx8fEaOnSopkyZoqioKK96h8OhlStXavXq1SouLpbVatXAgQM1bdo0derUyed7bNmyRUuXLtXXX38ti8WiHj16aNKkSRowYEDAnw8A0HwFtMV77NixWoPJl8TERJWUlATcqQULFmjVqlU6fPhwnRfgsNvtyszM1IoVK9S1a1eNGzdOV1xxhZYvX66xY8d6nWMsSU8++aTmzJkjp9Ope+65R9ddd502bNigkSNHqqioyKt+7dq1mjBhgvbt26eMjAylp6dr7969Gj9+vD788MOAPx8AoPkKaIs3IiJCFRUVftdXVFQoIiLwa3TMmTNHXbp0UUJCgpYtW6YXX3yx1trXX39dhYWFeuCBBzR9+nR3+wsvvKDXXntNK1as0EMPPeRu37p1q7Kzs5WSkqLly5e7t96HDx+uBx98UM8884zeeOMNd/2PP/6oOXPmKDY2Vjk5OerQoYMk6YEHHtCIESM0e/ZspaWlKTo6OuDPCQBofgJKxfj4eO3Zs8fv+j179gR1yciBAwcqISGhzjqn06ns7GxZrVavC3VMmjRJVqtV2dnZHu2u51OnTvXYZT548GClpqYqNzdXhw8fdrd/+OGHOnXqlDIzM92hK0kdOnRQZmamTpw4oY8++ijgzwgAaJ4CCt6+fftq69atOnjwYJ21Bw8e1NatW5WSkhJ05+pSVFSkY8eOqU+fPrJarR7LrFar+vTpowMHDujIkSPu9m3btrmX1XTddddJqrrzkovrcVpamle9q616PQAAFxLQrua7775ba9as0cMPP6zXX39d7du391l34sQJTZ06VZWVlbrzzjvrpaO+FBcXS5KSkpJ8Lk9KSlJubq6KiorUsWNH2e12lZSUyGazKTIy0qu+S5cuHuuV5J7zdS2rqz4QlZWVAdXVrHc6q35cj/1dH4JX21jAPMYidDAWVXzlii8BBW+PHj00duxYvfXWW/rNb36jMWPGqH///u5dsN9//70+/fRTrV69Wv/85z81fvx49ejRI/De+8l1y8Ha5ldd7aWlpQHVV7+Voeu1MTExftUHoqCgIOh6q9WqsjK7u39l0RHavXu37HZ7UH1BYAIdOzQcxiJ0NPex6Nu3r191AV+5aubMmbrkkkv0xhtvaMmSJVqyZInHcqfTqcjISD300EN65JFHAl19s5KcnOzXX0iVlZUqKCjwqm9TuEvR0Y6qx22sstlsDdZXVKltLGAeYxE6GIvABBy8FotFjz76qEaOHKm//e1v+vzzz3X8+HFJ0uWXX64+ffooIyNDnTt3rvfO1uTaCnVt9dXkandtmfpbX33rtvpWbWxsbJ31gYiMjAzol7RmvcVS9eN6zC+8OYGOHRoOYxE6GAv/BH2ThM6dO2vatGn12ZeAueZYfZ17W73dNQdstVoVFxengwcPqrKy0usXxDVXW30+NykpSV999ZWKi4u9gtdXPQAAFxL4SbYhJCkpSfHx8crPz/ea27Tb7crPz1diYqI6duzobk9NTXUvq2nLli2S5HEktutxbm6uV72rLTU19eI/DACgWWjSwWuxWDRq1CjZ7Xa9+uqrHsteffVV2e12jR492qPd9XzBggUqLy93t2/evFl5eXlKS0vzOIf41ltvVUxMjLKysnT06FF3+9GjR5WVlaXY2FgNGTKkIT4eACAMXdT9eBtKdna2duzYIUnavXu3u811vmzfvn01atQoSdKECRO0ceNGvfbaayosLNTVV1+tXbt2KTc3V8nJyRo7dqzHun/9619r1KhRys7OVkZGhgYPHqySkhKtX79e7dq10xNPPOFR37ZtW82aNUszZsxQenq6hg0bJklav369Tp48qXnz5nHVKgCA30IyeHfs2KGcnByPtvz8fI/dw67gtVqtysrKct8kYdu2bYqLi9N9992nyZMnq3Xr1l7rf/rpp2Wz2bR69Wq9/fbbslqtuummmzRt2jSfB4Xdcccdio2N1dKlS913W+rZs6cmTpyogQMH1udHBwCEOYvT6boEA0yprKzUF198od69e/t9OpGv+j/mFGhfSdWR1d3iovX/0pMbrM+oEujYoeEwFqGDsQhMk57jBQCgqSF4AQAwiOAFAMAgghcAAIMIXgAADCJ4AQAwiOAFAMAgghcAAIMIXgAADCJ4AQAwiOAFAMAgghcAAIMIXgAADCJ4AQAwiOAFAMAgghcAAIMIXgAADCJ4AQAwiOAFAMAgghcAAIMIXgAADCJ4AQAwiOAFAMAgghcAAIMIXgAADCJ4AQAwiOAFAMCgFo3dATQPizft1cETZZKkxNg2mnj9lY3cIwBoHAQvjDh4okz7SkobuxsA0OjY1QwAgEFs8aJBVN+13Kdzu8btDACEEIIXDaL6ruXE2DaN3BsACB3sagYAwCCCFwAAgwheAAAMInhhXKTF0thdAIBGw8FVMK5ju9ZcUANAs0XwolFwQQ0AzRW7mgEAMIjgBQDAIIIXAACDmONFyOHAKwDhjOBFUOozHGueXsSBVwDCGcGLoNRnOFY/vYgbKgAId8zxIiS4gvzY6XMe7VxsA0C4YYsXIY2LbQAINwQvQh5zvgDCCbuaAQAwiOAFAMAgghcAAIMIXgAADCJ4AQAwiOAFAMAgghcAAIMIXgAADCJ4AQAwiCtXwS/VL9vIjQwAIHgEL/xS/bKNibFtGrk3ANB0sasZF407CAGA/9jixUWreQchdkUDQO0IXtQLdkUDgH/Y1Ywmhd3aAJo6tnjRpNTcrZ0Y20YTr7+ykXsFAP4jeNHkVN+tDQBNDbuaAQAwiOBFk8acL4Cmhl3NaNKY8wXQ1BC8aPKY8wXQlLCrGQAAgwheAAAMIngBADCI4AUAwCAOrgoT9X1aDTc9AICGQfCGieqn1dTHKTXc9AAAGgbBG0Y4rQYAQh9zvAAAGETwAgBgEMELAIBBYTHHe8MNN+jQoUM+l6Wmpuqdd97xaCsvL9eyZcv0/vvv68iRI2rbtq3+5V/+RY888oguu+wyn+t5//339fbbb2vv3r1q2bKl+vTpo4cfflg9evSo98+D4HHTBAChLiyCV5JiYmI0duxYr/aEhASP5w6HQxMnTlRubq569+6tm2++WcXFxcrOztann36q1atXq3379h6vWbx4sebPn6+EhASNGTNGZ86c0X/+539qzJgxWrFihfr27dugnw3+46YJAEJd2ATvpZdeqt///vd11uXk5Cg3N1fDhw/XCy+8IMv/bSG9++67mj17tubPn6+nn37aXV9UVKSXX35ZSUlJ+vd//3fFxMRIku666y6NHj1as2bN0rp16xQRwV77UMHR3QBCWbNLi+zsbEnSo48+6g5dSRozZow6deqk//iP/9DZs2fd7WvWrNH58+c1ceJEd+hKUvfu3TV8+HDt27dPO3bsMPcBAABNWtgEb3l5udasWaMlS5YoKytLX375pVfNuXPn9OWXX+qKK67w2gVtsVg0cOBA2e12ffXVV+72vLw8SdK1117rtb60tDSPGgAA6hI2u5pLSkr0+OOPe7QlJyfrpZdeUufOnSVJ+/fvl8PhUFJSks91uNqLiorUr18/92Or1aq4uDiv+i5dukiSiouLg+pzZWVlQHU1653Oqp/qj10//q67NrWtu6GfN8S6L/a7qK62sYB5jEXoYCyqREZG+lUXFsGbkZGhvn37ymazyWq1qqioSG+++abWrl2rcePG6f3331d0dLROnz4tSYqOjva5Hld7aelP84OlpaVeB1vVrHetN1AFBQVB11utVpWV2d19PXcuRufOnVVpaanKoiO0e/du2e12v9ZrsVjUps1Pl4Vs3bq1ztay7oZ+Xt/rDvS78FegY4eGw1iEjuY+Fv4eaBsWwTtlyhSP5927d9dzzz0nSVq7dq2ys7M1fvz4xujaBSUnJ/v1F1JlZaUKCgq86tsU7lJ0tEOSdMklrXXJJU5FR0erTRurbDZbQH1ZtuU7Hfhn1ZHAfbu0UOs2Vp/rbujn9b3uYL6LC6ltLGAeYxE6GIvAhEXw1uZ3v/ud1q5dq/z8fI0fP959cFT1LdrqXO3Vt4irbynXVl/9oKtAREZGBvRLWrPeYqn6qf7Y9RPoL//BE2X69njV5+nUvk2t627o5w2x7ob4hyDQsUPDYSxCB2Phn7A5uMqX2NhYSXLvZuzUqZMiIiJUVFTks97VXn0OOCkpSXa7XSUlJV71rrld11wvAAB1Cevg3blzp6SfLqLRunVr9erVS999953Xla6cTqc++eQTWa1W9ezZ092ekpIiSfrHP/7htf7c3FxJVVfHAgDAH00+ePft26eysjKf7S+88IIk6bbbbnO3jx49WpL00ksvyek69FXSX//6Vx04cEC33XabWrdu7W7PyMhQixYttHjxYo9dzoWFhVq3bp26devGlatCmK9LSC7etFd/zCnQH3MKtHjT3kboFYDmrMnP8a5fv15vvvmmUlJS9POf/1xt2rRRUVGRPv74Y1VUVOihhx5yb7VKUnp6utavX69169bp4MGDSklJ0f79+7VhwwYlJibqkUce8Vj/FVdcoSlTpmj+/Pm64447dPPNN7svGSlJzzzzDFetCmE1LyHZp3M7rmwFoFE1+eDt37+/9u3bp8LCQn322Wc6e/asYmNjNWjQIN11113ui1y4REREaPHixVq2bJnWrl2rFStWqF27dho5cqQeeeQRn6cOTZw4UQkJCXrrrbf07rvvqmXLlurXr5+mTp3KTRKagOpBmxjbpo5qAGhYTT54U1NTA55jbdWqlaZMmeJ1GtKF3H777br99tsD7R4AAB7YRwoAgEEELwAABhG8AAAY1OTneBGcmkf6AgDMIHibKY70BYDGwa5mAAAMInjDkK+rNQEAQgO7msNQzas1Jca20cTrr2zkXgEAJII3bHFZRP+wdwCAaQQvmjX2DgAwjeBFs8feAQAmcXAVAAAGEbwAABhE8AIAYBDBCwCAQQQvUA2nFwFoaBzV3AwQJv7j9CIADY3gbQZqhgl3I7owTi8C0JAI3maCuxEBQGhgjhcAAIMIXgAADCJ4AQAwiOAFAMAgghe4AF+nYlk4PQvAReCoZuACap6K1bl9G93cObaRewWgKSN4gTpUPxUroV0bvb+nTN9/vkud2nNxDQCBI3iBAO3/4YwOnjov9jgDCAZzvAAAGETwAgBgEMELAIBBBC8QJO76BCAYHFwFBIlbCAIIBsELXARuIQggUOxqBgDAIIIXAACDCF6gnnCwFQB/MMcL1BMOtgLgD4IXqEccbAWgLuxqBgDAIIIXAACDCF4AAAwieAEAMIjgBRoIpxcB8IWjmoEGwulFAHwheIEGxOlFAGpiVzMAAAYRvAAAGETwAgBgEMELAIBBBC8AAAYRvAAAGETwAgBgEMELAIBBXEADaERc2QpofgheoBFxZSug+WFXMwAABhG8gCHcrQiAxK5mwJiadyvq07ld43YIQKMgeAGDqs/pJsa28VjGFjHQPBC8QIjg/r1A80DwAiGEo5yB8MfBVQAAGETwAiGKOV8gPLGrGQhRzPkC4YngBUJY9TlfX1vABDPQ9BC8QBPh6zxgDsYCmh6CF2hCOA8YaPoIXiBMMCcMNA0ELxBG2PUMhD5OJwIAwCCCFwAAgwheIExxsBUQmpjjBcJU9YOtONAKCB0ELxDGXAdbsfULhA6CF2gGONUICB0EL9BMcKoREBo4uApohtj1DDQetniBZsjXdZ+PnT7HrmjAAIIXaKZqXve5rjsh1cScMRAcgjcAO3fu1KJFi/T555/r/PnzstlsGjdunIYNG9bYXQPqVV1bxDXvjMSua8B/BK+ftm7dqgkTJqhVq1b6zW9+o6ioKG3YsEHTpk3T0aNHdd999zV2F4F6daEt4pp3RmLXNeA/gtcP58+f16xZs2SxWLRy5Up1795dkjR58mSNHDlSL730koYOHaqEhIRG7inQeC5213UwrFZrg6wXaEgErx+2bt2q/fv3KyMjwx26khQTE6N//dd/1cyZM5WTk6MpU6Y0Yi+B0FVzi7hz+zZ6aPCFt4AvdNWtxZv26sA/y1RWZtdVx76rc12BYv4aDYng9UNeXp4kKS0tzWuZq2379u1+r8/pdEqSysvLFRkZWWd9ZWWlz/rOsa0Vqap1/SympSLkVKScHo9rLgv156HUF1/PO8S0lOO8VVGtzzd6X5ra91pyutz9PC66pdZ9cVDHS8slSd3iovRjWYXHc9frI+RUeXlVu0uEnGphcahlhBQX5bmuy6Nb6ear43Uxqvfd1/vDk8PhUJs2bVRRUeH+96o5ioyMVEREhCx17OGxOF0pgFo9/PDD+vvf/66//e1v6tmzp9fya665Rm3bttWmTZv8Wl95ebkKCgrquZcAgMbWu3fvOjeo2OL1Q2lp1TxVTEyMz+XR0dE6ffq03+tr0aKFkpOT/frLCADQdERE1H1dKoK3EURERKhVq1aN3Q0AQCPgkpF+iI6OlqRat2pLS0tr3RoGAKA6gtcPSUlJkqTi4mKvZSUlJbLb7erSpYvhXgEAmiKC1w8pKSmSpNzcXK9lrjZXDQAAF0Lw+mHAgAHq1KmT1q1bp8LCQnf76dOntWTJErVs2VIjRoxovA4CAJoMTifyU22XjDx06JAee+wxLhkJAPALwRuAnTt3auHChR43SRg/fjw3SQAA+I3gBQDAIOZ4AQAwiOAFAMAgrlwVwnbu3KlFixZ5zCmPGzeOOeWL8P333+uDDz7Qxx9/rG+//VbHjx9X27Zt1adPH02YMEG/+tWvvF5TWlqqRYsWacOGDSopKVF8fLyGDh2qKVOmKCoqyqve4XBo5cqVWr16tYqLi2W1WjVw4EBNmzZNnTp1MvExm7Rly5bpxRdflCStWrVKvXv39ljOeDSs//qv/9Jf/vIX7dq1S3a7XXFxcerdu7f+7d/+TR07dnTXMQ7BY443RHEUdcN44YUX9Nprr6lz585KTU1V+/btVVxcrI8++khOp1Mvvviixx82drtdd911lwoLC5WWlqbu3bursLBQubm5Sk5O1sqVK3XJJZd4vMcTTzyh7Oxs/eIXv9DgwYN17NgxffDBB4qKitKqVavcF2SBt927d+u3v/2tWrRoIbvd7hW8jEfDcTqd+tOf/qRVq1apc+fOSktLU1RUlI4dO6bt27fr+eefV79+/SQxDhfNiZBTUVHhHDJkiLNnz57OXbt2udtPnTrlvPnmm509evRwHjx4sBF72HT9/e9/d27bts2rffv27c4ePXo4U1JSnOfOnXO3L1iwwGmz2ZzPP/+8R/3zzz/vtNlsziVLlni0f/rpp06bzea8++67PdazadMmp81mc9533331/InCR3l5uTM9Pd05atQo5/Tp0502m835+eefe9QwHg1nxYoVTpvN5pw9e7bz/PnzXssrKircjxmHi0PwhqAtW7Y4bTabc+bMmV7L1qxZ47TZbM5FixY1Qs/C23333ee02WzOnTt3Op1Op9PhcDjT0tKcvXv3dp45c8aj9syZM87evXs7b7zxRo/2Rx991Gmz2Zx5eXle68/MzHTabDbnoUOHGu5DNGELFy509uzZ07lnzx7nY4895hW8jEfDKSsrc6akpDhvvPFGj4D1hXG4eBxcFYLy8vIkSWlpaV7LXG3bt2832qfmoEWLFh7/LSoq0rFjx9SnTx9ZrVaPWqvVqj59+ujAgQM6cuSIu33btm3uZTVdd911kn4aX/zk66+/1pIlSzRlyhRdeeWVPmsYj4aTm5urH3/8UUOGDJHD4dCGDRu0bNkyvfvuu17XqGccLh4HV4WgoqIiSfJ544W4uDhZrVafN2xA8A4fPqxPPvlEcXFxstlskn66KUZtc09JSUnKzc1VUVGROnbsKLvdrpKSEtlsNp83wnaNJ2Pnqby8XI899piuuuoqTZgwodY6xqPhfP3115Kqbll62223uf8NcrWNGzdOjz32mCTGoT6wxRuCSktLJanWWw1GR0fXeotCBK6iokIzZsxQeXm5pk+f7v7HwfUdu24LWZOr3TVe/tYzdp4WLFigoqIizZ071+c/zC6MR8P54YcfJEkrVqxQTEyMsrOzlZ+fr5UrVyopKUnLly/XX/7yF0mMQ30geNGsORwOzZw5U9u3b9fo0aO52YVhn3/+uZYvX66JEye69zTAPOf/ndzSsmVLvfLKK+rVq5eioqLUr18/LViwQBEREXrzzTcbuZfhg+ANQXX9BVhaWlrr1jD853A49Ic//EHr1q3T7bffrqeeespjues7dv3lXpOr3TVe/tYzdlXOnz+vmTNn6pe//KUefPDBOusZj4bj+s569uypn/3sZx7LbDabOnXqpP379+vUqVOMQz1gjjcEueZOiouL1bNnT49lJSUlstvt6tWrVyP0LHw4HA49/vjjeu+99zR8+HA9++yziojw/DvUNfdUfb6rOle7a7ysVqvi4uJ08OBBVVZWeu02dc1h+Zq7b47sdrv7O6z5e+7yu9/9TpL0yiuvqFu3bpIYj4bQtWtXSbWHn6v97Nmz/H9RD9jiDUEpKSmSqo40rMnV5qpB4KqH7rBhw/Tcc8/5nFtMSkpSfHy88vPzZbfbPZbZ7Xbl5+crMTHR42o+qamp7mU1bdmyRRJj59KqVSuNHDnS54/rH+0bbrhBI0eOVEJCAuPRgPr37y9J+vbbb72WVVRUaP/+/bJarWrfvj3jUB8a+3wmeKuoqHDeeOONF7yAxoEDBxqxh01XZWWl+xzRhx9+uM5zFrlQQOPwdR6v08l4NCTXeeyrV6/2aH/55ZedNpvNOX36dHcb43BxuGRkiOKSkQ1j0aJFevnll2W1WnXvvfe6z9mtbsiQIerevbukqr/g77zzTn3zzTdKS0vT1VdfrV27drkvjZeVlaXWrVt7vL7mpfFKSkq0fv16RUVF6a9//auuuOIKI5+1KZs5c6ZycnJ8XjKS8WgY+/fv15gxY/TDDz/o+uuvV9euXbVr1y5t3bpVCQkJWrVqleLi4iQxDheL4A1hO3fu1MKFCz1ukjB+/HhuknARXP+gX8jcuXOVkZHhfn769Gn3xeCPHz+uuLg43XLLLZo8ebLPUyQcDoeysrJ8Xgy+c+fO9f6ZwlFtwSsxHg3pyJEjWrhwobZs2aKTJ0/q8ssv1w033KDJkyfrsssu86hlHIJH8AIAYBAHVwEAYBDBCwCAQQQvAAAGEbwAABhE8AIAYBDBCwCAQQQvAAAGEbwAABhE8AIAYBDBCwCAQQQvAAAGEbwAABj0/wE5MafrOKI5AgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 500x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "_ = sns.displot([len(sent) for sent in train_data])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "556e8d8c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocab size: 8076\n"
     ]
    }
   ],
   "source": [
    "vocab = build_vocab_from_iterator(train_data, specials=[\"<unk>\"], min_freq=MIN_WORD_FREQUENCY)\n",
    "vocab.set_default_index(vocab[\"<unk>\"])\n",
    "\n",
    "VOCAB_SIZE = len(vocab)\n",
    "print(f'Vocab size: {VOCAB_SIZE}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "05cde39b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['seaside', 'holiday', 'j', '.', 'salmon', ',', '1918', ',', '1921']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[102]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3c39a6a5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 3041, 1415, 3, 4855, 2, 2079, 2, 3157]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab(train_data[102])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "00736c8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def collate_fn(batch):\n",
    "    \"\"\"\n",
    "    Collate_fn for Skip-Gram model to be used with Dataloader.\n",
    "    `batch` is expected to be list of text paragrahs.\n",
    "\n",
    "    Context is represented as N=SKIPGRAM_N_WORDS past words \n",
    "    and N=SKIPGRAM_N_WORDS future words.\n",
    "\n",
    "    Long paragraphs will be truncated to contain\n",
    "    no more that MAX_SEQUENCE_LENGTH tokens.\n",
    "\n",
    "    Each element in `batch_input` is a middle word.\n",
    "    Each element in `batch_output` is a context word.\n",
    "    \"\"\"\n",
    "    batch_input, batch_output = [], []\n",
    "    for text in batch:\n",
    "        text_tokens_ids = vocab(text)\n",
    "\n",
    "        if MAX_SEQUENCE_LENGTH:\n",
    "            text_tokens_ids = text_tokens_ids[:MAX_SEQUENCE_LENGTH]\n",
    "\n",
    "        for idx in range(len(text_tokens_ids) - SKIPGRAM_N_WORDS * 2):\n",
    "            token_id_sequence = text_tokens_ids[idx : (idx + SKIPGRAM_N_WORDS * 2 + 1)]\n",
    "            input_ = token_id_sequence.pop(SKIPGRAM_N_WORDS)\n",
    "            outputs = token_id_sequence\n",
    "\n",
    "            batch_input += [input_] * len(outputs)\n",
    "            batch_output += outputs\n",
    "\n",
    "    batch_input = torch.tensor(batch_input, dtype=torch.long)\n",
    "    batch_output = torch.tensor(batch_output, dtype=torch.long)\n",
    "    return batch_input, batch_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1d544377",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = DataLoader(train_data, batch_size=BATCH_SIZE, num_workers=4,\n",
    "                          pin_memory=True, collate_fn=collate_fn, shuffle=True)\n",
    "valid_loader = DataLoader(valid_data, batch_size=BATCH_SIZE, num_workers=4,\n",
    "                          pin_memory=True, collate_fn=collate_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f1ab2158",
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "\n",
    "def plot_losses(train_losses, valid_losses):\n",
    "    clear_output()\n",
    "    plt.figure(figsize=(6, 4))\n",
    "    plt.plot(range(1, len(train_losses) + 1), train_losses, label='train', color='deepskyblue', linewidth=2)\n",
    "    plt.plot(range(1, len(valid_losses) + 1), valid_losses, label='valid', color='springgreen', linewidth=2)\n",
    "    plt.ylabel('loss')\n",
    "    plt.xlabel('epoch')\n",
    "    plt.legend()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5a903840",
   "metadata": {},
   "outputs": [],
   "source": [
    "def training_epoch(model, optimizer, criterion, train_loader, tqdm_desc):\n",
    "    train_loss, num_batches = 0.0, 1\n",
    "    model.train()\n",
    "    for inputs, outputs in tqdm(train_loader, desc=tqdm_desc):\n",
    "        inputs = inputs.to(device)\n",
    "        outputs = outputs.to(device)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        logits = model(inputs)\n",
    "        loss = criterion(logits, outputs)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        train_loss += loss.item()\n",
    "        num_batches += 1\n",
    "\n",
    "    train_loss /= num_batches\n",
    "    return train_loss\n",
    "\n",
    "\n",
    "@torch.no_grad()\n",
    "def validation_epoch(model, criterion, valid_loader, tqdm_desc):\n",
    "    valid_loss, num_batches = 0.0, 0\n",
    "    model.eval()\n",
    "    for inputs, outputs in tqdm(valid_loader, desc=tqdm_desc):\n",
    "        inputs = inputs.to(device)\n",
    "        outputs = outputs.to(device)\n",
    "\n",
    "        logits = model(inputs)\n",
    "        loss = criterion(logits, outputs)\n",
    "\n",
    "        valid_loss += loss.item()\n",
    "        num_batches += 1\n",
    "\n",
    "    valid_loss /= num_batches\n",
    "    return valid_loss\n",
    "\n",
    "\n",
    "def train(model, optimizer, scheduler, criterion, train_loader, valid_loader, num_epochs):\n",
    "    train_losses, valid_losses = [], []\n",
    "\n",
    "    for epoch in range(1, num_epochs + 1):\n",
    "        train_loss = training_epoch(\n",
    "            model, optimizer, criterion, train_loader,\n",
    "            tqdm_desc=f'Training {epoch}/{num_epochs}'\n",
    "        )\n",
    "        valid_loss = validation_epoch(\n",
    "            model, criterion, valid_loader,\n",
    "            tqdm_desc=f'Validating {epoch}/{num_epochs}'\n",
    "        )\n",
    "\n",
    "        if scheduler is not None:\n",
    "            scheduler.step()\n",
    "\n",
    "        train_losses += [train_loss]\n",
    "        valid_losses += [valid_loss]\n",
    "        plot_losses(train_losses, valid_losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "accac7cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Word2VecModel(nn.Module):\n",
    "    def __init__(self, embed_dim, embed_max_norm, vocab_size):\n",
    "        super().__init__()\n",
    "        self.embeds = nn.Embedding(num_embeddings=vocab_size, embedding_dim=embed_dim,\n",
    "                                   max_norm=embed_max_norm)\n",
    "        self.decoder = nn.Linear(embed_dim, vocab_size)\n",
    "        \n",
    "    def forward(self, tokens):\n",
    "        embeds = self.embeds(tokens)\n",
    "        logits = self.decoder(embeds)\n",
    "        return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b7342561",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "de72dded",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Word2VecModel(embed_dim=EMBED_DIM, embed_max_norm=EMBED_MAX_NORM,\n",
    "                      vocab_size=VOCAB_SIZE).to(device)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\n",
    "scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, NUM_EPOCHS)\n",
    "criterion = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "b69073d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiYAAAGECAYAAAAC8us+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAA9hAAAPYQGoP6dpAABYz0lEQVR4nO3deXhU5eH28e9M9pAASQghhARIIMgSjKyCUYvSaqlIpYoUXLD607cVajcQC4hVW9faIlCVVqQoKESKWJcKKIqgQADZ9yUBApKQQAJkm8yc94/JhKyQfc4k9+e6zpWZc86ceYaHITfPeRaLYRgGIiIiIiZgdXcBRERERFwUTERERMQ0FExERETENBRMRERExDQUTERERMQ0FExERETENBRMRERExDQUTERERMQ0FExERETENBRMRERExDS83V2Ay1m1ahWLFy9mz5495OXlER4eTmJiIpMnTyYyMvKyr924cSP33Xdftcefe+45Ro8e3dBFFhERkXowZTAxDIOZM2eyZMkSYmJiGDFiBK1atSIjI4OUlBTS09OvGExcBg0axKBBgyrt79mzZ0MXW0REROrJlMFk4cKFLFmyhHHjxjF9+nS8vLzKHS8uLq7xtQYNGsSkSZMauogiIiLSCEzXx6SgoIC5c+cSHR3NtGnTKoUSAG9vU+YpERERqSfT/YZft24dOTk5jB49GofDwcqVK0lNTSU4OJihQ4fSuXPnWl0vNTWVBQsWUFhYSEREBEOGDCEiIqKRSi8iIiL1Ybpgsnv3bgCsVisjR44kNTW19JjVamXChAk8/vjjNb7eRx99xEcffVT63Nvbm3vuuYcpU6ZU2RpTHYfDQXFxMVarFYvFUuPXiYiItHSGYeBwOPD29sZqvfzNGtMFk6ysLAAWLFhAr169SE5OJi4ujr179zJjxgzmz59PdHQ048aNu+x1QkND+f3vf8+wYcOIiooiPz+f7777jr/+9a8sWLAAi8XC1KlTa1yu4uJidu7cWa/PJiIi0pIlJCTg6+t72XMshmEYTVSeGpkxYwZLly7F39+flStXlrvtcuDAAUaNGkWnTp1YtWpVna6fmZnJ7bffTm5uLmvXriUsLKxGrysuLmb79u306tWrVi0t4n52u509e/ao7jyQ6s5zqe48V2PUneuaV1999RX7iZquxSQoKAiAPn36VOoLEh8fT3R0NGlpaeTm5tK6detaXz88PJybb76Z5ORktm/fzk033VSj17lu3/j6+upL5mHsdjuguvNEqjvPpbrzXI1Rd65r1qQrhOlG5cTGxgIQHBxc5XHX/oKCgjq/R0hICAD5+fl1voaIiIg0PNMFk8GDBwNw5MiRSsdsNhvHjh0jMDCQ0NDQOr/H9u3bAYiKiqrzNURERKThmS6YxMTEkJSURFpaGsnJyeWOzZs3j9zcXIYPH156jyo7O5vDhw+TnZ1d7txdu3ZVef1///vfbNy4kS5dupCQkNA4H0JERETqxHR9TABmzpzJ2LFjmT59OqtXryY2NpY9e/awYcMGoqKimDJlSum5ixYtYs6cOUycOLHcDK+//vWv8fb2Lu2rkp+fz/bt29mzZw+tW7fmpZde0n1PERERkzFlMImJiWHZsmW8+uqrfP3116xfv5527doxfvx4Hn300RqNpBk7dizr1q0jJSWFc+fOYbVa6dixI/fffz+/+MUv6NChQxN8EhEREakNUwYTgMjISJ577rkrnjdp0qQq18J5+OGHefjhhxujaCIi0kBsNlvpiA0xB1d9FBQUXPbOgpeXFz4+Pg3+/qYNJiIi0nzl5uZy5swZCgsL3V0UqcAwDLy9vUlLS7vi8F4/Pz/atWtXp+k7qqNgIiIiTSo3N5f09HSCgoJo164dPj4+WurDRAzDID8/n4CAgGrrxTAMbDYbOTk5pKenAzRYOFEwERGRJnXmzBmCgoLo1KmTAokJuda18ff3v2z9BAQEEBwczIkTJzhz5kyDBRPTDRduSfZchN8chE257i6JiEjTsNlsFBYW0qZNG4WSZsBisdCmTRsKCwux2WwNck0FEzf6zSGYlQ737nV3SUREmoarY2VjdJoU93DVZUN1YlYwcaMCh/PngXw4X+zesoiINCW1ljQfDV2XCiZuFBdw6fGRui/9IyIi0mwomLhRtzLB5JDWExQREVEwcac4/0uPDyuYiIiIKJi4U9lbOQomIiLS0GbPnk2PHj3YuHGju4tSYwombqRbOSIiLc/GjRvp0aMHs2fPdndRTEkTrLlRiA+EeMPZYrWYiIhIwxs/fjwjRoygY8eO7i5KjSmYuFlcAGw+D8cLodABfmrDEhGRBhIaGkpoaKi7i1Er+jXoZq7bOQ4gVUOGRUSatdmzZ3PfffcBMGfOHHr06FG6nThxgqlTp9KjRw+OHz/O/PnzGTFiBH369GHq1KkAnD59mldffZUxY8YwZMgQ+vTpw0033cRTTz1FVlZWle9XsY/JiRMn6NGjB1OnTiUtLY1HH32UgQMHkpiYyIQJE9i3b1/T/GFUQy0mblZxZE6PQPeVRUREGtegQYO44447WL58OYMGDWLQoEGlx8quNfPMM8+wfft2brzxRoYNG0ZYWBgAmzdv5q233uLaa6+lb9+++Pj4sGfPHt59913WrVvH8uXLCQ4OrlFZ0tPTGTNmDN27d+dnP/sZx44d4/PPP+f+++/n/fffJzDQPb+QFEzcTCNzREQuSc6AJ1PNOxt2sDc80wXubF+31w8ePBigNJhMmjSpyvP279/P8uXLK/UNufbaa1m3bh2tWrUqt/+DDz7g8ccf55133uGXv/xljcqyadMmfv/73/Pwww+X7vv73//Oa6+9xocffsijjz5am4/WYBRM3Ewjc0RELnnpOOzLc3cpLqPIWca6BpOaevDBB6vssOpqOalo1KhRPPPMM3zzzTc1DiadOnXioYceKrfvzjvv5LXXXmP37t21L3QDUTBxM7WYiIhcMiUaZqSau8VkcnTjv0/fvn2rPbZy5UqWLFnC7t27yc3NLbd4XkZGRo3fo2fPnlit5buadujQAYDz58/XssQNR8HEzSJ9IcAK+Q61mIiI3Nm+8VsjPEF1LSPz58/nhRdeIDQ0lOuuu44OHTrg7+/srPjvf/8bm81W4/cICgqqtM/b2xkLHA5HHUrdMBRM3Mxicbaa7LoIRwvAboCXFt0UEWnRqlqxt7i4mH/84x+Eh4ezYsWKcuHFMAz+9a9/NWURG42GC5uAa2ROkQHphe4ti4iINC4vLy+AcrdgauLs2bOcP3+ea665plKLys6dOykoaB5zTiiYmECcOsCKiLQYbdq0AeD777+v1evCwsLw9/dn9+7d5Odf+mWRk5PDs88+26BldCfdyjGBbhU6wN4U4r6yiIhI44qNjaV9+/Z8/PHH+Pr6EhERgcVi4d57773s66xWK+PGjWP+/PmMGjWKYcOGceHCBdauXUtUVBTt2zePzjkKJiagkTkiIi2Hl5cXc+bM4eWXX+ajjz7i4sWLANx+++1XfO3vfvc72rRpw/Lly1m8eDHt2rXjtttuY+LEiYwcObKxi94kLIZhGO4uhCew2+1s27aNxMTE0vuDDeVwPnQrmS34Z+3g/T4NevkWrzHrThqX6s5zVVd3BQUFHD16lK5du5aOJhFzMQyDvLw8AgMDq+yEW1FN6rQ232X1MTGBzn7gXVL3h5tH3yUREZE6UTAxAW+rM5yAs/VEbVgiItJSKZiYhKufyXk7ZNZ8fhwREZFmRcHEJCqOzBEREWmJFExMQiNzRERETD5ceNWqVSxevJg9e/aQl5dHeHg4iYmJTJ48mcjIyFpdq6ioiLvuuot9+/bRtWtX/ve//zVSqetGk6yJiIiYNJgYhsHMmTNZsmQJMTExjBgxglatWpGRkUFKSgrp6em1DiZz587l2LFjjVTi+it3K0cjc0REpIUyZTBZuHAhS5YsYdy4cUyfPr3SmOfi4tqth71jxw7++c9/Mm3aNJ5++umGLGqDiS0z9Fu3ckREpKUyXR+TgoIC5s6dS3R0NNOmTatyIhbXssw1UVhYyOOPP07//v0ZN25cQxa1QQV4QUdf52PdyhERkZbKdC0m69atIycnh9GjR+NwOFi5ciWpqakEBwczdOhQOnfuXKvrvfLKK5w6dYo33nijRjPYuVO3ADhZ5BwufL4Ygk1XOyIiIo3LdL/6du/eDTgXKxo5ciSpqamlx6xWKxMmTODxxx+v0bVSUlJYuHAhU6dOJSYmpkHKV9tlqmsj1t/C2hxnI9aBi3YSgxrtrVoUV501Zt1J41Ddea7q6s5ut2MYRukm5uOql5rWj6su7XZ7td/V2nyHTRdMsrKyAFiwYAG9evUiOTmZuLg49u7dy4wZM5g/fz7R0dFXvC2Tl5fHE088QWJi4hVXbKyNnTt3Nti1Kgoo7ABEAbB6byr4nGu092qJGrPupHGp7jxXVXXn7e1Nfn4+DofDDSWSmsrPr1m/gsLCQmw2G/v27WuQ9zVdMHElNB8fH+bOnUtERAQAAwYMYNasWYwaNYq33nrrisHkhRdeICMjg3/+859YrQ3XlSYhIaHRFhO7PtPCayX16ujQlcRo/W+iIdjtdnbu3NmodSeNQ3Xnuaqru4KCAtLS0ggICNAifiZlGAb5+fkEBATUqAuE1WrFx8eHbt26XXYRv5r+B8N0wSQoyHn/ok+fPqWhxCU+Pp7o6GjS0tLIzc2ldevWVV5j48aNvPfee0yZMoWuXbs2aPm8vLwa7R/I7oGXHh8ttKJ/hxtWY9adNC7VneeqWHdeXl5YLJbSTRrWvffey6ZNm9i/f3/pvo0bN3LfffcxceJEJk2aVKPr/N///R9btmwpd53quOqyob6npgsmsbGxAAQHB1d53LW/oKCg2mCyd+9eAF588UVefPHFSsePHj1Kjx49CA4OZvPmzQ1R7AahSdZERKSlM10wGTx4MABHjhypdMxms3Hs2DECAwMJDQ2t9hrx8fHceeedVR57//33CQ4O5pZbbiEgIKDKc9wlxAdCvSG7WHOZiIhIw+jbty+ffPIJISEh7i5KjZgumMTExJCUlMS6detITk7mrrvuKj02b948cnNzuf3220vnMsnOzubs2bOEhISUhpWhQ4cydOjQKq///vvv065dO/785z83/oepg7gAyD4Pxwuh0AF+pptpRkREPElAQABxcXHuLkaNmfLX3syZMwkLC2P69Ok88sgjvPDCC9x///28+uqrREVFMWXKlNJzFy1axIgRI1i0aJEbS9xwXLdzDOCoWk1ERJqVzZs306NHD5544okqj2dlZdG7d2/Gjh0LwK5du3j66ae57bbb6N+/P3379mXkyJHMmzcPm81Wo/fcuHEjPXr0YPbs2VWW55577iExMZHBgwfzm9/8hlOnTtX9AzYAUwaTmJgYli1bxujRo9m9ezdvv/02aWlpjB8/nuTkZMLDw91dxEajNXNERJqv/v37ExUVxcqVKyksLKx0/KOPPqK4uJhRo0YBsHTpUlatWkV8fDx33303d955J4Zh8Ne//pXf/e539SrLt99+y4QJE9i+fTu33HILY8aM4cSJE4wbN47c3Nx6Xbs+THcrxyUyMpLnnnvuiudNmjSpxr2MgRr1MHanOK2ZIyItWDK7eZI1nKfyL20zCMaPZxjGnfSu0+stFgu33347r732Gp9//jkjRowod3zFihX4+Pjw4x//GID/9//+HzNnziw32sUwDKZNm8ayZcvYsmUL/fv3r3U5HA4HM2bMoLi4mHfeeYcBAwaUXvsPf/gDH330UZ0+X0MwbTBpqTQyR0RaspdYzz7OuLsYl3Gel/imzsEEYNSoUbz22mt8+OGH5YLJ4cOH2b17N8OHD6dt27YAdOzYsdLrLRYL48ePZ9myZXz77bd1CiZbtmzh+PHjDBs2rDSUuK7929/+lk8//dRtMy4rmJhMuVs5CiYi0sJM4TpmmLzFZDJVD66oqa5du9K3b1/WrVtHdnZ26cCNDz/8EKD0Ng5AUVERixYt4uOPP+bIkSPk5eWVmyo+IyOjTmVwzdJaNpS4REVFERERwcmTJ+t07fpSMDGZDr4QaIU8h4KJiLQ8d9K7Xq0RnmLUqFHs2LGDTz/9lPHjx2MYBv/9739p06YNP/jBD0rP+/Wvf82aNWvo0qULI0aMICwsDG9vb3Jzc1m4cCFFRUV1ev/z588DEBYWVuXx0NBQBRNxslggNgB2XYQjBWA3wEuTI4qINCsjRozg+eef58MPP2T8+PGkpKSQnp7O3Xffja+vLwA7duxgzZo1JCUlMW/evHL9TLZt28bChQvr/P6uyUpd69NVlJ2dXedr15cpR+W0dK7bOTYDTpizNVNEROohNDSUpKQktm3bRlpaWultnNtvv730nOPHjwPwgx/8oNJU7/Wdtfyqq66q9jrp6emcPn26XtevDwUTE9LIHBGR5s/VlyQ5OZn//e9/dOrUqVxHVlfH1y1btpR73cGDB5k3b1693rt///506tSJL7/8slw4MQyDv/3tb27r+Aq6lWNKFUfm3OQZswiLiEgt3HTTTQQHB7NgwQJsNhv33ntvuYUN+/btS9++ffn000/JzMzk6quv5tSpU3zxxRfceOONfPbZZ3V+b6vVyjPPPMPDDz/MAw88wIgRI2jfvj0bNmwgMzOT7t27c/DgwYb4mLUvm1veVS5LI3NERJo/Pz8/br311tIZXMvexgHnSsxvvPEGP/vZzzh27BjvvPMOhw4dYsqUKUyePLne7z906FAWLFjA1Vdfzf/+9z+WLl1Kx44dWbRoUbWL5DYFi1F23JFUy263s23bNhITExt9+fUj+RC30fn4Z+3g/T6N+nbNXlPWnTQs1Z3nqq7uCgoKOHr0KF27dsXf3/8yVxB3MQyDvLw8AgMDy7XgVKcmdVqb77JaTEwoxg+8S/4uaJI1ERFpSRRMTMjbCl1KQufhAlCbloiItBQKJiblGplzwQ4ZNVtAUkRExOMpmJhUnDrAiohIC6RgYlIamSMiIi2RgolJaZVhERFpiRRMTEq3ckSkOdNMFc1HQ9elgolJxZadlr7AfeUQEWlIrjksXJOKiedz1WVDzTWkYGJSAV4Q5VxgUrdyRKTZ8PHxwc/Pj5ycHLWaNAOGYZCTk4Ofnx8+Pj4Nck2tlWNicQGQXgRnbJBbDK1VWyLSDLRr14709HROnDhBmzZt8PHxqdEMo9I0DMOgsLAQq9Vabb0YhoHNZiMnJ4cLFy4QFRXVYO+vX3Um1i0A1uY4Hx/Oh2uC3VseEZGG4FqH5cyZM6Snp7u5NFKRK3TUJDD6+fkRFRXVoGvrKJiYWMWROQomItJctG7dmtatW2Oz2bDb7e4ujpRht9vZt28f3bp1u2y/ES8vrwa7fVOWgomJaWSOiDR3Pj4+jfLLTerOFRT9/f3dsnimOr+aWLlJ1jQyR0REWgAFExOLKzNkWCNzRESkJVAwMbG2PhBacrNNt3JERKQlUDAxOdftnBOFUKD+YSIi0swpmJicqwOsARxVPxMREWnmFExMTiNzRESkJVEwMTmNzBERkZZEwcTkNDJHRERaElNPsLZq1SoWL17Mnj17yMvLIzw8nMTERCZPnkxkZORlX7t9+3YWL17M7t27ycjIID8/nw4dOtC7d28efPBBEhISmuhT1E833coREZEWxJTBxDAMZs6cyZIlS4iJiWHEiBG0atWKjIwMUlJSSE9Pv2Iw2bx5M+vXrycxMZFrr70Wf39/jh8/zhdffMH//vc/nn/+eX760582zQeqhwhfaGWFiw4FExERaf5MGUwWLlzIkiVLGDduHNOnT680JW5xcfEVr3HPPffw4IMPVtp/4MAB7rzzTl588UVGjRpl+hUtLRaIDYCdF52jcuwGeJm7yCIiInVmuj4mBQUFzJ07l+joaKZNm1blPP3e3lfOU35+flXuj4+PJy4ujqysLC5cuFDv8jYF1+0cmwHH1QFWRESaMdO1mKxbt46cnBxGjx6Nw+Fg5cqVpKamEhwczNChQ+ncuXO9rn/s2DGOHj1KZGQkwcGesVxvXIWROV0Cqj9XRETEk5kumOzevRsAq9XKyJEjSU1NLT1mtVqZMGECjz/+eI2vt2PHDr766iuKi4tJT0/niy++AOBPf/pTncrnjuW5u/pZcDVuHbjo4AetjSYvgydz1ZmWVvc8qjvPpbrzXI1Rd7W5lumCSVZWFgALFiygV69eJCcnExcXx969e5kxYwbz588nOjqacePG1eh6O3bsYM6cOaXP27VrxwsvvEBSUlKdyrdz5846va5eioOBeAC+Tctg0On0pi9DM+CWupMGobrzXKo7z+WuurMYhmGq/37PmDGDpUuX4u/vz8qVK4mIiCg9duDAAUaNGkWnTp1YtWpVra5bUFBAamoqCxYsYMWKFfzhD3+osnNsdex2O9u2bSMhIaHKfi+N6WgBdE9xvucdYQbJvRxN+v6ezm63s3PnTrfUndSP6s5zqe48V2PUneuaiYmJV7ym6VpMgoKCAOjTp0+5UALOjqvR0dGkpaWRm5tL69ata3xdf39/rrrqKp5//nmys7N5+eWXuf7664mPj69V+by8vJr8S9YlALwtUGzA4QKLvuR15I66k4ahuvNcqjvP5a66M92onNjYWIBqO6a69hcU1H14ynXXXYfD4WDz5s11vkZT8rZC15IZYA/ng7nauERERBqO6YLJ4MGDAThy5EilYzabjWPHjhEYGEhoaGid3yMjIwMAHx+fOl+jqblG5lx0QIbNvWURERFpLKYLJjExMSQlJZGWlkZycnK5Y/PmzSM3N5fhw4eXzmWSnZ3N4cOHyc7OLndudZ129u7dy3vvvYePjw9DhgxpnA/RCLRmjoiItASm62MCMHPmTMaOHcv06dNZvXo1sbGx7Nmzhw0bNhAVFcWUKVNKz120aBFz5sxh4sSJTJo0qXT/Y489hpeXF7179yYyMhKbzcbRo0f55ptvMAyDadOm0alTJ3d8vDqpuGbOdW3cVxYREZHGYspgEhMTw7Jly3j11Vf5+uuvWb9+Pe3atWP8+PE8+uijhIWFXfEajzzyCGvWrGH79u2sWbMGh8NBeHg4t912G/fccw9XX311E3yShhOnxfxERKQFMGUwAYiMjOS555674nmTJk0q11Licvfdd3P33Xc3RtHcomww0a0cERFprkzXx0SqFusPrrX71GIiIiLNlYKJh/D3gqiSdQkPayE/ERFpphRMPIhrZM4ZG+QUu7csIiIijUHBxINUHJkjIiLS3CiYeBB1gBURkeZOwcSDaMiwiIg0dwomHkS3ckREpLlTMPEgupUjIiLNnYKJB2njDWElU+JpyLCIiDRHCiYexnU750Qh5NvdWxYREZGGpmDiYcrezjmqVhMREWlmFEw8jEbmiIhIc6Zg4mE0MkdERJozBRMPo5E5IiLSnCmYeBjXejmgkTkiItL8KJh4mAhfaFVSa7qVIyIizY2CiYexWC7dzjlaAMUO95ZHRESkISmYeCBXMCk24Hihe8siIiLSkBRMPJBG5oiISHOlYOKBNDJHRESaKwUTD6SROSIi0lwpmHgg3coREZHmSsHEA0X7g4/F+Vi3ckREpDlRMPFAXhboUnI750g+GIZ7yyMiItJQFEw8lOt2zkUHnC5yb1lEREQaioKJh9LIHBERaY4UTDyURuaIiEhzpGDioTQyR0REmiMFEw+lWzkiItIcKZh4qK7+UDJiWC0mIiLSbHi7uwCXs2rVKhYvXsyePXvIy8sjPDycxMREJk+eTGRk5GVfu3nzZlavXs2mTZtIT08nLy+PqKgobr75Zh555BFat27dRJ+icfh7QSc/5yJ+CiYiItJcmDKYGIbBzJkzWbJkCTExMYwYMYJWrVqRkZFBSkoK6enpVwwmjz32GGfPnqV///6MGjUKi8XCpk2b+Ne//sVnn33Ge++9R7t27ZroEzWOuABnMMkqhnM2aOvj7hKJiIjUjymDycKFC1myZAnjxo1j+vTpeHl5lTteXFx8xWvcf//9jBo1ioiIiNJ9hmHwpz/9iXfffZe5c+cyc+bMBi97U4rzhy9LHh8ugP4KJiIi4uFM18ekoKCAuXPnEh0dzbRp0yqFEgBv7yvnqYcffrhcKAGwWCz86le/AiAlJaVhCuxGGpkjIiLNjelaTNatW0dOTg6jR4/G4XCwcuVKUlNTCQ4OZujQoXTu3Lle13eFmqoCj6fRyBwREWluTBdMdu/eDYDVamXkyJGkpqaWHrNarUyYMIHHH3+8ztdftmwZANddd12dXm+32+v83g2tix+AM2AdynNgt2vRnKq46sxMdSc1o7rzXKo7z9UYdVeba5kumGRlZQGwYMECevXqRXJyMnFxcezdu5cZM2Ywf/58oqOjGTduXK2vvXfvXubOnUtYWBgPPfRQncq3c+fOOr2uMeQZVuAaALafuci2vAPuLZDJmanupHZUd55Ldee53FV3pgsmRslSuT4+PsydO7e0n8iAAQOYNWsWo0aN4q233qp1MDl+/DgPP/wwdrudV155hdDQ0DqVLyEhwVS3gdp9a3Cm2MJpryASExPdXRxTstvt7Ny503R1J1emuvNcqjvP1Rh157pmTZgumAQFBQHQp0+fSp1X4+PjiY6OJi0tjdzc3BrPRXL8+HHuu+8+zp49y+zZs7n22mvrXD4vLy9TfcniAuDMeUgvslCEFwHmKZrpmK3upOZUd55Ldee53FV3phuVExsbC0BwcHCVx137CwpqtnKdK5RkZmby97//nWHDhjVMQU2i7Mico1rMT0REPFy9gsmpU6f49ttvyc+/NCTE4XAwb948xo4dy4QJE/jyyy9rdc3BgwcDcOTIkUrHbDYbx44dIzAwsEa3YsqGkr/97W8MHz68VmXxBBqZIyIizUm9gsmsWbP4zW9+U25ekddee41XXnmFbdu2sWHDBh599FF27NhR42vGxMSQlJREWloaycnJ5Y7NmzeP3Nxchg8fXvqe2dnZHD58mOzs7HLnukJJRkYGr7zyCj/84Q/r8UnNK05zmYiISDNSrz4mW7duZciQIfj4OKccNQyDRYsWERsby/z588nMzOSBBx7gzTffZNasWTW+7syZMxk7dizTp09n9erVxMbGsmfPHjZs2EBUVBRTpkwpPXfRokXMmTOHiRMnMmnSpNL9999/PydPniQxMZH9+/ezf//+Su9T9nxP1U0tJiIi0ozUK5hkZWXRsWPH0ud79+4lOzubiRMn0qFDBzp06MDw4cPZtGlTra4bExPDsmXLePXVV/n6669Zv3497dq1Y/z48Tz66KOEhYVd8Rrp6ekAbNu2jW3btlV5TnMIJmoxERGR5qRewcThcJQO7wXYtGkTFoul3KiXiIgIzpw5U+trR0ZG8txzz13xvEmTJlUZMKpqIWmO2vtAKytcdDjXyxEREfFk9epj0rFjx3L9R1avXk14eHjpyBqAzMzMGg/rldqzWC7dzkktgGKHe8sjIiJSH/VqMfnRj37E66+/zq9//Wt8fX3ZsmUL48ePL3fO4cOH6dSpU70KKZcXFwDbL0KxAccKITbgyq8RERExo3q1mDz44IMkJCSwcuVKPvroI+Lj48vdVklPT2fHjh2lQ4ClcaifiYiINBf1ajEJCgpi6dKlHDjgXKMlLi6u0ixxs2fPJiEhoT5vI1dQcWRO8xwYLSIiLUGDTEkfHx9f5f6oqCiioqIa4i3kMtRiIiIizUW9gsmFCxc4e/YsHTp0KJ3LBOCTTz7h888/x9/fn/Hjx9OrV696F1SqF+d/6bFG5oiIiCerVzB56aWX+PDDD/nmm29Kg8nixYt55plnSocRf/zxxyxbtoy4uLj6l1aqFO0PPhawGZpkTUREPFu9Or+mpKQwdOhQAgIu3Uv45z//SUREBO+88w5///vfMQyDN998s94Flep5WaBrSavJkXwoM7WMiIiIR6lXMMnMzCw3FPjw4cOcOnWKe++9lwEDBnDrrbdy0003sXnz5noXVC7P1c8kzwHfF7m3LCIiInVVr2BSVFRUrm+Ja+bX6667rnRfdHQ0p0+frs/bSA1ozRwREWkO6hVMOnToUG7q9y+//JI2bdpw1VVXle47d+4cgYGB9XkbqQGNzBERkeagXp1fr7/+ehYvXswLL7yAr68vX3/9NaNGjSp3ztGjR4mMjKxXIeXKNDJHRESag3oFk0ceeYQ1a9bw1ltvARAeHs5jjz1WejwrK4vvvvuu0jT10vB0K0dERJqDegWT8PBwPv74Y7799lsABg4cSFBQUOnxs2fPMnnyZJKSkupXSrmirgFgAQx0K0dERDxXvWd+9ff3Z9iwYVUe69atG926davvW0gN+Fmhkx8cL1QwERERz9UgU9IDnD59mr1793LhwgWCgoLo2bMnERERDXV5qYFuAc5gkl0MZ20Q4nPl14iIiJhJvYNJWloaTz31FBs2bKh0bMiQIcycOZPOnTvX922kBuICYM055+PD+TBAwURERDxMvYLJqVOnGDduHFlZWcTGxjJgwADat29PZmYmmzdv5ptvvmH8+PEkJydrZE4TqDgyZ0Br95VFRESkLuoVTObMmUNWVhYzZ85k7NixWCyWcsffe+89nnrqKebOncuzzz5br4LKlWlkjoiIeLp6BZN169YxbNgwfv7zn1d5fOzYsXz11VesXbu2Pm8jNaRJ1kRExNPVa+bXrKws4uPjL3tOfHw82dnZ9XkbqSEFExER8XT1CiahoaEcOnTosuccOnSI0NDQ+ryN1FBrbwgv6fCqWzkiIuKJ6hVMkpKS+OKLL0hOTq7y+Pvvv8+aNWu4/vrr6/M2UguuVpOTRZBvd29ZREREaqtefUwmTpzImjVrePLJJ1m4cCEDBw4kLCyMrKwsUlJSOHToECEhIUycOLGhyitX0C0ANuQ6Hx8pgN6t3FseERGR2qhXMOnYsSPvvvsuTz75JJs2beLgwYPljg8ePJinnnpKQ4WbUNkhw4fyFUxERMSz1HuCtS5durBw4UJOnTpVaeZXBZKmpw6wIiLiyRpsSvrIyEgFERPopmAiIiIerFbB5IknnqjTm1gsFv7yl7/U6bVSO3GaZE1ERDxYrYLJ8uXL6/QmCiZNJ9wHgrzggl0tJiIi4nlqFUw+//zzxiqHNBCLxXk7Z9sFSCsEmwN86jUoXEREpOnUKphERUU1VjmkAcX5O4NJsQGPH4EXYhVORETEMzRY59fGsGrVKhYvXsyePXvIy8sjPDycxMREJk+efMWOtllZWbz//vvs3r2bXbt2kZ6eDsD+/fubouhudWc4LDvjfPy3E7D5PCzpBZF+7i2XiIjIlZgymBiGwcyZM1myZAkxMTGMGDGCVq1akZGRQUpKCunp6VcMJocOHeKVV17BYrHQuXNnAgICyM9vGZ0u7m4P2cXwm0NgM+DrHLhmMyztDTe0dXfpREREqmfKYLJw4UKWLFnCuHHjmD59Ol5eXuWOFxcXX/EacXFxvPPOO/Ts2ZOgoCBuvfVWjh492lhFNhWLBX4VBf2C4K49cKIQTtvgpm3wfCz8Ptp5joiIiNmYrudBQUEBc+fOJTo6mmnTplUKJQDe3lfOU+3atWPgwIEEBQU1RjE9wrVtYGt/GB7ifG4HJh+BO3dD7pWznYiISJMzXYvJunXryMnJYfTo0TgcDlauXElqairBwcEMHTqUzp07u7V8drtnrYwX6gUf94Y/pVn4y3FnDv3PGdi5xSC5p4M+LWDKeledeVrdierOk6nuPFdj1F1trmW6YLJ7924ArFYrI0eOJDU1tfSY1WplwoQJPP74424qHezcudNt710fo4H2Aa2Zkd+V83hzMN/CtVthmv8xfuyT7e7iNQlPrTtR3Xky1Z3nclfdmS6YZGVlAbBgwQJ69epFcnIycXFx7N27lxkzZjB//nyio6MZN26cW8qXkJBQ5e0lT5AIjMiHu/cafHfRQgFezCjoysmQzrwca+Bnuht7DcNut7Nz506PrruWSnXnuVR3nqsx6s51zZowXTAxDAMAHx8f5s6dS0REBAADBgxg1qxZjBo1irfeesttwcTLy8ujv2Tdg2B9P5h0EN783rnvtVNWtl6A5N4Q7X/513syT6+7lkx157lUd57LXXVnuv8juzqr9unTpzSUuMTHxxMdHc2xY8fIzc11R/GahQAv+NdV8K8e4FcyOmfjeeeQ4lUt466OiIiYlOmCSWxsLADBwcFVHnftLygoaLIyNVcPRsI3/aBrSStJVjHcsgOeTQWH4daiiYhIC2W6YDJ48GAAjhw5UumYzWbj2LFjBAYGEhoa2tRFa5b6BcOW/vCTkj9OA5iRCrfvhLM2d5ZMRERaItMFk5iYGJKSkkhLSyM5ObncsXnz5pGbm8vw4cNL5zLJzs7m8OHDZGfrHkRdhfjAhwnwbFdwzbv2cTb02wJbz7u1aCIi0sKYrvMrwMyZMxk7dizTp09n9erVxMbGsmfPHjZs2EBUVBRTpkwpPXfRokXMmTOHiRMnMmnSpHLXmTp1aunjzMzMSvv+7//+j7i4uEb+NJ7BaoFpnWFQMIzbC2dskFoAQ7fC3HjnbR8REZHGZspgEhMTw7Jly3j11Vf5+uuvWb9+Pe3atWP8+PE8+uijhIWF1eg6y5cvv+y+O+64Q8Gkgh+GOmeLvXM3bDoPhQY8tB++yYE53Z0dZ0VERBqLKYMJQGRkJM8999wVz5s0aVKllhKXlrCScGOI9oe118DvD8Hck85987+HrRdgWW+IDXBv+UREpPkyXR8TMQc/K8yJh3d6QmDJ35JtF6D/FvjvGfeWTUREmi8FE7ms8RGwsR/El7SSnCuG23fBnbvgcL57yyYiIs2PgolcUZ8gSOkPo9td2rfsDPTcBL87BNkaViwiIg1EwURqpLU3vN8b/hkP7X2c+2wG/O0EdNsIfzsORQ73llFERDyfgonUmMUCD3WEQ4NhWgz4l/ztOVsMvzsMvTbBskwwNGusiIjUkYKJ1FqwNzwbCwcHwX0RlyZlO1zgHGZ8/XewUUsZiYhIHSiYSJ118od/94TN/WFY20v71+fCtVvh53vgqDrIiohILSiYSL31C4bPr4YP+0CPMnOcvJcBV22CKYfhnDrIiohIDSiYSIOwWGBkO9g5EOZ2h3YlHWSLDHjpuLOD7OwTYFMHWRERuQwFE2lQPlb4VZSzg+zUGPAr6YCSVQy/PgR9UmDFGXWQFRGRqimYSKNo4w3PxcL+wTC+/aX9B/Lhp7tg2DbYrA6yIiJSgYKJNKrO/vBOL9jUD65vc2n/VzkwcCvcuxeOF7ivfCIiYi4KJtIkBraGrxJheW/oXqaD7DunIX4T/PEI5Ba7rXgiImISCibSZCwW+Gk47BoIs7pBaMna1gUOeO6Ys4Ps82lwViN4RERaLAUTaXK+Vvh1Jzg8GP4QDb4lHWQzbfDEUYj+Fh47qDlQRERaIgUTcZu2PvBSHOwbBGPbX5pB9qIDXk13tqDctVuzyIqItCQKJuJ2XQPg3V5wcDBMjILAkr+VDuD9TOcssklbYXkm2DXMWESkWVMwEdOIC4DZ3eH4EPhzV4jwuXRsfS6M3u2cSfa1dMizu6+cIiLSeBRMxHRCfeCPnSFtCMzvAb0DLx07lA+/Oggx38KMo3C6yH3lFBGRhqdgIqblZ4UHIp3T3H+aADe3vXQsqxieTYPO38JD+2DPRbcVU0REGpCCiZiexQK3hsHqRPiuP9wTAd4lPWULDXjze+idAj/ZAWvOarp7ERFPpmAiHiUxGN7uCUcHw5RoaON16dgn2XDTdui/BRaf1oKBIiKeSMFEPFInf3ghztlR9m9xEON36dh3F2D8XojdCC8fgxzNKCsi4jEUTMSjBXvDb6Kdk7W91wsGBF86dqIQJh+BLpus/Lkghs/ParixiIjZKZhIs+BthbvbOxcL/CoRbg+7dOy83cJyWzi37PKi4zfw6AFYew4cCikiIqajYCLNisUCN7SFFQnOGWUfjoRA66UEkmGDf5yEG7c5p77/zUH4NkcdZkVEzELBRJqtHoHwRg84da2D5/yPcEeYgZ/l0vGTRTArHYZ+B102wOTDsDlXIUVExJ0UTKTZa+UFP/Q5S3IvB5nXwTs9YWQY+JQJKccK4eXjMHArdN8IfzwC2y8opIiINDUFE2lRgr1hfAR8mACnhzpnlr0lBMqMOuZwATx3DBI3Q68UeOoo7NUEbiIiTULBRFqsEB/nzLL/uxq+HwpvxMNNbct/KfblwZ/SnAGlbwr8OQ0O5bmrxCIizZ+3uwtwOatWrWLx4sXs2bOHvLw8wsPDSUxMZPLkyURGRl7x9Q6Hg0WLFrF06VLS0tIIDAxk6NCh/Pa3vyU6OroJPoF4ina+8HBH53a6yLmq8ZIMWJcDrrs5Oy/CzqMw/Sj0C3KOAhrdDroFXvbSIiJSC6YMJoZhMHPmTJYsWUJMTAwjRoygVatWZGRkkJKSQnp6eo2CyZNPPklycjLdu3fn3nvvJSMjg08//ZT169ezZMkSunTp0vgfRjxOhC88GuXc0gshOQOWZMKG3EvnbL3g3B4/ArH+cEuoc7uprfN2kYiI1I0p/wlduHAhS5YsYdy4cUyfPh0vL69yx4uLrzyV54YNG0hOTmbgwIHMnz8fX19fAG677TYefvhhnnnmGd58881GKb80H1F+zgncfhMNaQWwNMPZkrLlwqVzjhTAayedm7cFhra+FFSuCQKrpfrri4hIeabrY1JQUMDcuXOJjo5m2rRplUIJgLf3lfNUcnIyAI899lhpKAG48cYbGTRoEOvWrePkyZMNV3Bp9jr7w+QY2DwADg6C52PhB23Lj+4pNmBtDkw7CgO2QMQ3MG4P/Pt7OFXotqKLiHgM0wWTdevWkZOTw/Dhw3E4HKxcuZJ58+bx7rvvkpaWVuPrbNy4kcDAQPr161fp2PXXXw/Apk2bGqzc0rJ0C4THY2BNImRfBx/2gUc7QreA8uedscG7GTBhH3T8Fq5OgSmH4fOzUKhFBkVEKjHdrZzdu3cDYLVaGTlyJKmpqaXHrFYrEyZM4PHHH7/sNfLy8sjMzCQ+Pr7KFpfOnTsD1CrouNjt9lq/RtzLVWeNVXcBFhgR4twAjuTDqnMWVp618MU555T4LjsuOreXjjtnpL2xDfwoxOBHIQbxAc6Za+WSxq47aTyqO8/VGHVXm2uZLphkZWUBsGDBAnr16kVycjJxcXHs3buXGTNmMH/+fKKjoxk3bly11zh//jwAQUFBVR537XedVxs7d+6s9WvEHJqy7gaXbE8EwE57EN/aW7OhuDV7HYEYONNHnsPCp2fh07PO55GWQq71zmWIVy6DvHMJsqhJxUXfO8+luvNc7qo70wUTo2SqTR8fH+bOnUtERAQAAwYMYNasWYwaNYq33nrrssGkMSUkJFTZCiPmZbfb2blzp9vqbgDwQMnjMzYHq89aWHnW2apyquhSE8kpw4/ltnCW28KxYjAwGG5uazA8xODaYPA13Y3XxufuupO6U915rsaoO9c1a8J0wcTVmtGnT5/SUOISHx9PdHQ0aWlp5Obm0rp16yqvERwcDMCFCxeqPO7a7zqvNry8vPQl81BmqLsILxgf6dwMwzk3ymfZzu3rHCgqmTTFgYWN52HjeQt/OQ6BVrixLQwPcW4JrVrWbR8z1J3UjerOc7mr7kwXTGJjY4HqQ4Nrf0FBQbXBJDAwkPDwcE6cOIHdbq/0B+vqW+LqayLiDhYL9A1ybpNjIM8OX51zhpTVZ2F3mRlm8xzwabZzA2jvcymkDA+BaH+3fAQRkQZnusbhwYMHA3DkyJFKx2w2G8eOHSMwMJDQ0NDLXmfQoEHk5eWxdevWSse+/vprAAYOHNgAJa67Amz8l/1koTnOBQK94Mdh8PfusGsQpA+BhVfBfREQ6Vv+3AwbLM6AX+yHmA1w1UaYeAA+yIRzNveUX0SkIZgumMTExJCUlERaWlrpXCQu8+bNIzc3l+HDh5fOZZKdnc3hw4fJzs4ud+6YMWMAmDVrFkVFRaX7v/rqKzZt2kRSUhJRUVGN/Gku70E+5HbepSdzWMNRt5ZFzKejH9zbAf7d0xlSdg+EWd2cKyMHVWhd3Z8Pc0/CHbshbD0M2QozjjpbYIrUh1ZEPIjpbuUAzJw5k7FjxzJ9+nRWr15NbGwse/bsYcOGDURFRTFlypTScxctWsScOXOYOHEikyZNKt1/7bXXctddd5GcnMzo0aO58cYbyczM5JNPPqFt27ZMnz7dHR+tHGvJ6IxM8hjOQl5gOL9nKBZaUOcBqRGLBXq1cm6/7gQ2B2w677zls/qsc7r84tL+Kc7nG3Lh2bTy/VNuauu8RkvsSCsinsGUwSQmJoZly5bx6quv8vXXX7N+/XratWvH+PHjefTRRwkLC6vRdZ5++mni4+NZunQpCxcuJDAwkB/+8If89re/JSYmppE/xZXN4lYyuchnHMaBwWRWsYl03mQUwfi5u3hiYj5WuK6Nc5vZBc4XO1tHXEHlcv1TvC3QIwD6tCq/dQ0AL2ViEXEzi+EanyuXZbfb2bZtG4mJiQ3aS9mOg6f4kmdZW7qvF+H8h7vpQbsGe5+WrLHqzsxOFjpnl119FladhVNFV35NgBV6BZYPKwlB0NHXfSOAWmLdNReqO8/VGHVXm2uassWkJfHCyjPcxEA6ci/LyaWQPWQykHks5A5+Sk93F1E8kKt/yr0dnMOS9+Y5A8rGXNh1Efblga3Cf0nyHc7FCbdUGGXf1rty60qfVhDm03SfR0RaDgUTk7idq0jh/xjNEnaTyXmKuIMl/JHreZpheJmvn7J4iLL9U1xsDjiY7wwpZbdD+VCxCfVcMazLcW5ldfCtHFZ6B0KQ/lURkXrQPyEmEk87NvAQD/IhS3GuGfQXvmYzJ1nMzwgj0M0llObCx3oprIwpsz/f7mxdqRhYjlexMvL3Rc5t9dny+2P9S24DldwKSmgF3QOc7ykiciUKJiYThB/vcSeDiWIKq7BjsJLDDGAeyxhDPzq6u4jSjAV4Qb9g51ZWTjHsrhBWdl50rp5c0ZEC5/Zh1qV9vha4KvBSWHEFl2i/ljWDrYhcmYKJCVmw8DuG0o9IxpBMJnmkco7rmM/r3Mb9JLq7iNLCtPGGoW2cW1kZRZdCyq6LsPOC8+fFCnOnFBmXVlYmo8x1vcp3tE0oCSwh6r8i0mIpmJjYD+jKVh7hZyxlE+kUUMwEPmAjJ/g7t+Kr6hM3a+8LN/nCTSGX9jkMSC24FFh2XnD+3J8HFRc+z7HD+lznVlZHX2dA6R1ooZUtjAs50DMIwn3UwiLS3Ok3m8l1og1reYDH+JQ32ALAa2zmO77nfcYQRdXrBYm4i9UCsQHO7fYyI94LHc5wUjas7Kym/8rJIuf22Vkr0IVndjj3t/GC+ECID4DuJT/jA519WFrrXzORZkFfZQ/ghzevM5JBRPErPqYQOxs4QT/eIJm7uIEu7i6iyBX5WS8tWkiZhcPP2ZwTwpUNKzsvOkcDVZRjh5Tzzq2iDr7lg4rrcVyA871FxDMomHiQX9CPvkTwM5ZyjBwyuMhN/JuX+RGPca2msheP1Nbn0iy2LoYB6YWw/bydLw+lkxfaiYMFVg7kwbHCykOa4dIoobUVhjVbgc7+JWGlQitLjB94K7SImIqCiYcZQBRbeJifs4zVHMGOwW/5jI2k8y9upxW+V76IiMlZLNDJHyJ9oINvJondonBNFllgh8MFcCAPDuRf+nkwD05XMUrIARwtcG4rKwxt9rZAV3/oFlB56+KvNYVE3EHBxAO1oxX/4x6m8wXPsw6A99jFLjJYzt10o2ZrCYl4In8v6N3KuVWUU+wMKK7AcjD/0uPcij1vcS58eDDfuVXkammpKrTE+jvLISINT8HEQ3lh5TmGM5CO3M8HXKCIXWQwgHm8w2huo4e7iyjS5Np4w4DWzq0sw4AM26Wwsj/POcuta8tzVL5W2ZaWVRVaWixAJ7/qQ4tmvxWpO319PNxoetGLcO5gCfs4Qw6FjORdnuRGZnIjVk1lL4LFAhG+zu36tuWPGYazb0rZoFJ2q6qlxcA5muh4Iaw5V/l4K+ul94vwhQgf588OVewL8tIQaJGyFEyagasIZxP/xwN8wDL2AvA0X/E1afyFm7mWaDeXUMS8LBaI9HNuVYWWM7ZLIeVwhdCSVcXIIXBOMOeaAfdKAqxlQotP1YHGdTxYIUZaAAWTZiIYP5IZw0us5wk+x4HBGlIZwpv8kFie5EaS6OzuYop4FIsFwn2d25A2lY+ftVUOK6kFzk64p4vgbDXBpax8h/M1qTUMMR18y2+RvpX3Rfiq4654LgWTZsSChSkk0Z+OPMgK0nCOm1zFEVZxhGF04Ulu5Ea6aGixSAMI8YEBPpX7tLgUOZzT9p+2OW8XnS672co/r671pax8x6V+L1cS6l19cCm7hfo4J8UTMQsFk2boZmI5wCTeZgd/Zi1HOQfAGlJZQypJxPAkNzKcWAUUkUbka3UOe+7kf+VzbQ5nB93qwsv3ZX7WJMRkFzu3PXmXP8/H4gwvnfycW7Rfmcf+zp8dfMFL/1RIE1EwaaZ88eZB+nEfV7OYnfyZtRwkG4B1HONHvM1goniSG/kx3RVQRNzMxwpRfs7tSlwtMaeKLk0sV9V2qsjZynI5NsM5ad2xKpYGcPHC2QenXGgp87iTnzPcaLI6aQgKJs2cD17cTyL30Jcl7OJZ1rKXMwBsJJ2fsJj+RPIkNzKSHgooIh6gpi0xhgHn7VcOL+mFkFnF5HQuduBEoXOrjpXyLS9RvhaKCzsQl24hyBsCrRDo5ewnE+h16XnZnwFWhRtRMGkxvLAyjr7cTR+WsYdnWMuukvXnt3CKUbzH1UQwgxu5g6s0zFikGbBYnIsbtvZ2TsN/OQV2SC9yho/jBZeCyImSYdEnCp23mqrjwPn69CLYeB6cUSUKjtSuzD6WakJLmeetrM7+PSHezi20zOMQ70vH1AHYMymYtDBeWBlDH+6kFyvYz9N8xTa+B2A7p7mTpfQmnOncwF30xksBRaRF8PdyLngYF1D9OYUOZ+tKxcBSNsxUtSxAbdgM52KNOVXMH1NbFQNMpRBT8ji05HFbb+cyBVacHYIr/rRQ/bFy55Q819DuulEwaaGsWLmDnvyUq/iYAzzNV6RwEoDdZPJzlvEUXzKdGxhLH7zR/NsiLZ2fFWIDnFt1ikrCS2q+ne8OHqVDTCwFWMmzO2fYzXdQ+rjsz8vuv0I/mepcdMDFK9yCamyuwOJnucwtrGr2l/0ZYK3+tQElt8H8rc0jDCmYtHAWLNxGD35CPJ9xiKf5im85AcB+sriX5fyJr5jG9YynLz4KKCJyGb5W6BoAMb7Q2juHxPZG6QKMdWUYUOC4FFbO2+FcsXOemGyb8+fZYue8MmdLRiOdLbM/2waFVS1J3QQcgMNwrst0sY4Bqzb8SwJKQNnNq/Lzy53jZ7FgLw7iajf9mSmYCOAMKLfSnVvoxhcc5Wm+Yi1pABwimwdYwZ/4ij9yPfdzNb76qyMiTcRiKfnF6QVhPnW7Rr79ygHmXDHYDeeSAw7jUqio+NOg+mPlzil5bject8EqtgY1wN2qSgoczu1cva5iBXqQd9LBYzENUqxa0W8XKceChZuJ5WZi+YpUnuErPucoAKmc42H+y1N8yVj6cDe9GUiURvKIiOm5gk3HGgzHbgqG4exPU90trJrc6rpYsj+/JIzkl3meX+F5XRo/6noLrb4UTKRaN9KFG+nCeo7xDF/xGYcBOMl5XuFbXuFbutKWMfRmLH24mg4KKSIiNWCxgK/FeeurTSP/Ji4bgvIrhpkqAs3FYgd5J47yaFSXxi1YNRRM5IquI4b/cS+bOMFzrONjDmDDGaWPco4XWM8LrCeesNKWlF60d3OpRUQEah+C7HaDbafPuW24tYKJ1NggOrGcsWSTx3L2sYRdfMFR7CWNhAfI4mm+4mm+og/tuZve3E0fuhPm5pKLiIinUDCRWgslkAfpx4P0I4MLLGMvS9jFWtJK72PuIoNdZDCDNfQjsjSkdKatO4suIiImp2Ai9dKeIH7JQH7JQE6SSzJ7WMKu0iHHAFs5xVZO8TiruZZO3E1v7qI3UVSzJKuIiLRYpgwmN910E+np6VUeGzRoEG+//XaNrpOTk8Prr7/O6tWrOXXqFEFBQQwaNIhJkybRvXv3hiyyAB1pzWNcy2NcSxrnWMpulrCLLZwqPWcDJ9jACX7HZ1xPZ+6mN3fSi/YEubHkIiJiFqYMJgDBwcHcf//9lfZHRUXV6PVnz55l7NixpKamcs0113DzzTeTmZnJZ599xtq1a/n3v//N1Vdf3dDFlhKdactkrmMy13GILJaUhJSdJevzGMBa0lhLGpP4lJvoyt30ZhRXEU4r9xZeRETcxrTBpHXr1kyaNKnOr589ezapqak88MADTJ06tXT/Pffcw/jx4/njH//If//7X6xWrQXT2LoRxjRuYBo3sJdMlrCL99jFfrIAcGCwmiOs5giP8BHXEc1PuYqfchWxhLq59CIi0pSa7W/lzz//HKvVWincXHPNNQwbNoxDhw6xadMmN5Wu5epJOE8xjL1MZBv/jydIomuZDrEODL7mGL9nJXG8Sl/+wZN8wVZOYtRpiiAREfEkpm0xKSoq4j//+Q8ZGRkEBQWRkJBQq1svZ86cISQkhFatKt8W6NSpEwAbNmzg2muvbbAyS81ZsHA1HbiaDvyZm9nMSf7DXj5gH/s4U3reTjLYSQbPsJZoWpe2pFxPZ63bIyLSDJk2mGRmZvLEE0+U25eQkMArr7xCTMyVJ+8PCQkhKyuLixcvVgonJ044R4ykpqbWulx2e2OsbiD96EA/OvAsw9jHGT607GeFZT8bLZc6QR8nl9lsYjabCDH8+YnRnVFGD35EHK3wrfbarjpT3Xke1Z3nUt15rsaou9pcy2IYhunax+fMmUP//v2Jj48nMDCQ1NRU3nrrLVasWEFUVBQffvghQUGXH8XxxBNP8J///Idf/OIXPP7446X7t2/fzrhx4yguLiYpKYk333yzRmWy2+1s27atPh9L6iDTp4C1bb7nq7bfsyn4DMXWyn9d/RxWBuWG84NzHbghJ4KQYpMshiEiIuUkJibidYXlpk0ZTKozZcoUVqxYwdSpU3nggQcue+7333/PnXfeSWZmJv369SMxMZGMjAw+++wzYmNj2b9/P9dffz3/+te/avTermCSkJBwxT9UaRy5FPI/yyFWsJ9PLAc5bymqdI7VsDCUaEYZPRhl9CCWEOx2Ozt37lTdeSDVnedS3Xmuxqg71zVrEkxMeyunKnfffTcrVqxg69atVwwmHTp0YNmyZbz66qusXbuWnTt30qFDB37961/TqVMnfvvb3xIWVvup0r28vPQlc5MQAvk5ffk5fSmkmC9J5QP2sYJ9nOICAA6LwTqOsc5yjMmsIoH23GaJxy/0AulerQj3akUYAYQSQAgBeDXf/t/Nir53nkt157ncVXceFUxCQkIAyMvLq9H5ERER/PnPf660f/bs2QD06dOn4QonTcoPb26hG7fQjbmMIIWTfMC+qjvPWjOgK8C2Stdpi39pUAkjsOSn83lV+8IIpA1+WBVoREQahUcFkx07dgA1n2StKna7nY8//hhvb29+9KMfNVTRxI2sWBlMJwbTiecYzj4yWcF+PmAfG8pMjV+VcxRwjgIOc7YW72chBH9CCaAboQwkigF0ZAAdiSS4vh9HRKRFM10wOXz4MB07diQgIKDS/pdffhmAkSNHlu4/f/48GRkZBAcH0759+9L9NpsNu92Ov79/6T6Hw8ELL7zA0aNHmTBhAhEREY38acQdriKcqwjncZI4xXm+dBxl24kDBHQK45y1gCzyySafLPJKfuZzlvwaz5LiwCCr5HUHyeZTDpUeiyK4NKQMJIr+RNJOM9mKiNSY6YLJJ598wltvvcXAgQNLA0pqaipr167FZrPxyCOPMHDgwNLzV61axRNPPMEdd9zB888/X7o/KyuLn/zkJ1x33XV06tQJm83GunXrOHLkCD/4wQ/4/e9/746PJ00skmDGGL2Jz7SRGJWIVzVznzhwcI4CsktDS+XwUvF5Jhc5T/kOuOmcJ539rGB/6b4utGVgmbDSj0ja4F+xCCIiggmDyeDBgzl8+DB79+5l8+bNFBQUEBISwg033MC4ceNISkqq0XWCgoK4+eab2bp1K19++SXe3t7Ex8fz7LPP8rOf/UxT0Us5VqyEEkgogTV+jYFBGudI4SSbOUkK6WzhFLkUljsvlXOkco5k9pTuiyesNKwMoCPXEHnZuVhERFoK0wWTQYMGMWjQoBqfP3r0aEaPHl1pf1BQEC+++GJDFk2kHAsWuhBCF0K4i96As+XlENnlwsp3fE8etnKvPUAWB8hiETsBZ7+VXoSXtKp05Go60JW2dCBIHW1FpEUxXTAR8WRWrMTTjnjaMZ6+ABRjZx9nyoWV7ZymiEszITow2EUGu8hgQZnRQ7540Zk2dCWELrSlC23pWvKzC22JIAgLlqb+mCIijUbBRKSReeNFHyLoQwQPcA0ARRSzi4xyYWUXGdgrdMEtws5BsjlIdpXX9se7ysDifB5COwIVXETEoyiYiLiBL970oyP96MgjJfvysbGd70nhJAfIIpVzHOUsqZzjYoVbQS4FFLOPM+XmbikrEJ9yoSWaNrTBj9b4EVzy0/nYt/SxH94KMyLiNgomIiYRgA/XEs21RJfbb2CQRR5HSzrRlg0sri2f4iqvmYeNPWSyh8wal8Mba6WwElxFgCkbbFrjRwxtiCWEAHzq9ecgIi2bgomIyVmw0I5WtKMVA6k8uaCBQQYXqwwtRzlHGucopOYrexbjKB02XRcdCSaOEOIIrfQzlAC1xojIZSmYiHg4CxYiCCKCIAbTqdJxBw5OlwSXdHI5TxG5FHKewpKfzudl95XdX1BNa0x1TnKek5zna45VOtYGv9KgElshtHSitdYuEhEFE5HmzoqVSILrPF2+DftlQ8x5isgij1TOcZizHCabTKpezyqHQrZyiq2cqnTMFy+60LZcK0sX2pIbeI5WZBFGK9rgh6/+2RJp1vQNF5HL8sGr1pPP5VLAEc6WBhXXzyOcJY0cHFUsAFCEvXR+l1JeQE+Ar0t3BeJDW/zLbW3wq7Svuk3BRsTc9A0VkQbXGn8SiSSRyErHbNhJK9O6UvbnEc5Wmoyuojxs5GHjJOfrVLYAvGmLP8H40QofAvGhFb4lP+v33Bcv9aERqScFExFpUj540Y0wuhFW6ZiBwWkulIaVQ45s9p85hk+7YHKshaWrQbu2CxXWKqqJfIrJ5wKnuNAQH6ccLywE4oM/3gSU/vTGv2SruK+659Wd44c3fnjhX8Vjb6wKRdIsKJiIiGlYsNCBYDoQzHXEYDfsbDu+jcSwqhdgLMZODpUDS9ktp4p9Z0tCzUWKsOFosPLbMThPUaXFHZuCBaoMLLV97F/P5354aRkFqRcFExHxWN54EUYgYbXo/1KRDTt52LhIUclPW72e52Ejn2IKSrZ8bLUarl1XBq7WoNqNomoMPljxt3rj0xdaW7+mVTW3v6p77Dqvutf4V/jVZZR7bFR7rOLxyj2dnOtWqfXJvRRMRKRF88GLNnjRBv9Gew8HDgqxlwaVgpIAUTa8lN1X3TmuaxRSXOvHVf0Sbiw2HNgsReAD2W5oPWoIFpyTDXphxaskrFR87F3yvLrHZV9jbcSgYymJUa4wZSmzj2oeVzy33GOrgV+nQl6iG1G0abRyV0fBRESkkVmxEoCVAHwIIaDJ39/AoBhHucBSWBJ4yj4uqBBm6vO8wLCRW5SHzdfCRYvtip2azcagJGA14K0+j2EBIqCTYyMv8qMmf3sFExGRZs6CBR+88MGLYPya5D3tDjvbdm0jMTERLy8vDAzyS297VX0rrKrbZFWdV0BxuVstFdsiKt6GudzxsscMnCt9F+PAjgN7DR4Xlzyv+Ljigpyext/uRRIxbnlvBRMREWl0FiwE4ksgvrRzd2GagIFRLuQ0VkwxSnrNuPrOGGX2Uc3jiudWfJ3dbufYjgMM6du9kUp9eQomIiIiDcyCpbTPiaexYyfT4b544Hl/YiIiItJsKZiIiIiIaSiYiIiIiGkomIiIiIhpKJiIiIiIaSiYiIiIiGkomIiIiIhpKJiIiIiIaSiYiIiIiGkomIiIiIhpaEr6GjIM53oCdrvdzSWR2nLVmerO86juPJfqznM1Rt25ruX6XXo5FqMmZwlFRUXs3LnT3cUQERHxWAkJCfj6+l72HAWTGnI4HBQXF2O1WrFYKi6iLSIiItUxDAOHw4G3tzdW6+V7kSiYiIiIiGmo86uIiIiYhoKJiIiImIaCiYiIiJiGgomIiIiYhoKJiIiImIaCiYiIiJiGgomIiIiYhoKJiIiImIbWypFm6aabbiI9Pb3KY4MGDeLtt99u4hJJRStWrGDLli3s2rWLAwcOYLPZeO655xg9enSV51+4cIHZs2ezcuVKMjMzad++PbfccgsTJ06kVatWTVz6lq02dTd79mzmzJlT7bU+//xzOnXq1JjFlRKnT5/m008/Ze3atRw5coQzZ87Qpk0b+vXrx0MPPcTVV19d6TXu+N4pmEizFRwczP33319pf1RUlBtKIxXNmjWL9PR0QkJCaN++fbVBEiAvL4977rmHvXv3kpSUxE9+8hP27t3L/PnzSUlJYdGiRfj5+TVh6Vu22tSdyx133FHld69169aNUUSpwttvv80///lPYmJiuO666wgNDSUtLY3Vq1ezevVq/vrXvzJixIjS8932vTNEmqFhw4YZw4YNc3cx5DLWr19vnDhxwjAMw3jjjTeM+Ph4Y9myZVWeO2vWLCM+Pt546aWXyu1/6aWXjPj4eOP1119v9PLKJbWpu1dffdWIj483NmzY0JRFlCp89tlnxsaNGyvtT0lJMXr37m0MHDjQKCwsLN3vru+d+piIiFsMHTq0Rq1XhmGQnJxMYGAgv/rVr8od+9WvfkVgYCDJycmNVUypQk3rTszlRz/6EYMGDaq0f8CAAQwePJicnBz2798PuPd7p1s50mwVFRXxn//8h4yMDIKCgkhISKjyHqqYW2pqKhkZGSQlJREYGFjuWGBgIP369WPdunWcOnWKyMhIN5VSriQlJYXt27djtVrp0qULQ4YMUd8gE/H29i73053fOwUTabYyMzN54oknyu1LSEjglVdeISYmxk2lktpKS0sDoEuXLlUe79KlC+vWrSM1NVXBxMRmz55d7nnr1q2ZNm0aP/3pT91TICl18uRJvvnmG8LDw4mPjwfc+71TMJFmafTo0fTv35/4+HgCAwNJTU3lrbfeYsWKFUyYMIEPP/yQoKAgdxdTauD8+fMA1daXa/+FCxearExSc1dddRV/+ctfGDRoEO3btyczM5Mvv/ySV199lalTpxIcHMzNN9/s7mK2WDabjSlTplBUVMQf/vAHvLy8APd+7xRMpFmaOHFiuec9e/bkxRdfBJxDHZOTk3nggQfcUTSRFuWHP/xhueedOnXinnvuIS4ujgceeIC///3vCiZu4nA4mDp1KikpKYwZM8Y0rVfq/Cotyt133w3A1q1b3VwSqang4GCg+v+ZufarBcyzDBkyhJiYGA4cOKDWLjdwOBz88Y9/5KOPPuL222/nT3/6U7nj7vzeKZhIixISEgI4x+eLZ+jcuTPg7IxXFdf+6u6Fi3m5vo/5+fluLknL4nA4eOKJJ1i+fDm33XYbzz//PFZr+Tjgzu+dgom0KDt27AA0yZon6dKlC+3bt2fr1q2VAmVeXh5bt26lU6dO6vjqYfLy8jh48CCBgYGlAUUanyuUfPDBB4wYMYIXX3yxtF9JWe783imYSLNz+PDhKv8HdvjwYV5++WUARo4c2dTFkjqyWCzcdddd5OXl8Y9//KPcsX/84x/k5eUxZswYN5VOLufChQscPXq00v6CggJmzJjBxYsXufXWW0uHqErjct2++eCDD7j11lt56aWXqgwl4N7vncUwDKNRriziJrNnz+att95i4MCBdOzYkYCAAFJTU1m7di02m41HHnmE3/3ud+4uZouXnJzMli1bADhw4AC7d++mX79+pU3I/fv356677gKc/0P7+c9/zr59+0hKSqJXr17s2bOHdevWkZCQwDvvvIO/v7/bPktLU9O6O3HiBMOHDychIYG4uDjatWtHVlYW33zzDd9//z3x8fEsXLhQLSZNxLVuUWBgIPfdd1+VgXD48OH07NkTcN/3TjFVmp3Bgwdz+PBh9u7dy+bNmykoKCAkJIQbbriBcePGkZSU5O4iCrBlyxaWL19ebt/WrVvLdUx2BZPAwEDeeeed0sXENm7cSHh4OL/4xS949NFHFUqaWE3rrm3btowbN44dO3bw1VdfkZubi5+fH3Fxcdx7773cc889qrsm5FrTKC8vj9dff73Kc6KiokqDibu+d2oxEREREdNQHxMRERExDQUTERERMQ0FExERETENBRMRERExDQUTERERMQ0FExERETENBRMRERExDQUTERERMQ0FExERETENBRMRkTJOnDhBjx49mDp1qruLItIiKZiIiIiIaSiYiIiIiGkomIiIiIhpeLu7ACLSvKWkpPDmm2/y3XffcfHiRTp27MiPf/xj/t//+38EBAQAsHHjRu677z4mTpzIkCFDmDVrFrt27cLLy4shQ4bwhz/8gc6dO1e69oEDB5g7dy6bNm3i/PnztG/fnptvvplf/epXhISEVDo/KyuLf/7zn6xZs4ZTp07h7+9P586dufXWW3nwwQcrnZ+WlsaLL77Ipk2bsNlsJCYmMnXqVK666qqG/4MSEQAshmEY7i6EiDRPixcv5umnn6Z169YMGzaM0NBQdu3axaZNm7jmmmtYuHAhvr6+pcEkKSmJjRs3cv3119O9e3cOHjzImjVrCAkJYenSpURHR5dee/PmzTz00EPYbDZuueUWoqKi2LZtG5s2bSImJoYlS5YQGhpaev6RI0e47777yMzMpH///lxzzTXk5+dz8OBB9u/fz6ZNmwBn59ebb76ZQYMGceDAAbp3706fPn04duwYn3/+OW3atOGTTz6hXbt2Tf7nKdIiGCIijeDgwYNGr169jNtvv93Izs4ud+yNN94w4uPjjTfffNMwDMPYsGGDER8fb8THxxvvvvtuuXPfffddIz4+3njkkUdK99ntdmP48OFGfHy8sXbt2nLnv/DCC0Z8fLzxxBNPlNs/evRoIz4+3liyZEmlsp46dar08fHjx0vL8sYbb5Q7729/+1uV+0Wk4aiPiYg0ivfee4/i4mJmzJhR6bbKQw89RGhoKB999FG5/V26dGHMmDHl9o0ZM4YuXbrw5Zdfkp2dDcDWrVs5duwYN9xwA9dff3258x999FHatm3LRx99RFFREQA7duxg165dDBw4sNL1ATp06FBpX6dOnXjooYfK7bvzzjsB2LlzZ03+CESkDtTHREQaxfbt2wH4+uuv+fbbbysd9/b25ujRo+X29evXD6u1/P+XrFYr/fr1IzU1lX379jF06FD27NkDwKBBgypdt1WrVvTp04d169Zx9OhRevTowY4dOwC47rrralz+nj17ViqLK8Dk5ubW+DoiUjsKJiLSKHJycgB4/fXXa/ya6vpthIWFAXD+/HkALly4cNnzw8PDy53nel1ERESNyxIUFFRpn7e3859Mh8NR4+uISO0omIhIo3D9Yt+yZUuVv+SrcubMmSr3Z2VlARAcHFzu2tWdn5mZWe681q1bA3D69OkalUNE3Ed9TESkUfTt2xe4dEunJrZu3VqpNcLhcLB161YsFkvpMN1evXoBlI6kKSsvL49du3bh7+9P165dAUhISABg/fr1tf8gItKkFExEpFGMGzcOb29vnnnmGU6ePFnpeG5ubmlfEZfU1FSWLl1abt/SpUtJTU3lBz/4Qenw3379+hETE8PatWv55ptvyp3/2muvce7cOX7yk5/g6+sLOENSQkICKSkpla4PakkRMRPdyhGRRhEfH8/MmTN56qmnuPXWW7nxxhuJjo7m4sWLnDhxgk2bNnHHHXfw9NNPl74mKSmJZ599lq+++qrSPCbTpk0rPc9qtfLcc8/x0EMP8fDDD5fOY/Ldd9+VzmPyhz/8oVx5Xn75Ze69915mzJjBihUrSExMpLCwkEOHDrF37142btzYZH82IlI9BRMRaTRjxozhqquuYsGCBaSkpLBmzRqCgoLo2LEjEyZM4Kc//Wm58xMTE/nlL3/JrFmzePvtt7FarQwfPpzJkyeXm1wNYMCAASxZsoS5c+eyfv16Lly4QPv27bnvvvv45S9/WW5yNXAORV6+fDlvvPEGa9as4d///jetWrWic+fO/PKXv2zsPwoRqSHN/Coibld2SvpJkya5uzgi4kbqYyIiIiKmoWAiIiIipqFgIiIiIqahPiYiIiJiGmoxEREREdNQMBERERHTUDARERER01AwEREREdNQMBERERHTUDARERER01AwEREREdNQMBERERHT+P+1BoKKnretLwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 600x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train(model, optimizer, scheduler, criterion, train_loader, valid_loader, NUM_EPOCHS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d9c22212",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "\n",
    "def get_similarities(embed_vector):\n",
    "    similarities = embed_matrix @ embed_vector / embed_norms / torch.norm(embed_vector)\n",
    "    top_indices = similarities.argsort(descending=True)\n",
    "    return similarities, top_indices\n",
    "\n",
    "\n",
    "def get_similar(query, num_similar=5):\n",
    "    print(f'Processing query \\\"{query}\\\"')\n",
    "    split_query = re.split('([^a-zA-Z0-9])', query)\n",
    "    index = vocab[split_query[0]]\n",
    "    embed_vector = torch.clone(embed_matrix[index])\n",
    "    if index > 0:\n",
    "        print(f'Found word \\\"{split_query[0]}\\\" in vocab')\n",
    "    else:\n",
    "        print(f'Word \\\"{split_query[0]}\\\" not in vocab, using <unk>')\n",
    "\n",
    "    for i in range(2, len(split_query), 2):\n",
    "        index = vocab[split_query[i]]\n",
    "        if index > 0:\n",
    "            print(f'Found word \\\"{split_query[i]}\\\" in vocab, ', end='')\n",
    "        else:\n",
    "            print(f'Word \\\"{split_query[i]}\\\" not in vocab, using <unk>, ', end='')\n",
    "        print(f'operation \\\"{split_query[i - 1]}\\\"')\n",
    "\n",
    "        if split_query[i - 1] == '+':\n",
    "            embed_vector += embed_matrix[index]\n",
    "        elif split_query[i - 1] == '-':\n",
    "            embed_vector -= embed_matrix[index]\n",
    "        else:\n",
    "            raise ValueError('Unknown operation')\n",
    "\n",
    "    similarities, top_indices = get_similarities(embed_vector)\n",
    "    itos = vocab.get_itos()\n",
    "    print('Top similar:')\n",
    "    for top_index in top_indices[:num_similar]:\n",
    "        print(f'    {itos[top_index]}: sim = {similarities[top_index]:.4f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "17e53619",
   "metadata": {},
   "outputs": [],
   "source": [
    "embed_matrix = model.embeds.weight.detach()\n",
    "embed_norms = (embed_matrix ** 2).sum(1).sqrt()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "ff777fce",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing query \"woman\"\n",
      "Found word \"woman\" in vocab\n",
      "Top similar:\n",
      "    woman: sim = 1.0000\n",
      "    girl: sim = 0.7951\n",
      "    man: sim = 0.7619\n",
      "    lover: sim = 0.7379\n",
      "    jealous: sim = 0.7238\n"
     ]
    }
   ],
   "source": [
    "get_similar('woman')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "f24a1844",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing query \"kick\"\n",
      "Found word \"kick\" in vocab\n",
      "Top similar:\n",
      "    kick: sim = 1.0000\n",
      "    touchdown: sim = 0.8346\n",
      "    ecu: sim = 0.8276\n",
      "    punt: sim = 0.8110\n",
      "    tech: sim = 0.7963\n"
     ]
    }
   ],
   "source": [
    "get_similar('kick')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "6996edc3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing query \"help\"\n",
      "Found word \"help\" in vocab\n",
      "Top similar:\n",
      "    help: sim = 1.0000\n",
      "    keep: sim = 0.6913\n",
      "    try: sim = 0.6778\n",
      "    accept: sim = 0.6642\n",
      "    kill: sim = 0.6633\n"
     ]
    }
   ],
   "source": [
    "get_similar('help')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "43a72873",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing query \"king-man+woman\"\n",
      "Found word \"king\" in vocab\n",
      "Found word \"man\" in vocab, operation \"-\"\n",
      "Found word \"woman\" in vocab, operation \"+\"\n",
      "Top similar:\n",
      "    king: sim = 0.8269\n",
      "    son: sim = 0.6992\n",
      "    hairan: sim = 0.6945\n",
      "    lord: sim = 0.6846\n",
      "    henry: sim = 0.6820\n",
      "    eldest: sim = 0.6809\n",
      "    leo: sim = 0.6657\n",
      "    nephew: sim = 0.6584\n",
      "    daughter: sim = 0.6580\n",
      "    gerard: sim = 0.6577\n"
     ]
    }
   ],
   "source": [
    "v = get_similar('king-man+woman', num_similar=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "10ae5d04",
   "metadata": {},
   "source": [
    "# FastText for sentiment classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d733811",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "kaggle datasets download -d shivkumarganesh/bumble-dating-app-google-play-store-review\n",
    "unzip bumble-dating-app-google-play-store-review.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "6114b43f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import fasttext\n",
    "import fasttext.util\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "sns.set_style('whitegrid')\n",
    "plt.rcParams.update({'font.size': 14})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "b2bf448a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reviewId</th>\n",
       "      <th>userName</th>\n",
       "      <th>userImage</th>\n",
       "      <th>content</th>\n",
       "      <th>score</th>\n",
       "      <th>thumbsUpCount</th>\n",
       "      <th>reviewCreatedVersion</th>\n",
       "      <th>at</th>\n",
       "      <th>replyContent</th>\n",
       "      <th>repliedAt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>96d20295-5ea1-4cb0-a0b0-5bc27f843e00</td>\n",
       "      <td>Eric-Martin</td>\n",
       "      <td>https://play-lh.googleusercontent.com/a-/ACNPE...</td>\n",
       "      <td>love it</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>5.293.1</td>\n",
       "      <td>2022-11-06 02:13:12</td>\n",
       "      <td>That’s what we love to hear! It sure is the pl...</td>\n",
       "      <td>2022-11-06 16:12:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fdcd745a-1831-41ca-9921-579ee2a2675e</td>\n",
       "      <td>Amanda Turel</td>\n",
       "      <td>https://play-lh.googleusercontent.com/a/ALm5wu...</td>\n",
       "      <td>My experience with Bumble has been terrible. M...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2022-11-06 02:03:09</td>\n",
       "      <td>Our team fully investigates all reports before...</td>\n",
       "      <td>2022-11-06 16:12:02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6c3dad16-73cd-4cf1-8c3f-f7b8dc2fd55b</td>\n",
       "      <td>Money Making Mitch</td>\n",
       "      <td>https://play-lh.googleusercontent.com/a/ALm5wu...</td>\n",
       "      <td>Can't get into my account</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>5.293.1</td>\n",
       "      <td>2022-11-06 02:00:30</td>\n",
       "      <td>This doesn’t sound ideal and we'd love to inve...</td>\n",
       "      <td>2022-11-06 16:11:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>450b87e9-2434-42a5-8093-526317830bcc</td>\n",
       "      <td>kirsten roberts</td>\n",
       "      <td>https://play-lh.googleusercontent.com/a/ALm5wu...</td>\n",
       "      <td>I feel undecided and cheated by this dating ap...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>5.293.1</td>\n",
       "      <td>2022-11-06 01:42:45</td>\n",
       "      <td>Hi Kirsten, we're so sorry to hear that you're...</td>\n",
       "      <td>2022-11-06 16:49:33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3f6d0a04-8534-444d-95c9-4f3c46e563e1</td>\n",
       "      <td>MAB</td>\n",
       "      <td>https://play-lh.googleusercontent.com/a-/ACNPE...</td>\n",
       "      <td>It's been crashing I try to login and it's get...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>5.293.1</td>\n",
       "      <td>2022-11-06 01:36:28</td>\n",
       "      <td>This doesn’t sound ideal and we'd love to inve...</td>\n",
       "      <td>2022-11-06 16:21:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119413</th>\n",
       "      <td>61bc7cdc-3085-4631-ac95-d8e42e2e0742</td>\n",
       "      <td>A Google user</td>\n",
       "      <td>https://play-lh.googleusercontent.com/EGemoI2N...</td>\n",
       "      <td>Finally here!</td>\n",
       "      <td>5</td>\n",
       "      <td>54</td>\n",
       "      <td>1.0.1</td>\n",
       "      <td>2015-12-01 02:17:24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119414</th>\n",
       "      <td>24d07489-88ad-496a-80ce-8f137c130323</td>\n",
       "      <td>A Google user</td>\n",
       "      <td>https://play-lh.googleusercontent.com/EGemoI2N...</td>\n",
       "      <td>Finally!</td>\n",
       "      <td>5</td>\n",
       "      <td>75</td>\n",
       "      <td>1.0.1</td>\n",
       "      <td>2015-12-01 02:09:05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119415</th>\n",
       "      <td>22a42d0d-7a12-400d-a2af-826f35d21581</td>\n",
       "      <td>A Google user</td>\n",
       "      <td>https://play-lh.googleusercontent.com/EGemoI2N...</td>\n",
       "      <td>Finally, an app where women have to start the ...</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0.1</td>\n",
       "      <td>2015-12-01 00:18:53</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119416</th>\n",
       "      <td>2480a41f-8863-4948-9f83-7d856bd9dc78</td>\n",
       "      <td>A Google user</td>\n",
       "      <td>https://play-lh.googleusercontent.com/EGemoI2N...</td>\n",
       "      <td>At last we have Android version!</td>\n",
       "      <td>5</td>\n",
       "      <td>60</td>\n",
       "      <td>1.0.0</td>\n",
       "      <td>2015-11-30 21:27:51</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119417</th>\n",
       "      <td>6ebb270a-3b4b-4ff5-b98c-8899dcfd40ee</td>\n",
       "      <td>A Google user</td>\n",
       "      <td>https://play-lh.googleusercontent.com/EGemoI2N...</td>\n",
       "      <td>Nice and smooth.</td>\n",
       "      <td>5</td>\n",
       "      <td>52</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-11-29 23:15:25</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>119412 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    reviewId            userName  \\\n",
       "0       96d20295-5ea1-4cb0-a0b0-5bc27f843e00         Eric-Martin   \n",
       "1       fdcd745a-1831-41ca-9921-579ee2a2675e        Amanda Turel   \n",
       "2       6c3dad16-73cd-4cf1-8c3f-f7b8dc2fd55b  Money Making Mitch   \n",
       "3       450b87e9-2434-42a5-8093-526317830bcc     kirsten roberts   \n",
       "4       3f6d0a04-8534-444d-95c9-4f3c46e563e1                 MAB   \n",
       "...                                      ...                 ...   \n",
       "119413  61bc7cdc-3085-4631-ac95-d8e42e2e0742       A Google user   \n",
       "119414  24d07489-88ad-496a-80ce-8f137c130323       A Google user   \n",
       "119415  22a42d0d-7a12-400d-a2af-826f35d21581       A Google user   \n",
       "119416  2480a41f-8863-4948-9f83-7d856bd9dc78       A Google user   \n",
       "119417  6ebb270a-3b4b-4ff5-b98c-8899dcfd40ee       A Google user   \n",
       "\n",
       "                                                userImage  \\\n",
       "0       https://play-lh.googleusercontent.com/a-/ACNPE...   \n",
       "1       https://play-lh.googleusercontent.com/a/ALm5wu...   \n",
       "2       https://play-lh.googleusercontent.com/a/ALm5wu...   \n",
       "3       https://play-lh.googleusercontent.com/a/ALm5wu...   \n",
       "4       https://play-lh.googleusercontent.com/a-/ACNPE...   \n",
       "...                                                   ...   \n",
       "119413  https://play-lh.googleusercontent.com/EGemoI2N...   \n",
       "119414  https://play-lh.googleusercontent.com/EGemoI2N...   \n",
       "119415  https://play-lh.googleusercontent.com/EGemoI2N...   \n",
       "119416  https://play-lh.googleusercontent.com/EGemoI2N...   \n",
       "119417  https://play-lh.googleusercontent.com/EGemoI2N...   \n",
       "\n",
       "                                                  content  score  \\\n",
       "0                                                 love it      5   \n",
       "1       My experience with Bumble has been terrible. M...      1   \n",
       "2                               Can't get into my account      1   \n",
       "3       I feel undecided and cheated by this dating ap...      2   \n",
       "4       It's been crashing I try to login and it's get...      1   \n",
       "...                                                   ...    ...   \n",
       "119413                                      Finally here!      5   \n",
       "119414                                           Finally!      5   \n",
       "119415  Finally, an app where women have to start the ...      4   \n",
       "119416                   At last we have Android version!      5   \n",
       "119417                                   Nice and smooth.      5   \n",
       "\n",
       "        thumbsUpCount reviewCreatedVersion                   at  \\\n",
       "0                   0              5.293.1  2022-11-06 02:13:12   \n",
       "1                   0                  NaN  2022-11-06 02:03:09   \n",
       "2                   0              5.293.1  2022-11-06 02:00:30   \n",
       "3                   0              5.293.1  2022-11-06 01:42:45   \n",
       "4                   0              5.293.1  2022-11-06 01:36:28   \n",
       "...               ...                  ...                  ...   \n",
       "119413             54                1.0.1  2015-12-01 02:17:24   \n",
       "119414             75                1.0.1  2015-12-01 02:09:05   \n",
       "119415              2                1.0.1  2015-12-01 00:18:53   \n",
       "119416             60                1.0.0  2015-11-30 21:27:51   \n",
       "119417             52                  NaN  2015-11-29 23:15:25   \n",
       "\n",
       "                                             replyContent            repliedAt  \n",
       "0       That’s what we love to hear! It sure is the pl...  2022-11-06 16:12:03  \n",
       "1       Our team fully investigates all reports before...  2022-11-06 16:12:02  \n",
       "2       This doesn’t sound ideal and we'd love to inve...  2022-11-06 16:11:04  \n",
       "3       Hi Kirsten, we're so sorry to hear that you're...  2022-11-06 16:49:33  \n",
       "4       This doesn’t sound ideal and we'd love to inve...  2022-11-06 16:21:03  \n",
       "...                                                   ...                  ...  \n",
       "119413                                                NaN                  NaN  \n",
       "119414                                                NaN                  NaN  \n",
       "119415                                                NaN                  NaN  \n",
       "119416                                                NaN                  NaN  \n",
       "119417                                                NaN                  NaN  \n",
       "\n",
       "[119412 rows x 10 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('bumble_google_play_reviews.csv')\n",
    "data = data[data.content.notna()]\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "b40ef514",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd0AAAHdCAYAAABYPaNuAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAA9hAAAPYQGoP6dpAABN6ElEQVR4nO3de1xUdf4/8NcwSnqAkAoWA3S0HPOCqyKoLEoXzUJalcRVwxRDXcGWKNes9Z77te/WN0HKvOca1spskm2R60a/0Km4BLlSsiYmKKLBVipwSGDm/P6YnbOMMyozDMdhej0fjx4Ln/M+l3mvnNecOZdRSZIkgYiIiDqdx83eACIiop8Lhi4REZFCGLpEREQKYegSEREphKFLRESkEIYuERGRQhi6RERECmHodgGSJMFgMIC3VBMRdW0M3S7AaDTi6NGjMBqNDs9/7Ngxh+d3F+yDCftgwj6YsA8mSvWBofszIEkSWlpafvZHyuyDCftgwj6YsA8mSvWBoUtERKQQhi4REZFCGLpEREQKYegSEREphKFLRESkEIYuERGRQhi6RERECmHoEhERKYShS0REpBCGLhERkUIYukRERAph6BIRESmEoUtERKQQhi4REZFCGLpEREQKYegSEREppNvN3oCr7d+/H88999x1a8aMGYM///nP8u8NDQ3IzMzEoUOHUFdXh4CAAEyaNAlLliyBl5eX1fxGoxF79+5FdnY2qqqqIAgCIiMjkZaWhpCQEJvrPHLkCLZu3Yqvv/4aKpUKQ4YMQXJyMsaOHWuz/vTp00hPT0dBQQGampqg0Wgwc+ZMzJo1CyqVyo6OEBGRu3C50B00aBCWLFlic9rf//53nDx5ElFRUfKYKIpISEhAeXk5oqKiMHnyZJSXl2PXrl0oLi7G3r17ccstt1gsZ9WqVdDpdBgwYADmzJmD2tpafPjhh/j000+xb98+aDQai/oDBw5g2bJluO222xAXFwcAyM3NRWJiItLT0/HQQw9Z1FdUVGDmzJn46aef8PDDDyMgIAD5+flYu3YtTp06hZUrVzqhU/bp3r274uskIqKrSF3ElStXpIiICGnw4MFSXV2dPJ6RkSFptVrppZdesqh/6aWXJK1WK23ZssVi/PPPP5e0Wq302GOPSVeuXJHHP/nkE0mr1Urz58+3qL948aI0atQoafTo0dL58+fl8fPnz0ujR4+WRo8eLdXX11vM89hjj0larVb65JNPLLZ/9uzZklarlUpLS+167a2trdIXX3whtba22jVf2/lbHJjXYDA6tD5X1dE+ugv2wYR9MGEfTJTqg8sd6V7LRx99hIsXL2LChAm44447AACSJEGn00EQBCQnJ1vUJycnY+/evdDpdFi0aJE8rtPpAACpqanw9PSUx6OjoxEREQG9Xo+amhrceeedAICDBw/i8uXLePLJJxEYGCjXBwYGIiEhAZmZmfjoo48wdepUAKaPlYuLizF69GhER0fL9Z6enkhNTcWcOXOQnZ2NESNGOLdBN9BNrUZm3kmcu9jUrvogv5548v4BnbxVREQ/L13mQqq//vWvAID4+Hh5rLKyErW1tRg5ciQEQbCoFwQBI0eOxNmzZ3H+/Hl5vLCwUJ52tXHjxgEAioqK5DHzz20/0jYzj7W3PiwsDIIgoLi4+AavtnOcu9iEyu8b2/XfuR/bF85ERNR+XeJI99y5c/j8888RGBgoByMAVFVVAYDVOVgzjUYDvV6PyspK9O7dG6Iooq6uDlqtFmq12qq+b9++FssFTMHedlpH6tVqNYKDg1FRUYHW1lZ062Zf+w0Gg131Zkaj0fR6JQmSZN+8jq7TFZlfizu9JkewDybsgwn7YOKMPtjKlat1idDdv38/jEYjpk2bZvGi6uvrAQDe3t425zOPNzQ02FVvrms7r4+PT4frAcDLywtGoxGNjY3w9fW1WXMtZWVldtWb9ezZE4MHD4bY1CRv342IgukK6xMnTqCpyb2Oeh3to7thH0zYBxP2waQjfQgLC7thjcuHrtFoxP79+6FSqfDoo4/e7M25qUJDQ9v1TupqRqMRACD07Alv7/Yd6po/rh84cKDd63NVBoMBZWVlDvfRXbAPJuyDCftgolQfXD50P/vsM9TU1GDs2LFW99CajyavdfRmHjcfkba3vu1RatujWT8/P7vqbWlsbIRKpbJ5//CNqNXqjv1jUKlg7y3C7vhH2OE+ugn2wYR9MGEfTDq7Dy5/IZX5auO2F1CZmc+bms+jXs08bj7nKwgC/P39UV1dbfNze/O52bbnY83ztj1v62i9wWBAdXU1goOD7T6fS0REXZ9Lh+6PP/6IvLw89OrVCxMnTrSartFoEBAQgNLSUoiiaDFNFEWUlpYiODgYvXv3lscjIiLkaVc7cuQIACA8PFweM/+s1+ut6s1jERER7aovKSmBKIoWyyciop8Plw7dAwcOoKWlBY888ojFPbVmKpUK8fHxEEURmzdvtpi2efNmiKKIGTNmWIybf8/IyEBzc7M8np+fj6KiIkRFRSEoKEgef/jhh+Hj44OsrCxcuHBBHr9w4QKysrLg5+eHCRMmyOP9+/dHeHg4CgsLkZ+fL483NzcjIyMDgO2jdiIicn8u/RnnO++8A+D6IZWUlIS8vDxs374d5eXlGDx4MI4fPw69Xo/Q0FDMnTvXon7MmDGIj4+HTqdDXFwcoqOjUVdXh9zcXPTq1QsrVqywqPf19cXKlSuxbNkyTJs2DTExMQBMj4G8ePEiNm7caHU19OrVqzFr1iykpKQgJiYG/v7+yM/Px8mTJ5GQkGDzHmEiInJ/Lhu6x44dwzfffINhw4Zd9wpaQRCQlZUlf+FBYWEh/P39MX/+fKSkpKBHjx5W86xbtw5arRbZ2dnYs2cPBEHAxIkTkZaWhj59+ljVT5kyBX5+fti6dSv2798PABg6dCgWL16MyMhIq/oBAwYgOzsb6enpyM/PhyiK0Gg0WLVqFWbPnt2BrhARUVfmsqE7bNgwnDhxol21Pj4+eP755/H888+3q97DwwOPP/44Hn/88XZvz/jx4zF+/Ph21/fv3x+bNm1qdz0REbk/lz6nS0RE5E4YukRERAph6BIRESmEoUtERKQQhi4REZFCGLpEREQKYegSEREphKFLRESkEIYuERGRQhi6RERECmHoEhERKYShS0REpBCGLhERkUIYukRERAph6BIRESmEoUtERKQQhi4REZFCGLpEREQKYegSEREphKFLRESkEIYuERGRQhi6RERECmHoEhERKYShS0REpBCGLhERkUIYukRERAph6BIRESmEoUtERKQQhi4REZFCGLpEREQKYegSEREphKFLRESkEIYuERGRQhi6RERECmHoEhERKYShS0REpBCGLhERkUIYukRERAph6BIRESmEoUtERKQQhi4REZFCGLpEREQKYegSEREppNvN3oDr+cc//oG33noLx48fhyiK8Pf3x/Dhw/H73/8evXv3lusaGhqQmZmJQ4cOoa6uDgEBAZg0aRKWLFkCLy8vq+UajUbs3bsX2dnZqKqqgiAIiIyMRFpaGkJCQmxuy5EjR7B161Z8/fXXUKlUGDJkCJKTkzF27Fib9adPn0Z6ejoKCgrQ1NQEjUaDmTNnYtasWVCpVM5pEBERdSkuGbqSJGH16tXYt28f+vTpg5iYGHh5eaG2thbFxcU4d+6cHLqiKCIhIQHl5eWIiorC5MmTUV5ejl27dqG4uBh79+7FLbfcYrH8VatWQafTYcCAAZgzZw5qa2vx4Ycf4tNPP8W+ffug0Wgs6g8cOIBly5bhtttuQ1xcHAAgNzcXiYmJSE9Px0MPPWRRX1FRgZkzZ+Knn37Cww8/jICAAOTn52Pt2rU4deoUVq5c2XnNIyIil+WSobtnzx7s27cPs2fPxooVK6BWqy2mt7a2yj/v2LED5eXlWLBgAZYuXSqPv/zyy9i+fTt2796NRYsWyeMFBQXQ6XQIDw/Hrl274OnpCQCIjY3FwoUL8cILL2Dnzp1y/aVLl7B+/Xr4+fkhJycHgYGBAIAFCxZg6tSpWLNmDaKiouDt7S3Ps2bNGtTX12Pbtm2Ijo4GAKSmpiIxMRFZWVmIjY3FiBEjnNgxIiLqClzunO5PP/2E1157DSEhIfjDH/5gFbgA0K2b6b2CJEnQ6XQQBAHJyckWNcnJyRAEATqdzmLc/HtqaqocuAAQHR2NiIgI6PV61NTUyOMHDx7E5cuXkZCQIAcuAAQGBiIhIQE//vgjPvroI3n89OnTKC4uxujRo+XABQBPT0+kpqYCALKzs+3uCxERdX0uF7p6vR6XLl3ChAkTYDQacejQIWzbtg1vv/02qqqqLGorKytRW1uLkSNHQhAEi2mCIGDkyJE4e/Yszp8/L48XFhbK0642btw4AEBRUZE8Zv45KirKqt481t76sLAwCIKA4uLi6zeBiIjckst9vPz1118DADw8PPDII4+gsrJSnubh4YF58+bh2WefBQA5hK8+B2um0Wig1+tRWVmJ3r17QxRF1NXVQavV2jyC7tu3r8VyAcjrN0/rSL1arUZwcDAqKirQ2toqH7G3l8FgsKvezGg0ml6vJEGS7JvX0XW6IvNrcafX5Aj2wYR9MGEfTJzRB1u5cjWXC93vv/8eALB7924MHjwYOp0Od911F8rLy7Fy5Urs2rULISEhmD17Nurr6wHA4nxqW+bxhoYGAGh3vbmu7bw+Pj4drgcALy8vGI1GNDY2wtfX12bNtZSVldlVb9azZ08MHjwYYlOTvH03IgqmK6xPnDiBpqYmh9brqhzto7thH0zYBxP2waQjfQgLC7thjcuFrvSfQ7Hu3bvjtddewy9+8QsAwKhRo5CRkYEpU6bgjTfewOzZs2/mZt4UoaGh7XondTWj0QgAEHr2hLd3+w51zR/XDxw40O71uSqDwYCysjKH++gu2AcT9sGEfTBRqg8uF7rmo8ehQ4fKgWum1WoREhKCqqoqXL58WT6avNbRm3ncvMz21rc9Sm17NOvn52dXvS2NjY1QqVQ27x++EbVa3bF/DCoV7L1F2B3/CDvcRzfBPpiwDybsg0ln98HlLqTq378/gGt/PGse/+mnn+Tzpm3P+7ZlHjef8xUEAf7+/qiurrb5ub353Gzb87Hmea++iMuReoPBgOrqagQHB9t9PpeIiLo+lwvd0aNHAwC+/fZbq2ktLS04c+YMBEHAbbfdBo1Gg4CAAJSWlkIURYtaURRRWlqK4OBgi6dXRUREyNOuduTIEQBAeHi4PGb+Wa/XW9WbxyIiItpVX1JSAlEULZZPREQ/Hy4Xun369EFUVBSqqqqs7rHdtm0bLl++jAkTJqBbt25QqVSIj4+HKIrYvHmzRe3mzZshiiJmzJhhMW7+PSMjA83NzfJ4fn4+ioqKEBUVhaCgIHn84Ycfho+PD7KysnDhwgV5/MKFC8jKyoKfnx8mTJggj/fv3x/h4eEoLCxEfn6+PN7c3IyMjAwAQHx8vKPtISKiLswlP+NcvXo1Zs6ciRUrVuCjjz5C//79cfz4cRQUFCAoKAjLli2Ta5OSkpCXl4ft27ejvLwcgwcPxvHjx6HX6xEaGoq5c+daLHvMmDGIj4+HTqdDXFwcoqOjUVdXh9zcXPTq1QsrVqywqPf19cXKlSuxbNkyTJs2DTExMQBMj4G8ePEiNm7caHU19OrVqzFr1iykpKQgJiYG/v7+yM/Px8mTJ5GQkGDzHmEiInJ/Lhm6ffr0wTvvvINNmzbhyJEj+PTTT3HHHXfgscceQ0pKCm6//Xa5VhAEZGVlyV94UFhYCH9/f8yfPx8pKSno0aOH1fLXrVsHrVaL7Oxs7NmzB4IgYOLEiUhLS0OfPn2s6qdMmQI/Pz9s3boV+/fvB2C60Gvx4sWIjIy0qh8wYACys7ORnp6O/Px8iKIIjUaDVatW/SyvuiYiIhOVJNn7uARSmsFgwNGjRzF8+HCHrqozGAxQq9VY/s4xVH7f2K55NLd74cVHh9m9LlfW0T66C/bBhH0wYR9MlOqDy53TJSIiclcMXSIiIoUwdImIiBTC0CUiIlIIQ5eIiEghDF0iIiKFMHSJiIgUwtAlIiJSCEOXiIhIIQxdIiIihTB0iYiIFMLQJSIiUghDl4iISCEMXSIiIoUwdImIiBTC0CUiIlIIQ5eIiEghDF0iIiKFMHSJiIgUwtAlIiJSCEOXiIhIIQxdIiIihTB0iYiIFMLQJSIiUghDl4iISCEMXSIiIoUwdImIiBTC0CUiIlIIQ5eIiEghDF0iIiKFMHSJiIgUwtAlIiJSCEOXiIhIIQxdIiIihTB0iYiIFMLQJSIiUghDl4iISCEMXSIiIoUwdImIiBTC0CUiIlIIQ5eIiEghDF0iIiKFMHSJiIgU0u1mb4At999/P86dO2dzWkREBN58802LsebmZmzbtg3vvfcezp8/D19fX9x333146qmncPvtt9tcznvvvYc9e/agoqIC3bt3x8iRI/G73/0OQ4YMsVl/7NgxZGZm4ssvv0Rrayu0Wi3mzZuHmJgYm/W1tbVIT0/H4cOHcenSJdx5552YOnUqkpKS0L17dzu6QURE7sIlQxcAfHx8MHfuXKvxoKAgi9+NRiMWL14MvV6P4cOH48EHH0RVVRV0Oh0+//xzZGdn47bbbrOY5/XXX0d6ejqCgoIwc+ZMNDY24oMPPsDMmTOxe/duhIWFWdQXFBQgKSkJnp6emDx5Mry8vHDo0CGkpaXhwoULmD9/vkV9XV0dZsyYgQsXLmDixIno27cviouLkZ6ejmPHjmHz5s1QqVRO6hQREXUVLhu6t956K5588skb1uXk5ECv1yM2NhYvv/yyHGZvv/021qxZg/T0dKxbt06ur6ysxKuvvgqNRoO//vWv8PHxAQDMnj0bM2bMwMqVK/H+++/Dw8P0yXtraytWrlwJlUqFvXv3YtCgQQCAlJQUTJ8+Ha+88gomTZpk8Wbg5Zdfxvnz57FmzRrMmjULACBJEp555hl88MEH+OCDDxAbG+ucRhERUZfR5c/p6nQ6AMDTTz9tcfQ4c+ZMhISE4G9/+xt++ukneXz//v1obW3F4sWL5cAFgEGDBiE2NhanTp1CSUmJPF5QUIAzZ84gNjZWDlzAdCT+29/+Fi0tLcjJyZHHGxoakJubi5CQEMycOVMeV6lUeOaZZwAA2dnZTuwAERF1FS4bus3Nzdi/fz+2bNmCrKws/POf/7SquXLlCv75z3+iX79+Vh87q1QqREZGQhRFfPXVV/J4UVERAOBXv/qV1fKioqIsatr+bJ5mq764uFgeO3r0KJqbmxEZGWn1EXJQUBD69euH0tJSGAyG6zeAiIjcjst+vFxXV4fnnnvOYiw0NBSvvPIK+vTpAwA4c+YMjEYjNBqNzWWYxysrKzFq1Cj5Z0EQ4O/vb1Xft29fAEBVVZU8VllZaTGtLX9/fwiCYFFv/vl623T69GnU1NQgJCTEZs21OBrURqMRarUakCRIkn3zutObA/NrcafX5Aj2wYR9MGEfTJzRB7VafcMalwzduLg4hIWFQavVQhAEVFZW4o033sCBAwcwb948vPfee/D29kZ9fT0AwNvb2+ZyzOMNDQ3yWENDg9WFVVfXm5fbdt62H0VfPU/bevPP16r38vKyWkd7lZWV2T0PAPTs2RODBw+G2NRk0YvrEQXTUfqJEyfQ1NTk0HpdlaN9dDfsgwn7YMI+mHSkD1dfhGuLS4bukiVLLH4fNGgQ/vSnPwEADhw4AJ1Oh8TExJuxaTdVaGhou95JXc1oNAIAhJ494e3dvkNdQRAAAAMHDrR7fa7KYDCgrKzM4T66C/bBhH0wYR9MlOqDS4butfzmN7/BgQMHUFpaisTERPlo8lpHb+bxtkfCVx+Z2qpve5Rq6+j36nl8fX3l383zXqu+sbHRah3tpVarO/aPQaWCvXcqueMfYYf76CbYBxP2wYR9MOnsPrjshVS2+Pn5AQBEUQQAhISEwMPDQz7vejXzeNvzqxqNBqIooq6uzqrefD627flb87xtz9ua1dXVQRRFi3rzz9fbpu7du6N37942pxMRkfvqUqF77NgxAP99QEaPHj0wbNgwnD592uoJVpIk4bPPPoMgCBg6dKg8Hh4eDgD49NNPrZav1+sBmJ56dXW9eZqtenMNAAwfPhzdu3fHZ599Bumqq5bOnTuH06dPY+TIkejWrUt9yEBERE7gcqF76tQpmxfunDp1Ci+//DIA4JFHHpHHZ8yYAQB45ZVXLELuL3/5C86ePYtHHnkEPXr0kMfj4uLQrVs3vP766xYfAZeXl+P999/HXXfdZXEyfOzYsQgJCcH777+P8vJyeby+vh5btmxB9+7dMXXqVHnc29sbkydPxtmzZ/GXv/xFHpckCa+88orFNhMR0c+Lyx1u5ebm4o033kB4eDjuvPNO9OzZE5WVlTh8+DBaWlqwaNEiiyPLadOmITc3F++//z6qq6sRHh6OM2fO4NChQwgODsZTTz1lsfx+/fphyZIlSE9Px5QpU/Dggw/Kj4EEgBdeeEF+GhUAdOvWDevXr0dSUhIee+wxi8dAnjt3Ds8++yyCg4Mt1vHMM8+gsLAQa9euxeeff44+ffqguLgYR48exX333YfJkyd3XgOJiMghSjwX3+VCd/To0Th16hTKy8vxxRdf4KeffoKfnx/Gjx+P2bNnWz2kwsPDA6+//jq2bduGAwcOYPfu3ejVqxemT5+Op556yubtQYsXL0ZQUBD+/Oc/4+2330b37t0xatQopKam2vzCgzFjxuCtt97Cpk2bkJubK3/hwdKlS21+4UFAQACys7ORnp6O/Px8fPzxxwgKCkJqaiqSkpL43GUiok5kNErw8LBvP6tWqzH4Gl9440wq6eoTj+RyDAYDjh49iuHDhzt0VZ3BYIBarcbyd46h8vvGds2jud0LLz46zO51ubKO9tFdsA8m7IOJu/Yh8+OTOPdj+58xENSrJ558YIC8v+wsLnekS0RE1FHnfmxq90GGklzuQioiIiJ3xdAlIiJSCEOXiIhIIQxdIiIihTB0iYiIFMLQJSIiUghDl4iISCEMXSIiIoUwdImIiBTC0CUiIlIIQ5eIiEghDF0iIiKFMHSJiIgUwtAlIiJSCEOXiIhIIQxdIiIihTB0iYiIFMLQJSIiUghDl4iISCEMXSIiIoUwdImIiBTC0CUiIlIIQ5eIiEghDF0iIiKFMHSJiIgUwtAlIiJSCEOXiIhIIQxdIiIihTB0iYiIFMLQJSIiUghDl4iISCEMXSIiIoUwdImIiBTC0CUiIlIIQ5eIiEghDF0iIiKFMHSJiIgUwtAlIiJSCEOXiIhIIQxdIiIihTgcusXFxaipqbluzfnz51FcXOzoKoiIiNyKw6H7+OOPY//+/deteffdd/H44487ugoiIiK34nDoSpJ0wxqj0QiVSuXoKixs27YNAwcOxMCBA3H06FGr6Q0NDdiwYQPuu+8+DB06FPfffz/+93//F42NjdfctjfffBOPPPIIhg0bhjFjxuDpp5/G2bNnr7kNR44cQUJCAkaMGIGRI0dizpw5+Pzzz69Zf/r0aaSmpmL06NEYNmwYfv3rX+Ott95qV++IiMj9dOo53aqqKvj4+HR4Od988w0yMzMhCILN6aIoIiEhAbt370b//v0xb9489OvXD7t27cLcuXNx5coVq3lWrVqF9evXQ5IkzJkzB+PGjcOhQ4cwffp0VFZWWtUfOHAASUlJOHXqFOLi4jBt2jRUVFQgMTERBw8etKqvqKhAfHw88vLyMH78eMyZMwdGoxFr167F+vXrO9wTIiLqerrZU/zcc89Z/J6Xl4dz585Z1RmNRpw/fx5ffPEFxo8f36ENbGlpwfLlyzFo0CD07dsX7733nlXNjh07UF5ejgULFmDp0qXy+Msvv4zt27dj9+7dWLRokTxeUFAAnU6H8PBw7Nq1C56engCA2NhYLFy4EC+88AJ27twp11+6dAnr16+Hn58fcnJyEBgYCABYsGABpk6dijVr1iAqKgre3t7yPGvWrEF9fT22bduG6OhoAEBqaioSExORlZWF2NhYjBgxokO9ISKirsWu0M3JyZF/VqlUKC8vR3l5uc1alUqF0NBQq6C215YtW3Dy5Enk5ORgx44dVtMlSYJOp4MgCEhOTraYlpycjL1790Kn01mErk6nA2AKQXPgAkB0dDQiIiKg1+tRU1ODO++8EwBw8OBBXL58GU8++aQcuAAQGBiIhIQEZGZm4qOPPsLUqVMBmD5WLi4uxujRo+XABQBPT0+kpqZizpw5yM7OZugSEf3M2BW6eXl5AExBN2HCBMydO9fmhVJqtRq33nrrNT8Obq+vv/4aW7Zswe9+9zvcfffdNmsqKytRW1uLqKgoq/UJgoCRI0dCr9fj/Pnz6N27NwCgsLBQnna1cePGoaioCEVFRXKIFhUVAQCioqKs6qOiopCZmdnu+rCwMAiCwKu6iYh+huwK3aCgIPnnDRs2YNCgQRZjztTc3Ixnn30W99xzD5KSkq5ZV1VVBQDQaDQ2p2s0Guj1elRWVqJ3794QRRF1dXXQarVQq9VW9X379rVYLgD5HK95Wkfq1Wo1goODUVFRgdbWVnTr1v7/CwwGQ7tr2zIajabXKkmw9xouR9fpisyvxZ1ekyPYBxP2wcQd+2Det9u1v/tPsdFo7PB6r8eu0G1r2rRpjs7aLhkZGaisrMT+/fuv+0Lq6+sBwOJ8alvm8YaGBrvqzXVt57V1UZi99QDg5eUFo9GIxsZG+Pr62qyxpaysrN21bfXs2RODBw+G2NQkb9uNiILpqvMTJ06gqanJofW6Kkf76G7YBxP2wcRd+iDv70Sx3fs74L/7vJMnTzq8zwsLC7thjcOha3bs2DGUlZXh8uXLNt8pqVQqpKSk2LXML7/8Ert27cKSJUug1Wo7uoluIzQ0tF3vpK5mfucm9OwJb+/2vfUzf1Q/cOBAu9fnqgwGA8rKyhzuo7tgH0zYBxN37YMgCO3e3wGm/SMADBgwAB4enXdjj8Ohe/HiRaSkpKC0tPS6953aG7qtra1Yvnw5Bg4ciIULF96w3nw0ea13NOZx8xFpe+vbHqW2PZr18/Ozq96WxsZGqFQqeHl5Xetl2aRWqzv2R6FSwd7bpt3pj9Csw310E+yDCftg4o59sGt/959iDw+PTu2Dw6H74osvoqSkBBEREZg2bRoCAwOdsqGiKMrnRIcOHWqz5je/+Q0A4LXXXsNdd90FADbvrW07bj7nKwgC/P39UV1dDYPBYLXN5nOzbc/HajQafPXVV6iqqrIK3WvVt53WlsFgQHV1NYKDg+06n0tERF2fw3v9//f//h+GDRuGP//5z0576hRguq1m+vTpNqd98cUXqKysxP3334/bbrsNQUFB0Gg0CAgIQGlpKURRtLiCWRRFlJaWIjg4WL5yGQAiIiLwwQcfoLS0FOHh4RbrOHLkCABYjIeHh+P999+HXq/H8OHDLer1er28zLb15mlXH62XlJRAFEU89NBD7W0JERG5CYc/uL5y5QpGjRrl1MAFgB49euCPf/yjzf/M97UuWrQIf/zjHzFo0CCoVCrEx8dDFEVs3rzZYlmbN2+GKIqYMWOGxbj594yMDDQ3N8vj+fn5KCoqQlRUlMVV2Q8//DB8fHyQlZWFCxcuyOMXLlxAVlYW/Pz8MGHCBHm8f//+CA8PR2FhIfLz8+Xx5uZmZGRkAADi4+M72ioiIupiHD7Sveeee2w+jepmSEpKQl5eHrZv347y8nIMHjwYx48fh16vR2hoKObOnWtRP2bMGMTHx0On0yEuLg7R0dGoq6tDbm4uevXqhRUrVljU+/r6YuXKlVi2bBmmTZuGmJgYAEBubi4uXryIjRs3Wl0NvXr1asyaNQspKSmIiYmBv78/8vPzcfLkSSQkJNi8R5iIiNybw6G7ZMkSLF68GEePHrX6yFVpgiAgKysLmZmZOHToEAoLC+Hv74/58+cjJSUFPXr0sJpn3bp10Gq1yM7Oxp49eyAIAiZOnIi0tDT06dPHqn7KlCnw8/PD1q1b5W9XGjp0KBYvXozIyEir+gEDBiA7Oxvp6enIz8+HKIrQaDRYtWoVZs+e7fwmEBGRy3M4dP/973/j3nvvRUJCAh555BEMGTLkmve+mp/U1FEvvvgiXnzxRZvTfHx88Pzzz+P5559v17I8PDzw+OOP2/XVg+PHj7frWdL9+/fHpk2b2l1PRETuzeHQXb58OVQqFSRJQk5ODnJycqzO70qSBJVK5bTQJSIi6socDt0NGzY4czuIiIjcnss+BpKIiMjddOqX2BMREdF/OXykW1NT0+5a8/fSEhER/Zw5HLr3339/ux6MoVKpcPz4cUdXQ0RE5DYcDt2pU6faDN36+nr861//QnV1NcLDwxEcHNyhDSQiInIXHfrCg2uRJAm7du3Cjh078D//8z+OroKIiMitdMqFVCqVCk888QTuvvtu/OlPf+qMVRAREXU5nXr18tChQ1FQUNCZqyAiIuoyOjV0z549i9bW1s5cBRERUZfh9G9RNxqN+O6777B//37k5eVh7Nixzl4FERFRl9Shr/a73i1DkiTB19cXzz77rKOrICIicisOh254eLjNcQ8PD/j6+mLo0KF49NFHcfvttzu8cURERO7E4dB98803nbkdREREbo/PXiYiIlKIUy6kKikpwb/+9S80NDTA29sb99xzD8LCwpyxaCIiIrfRodAtLS3Fc889hzNnzgD475fWA0Dfvn2xYcMGjBgxouNbSURE5AYcDt2TJ0/iiSeeQFNTE371q19h9OjR8Pf3R11dHQoLC/Hpp5/iiSeeQHZ2Nu6++25nbjMREVGX5HDovvbaa2hpacG2bdswfvx4i2kLFy7E4cOHkZycjNdeew0bN27s8IYSERF1dQ5fSFVUVIRJkyZZBa7Z+PHjMWnSJBQWFjq8cURERO7E4dCtr6+/4df2BQcHo76+3tFVEBERuRWHQzcgIABHjx69bs0///lPBAQEOLoKIiIit+Jw6N5///0oKipCeno6rly5YjHtypUr2LRpEwoLC/HAAw90eCOJiIjcgcMXUiUnJ+OTTz7B1q1bsW/fPgwbNgy33347vv/+e5SVleGHH35ASEgIkpOTnbm9REREXZbDoevn54d9+/bhpZdeQm5uLvLz8+Vpt9xyC+Li4rB06VL06tXLGdtJRETU5XXo4Ri33XYbNmzYgHXr1uHbb7+Vn0jVv39/dO/e3VnbSERE5BbsDt3XX38dTU1NePLJJ+Vg7d69OwYOHCjXNDc3Y+PGjfDy8sLChQudt7VERERdmF0XUn322WfYtGkTevXqdd0jWU9PT/Tq1QsbN25EQUFBhzeSiIjIHdgVuu+++y5uvfVWJCQk3LD2scceg6+vL/bv3+/wxhEREbkTu0L3yy+/RGRkJDw9PW9Y6+npicjISJSWljq8cURERO7ErtCtra1FSEhIu+uDg4NRV1dn90YRERG5I7tC18PDAy0tLe2ub2lpgYeHw8/fICIicit2JWJAQABOnjzZ7vqTJ0/yMZBERET/YVfohoWFoaCgANXV1Tesra6uRkFBAcLDwx3eOCIiIndiV+g+9thjaG1txe9+9zv88MMP16z78ccfkZqaCoPBgFmzZnV4I4mIiNyBXQ/HGDJkCObOnYs///nPmDx5MmbOnInRo0cjMDAQAPDdd9/h888/R3Z2Nn744QckJiZiyJAhnbLhREREXY3dT6Ravnw5brnlFuzcuRNbtmzBli1bLKZLkgS1Wo1FixbhqaeectZ2EhERdXl2h65KpcLTTz+N6dOn45133sGXX36Jf//73wCAO+64AyNHjkRcXBz69Onj9I0lIiLqyhz+woM+ffogLS3NmdtCRETk1ngTLRERkUIYukRERAph6BIRESmkQ19i3xmuXLmCV155BV999RWqqqpw6dIl3HrrrQgJCUF8fDx+/etfW32tYENDAzIzM3Ho0CHU1dUhICAAkyZNwpIlS+Dl5WW1DqPRiL179yI7OxtVVVUQBAGRkZFIS0u75rOljxw5gq1bt+Lrr7+GSqXCkCFDkJycjLFjx9qsP336NNLT01FQUICmpiZoNBrMnDkTs2bNgkql6nijiIioy3G5I93Gxka8/fbbUKlUuPfee5GYmIgJEyagtrYWzz//PH7729/CaDTK9aIoIiEhAbt370b//v0xb9489OvXD7t27cLcuXNx5coVq3WsWrUK69evhyRJmDNnDsaNG4dDhw5h+vTpqKystKo/cOAAkpKScOrUKcTFxWHatGmoqKhAYmIiDh48aFVfUVGB+Ph45OXlYfz48ZgzZw6MRiPWrl2L9evXO7VfRETUdbjckW6vXr3wxRdfWH19YGtrKxITE6HX63H48GHce++9AIAdO3agvLwcCxYswNKlS+X6l19+Gdu3b8fu3buxaNEiebygoAA6nQ7h4eHYtWuXvJ7Y2FgsXLgQL7zwAnbu3CnXX7p0CevXr4efnx9ycnLkB4EsWLAAU6dOxZo1axAVFQVvb295njVr1qC+vh7btm1DdHQ0ACA1NRWJiYnIyspCbGwsRowY4dzGERGRy3O5I10PDw+b39fbrVs3TJw4EQBQVVUFwPQgDp1OB0EQkJycbFGfnJwMQRCg0+ksxs2/p6amWqwnOjoaERER0Ov1qKmpkccPHjyIy5cvIyEhQQ5cAAgMDERCQgJ+/PFHfPTRR/L46dOnUVxcjNGjR8uBC5i+Xzg1NRUAkJ2dbV9TiIjILbhc6F6L0WjEkSNHAABarRYAUFlZidraWowcORKCIFjUC4KAkSNH4uzZszh//rw8XlhYKE+72rhx4wAARUVF8pj556ioKKt681h768PCwiAIAoqLi9vxiomIyN243MfLZs3Nzdi6dSskScLFixfx+eef49tvv0VcXJx88ZL5iFej0dhchkajgV6vR2VlJXr37g1RFFFXVwetVgu1Wm1V37dvX4vlApDP8ZqndaRerVYjODgYFRUVaG1tRbdu9rXfYDDYVW9mNBpNr1eSIEn2zevoOl2R+bW402tyBPtgwj6YuGMfzPt3u/Z3/ylue82Qo+u9HpcN3ZaWFrz66qvy7yqVCvPnz8czzzwjj9XX1wOAxfnUtszjDQ0NdtWb69rO6+Pj0+F6APDy8oLRaERjYyN8fX1t1lxLWVmZXfVmPXv2xODBgyE2NcnbdyOiYLrC+sSJE2hqanJova7K0T66G/bBhH0wcZc+yPs7UWz3/g747z7v5MmTDu/zwsLCbljjsqHr5eWFEydOwGg0ora2Fh9//DE2btyIo0ePYvv27dcMTncWGhrarndSVzO/cxN69oS3d/ve+pk/rh84cKDd63NVBoMBZWVlDvfRXbAPJuyDibv2QRCEdu/vANP+EQAGDBgAD4/OO/PqsqFr5uHhgcDAQMyePRt+fn546qmn8Prrr+P3v/+9fDR5rXcz5nFzQLe3vu1RatujWT8/P7vqbWlsbIRKpbJ5//CNqNXqjv1RqFSw9xZhd/ojNOtwH90E+2DCPpi4Yx/s2t/9p9jDw6NT+9BlLqQCrC9cMp83tXVvbdtx8zlfQRDg7++P6upqm+cvzOdm256PNc/b9ryto/UGgwHV1dUIDg62+3wuERF1fV0qdGtrawFADiyNRoOAgACUlpZCFEWLWlEUUVpaiuDgYPTu3Vsej4iIkKddzXx1dHh4uDxm/lmv11vVm8ciIiLaVV9SUgJRFC2WT0REPx8uF7oVFRU2T2I3NTVhw4YNACDf/6pSqRAfHw9RFLF582aL+s2bN0MURcyYMcNi3Px7RkYGmpub5fH8/HwUFRUhKioKQUFB8vjDDz8MHx8fZGVl4cKFC/L4hQsXkJWVBT8/P0yYMEEe79+/P8LDw1FYWIj8/Hx5vLm5GRkZGQCA+Ph4+5pCRERuweU+4/zwww/xxhtvICwsDEFBQfD29sZ3332Hw4cP4+LFixg1ahTmzZsn1yclJSEvLw/bt29HeXk5Bg8ejOPHj0Ov1yM0NBRz5861WP6YMWMQHx8PnU6HuLg4REdHo66uDrm5uejVqxdWrFhhUe/r64uVK1di2bJlmDZtGmJiYgAAubm5uHjxIjZu3Gh1Udfq1asxa9YspKSkICYmBv7+/sjPz8fJkyeRkJBg8x5hIiJyfy4Xuvfeey9qa2vx5Zdf4ujRoxBFEd7e3hg4cCAmT56MRx991OJ8qCAIyMrKkr/woLCwEP7+/pg/fz5SUlLQo0cPq3WsW7cOWq0W2dnZ2LNnDwRBwMSJE5GWloY+ffpY1U+ZMgV+fn7YunUr9u/fDwAYOnQoFi9ejMjISKv6AQMGIDs7G+np6cjPz4coitBoNFi1ahVmz57txG4REVFX4nKhGxoaitDQULvm8fHxwfPPP4/nn3++XfUeHh54/PHH8fjjj7d7HePHj8f48ePbXd+/f39s2rSp3fVEROT+XO6cLhERkbti6BIRESmEoUtERKQQhi4REZFCGLpEREQKYegSEREphKFLRESkEIYuERGRQhi6RERECmHoEhERKYShS0REpBCGLhERkUIYukRERAph6BIRESmEoUtERKQQhi4REZFCGLpEREQKYegSEREphKFLRESkEIYuERGRQhi6RERECmHoEhERKYShS0REpBCGLhERkUIYukRERAph6BIRESmEoUtERKQQhi4REZFCGLpEREQKYegSEREphKFLRESkEIYuERGRQhi6RERECmHoEhERKYShS0REpBCGLhERkUIYukRERAph6BIRESmEoUtERKQQhi4REZFCGLpEREQKYegSEREphKFLRESkkG43ewOu9t133+HDDz/E4cOH8e233+Lf//43fH19MXLkSCQlJeGXv/yl1TwNDQ3IzMzEoUOHUFdXh4CAAEyaNAlLliyBl5eXVb3RaMTevXuRnZ2NqqoqCIKAyMhIpKWlISQkxOZ2HTlyBFu3bsXXX38NlUqFIUOGIDk5GWPHjrVZf/r0aaSnp6OgoABNTU3QaDSYOXMmZs2aBZVK1bEmERFRl+RyR7pvvvkmNmzYgLNnz+JXv/oVEhMTERYWhry8PMycORO5ubkW9aIoIiEhAbt370b//v0xb9489OvXD7t27cLcuXNx5coVq3WsWrUK69evhyRJmDNnDsaNG4dDhw5h+vTpqKystKo/cOAAkpKScOrUKcTFxWHatGmoqKhAYmIiDh48aFVfUVGB+Ph45OXlYfz48ZgzZw6MRiPWrl2L9evXO61XRETUtbjcke6wYcPw5ptvIiIiwmL8iy++wLx587BmzRpMmDABnp6eAIAdO3agvLwcCxYswNKlS+X6l19+Gdu3b8fu3buxaNEiebygoAA6nQ7h4eHYtWuXvJzY2FgsXLgQL7zwAnbu3CnXX7p0CevXr4efnx9ycnIQGBgIAFiwYAGmTp2KNWvWICoqCt7e3vI8a9asQX19PbZt24bo6GgAQGpqKhITE5GVlYXY2FiMGDHCyZ0jIiJX53JHug8++KBV4ALAqFGjMHr0aFy6dAknTpwAAEiSBJ1OB0EQkJycbFGfnJwMQRCg0+ksxs2/p6amyoELANHR0YiIiIBer0dNTY08fvDgQVy+fBkJCQly4AJAYGAgEhIS8OOPP+Kjjz6Sx0+fPo3i4mKMHj1aDlwA8PT0RGpqKgAgOzvb7r4QEVHX53JHutfTrVs3i/+trKxEbW0toqKiIAiCRa0gCBg5ciT0ej3Onz+P3r17AwAKCwvlaVcbN24cioqKUFRUhKlTpwIAioqKAABRUVFW9VFRUcjMzGx3fVhYGARBQHFxsQOvHjAYDA7NZzQaoVarAUmCJCmzTldkfi3u9JocwT6YsA8m7tgHtVoNAPbt7/5TbDQaO7ze6+kyoVtTU4PPPvsM/v7+0Gq1AICqqioAgEajsTmPRqOBXq9HZWUlevfuDVEUUVdXB61Wa7M5ffv2tVguAPkcr3laR+rVajWCg4NRUVGB1tZW+c1De5WVldlVb9azZ08MHjwYYlMTGhoa2jWPKJgu9jpx4gSampocWq+rcrSP7oZ9MGEfTNylD/L+ThTbvb8D/rvPO3nypMP7vLCwsBvWdInQbWlpwbJly9Dc3IylS5fKgVlfXw8AFudT2zKPmxvf3npzXdt5fXx8OlwPAF5eXjAajWhsbISvr6/NmmsJDQ1t1zupq5nfuQk9e8Lbu31v/cyfHAwcONDu9bkqg8GAsrIyh/voLtgHE/bBxF37IAhCu/d3gGn/CAADBgyAh0fnnXl1+dA1Go1Yvnw5iouLMWPGDPlj3J8jtVrdsT8KlQr23q3kTn+EZh3uo5tgH0zYBxN37INd+7v/FHt4eHRqH1zuQqq2jEYjnn/+ebz//vv49a9/jbVr11pMNx9NXusjBPO4+Yi0vfVtj1JtHc06Wg8AjY2NUKlUNu8fJiIi9+ayoWs0GvHcc88hJycHsbGxePHFF60O+c3nTW3dW9t23HzOVxAE+Pv7o7q62uZFA+Zzs23Px5rnbXve1tF6g8GA6upqBAcH230+l4iIuj6XDF1z4L777ruIiYnBn/70J5uH+xqNBgEBASgtLYUoihbTRFFEaWkpgoOD5SuXASAiIkKedrUjR44AAMLDw+Ux8896vd6q3jzW9han69WXlJRAFEWL5RMR0c+Hy4Wu+SPld999Fw899BBeeumla36+rlKpEB8fD1EUsXnzZotpmzdvhiiKmDFjhsW4+feMjAw0NzfL4/n5+SgqKkJUVBSCgoLk8Ycffhg+Pj7IysrChQsX5PELFy4gKysLfn5+mDBhgjzev39/hIeHo7CwEPn5+fJ4c3MzMjIyAADx8fH2toWIiNyAy33G+dprryEnJweCIECj0eD111+3qpkwYQIGDRoEAEhKSkJeXh62b9+O8vJyDB48GMePH4der0doaCjmzp1rMe+YMWMQHx8PnU6HuLg4REdHo66uDrm5uejVqxdWrFhhUe/r64uVK1di2bJlmDZtGmJiYgAAubm5uHjxIjZu3Gh1NfTq1asxa9YspKSkICYmBv7+/sjPz8fJkyeRkJBg8x5hIiJyfy4XuufOnQNg+nh4y5YtNmuCgoLk0BUEAVlZWfIXHhQWFsLf3x/z589HSkoKevToYTX/unXroNVqkZ2djT179kAQBEycOBFpaWno06ePVf2UKVPg5+eHrVu3Yv/+/QCAoUOHYvHixYiMjLSqHzBgALKzs5Geno78/HyIogiNRoNVq1Zh9uzZDveGiIi6NpUk2fuMIlKawWDA0aNHMXz4cIcuZTcYDFCr1Vj+zjFUft/Yrnk0t3vhxUeH2b0uV9bRProL9sGEfTBx1z7Ys78D/rvPM+8vO4vLndMlIiJyVwxdIiIihTB0iYiIFMLQJSIiUghDl4iISCEMXSIiIoUwdImIiBTC0CUiIlIIQ5eIiEghDF0iIiKFMHSJiIgUwtAlIiJSCEOXiIhIIQxdIiIihTB0iYiIFMLQJSIiUghDl4iISCEMXSIiIoUwdImIiBTC0CUiIlIIQ5eIiEghDF0iIiKFMHSJiIgUwtAlIiJSCEOXiIhIIQxdIiIihTB0iYiIFMLQJSIiUghDl4iISCEMXSIiIoUwdImIiBTC0CUiIlIIQ5eIiEghDF0iIiKFMHSJiIgUwtAlIiJSCEOXiIhIIQxdIiIihTB0iYiIFMLQJSIiUghDl4iISCEMXSIiIoUwdImIiBTS7WZvgC0HDhxASUkJvvrqK3zzzTdoaWnBhg0bEBcXZ7O+oaEBmZmZOHToEOrq6hAQEIBJkyZhyZIl8PLysqo3Go3Yu3cvsrOzUVVVBUEQEBkZibS0NISEhNhcx5EjR7B161Z8/fXXUKlUGDJkCJKTkzF27Fib9adPn0Z6ejoKCgrQ1NQEjUaDmTNnYtasWVCpVI43h4iIuiyXPNLNyMjAvn37UFNTg4CAgOvWiqKIhIQE7N69G/3798e8efPQr18/7Nq1C3PnzsWVK1es5lm1ahXWr18PSZIwZ84cjBs3DocOHcL06dNRWVlpVX/gwAEkJSXh1KlTiIuLw7Rp01BRUYHExEQcPHjQqr6iogLx8fHIy8vD+PHjMWfOHBiNRqxduxbr1693uC9ERNS1ueSR7vr169G3b18EBQVh27Zt+L//+79r1u7YsQPl5eVYsGABli5dKo+//PLL2L59O3bv3o1FixbJ4wUFBdDpdAgPD8euXbvg6ekJAIiNjcXChQvxwgsvYOfOnXL9pUuXsH79evj5+SEnJweBgYEAgAULFmDq1KlYs2YNoqKi4O3tLc+zZs0a1NfXY9u2bYiOjgYApKamIjExEVlZWYiNjcWIESOc0ywiIuoyXPJINzIyEkFBQTeskyQJOp0OgiAgOTnZYlpycjIEQYBOp7MYN/+empoqBy4AREdHIyIiAnq9HjU1NfL4wYMHcfnyZSQkJMiBCwCBgYFISEjAjz/+iI8++kgeP336NIqLizF69Gg5cAHA09MTqampAIDs7Oz2tIGIiNyMSx7ptldlZSVqa2sRFRUFQRAspgmCgJEjR0Kv1+P8+fPo3bs3AKCwsFCedrVx48ahqKgIRUVFmDp1KgCgqKgIABAVFWVVHxUVhczMzHbXh4WFQRAEFBcXO/R6DQaDQ/MZjUao1WpAkiBJyqzTFZlfizu9JkewDybsg4k79kGtVgOAffu7/xQbjcYOr/d6unToVlVVAQA0Go3N6RqNBnq9HpWVlejduzdEUURdXR20Wq3N5vTt29diuQDkc7zmaR2pV6vVCA4ORkVFBVpbW9Gtm33tLysrs6verGfPnhg8eDDEpiY0NDS0ax5RMF3sdeLECTQ1NTm0XlflaB/dDftgwj6YuEsf5P2dKLZ7fwf8d5938uRJh/d5YWFhN6zp0qFbX18PABbnU9syj5sb3956c13beX18fDpcDwBeXl4wGo1obGyEr6+vzZprCQ0Nbdc7qauZ37kJPXvC27t9b/3MnxwMHDjQ7vW5KoPBgLKyMof76C7YBxP2wcRd+yAIQrv3d4Bp/wgAAwYMgIdH55157dKh+3OjVqs79kehUsHeu5Xc6Y/QrMN9dBPsgwn7YOKOfbBrf/efYg8Pj07tg0teSNVe5qPJa32EYB43H5G2t77tUaqto1lH6wGgsbERKpXK5v3DRETk3rp06JrPm9q6t7btuPmcryAI8Pf3R3V1tc2LBsznZtuejzXP2/a8raP1BoMB1dXVCA4Otvt8LhERdX1dOnQ1Gg0CAgJQWloKURQtpomiiNLSUgQHB8tXLgNARESEPO1qR44cAQCEh4fLY+af9Xq9Vb15LCIiol31JSUlEEXRYvlERPTz0aVDV6VSIT4+HqIoYvPmzRbTNm/eDFEUMWPGDItx8+8ZGRlobm6Wx/Pz81FUVISoqCiLe4Qffvhh+Pj4ICsrCxcuXJDHL1y4gKysLPj5+WHChAnyeP/+/REeHo7CwkLk5+fL483NzcjIyAAAxMfHO+HVExFRV+OSn3HqdDqUlJQAAL755ht5zHwPbFhYmBxcSUlJyMvLw/bt21FeXo7Bgwfj+PHj0Ov1CA0Nxdy5cy2WPWbMGMTHx0On0yEuLg7R0dGoq6tDbm4uevXqhRUrVljU+/r6YuXKlVi2bBmmTZuGmJgYAEBubi4uXryIjRs3Wl0NvXr1asyaNQspKSmIiYmBv78/8vPzcfLkSSQkJNi8R5iIiNyfS4ZuSUkJcnJyLMZKS0stPhI2h64gCMjKypK/8KCwsBD+/v6YP38+UlJS0KNHD6vlr1u3DlqtFtnZ2dizZw8EQcDEiRORlpaGPn36WNVPmTIFfn5+2Lp1K/bv3w8AGDp0KBYvXozIyEir+gEDBiA7Oxvp6enIz8+HKIrQaDRYtWoVZs+e3aHeEBFR16WSJHufUURKMxgMOHr0KIYPH+7QpewGgwFqtRrL3zmGyu8b2zWP5nYvvPjoMLvX5co62kd3wT6YsA8m7toHe/Z3wH/3eeb9ZWfp0ud0iYiIuhKGLhERkUIYukRERAph6BIRESmEoUtERKQQhi4REZFCGLpEREQKYegSEREphKFLRESkEIYuERGRQhi6RERECmHoEhERKYShS0REpBCGLhERkUIYukRERAph6BIRESmEoUtERKQQhi4REZFCGLpEREQKYegSEREphKFLRESkEIYuERGRQhi6RERECmHoEhERKYShS0REpBCGLhERkUIYukRERAph6BIRESmEoUtERKQQhi4REZFCGLpEREQKYegSEREphKFLRESkEIYuERGRQhi6RERECmHoEhERKYShS0REpBCGLhERkUIYukRERAph6BIRESmEoUtERKQQhi4REZFCGLpEREQKYeh2omPHjmHBggUYNWoUhg8fjhkzZiA3N/dmbxYREd0k3W72BrirgoICJCUlwdPTE5MnT4aXlxcOHTqEtLQ0XLhwAfPnz7/Zm0hERApj6HaC1tZWrFy5EiqVCnv37sWgQYMAACkpKZg+fTpeeeUVTJo0CUFBQTd5S4mISEn8eLkTFBQU4MyZM4iNjZUDFwB8fHzw29/+Fi0tLcjJybmJW0hERDcDj3Q7QVFREQAgKirKapp5rLi4uN3LkyQJANDc3Ay1Wm339hiNRgBAn163oJtKatc8d/r2gMFggMFgsHt9rspoNOKWW25BS0uLW70ue7EPJuyDiTv2Qa1Wo0+vHu3e3wHAnb63wGAwdKgParUaHh4eUKlU16xRSeY9OjnN7373O/z973/HO++8g6FDh1pNHzFiBHx9ffHJJ5+0a3nNzc0oKytz8lYSEZGzDR8+/LoHRzzS7QQNDQ0ATB8n2+Lt7Y36+vp2L69bt24IDQ294TsoIiK6uTw8rn/WlqHbBXh4eMDT0/NmbwYREXUQL6TqBN7e3gBwzaPZhoaGax4FExGR+2LodgKNRgMAqKqqsppWV1cHURTRt29fhbeKiIhuNoZuJwgPDwcA6PV6q2nmMXMNERH9fDB0O8HYsWMREhKC999/H+Xl5fJ4fX09tmzZgu7du2Pq1Kk3bwOJiOim4C1DneRaj4E8d+4cnn32WT4GkojoZ4ih24mOHTuGTZs24csvv0Rrayu0Wi0SExMRExNzszeNiIhuAoYuERGRQnhOl4iISCEMXSIiIoXwiVRd1LFjx5CZmWlxvnjevHl2nS9ubm7Gtm3b8N577+H8+fPw9fXFfffdh6eeegq33357J26983SkD5Ik4fDhw/j4449RWlqKmpoatLa2om/fvoiJiUFiYiJuueUWBV5Fxznj30Nbly5dQmxsLGpraxEVFYWdO3c6eYs7h7P68P3332Pr1q345JNPcP78eQiCAI1GgylTpmD27NmdtPXO4YwefPfdd9i+fTs+++wz1NTUQBAE9O3bF7/5zW/wyCOPOPTFK0o6cOAASkpK8NVXX+Gbb75BS0sLNmzYgLi4OLuWYzQasXfvXmRnZ6OqqgqCICAyMhJpaWkICQlxaNt4TrcLcsaV0UajEQsWLIBer8fw4cMRHh6Oqqoq/OMf/0BwcDCys7Nx2223KfBqHNfRPly5cgXDhg2Dp6cnIiIioNVq0dzcDL1ej8rKSoSGhuLNN99Ez549FXpFjumMK+WfeeYZfPzxxxBFscuErrP6UF5ejvnz5+Py5cuIjo7GXXfdBVEUcerUKXTv3h3bt2/v5FfiOGf04OzZs4iPj8fFixcRFRWFgQMHoqGhAXl5eairq0NcXBw2bNigwKtx3P33349z587Bz88PgiDg3LlzDoXuihUroNPpMGDAAERHR6O2thYffvghvLy8sG/fPvlBSHaRqEtpaWmRJkyYIA0dOlQ6fvy4PH758mXpwQcflIYMGSJVV1ffcDl//etfJa1WKz399NOS0WiUx9966y1Jq9VKK1eu7JTtdxZn9KG5uVnavHmzdPHiRavxRYsWSVqtVtq+fXunbL+zOOvfQ1sHDx6UtFqtlJWVJWm1Wmn+/PnO3mync1Yf6uvrpXvvvVcaM2aMVF5ebnM9rspZPVi9erWk1Wql3bt3W4xfunRJuvfeeyWtVmv3vymlffrpp/I2bt26VdJqtdI777xj1zI+//xzSavVSo899ph05coVefyTTz7p0N8Fz+l2MQUFBThz5gxiY2MxaNAgedzHxwe//e1v0dLSgpycnBsuR6fTAQCefvppi28umjlzJkJCQvC3v/0NP/30k/NfgJM4ow/du3fH4sWL4evrazW+aNEiAPZ97/HN4Kx/D2Y//PAD1qxZgylTpiA6OrozNrlTOKsPb731FmpqavDMM8/gnnvusZrerZvrnpFzVg/Onj0LAFb//996660YOXIkAODHH3904pY7X2RkJIKCgjq0DPM+MjU11eILZ6KjoxEREQG9Xo+amhq7l8vQ7WKKiooAAFFRUVbTzGM3CoorV67gn//8J/r162f1D1OlUiEyMhKiKOKrr75y0lY7nzP6cD3mnaurn7tydh9Wr14NtVqNP/zhD87ZQIU4qw+5ublQqVSYNGkSvv32W7z55pvYvn078vLy0Nzc7NyNdjJn9UCr1QIA8vPzLcYvX76ML7/8Ev7+/rj77rs7urkur7CwEIIgyG802ho3bhyA//bcHq77to1sqqysBACbX5jg7+8PQRBsftFCW2fOnIHRaLzm+QjzeGVlJUaNGtWRze00zujD9bzzzjsAgF/96lcOL0MJzuzDgQMHcOjQIbz22mvw9fW16zufbzZn9KG5uRnffPMNbrvtNrz55pvIzMyE0WiUp4eEhOC1117DwIEDnbrtzuKsfwtPPPEEPv74Y2zYsAFHjhyxOKfbo0cPvPrqq+jRo4ezN9+liKKIuro6aLVam2+8zT12ZB/DI90upqGhAQCu+dWA3t7eN9xZmqebv4LQ1jLarssVOaMP15Kfn499+/bhrrvuQnx8vMPbqARn9eG7777DH//4R8TGxmLChAlO3UYlOKMPly5dgsFgwMWLF7F582b8/ve/x2effYbDhw8jOTkZ1dXVWLx4Ma5cueL07XcGZ/1buOOOO7Bv3z6MGzcOR44cwY4dO/CXv/wF9fX1mDp1qs2P3d1Ne/eRjuxjGLpEbRw7dgxpaWnw8fFBRkaGxbkcd7ZixQp069aty32s7Ezmo1qDwYBZs2Zh/vz5uP322/GLX/wCqampeOihh3Du3DkcPHjwJm9p56qqqsKsWbPwww8/YO/evSgtLUV+fj5SUlKwefNmzJs3DwaD4WZvZpfF0O1ibvQOq6Gh4ZrvdM3M0691JGsev9a7PFfgjD5craysDE888QQ8PDywY8cODBgwoMPb2dmc0YecnBwcPnwYq1atcvnbxK7FmX8XgOmWk6uZx1z1Wgdn/U0sX74cNTU12LJlC0aNGgUvLy8EBgZi4cKFSEhIwJdffokPPvjAqdvuatq7j7R3HwMwdLsc8/lWW+cS6urqIIqizXM6bYWEhMDDw0M+B3Q187hD96ApxBl9aKusrAzz58+H0WjEzp07MWzYMGdtaqdyRh+OHz8OwHSV5sCBA+X/HnjgAQCm74AeOHAgpkyZ4tyNdyJn9EEQBPziF78AYLpS92rmMVf9eNkZPWhoaEBpaSnuuusu+Pv7W00fPXo0AFh8Zak7EgQB/v7+qK6utnlUb+6xPfsYM4ZuFxMeHg7AtCO8mnnMXHMtPXr0wLBhw3D69GmcO3fOYpokSfjss88gCAKGDh3qpK12Pmf0wcwcuAaDATt27MAvf/lL521oJ3NGH0aMGIHp06db/Wd+glFgYCCmT5+OiRMnOnnrncdZ/x7GjBkDAKioqLCaZh7r6K0oncUZPWhpaQFw7VuCfvjhBwD4WZx2iYiIgCiKKC0ttZp25MgRAO3fx1hw6O5eumlaWlqkBx544Lo3wJ89e1Ye/+6776SKigrp8uXLFstxh4djOKMPZWVl0qhRo6Thw4dLX3zxhWLb7yzO6oMtZ8+e7VIPx3BGH0pKSiStVitNnjxZunTpkjxeW1srjRs3Trrnnnukb7/9tvNfkAOc1YNJkyZJWq1Wys7Othi/dOmS9NBDD0larVb69NNPO/fFONGNHo7x/fffSxUVFdL3339vMd5ZD8fgYyC7IHse9bZ8+XLk5ORYPQLN1mMgz5w5g0OHDiEoKAg6nc7lz+91tA8XL17Egw8+iEuXLmHcuHE2j3B9fHwwb948pV6SQ5zx78GW6upqPPDAA275GMjr9eHFF1/EG2+8gd69e+O+++5Da2sr8vLy8P333+Ppp5+WH5ziipzRg/z8fCQnJ6O1tRVjx47FoEGDcPnyZXz88cf44YcfMGnSJGzatOlmvLx20+l0KCkpAQB88803+PrrrzFy5Ej54+CwsDD5zoTMzEy8+uqrWLJkCZ588kmL5Vz9GMi6ujrk5ubCy8sLf/nLX9CvXz+7t4336XZBY8aMwVtvvYVNmzYhNzdXfqj50qVL2/1Qcw8PD7z++uvYtm0bDhw4gN27d6NXr16YPn06nnrqKZcPXKDjfWhoaMClS5cAmD4uMn9k1FZQUJDLh64z/j24A2f1Yfny5dBqtdi7dy9ycnKgUqkwaNAgrF271qU/Ygec04Po6Gi8/fbb2LlzJ0pKSlBcXAxPT0/cddddSElJwaxZszr5VXRcSUmJ1dO3SktLLT4qbs/tgOvWrYNWq0V2djb27NkDQRAwceJEpKWloU+fPg5tG490iYiIFMILqYiIiBTC0CUiIlIIQ5eIiEghDF0iIiKFMHSJiIgUwtAlIiJSCEOXiIhIIQxdIiIihTB0iYiIFMLQJSIiUghDl4iISCEMXSIiIoX8f6KWAU1i0ulkAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 500x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X = data.content.apply(tokenizer).to_list()\n",
    "y = (data.score >= 4).to_numpy().astype(int)\n",
    "_ = sns.displot(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "694fc868",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkkAAAG6CAYAAAAcbtIeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA2CUlEQVR4nO3df1zV9f3///vhAAocf5VQDUkqB6JCmT9QP2yZ+S6ntpllaa5Sw7Ll3mZvC9zmVhd10iz3RrdmzcpVmuUuKe85M2eulqujBDRJSUZNSNQgrYSwhMP5/uH3nDycJwiHA+cAt+vl0qXO8/U8r9fjdZ6I916v53k9LU6n0ykAAAB4CAl0AQAAAMGIkAQAAGBASAIAADAgJAEAABgQkgAAAAwISQAAAAaEJAAAAIPQQBfQUdXX16uurk4hISGyWCyBLgcAADSD0+lUfX29QkNDFRLS9LUiQpKP6urqVFhYGOgyAACAD5KTkxUeHt5kH0KSj1zpMzk5WVar1W/7dTgcKiws9Pt+4R+MT/BibIIXYxPcutr4uM73fFeRJEKSz1y32KxWa5v8ULXVfuEfjE/wYmyCF2MT3Lra+DRnqgwTtwEAAAwISQAAAAaEJAAAAANCEgAAgAEhCQAAwICQBAAAYEBIAgAAMCAkAQAAGBCSAAAADAhJAAAABoQkAAAAA0ISAACAASEJAADAgJAEAABgQEgKQmFhYZLFc2gc9c4AVQMAQNcUGugC4C00NFTWEIsWbCpQSUW1BsTYlD19aKDLAgCgSyEkBbGSimodOHoq0GUAANAlcbsNAADAgJAEAABgQEgCAAAwICQBAAAYEJIAAAAMCEkAAAAGhCQAAAADQhIAAIABIQkAAMCAkAQAAGBASAIAADAgJAEAABgQkgAAAAwISQAAAAaEJAAAAANCEgAAgAEhCQAAwICQBAAAYEBIAgAAMCAkAQAAGBCSAAAADAhJHUC0rZsc9U6vdlMbAADwj9BAF4Dz6xkRKmuIRQs2FaikolqSNCDGpuzpQwNcGQAAnRchqQMpqajWgaOnAl0GAABdArfbAAAADAhJAAAABoQkAAAAA0ISAACAASEJAADAgJAEAABgQEgCAAAwICQBAAAYEJI6KNNSJSxTAgCA//DE7Q6q4VIlLFMCAIB/EZI6OJYqAQCgbXC7TdKvfvUrJSYm6sUXXwx0KQAAIEh0+ZD097//Xe+//75iYmICXQoAAAgiXTokffbZZ3rkkUf0m9/8RmFhYYEuBwAABJGgnZOUk5OjvLw8ffDBByouLlZtba1WrFihqVOnNvqe/fv3a82aNSooKFBdXZ0SEhI0a9YsTZw40dh/8eLFuuOOO5SYmNhWpwEAADqooA1J2dnZKi8vV58+fRQTE6Py8vIm+9vtdqWnpys8PFyTJk1SVFSUdu7cqYULF+r48eOaM2eOR/8XX3xRp0+f9moHAACQgjgkLVu2TP3791dsbKyefvppPfHEE432raur05IlS2SxWLRhwwYlJSVJku6//37dcsstWrVqlW644QbFxsZKkj766CM9+eSTeuWVVxQS0qXvOAIAgEYEbUIYM2aMO9Scj91uV1lZmSZPnuwOSJLUo0cPzZs3T7W1tdqyZYu7/V//+pdOnjyp66+/XoMGDdKgQYNUXl6u5cuX60c/+pHfzwUAAHQ8QXslqSX27dsnSUpLS/Pa5mrLzc11t40fP15Dhgzx6Hf33Xdr6tSpTc55MnE4HC0tt0335+964Mn1+fI5Bx/GJngxNsGtq41PS86zU4Skw4cPS5L69+/vtS06OlqRkZEqLS11t/Xs2VM9e/b06BcWFqbo6GjjPppSWFjY8oLPIyIiwuf3Hjp0SKdPn/ZjNTBpi3GHfzA2wYuxCW6Mj7dOEZKqq6slnb29ZmKz2VRVVdUmx05OTpbVavXb/hwOh0pKSnx+P9/Ua1sOh0OFhYV+H3e0HmMTvBib4NbVxsd1vs3RKUKSP+zevdun91mt1qD6oQqmWjqzYBt3fIuxCV6MTXBjfLwF7cTtlrDZbJLU6NWi6urqRq8ydRbRtm5y1Du92k1tAADg/DrFlaT4+HhJUmlpqdeE7MrKStXU1CglJSUAlbWfnhGhsoZYtGBTgUoqzt5+HBBjU/b0oQGuDACAjqlTXEkaMWKEJGnPnj1e21xtrj6dXUlFtQ4cPaUDR0+5wxIAAGi5ThGSRo8erbi4OG3btk1FRUXu9qqqKq1du1ZhYWGaMmVK4AoEAAAdTtDebtu8ebPy8vIkScXFxe421zORhg0bpmnTpkmSQkNDtWzZMqWnp2vmzJkey5KUl5crIyND/fr1C8yJAACADiloQ1JeXp7HU7IlKT8/X/n5+e7XrpAkSaNGjdLGjRu1evVqbd++3b3A7aJFixpd4BYAAKAxQRuSsrKylJWV1aL3pKSkaN26dW1UUcfj+sabNcTibmv4GgAAmAVtSELrNfzGG992AwCg+QhJXYDrG28AAKD5OsW32wAAAPyNkAQAAGBASAIAADAgJAEAABgQkgAAAAwISQAAAAaEpC7E9XDJhkxtAAB0dTwnqQtp+HBJSTxgEgCARhCSuiAeLgkAwPlxuw0AAMCAkAQAAGBASAIAADAgJAEAABgQkgAAAAwISQAAAAaEJAAAAANCUhdnego3T+AGAICHSXZ5DZ/CzRO4AQA4i5AESTyFGwCAhrjdBgAAYEBIAgAAMCAkAQAAGBCSAAAADAhJAAAABoQkAAAAA0ISAACAASEJAADAgJAEAABgQEgCAAAwICTBg2nBW4lFbwEAXQ9rt8FDwwVvJbHoLQCgSyIkwejcBW9dV5esIRb39oavAQDobAhJOK+GV5e4sgQA6AoISWi2c68uNYfpahNXoAAAHQUhCW2GuU0AgI6MkIQ21dKrTwAABAseAQAAAGBASAIAADAgJAEAABgQktBuTE/z5kneAIBgxcRttJjp4ZLS+b/ez/OWAAAdCSEJLdbapUv4xhsAoCMgJMFn51u6BACAjoyQBL9oeHVpbGK0HrphYKDLAgDAZ0zchl+5ri59crIm0KUAANAqhCQAAAADQhIAAIABIQkAAMCAkAQAAGBASAIAADAgJAEAABgQkgAAAAwISQAAAAZdNiStX79ekyZN0tChQzV8+HDdeeed+te//hXosgAAQJDosiHpO9/5jjIzM5WTk6NNmzapf//+uvvuu/X5558HujQAABAEumxIuv766/W9731Pl156qQYMGKCMjAxVVVXp3//+d6BLAwAAQSBoF7jNyclRXl6ePvjgAxUXF6u2tlYrVqzQ1KlTG33P/v37tWbNGhUUFKiurk4JCQmaNWuWJk6c2OSxzpw5o5dfflm9evVSQkKCv08FAAB0QEEbkrKzs1VeXq4+ffooJiZG5eXlTfa32+1KT09XeHi4Jk2apKioKO3cuVMLFy7U8ePHNWfOHK/3vPfee5o7d66+/vpr9e3bV88++6x69+7dRmcEAAA6kqC93bZs2TLt3r1bdrtd06dPb7JvXV2dlixZIovFog0bNmjp0qXu+Ubx8fFatWqVMWQNGTJEW7du1aZNm/T9739fDzzwgE6ePNlWpwQAADqQoA1JY8aMUWxsbLP62u12lZWVafLkyUpKSnK39+jRQ/PmzVNtba22bNni9b7u3burf//+uvLKK7V8+XKFhITo1Vdf9ds5AACAjitob7e1xL59+yRJaWlpXttcbbm5uefdj9Pp1JkzZ1p0bIfD0aL+7b2/jqAjnbOr1o5Uc1fB2AQvxia4dbXxacl5doqQdPjwYUlS//79vbZFR0crMjJSpaWlHu0rV67UuHHjdPHFF+vLL7/Uxo0bdfz4cd1www0tOnZhYaHPdTcmIiLC7/sMZocOHdLp06cDXUaLtMW4wz8Ym+DF2AQ3xsdbpwhJ1dXVks7eXjOx2WyqqqryaKuoqNCDDz6oEydOqHfv3hoyZIg2bNigK664okXHTk5OltVq9a1wA4fDoZKSEr/tryNITEwMdAnN5nA4VFhY6PdxR+sxNsGLsQluXW18XOfbHJ0iJPli5cqVftmP1WrtEj9Ubakjfn6Me/BibIIXYxPcGB9vQTtxuyVsNpskeV0tcqmurm70KhMAAIBJpwhJ8fHxkuQ170iSKisrVVNTY5yvBAAA0JhOEZJGjBghSdqzZ4/XNlebqw8AAEBzdIqQNHr0aMXFxWnbtm0qKipyt1dVVWnt2rUKCwvTlClTAlcgAADocIJ24vbmzZuVl5cnSSouLna3uZ6JNGzYME2bNk2SFBoaqmXLlik9PV0zZ870WJakvLxcGRkZ6tevX2BOBAAAdEhBG5Ly8vK8npKdn5+v/Px892tXSJKkUaNGaePGjVq9erW2b9/uXuB20aJF513gFgAAoKGgDUlZWVnKyspq0XtSUlK0bt26NqoIAAB0JZ1iThIAAIC/EZIAAAAMCEkImGhbNznqnV7tpjYAANpb0M5JQufXMyJU1hCLFmwqUEnF2fX3BsTYlD19aIArAwCAkIQgUFJRrQNHTwW6DAAAPHC7DQAAwICQBAAAYEBIAgAAMCAkAQAAGBCSAAAADAhJAAAABoQkAAAAA0ISAACAASEJAADAgJAEAABg4HNI2rp1qz788MMm+xQXF2vr1q2+HgIAACBgfA5JmZmZ2rVrV5N93njjDS1evNjXQwAAAARMm95uczgcCgnhjh4AAOh42jTBFBUVqVevXm15CAAAgDYR2pLOd955p8frLVu2aN++fV796uvrdfz4cZWXl+sHP/hB6yoEAAAIgBaFpHMDkcViUXl5ucrLy736hYSEqFevXpowYYJ+9rOftb5KAACAdtaikHTut9kGDhyo+fPna/78+X4vCgAAINBaFJLO9fzzzys2NtaftQAAAAQNn0PSyJEj/VkHAABAUPE5JEnSmTNntGvXLhUWFqqqqkoOh8Orj8Vi0a9//evWHAYAAKDd+RySysvLNWfOHJWVlcnpdDbaj5AEAAA6Ip9D0ooVK1RaWqof/ehHuvnmm3XxxRfLarX6szYAAICA8Tkk2e12jR49Wo899pg/6wEAAAgKPj9xu76+XklJSf6sBQAAIGj4HJKuvPJKffzxx/6sBQAAIGj4HJL+53/+R3a7XTt27PBnPQAAAEHB5zlJb775plJTU7Vw4UJt3LhRgwcPVlRUlFc/i8Wi+++/v1VFAgAAtDefQ9Lvfvc793/v27fPuNCtREgCAAAdU6uWJQEAAOisWJYEAADAwOeJ2wAAAJ2Zz1eScnNzm913xIgRvh4GAAAgIHwOSXfccYcsFkuz+hYVFfl6GAAAgIDwOSTdf//9xpBUVVWlgwcPKjc3V2PHjtWQIUNaVSAAAEAg+BySfvrTnza5fceOHVq8ePF5+wEAAASjNpu4PWHCBKWmpmrVqlVtdQgAAIA206bfbrv88stVUFDQlocAAABoE20akoqKihQSwlMGAABAx+PznKSjR48a2x0Ohz799FO9+uqrstvtGj9+vM/FAQAABIrPIWncuHFNPgLA6XTq0ksv1eLFi309BAAAQMD4HJKmTJliDEkWi0W9evVScnKyrrvuOnXr1q1VBQIAAASCzyEpKyvLn3UAAAAEFWZVAwAAGPh8JcmlpqZGu3btUlFRkb766itFRUUpKSlJ48ePV2RkpD9qBAAAaHetCkmvv/66fvnLX+rUqVNyOp3udovFop49e2rp0qW6/vrrW10kuo5oWzc56p2yhnw7363hawAA2oPPISk/P18PPvigQkJCNG3aNKWmpio6OlqfffaZ7Ha7tm7dqgcffFAvvPCChg4d6s+a0Yn1jAiVNcSiBZsKVFJRrQExNmVP5+cHAND+fA5JTz31lMLDw/XSSy9p4MCBHtsmTpyo22+/XTNmzNBTTz2ltWvXtrpQdC0lFdU6cPRUoMsAAHRhPk/cfv/99/WDH/zAKyC5DBw4UBMmTGBZEgAA0CH5HJJOnz6tvn37Ntmnb9++On36tK+HAAAACBifQ1JsbKz++c9/Ntnn3XffVWxsrK+HaFNPPfWUpk6dqqFDh2rMmDFasGCBjhw5EuiyAABAkPA5JP3gBz/QgQMHlJGRoU8//dRjW0VFhTIzM3XgwAFNnDix1UW2hX379umOO+7Q5s2btW7dOn355ZeaO3eu6urqAl0aAAAIAj5P3J47d67efvtt5eTkaPv27erfv78uvPBCnThxQqWlpaqtrVVKSormzp3rz3r95plnnvF4vXTpUo0fP14lJSWNzrMCAABdh88hKSIiQhs2bNDTTz+tnJwclZSUqKSkRJIUFxenKVOmaO7cuQoPD/dp/zk5OcrLy9MHH3yg4uJi1dbWasWKFZo6dWqj79m/f7/WrFmjgoIC1dXVKSEhQbNmzWrW1azq6mpJUu/evX2qFwAAdC6tephkeHi45s+fr/nz56u6utr9xG2bzdbqwrKzs1VeXq4+ffooJiZG5eXlTfa32+1KT09XeHi4Jk2apKioKO3cuVMLFy7U8ePHNWfOnEbf63A49Nhjj+maa67RxRdf3OraAQBAx+fznKS8vDytWLFClZWVkiSbzaaLLrrIHZAqKiq0YsUKvf/++z7tf9myZdq9e7fsdrumT5/eZN+6ujotWbJEFotFGzZs0NKlS5WZmamcnBzFx8dr1apVjYYsp9OpX/3qVzp27BiL9gIAADefQ9L69ev197//XdHR0cbtMTExevPNN7V+/Xqf9j9mzJhmfzPObrerrKxMkydPVlJSkru9R48emjdvnmpra7Vlyxav9zmdTj3yyCN65513tH79el1wwQU+1QoAADofn2+3FRYWavTo0U32GT58uN555x1fD9Fs+/btkySlpaV5bXO15ebmerQ7nU49+uijeuutt/TCCy/okksu8enYDofDp/e11/46i2D5XFx1BEs9+BZjE7wYm+DW1canJefpc0g6ceKEYmJimuzTt29fnThxwtdDNNvhw4clSf379/faFh0drcjISJWWlnq0P/roo/rrX/+qP/zhD+revbv7tmGvXr1aNNm8sLDQ98IbERER4fd9dnSHDh0KqgeTtsW4wz8Ym+DF2AQ3xsebzyGpZ8+eOnbsWJN9jh49qsjISF8P0Wyub6b16NHDuN1ms6mqqsqj7aWXXpIkzZw506P9+eefV2pqarOPnZycLKvV2pJym+RwONzfEsS3EhMTA12CpLPjU1hY6PdxR+sxNsGLsQluXW18XOfbHD6HpCuvvFJ/+9vfdOzYMeOtqqNHj2rXrl0aNWqUr4doU4cOHfLLfqxWa5f4oQq0YPuMGffgxdgEL8YmuDE+3nyeuD179mx9/fXXmjFjhrZu3aqKigpJZ7/VtmXLFs2YMUPffPNNk1+99xfXN+oaXi1yqa6ubvQqEwAAgInPV5JGjBihzMxMPfbYY1q8eLEkyWKxyOl0SpJCQkL085//XCNGjPBPpU2Ij4+XJJWWlmrIkCEe2yorK1VTU6OUlJQ2rwMAAHQePl9JkqS77rpLr776qm677TYNGjRIcXFxGjx4sGbMmKEtW7Z4zfdpK64gtmfPHq9trrb2CGsAAKDzaNUTtyVp4MCBeuSRR/xQiu9Gjx6tuLg4bdu2TXfeeaf7WUlVVVVau3atwsLCNGXKlIDWCAAAOpZWh6S2snnzZuXl5UmSiouL3W2uZyINGzZM06ZNkySFhoZq2bJlSk9P18yZMz2WJSkvL1dGRob69esXmBNBq0TbuslR75Q1xOLRbmoDAMCfgjYk5eXleT0lOz8/X/n5+e7XrpAkSaNGjdLGjRu1evVqbd++3b3A7aJFi5q1wC2CU8+IUFlDLFqwqUAlFWcf9TAgxqbs6UMDXBkAoLML2pCUlZXV4rXUUlJStG7dujaqCIFUUlGtA0dPBboMAEAX0qqJ2wAAAJ0VIQkdjmue0rkavgYAoLWC9nYb0JiG85SYowQAaAuEJHRYzFMCALQlbrcBAAAYEJIAAAAMCEkAAAAGhCQAAAADQhIAAIABIQkAAMCAkAQAAGBASAIAADAgJAEAABgQkgAAAAwISQAAAAaEJAAAAANCEgAAgAEhCQAAwICQBAAAYEBIQocXbesmR73Tq93UBgBAc4UGugCgtXpGhMoaYtGCTQUqqaiWJA2IsSl7+tAAVwYA6MgISeg0SiqqdeDoqUCXAQDoJLjdBgAAYEBIAgAAMCAkoVMyTeZmIjcAoCWYk4ROqeFkbiZyAwBaipCETo3J3AAAX3G7DQAAwICQBAAAYEBIAgAAMCAkoUto7tIlLG8CAHBh4ja6hOYuXcLyJgAAF0ISupTmfNuNb8QBACRutwEAABgRkgAAAAwISQAAAAaEJAAAAANCEgAAgAEhCQAAwICQBAAAYEBIAgAAMCAkAQAAGBCS0GU1tp4bAAASy5KgC2u4ntvYxGg9dMPA874vLCysHaoDAAQaV5LQ5bnWavvkZI3XtoZXm6xWq5IGDW7P8gAAAcKVJKAJDa82DYixKXv6UDkcjkCXBgBoY4QkoBlcV5sAAF0Ht9sAAAAMCEkAAAAGhCQAAAADQhLQTho+k8n0jKbm9AEAtA8mbgPtxPQtOV/6AADaByEJaAHXc5OsVqtHu6PeKWuI5bzvb8635PgmHQAEB0IS0AINn5skiSs+ANBJEZIAH3C1BwA6vy47cXvnzp2aPXu2Ro4cqcTERB05ciTQJQEAgCDSZUNSTU2Nhg8frv/+7/8OdCkAACAIddnbbVOmTJEkFRcXB7YQdHjuydznTNxu7kRuAEDwCtqQlJOTo7y8PH3wwQcqLi5WbW2tVqxYoalTpzb6nv3792vNmjUqKChQXV2dEhISNGvWLE2cOLEdK0dX03Ay99jEaD10w0CPyd2uNgBAxxG0ISk7O1vl5eXq06ePYmJiVF5e3mR/u92u9PR0hYeHa9KkSYqKitLOnTu1cOFCHT9+XHPmzGmnytFVuSZzXxEd5fFakrsNANBxBO2cpGXLlmn37t2y2+2aPn16k33r6uq0ZMkSWSwWbdiwQUuXLlVmZqZycnIUHx+vVatWnTdkAe3JdYsOABC8gjYkjRkzRrGxsc3qa7fbVVZWpsmTJyspKcnd3qNHD82bN0+1tbXasmVLW5UKtNi5t+gmrX5bk1a/rZWvfxjosgAA5wja220tsW/fPklSWlqa1zZXW25ubpsc2+FwBPX+ENyac0uOn4nmcX1OfF7Bh7EJbl1tfFpynp0iJB0+fFiS1L9/f69t0dHRioyMVGlpqUf7F198oWPHjqmsrEyS9NFHH6mqqkqXXHKJevfu3exjFxYW+lx3YyIiIvy+T3Rchw4d0unTpwNdRofRFn8m4R+MTXBjfLx1ipBUXX32G0Q9evQwbrfZbKqqqvJo2717txYvXux+fc8990jSeb9B11BycrLXOl6t4XA4VFJS4rf9oeNLTEwMdAkdgsPhUGFhod//TKL1GJvg1tXGx3W+zdEpQpIvpk6d2qIw1Bir1dolfqgQOPx8tQx/JoMXYxPcGB9vQTtxuyVsNpskeV0tcqmurm70KhMAAIBJpwhJ8fHxkuQ170iSKisrVVNTY5yvBAAA0JhOEZJGjBghSdqzZ4/XNlebqw8AAEBzdIqQNHr0aMXFxWnbtm0qKipyt1dVVWnt2rUKCwtzr9UGAADQHEE7cXvz5s3Ky8uT9O0itJs3b3Y/E2nYsGGaNm2aJCk0NFTLli1Tenq6Zs6c6bEsSXl5uTIyMtSvX7/AnAjgI9PCuRKL5wJAewnakJSXl+f1lOz8/Hzl5+e7X7tCkiSNGjVKGzdu1OrVq7V9+3b3AreLFi1igVt0SA0XzpWkATE2ZU8fGuDKAKBrCNqQlJWVpaysrBa9JyUlRevWrWujioDAOPep3ACA9tMp5iQBAAD4GyEJ6EBc85TO1fA1AMA/gvZ2GwBvDecpMUcJANoOIQnogJinBABtj9ttAAAABoQkAAAAA0ISAACAASEJAADAgJAEAABgQEgCAAAwICQBAAAYEJIAAAAMCEkAAAAGhCQAAAADQhIAAIABIQkAAMCAkAQAAGBASAIAADAgJAFdlKPe2eRrAOjqQgNdAIDAsIZYtGBTgUoqqjUgxqbs6UMDXRIABBVCEtCFlVRU68DRU4EuAwCCErfbAAAADAhJAAAABoQkAAAAA0ISAACAASEJAADAgJAEAABgQEgCOrBoWzfjQyDb80GRzTk+AHREPCcJ6MB6RoR6PBRSkvHBkA37jE2M1kM3DPRLDc05PgB0RIQkoBNozkMhz+1zRXRUux8fADoabrcBAAAYEJIAAAAMCEkAAAAGhCQAAAADQhIAAIABIQkAAMCAkAQAAGBASAIAADAgJAEAABgQkgAAAAwISUAn09iit/7QnP2aju9rPS3dT1hYmGTx/rUWqHoAdGys3QZ0Mg0XvW3LxWxN+254/NYseNvS/YSGei/466oxEPUA6NgISUAn5Vp0ti0Xs21q3/5a9NaX/ZhqDGQ9ADombrcBAAAYEJIAAAAMCEkAAAAGhCQAAAADQhIAAIABIQkAAMCAkAQAAGBASAIAADAgJAEAABgQkgAAAAwISQAAAAZdOiT96U9/0rXXXqvk5GTNmDFDH374YaBLAgAAQaLLhqS//OUvevzxx7VgwQK9+uqr6t+/v+6++25VV1cHujQAABAEumxIWr9+vaZPn64pU6bou9/9rpYtW6a6ujr95S9/CXRpAAAgCARtSMrJydEvf/lLTZ06VUOGDFFiYqJeffXVJt+zf/9+zZ07V8OHD9dVV12lW2+9Vdu3b/fqd+bMGRUVFWnMmDHuttDQUKWmpur999/396kAAIAOKDTQBTQmOztb5eXl6tOnj2JiYlReXt5kf7vdrvT0dIWHh2vSpEmKiorSzp07tXDhQh0/flxz5sxx9/3888/lcDh04YUXeuzjggsu0CeffNIm5wMAADqWoL2StGzZMu3evVt2u13Tp09vsm9dXZ2WLFkii8WiDRs2aOnSpcrMzFROTo7i4+O1atWq84YsAACAcwVtSBozZoxiY2Ob1ddut6usrEyTJ09WUlKSu71Hjx6aN2+eamtrtWXLFnd7nz59ZLVadeLECY/9nDx5Un379vXPCQAAgA4taG+3tcS+ffskSWlpaV7bXG25ubnutvDwcCUlJendd9/VtddeK+ns1ai9e/fqgQceaNGxHQ6Hj1W3z/6Aljjfz5/Vam2zfTf3eI3tp76+Xt27dw+aevAt12fEZxWcutr4tOQ8O0VIOnz4sCSpf//+Xtuio6MVGRmp0tJSj/ZZs2bp5z//uQYPHqzBgwfrmWeeUWhoqG688cYWHbuwsNDnuhsTERHh930CzXHo0CGdPn1akhQWFqakQYMVFup7MJKkaFs3OeqdXgHDUe+UNcTS6OuW7sdqteryyy/3uZ7aOoeKDh5QbW2t8X0REREaNGiQV/v5PjPTeZ17rMY+55b28ZVp3w3368t5mbTF70v4TyDHpy1/xlujU4Qk17ONevToYdxus9lUVVXl0XbjjTfq5MmT+u1vf6vPPvtMQ4YM0TPPPCObzdaiYycnJ7fq/64bcjgcKikp8dv+gJZITEz0eG21WrVgU4FKKqo1NjFaD90wsMX77BkRKmuIxb0fSe59Ndy3qY8v+2lpPQNibMqePlSDBw9u8fk15zM737HOfU9r+vjq3H03tl9fzsvF4XCosLDQ778v4R/BMj5t+TN+Ltf5NkenCEm+uuuuu3TXXXe1ah9Wq5U/9Og0TD/LJRXVOnD0lK6IjmrVvl37keTeV8N9m/r4sp+W1uPiy5/l5nxmzTmWv/r4quG+/XVeDbfx+zJ4BcP4tOXPuC+CduJ2S7iu/jS8WuRSXV3d6FUmAAAAk04RkuLj4yXJa96RJFVWVqqmpsY4XwkAAKAxnSIkjRgxQpK0Z88er22uNlcfAACA5ugUIWn06NGKi4vTtm3bVFRU5G6vqqrS2rVrFRYWpilTpgSuQAAA0OEE7cTtzZs3Ky8vT5JUXFzsbnM9E2nYsGGaNm2apLPrri1btkzp6emaOXOmx7Ik5eXlysjIUL9+/QJzIgAAoEMK2pCUl5fn8ZRsScrPz1d+fr77tSskSdKoUaO0ceNGrV69Wtu3b1ddXZ0SEhK0aNEiTZw4sd3qBgAAnUPQhqSsrCxlZWW16D0pKSlat25dG1UEAAC6kk4xJwkAAMDfCEkAAAAGhCQAAAADQhIAAIABIQkAAMCAkAQAAGAQtI8ACHZOp1OS5HA4/Lpfh8Mhp9Mph8OhblYpItSisBDP15K82uhDn9b06WZt/Ge5JfsJ9Hn42qep82/s8/D1M2vsff7q46u2Oi/p23Z//76EfwTT+LTlz7iLa5+uv8ebYnE2pxe8nDlzRoWFhYEuAwAA+CA5OVnh4eFN9iEk+ai+vl51dXUKCQmRxWIJdDkAAKAZnE6n6uvrFRoaqpCQpmcdEZIAAAAMmLgNAABgQEgCAAAwICQBAAAYEJIAAAAMCEkAAAAGhCQAAAADQhIAAIABIQkAAMCAtduCxP79+7VmzRoVFBSorq5OCQkJmjVrliZOnBjo0jqNTz/9VK+99pr+8Y9/6OOPP9Znn32mXr166eqrr1Z6erquvPJKr/dUV1drzZo12rlzpyorKxUTE6MbbrhB8+fPV1RUlFf/+vp6bdiwQa+88opKS0sVGRmpMWPGaOHChYqLi2uP0+xUnn76aT3xxBOSpJdffllXXXWVx3bGp/397W9/08aNG3Xw4EHV1NQoOjpaV111lR566CFdcskl7n6MTftyOp3629/+phdeeEH/+c9/VFVVpYsvvlipqamaO3eu12fI+DQPT9wOAna7Xenp6QoPD9ekSZMUFRWlnTt3qry8XBkZGZozZ06gS+wUHn/8cf3xj3/UpZdeqpEjR+qCCy5QaWmpdu3aJafTqSeeeMIjlNbU1Oj2229XUVGR0tLSlJSUpKKiIu3Zs0fJycnasGGDunXr5nGMX/ziF9q8ebO++93v6pprrlFFRYVee+01RUVF6eWXX1Z8fHw7n3XHVVxcrJtvvlmhoaGqqanxCkmMT/tyOp361a9+pZdfflmXXnqp0tLSFBUVpYqKCuXm5mrlypUaPny4JMYmELKysvTcc88pOjpa1113nWw2mz788EP985//VGRkpDZt2qSEhARJjE+LOBFQtbW1zvHjxzuHDBniPHjwoLv91KlTzuuvv945ePBg55EjRwJYYefx+uuvO/fu3evVnpub6xw8eLBzxIgRzm+++cbdnp2d7UxISHCuXLnSo//KlSudCQkJzrVr13q0v/vuu86EhATnzJkzPfbz5ptvOhMSEpxz5szx8xl1XmfOnHHedNNNzmnTpjkXLVrkTEhIcBYUFHj0YXza1/r1650JCQnORx55xFlXV+e1vba21v3fjE37qqiocA4cONB57bXXOk+dOuWx7bnnnnMmJCQ4MzMz3W2MT/MxJynA7Ha7ysrKNHnyZCUlJbnbe/TooXnz5qm2tlZbtmwJYIWdx/XXX6+RI0d6tQ8fPlypqan68ssvdejQIUln/6958+bNioyM1E9+8hOP/j/5yU8UGRmpzZs3e7S7Xi9YsMBjZelrrrlGI0eO1J49e3T06FF/n1antHbtWv373//Wr3/9a1mtVq/tjE/7+vrrr/X73/9ecXFx+vnPf24ck9DQs7M3GJv2V15ervr6eg0dOlQ9evTw2DZ27FhJ0ueffy6J8WkpQlKA7du3T5KUlpbmtc3Vlpub2641dUWuX/Cufx8+fFgVFRW6+uqrFRkZ6dE3MjJSV199tT755BMdO3bM3b537173toa+973vSfp2vNG4AwcOaO3atZo/f74GDBhg7MP4tK89e/boyy+/1Pjx41VfX6+dO3fq6aef1ksvvaTS0lKPvoxN++vfv7/CwsJUUFCg6upqj21vvvmmJGnUqFGSGJ+WYuJ2gB0+fFjS2R/yhqKjoxUZGen1Swj+dfToUb3zzjuKjo5237N3feaN3WePj4/Xnj17dPjwYV1yySWqqalRZWWlEhISjP+X7RpfxrJpZ86cUUZGhgYOHKj09PRG+zE+7evAgQOSpJCQEN14443u31uutlmzZikjI0MSYxMIffr00aJFi5SVlaUJEyZ4zEnau3evbr/9dv34xz+WxPi0FCEpwFypv+ElUhebzaaqqqr2LKlLqa2t1cMPP6wzZ85o0aJF7l8Crs/cZrMZ3+dqd41fc/szlk3Lzs7W4cOH9eqrrxp/IbswPu3rxIkTkqT169dr0KBB2rx5s6644goVFRVpyZIlevbZZxUXF6fbb7+dsQmQWbNmKSYmRr/4xS+0adMmd/uwYcM0efJk91VyxqdluN2GLqu+vl6ZmZnKzc3VrbfeqilTpgS6pC6toKBAzz77rO677z73FT0EB+f//yXosLAw/f73v1dKSoqioqI0fPhwZWdnKyQkRM8991yAq+zafve73+nhhx/WvHnz9NZbbyk/P18bNmzQN998ozvvvFNvvPFGoEvskAhJAXa+FF5dXd3oVSb4rr6+Xj/72c+0bds2/fCHP9Sjjz7qsd31mTe8v+/ianeNX3P7M5ZmdXV1yszMVGJiou65557z9md82pfrcxwyZIguuugij20JCQmKi4tTWVmZTp06xdgEwDvvvKM1a9Zo5syZuueee3TxxRe7Q+zatWsVGhqqxx57TBJ/dlqK220B5rovXFpaqiFDhnhsq6ysVE1NjVJSUgJQWedVX1+vxYsXa+vWrZo8ebKysrIUEuL5/wuu++znzr04l6vdNX6RkZGKjo7WkSNH5HA4vG4Vue7Xm+ae4exzW1yfacM/By633XabJOn3v/+9rrjiCkmMT3u5/PLLJTX+F6Gr/euvv+bPTgD84x//kCSlpqZ6bYuOjtbll1+ugwcP6quvvmJ8WogrSQE2YsQISWe/PdKQq83VB613bkCaOHGifvOb3xjnvsTHxysmJkb5+fmqqanx2FZTU6P8/Hz169fP4wnDI0eOdG9r6O2335bEWDYmPDxct9xyi/Ef1y/rcePG6ZZbblFsbCzj085cf/l+/PHHXttqa2tVVlamyMhIXXDBBYxNANTW1kqSTp48adx+8uRJhYSEKCwsjPFpIUJSgI0ePVpxcXHatm2bioqK3O1VVVVau3atwsLCmCvjJ65bbFu3btWECRO0cuXKRicHWywWTZs2TTU1NXryySc9tj355JOqqanRrbfe6tHuep2dna0zZ86429966y3t27dPaWlpio2N9fNZdQ7du3fX8uXLjf8MHTpUknTvvfdq+fLlSkpKYnzamesJ26WlpV7P0Hn66ad16tQpjR8/XqGhoYxNALi+mr9+/XqvqRsvvfSSjh8/rquuukrh4eGMTwuxLEkQYFmS9rFmzRr97ne/U2RkpO688073tz3ONX78ePdDPWtqajRjxgx9+OGHSktL06BBg3Tw4EH3o/tffPFFde/e3eP9DR/dX1lZqe3btysqKkqbNm3SZZdd1i7n2plkZmZqy5YtxmVJGJ/2U1ZWpunTp+vEiRMaO3as+xaO3W5XbGysXn75ZUVHR0tibNqbw+HQXXfdpdzcXF144YUaN26cevTo4R6f7t2764UXXnBP3WB8mo+QFCT279+v1atXeyxwO3v2bBa49SPXX7ZNWbFihaZOnep+XVVV5V4E8rPPPlN0dLQmTJig+++/3/iV2Pr6er344ovGRSAvvfRSv59TV9BYSJIYn/Z27NgxrV69Wm+//ba++OIL9e3bV+PGjdP999+vCy+80KMvY9O+zpw5o/Xr1+u1117Tf/7zH9XW1urCCy9Uamqq5s2b557H58L4NA8hCQAAwIA5SQAAAAaEJAAAAANCEgAAgAEhCQAAwICQBAAAYEBIAgAAMCAkAQAAGBCSAAAADAhJAAAABoQkAOiA7rjjDiUmJga6DKBTIyQBAAAYEJIAAAAMCEkAAAAGhCQArbZ3714lJiZqzZo1Kiws1OzZszV06FANGzZM999/v44cOeLR/8iRI0pMTFRmZqZxf4mJibrjjjs82lxzcM6cOaNVq1Zp7NixSklJ0dSpU/XOO+9IkqqqqvToo48qLS1NycnJuu2227R///5mncOf/vQnJSYmaseOHR7ty5cvV2JiombMmGE8h8WLF3u0nzx5UsuXL9e4ceM0ZMgQjR49WgsWLFBxcbHXMTMzM5WYmKhPPvlEzz77rCZOnKghQ4Z4fC7vvfeefvzjH+uqq65SamqqHnjgAR07dsx4Dt98842effZZ/fCHP9SwYcN01VVXady4cVqwYIE+/PDDZn0OAL4VGugCAHQehYWFWrdunVJTUzV9+nQdPHhQu3btUnFxsbZt26Zu3bq1+hgPPPCAiouLNW7cOH399df6y1/+onvvvVcvvfSSfvnLX6q2tlYTJkzQ559/ru3btys9PV1vvPGGevTo0eR+U1NTJZ0NfBMmTHC37927131uNTU1ioyM9Gh3vU86G5Buu+02lZWVaeTIkZo0aZKOHDmi119/XW+99ZbWrVun4cOHex176dKl+te//qVrrrlG1157rS688EJJ0rvvvqu5c+fKYrFo4sSJiomJ0bvvvqsZM2aoZ8+eXvvJyMjQa6+9psTERE2dOlXh4eE6fvy49u7dq8LCQg0cOLAlHzXQ5RGSAPjNW2+9pd/+9reaOHGiu+3hhx9WTk6Odu3apUmTJrX6GF988YX+7//+zx1W0tLStHDhQs2ePVtjxozRE088odDQs7/aBg4cqMcff1x//vOfNXv27Cb3m5iYqN69e7vDjyR9/vnnKi4u1ujRo/Xuu+8qPz9faWlpkswhaeXKlSorK9O9996rBx980N3+1ltv6Z577tHPfvYz7dixQyEhnhfxDx06pC1btug73/mOu62+vl5LlixRXV2dXnzxRXe4cjqdWrRokbZt2+axj6qqKu3YsUODBw/W5s2bZbVa3dscDoe++uqr83yyABridhsAvxkxYoRHQJKkm2++WdLZKzH+sHDhQndAkqQJEyYoLCxMp06dUkZGhjsgSdLkyZMlqVm3miwWi1JTU/XRRx+psrJSkrRv3z45nU7dd999Cg8Pl91ud/ffu3ev4uLidMkll0iSzpw5o7/+9a/q3bu37rvvPo99X3PNNfp//+//qbS0VPn5+V7Hvvvuuz0CkiTl5eXpk08+0dixYz2uPlksFj344IMeIcjV7nQ61a1bN68QZrVajVeeADSNkATAbwYPHuzVdvHFF0uSTp065ZdjJCUlebwOCQnRBRdcoF69enkFjejoaElSRUVFs/Z97i0317+joqLc83tc7aWlpTp+/LjHVaSPP/5Y33zzjVJSUhQREdHovouKiry2paSkeLW5gp3p9lxsbKz7c3Wx2Wy65pprlJ+fr5tuuklr165Vfn6+amtrm3XuALwRkgD4jc1m82pzXfGor69vs2OEhoY22i5JdXV1zdq3KSSNGDFCoaGhSk1N1YEDB1RdXW281VZdXS1J6tu3r3HfrsDm6ncu1xykc1VVVTW6rbHjZGdna968eaqqqtJvf/tbzZgxQ6NGjdLy5ct1+vRp80kDaBQhCUC7c90OMoUXVzgIhAEDBqhv377au3evTpw4oZKSEncQSk1NlcPh0HvvvWcMSa6Q9tlnnxn37Wo3hTmLxeLV5ppofuLEiSb3d66IiAgtXLhQb7zxhnbt2qXly5frsssu0/PPP68VK1Y0et4AzAhJANqda37Mp59+6rXt4MGD7V2Oh5EjR6q0tFQ5OTmSpFGjRkmSrrzySnXv3l12u1179+5VfHy8LrroIvf7Lr/8cnXr1k2FhYXGqzauYNXwdmFjXN9Ee++997y2lZeX6/jx402+Py4uTrfccotefPFFRUZGavfu3c06LoBvEZIAtDubzabLLrtM+fn5Ki0tdbdXV1dr1apVAazs26tDf/zjH9W7d293WAkPD9fVV1+tnJwcVVZWelxFcm2fNGmSPv/8cz311FMe2/7xj39oz5496t+/v66++upm1TFs2DD169dPb775pkdQcjqdWrVqlRwOh0f/kydPGp/F9OWXX6q2tlbh4eHNOi6Ab/EIAAABMWfOHC1ZskS33XabJkyYoPr6er399ttKTk4OaF2u8HPy5En913/9l8c3xVJTU90PrmwYkiTpoYceUm5urv7whz+ooKBAV155pcrLy7Vjxw5FRETo17/+tdc3zxoTEhKipUuX6p577tHs2bPdz0my2+2qrKxUYmKiDh065O7/6aefasqUKRo4cKASExN10UUX6YsvvtAbb7yh2tpa3X333a35WIAuiZAEICBuvfVW1dbW6vnnn9ef//xnRUdH66abbtJ9992n119/PWB1XXbZZbrooov06aefum+1uZwbjEaOHOn13gsuuECvvPKKnnzySe3evVt5eXmy2Wy67rrrNH/+fCUkJLSoljFjxmj9+vX63//9X+3YsUPdu3fXqFGjlJ2drYyMDI++sbGx+ulPfyq73a533nlHX3zxhfr06aNBgwbpzjvv1Pe///0WHRuAZHE6nc5AFwEAABBsmJMEAABgQEgCAAAwICQBAAAYEJIAAAAMCEkAAAAGhCQAAAADQhIAAIABIQkAAMCAkAQAAGBASAIAADAgJAEAABgQkgAAAAz+P4Dm1PqVy2jHAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist([len(sent) for sent in X], bins=100)\n",
    "plt.yscale('log')\n",
    "plt.ylabel('count')\n",
    "plt.xlabel('num words')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "16a144be",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(89559, 29853)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42)\n",
    "len(X_train), len(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "d021d805",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n"
     ]
    }
   ],
   "source": [
    "fasttext.util.download_model('en', if_exists='ignore')\n",
    "ft = fasttext.load_model('cc.en.300.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "78839cc8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(300,)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ft.get_word_vector('mother').shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "5716953b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.8316885828971863, 'father'),\n",
       " (0.8139842748641968, 'grandmother'),\n",
       " (0.8042423725128174, 'daughter'),\n",
       " (0.7595430612564087, 'aunt'),\n",
       " (0.7551708221435547, 'stepfather'),\n",
       " (0.7433159947395325, 'mom'),\n",
       " (0.7390366792678833, 'step-father'),\n",
       " (0.7379246950149536, 'stepmother'),\n",
       " (0.7268661856651306, 'step-mother'),\n",
       " (0.7265076041221619, 'sister')]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ft.get_nearest_neighbors('mother')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "1b1bb3a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "dfe2eb0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_sents = [' '.join(sent) for sent in X_train]\n",
    "X_test_sents = [' '.join(sent) for sent in X_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "02ea04e8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((89559, 7615), (29853, 7615))"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_idf = TfidfVectorizer(min_df=5).fit(X_train_sents)\n",
    "tf_idf_train = tf_idf.transform(X_train_sents)\n",
    "tf_idf_test = tf_idf.transform(X_test_sents)\n",
    "\n",
    "tf_idf_train.shape, tf_idf_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "8e80207e",
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = LogisticRegression(max_iter=1000).fit(tf_idf_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "3fcf9b2f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tf-idf + LogRegr train acc: 0.8965\n",
      "Tf-idf + LogRegr test acc: 0.8845\n"
     ]
    }
   ],
   "source": [
    "print(f'Tf-idf + LogRegr train acc: {accuracy_score(lr.predict(tf_idf_train), y_train):.4f}')\n",
    "print(f'Tf-idf + LogRegr test acc: {accuracy_score(lr.predict(tf_idf_test), y_test):.4f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "c5002797",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_sentence_embedding(sent):\n",
    "    embeds = []\n",
    "    for word in sent:\n",
    "        embeds += [ft.get_word_vector(word)]\n",
    "\n",
    "    return np.stack(embeds, axis=0).mean(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "9c2de438",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_embeds = np.stack([get_sentence_embedding(sent) for sent in X_train], axis=0)\n",
    "test_embeds = np.stack([get_sentence_embedding(sent) for sent in X_test], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "047b4f6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = LogisticRegression(max_iter=1000).fit(train_embeds, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "9d5c8102",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FastText + LogRegr train acc: 0.8564\n",
      "FastText + LogRegr test acc: 0.8558\n"
     ]
    }
   ],
   "source": [
    "print(f'FastText + LogRegr train acc: {accuracy_score(lr.predict(train_embeds), y_train):.4f}')\n",
    "print(f'FastText + LogRegr test acc: {accuracy_score(lr.predict(test_embeds), y_test):.4f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "5e4a9938",
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 128\n",
    "MAX_SEQUENCE_LENGTH = 256\n",
    "MODEL_BLOCKS = 2\n",
    "MODEL_CHANNELS = 300\n",
    "NUM_EPOCHS = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "bd885090",
   "metadata": {},
   "outputs": [],
   "source": [
    "class EmbeddingsDataset(Dataset):\n",
    "    def __init__(self, sents, targets, max_length):\n",
    "        super().__init__()\n",
    "        self.max_length = max_length\n",
    "        self.sents = sents\n",
    "        self.targets = targets\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.sents)\n",
    "\n",
    "    def __getitem__(self, item):\n",
    "        sent = self.sents[item]\n",
    "        length = min(len(sent), self.max_length)\n",
    "\n",
    "        embeds = []\n",
    "        for word in sent[:length]:\n",
    "            embeds += [torch.from_numpy(ft.get_word_vector(word).astype(np.float32))]\n",
    "        \n",
    "        embeds = torch.stack(embeds, dim=1)\n",
    "        padding = torch.zeros((embeds.shape[0], self.max_length - length))\n",
    "        embeds = torch.cat([embeds, padding], dim=1)\n",
    "        target = self.targets[item]\n",
    "        return embeds, length, target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "35f890e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set = EmbeddingsDataset(X_train, y_train, max_length=MAX_SEQUENCE_LENGTH)\n",
    "test_set = EmbeddingsDataset(X_test, y_test, max_length=MAX_SEQUENCE_LENGTH)\n",
    "\n",
    "train_loader = DataLoader(train_set, batch_size=BATCH_SIZE, num_workers=4,\n",
    "                          pin_memory=True, shuffle=True)\n",
    "test_loader = DataLoader(test_set, batch_size=BATCH_SIZE, num_workers=4,\n",
    "                         pin_memory=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "95d9476a",
   "metadata": {},
   "outputs": [],
   "source": [
    "class BasicBlock(nn.Module):\n",
    "    def __init__(self, num_channels, kernel_size):\n",
    "        super().__init__()\n",
    "        padding = (kernel_size - 1) // 2\n",
    "        self.conv1 = nn.Conv1d(num_channels, num_channels, kernel_size,\n",
    "                               padding=padding, bias=False)\n",
    "        self.bn1 = nn.BatchNorm1d(num_channels)\n",
    "        self.conv2 = nn.Conv1d(num_channels, num_channels, kernel_size,\n",
    "                               padding=padding, bias=False)\n",
    "        self.bn2 = nn.BatchNorm1d(num_channels)\n",
    "        self.relu = nn.LeakyReLU(0.1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = self.conv1(x)\n",
    "        out = self.bn1(out)\n",
    "        out = self.relu(out)\n",
    "        out = self.conv2(out)\n",
    "        out = self.bn2(out)\n",
    "        out = self.relu(out + x)\n",
    "        return out\n",
    "\n",
    "class TextCNN(nn.Module):\n",
    "    def __init__(self, num_blocks, num_channels, num_classes, kernel_size=5):\n",
    "        super().__init__()\n",
    "        self.backbone = nn.Sequential(\n",
    "            *[BasicBlock(num_channels, kernel_size) for _ in range(num_blocks)]\n",
    "        )\n",
    "\n",
    "        self.classifier = nn.Linear(num_channels, num_classes)\n",
    "\n",
    "    @staticmethod\n",
    "    def get_lengths_mask(embeds, lengths):\n",
    "        mask = torch.arange(embeds.shape[-1]).unsqueeze(0).to(embeds.device)\n",
    "        mask = (mask < lengths.unsqueeze(1)).to(torch.float)\n",
    "        return mask.unsqueeze(1)\n",
    "\n",
    "    def forward(self, embeds, lengths):\n",
    "        # embeds: (B, C, T), lengths: (B, )\n",
    "        features = self.backbone(embeds)\n",
    "        # features: (B, C, T)\n",
    "        mask = self.get_lengths_mask(embeds, lengths)\n",
    "        # mask: (B, 1, T)\n",
    "        features = (features * mask).sum(-1) / lengths.unsqueeze(1)\n",
    "        # features: (B, C)\n",
    "        logits = self.classifier(features)\n",
    "        # logits: (B, num_classes)\n",
    "        return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "7986a573",
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "\n",
    "def plot_losses(train_losses, test_losses, train_accuracies, test_accuracies):\n",
    "    clear_output()\n",
    "    fig, axs = plt.subplots(1, 2, figsize=(13, 4))\n",
    "    axs[0].plot(range(1, len(train_losses) + 1), train_losses, label='train', color='deepskyblue', linewidth=2)\n",
    "    axs[0].plot(range(1, len(test_losses) + 1), test_losses, label='test', color='springgreen', linewidth=2)\n",
    "    axs[0].set_ylabel('loss')\n",
    "\n",
    "    axs[1].plot(range(1, len(train_accuracies) + 1), train_accuracies, label='train',\n",
    "                color='deepskyblue', linewidth=2)\n",
    "    axs[1].plot(range(1, len(test_accuracies) + 1), test_accuracies, label='test',\n",
    "                color='springgreen', linewidth=2)\n",
    "    axs[1].set_ylabel('accuracy')\n",
    "\n",
    "    for ax in axs:\n",
    "        ax.set_xlabel('epoch')\n",
    "        ax.legend()\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "a45bf024",
   "metadata": {},
   "outputs": [],
   "source": [
    "def training_epoch(model, optimizer, criterion, train_loader, tqdm_desc):\n",
    "    train_loss, train_accuracy = 0.0, 0.0\n",
    "    model.train()\n",
    "    for embeds, lengths, labels in tqdm(train_loader, desc=tqdm_desc):\n",
    "        embeds = embeds.to(device)\n",
    "        lengths = lengths.to(device)\n",
    "        labels = labels.to(device)\n",
    "        embeds = embeds[..., :lengths.max()]\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        logits = model(embeds, lengths)\n",
    "        loss = criterion(logits, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        train_loss += loss.item() * embeds.shape[0]\n",
    "        train_accuracy += (logits.argmax(1) == labels).sum().item()\n",
    "\n",
    "    train_loss /= len(train_loader.dataset)\n",
    "    train_accuracy /= len(train_loader.dataset)\n",
    "    return train_loss, train_accuracy\n",
    "\n",
    "\n",
    "@torch.no_grad()\n",
    "def validation_epoch(model, criterion, valid_loader, tqdm_desc):\n",
    "    valid_loss, valid_accuracy = 0.0, 0.0\n",
    "    model.eval()\n",
    "    for embeds, lengths, labels in tqdm(valid_loader, desc=tqdm_desc):\n",
    "        embeds = embeds.to(device)\n",
    "        lengths = lengths.to(device)\n",
    "        labels = labels.to(device)\n",
    "        embeds = embeds[..., :lengths.max()]\n",
    "\n",
    "        logits = model(embeds, lengths)\n",
    "        loss = criterion(logits, labels)\n",
    "\n",
    "        valid_loss += loss.item() * embeds.shape[0]\n",
    "        valid_accuracy += (logits.argmax(1) == labels).sum().item()\n",
    "\n",
    "    valid_loss /= len(valid_loader.dataset)\n",
    "    valid_accuracy /= len(valid_loader.dataset)\n",
    "    return valid_loss, valid_accuracy\n",
    "\n",
    "\n",
    "def train(model, optimizer, scheduler, criterion, train_loader, valid_loader, num_epochs):\n",
    "    train_losses, valid_losses = [], []\n",
    "    train_accuracies, valid_accuracies = [], []\n",
    "\n",
    "    for epoch in range(1, num_epochs + 1):\n",
    "        train_loss, train_accuracy = training_epoch(\n",
    "            model, optimizer, criterion, train_loader,\n",
    "            tqdm_desc=f'Training {epoch}/{num_epochs}'\n",
    "        )\n",
    "        valid_loss, valid_accuracy = validation_epoch(\n",
    "            model, criterion, valid_loader,\n",
    "            tqdm_desc=f'Validating {epoch}/{num_epochs}'\n",
    "        )\n",
    "\n",
    "        if scheduler is not None:\n",
    "            scheduler.step()\n",
    "\n",
    "        train_losses += [train_loss]\n",
    "        train_accuracies += [train_accuracy]\n",
    "        valid_losses += [valid_loss]\n",
    "        valid_accuracies += [valid_accuracy]\n",
    "        plot_losses(train_losses, valid_losses, train_accuracies, valid_accuracies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "460dcee6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "b602cb49",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = TextCNN(num_blocks=MODEL_BLOCKS, num_channels=MODEL_CHANNELS, num_classes=2).to(device)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\n",
    "scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, NUM_EPOCHS)\n",
    "criterion = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec787850",
   "metadata": {},
   "outputs": [],
   "source": [
    "train(model, optimizer, scheduler, criterion, train_loader, test_loader, NUM_EPOCHS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02744823",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}