Train_using_embeddings_aug_r18.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "aaee65d4-fdcd-492e-8a12-24a9f6e6e9af",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Goal : To classify the type of ovarian cancer from microscopy scans of biopsy samples.\n",
    "\n",
    "# Search SoTA model on Kaggle Approach, Papers with code - Medical Data, Graph Transformer, GNN, Global and Local Analysis\n",
    "\n",
    "# You should manage, including differences in image dimensions, quality, slide staining techniques, and more\n",
    "\n",
    "# There are 2 types of images - TMA and WSI (25 images are TMA and 513(rest) are WSI)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9d1971b2-09c3-4a8d-b158-355c9772dfa9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/projectnb/cs640grp/students/avarshn/.conda/envs/ocean'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "sys.prefix"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48222a33-507b-4f98-8df3-8538ed479f1c",
   "metadata": {},
   "source": [
    "# Check if GPU is available"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "143df7b3-2bc3-4651-b9a8-ed3f5a8108be",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c193c329-3591-4572-9640-35b274bf754e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3c04ae72-d26a-4c3c-8415-ba36cb557c94",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/bin/bash: nvidia: command not found\n"
     ]
    }
   ],
   "source": [
    "!nvidia smi\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "150de13e-2b4e-499e-9092-e541baa686b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "!lspci | grep -i nvidia"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75827adc-9618-4d47-8fdc-44dbdc43cbe8",
   "metadata": {},
   "source": [
    "# Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b7db4971-169f-47a7-9938-cc2d20e23b09",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from time import time\n",
    "from collections import defaultdict\n",
    "from tqdm import tqdm\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.utils.class_weight import compute_class_weight\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score, f1_score, confusion_matrix, classification_report\n",
    "\n",
    "from PIL import Image\n",
    "import cv2\n",
    "\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.optim import AdamW\n",
    "\n",
    "from torchsummary import summary\n",
    "\n",
    "import torchvision\n",
    "from torchvision import transforms, utils\n",
    "\n",
    "from torchvision.models import efficientnet_b0, EfficientNet_B0_Weights\n",
    "from torchvision.models import resnet18, ResNet18_Weights"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5c70d5b-6ba8-4faf-a3ff-5e37b6a0c467",
   "metadata": {},
   "source": [
    "# Configurations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "13ca183a-b75e-4d06-ae54-1c3af198cae6",
   "metadata": {},
   "outputs": [],
   "source": [
    "CONFIG = {\n",
    "    \"model_name\" : \"resnet18\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "c49af72f-38c9-44c7-b1a5-60dd6203ba46",
   "metadata": {},
   "outputs": [],
   "source": [
    "if CONFIG[\"model_name\"] == \"resnet18\":\n",
    "\n",
    "    CONFIG.update({\n",
    "            \"seed\": 42,\n",
    "            \"num_classes\": 5,\n",
    "            \"device\": torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\"),\n",
    "            \n",
    "            # \"valid_batch_size\": 32,\n",
    "            \"embedding_save_folder\" : \"./models/resnet18/embeddings/\",\n",
    "            \"feature_input_size\" : 512,\n",
    "            \"model_save_folder\" : \"./models/resnet18/model_checkpoints/\",\n",
    "        \n",
    "            \"batch_size\" : 8,\n",
    "            \"train_batch_size\" : 8,\n",
    "            \"epochs\" : 20,\n",
    "            \"learning_rate\" : 1e-5,\n",
    "            \"weight_decay\" : 1e-3\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c572c798-bfbf-49bb-a47e-95935ba372dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "if CONFIG[\"model_name\"] == \"efficientnet_b3\":\n",
    "\n",
    "    CONFIG.update({\n",
    "            \"seed\": 42,\n",
    "            \"num_classes\": 5,\n",
    "            \"device\": torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\"),\n",
    "            \n",
    "            # \"valid_batch_size\": 32,\n",
    "            \"embedding_save_folder\" : \"./models/efficientnet_b3/embeddings/\",\n",
    "            \"feature_input_size\" : 1536,\n",
    "            \"model_save_folder\" : \"./models/efficientnet_b3/model_checkpoints/\",\n",
    "        \n",
    "            \"batch_size\" : 8,\n",
    "            \"train_batch_size\" : 8,\n",
    "            \"epochs\" : 10,\n",
    "            \"learning_rate\" : 1e-5,\n",
    "            \"weight_decay\" : 1e-3\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "4dcff8b2-936b-458c-a5f1-b7b711030b2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "if CONFIG[\"model_name\"] == \"efficientnet_b0\":\n",
    "\n",
    "    CONFIG.update({\n",
    "            \"seed\": 42,\n",
    "            \"num_classes\": 5,\n",
    "            \"device\": torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\"),\n",
    "            \n",
    "            # \"valid_batch_size\": 32,\n",
    "            \"embedding_save_folder\" : \"./models/efficientnet_b0/embeddings/\",\n",
    "            \"feature_input_size\" : 1280,\n",
    "            \"model_save_folder\" : \"./models/efficientnet_b0/model_checkpoints/\",\n",
    "        \n",
    "            \"batch_size\" : 8,\n",
    "            \"epochs\" : 40,\n",
    "            \"learning_rate\" : 1e-5,\n",
    "            \"weight_decay\" : 1e-2\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d5fbe965-3c93-4b23-bb36-f4f56aca0000",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'model_name': 'resnet18',\n",
       " 'seed': 42,\n",
       " 'num_classes': 5,\n",
       " 'device': device(type='cpu'),\n",
       " 'embedding_save_folder': './models/resnet18/embeddings/',\n",
       " 'feature_input_size': 512,\n",
       " 'model_save_folder': './models/resnet18/model_checkpoints/',\n",
       " 'batch_size': 8,\n",
       " 'train_batch_size': 8,\n",
       " 'epochs': 20,\n",
       " 'learning_rate': 0.0001,\n",
       " 'weight_decay': 0.001}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CONFIG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "59ef42e6-00e5-40ff-a2b9-9e44b718dcee",
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_seed(seed=42):\n",
    "    '''Sets the seed of the entire notebook so results are the same every time we run.\n",
    "    This is for REPRODUCIBILITY.'''\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    # When running on the CuDNN backend, two further options must be set\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    torch.backends.cudnn.benchmark = False\n",
    "    # Set a fixed value for the hash seed\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    \n",
    "set_seed(CONFIG['seed'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b28ab07-67fb-4555-9187-88e860e73c34",
   "metadata": {},
   "source": [
    "# Merge Numpy arrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "859bccec-19a7-4315-9448-32e9edaa9b1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings_arr_list = []\n",
    "labels_arr_list = []\n",
    "\n",
    "for file in os.listdir(CONFIG[\"embedding_save_folder\"]):\n",
    "    if file.endswith(\".npz\"):\n",
    "        if \"train_embeddings_and_labels_\" in file:\n",
    "            np_arr = np.load(CONFIG[\"embedding_save_folder\"] + file)\n",
    "            embeddings_arr_list.append(np_arr[\"embeddings\"])\n",
    "            labels_arr_list.append(np_arr[\"labels\"])\n",
    "            # labels_arr = np.concatenate((labels_arr, np_arr[\"labels\"]), axis = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9c04dbc4-4ea3-436a-8f6e-6839333c4503",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1505, 512)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_embeddings = np.concatenate(embeddings_arr_list)\n",
    "all_embeddings.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4d2675b4-3009-47a7-bf8d-1f87046b58c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1505,)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_labels = np.concatenate(labels_arr_list)\n",
    "all_labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "df552edb-4d06-45ca-8fcb-95f99079fe1d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1, 2, 3, 4]), array([315, 300, 280, 308, 302]))"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Balanced Dataset with Augmentation\n",
    "np.unique(all_labels, return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7c39fe9e-d3ec-4df1-aee3-b8c790f94faa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./models/resnet18/embeddings/resnet18_train_embeddings_and_labels.npz\n"
     ]
    }
   ],
   "source": [
    "save_location = f'{CONFIG[\"embedding_save_folder\"]}{CONFIG[\"model_name\"]}_train_embeddings_and_labels.npz'\n",
    "print(save_location)\n",
    "\n",
    "np.savez(save_location, embeddings = all_embeddings, labels = all_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f3350a3-92d5-4526-8c18-60568057476c",
   "metadata": {},
   "source": [
    "# Custom Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "02dd6b7a-b315-49e5-ab7d-50141f6a92c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "Image.MAX_IMAGE_PIXELS = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "002b334d-0f52-4bde-828c-eddcbb34a033",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ocean_embedding_dataset(Dataset):\n",
    "    \n",
    "    def __init__(self, numpy_file_name):\n",
    "        \n",
    "        self.ocean_np_arr = np.load(numpy_file_name)\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.ocean_np_arr[\"labels\"].shape[0]\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        if torch.is_tensor(idx):\n",
    "            idx = idx.tolist()\n",
    "\n",
    "        image = self.ocean_np_arr[\"embeddings\"][idx]\n",
    "        image = torch.tensor(image)\n",
    "        \n",
    "        label = self.ocean_np_arr[\"labels\"][idx]\n",
    "\n",
    "        return image, label\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33858d05-4c7f-4766-8cb5-84cc5b1dc770",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "01054b83-dfda-4717-a6b5-64608e69e9d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_np_arr_path = f'{CONFIG[\"embedding_save_folder\"]}{CONFIG[\"model_name\"]}_train_embeddings_and_labels.npz'\n",
    "test_np_arr_path = f'{CONFIG[\"embedding_save_folder\"]}{CONFIG[\"model_name\"]}_test_embeddings_and_labels.npz'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "27eba013-ad6c-4dc8-b686-e7e327206c69",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = ocean_embedding_dataset(train_np_arr_path)\n",
    "\n",
    "test_dataset = ocean_embedding_dataset(test_np_arr_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18e15d1c-a392-41a3-a03f-5642fcdbfb96",
   "metadata": {},
   "source": [
    "## Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "50fbd20d-5ad0-44d1-8c8a-5bd32dcbc02e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1505"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset.__len__()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "86204af9-161a-488c-b890-b8f11b17911b",
   "metadata": {},
   "outputs": [],
   "source": [
    "a,b = train_dataset.__getitem__(torch.tensor(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "52061b6a-000b-4ff2-8595-e37f9007f17f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([6.5362e-01, 2.0221e-01, 1.0587e+00, 1.1869e+00, 6.1350e-01, 1.8662e-02,\n",
       "        1.3995e-01, 4.8100e-01, 2.5939e+00, 2.9080e-01, 2.6905e-02, 3.1879e-01,\n",
       "        8.4294e-01, 1.6270e+00, 2.0928e-01, 2.7919e-01, 1.0088e+00, 9.0720e-01,\n",
       "        2.1647e+00, 8.1846e-01, 1.2649e+00, 5.1603e-01, 1.3669e+00, 3.6165e-01,\n",
       "        1.4862e+00, 1.4949e+00, 9.1102e-01, 9.5804e-01, 5.2099e-02, 8.4702e-01,\n",
       "        1.6026e-01, 3.7481e-01, 1.0356e+00, 1.4091e+00, 1.9095e-01, 4.6272e-01,\n",
       "        3.8691e-02, 2.7147e-01, 1.6918e-01, 1.0337e-01, 6.9889e-01, 7.9704e-02,\n",
       "        5.2884e-01, 4.8314e-01, 2.5355e+00, 1.0056e+00, 4.2173e-01, 3.6864e-01,\n",
       "        1.3022e+00, 5.9585e-01, 1.4884e-01, 1.3268e+00, 1.0003e+00, 4.9262e-01,\n",
       "        2.7224e+00, 1.8266e+00, 2.0147e+00, 7.8689e-02, 1.0667e+00, 2.8156e-01,\n",
       "        6.1578e-03, 2.2887e-01, 3.3332e-01, 7.5356e-01, 2.9631e-01, 1.4279e+00,\n",
       "        4.7247e-01, 4.5107e-01, 6.6089e-01, 2.6366e-01, 1.4474e+00, 3.8410e-01,\n",
       "        7.9504e-02, 8.8031e-02, 1.9793e+00, 1.7075e-01, 2.3522e-01, 7.8605e-01,\n",
       "        3.6943e-01, 9.9700e-01, 4.7059e-02, 7.1929e-01, 1.6755e+00, 3.4487e-02,\n",
       "        8.4117e-01, 7.5118e-01, 1.6835e+00, 1.2844e+00, 1.7715e+00, 3.3964e-01,\n",
       "        2.0725e-01, 1.0772e+00, 8.1649e-01, 3.4021e-01, 3.6972e-01, 4.2603e-01,\n",
       "        1.3698e+00, 1.6426e-01, 8.6439e-01, 5.6322e-01, 1.3501e+00, 1.8099e+00,\n",
       "        8.4551e-01, 8.6630e-01, 2.6309e+00, 1.1500e+00, 1.0046e+00, 4.1891e-01,\n",
       "        1.2140e-01, 6.8928e-01, 2.8285e+00, 2.7281e-01, 2.2904e-01, 6.6781e-01,\n",
       "        5.1537e-02, 1.8239e+00, 1.8068e-02, 2.1193e-01, 6.2228e-01, 1.0520e+00,\n",
       "        9.5836e-01, 2.2772e-02, 1.4909e-01, 7.5809e-01, 1.8880e+00, 4.6047e-01,\n",
       "        3.0726e+00, 2.7665e+00, 8.7710e-01, 2.2607e-01, 4.4436e-01, 1.0459e+00,\n",
       "        8.9781e-01, 2.1762e-01, 1.2653e+00, 1.5277e+00, 4.1478e-01, 2.3718e+00,\n",
       "        9.2508e-01, 6.8808e-01, 5.4102e-01, 2.6691e-01, 9.5721e-01, 1.1273e+00,\n",
       "        1.1225e+00, 5.9688e-01, 3.2859e-01, 4.3876e+00, 1.2461e+00, 2.0410e+00,\n",
       "        1.2285e-01, 1.0157e+00, 1.2139e+00, 2.1711e+00, 1.0166e+00, 3.6994e-01,\n",
       "        6.6677e-01, 7.7575e-01, 5.7554e-01, 9.1585e-01, 2.6324e-01, 1.8273e-01,\n",
       "        4.1946e-01, 1.3043e+00, 3.0875e-01, 7.0528e-01, 1.9202e+00, 2.2491e-01,\n",
       "        2.3100e+00, 6.3286e-01, 3.0424e-01, 2.1733e-01, 3.6160e-01, 6.4134e-02,\n",
       "        8.7749e-01, 1.3682e+00, 8.7443e-02, 1.0437e+00, 2.5350e-03, 5.2828e-01,\n",
       "        7.9861e-01, 7.8737e-01, 8.5589e-01, 6.8813e-01, 7.1980e-01, 2.3285e+00,\n",
       "        2.3211e-01, 2.6541e+00, 5.8400e-01, 1.7158e+00, 7.3791e-01, 3.9697e-01,\n",
       "        7.0244e-01, 1.2293e+00, 4.9882e-01, 1.1239e+00, 1.9350e+00, 8.7470e-01,\n",
       "        2.8713e-01, 1.4773e+00, 2.3232e+00, 3.5911e-01, 8.6046e-01, 2.4015e-01,\n",
       "        1.7383e-02, 2.3290e+00, 4.3054e-01, 5.9906e-01, 1.4056e-01, 4.8136e+00,\n",
       "        8.6527e-01, 1.7450e+00, 2.7433e-01, 7.9066e-02, 9.3327e-01, 1.9677e+00,\n",
       "        3.2577e-02, 9.5169e-01, 5.1705e-01, 6.4012e-01, 5.2173e-01, 2.5285e-01,\n",
       "        5.5273e-01, 8.8236e-01, 3.1306e-01, 0.0000e+00, 1.6144e+00, 3.3689e-01,\n",
       "        7.6653e-01, 4.0192e-01, 6.7616e-01, 3.2001e-02, 2.1021e-01, 2.9882e+00,\n",
       "        6.6848e-01, 4.7189e-01, 1.0047e+00, 7.8403e-02, 1.1228e-01, 7.8580e-01,\n",
       "        3.3846e-01, 2.8764e+00, 5.2861e-02, 1.7285e+00, 4.5902e-01, 1.4326e+00,\n",
       "        1.1888e-01, 3.0592e-01, 3.1053e+00, 1.2477e-01, 2.7606e+00, 1.4813e+00,\n",
       "        2.7469e-01, 2.2708e-01, 5.9823e-01, 7.5395e-01, 3.0877e-02, 6.4384e-01,\n",
       "        7.9977e-01, 1.3686e+00, 2.5819e+00, 5.4982e-01, 1.2344e+00, 3.9766e-01,\n",
       "        5.4059e-01, 3.4938e-03, 4.0506e-01, 3.9834e-01, 3.0830e+00, 7.4929e-02,\n",
       "        7.6172e-01, 1.3614e+00, 1.1995e+00, 1.1657e+00, 1.0338e-01, 2.0260e-02,\n",
       "        1.5844e-01, 5.0244e-01, 3.1147e-01, 4.3949e-01, 1.3744e-01, 2.0459e-01,\n",
       "        1.0578e-01, 1.5121e+00, 8.0043e-01, 4.0843e+00, 1.3413e+00, 6.4077e-01,\n",
       "        9.2891e-01, 8.9988e-01, 3.5731e-01, 1.0629e+00, 4.9882e-01, 2.8234e+00,\n",
       "        1.5938e+00, 9.9663e-01, 8.8590e-01, 0.0000e+00, 1.6658e-01, 1.5255e+00,\n",
       "        1.5370e+00, 1.4311e-01, 6.7959e-01, 4.8970e-01, 1.0477e-01, 1.6770e+00,\n",
       "        1.3223e+00, 2.8898e-01, 5.7595e-01, 4.6904e-01, 1.0277e+00, 7.8714e-01,\n",
       "        1.4311e+00, 3.8747e-01, 4.3243e-01, 4.5868e-01, 1.2318e+00, 1.3748e-01,\n",
       "        3.1229e-01, 2.6990e-02, 1.0155e+00, 1.4641e+00, 1.9139e-01, 2.7481e-01,\n",
       "        3.2148e-01, 1.0985e+00, 6.8144e-02, 5.8288e-02, 2.3114e+00, 6.6723e-01,\n",
       "        7.4131e-02, 1.0207e-01, 1.3079e+00, 1.1664e-01, 2.4211e-01, 5.5466e-02,\n",
       "        2.0050e+00, 2.2654e+00, 6.4973e-01, 2.3027e-01, 1.4785e+00, 1.3587e+00,\n",
       "        1.0028e+00, 1.2708e+00, 5.2931e-01, 1.3360e-01, 5.2782e-01, 1.6089e+00,\n",
       "        3.2664e-01, 1.9654e-01, 2.9977e-01, 2.3970e-01, 2.4797e-01, 2.5094e-01,\n",
       "        5.9120e-01, 5.2311e-01, 6.1031e-01, 1.5726e+00, 2.8808e-01, 1.7435e+00,\n",
       "        1.0317e+00, 4.6387e-01, 2.9081e-01, 1.8434e+00, 1.1343e+00, 5.6940e-01,\n",
       "        7.4516e-02, 1.9692e-01, 5.3965e-01, 7.9796e-01, 1.4866e+00, 6.8832e-01,\n",
       "        1.3068e+00, 2.3887e-01, 2.0587e-01, 3.0018e+00, 1.3179e+00, 8.1371e-01,\n",
       "        7.1122e-01, 6.8969e-01, 4.7434e-01, 5.4055e-01, 1.8518e-02, 3.6572e-01,\n",
       "        1.8395e+00, 5.2198e-01, 2.5948e-01, 9.8313e-01, 4.3202e-01, 5.6275e-01,\n",
       "        1.4851e-01, 2.2254e-01, 3.2947e-01, 1.2066e-01, 4.5594e-01, 8.6891e-02,\n",
       "        1.3070e+00, 7.6446e-01, 4.1475e-01, 1.2683e-01, 5.1695e-01, 5.3638e+00,\n",
       "        9.7923e-01, 1.1120e+00, 9.2010e-01, 6.0496e-01, 2.3375e-01, 2.8529e-01,\n",
       "        2.0599e+00, 1.8868e-01, 5.7743e-01, 1.5140e+00, 1.1566e-02, 1.4247e-02,\n",
       "        6.5870e-01, 7.3592e-01, 1.3653e-01, 1.7873e+00, 1.0893e+00, 8.1312e-01,\n",
       "        5.3813e-01, 2.4455e+00, 1.2815e+00, 4.4845e-01, 1.2617e+00, 3.0300e-01,\n",
       "        8.3324e-01, 1.8979e+00, 1.1271e+00, 8.2701e-01, 1.4273e+00, 3.0823e-02,\n",
       "        7.4616e-02, 9.7777e-01, 2.5342e-02, 2.9901e+00, 1.8249e+00, 5.5404e-01,\n",
       "        5.5604e-01, 2.8607e-01, 1.5532e+00, 1.3627e+00, 4.4004e-01, 1.9062e-01,\n",
       "        2.7506e-01, 9.1821e-01, 1.1393e+00, 4.5592e-01, 8.0088e-02, 4.2671e+00,\n",
       "        4.5562e-01, 6.6573e-01, 3.3479e-01, 1.6018e-01, 2.0616e+00, 1.0787e+00,\n",
       "        1.2359e+00, 7.0099e-01, 4.3987e+00, 1.2915e+00, 9.1558e-01, 7.9177e-01,\n",
       "        1.1285e-01, 1.0800e+00, 1.4450e-01, 4.3995e-01, 4.7668e-01, 5.5384e-01,\n",
       "        1.5562e+00, 6.6149e-01, 9.1845e-01, 6.6017e-01, 2.2385e-01, 1.4554e+00,\n",
       "        8.9294e-01, 5.0199e-01, 6.8730e-01, 3.4715e+00, 1.0672e+00, 2.0954e-02,\n",
       "        7.1006e-01, 1.9653e-01, 2.2586e+00, 1.7519e+00, 7.8075e-01, 2.0830e+00,\n",
       "        1.7855e+00, 8.7283e-01, 3.3911e-01, 7.4359e-01, 6.3987e-04, 1.5091e+00,\n",
       "        7.4969e-01, 8.3303e-01, 2.1438e-01, 0.0000e+00, 3.1121e-01, 6.5357e-01,\n",
       "        1.8666e+00, 1.4380e+00, 2.2410e+00, 1.0402e+00, 3.6195e-02, 6.6213e-01,\n",
       "        1.5954e+00, 1.5891e+00, 1.1606e+00, 5.5537e-01, 7.4682e-01, 1.6598e+00,\n",
       "        3.0149e-01, 1.0166e+00])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "bd5247e2-7402-4bdd-b77d-f5d75b2ce125",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "b46cb26c-214f-4d06-a8c4-905d0ec08154",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4, 0, 2, ..., 1, 2, 2])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset.ocean_np_arr[\"labels\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20401933-47b3-404e-ba4e-be65e40f6ae6",
   "metadata": {},
   "source": [
    "# Class Weights - From Training Data since the dataset is imbalanced"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d5ad94e-b4fd-4454-8b6b-f8a607065cc8",
   "metadata": {},
   "source": [
    "## Not using Class Weights in nn.CrossEntropyLoss() since we have balanced data while Augmentation itself"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "31f40cfe-c181-440a-8f3e-d440782b30c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4, 0, 2, ..., 1, 2, 2])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset.ocean_np_arr[\"labels\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e8ff4c7-62e4-4654-a53b-209323c92e3f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75b28ad6-975a-4fc2-a19b-73bde1c09870",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "2b246783-1d97-4989-ac02-f7e263919015",
   "metadata": {},
   "source": [
    "* Class weight Formula -\n",
    "  \n",
    "class_weight_of_that_class = $\\frac{\\text{Total samples}}{\\text{Total classes * samples of that class}}$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "d3548690-c5de-462c-b869-e2ed6836beb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# NumPy array of class labels - labels_arr\n",
    "\n",
    "labels_arr = train_dataset.ocean_np_arr[\"labels\"].copy()\n",
    "\n",
    "class_weights = compute_class_weight('balanced',classes = np.unique(labels_arr), y = labels_arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "3c848bee-deca-4c3d-ac8f-b6b0c58ae80e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.95555556, 1.00333333, 1.075     , 0.97727273, 0.99668874])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "42486cca-117d-4dcb-9a8b-a2a118939821",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1, 2, 3, 4]), array([315, 300, 280, 308, 302]))"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# label_mapper = {'HGSC' : 0, 'EC' : 1, 'MC' : 2, 'CC' : 3, 'LGSC' : 4}\n",
    "# Train Label Distribution\n",
    "np.unique(labels_arr, return_counts = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "4b19a847-2296-4df0-9897-8e5a295e6c6b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.9556, 1.0033, 1.0750, 0.9773, 0.9967], dtype=torch.float64)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_weights_tensor = torch.tensor(class_weights)\n",
    "class_weights_tensor"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c34038c-0762-4061-85ee-cafd44eddddf",
   "metadata": {},
   "source": [
    "# Train - Evaluation, Training, Architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "fd3de3db-b023-41f9-b2d9-cf9f8c245a37",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use `torch.no_grad()` here to disable gradient calculation. \n",
    "# It will reduce memory consumption as we don't need to compute gradients in inference.\n",
    "\n",
    "@torch.no_grad()\n",
    "def evaluate(model, testloader, criterion, device):\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    # Keep track of loss and other evaluation metrics\n",
    "    test_loss = 0.0\n",
    "\n",
    "    YPredict = []\n",
    "    YTrue = []\n",
    "    \n",
    "    # Loop through each batch on test set\n",
    "    for i, (images, labels) in enumerate(testloader, 1):\n",
    "        \n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        \n",
    "        logits = model(images)\n",
    "\n",
    "        loss = criterion(logits, labels)\n",
    "        \n",
    "        test_loss += loss.detach().item()\n",
    "\n",
    "        predictions = torch.argmax(logits, dim = 1)\n",
    "        predictions = predictions.view(-1)\n",
    "        YTrue.extend(labels.tolist())\n",
    "        YPredict.extend(predictions.tolist())\n",
    "\n",
    "    # Compute Scores\n",
    "    accuracy = accuracy_score(YTrue, YPredict)\n",
    "    f1 = f1_score(YTrue, YPredict, average = \"macro\")\n",
    "    print(\"Classification report :\")\n",
    "    print(classification_report(YTrue, YPredict))\n",
    "\n",
    "    return (test_loss/i, accuracy, f1) ## avg loss and acc over all batches\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9af0610-8e73-45cd-987c-9556427a514a",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "9857e5e7-7f67-45c6-90f9-9a6586b650c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, trainloader, testloader, CONFIG):\n",
    "    \n",
    "    optimizer = AdamW(model.parameters(), lr = CONFIG[\"learning_rate\"], weight_decay = CONFIG[\"weight_decay\"])\n",
    "\n",
    "    # criterion = nn.CrossEntropyLoss(weight = class_weights_tensor)\n",
    "\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "\n",
    "    ## Measure runtime\n",
    "    t_start = time()\n",
    "    \n",
    "    ## Store training log\n",
    "    history = defaultdict(list)\n",
    "    \n",
    "    # We will train the model `num_epochs` times\n",
    "    for i in range(1, CONFIG[\"epochs\"]+1):\n",
    "\n",
    "        # Training\n",
    "        train_loss, train_acc, train_f1 = train_one_epoch(model = model, trainloader = trainloader, optimizer = optimizer, criterion = criterion, device = CONFIG[\"device\"])\n",
    "\n",
    "        # Model Checkpointing\n",
    "        if i%10==0:\n",
    "            torch.save(model.state_dict(), f'{CONFIG[\"model_save_folder\"]}epoch{i}_batch_size{CONFIG[\"batch_size\"]}.pth')\n",
    "\n",
    "        # Evaluate on test set\n",
    "        test_loss, test_acc, test_f1 = evaluate(model = model, testloader = testloader, criterion = criterion, device = CONFIG[\"device\"])\n",
    "\n",
    "        ## Store train/test loss, accuracy, f1 scores\n",
    "        history[\"train_loss\"].append(train_loss)\n",
    "        history[\"train_acc\"].append(train_acc)\n",
    "        history[\"train_f1\"].append(train_f1)\n",
    "        \n",
    "        history[\"test_loss\"].append(test_loss)\n",
    "        history[\"test_acc\"].append(test_acc)\n",
    "        history[\"test_f1\"].append(test_f1)\n",
    "\n",
    "        \n",
    "        \n",
    "        ## print out train/test loss, accuracy\n",
    "        print(f'Epoch: {i} | Runtime: {((time()-t_start)/60):.2f}[m] | train_loss: {train_loss:.3f} | test_loss: {test_loss:.3f} | train_acc: {train_acc:.3f} |  test_acc: {test_acc:.3f} | train_f1: {train_f1:.3f} | test_f1: {test_f1:.3f}')    \n",
    "    return history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "44a71bce-186f-4c0c-9b32-308f9819afc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_one_epoch(model, trainloader, device, optimizer, criterion):\n",
    "    \"\"\"\n",
    "      Train 1 epoch on trainloader.\"\n",
    "    \"\"\"\n",
    "\n",
    "    model = model.train()\n",
    "    \n",
    "    # Keep track of loss and other evaluation metrics\n",
    "    train_loss = 0.0\n",
    "\n",
    "    YPredict = []\n",
    "    YTrue = []\n",
    "\n",
    "    ## Loop over all the batches\n",
    "    for i, (images, labels) in tqdm(enumerate(trainloader, 1), total=len(trainloader), desc=f\"Training on whole batch...\"):\n",
    "            \n",
    "        # Move images and labels to `device` (CPU or GPU)\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "\n",
    "        logits = model(images)\n",
    "\n",
    "        # print(logits)\n",
    "        # print(labels)\n",
    "        # print(logits.dtype, labels.dtype)\n",
    "        # print(type(labels[0]))\n",
    "        # print(type(labels))\n",
    "        loss = criterion(logits, labels)\n",
    "        # loss = criterion(logits, labels.to(torch.long))\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        # Compute loss and accuracy for this batch\n",
    "        train_loss += loss.detach().item()\n",
    "\n",
    "        predictions = torch.argmax(logits, dim = 1)\n",
    "        predictions = predictions.view(-1)\n",
    "        YTrue.extend(labels.tolist())\n",
    "        YPredict.extend(predictions.tolist())\n",
    "\n",
    "\n",
    "    # Compute Scores\n",
    "    accuracy = accuracy_score(YTrue, YPredict)\n",
    "    f1 = f1_score(YTrue, YPredict, average = \"macro\")\n",
    "\n",
    "    return (train_loss/i, accuracy, f1) ## avg loss and acc over all batches\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "05d4166d-2d7d-4c79-a87c-9366953343ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Baseline_Model(nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "        # self.base_model_headless = base_layers\n",
    "        # self.base_model_headless.requires_grad = False\n",
    "        \n",
    "        self.fc1 = nn.Linear(CONFIG[\"feature_input_size\"], 32)\n",
    "        self.fc2 = nn.Linear(32, 5)\n",
    "        self.relu = nn.ReLU()\n",
    "        self.dropout = nn.Dropout(p = 0.1)\n",
    "\n",
    "\n",
    "    def forward(self, x):\n",
    "        # x = self.base_model_headless(x)\n",
    "        # # print(\"Output shape after Passing through Pretrained Model\")\n",
    "        # # print(x.shape)\n",
    "\n",
    "        # # Flatten\n",
    "        # x = x.view((x.shape[0], -1))\n",
    "\n",
    "        # FCNN\n",
    "        x = self.fc1(x)\n",
    "        x = self.relu(x)\n",
    "        x = self.dropout(x)\n",
    "        x = self.fc2(x)\n",
    "        # x = self.relu(x)\n",
    "        # x = self.dropout(x)\n",
    "        # x = self.fc3(x)\n",
    "        # print(x.shape)\n",
    "\n",
    "        return x      "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "184b00e2-750e-40e8-9699-67daeaadde48",
   "metadata": {},
   "source": [
    "# Train the Model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8774b78-d7f9-4b3a-a710-db583ebdaa3c",
   "metadata": {},
   "source": [
    "## DataLoader"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "297e5438-91dd-4722-9a89-3ff7c49a883e",
   "metadata": {},
   "source": [
    "* The EOFError you encountered in your DataLoader worker process could be related to using multiple workers (num_workers = 2)\n",
    "\n",
    "\n",
    "1) Concurrency Issues:\n",
    "\n",
    "    * When using multiple worker processes (num_workers > 0), there is potential for concurrency issues, especially when reading data from a file or other data sources. If your dataset is not designed to handle concurrent access properly, it can result in unexpected errors like EOFError.\n",
    "  \n",
    "2) Use torch.utils.data.SequentialSampler:\n",
    "    * Instead of relying on the default random sampler, you can use SequentialSampler to ensure that the data is loaded in a sequential order, which might help in scenarios where the order of loading matters.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "98abd35d-befe-40e6-ab4e-c678d48864ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainloader = DataLoader(train_dataset, batch_size = CONFIG['train_batch_size'], shuffle = True, drop_last = True)\n",
    "\n",
    "testloader = DataLoader(test_dataset, batch_size = CONFIG['batch_size'], shuffle = False, drop_last = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "1a15c053-9780-47d1-882f-15fbbb06ac7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize Model\n",
    "model = Baseline_Model().to(CONFIG[\"device\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "6f956533-8f0b-4dc3-bbc6-03ac4975d84d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Linear-1                   [-1, 32]          16,416\n",
      "              ReLU-2                   [-1, 32]               0\n",
      "           Dropout-3                   [-1, 32]               0\n",
      "            Linear-4                    [-1, 5]             165\n",
      "================================================================\n",
      "Total params: 16,581\n",
      "Trainable params: 16,581\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.00\n",
      "Forward/backward pass size (MB): 0.00\n",
      "Params size (MB): 0.06\n",
      "Estimated Total Size (MB): 0.07\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "summary(model, input_size = (CONFIG[\"feature_input_size\"],))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "3b3268f2-b071-4f84-a307-a41e365ce456",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ## compute accuracy score in a batch\n",
    "# def compute_accuracy(logits, labels, batch_size):\n",
    "#     ''' \n",
    "#       Obtain accuracy for a training batch \n",
    "#       logits: float Tensor, shape (batch_size, num_classes),  output from the model\n",
    "#       labels: Long Tensor, shape (batch_size, ), contains labels for the predictions\n",
    "#       batch_size: int, batch size\n",
    "\n",
    "#       Return accuracy for this batch, which should be a float number\n",
    "#     '''\n",
    "    \n",
    "#     # Write your code in this block ----------------\n",
    "    \n",
    "#     predictions = torch.argmax(logits, dim = 1)\n",
    "#     predictions = predictions.view(-1)\n",
    "    \n",
    "#     total_correct = (labels == predictions).sum().item()\n",
    "#     accuracy = total_correct/batch_size\n",
    "    \n",
    "#     return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb41d874-840c-46ba-a732-b538cff27f27",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84b0e68d-1d4a-4885-92ee-29a9abf92b05",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training on whole batch...: 100%|██████████| 188/188 [00:06<00:00, 30.92it/s]\n",
      "/projectnb/cs640grp/students/avarshn/.conda/envs/ocean/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/projectnb/cs640grp/students/avarshn/.conda/envs/ocean/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/projectnb/cs640grp/students/avarshn/.conda/envs/ocean/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.00      0.00      0.00        53\n",
      "           1       0.18      0.43      0.26        30\n",
      "           2       0.06      0.18      0.10        11\n",
      "           3       0.12      0.12      0.12        24\n",
      "           4       0.00      0.00      0.00        10\n",
      "\n",
      "    accuracy                           0.14       128\n",
      "   macro avg       0.07      0.15      0.10       128\n",
      "weighted avg       0.07      0.14      0.09       128\n",
      "\n",
      "Epoch: 1 | Runtime: 0.10[m] | train_loss: 1.647 | test_loss: 1.623 | train_acc: 0.193 |  test_acc: 0.141 | train_f1: 0.149 | test_f1: 0.095\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training on whole batch...: 100%|██████████| 188/188 [00:06<00:00, 31.00it/s]\n",
      "/projectnb/cs640grp/students/avarshn/.conda/envs/ocean/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/projectnb/cs640grp/students/avarshn/.conda/envs/ocean/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/projectnb/cs640grp/students/avarshn/.conda/envs/ocean/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.50      0.06      0.10        53\n",
      "           1       0.20      0.50      0.29        30\n",
      "           2       0.05      0.09      0.06        11\n",
      "           3       0.11      0.12      0.12        24\n",
      "           4       0.00      0.00      0.00        10\n",
      "\n",
      "    accuracy                           0.17       128\n",
      "   macro avg       0.17      0.15      0.11       128\n",
      "weighted avg       0.28      0.17      0.14       128\n",
      "\n",
      "Epoch: 2 | Runtime: 0.21[m] | train_loss: 1.627 | test_loss: 1.612 | train_acc: 0.214 |  test_acc: 0.172 | train_f1: 0.189 | test_f1: 0.114\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training on whole batch...: 100%|██████████| 188/188 [00:06<00:00, 30.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.54      0.13      0.21        53\n",
      "           1       0.18      0.40      0.25        30\n",
      "           2       0.05      0.09      0.06        11\n",
      "           3       0.10      0.12      0.11        24\n",
      "           4       0.00      0.00      0.00        10\n",
      "\n",
      "    accuracy                           0.18       128\n",
      "   macro avg       0.18      0.15      0.13       128\n",
      "weighted avg       0.29      0.18      0.17       128\n",
      "\n",
      "Epoch: 3 | Runtime: 0.31[m] | train_loss: 1.611 | test_loss: 1.602 | train_acc: 0.219 |  test_acc: 0.180 | train_f1: 0.200 | test_f1: 0.128\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training on whole batch...: 100%|██████████| 188/188 [00:06<00:00, 28.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.52      0.21      0.30        53\n",
      "           1       0.18      0.33      0.24        30\n",
      "           2       0.07      0.09      0.08        11\n",
      "           3       0.11      0.17      0.13        24\n",
      "           4       0.00      0.00      0.00        10\n",
      "\n",
      "    accuracy                           0.20       128\n",
      "   macro avg       0.18      0.16      0.15       128\n",
      "weighted avg       0.29      0.20      0.21       128\n",
      "\n",
      "Epoch: 4 | Runtime: 0.42[m] | train_loss: 1.603 | test_loss: 1.598 | train_acc: 0.234 |  test_acc: 0.203 | train_f1: 0.225 | test_f1: 0.149\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training on whole batch...: 100%|██████████| 188/188 [00:06<00:00, 30.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.46      0.25      0.32        53\n",
      "           1       0.19      0.30      0.23        30\n",
      "           2       0.08      0.09      0.08        11\n",
      "           3       0.14      0.21      0.17        24\n",
      "           4       0.00      0.00      0.00        10\n",
      "\n",
      "    accuracy                           0.22       128\n",
      "   macro avg       0.17      0.17      0.16       128\n",
      "weighted avg       0.27      0.22      0.23       128\n",
      "\n",
      "Epoch: 5 | Runtime: 0.53[m] | train_loss: 1.596 | test_loss: 1.593 | train_acc: 0.243 |  test_acc: 0.219 | train_f1: 0.236 | test_f1: 0.160\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training on whole batch...: 100%|██████████| 188/188 [00:06<00:00, 30.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.45      0.25      0.32        53\n",
      "           1       0.18      0.27      0.21        30\n",
      "           2       0.07      0.09      0.08        11\n",
      "           3       0.14      0.21      0.17        24\n",
      "           4       0.00      0.00      0.00        10\n",
      "\n",
      "    accuracy                           0.21       128\n",
      "   macro avg       0.17      0.16      0.15       128\n",
      "weighted avg       0.26      0.21      0.22       128\n",
      "\n",
      "Epoch: 6 | Runtime: 0.63[m] | train_loss: 1.589 | test_loss: 1.591 | train_acc: 0.258 |  test_acc: 0.211 | train_f1: 0.255 | test_f1: 0.155\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training on whole batch...: 100%|██████████| 188/188 [00:05<00:00, 32.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.44      0.26      0.33        53\n",
      "           1       0.16      0.23      0.19        30\n",
      "           2       0.07      0.09      0.08        11\n",
      "           3       0.14      0.21      0.17        24\n",
      "           4       0.00      0.00      0.00        10\n",
      "\n",
      "    accuracy                           0.21       128\n",
      "   macro avg       0.16      0.16      0.15       128\n",
      "weighted avg       0.25      0.21      0.22       128\n",
      "\n",
      "Epoch: 7 | Runtime: 0.73[m] | train_loss: 1.579 | test_loss: 1.587 | train_acc: 0.275 |  test_acc: 0.211 | train_f1: 0.272 | test_f1: 0.154\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training on whole batch...: 100%|██████████| 188/188 [00:05<00:00, 37.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.47      0.32      0.38        53\n",
      "           1       0.17      0.23      0.20        30\n",
      "           2       0.07      0.09      0.08        11\n",
      "           3       0.16      0.21      0.18        24\n",
      "           4       0.00      0.00      0.00        10\n",
      "\n",
      "    accuracy                           0.23       128\n",
      "   macro avg       0.17      0.17      0.17       128\n",
      "weighted avg       0.27      0.23      0.25       128\n",
      "\n",
      "Epoch: 8 | Runtime: 0.82[m] | train_loss: 1.574 | test_loss: 1.582 | train_acc: 0.289 |  test_acc: 0.234 | train_f1: 0.288 | test_f1: 0.168\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training on whole batch...: 100%|██████████| 188/188 [00:04<00:00, 37.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.49      0.32      0.39        53\n",
      "           1       0.17      0.23      0.20        30\n",
      "           2       0.08      0.09      0.08        11\n",
      "           3       0.17      0.25      0.20        24\n",
      "           4       0.00      0.00      0.00        10\n",
      "\n",
      "    accuracy                           0.24       128\n",
      "   macro avg       0.18      0.18      0.17       128\n",
      "weighted avg       0.28      0.24      0.25       128\n",
      "\n",
      "Epoch: 9 | Runtime: 0.90[m] | train_loss: 1.570 | test_loss: 1.582 | train_acc: 0.309 |  test_acc: 0.242 | train_f1: 0.311 | test_f1: 0.175\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training on whole batch...: 100%|██████████| 188/188 [00:04<00:00, 37.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.45      0.34      0.39        53\n",
      "           1       0.18      0.23      0.20        30\n",
      "           2       0.07      0.09      0.08        11\n",
      "           3       0.16      0.21      0.18        24\n",
      "           4       0.00      0.00      0.00        10\n",
      "\n",
      "    accuracy                           0.24       128\n",
      "   macro avg       0.17      0.17      0.17       128\n",
      "weighted avg       0.26      0.24      0.25       128\n",
      "\n",
      "Epoch: 10 | Runtime: 0.99[m] | train_loss: 1.563 | test_loss: 1.576 | train_acc: 0.303 |  test_acc: 0.242 | train_f1: 0.305 | test_f1: 0.170\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training on whole batch...: 100%|██████████| 188/188 [00:04<00:00, 37.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.45      0.34      0.39        53\n",
      "           1       0.18      0.23      0.20        30\n",
      "           2       0.08      0.09      0.08        11\n",
      "           3       0.16      0.21      0.18        24\n",
      "           4       0.00      0.00      0.00        10\n",
      "\n",
      "    accuracy                           0.24       128\n",
      "   macro avg       0.17      0.17      0.17       128\n",
      "weighted avg       0.26      0.24      0.25       128\n",
      "\n",
      "Epoch: 11 | Runtime: 1.07[m] | train_loss: 1.557 | test_loss: 1.574 | train_acc: 0.333 |  test_acc: 0.242 | train_f1: 0.334 | test_f1: 0.170\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training on whole batch...: 100%|██████████| 188/188 [00:04<00:00, 37.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.41      0.28      0.33        53\n",
      "           1       0.17      0.23      0.20        30\n",
      "           2       0.07      0.09      0.08        11\n",
      "           3       0.15      0.21      0.18        24\n",
      "           4       0.00      0.00      0.00        10\n",
      "\n",
      "    accuracy                           0.22       128\n",
      "   macro avg       0.16      0.16      0.16       128\n",
      "weighted avg       0.24      0.22      0.22       128\n",
      "\n",
      "Epoch: 12 | Runtime: 1.16[m] | train_loss: 1.555 | test_loss: 1.572 | train_acc: 0.337 |  test_acc: 0.219 | train_f1: 0.338 | test_f1: 0.157\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training on whole batch...: 100%|██████████| 188/188 [00:05<00:00, 37.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.41      0.28      0.33        53\n",
      "           1       0.17      0.23      0.20        30\n",
      "           2       0.05      0.09      0.06        11\n",
      "           3       0.17      0.21      0.19        24\n",
      "           4       0.00      0.00      0.00        10\n",
      "\n",
      "    accuracy                           0.22       128\n",
      "   macro avg       0.16      0.16      0.16       128\n",
      "weighted avg       0.25      0.22      0.23       128\n",
      "\n",
      "Epoch: 13 | Runtime: 1.24[m] | train_loss: 1.548 | test_loss: 1.569 | train_acc: 0.349 |  test_acc: 0.219 | train_f1: 0.352 | test_f1: 0.157\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training on whole batch...: 100%|██████████| 188/188 [00:04<00:00, 42.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.45      0.32      0.37        53\n",
      "           1       0.15      0.17      0.16        30\n",
      "           2       0.07      0.18      0.11        11\n",
      "           3       0.15      0.17      0.16        24\n",
      "           4       0.00      0.00      0.00        10\n",
      "\n",
      "    accuracy                           0.22       128\n",
      "   macro avg       0.16      0.17      0.16       128\n",
      "weighted avg       0.25      0.22      0.23       128\n",
      "\n",
      "Epoch: 14 | Runtime: 1.32[m] | train_loss: 1.545 | test_loss: 1.565 | train_acc: 0.344 |  test_acc: 0.219 | train_f1: 0.347 | test_f1: 0.158\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training on whole batch...: 100%|██████████| 188/188 [00:04<00:00, 42.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.42      0.30      0.35        53\n",
      "           1       0.15      0.17      0.16        30\n",
      "           2       0.08      0.18      0.11        11\n",
      "           3       0.17      0.21      0.19        24\n",
      "           4       0.00      0.00      0.00        10\n",
      "\n",
      "    accuracy                           0.22       128\n",
      "   macro avg       0.16      0.17      0.16       128\n",
      "weighted avg       0.25      0.22      0.23       128\n",
      "\n",
      "Epoch: 15 | Runtime: 1.39[m] | train_loss: 1.541 | test_loss: 1.562 | train_acc: 0.338 |  test_acc: 0.219 | train_f1: 0.340 | test_f1: 0.161\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training on whole batch...:  21%|██▏       | 40/188 [00:00<00:03, 42.26it/s]"
     ]
    }
   ],
   "source": [
    "# Train Model\n",
    "\n",
    "history_model = train_model(model, trainloader, testloader, CONFIG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad0b4e03-74a6-4f50-ab92-66ae5087fb36",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "fe441e3b-ca49-4169-9237-25c32dd4891b",
   "metadata": {},
   "source": [
    "# Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "4d6bb5c3-1575-43e9-94b0-35e272d47f45",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x15169225aee0>"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkgAAAGwCAYAAABSN5pGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/SrBM8AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA68ElEQVR4nO3de1RVZeL/8c8RBbkIiAocDATRUAlvmY7aSh0xxTJNl34zK82cprLxa5IVU1pqaVlTlGPZ5OStpppJpZouGhpekFA01NQwCEURtEw44gUU9u+Pvu5fZ4uGBB7Q92utvWLv/eznBnY+a+/nnGMzDMMQAAAATA1c3QEAAIC6hoAEAABgQUACAACwICABAABYEJAAAAAsCEgAAAAWBCQAAACLhq7uQH1VUVGhQ4cOqUmTJrLZbK7uDgAAqALDMHT8+HGFhISoQYML3yciIFXToUOHFBoa6upuAACAajhw4ICuueaaC54nIFVTkyZNJP0ywb6+vi7uDQAAqAqHw6HQ0FDzdfxCCEjVdO6xmq+vLwEJAIB65reWx7BIGwAAwIKABAAAYEFAAgAAsGANEgAAlSgvL9eZM2dc3Q1cokaNGsnNze1310NAAgDgVwzDUGFhoYqKilzdFVSTv7+/goODf9fnFBKQAAD4lXPhKDAwUF5eXnwYcD1iGIZOnjypI0eOSJLsdnu16yIgAQDwf8rLy81w1KxZM1d3B9Xg6ekpSTpy5IgCAwOr/biNRdoAAPyfc2uOvLy8XNwT/B7nfn+/Zw0ZAQkAAAseq9VvNfH7IyABAABYEJAAAAAsCEgAAOA84eHhSkxMdHkdrsK72AAAuAL07dtXnTt3rrFAsmXLFnl7e9dIXfURAQkAgKuEYRgqLy9Xw4a//fLfokWLy9CjuotHbAAAXIRhGDpZdtYlm2EYVerjuHHjtG7dOr366quy2Wyy2Wzat2+fUlJSZLPZ9Pnnn+v666+Xh4eHNm7cqJycHA0dOlRBQUHy8fHRDTfcoOTkZKc6rY/HbDabFi5cqNtvv11eXl5q27atPv7440uay7y8PA0dOlQ+Pj7y9fXVqFGjdPjwYfP89u3b1a9fPzVp0kS+vr66/vrrlZGRIUnav3+/hgwZoqZNm8rb21vR0dH67LPPLqn9S8EdJAAALuLUmXJ1mL7KJW3vnjlQXu6//VL96quvau/evbruuus0c+ZMSb/cAdq3b58k6YknntBLL72k1q1bq2nTpjpw4IAGDx6s5557Th4eHlq6dKmGDBmirKwshYWFXbCdGTNmaO7cuXrxxRc1b948jRkzRvv371dAQMBv9rGiosIMR+vWrdPZs2c1ceJE/c///I9SUlIkSWPGjFGXLl30xhtvyM3NTZmZmWrUqJEkaeLEiSorK9P69evl7e2t3bt3y8fH5zfbrS4CEgAA9Zyfn5/c3d3l5eWl4ODg887PnDlTAwYMMPcDAgLUqVMnc3/WrFlauXKlPv74Yz388MMXbGfcuHEaPXq0JGn27Nl67bXXtHnzZg0aNOg3+7hmzRrt3LlTubm5Cg0NlSQtXbpU0dHR2rJli2644Qbl5eVp6tSpateunSSpbdu25vV5eXkaMWKEYmJiJEmtW7f+zTZ/DwISAAAX4dnITbtnDnRZ2zWhW7duTvslJSV65pln9Omnn6qgoEBnz57VqVOnlJeXd9F6OnbsaP7s7e0tX19f83vPfsuePXsUGhpqhiNJ6tChg/z9/bVnzx7dcMMNmjJliiZMmKBly5YpNjZWI0eOVGRkpCRp0qRJevDBB7V69WrFxsZqxIgRTv2paaxBAgDgImw2m7zcG7pkq6lP9La+G+3RRx/VypUrNXv2bG3YsEGZmZmKiYlRWVnZRes597jr13NTUVFRI32UpGeeeUa7du3SLbfcorVr16pDhw5auXKlJGnChAn64YcfdPfdd2vnzp3q1q2b5s2bV2NtWxGQAAC4Ari7u6u8vLxKZVNTUzVu3DjdfvvtiomJUXBwsLleqba0b99eBw4c0IEDB8xju3fvVlFRkTp06GAeu/baa/XII49o9erVGj58uBYtWmSeCw0N1QMPPKAVK1YoPj5eb731Vq31l4AEAMAVIDw8XOnp6dq3b59++umni97Zadu2rVasWKHMzExt375dd955Z43eCapMbGysYmJiNGbMGG3btk2bN2/WPffcoz59+qhbt246deqUHn74YaWkpGj//v1KTU3Vli1b1L59e0nS5MmTtWrVKuXm5mrbtm366quvzHO1gYAEAMAV4NFHH5Wbm5s6dOigFi1aXHQ90csvv6ymTZuqV69eGjJkiAYOHKiuXbvWav9sNps++ugjNW3aVDfddJNiY2PVunVrffDBB5IkNzc3HT16VPfcc4+uvfZajRo1SnFxcZoxY4Ykqby8XBMnTlT79u01aNAgXXvttXr99ddrr79GVT9kAU4cDof8/PxUXFwsX19fV3cHAFADTp8+rdzcXEVERKhx48au7g6q6WK/x6q+fnMHCQAAwIKABAAAYEFAAgAAsCAgAQAAWBCQAAAALAhIAAAAFgQkAAAACwISAACAhUsD0vr16zVkyBCFhITIZrMpKSnpouVTUlJks9nO2woLC80yx48f1+TJk9WqVSt5enqqV69e2rJli1M948aNO6+OQYMG1cYQAQC44vXt21eTJ092dTdqlEsD0okTJ9SpUyfNnz//kq7LyspSQUGBuQUGBprnJkyYoC+//FLLli3Tzp07dfPNNys2Nlb5+flOdQwaNMipjvfee69GxgQAgCvURkgZN26chg0bVqN11hcNXdl4XFyc4uLiLvm6wMBA+fv7n3f81KlTWr58uT766CPddNNNkqRnnnlGn3zyid544w09++yzZlkPDw8FBwdXu+8AAODKVS/XIHXu3Fl2u10DBgxQamqqefzs2bMqLy8/73tXPD09tXHjRqdjKSkpCgwMVFRUlB588EEdPXr0om2WlpbK4XA4bQAA1AXjxo3TunXr9Oqrr5pLR/bt2ydJ+vbbbxUXFycfHx8FBQXp7rvv1k8//WRe++GHHyomJkaenp5q1qyZYmNjdeLECT3zzDNasmSJPvroI7POlJSUKvXn2LFjuueee9S0aVN5eXkpLi5O33//vXl+//79GjJkiJo2bSpvb29FR0frs88+M68dM2aMWrRoIU9PT7Vt21aLFi2qsbmqKpfeQbpUdrtdCxYsULdu3VRaWqqFCxeqb9++Sk9PV9euXdWkSRP17NlTs2bNUvv27RUUFKT33ntPaWlpatOmjVnPoEGDNHz4cEVERCgnJ0d//etfFRcXp7S0NLm5uVXa9pw5c8xvFAYAXEUMQzpz0jVtN/KSbLbfLPbqq69q7969uu666zRz5kxJUosWLVRUVKQ//vGPmjBhgl555RWdOnVKjz/+uEaNGqW1a9eqoKBAo0eP1ty5c3X77bfr+PHj2rBhgwzD0KOPPqo9e/bI4XCYASUgIKBK3R43bpy+//57ffzxx/L19dXjjz+uwYMHa/fu3WrUqJEmTpyosrIyrV+/Xt7e3tq9e7d8fHwkSdOmTdPu3bv1+eefq3nz5srOztapU6eqOYHVV68CUlRUlKKiosz9Xr16KScnR6+88oqWLVsmSVq2bJnGjx+vli1bys3NTV27dtXo0aO1detW87o77rjD/DkmJkYdO3ZUZGSkUlJS1L9//0rbTkhI0JQpU8x9h8Oh0NDQmh4iAKCuOXNSmh3imrb/ekhy9/7NYn5+fnJ3d5eXl5fT8pG///3v6tKli2bPnm0ee/vttxUaGqq9e/eqpKREZ8+e1fDhw9WqVStJv7wunuPp6anS0tJLWpJyLhilpqaqV69ekqR3331XoaGhSkpK0siRI5WXl6cRI0aYbbVu3dq8Pi8vT126dFG3bt0kSeHh4VVuuybVy0dsv9a9e3dlZ2eb+5GRkVq3bp1KSkp04MABbd68WWfOnHGafKvWrVubKfVCPDw85Ovr67QBAFCXbd++XV999ZV8fHzMrV27dpKknJwcderUSf3791dMTIxGjhypt956S8eOHftdbe7Zs0cNGzZUjx49zGPNmjVTVFSU9uzZI0maNGmSnn32WfXu3VtPP/20duzYYZZ98MEH9f7776tz58567LHHtGnTpt/Vn+qqV3eQKpOZmSm73X7ecW9vb3l7e+vYsWNatWqV5s6de8E6Dh48qKNHj1ZaDwDgKtfI65c7Oa5q+3coKSnRkCFD9MILL5x3zm63y83NTV9++aU2bdqk1atXa968eXryySeVnp6uiIiI39X2xUyYMEEDBw7Up59+qtWrV2vOnDn629/+pr/85S+Ki4vT/v379dlnn+nLL79U//79NXHiRL300ku11p/KuPQOUklJiTIzM5WZmSlJys3NVWZmpvLy8iT98ljrnnvuMcsnJibqo48+UnZ2tr799ltNnjxZa9eu1cSJE80yq1at0hdffKHc3Fx9+eWX6tevn9q1a6d7773XbHPq1Kn6+uuvtW/fPq1Zs0ZDhw5VmzZtNHDgwMs3eABA/WCz/fKYyxVbFdYfnePu7q7y8nKnY127dtWuXbsUHh6uNm3aOG3e3t7/NzybevfurRkzZuibb76Ru7u7Vq5cecE6f0v79u119uxZpaenm8eOHj2qrKwsdejQwTwWGhqqBx54QCtWrFB8fLzeeust81yLFi00duxYvfPOO0pMTNQ//vGPS+pDTXBpQMrIyFCXLl3UpUsXSdKUKVPUpUsXTZ8+XZJUUFBghiVJKisrU3x8vGJiYtSnTx9t375dycnJTuuGiouLNXHiRLVr10733HOPbrzxRq1atUqNGjWSJLm5uWnHjh267bbbdO211+q+++7T9ddfrw0bNsjDw+Myjh4AgJoTHh6u9PR07du3Tz/99JMqKio0ceJE/fzzzxo9erS2bNminJwcrVq1Svfee6/Ky8uVnp6u2bNnKyMjQ3l5eVqxYoV+/PFHtW/f3qxzx44dysrK0k8//aQzZ878Zj/atm2roUOH6k9/+pM2btyo7du366677lLLli01dOhQSdLkyZO1atUq5ebmatu2bfrqq6/MNqdPn27eDNm1a5f++9//mucuKwPVUlxcbEgyiouLXd0VAEANOXXqlLF7927j1KlTru7KJcvKyjL+8Ic/GJ6enoYkIzc31zAMw9i7d69x++23G/7+/oanp6fRrl07Y/LkyUZFRYWxe/duY+DAgUaLFi0MDw8P49prrzXmzZtn1nnkyBFjwIABho+PjyHJ+Oqrryptu0+fPsb//u//mvs///yzcffddxt+fn6Gp6enMXDgQGPv3r3m+YcfftiIjIw0PDw8jBYtWhh333238dNPPxmGYRizZs0y2rdvb3h6ehoBAQHG0KFDjR9++OGS5uJiv8eqvn7bDMMwLn8sq/8cDof8/PxUXFzMgm0AuEKcPn1aubm5ioiIOO8z9VB/XOz3WNXX73r/LjYAAICaRkACAACwICABAABYEJAAAAAsCEgAAFjw/qX6rSZ+fwQkAAD+z7nPzDt50kVfTosace73d+73WR31/qtGAACoKW5ubvL399eRI0ckSV5eXrJdwqdZw7UMw9DJkyd15MgR+fv7y83Nrdp1EZAAAPiVc99cfy4kof7x9/c3f4/VRUACAOBXbDab7Ha7AgMDq/TVGqhbGjVq9LvuHJ1DQAIAoBJubm418kKL+olF2gAAABYEJAAAAAsCEgAAgAUBCQAAwIKABAAAYEFAAgAAsCAgAQAAWBCQAAAALAhIAAAAFgQkAAAACwISAACABQEJAADAgoAEAABgQUACAACwICABAABYEJAAAAAsCEgAAAAWBCQAAAALAhIAAIAFAQkAAMCCgAQAAGBBQAIAALAgIAEAAFgQkAAAACwISAAAABYEJAAAAAsCEgAAgAUBCQAAwIKABAAAYEFAAgAAsCAgAQAAWLg0IK1fv15DhgxRSEiIbDabkpKSLlo+JSVFNpvtvK2wsNAsc/z4cU2ePFmtWrWSp6enevXqpS1btjjVYxiGpk+fLrvdLk9PT8XGxur777+vjSECAIB6yKUB6cSJE+rUqZPmz59/SddlZWWpoKDA3AIDA81zEyZM0Jdffqlly5Zp586duvnmmxUbG6v8/HyzzNy5c/Xaa69pwYIFSk9Pl7e3twYOHKjTp0/X2NgAAED9ZTMMw3B1JyTJZrNp5cqVGjZs2AXLpKSkqF+/fjp27Jj8/f3PO3/q1Ck1adJEH330kW655Rbz+PXXX6+4uDg9++yzMgxDISEhio+P16OPPipJKi4uVlBQkBYvXqw77rij0rZLS0tVWlpq7jscDoWGhqq4uFi+vr7VGzQAALisHA6H/Pz8fvP1u16uQercubPsdrsGDBig1NRU8/jZs2dVXl6uxo0bO5X39PTUxo0bJUm5ubkqLCxUbGysed7Pz089evRQWlraBducM2eO/Pz8zC00NLSGRwUAAOqKehWQ7Ha7FixYoOXLl2v58uUKDQ1V3759tW3bNklSkyZN1LNnT82aNUuHDh1SeXm53nnnHaWlpamgoECSzPVKQUFBTnUHBQU5rWWySkhIUHFxsbkdOHCglkYJAABcraGrO3ApoqKiFBUVZe736tVLOTk5euWVV7Rs2TJJ0rJlyzR+/Hi1bNlSbm5u6tq1q0aPHq2tW7f+rrY9PDzk4eHxu+oAAAD1Q726g1SZ7t27Kzs729yPjIzUunXrVFJSogMHDmjz5s06c+aMWrduLUkKDg6WJB0+fNipnsOHD5vnAADA1a3eB6TMzEzZ7fbzjnt7e8tut+vYsWNatWqVhg4dKkmKiIhQcHCw1qxZY5Z1OBxKT09Xz549L1u/AQBA3eXSR2wlJSVOd39yc3OVmZmpgIAAhYWFKSEhQfn5+Vq6dKkkKTExUREREYqOjtbp06e1cOFCrV27VqtXrzbrWLVqlQzDUFRUlLKzszV16lS1a9dO9957r6Rf3i03efJkPfvss2rbtq0iIiI0bdo0hYSEXPQddAAA4Orh0oCUkZGhfv36mftTpkyRJI0dO1aLFy9WQUGB8vLyzPNlZWWKj49Xfn6+vLy81LFjRyUnJzvVUVxcrISEBB08eFABAQEaMWKEnnvuOTVq1Mgs89hjj+nEiRO6//77VVRUpBtvvFFffPHFee9+AwAAV6c68zlI9U1VP0cBAADUHVf05yABAADUJgISAACABQEJAADAgoAEAABgQUACAACwICABAABYEJAAAAAsCEgAAAAWBCQAAAALAhIAAIAFAQkAAMCCgAQAAGBBQAIAALAgIAEAAFgQkAAAACwISAAAABYEJAAAAAsCEgAAgAUBCQAAwIKABAAAYEFAAgAAsCAgAQAAWBCQAAAALAhIAAAAFgQkAAAACwISAACABQEJAADAgoAEAABgQUACAACwICABAABYEJAAAAAsCEgAAAAWBCQAAAALAhIAAIAFAQkAAMCCgAQAAGBBQAIAALAgIAEAAFgQkAAAACwISAAAABYEJAAAAAuXBqT169dryJAhCgkJkc1mU1JS0kXLp6SkyGaznbcVFhaaZcrLyzVt2jRFRETI09NTkZGRmjVrlgzDMMuMGzfuvDoGDRpUW8MEAAD1TENXNn7ixAl16tRJ48eP1/Dhw6t8XVZWlnx9fc39wMBA8+cXXnhBb7zxhpYsWaLo6GhlZGTo3nvvlZ+fnyZNmmSWGzRokBYtWmTue3h4/M7RAACAK4VLA1JcXJzi4uIu+brAwED5+/tXem7Tpk0aOnSobrnlFklSeHi43nvvPW3evNmpnIeHh4KDgy+5bQAAcOWrl2uQOnfuLLvdrgEDBig1NdXpXK9evbRmzRrt3btXkrR9+3Zt3LjxvCCWkpKiwMBARUVF6cEHH9TRo0cv2mZpaakcDofTBgAArkwuvYN0qex2uxYsWKBu3bqptLRUCxcuVN++fZWenq6uXbtKkp544gk5HA61a9dObm5uKi8v13PPPacxY8aY9QwaNEjDhw9XRESEcnJy9Ne//lVxcXFKS0uTm5tbpW3PmTNHM2bMuCzjBAAArmUzfr162YVsNptWrlypYcOGXdJ1ffr0UVhYmJYtWyZJev/99zV16lS9+OKLio6OVmZmpiZPnqyXX35ZY8eOrbSOH374QZGRkUpOTlb//v0rLVNaWqrS0lJz3+FwKDQ0VMXFxU7roQAAQN3lcDjk5+f3m6/f9eoOUmW6d++ujRs3mvtTp07VE088oTvuuEOSFBMTo/3792vOnDkXDEitW7dW8+bNlZ2dfcGA5OHhwUJuAACuEvVyDdKvZWZmym63m/snT55UgwbOw3Jzc1NFRcUF6zh48KCOHj3qVA8AALh6ufQOUklJibKzs8393NxcZWZmKiAgQGFhYUpISFB+fr6WLl0qSUpMTFRERISio6N1+vRpLVy4UGvXrtXq1avNOoYMGaLnnntOYWFhio6O1jfffKOXX35Z48ePN9ucMWOGRowYoeDgYOXk5Oixxx5TmzZtNHDgwMs7AQAAoE5yaUDKyMhQv379zP0pU6ZIksaOHavFixeroKBAeXl55vmysjLFx8crPz9fXl5e6tixo5KTk53qmDdvnqZNm6aHHnpIR44cUUhIiP785z9r+vTpkn65m7Rjxw4tWbJERUVFCgkJ0c0336xZs2bxCA0AAEiqQ4u065uqLvICAAB1R1Vfv+v9GiQAAICaRkACAACwICABAABYEJAAAAAsCEgAAAAWBCQAAAALAhIAAIAFAQkAAMCCgAQAAGBBQAIAALAgIAEAAFgQkAAAACwISAAAABYEJAAAAAsCEgAAgAUBCQAAwIKABAAAYEFAAgAAsCAgAQAAWBCQAAAALAhIAAAAFgQkAAAACwISAACABQEJAADAgoAEAABgQUACAACwqFZAWrJkiT799FNz/7HHHpO/v7969eql/fv311jnAAAAXKFaAWn27Nny9PSUJKWlpWn+/PmaO3eumjdvrkceeaRGOwgAAHC5NazORQcOHFCbNm0kSUlJSRoxYoTuv/9+9e7dW3379q3J/gEAAFx21bqD5OPjo6NHj0qSVq9erQEDBkiSGjdurFOnTtVc7wAAAFygWneQBgwYoAkTJqhLly7au3evBg8eLEnatWuXwsPDa7J/AAAAl1217iDNnz9fPXv21I8//qjly5erWbNmkqStW7dq9OjRNdpBAACAy81mGIbh6k7URw6HQ35+fiouLpavr6+ruwMAAKqgqq/f1bqD9MUXX2jjxo3m/vz589W5c2fdeeedOnbsWHWqBAAAqDOqFZCmTp0qh8MhSdq5c6fi4+M1ePBg5ebmasqUKTXaQQAAgMutWou0c3Nz1aFDB0nS8uXLdeutt2r27Nnatm2buWAbAACgvqrWHSR3d3edPHlSkpScnKybb75ZkhQQEGDeWQIAAKivqnUH6cYbb9SUKVPUu3dvbd68WR988IEkae/evbrmmmtqtIMAAACXW7XuIP39739Xw4YN9eGHH+qNN95Qy5YtJUmff/65Bg0aVKMdBAAAuNx4m3818TZ/AADqn6q+flfrEZsklZeXKykpSXv27JEkRUdH67bbbpObm1t1qwQAAKgTqhWQsrOzNXjwYOXn5ysqKkqSNGfOHIWGhurTTz9VZGRkjXYSAADgcqrWGqRJkyYpMjJSBw4c0LZt27Rt2zbl5eUpIiJCkyZNqnI969ev15AhQxQSEiKbzaakpKSLlk9JSZHNZjtvKywsNMuUl5dr2rRpioiIkKenpyIjIzVr1iz9+kmiYRiaPn267Ha7PD09FRsbq++///6S5wEAAFyZqnUHad26dfr6668VEBBgHmvWrJmef/559e7du8r1nDhxQp06ddL48eM1fPjwKl+XlZXl9NwwMDDQ/PmFF17QG2+8oSVLlig6OloZGRm699575efnZ4a3uXPn6rXXXtOSJUsUERGhadOmaeDAgdq9e7caN25c5X4AAIArU7UCkoeHh44fP37e8ZKSErm7u1e5nri4OMXFxV1y+4GBgfL396/03KZNmzR06FDdcsstkqTw8HC999572rx5s6Rf7h4lJibqqaee0tChQyVJS5cuVVBQkJKSknTHHXdccn8AAMCVpVqP2G699Vbdf//9Sk9Pl2EYMgxDX3/9tR544AHddtttNd3H83Tu3Fl2u10DBgxQamqq07levXppzZo12rt3ryRp+/bt2rhxoxnEcnNzVVhYqNjYWPMaPz8/9ejRQ2lpaRdss7S0VA6Hw2kDAABXpmoFpNdee02RkZHq2bOnGjdurMaNG6tXr15q06aNEhMTa7iL/5/dbteCBQu0fPlyLV++XKGhoerbt6+2bdtmlnniiSd0xx13qF27dmrUqJG6dOmiyZMna8yYMZJkrlcKCgpyqjsoKMhpLZPVnDlz5OfnZ26hoaG1MEIAAFAXVOsRm7+/vz766CNlZ2ebb/Nv37692rRpU6Ods4qKijLfNSf9crcoJydHr7zyipYtWyZJ+ve//613331X//rXvxQdHa3MzExNnjxZISEhGjt2bLXbTkhIcPoiXofDQUgCAOAKVeWA9OtwUJmvvvrK/Pnll1+ufo8uUffu3bVx40Zzf+rUqeZdJEmKiYnR/v37NWfOHI0dO1bBwcGSpMOHD8tut5vXHT58WJ07d75gOx4eHvLw8KidQQAAgDqlygHpm2++qVI5m81W7c5UR2ZmplPQOXnypBo0cH5y6ObmpoqKCklSRESEgoODtWbNGjMQORwOpaen68EHH7xs/QYAAHVXlQPSr+8Q1ZSSkhJlZ2eb+7m5ucrMzFRAQIDCwsKUkJCg/Px8LV26VJKUmJioiIgIRUdH6/Tp01q4cKHWrl2r1atXm3UMGTJEzz33nMLCwhQdHa1vvvlGL7/8ssaPHy/plwA3efJkPfvss2rbtq35Nv+QkBANGzasxscIAADqn2p/1UhNyMjIUL9+/cz9c4/xxo4dq8WLF6ugoEB5eXnm+bKyMsXHxys/P19eXl7q2LGjkpOTneqYN2+epk2bpoceekhHjhxRSEiI/vznP2v69Olmmccee0wnTpzQ/fffr6KiIt1444364osv+AwkAAAgiS+rrTa+rBYAgPqnqq/f1XqbPwAAwJWMgAQAAGBBQAIAALAgIAEAAFgQkAAAACwISAAAABYEJAAAAAsCEgAAgAUBCQAAwIKABAAAYEFAAgAAsCAgAQAAWBCQAAAALAhIAAAAFgQkAAAACwISAACABQEJAADAgoAEAABgQUACAACwICABAABYEJAAAAAsCEgAAAAWBCQAAAALAhIAAIAFAQkAAMCCgAQAAGBBQAIAALAgIAEAAFgQkAAAACwISAAAABYEJAAAAAsCEgAAgAUBCQAAwIKABAAAYEFAAgAAsCAgAQAAWBCQAAAALAhIAAAAFgQkAAAACwISAACABQEJAADAgoAEAABg4dKAtH79eg0ZMkQhISGy2WxKSkq6aPmUlBTZbLbztsLCQrNMeHh4pWUmTpxolunbt+955x944IHaGiYAAKhnGrqy8RMnTqhTp04aP368hg8fXuXrsrKy5Ovra+4HBgaaP2/ZskXl5eXm/rfffqsBAwZo5MiRTnX86U9/0syZM819Ly+v6gwBAABcgVwakOLi4hQXF3fJ1wUGBsrf37/Scy1atHDaf/755xUZGak+ffo4Hffy8lJwcHCV2ywtLVVpaam573A4qt5hAABQr9TLNUidO3eW3W7XgAEDlJqaesFyZWVleueddzR+/HjZbDanc++++66aN2+u6667TgkJCTp58uRF25wzZ478/PzMLTQ0tEbGAgAA6h6X3kG6VHa7XQsWLFC3bt1UWlqqhQsXqm/fvkpPT1fXrl3PK5+UlKSioiKNGzfO6fidd96pVq1aKSQkRDt27NDjjz+urKwsrVix4oJtJyQkaMqUKea+w+EgJAEAcIWyGYZhuLoTkmSz2bRy5UoNGzbskq7r06ePwsLCtGzZsvPODRw4UO7u7vrkk08uWsfatWvVv39/ZWdnKzIyskrtOhwO+fn5qbi42Gk9FAAAqLuq+vpdLx+x/Vr37t2VnZ193vH9+/crOTlZEyZM+M06evToIUmV1gMAAK4+9T4gZWZmym63n3d80aJFCgwM1C233FKlOiRVWg8AALj6uHQNUklJidNdm9zcXGVmZiogIEBhYWFKSEhQfn6+li5dKklKTExURESEoqOjdfr0aS1cuFBr167V6tWrneqtqKjQokWLNHbsWDVs6DzEnJwc/etf/9LgwYPVrFkz7dixQ4888ohuuukmdezYsfYHDQAA6jyXBqSMjAz169fP3D+3CHrs2LFavHixCgoKlJeXZ54vKytTfHy88vPz5eXlpY4dOyo5OdmpDklKTk5WXl6exo8ff16b7u7uSk5OVmJiok6cOKHQ0FCNGDFCTz31VC2NEgAA1Dd1ZpF2fcMibQAA6p+rZpE2AABATSMgAQAAWBCQAAAALAhIAAAAFgQkAAAACwISAACABQEJAADAgoAEAABgQUACAACwICABAABYEJAAAAAsCEgAAAAWBCQAAAALAhIAAIAFAQkAAMCCgAQAAGBBQAIAALAgIAEAAFgQkAAAACwISAAAABYEJAAAAAsCEgAAgAUBCQAAwIKABAAAYEFAAgAAsCAgAQAAWBCQAAAALAhIAAAAFgQkAAAACwISAACABQEJAADAgoAEAABgQUACAACwICABAABYEJAAAAAsCEgAAAAWBCQAAAALAhIAAIAFAQkAAMCCgAQAAGBBQAIAALBwaUBav369hgwZopCQENlsNiUlJV20fEpKimw223lbYWGhWSY8PLzSMhMnTjTLnD59WhMnTlSzZs3k4+OjESNG6PDhw7U1TAAAUM+4NCCdOHFCnTp10vz58y/puqysLBUUFJhbYGCgeW7Lli1O57788ktJ0siRI80yjzzyiD755BP95z//0bp163To0CENHz68ZgYFAADqvYaubDwuLk5xcXGXfF1gYKD8/f0rPdeiRQun/eeff16RkZHq06ePJKm4uFj//Oc/9a9//Ut//OMfJUmLFi1S+/bt9fXXX+sPf/jDJfcHAABcWerlGqTOnTvLbrdrwIABSk1NvWC5srIyvfPOOxo/frxsNpskaevWrTpz5oxiY2PNcu3atVNYWJjS0tIuWFdpaakcDofTBgAArkz1KiDZ7XYtWLBAy5cv1/LlyxUaGqq+fftq27ZtlZZPSkpSUVGRxo0bZx4rLCyUu7v7eXeggoKCnNYyWc2ZM0d+fn7mFhoaWhNDAgAAdZBLH7FdqqioKEVFRZn7vXr1Uk5Ojl555RUtW7bsvPL//Oc/FRcXp5CQkN/ddkJCgqZMmWLuOxwOQhIAAFeoehWQKtO9e3dt3LjxvOP79+9XcnKyVqxY4XQ8ODhYZWVlKioqcrqLdPjwYQUHB1+wHQ8PD3l4eNRYvwEAQN1Vrx6xVSYzM1N2u/2844sWLVJgYKBuueUWp+PXX3+9GjVqpDVr1pjHsrKylJeXp549e9Z6fwEAQN3n0jtIJSUlys7ONvdzc3OVmZmpgIAAhYWFKSEhQfn5+Vq6dKkkKTExUREREYqOjtbp06e1cOFCrV27VqtXr3aqt6KiQosWLdLYsWPVsKHzEP38/HTfffdpypQpCggIkK+vr/7yl7+oZ8+evIMNAABIcnFAysjIUL9+/cz9c2t8xo4dq8WLF6ugoEB5eXnm+bKyMsXHxys/P19eXl7q2LGjkpOTneqQpOTkZOXl5Wn8+PGVtvvKK6+oQYMGGjFihEpLSzVw4EC9/vrrtTBCAABQH9kMwzBc3Yn6yOFwyM/PT8XFxfL19XV1dwAAQBVU9fW73q9BAgAAqGkEJAAAAAsCEgAAgAUBCQAAwIKABAAAYEFAAgAAsCAgAQAAWBCQAAAALAhIAAAAFgQkAAAACwISAACABQEJAADAgoAEAABgQUACAACwICABAABYEJAAAAAsCEgAAAAWBCQAAAALAhIAAIAFAQkAAMCCgAQAAGBBQAIAALAgIAEAAFgQkAAAACwISAAAABYEJAAAAAsCEgAAgAUBCQAAwIKABAAAYEFAAgAAsCAgAQAAWBCQAAAALAhIAAAAFgQkAAAACwISAACABQEJAADAgoAEAABgQUACAACwICABAABYEJAAAAAsCEgAAAAWBCQAAAALlwak9evXa8iQIQoJCZHNZlNSUtJFy6ekpMhms523FRYWOpXLz8/XXXfdpWbNmsnT01MxMTHKyMgwz48bN+68OgYNGlQbQwQAAPVQQ1c2fuLECXXq1Enjx4/X8OHDq3xdVlaWfH19zf3AwEDz52PHjql3797q16+fPv/8c7Vo0ULff/+9mjZt6lTHoEGDtGjRInPfw8Pjd4wEAABcSVwakOLi4hQXF3fJ1wUGBsrf37/Scy+88IJCQ0Odwk9ERMR55Tw8PBQcHHzJbQMAgCtfvVyD1LlzZ9ntdg0YMECpqalO5z7++GN169ZNI0eOVGBgoLp06aK33nrrvDpSUlIUGBioqKgoPfjggzp69OhF2ywtLZXD4XDaAADAlaleBSS73a4FCxZo+fLlWr58uUJDQ9W3b19t27bNLPPDDz/ojTfeUNu2bbVq1So9+OCDmjRpkpYsWWKWGTRokJYuXao1a9bohRde0Lp16xQXF6fy8vILtj1nzhz5+fmZW2hoaK2OFQAAuI7NMAzD1Z2QJJvNppUrV2rYsGGXdF2fPn0UFhamZcuWSZLc3d3VrVs3bdq0ySwzadIkbdmyRWlpaZXW8cMPPygyMlLJycnq379/pWVKS0tVWlpq7jscDoWGhqq4uNhpPRQAAKi7HA6H/Pz8fvP1u17dQapM9+7dlZ2dbe7b7XZ16NDBqUz79u2Vl5d3wTpat26t5s2bO9Vj5eHhIV9fX6cNAABcmep9QMrMzJTdbjf3e/furaysLKcye/fuVatWrS5Yx8GDB3X06FGnegAAwNXLpe9iKykpcbprk5ubq8zMTAUEBCgsLEwJCQnKz8/X0qVLJUmJiYmKiIhQdHS0Tp8+rYULF2rt2rVavXq1WccjjzyiXr16afbs2Ro1apQ2b96sf/zjH/rHP/5htjljxgyNGDFCwcHBysnJ0WOPPaY2bdpo4MCBl3cCAABAneTSgJSRkaF+/fqZ+1OmTJEkjR07VosXL1ZBQYHTo7GysjLFx8crPz9fXl5e6tixo5KTk53quOGGG7Ry5UolJCRo5syZioiIUGJiosaMGSNJcnNz044dO7RkyRIVFRUpJCREN998s2bNmsVnIQEAAEl1aJF2fVPVRV4AAKDuuGoWaQMAANQ0AhIAAIAFAQkAAMCCgAQAAGBBQAIAALAgIAEAAFgQkAAAACwISAAAABYu/STt+uzc52s6HA4X9wQAAFTVudft3/qcbAJSNR0/flySFBoa6uKeAACAS3X8+HH5+fld8DxfNVJNFRUVOnTokJo0aSKbzebq7ricw+FQaGioDhw4wFev1CLm+fJgni8P5vnyYJ6dGYah48ePKyQkRA0aXHilEXeQqqlBgwa65pprXN2NOsfX15d/gJcB83x5MM+XB/N8eTDP/9/F7hydwyJtAAAACwISAACABQEJNcLDw0NPP/20PDw8XN2VKxrzfHkwz5cH83x5MM/VwyJtAAAAC+4gAQAAWBCQAAAALAhIAAAAFgQkAAAACwISquTnn3/WmDFj5OvrK39/f913330qKSm56DWnT5/WxIkT1axZM/n4+GjEiBE6fPhwpWWPHj2qa665RjabTUVFRbUwgvqhNuZ5+/btGj16tEJDQ+Xp6an27dvr1Vdfre2h1Cnz589XeHi4GjdurB49emjz5s0XLf+f//xH7dq1U+PGjRUTE6PPPvvM6bxhGJo+fbrsdrs8PT0VGxur77//vjaHUG/U5FyfOXNGjz/+uGJiYuTt7a2QkBDdc889OnToUG0Po86r6b/pX3vggQdks9mUmJhYw72uZwygCgYNGmR06tTJ+Prrr40NGzYYbdq0MUaPHn3Rax544AEjNDTUWLNmjZGRkWH84Q9/MHr16lVp2aFDhxpxcXGGJOPYsWO1MIL6oTbm+Z///KcxadIkIyUlxcjJyTGWLVtmeHp6GvPmzavt4dQJ77//vuHu7m68/fbbxq5du4w//elPhr+/v3H48OFKy6emphpubm7G3Llzjd27dxtPPfWU0ahRI2Pnzp1mmeeff97w8/MzkpKSjO3btxu33XabERERYZw6depyDatOqum5LioqMmJjY40PPvjA+O6774y0tDSje/fuxvXXX385h1Xn1Mbf9DkrVqwwOnXqZISEhBivvPJKLY+kbiMg4Tft3r3bkGRs2bLFPPb5558bNpvNyM/Pr/SaoqIio1GjRsZ//vMf89iePXsMSUZaWppT2ddff93o06ePsWbNmqs6INX2PP/aQw89ZPTr16/mOl+Hde/e3Zg4caK5X15eboSEhBhz5syptPyoUaOMW265xelYjx49jD//+c+GYRhGRUWFERwcbLz44ovm+aKiIsPDw8N47733amEE9UdNz3VlNm/ebEgy9u/fXzOdrodqa54PHjxotGzZ0vj222+NVq1aXfUBiUds+E1paWny9/dXt27dzGOxsbFq0KCB0tPTK71m69atOnPmjGJjY81j7dq1U1hYmNLS0sxju3fv1syZM7V06dKLfmng1aA259mquLhYAQEBNdf5OqqsrExbt251mp8GDRooNjb2gvOTlpbmVF6SBg4caJbPzc1VYWGhUxk/Pz/16NHjonN+pauNua5McXGxbDab/P39a6Tf9U1tzXNFRYXuvvtuTZ06VdHR0bXT+Xrm6n5FQpUUFhYqMDDQ6VjDhg0VEBCgwsLCC17j7u5+3v/EgoKCzGtKS0s1evRovfjiiwoLC6uVvtcntTXPVps2bdIHH3yg+++/v0b6XZf99NNPKi8vV1BQkNPxi81PYWHhRcuf+++l1Hk1qI25tjp9+rQef/xxjR49+qr90tXamucXXnhBDRs21KRJk2q+0/UUAekq9sQTT8hms110++6772qt/YSEBLVv31533XVXrbVRF7h6nn/t22+/1dChQ/X000/r5ptvvixtAjXhzJkzGjVqlAzD0BtvvOHq7lxRtm7dqldffVWLFy+WzWZzdXfqjIau7gBcJz4+XuPGjbtomdatWys4OFhHjhxxOn727Fn9/PPPCg4OrvS64OBglZWVqaioyOnuxuHDh81r1q5dq507d+rDDz+U9Ms7gySpefPmevLJJzVjxoxqjqxucfU8n7N79271799f999/v5566qlqjaW+ad68udzc3M5792Rl83NOcHDwRcuf++/hw4dlt9udynTu3LkGe1+/1MZcn3MuHO3fv19r1669au8eSbUzzxs2bNCRI0ec7uSXl5crPj5eiYmJ2rdvX80Oor5w9SIo1H3nFg9nZGSYx1atWlWlxcMffviheey7775zWjycnZ1t7Ny509zefvttQ5KxadOmC74b40pWW/NsGIbx7bffGoGBgcbUqVNrbwB1VPfu3Y2HH37Y3C8vLzdatmx50QWtt956q9Oxnj17nrdI+6WXXjLPFxcXs0jbqPm5NgzDKCsrM4YNG2ZER0cbR44cqZ2O1zM1Pc8//fST0/+Ld+7caYSEhBiPP/648d1339XeQOo4AhKqZNCgQUaXLl2M9PR0Y+PGjUbbtm2d3n5+8OBBIyoqykhPTzePPfDAA0ZYWJixdu1aIyMjw+jZs6fRs2fPC7bx1VdfXdXvYjOM2pnnnTt3Gi1atDDuuusuo6CgwNyulheb999/3/Dw8DAWL15s7N6927j//vsNf39/o7Cw0DAMw7j77ruNJ554wiyfmppqNGzY0HjppZeMPXv2GE8//XSlb/P39/c3PvroI2PHjh3G0KFDeZu/UfNzXVZWZtx2223GNddcY2RmZjr9/ZaWlrpkjHVBbfxNW/EuNgISqujo0aPG6NGjDR8fH8PX19e49957jePHj5vnc3NzDUnGV199ZR47deqU8dBDDxlNmzY1vLy8jNtvv90oKCi4YBsEpNqZ56efftqQdN7WqlWryzgy15o3b54RFhZmuLu7G927dze+/vpr81yfPn2MsWPHOpX/97//bVx77bWGu7u7ER0dbXz66adO5ysqKoxp06YZQUFBhoeHh9G/f38jKyvrcgylzqvJuT73917Z9ut/A1ejmv6btiIgGYbNMP5v4QcAAAAk8S42AACA8xCQAAAALAhIAAAAFgQkAAAACwISAACABQEJAADAgoAEAABgQUACAACwICABQA165plnruovrQWuFAQkAPXGjz/+KHd3d504cUJnzpyRt7e38vLyXN0tAFcgAhKAeiMtLU2dOnWSt7e3tm3bpoCAAIWFhbm6WwCuQAQkAPXGpk2b1Lt3b0nSxo0bzZ/PKSoq0oQJE9SiRQv5+vrqj3/8o7Zv326eP/f4680331RoaKi8vLw0atQoFRcXm2UqKio0c+ZMXXPNNfLw8FDnzp31xRdfOLVz8OBBjR49WgEBAfL29la3bt2Unp7uVGbZsmUKDw+Xn5+f7rjjDh0/ftw89+GHHyomJkaenp5q1qyZYmNjdeLEiRqbJwC/X0NXdwAALiYvL08dO3aUJJ08eVJubm5avHixTp06JZvNJn9/f9155516/fXXNXLkSHl6eurzzz+Xn5+f3nzzTfXv31979+5VQECAJCk7O1v//ve/9cknn8jhcOi+++7TQw89pHfffVeS9Oqrr+pvf/ub3nzzTXXp0kVvv/22brvtNu3atUtt27ZVSUmJ+vTpo5YtW+rjjz9WcHCwtm3bpoqKCrPPOTk5SkpK0n//+18dO3ZMo0aN0vPPP6/nnntOBQUFGj16tObOnavbb79dx48f14YNG8T3hgN1jAEAddiZM2eM3NxcY/v27UajRo2M7du3G9nZ2YaPj4+xbt06Izc31/jxxx+NDRs2GL6+vsbp06edro+MjDTefPNNwzAM4+mnnzbc3NyMgwcPmuc///xzo0GDBkZBQYFhGIYREhJiPPfcc0513HDDDcZDDz1kGIZhvPnmm0aTJk2Mo0ePVtrfp59+2vDy8jIcDod5bOrUqUaPHj0MwzCMrVu3GpKMffv2/c6ZAVCbeMQGoE5r2LChwsPD9d133+mGG25Qx44dVVhYqKCgIN10000KDw9X8+bNtX37dpWUlKhZs2by8fExt9zcXOXk5Jj1hYWFqWXLluZ+z549VVFRoaysLDkcDh06dOi8R3e9e/fWnj17JEmZmZnq0qWLeUeqMuHh4WrSpIm5b7fbdeTIEUlSp06d1L9/f8XExGjkyJF66623dOzYsRqZKwA1h0dsAOq06Oho7d+/X2fOnFFFRYV8fHx09uxZnT17Vj4+PmrVqpV27dqlkpIS2e12paSknFeHv79/jfXH09PzN8s0atTIad9ms5mP4Nzc3PTll19q06ZNWr16tebNm6cnn3xS6enpioiIqLF+Avh9uIMEoE777LPPlJmZqeDgYL3zzjvKzMzUddddp8TERGVmZuqzzz6TJHXt2lWFhYVq2LCh2rRp47Q1b97crC8vL0+HDh0y97/++ms1aNBAUVFR8vX1VUhIiFJTU536kJqaqg4dOkiSOnbsqMzMTP3888/VHpPNZlPv3r01Y8YMffPNN3J3d9fKlSurXR+AmkdAAlCntWrVSj4+Pjp8+LCGDh2q0NBQ7dq1SyNGjFCbNm3UqlUrSVJsbKx69uypYcOGafXq1dq3b582bdqkJ598UhkZGWZ9jRs31tixY7V9+3Zt2LBBkyZN0qhRoxQcHCxJmjp1ql544QV98MEHysrK0hNPPKHMzEz97//+ryRp9OjRCg4O1rBhw5SamqoffvhBy5cvV1paWpXGk56ertmzZysjI0N5eXlasWKFfvzxR7Vv376GZw7A78EjNgB1XkpKim644QY1btxYGzZs0DXXXCO73e5Uxmaz6bPPPtOTTz6pe++9Vz/++KOCg4N10003KSgoyCzXpk0bDR8+XIMHD9bPP/+sW2+9Va+//rp5ftKkSSouLlZ8fLyOHDmiDh066OOPP1bbtm0lSe7u7lq9erXi4+M1ePBgnT17Vh06dND8+fOrNBZfX1+tX79eiYmJcjgcatWqlf72t78pLi6uBmYKQE2xGQbvLQVwdXjmmWeUlJSkzMxMV3cFQB3HIzYAAAALAhIAAIAFj9gAAAAsuIMEAABgQUACAACwICABAABYEJAAAAAsCEgAAAAWBCQAAAALAhIAAIAFAQkAAMDi/wGtEEppDJtOmwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "## Plot train_loss, test_loss\n",
    "num_epochs = CONFIG['epochs']\n",
    "plt.plot(np.arange(num_epochs), history_model[\"train_loss\"], label='train loss')\n",
    "plt.plot(np.arange(num_epochs), history_model[\"test_loss\"], label='test loss')\n",
    "plt.xlabel('#epochs')\n",
    "plt.ylabel('loss')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "7060a727-2c6c-4528-ba8b-ca08f4c4123e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x151692e387c0>"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkgAAAGwCAYAAABSN5pGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/SrBM8AAAACXBIWXMAAA9hAAAPYQGoP6dpAABAuklEQVR4nO3de1hVVeL/8c8B5aIIgshNUbyUZQEaKF8tu8kAOmaaVtJMXr7lTNptokzNEo0pQa3R1MwsS/uW2tWcLmShxxwjr4F5LUkjlYtigGAJwv790c8znS0oInDA3q/n2c941l577bW2NufzrL3O3hbDMAwBAADAxsnRHQAAAGhsCEgAAAAmBCQAAAATAhIAAIAJAQkAAMCEgAQAAGBCQAIAADBp5ugONFWVlZU6cuSIWrVqJYvF4ujuAACAGjAMQydOnFBQUJCcnKqfJyIg1dKRI0cUHBzs6G4AAIBa+Omnn9S+fftq9xOQaqlVq1aSfrvAnp6eDu4NAACoieLiYgUHB9u+x6vj8IC0YMECzZo1S7m5uQoPD9e8efPUu3fvKusuXrxYy5Yt086dOyVJERERevbZZ+3qjx49WkuXLrU7LjY2VqmpqbbP3333nSZMmKCNGzeqrKxMYWFhSkpK0k033VTjfp+5rebp6UlAAgCgiTnf8hiHLtJeuXKlEhISlJiYqO3btys8PFyxsbHKz8+vsr7ValV8fLzWrVun9PR0BQcHKyYmRocPH7arFxcXp5ycHNu2fPlyu/2DBg3S6dOntXbtWm3btk3h4eEaNGiQcnNz622sAACg6bA48mW1UVFR6tWrl+bPny/pt4XPwcHBevDBBzVp0qTzHl9RUSFvb2/Nnz9fI0eOlPTbDFJhYaFWrVpV5THHjh1T27Zt9eWXX6pfv36SpBMnTsjT01Off/65oqOja9T34uJieXl5qaioiBkkAACaiJp+fztsBqmsrEzbtm2zCyROTk6Kjo5Wenp6jdo4efKkysvL5ePjY1dutVrl5+enbt26ady4cSooKLDta9Omjbp166Zly5aptLRUp0+f1qJFi+Tn56eIiIhqz3Xq1CkVFxfbbQAA4NLksDVIx44dU0VFhfz9/e3K/f39tXfv3hq1MXHiRAUFBdmFrLi4ON12223q1KmTsrKy9MQTT2jAgAFKT0+Xs7OzLBaLvvjiCw0ZMkStWrWSk5OT/Pz8lJqaKm9v72rPNWPGDE2fPr12gwUANGkVFRUqLy93dDdQA82bN5ezs/NFt+PwRdq1lZycrBUrVshqtcrNzc1WPmLECNufQ0NDFRYWpi5dushqtap///4yDEP333+//Pz8tGHDBrm7u+uVV17RLbfcoi1btigwMLDK802ePFkJCQm2z2dWwQMALl2GYSg3N1eFhYWO7gouQOvWrRUQEHBRzyl0WEDy9fWVs7Oz8vLy7Mrz8vIUEBBwzmNnz56t5ORkffHFFwoLCztn3c6dO8vX11f79+9X//79tXbtWn300Uf6+eefbfceX3zxRX3++edaunRptWufXF1d5erqegEjBAA0dWfCkZ+fn1q0aMGDgRs5wzB08uRJ24+9qpv0qAmHBSQXFxdFREQoLS1NQ4YMkfTbIu20tDQ98MAD1R43c+ZMPfPMM/rss88UGRl53vMcOnRIBQUFtot08uRJSTrr6ZlOTk6qrKys5WgAAJeaiooKWzhq06aNo7uDGnJ3d5ck5efny8/Pr9a32xz6M/+EhAQtXrxYS5cu1Z49ezRu3DiVlpZqzJgxkqSRI0dq8uTJtvopKSl66qmntGTJEoWEhCg3N1e5ubkqKSmRJJWUlGjChAn6+uuvdfDgQaWlpenWW29V165dFRsbK0nq06ePvL29NWrUKGVmZtqeiXTgwAH9+c9/bviLAABolM6sOWrRooWDe4ILdebv7GLWjTl0DdKdd96po0ePaurUqcrNzVWPHj2UmppqW7idnZ1tN9OzcOFClZWVafjw4XbtJCYmatq0aXJ2dtaOHTu0dOlSFRYWKigoSDExMUpKSrLdHvP19VVqaqqmTJmim2++WeXl5brqqqv04YcfKjw8vOEGDwBoErit1vTUxd+ZQ5+D1JTxHCQAuLT9+uuvOnDggDp16mT3YyA0fuf6u2v0z0ECAABorAhIAADgnEJCQjRnzhxHd6NBNdnnIAEAgKrdeOON6tGjR52Fmi1btqhly5Z10lZTQUACAOAPyDAMVVRUqFmz80eBtm3bNkCPGhdusQEAUEOGYehk2ekG3y7k91SjR4/W+vXrNXfuXFksFlksFh08eFBWq1UWi0WffvqpIiIi5Orqqv/85z/KysrSrbfeKn9/f3l4eKhXr1764osv7No032KzWCx65ZVXNHToULVo0UKXXXaZVq9efc5+vfHGG4qMjFSrVq0UEBCgu+66y/ZAxzN27dqlQYMGydPTU61atVK/fv2UlZVl279kyRJdddVVcnV1VWBg4Dmfm3ixmEECAKCGfimvUPepnzX4eXc/HasWLjX7yp47d66+++47XX311Xr66acl/TYDdPDgQUnSpEmTNHv2bHXu3Fne3t766aefNHDgQD3zzDNydXXVsmXLdMstt2jfvn3q0KFDteeZPn26Zs6cqVmzZmnevHn6y1/+oh9//PGsF8ifUV5erqSkJHXr1k35+flKSEjQ6NGj9cknn0iSDh8+rOuvv1433nij1q5dK09PT23cuFGnT5+W9NujfhISEpScnKwBAwaoqKhIGzdurOklvGAEJAAALiFeXl5ycXFRixYtqnx119NPP60//elPts8+Pj52zwFMSkrSBx98oNWrV59zhmb06NGKj4+XJD377LN64YUXtHnzZsXFxVVZ/3//939tf+7cubNeeOEF9erVSyUlJfLw8NCCBQvk5eWlFStWqHnz5pKkyy+/3HbMP//5Tz366KN6+OGHbWW9evU63+WoNQISAAA15N7cWbufjnXIeeuK+TVdJSUlmjZtmj7++GPl5OTo9OnT+uWXX5SdnX3Odn7/LtSWLVvK09PzrFtmv7dt2zZNmzZNmZmZ+vnnn22v98rOzlb37t2VkZGhfv362cLR7+Xn5+vIkSPq37//hQz1ohCQAACoIYvFUuNbXY2V+ddojz32mD7//HPNnj1bXbt2lbu7u4YPH66ysrJztmMOMhaLpdp3mpaWlio2NlaxsbF688031bZtW2VnZys2NtZ2njPvUKvKufbVFxZpAwBwiXFxcVFFRUWN6m7cuFGjR4/W0KFDFRoaqoCAANt6pbqyd+9eFRQUKDk5Wf369dMVV1xx1mxTWFiYNmzYUOX701q1aqWQkBClpaXVab/OhYAEAMAlJiQkRJs2bdLBgwd17Nixamd2JOmyyy7T+++/r4yMDGVmZuquu+46Z/3a6NChg1xcXDRv3jz98MMPWr16tZKSkuzqPPDAAyouLtaIESO0detWff/993rjjTe0b98+SdK0adP03HPP6YUXXtD333+v7du3a968eXXaz98jIAEAcIl57LHH5OzsrO7du9tuZ1Xn+eefl7e3t/r27atbbrlFsbGxuuaaa+q0P23bttXrr7+ud955R927d1dycrJmz55tV6dNmzZau3atSkpKdMMNNygiIkKLFy+23cobNWqU5syZoxdffFFXXXWVBg0apO+//75O+/l7vKy2lnhZLQBc2nhZbdPFy2oBAADqAQEJAADAhIAEAABgQkACAAAwISABAACYEJAAAABMCEgAAAAmBCQAAAATAhIAAIAJAQkAgEvMjTfeqH/84x912ubo0aM1ZMiQOm2zMSMgAQAAmBCQAAC4hIwePVrr16/X3LlzZbFYZLFYdPDgQUnSzp07NWDAAHl4eMjf31933323jh07Zjv23XffVWhoqNzd3dWmTRtFR0ertLRU06ZN09KlS/Xhhx/a2rRarVWePzU1Vdddd51at26tNm3aaNCgQcrKyrKrc+jQIcXHx8vHx0ctW7ZUZGSkNm3aZNv/73//W7169ZKbm5t8fX01dOjQOr9O50NAAgCgpgxDKitt+O0C3is/d+5c9enTR2PHjlVOTo5ycnIUHByswsJC3XzzzerZs6e2bt2q1NRU5eXl6Y477pAk5eTkKD4+Xv/7v/+rPXv2yGq16rbbbpNhGHrsscd0xx13KC4uztZm3759qzx/aWmpEhIStHXrVqWlpcnJyUlDhw5VZWWlJKmkpEQ33HCDDh8+rNWrVyszM1OPP/64bf/HH3+soUOHauDAgfrmm2+Ulpam3r17X+Rf3IVr1uBnBACgqSo/KT0b1PDnfeKI5NKyRlW9vLzk4uKiFi1aKCAgwFY+f/589ezZU88++6ytbMmSJQoODtZ3332nkpISnT59Wrfddps6duwoSQoNDbXVdXd316lTp+zarMqwYcPsPi9ZskRt27bV7t27dfXVV+utt97S0aNHtWXLFvn4+EiSunbtaqv/zDPPaMSIEZo+fbqtLDw8vEZjr0vMIAEA8AeQmZmpdevWycPDw7ZdccUVkqSsrCyFh4erf//+Cg0N1e23367Fixfr559/vuDzfP/994qPj1fnzp3l6empkJAQSVJ2drYkKSMjQz179rSFI7OMjAz179+/doOsQ8wgAQBQU81b/Dab44jzXqSSkhLdcsstSklJOWtfYGCgnJ2d9fnnn+urr77SmjVrNG/ePE2ZMkWbNm1Sp06danyeW265RR07dtTixYsVFBSkyspKXX311SorK5P020zUuZxvf0NhBgkAgJqyWH671dXQm8VyQd10cXFRRUWFXdk111yjXbt2KSQkRF27drXbWrZs+f+HZ9G1116r6dOn65tvvpGLi4s++OCDats0Kygo0L59+/Tkk0+qf//+uvLKK8+ahQoLC1NGRoaOHz9eZRthYWFKS0u7oPHWBwISAACXmJCQEG3atEkHDx7UsWPHVFlZqfvvv1/Hjx9XfHy8tmzZoqysLH322WcaM2aMKioqtGnTJj377LPaunWrsrOz9f777+vo0aO68sorbW3u2LFD+/bt07Fjx1ReXn7Web29vdWmTRu9/PLL2r9/v9auXauEhAS7OvHx8QoICNCQIUO0ceNG/fDDD3rvvfeUnp4uSUpMTNTy5cuVmJioPXv26Ntvv61y1qu+EZAAALjEPPbYY3J2dlb37t3Vtm1bZWdnKygoSBs3blRFRYViYmIUGhqqf/zjH2rdurWcnJzk6empL7/8UgMHDtTll1+uJ598Us8995wGDBggSRo7dqy6deumyMhItW3bVhs3bjzrvE5OTlqxYoW2bdumq6++Wo888ohmzZplV8fFxUVr1qyRn5+fBg4cqNDQUCUnJ8vZ2VnSbw+5fOedd7R69Wr16NFDN998szZv3lz/F83EYhgX8NtB2BQXF8vLy0tFRUXy9PR0dHcAAHXs119/1YEDB9SpUye5ubk5uju4AOf6u6vp9zczSAAAACYEJAAAABMCEgAAgAkBCQAAwISABADAOfBbpqanLv7OCEgAAFShefPmkqSTJ086uCe4UGf+zs78HdYGrxoBAKAKzs7Oat26tfLz8yVJLVq0kOUCn2iNhmUYhk6ePKn8/Hy1bt3a9myl2iAgAQBQjTNvrj8TktA0tG7d2vZ3V1sEJAAAqmGxWBQYGCg/P78qX62Bxqd58+YXNXN0BgEJAIDzcHZ2rpMvXTQdLNIGAAAwISABAACYEJAAAABMCEgAAAAmjSIgLViwQCEhIXJzc1NUVJQ2b95cbd3FixerX79+8vb2lre3t6Kjo8+qP3r0aFksFrstLi7Ott9qtZ61/8y2ZcuWehsnAABoGhwekFauXKmEhAQlJiZq+/btCg8PV2xsbLXPnLBarYqPj9e6deuUnp6u4OBgxcTE6PDhw3b14uLilJOTY9uWL19u29e3b1+7fTk5Obr33nvVqVMnRUZG1ut4AQBA42cxHPySmaioKPXq1Uvz58+XJFVWVio4OFgPPvigJk2adN7jKyoq5O3trfnz52vkyJGSfptBKiws1KpVq2rUh/LycrVr104PPvignnrqqRodU1xcLC8vLxUVFcnT07NGxwAAAMeq6fe3Q2eQysrKtG3bNkVHR9vKnJycFB0drfT09Bq1cfLkSZWXl8vHx8eu3Gq1ys/PT926ddO4ceNUUFBQbRurV69WQUGBxowZU22dU6dOqbi42G4DAACXJocGpGPHjqmiokL+/v525f7+/srNza1RGxMnTlRQUJBdyIqLi9OyZcuUlpamlJQUrV+/XgMGDFBFRUWVbbz66quKjY1V+/btqz3PjBkz5OXlZduCg4Nr1D8AAND0NOknaScnJ2vFihWyWq1yc3OzlY8YMcL259DQUIWFhalLly6yWq3q37+/XRuHDh3SZ599prfffvuc55o8ebISEhJsn4uLiwlJAABcohw6g+Tr6ytnZ2fl5eXZlefl5Z33JXOzZ89WcnKy1qxZo7CwsHPW7dy5s3x9fbV///6z9r322mtq06aNBg8efM42XF1d5enpabcBAIBLk0MDkouLiyIiIpSWlmYrq6ysVFpamvr06VPtcTNnzlRSUpJSU1Nr9KuzQ4cOqaCgQIGBgXblhmHotdde08iRI9W8efPaDwQAAFxSHP4z/4SEBC1evFhLly7Vnj17NG7cOJWWltoWTI8cOVKTJ0+21U9JSdFTTz2lJUuWKCQkRLm5ucrNzVVJSYkkqaSkRBMmTNDXX3+tgwcPKi0tTbfeequ6du2q2NhYu3OvXbtWBw4c0L333ttwAwYAAI2ew9cg3XnnnTp69KimTp2q3Nxc9ejRQ6mpqbaF29nZ2XJy+m+OW7hwocrKyjR8+HC7dhITEzVt2jQ5Oztrx44dWrp0qQoLCxUUFKSYmBglJSXJ1dXV7phXX31Vffv21RVXXFH/AwUAAE2Gw5+D1FTxHCQAAJqeJvEcJAAAgMaIgAQAAGBCQAIAADAhIAEAAJgQkAAAAEwISAAAACYEJAAAABMCEgAAgAkBCQAAwISABAAAYEJAAgAAMCEgAQAAmBCQAAAATAhIAAAAJgQkAAAAEwISAACACQEJAADAhIAEAABgQkACAAAwISABAACYEJAAAABMCEgAAAAmBCQAAAATAhIAAIAJAQkAAMCEgAQAAGBCQAIAADAhIAEAAJgQkAAAAEwISAAAACYEJAAAABMCEgAAgAkBCQAAwISABAAAYEJAAgAAMCEgAQAAmBCQAAAATAhIAAAAJgQkAAAAEwISAACACQEJAADAhIAEAABgQkACAAAwISABAACYEJAAAABMCEgAAAAmBCQAAAAThwekBQsWKCQkRG5uboqKitLmzZurrbt48WL169dP3t7e8vb2VnR09Fn1R48eLYvFYrfFxcWd1dbHH3+sqKgoubu7y9vbW0OGDKnroQEAgCbKoQFp5cqVSkhIUGJiorZv367w8HDFxsYqPz+/yvpWq1Xx8fFat26d0tPTFRwcrJiYGB0+fNiuXlxcnHJycmzb8uXL7fa/9957uvvuuzVmzBhlZmZq48aNuuuuu+ptnAAAoGmxGIZhOOrkUVFR6tWrl+bPny9JqqysVHBwsB588EFNmjTpvMdXVFTI29tb8+fP18iRIyX9NoNUWFioVatWVXnM6dOnFRISounTp+uee+6pdd+Li4vl5eWloqIieXp61rodAADQcGr6/e2wGaSysjJt27ZN0dHR/+2Mk5Oio6OVnp5eozZOnjyp8vJy+fj42JVbrVb5+fmpW7duGjdunAoKCmz7tm/frsOHD8vJyUk9e/ZUYGCgBgwYoJ07d57zXKdOnVJxcbHdBgAALk0OC0jHjh1TRUWF/P397cr9/f2Vm5tbozYmTpyooKAgu5AVFxenZcuWKS0tTSkpKVq/fr0GDBigiooKSdIPP/wgSZo2bZqefPJJffTRR/L29taNN96o48ePV3uuGTNmyMvLy7YFBwdf6JABAEAT4fBF2rWVnJysFStW6IMPPpCbm5utfMSIERo8eLBCQ0M1ZMgQffTRR9qyZYusVquk327jSdKUKVM0bNgwRURE6LXXXpPFYtE777xT7fkmT56soqIi2/bTTz/V6/gAAIDjOCwg+fr6ytnZWXl5eXbleXl5CggIOOexs2fPVnJystasWaOwsLBz1u3cubN8fX21f/9+SVJgYKAkqXv37rY6rq6u6ty5s7Kzs6ttx9XVVZ6ennYbAAC4NDksILm4uCgiIkJpaWm2ssrKSqWlpalPnz7VHjdz5kwlJSUpNTVVkZGR5z3PoUOHVFBQYAtGERERcnV11b59+2x1ysvLdfDgQXXs2PEiRgQAAC4VzRx58oSEBI0aNUqRkZHq3bu35syZo9LSUo0ZM0aSNHLkSLVr104zZsyQJKWkpGjq1Kl66623FBISYlur5OHhIQ8PD5WUlGj69OkaNmyYAgIClJWVpccff1xdu3ZVbGysJMnT01P33XefEhMTFRwcrI4dO2rWrFmSpNtvv90BVwEAADQ2Dg1Id955p44ePaqpU6cqNzdXPXr0UGpqqm3hdnZ2tpyc/jvJtXDhQpWVlWn48OF27SQmJmratGlydnbWjh07tHTpUhUWFiooKEgxMTFKSkqSq6urrf6sWbPUrFkz3X333frll18UFRWltWvXytvbu2EGDgAAGjWHPgepKeM5SAAAND2N/jlIAAAAjRUBCQAAwISABAAAYEJAAgAAMCEgAQAAmBCQAAAATAhIAAAAJgQkAAAAEwISAACACQEJAADAhIAEAABgQkACAAAwISABAACYEJAAAABMCEgAAAAmBCQAAAATAhIAAIAJAQkAAMCEgAQAAGBCQAIAADAhIAEAAJgQkAAAAEwISAAAACYEJAAAABMCEgAAgAkBCQAAwISABAAAYFKrgLRu3bq67gcAAECjUauAFBcXpy5duuif//ynfvrpp7ruEwAAgEPVKiAdPnxYDzzwgN5991117txZsbGxevvtt1VWVlbX/QMAAGhwtQpIvr6+euSRR5SRkaFNmzbp8ssv1/jx4xUUFKSHHnpImZmZdd1PAACABnPRi7SvueYaTZ48WQ888IBKSkq0ZMkSRUREqF+/ftq1a1dd9BEAAKBB1ToglZeX691339XAgQPVsWNHffbZZ5o/f77y8vK0f/9+dezYUbfffntd9hUAAKBBWAzDMC70oAcffFDLly+XYRi6++67de+99+rqq6+2q5Obm6ugoCBVVlbWWWcbk+LiYnl5eamoqEienp6O7g4AAKiBmn5/N6tN47t379a8efN02223ydXVtco6vr6+PA4AAAA0SbWaQQIzSAAANEU1/f6u1RqkGTNmaMmSJWeVL1myRCkpKbVpEgAAoNGoVUBatGiRrrjiirPKr7rqKr300ksX3SkAAABHqlVAys3NVWBg4Fnlbdu2VU5OzkV3CgAAwJFqFZCCg4O1cePGs8o3btyooKCgi+4UAACAI9XqV2xjx47VP/7xD5WXl+vmm2+WJKWlpenxxx/Xo48+WqcdBAAAaGi1CkgTJkxQQUGBxo8fb3v/mpubmyZOnKjJkyfXaQcBAAAa2kX9zL+kpER79uyRu7u7LrvssmqfiXQp4mf+AAA0PfX6oMgzPDw81KtXr4tpAgAAoNGpdUDaunWr3n77bWVnZ9tus53x/vvvX3THAAAAHKVWv2JbsWKF+vbtqz179uiDDz5QeXm5du3apbVr18rLy6uu+wgAANCgahWQnn32Wf3rX//Sv//9b7m4uGju3Lnau3ev7rjjDnXo0KGu+wgAANCgahWQsrKy9Oc//1mS5OLiotLSUlksFj3yyCN6+eWXL7i9BQsWKCQkRG5uboqKitLmzZurrbt48WL169dP3t7e8vb2VnR09Fn1R48eLYvFYrfFxcXZ1QkJCTmrTnJy8gX3HQAAXHpqFZC8vb114sQJSVK7du20c+dOSVJhYaFOnjx5QW2tXLlSCQkJSkxM1Pbt2xUeHq7Y2Fjl5+dXWd9qtSo+Pl7r1q1Tenq6goODFRMTo8OHD9vVi4uLU05Ojm1bvnz5WW09/fTTdnUefPDBC+o7AAC4NNVqkfb111+vzz//XKGhobr99tv18MMPa+3atfr888/Vv3//C2rr+eef19ixYzVmzBhJ0ksvvaSPP/5YS5Ys0aRJk86q/+abb9p9fuWVV/Tee+8pLS1NI0eOtJW7uroqICDgnOdu1arVeeuccerUKZ06dcr2ubi4uEbHAQCApqdWM0jz58/XiBEjJElTpkxRQkKC8vLyNGzYML366qs1bqesrEzbtm1TdHT0fzvk5KTo6Gilp6fXqI2TJ0+qvLxcPj4+duVWq1V+fn7q1q2bxo0bp4KCgrOOTU5OVps2bdSzZ0/NmjVLp0+frvY8M2bMkJeXl20LDg6u4SgBAEBTc8EzSKdPn9ZHH32k2NhYSb8Fmqpmemri2LFjqqiokL+/v125v7+/9u7dW6M2Jk6cqKCgILuQFRcXp9tuu02dOnVSVlaWnnjiCQ0YMEDp6elydnaWJD300EO65ppr5OPjo6+++kqTJ09WTk6Onn/++SrPM3nyZCUkJNg+FxcXE5IAALhEXXBAatasme677z7t2bOnPvpzQZKTk7VixQpZrVa5ubnZys/MbklSaGiowsLC1KVLF1mtVtstwN+HnbCwMLm4uOjvf/+7ZsyYUeUTwV1dXf9QTwoHAOCPrFa32Hr37q2MjIyLPrmvr6+cnZ2Vl5dnV56Xl3fetUGzZ89WcnKy1qxZo7CwsHPW7dy5s3x9fbV///5q60RFRen06dM6ePBgjfsPAAAuTbVapD1+/HglJCTop59+UkREhFq2bGm3/3yB5QwXFxdFREQoLS1NQ4YMkSRVVlYqLS1NDzzwQLXHzZw5U88884w+++wzRUZGnvc8hw4dUkFBgQIDA6utk5GRIScnJ/n5+dWo7wAA4NJVq4B05hbWQw89ZCuzWCwyDEMWi0UVFRU1bishIUGjRo1SZGSkevfurTlz5qi0tNT2q7aRI0eqXbt2mjFjhiQpJSVFU6dO1VtvvaWQkBDl5uZK+u29cB4eHiopKdH06dM1bNgwBQQEKCsrS48//ri6du1qWzeVnp6uTZs26aabblKrVq2Unp6uRx55RH/961/l7e1dm0sCAAAuIbUKSAcOHKizDtx55506evSopk6dqtzcXPXo0UOpqam2hdvZ2dlycvrvncCFCxeqrKxMw4cPt2snMTFR06ZNk7Ozs3bs2KGlS5eqsLBQQUFBiomJUVJSkm0Nkaurq1asWKFp06bp1KlT6tSpkx555BG7dUkAAOCPy2IYhuHoTjRFxcXF8vLyUlFRkTw9PR3dHQAAUAM1/f6u1QzSsmXLzrn/9w9sBAAAaGpqNYNkXqdTXl6ukydPysXFRS1atNDx48frrIONFTNIAAA0PTX9/q7Vz/x//vlnu62kpET79u3TddddV+U7zwAAAJqSWgWkqlx22WVKTk7Www8/XFdNAgAAOESdBSTpt6dsHzlypC6bBAAAaHC1WqS9evVqu8+GYSgnJ0fz58/XtddeWycdAwAAcJRaBaQzT70+w2KxqG3btrr55pv13HPP1UW/AAAAHKZWAamysrKu+wEAANBo1OkaJAAAgEtBrQLSsGHDlJKSclb5zJkzdfvtt190pwAAABypVgHpyy+/1MCBA88qHzBggL788suL7hQAAIAj1SoglZSUyMXF5azy5s2bq7i4+KI7BQAA4Ei1CkihoaFauXLlWeUrVqxQ9+7dL7pTAAAAjlSrX7E99dRTuu2225SVlaWbb75ZkpSWlqbly5frnXfeqdMOAgAANLRaBaRbbrlFq1at0rPPPqt3331X7u7uCgsL0xdffKEbbrihrvsIAADQoCyGYRiO7kRTVNO3AQMAgMajpt/ftVqDtGXLFm3atOms8k2bNmnr1q21aRIAAKDRqFVAuv/++/XTTz+dVX748GHdf//9F90pAAAAR6pVQNq9e7euueaas8p79uyp3bt3X3SnAAAAHKlWAcnV1VV5eXlnlefk5KhZs1qt+wYAAGg0ahWQYmJiNHnyZBUVFdnKCgsL9cQTT+hPf/pTnXUOAADAEWo13TN79mxdf/316tixo3r27ClJysjIkL+/v95444067SAAAEBDq1VAateunXbs2KE333xTmZmZcnd315gxYxQfH6/mzZvXdR8BAAAaVK0XDLVs2VLXXXedOnTooLKyMknSp59+KkkaPHhw3fQOAADAAWoVkH744QcNHTpU3377rSwWiwzDkMVise2vqKiosw4CAAA0tFot0n744YfVqVMn5efnq0WLFtq5c6fWr1+vyMhIWa3WOu4iAABAw6rVDFJ6errWrl0rX19fOTk5ydnZWdddd51mzJihhx56SN98801d9xMAAKDB1GoGqaKiQq1atZIk+fr66siRI5Kkjh07at++fXXXOwAAAAeo1QzS1VdfrczMTHXq1ElRUVGaOXOmXFxc9PLLL6tz58513UcAAIAGVauA9OSTT6q0tFSS9PTTT2vQoEHq16+f2rRpo5UrV9ZpBwEAABqaxTAMoy4aOn78uLy9ve1+zXYpKy4ulpeXl4qKiuTp6eno7gAAgBqo6fd3nb04zcfHp66aAgAAcKhaLdIGAAC4lBGQAAAATAhIAAAAJgQkAAAAEwISAACACQEJAADAhIAEAABgQkACAAAwISABAACYEJAAAABMCEgAAAAmBCQAAAATAhIAAIAJAQkAAMCkUQSkBQsWKCQkRG5uboqKitLmzZurrbt48WL169dP3t7e8vb2VnR09Fn1R48eLYvFYrfFxcVV2d6pU6fUo0cPWSwWZWRk1OWwAABAE+XwgLRy5UolJCQoMTFR27dvV3h4uGJjY5Wfn19lfavVqvj4eK1bt07p6ekKDg5WTEyMDh8+bFcvLi5OOTk5tm358uVVtvf4448rKCiozscFAACaLocHpOeff15jx47VmDFj1L17d7300ktq0aKFlixZUmX9N998U+PHj1ePHj10xRVX6JVXXlFlZaXS0tLs6rm6uiogIMC2eXt7n9XWp59+qjVr1mj27Nn1MjYAANA0OTQglZWVadu2bYqOjraVOTk5KTo6Wunp6TVq4+TJkyovL5ePj49dudVqlZ+fn7p166Zx48apoKDAbn9eXp7Gjh2rN954Qy1atDjveU6dOqXi4mK7DQAAXJocGpCOHTumiooK+fv725X7+/srNze3Rm1MnDhRQUFBdiErLi5Oy5YtU1pamlJSUrR+/XoNGDBAFRUVkiTDMDR69Gjdd999ioyMrNF5ZsyYIS8vL9sWHBxcw1ECAICmppmjO3AxkpOTtWLFClmtVrm5udnKR4wYYftzaGiowsLC1KVLF1mtVvXv31/z5s3TiRMnNHny5Bqfa/LkyUpISLB9Li4uJiQBAHCJcugMkq+vr5ydnZWXl2dXnpeXp4CAgHMeO3v2bCUnJ2vNmjUKCws7Z93OnTvL19dX+/fvlyStXbtW6enpcnV1VbNmzdS1a1dJUmRkpEaNGlVlG66urvL09LTbAADApcmhAcnFxUURERF2C6zPLLju06dPtcfNnDlTSUlJSk1NrdEtskOHDqmgoECBgYGSpBdeeEGZmZnKyMhQRkaGPvnkE0m//aLumWeeuchRAQCAps7ht9gSEhI0atQoRUZGqnfv3pozZ45KS0s1ZswYSdLIkSPVrl07zZgxQ5KUkpKiqVOn6q233lJISIhtrZKHh4c8PDxUUlKi6dOna9iwYQoICFBWVpYef/xxde3aVbGxsZKkDh062PXBw8NDktSlSxe1b9++oYYOAAAaKYcHpDvvvFNHjx7V1KlTlZubqx49eig1NdW2cDs7O1tOTv+d6Fq4cKHKyso0fPhwu3YSExM1bdo0OTs7a8eOHVq6dKkKCwsVFBSkmJgYJSUlydXVtUHHBgAAmiaLYRiGozvRFBUXF8vLy0tFRUWsRwIAoImo6fe3wx8UCQAA0NgQkAAAAEwISAAAACYEJAAAABMCEgAAgAkBCQAAwISABAAAYEJAAgAAMCEgAQAAmBCQAAAATAhIAAAAJgQkAAAAEwISAACACQEJAADAhIAEAABgQkACAAAwISABAACYEJAAAABMCEgAAAAmBCQAAAATAhIAAIAJAQkAAMCEgAQAAGBCQAIAADAhIAEAAJgQkAAAAEwISAAAACYEJAAAABMCEgAAgAkBCQAAwISABAAAYEJAAgAAMCEgAQAAmBCQAAAATAhIAAAAJgQkAAAAEwISAACACQEJAADAhIAEAABgQkACAAAwISABAACYEJAAAABMCEgAAAAmBCQAAAATAhIAAIAJAQkAAMCEgAQAAGDSKALSggULFBISIjc3N0VFRWnz5s3V1l28eLH69esnb29veXt7Kzo6+qz6o0ePlsVisdvi4uLs6gwePFgdOnSQm5ubAgMDdffdd+vIkSP1Mj4AANC0ODwgrVy5UgkJCUpMTNT27dsVHh6u2NhY5efnV1nfarUqPj5e69atU3p6uoKDgxUTE6PDhw/b1YuLi1NOTo5tW758ud3+m266SW+//bb27dun9957T1lZWRo+fHi9jRMAADQdFsMwDEd2ICoqSr169dL8+fMlSZWVlQoODtaDDz6oSZMmnff4iooKeXt7a/78+Ro5cqSk32aQCgsLtWrVqhr3Y/Xq1RoyZIhOnTql5s2bn7d+cXGxvLy8VFRUJE9PzxqfBwAAOE5Nv78dOoNUVlambdu2KTo62lbm5OSk6Ohopaen16iNkydPqry8XD4+PnblVqtVfn5+6tatm8aNG6eCgoJq2zh+/LjefPNN9e3bt9pwdOrUKRUXF9ttAADg0uTQgHTs2DFVVFTI39/frtzf31+5ubk1amPixIkKCgqyC1lxcXFatmyZ0tLSlJKSovXr12vAgAGqqKg469iWLVuqTZs2ys7O1ocffljteWbMmCEvLy/bFhwcfAEjBQAATYnD1yBdjOTkZK1YsUIffPCB3NzcbOUjRozQ4MGDFRoaqiFDhuijjz7Sli1bZLVa7Y6fMGGCvvnmG61Zs0bOzs4aOXKkqrvjOHnyZBUVFdm2n376qT6HBgAAHKiZI0/u6+srZ2dn5eXl2ZXn5eUpICDgnMfOnj1bycnJ+uKLLxQWFnbOup07d5avr6/279+v/v37253f19dXl19+ua688koFBwfr66+/Vp8+fc5qw9XVVa6urhcwOgAA0FQ5dAbJxcVFERERSktLs5VVVlYqLS2typByxsyZM5WUlKTU1FRFRkae9zyHDh1SQUGBAgMDq61TWVkp6be1RgAA4I/NoTNIkpSQkKBRo0YpMjJSvXv31pw5c1RaWqoxY8ZIkkaOHKl27dppxowZkqSUlBRNnTpVb731lkJCQmxrlTw8POTh4aGSkhJNnz5dw4YNU0BAgLKysvT444+ra9euio2NlSRt2rRJW7Zs0XXXXSdvb29lZWXpqaeeUpcuXc4ZzAAAwB+DwwPSnXfeqaNHj2rq1KnKzc1Vjx49lJqaalu4nZ2dLSen/050LVy4UGVlZWc9sygxMVHTpk2Ts7OzduzYoaVLl6qwsFBBQUGKiYlRUlKS7RZZixYt9P777ysxMVGlpaUKDAxUXFycnnzySW6jAQAAxz8HqaniOUgAADQ9TeI5SAAAAI0RAQkAAMCEgAQAAGBCQAIAADAhIAEAAJgQkAAAAEwISAAAACYEJAAAABMCEgAAgAkBCQAAwISABAAAYEJAAgAAMCEgAQAAmBCQAAAATAhIAAAAJgQkAAAAEwISAACACQEJAADAhIAEAABgQkACAAAwISABAACYEJAAAABMCEgAAAAmBCQAAAATAhIAAIAJAQkAAMCEgAQAAGBCQAIAADAhIAEAAJgQkAAAAEwISAAAACYEJAAAABMCEgAAgAkBCQAAwISABAAAYEJAAgAAMCEgAQAAmBCQAAAATAhIAAAAJgQkAAAAEwISAACACQEJAADAhIAEAABgQkACAAAwISABAACYEJAAAABMCEgAAAAmjSIgLViwQCEhIXJzc1NUVJQ2b95cbd3FixerX79+8vb2lre3t6Kjo8+qP3r0aFksFrstLi7Otv/gwYO655571KlTJ7m7u6tLly5KTExUWVlZvY0RAAA0HQ4PSCtXrlRCQoISExO1fft2hYeHKzY2Vvn5+VXWt1qtio+P17p165Senq7g4GDFxMTo8OHDdvXi4uKUk5Nj25YvX27bt3fvXlVWVmrRokXatWuX/vWvf+mll17SE088Ua9jBQAATYPFMAzDkR2IiopSr169NH/+fElSZWWlgoOD9eCDD2rSpEnnPb6iokLe3t6aP3++Ro4cKem3GaTCwkKtWrWqxv2YNWuWFi5cqB9++KFG9YuLi+Xl5aWioiJ5enrW+DwAAMBxavr97dAZpLKyMm3btk3R0dG2MicnJ0VHRys9Pb1GbZw8eVLl5eXy8fGxK7darfLz81O3bt00btw4FRQUnLOdoqKis9r4vVOnTqm4uNhuAwAAlyaHBqRjx46poqJC/v7+duX+/v7Kzc2tURsTJ05UUFCQXciKi4vTsmXLlJaWppSUFK1fv14DBgxQRUVFlW3s379f8+bN09///vdqzzNjxgx5eXnZtuDg4Br1DwAAND3NHN2Bi5GcnKwVK1bIarXKzc3NVj5ixAjbn0NDQxUWFqYuXbrIarWqf//+dm0cPnxYcXFxuv322zV27NhqzzV58mQlJCTYPhcXFxOSAAC4RDl0BsnX11fOzs7Ky8uzK8/Ly1NAQMA5j509e7aSk5O1Zs0ahYWFnbNu586d5evrq/3799uVHzlyRDfddJP69u2rl19++ZxtuLq6ytPT024DAACXJocGJBcXF0VERCgtLc1WVllZqbS0NPXp06fa42bOnKmkpCSlpqYqMjLyvOc5dOiQCgoKFBgYaCs7fPiwbrzxRkVEROi1116Tk5PDf9AHAAAaCYffYktISNCoUaMUGRmp3r17a86cOSotLdWYMWMkSSNHjlS7du00Y8YMSVJKSoqmTp2qt956SyEhIba1Sh4eHvLw8FBJSYmmT5+uYcOGKSAgQFlZWXr88cfVtWtXxcbGSvpvOOrYsaNmz56to0eP2vpzvpkrAABw6XN4QLrzzjt19OhRTZ06Vbm5uerRo4dSU1NtC7ezs7PtZncWLlyosrIyDR8+3K6dxMRETZs2Tc7OztqxY4eWLl2qwsJCBQUFKSYmRklJSXJ1dZUkff7559q/f7/279+v9u3b27Xj4KceAACARsDhz0FqqngOEgAATU+TeA4SAABAY0RAAgAAMCEgAQAAmBCQAAAATAhIAAAAJgQkAAAAEwISAACACQEJAADAhIAEAABg4vBXjTRVZx5AXlxc7OCeAACAmjrzvX2+F4kQkGrpxIkTkqTg4GAH9wQAAFyoEydOyMvLq9r9vIutliorK3XkyBG1atVKFovF0d1xqOLiYgUHB+unn37ivXT1jGvdMLjODYPr3DC4zvYMw9CJEycUFBQkJ6fqVxoxg1RLTk5Oat++vaO70ah4enryH18D4Vo3DK5zw+A6Nwyu83+da+boDBZpAwAAmBCQAAAATAhIuGiurq5KTEyUq6uro7tyyeNaNwyuc8PgOjcMrnPtsEgbAADAhBkkAAAAEwISAACACQEJAADAhIAEAABgQkBCjRw/flx/+ctf5OnpqdatW+uee+5RSUnJOY/59ddfdf/996tNmzby8PDQsGHDlJeXV2XdgoICtW/fXhaLRYWFhfUwgqahPq5zZmam4uPjFRwcLHd3d1155ZWaO3dufQ+lUVmwYIFCQkLk5uamqKgobd68+Zz133nnHV1xxRVyc3NTaGioPvnkE7v9hmFo6tSpCgwMlLu7u6Kjo/X999/X5xCajLq81uXl5Zo4caJCQ0PVsmVLBQUFaeTIkTpy5Eh9D6PRq+t/07933333yWKxaM6cOXXc6ybGAGogLi7OCA8PN77++mtjw4YNRteuXY34+PhzHnPfffcZwcHBRlpamrF161bjf/7nf4y+fftWWffWW281BgwYYEgyfv7553oYQdNQH9f51VdfNR566CHDarUaWVlZxhtvvGG4u7sb8+bNq+/hNAorVqwwXFxcjCVLlhi7du0yxo4da7Ru3drIy8ursv7GjRsNZ2dnY+bMmcbu3buNJ5980mjevLnx7bff2uokJycbXl5exqpVq4zMzExj8ODBRqdOnYxffvmloYbVKNX1tS4sLDSio6ONlStXGnv37jXS09ON3r17GxEREQ05rEanPv5Nn/H+++8b4eHhRlBQkPGvf/2rnkfSuBGQcF67d+82JBlbtmyxlX366aeGxWIxDh8+XOUxhYWFRvPmzY133nnHVrZnzx5DkpGenm5X98UXXzRuuOEGIy0t7Q8dkOr7Ov/e+PHjjZtuuqnuOt+I9e7d27j//vttnysqKoygoCBjxowZVda/4447jD//+c92ZVFRUcbf//53wzAMo7Ky0ggICDBmzZpl219YWGi4uroay5cvr4cRNB11fa2rsnnzZkOS8eOPP9ZNp5ug+rrOhw4dMtq1a2fs3LnT6Nix4x8+IHGLDeeVnp6u1q1bKzIy0lYWHR0tJycnbdq0qcpjtm3bpvLyckVHR9vKrrjiCnXo0EHp6em2st27d+vpp5/WsmXLzvnSwD+C+rzOZkVFRfLx8am7zjdSZWVl2rZtm931cXJyUnR0dLXXJz093a6+JMXGxtrqHzhwQLm5uXZ1vLy8FBUVdc5rfqmrj2tdlaKiIlksFrVu3bpO+t3U1Nd1rqys1N13360JEyboqquuqp/ONzF/7G8k1Ehubq78/Pzsypo1ayYfHx/l5uZWe4yLi8tZ/yfm7+9vO+bUqVOKj4/XrFmz1KFDh3rpe1NSX9fZ7KuvvtLKlSv1t7/9rU763ZgdO3ZMFRUV8vf3tys/1/XJzc09Z/0z/3shbf4R1Me1Nvv11181ceJExcfH/2Ffulpf1zklJUXNmjXTQw89VPedbqIISH9gkyZNksViOee2d+/eejv/5MmTdeWVV+qvf/1rvZ2jMXD0df69nTt36tZbb1ViYqJiYmIa5JxAXSgvL9cdd9whwzC0cOFCR3fnkrJt2zbNnTtXr7/+uiwWi6O702g0c3QH4DiPPvqoRo8efc46nTt3VkBAgPLz8+3KT58+rePHjysgIKDK4wICAlRWVqbCwkK72Y28vDzbMWvXrtW3336rd999V9JvvwySJF9fX02ZMkXTp0+v5cgaF0df5zN2796t/v37629/+5uefPLJWo2lqfH19ZWzs/NZv56s6vqcERAQcM76Z/43Ly9PgYGBdnV69OhRh71vWurjWp9xJhz9+OOPWrt27R929kiqn+u8YcMG5efn283kV1RU6NFHH9WcOXN08ODBuh1EU+HoRVBo/M4sHt66daut7LPPPqvR4uF3333XVrZ37167xcP79+83vv32W9u2ZMkSQ5Lx1VdfVftrjEtZfV1nwzCMnTt3Gn5+fsaECRPqbwCNVO/evY0HHnjA9rmiosJo167dORe0Dho0yK6sT58+Zy3Snj17tm1/UVERi7SNur/WhmEYZWVlxpAhQ4yrrrrKyM/Pr5+ONzF1fZ2PHTtm9//F3377rREUFGRMnDjR2Lt3b/0NpJEjIKFG4uLijJ49exqbNm0y/vOf/xiXXXaZ3c/PDx06ZHTr1s3YtGmTrey+++4zOnToYKxdu9bYunWr0adPH6NPnz7VnmPdunV/6F+xGUb9XOdvv/3WaNu2rfHXv/7VyMnJsW1/lC+bFStWGK6ursbrr79u7N692/jb3/5mtG7d2sjNzTUMwzDuvvtuY9KkSbb6GzduNJo1a2bMnj3b2LNnj5GYmFjlz/xbt25tfPjhh8aOHTuMW2+9lZ/5G3V/rcvKyozBgwcb7du3NzIyMuz+/Z46dcohY2wM6uPftBm/YiMgoYYKCgqM+Ph4w8PDw/D09DTGjBljnDhxwrb/wIEDhiRj3bp1trJffvnFGD9+vOHt7W20aNHCGDp0qJGTk1PtOQhI9XOdExMTDUlnbR07dmzAkTnWvHnzjA4dOhguLi5G7969ja+//tq274YbbjBGjRplV//tt982Lr/8csPFxcW46qqrjI8//thuf2VlpfHUU08Z/v7+hqurq9G/f39j3759DTGURq8ur/WZf+9Vbb//b+CPqK7/TZsRkAzDYhj/f+EHAAAAJPErNgAAgLMQkAAAAEwISAAAACYEJAAAABMCEgAAgAkBCQAAwISABAAAYEJAAgAAMCEgAUAdmjZt2h/6pbXApYKABKDJOHr0qFxcXFRaWqry8nK1bNlS2dnZju4WgEsQAQlAk5Genq7w8HC1bNlS27dvl4+Pjzp06ODobgG4BBGQADQZX331la699lpJ0n/+8x/bn88oLCzUvffeq7Zt28rT01M333yzMjMzbfvP3P5atGiRgoOD1aJFC91xxx0qKiqy1amsrNTTTz+t9u3by9XVVT169FBqaqrdeQ4dOqT4+Hj5+PioZcuWioyM1KZNm+zqvPHGGwoJCZGXl5dGjBihEydO2Pa9++67Cg0Nlbu7u9q0aaPo6GiVlpbW2XUCcPGaOboDAHAu2dnZCgsLkySdPHlSzs7Oev311/XLL7/IYrGodevWuuuuu/Tiiy/q9ttvl7u7uz799FN5eXlp0aJF6t+/v7777jv5+PhIkvbv36+3335b//73v1VcXKx77rlH48eP15tvvilJmjt3rp577jktWrRIPXv21JIlSzR48GDt2rVLl112mUpKSnTDDTeoXbt2Wr16tQICArR9+3ZVVlba+pyVlaVVq1bpo48+0s8//6w77rhDycnJeuaZZ5STk6P4+HjNnDlTQ4cO1YkTJ7Rhwwbx3nCgkTEAoBErLy83Dhw4YGRmZhrNmzc3MjMzjf379xseHh7G+vXrjQMHDhhHjx41NmzYYHh6ehq//vqr3fFdunQxFi1aZBiGYSQmJhrOzs7GoUOHbPs//fRTw8nJycjJyTEMwzCCgoKMZ555xq6NXr16GePHjzcMwzAWLVpktGrVyigoKKiyv4mJiUaLFi2M4uJiW9mECROMqKgowzAMY9u2bYYk4+DBgxd5ZQDUJ26xAWjUmjVrppCQEO3du1e9evVSWFiYcnNz5e/vr+uvv14hISHy9fVVZmamSkpK1KZNG3l4eNi2AwcOKCsry9Zehw4d1K5dO9vnPn36qLKyUvv27VNxcbGOHDly1q27a6+9Vnv27JEkZWRkqGfPnrYZqaqEhISoVatWts+BgYHKz8+XJIWHh6t///4KDQ3V7bffrsWLF+vnn3+uk2sFoO5wiw1Ao3bVVVfpxx9/VHl5uSorK+Xh4aHTp0/r9OnT8vDwUMeOHbVr1y6VlJQoMDBQVqv1rDZat25dZ/1xd3c/b53mzZvbfbZYLLZbcM7Ozvr888/11Vdfac2aNZo3b56mTJmiTZs2qVOnTnXWTwAXhxkkAI3aJ598ooyMDAUEBOj//u//lJGRoauvvlpz5sxRRkaGPvnkE0nSNddco9zcXDVr1kxdu3a123x9fW3tZWdn68iRI7bPX3/9tZycnNStWzd5enoqKChIGzdutOvDxo0b1b17d0lSWFiYMjIydPz48VqPyWKx6Nprr9X06dP1zTffyMXFRR988EGt2wNQ9whIABq1jh07ysPDQ3l5ebr11lsVHBysXbt2adiwYeratas6duwoSYqOjlafPn00ZMgQrVmzRgcPHtRXX32lKVOmaOvWrbb23NzcNGrUKGVmZmrDhg166KGHdMcddyggIECSNGHCBKWkpGjlypXat2+fJk2apIyMDD388MOSpPj4eAUEBGjIkCHauHGjfvjhB7333ntKT0+v0Xg2bdqkZ599Vlu3blV2drbef/99HT16VFdeeWUdXzkAF4NbbAAaPavVql69esnNzU0bNmxQ+/btFRgYaFfHYrHok08+0ZQpUzRmzBgdPXpUAQEBuv766+Xv72+r17VrV912220aOHCgjh8/rkGDBunFF1+07X/ooYdUVFSkRx99VPn5+erevbtWr16tyy67TJLk4uKiNWvW6NFHH9XAgQN1+vRpde/eXQsWLKjRWDw9PfXll19qzpw5Ki4uVseOHfXcc89pwIABdXClANQVi2Hw21IAfwzTpk3TqlWrlJGR4eiuAGjkuMUGAABgQkACAAAw4RYbAACACTNIAAAAJgQkAAAAEwISAACACQEJAADAhIAEAABgQkACAAAwISABAACYEJAAAABM/h8Ls6p5p+gciQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "## Plot accuracy\n",
    "\n",
    "plt.plot(np.arange(num_epochs), history_model[\"train_acc\"], label='train acc')\n",
    "plt.plot(np.arange(num_epochs), history_model[\"test_acc\"], label='test acc')\n",
    "plt.xlabel('#epochs')\n",
    "plt.ylabel('accuracy')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "82512b88-1775-4423-832e-7f1d865de437",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x151692fffdf0>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAGwCAYAAABB4NqyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/SrBM8AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA4P0lEQVR4nO3df1xVVaL///cB5Iciv0RBCD2aTqUpEAiDzlyd6VzNmrJG07gzoTZTt1tj46W84rcJLEcBpQkTs3JqtF9XZxptvE2RSVKm5A8IS00rR4VUQMYEBQM8Z3/+6OtpToKiAgfcr+fjsR9x1l577bUW1Hm3z9r7WAzDMAQAAGAiHu7uAAAAQEcjAAEAANMhAAEAANMhAAEAANMhAAEAANMhAAEAANMhAAEAANPxcncHOiOHw6EjR46oZ8+eslgs7u4OAABoBcMwdPLkSUVERMjD4/zXeAhAzThy5IiioqLc3Q0AAHAJysvLddVVV523DgGoGT179pT07QQGBAS4uTcAAKA1amtrFRUV5XwfPx8CUDPOfuwVEBBAAAIAoItpzfIVFkEDAADTIQABAADTIQABAADTYQ0QAACtYLfb1dTU5O5umFq3bt3k6enZJm0RgAAAOA/DMFRRUaETJ064uyuQFBQUpPDw8Mt+Th8BCACA8zgbfvr06aPu3bvzgFw3MQxD9fX1qqqqkiT17dv3stojAAEA0AK73e4MP7169XJ3d0zPz89PklRVVaU+ffpc1sdhnWIR9NKlS2W1WuXr66vExERt27atxbrLly/Xj3/8YwUHBys4OFg2m+289e+//35ZLBbl5ua2Q88BAFeys2t+unfv7uae4Kyzv4vLXY/l9gC0evVqpaamKiMjQyUlJYqOjta4ceOcl7i+r7CwUMnJydq4caOKiooUFRWlsWPH6vDhw+fUXbt2rT766CNFRES09zAAAFcwPvbqPNrqd+H2APSHP/xB9957r6ZPn64hQ4bo2WefVffu3fXiiy82W//VV1/VAw88oJiYGF177bX64x//KIfDoYKCApd6hw8f1owZM/Tqq6+qW7du5+1DQ0ODamtrXTYAAHDlcmsAamxsVHFxsWw2m7PMw8NDNptNRUVFrWqjvr5eTU1NCgkJcZY5HA7dfffdmjVrloYOHXrBNjIzMxUYGOjc+CJUAACubG4NQNXV1bLb7QoLC3MpDwsLU0VFRavamD17tiIiIlxCVHZ2try8vPTQQw+1qo05c+aopqbGuZWXl7d+EAAAmIDVar3s9bT19fWaOHGiAgICZLFY3PpogS59F1hWVpZWrVqlwsJC+fr6SpKKi4u1ePFilZSUtPpzQh8fH/n4+LRnVwEA6FBjxoxRTExMm90EtH37dvXo0eOy2li5cqU2bdqkLVu2KDQ0VIGBgVqzZo2effZZFRcX6/jx4/r4448VExPTJn0+H7deAQoNDZWnp6cqKytdyisrKxUeHn7eY3NycpSVlaX169dr+PDhzvJNmzapqqpK/fr1k5eXl7y8vHTo0CE9/PDDslqt7TEMAAC6JMMwdObMmVbV7d2792XfDbd//35dd911uv76650PM6yrq9OPfvQjZWdnX1bbF8utAcjb21txcXEuC5jPLmhOSkpq8biFCxdq3rx5ys/PV3x8vMu+u+++W5988olKS0udW0REhGbNmqV33nmn3cYCADAHwzBU33jGLZthGK3q47Rp0/T+++9r8eLFslgsslgsOnjwoAoLC2WxWPT2228rLi5OPj4++vDDD7V//35NmDBBYWFh8vf314gRI7RhwwaXNr//EZjFYtEf//hH3XHHHerevbsGDx6sdevWtdinMWPG6Mknn9QHH3wgi8WiMWPGSPr2fTs9Pd1lKUtHcPtHYKmpqZo6dari4+OVkJCg3Nxc1dXVafr06ZKklJQURUZGKjMzU9K363vS09P12muvyWq1OtcK+fv7y9/fX7169TrnYVXdunVTeHi4rrnmmo4dHADginO6ya4h6e75H+o9T4xTd+8Lv3UvXrxYn3/+ua6//no98cQTkr69gnPw4EFJUlpamnJycjRw4EAFBwervLxcN998s+bPny8fHx+99NJLuvXWW7Vv3z7169evxfM8/vjjWrhwoRYtWqQlS5boF7/4hQ4dOuRyY9JZa9asUVpamnbt2qU1a9bI29v70iahjbg9AE2ZMkXHjh1Tenq6KioqFBMTo/z8fOfC6LKyMnl4fHehatmyZWpsbNSkSZNc2snIyNDcuXM7susAAHRKgYGB8vb2Vvfu3ZtdUvLEE0/o3//9352vQ0JCFB0d7Xw9b948rV27VuvWrdNvfvObFs8zbdo0JScnS5IWLFigp59+Wtu2bdNNN910Tt2QkBB1795d3t7eF1zm0hHcHoAk6Te/+U2LE1xYWOjy+mx6vRiXcgwAAM3x6+apPU+Mc9u528L3l4+cOnVKc+fO1d///ncdPXpUZ86c0enTp1VWVnbedv51DW6PHj0UEBDQ4oOMO5tOEYAAAOgqLBZLqz6G6sy+fzfXI488onfffVc5OTkaNGiQ/Pz8NGnSJDU2Np63ne8/aNhiscjhcLR5f9tD1/4NAgCAZnl7e8tut7eq7ubNmzVt2jTdcccdkr69InSlf3pCAAIA4ApktVq1detWHTx4UP7+/s0uTD5r8ODBWrNmjW699VZZLBY99thjHXYl5/jx4yorK9ORI0ckSfv27ZMkhYeHt+taIbd/FxgAAGh7jzzyiDw9PTVkyBD17t37vOt5/vCHPyg4OFgjR47UrbfeqnHjxumGG27okH6uW7dOsbGxuuWWWyRJd911l2JjY/Xss8+263ktRmsfKmAitbW1CgwMVE1NjQICAtzdHQCAm3zzzTc6cOCABgwY4PzGAbjX+X4nF/P+zRUgAABgOgQgAABgOgQgAABgOgQgAABgOgQgAABgOgQgAABgOgQgAABgOgQgAABgOgQgAADQpgzD0H333aeQkBBZLBaVlpa6u0vnIAABAHAFGjNmjGbOnNmmbU6bNk233377Bevl5+drxYoVevPNN3X06FFdf/31+uCDD3TrrbcqIiJCFotFb7zxRpv27WIRgAAAQJvav3+/+vbtq5EjRyo8PFxeXl6qq6tTdHS0li5d6u7uSSIAAQBwxZk2bZref/99LV68WBaLRRaLRQcPHpQk7dq1S+PHj5e/v7/CwsJ09913q7q62nns66+/rmHDhsnPz0+9evWSzWZTXV2d5s6dq5UrV+pvf/ubs83CwsJmzz1jxgyVlZXJYrHIarVKksaPH6/f//73uuOOOzpgBi7My90dAACgSzEMqanePefu1l2yWC5YbfHixfr88891/fXX64knnpAk9e7dWydOnNBPf/pT/frXv9ZTTz2l06dPa/bs2Zo8ebLee+89HT16VMnJyVq4cKHuuOMOnTx5Ups2bZJhGHrkkUf02Wefqba2Vn/6058kSSEhIc2e++qrr9bzzz+v7du3y9PTs23noI0QgAAAuBhN9dKCCPec+/87Inn3uGC1wMBAeXt7q3v37goPD3eW5+XlKTY2VgsWLHCWvfjii4qKitLnn3+uU6dO6cyZM/r5z3+u/v37S5KGDRvmrOvn56eGhgaXNps7d8+ePeXp6Xneeu5GAAIAwCR27typjRs3yt/f/5x9+/fv19ixY3XjjTdq2LBhGjdunMaOHatJkyYpODjYDb1tXwQgAAAuRrfu316Jcde5L8OpU6d06623Kjs7+5x9ffv2laenp959911t2bJF69ev15IlS/Too49q69atGjBgwGWdu7MhAAEAcDEsllZ9DOVu3t7estvtLmU33HCD/vrXv8pqtcrLq/kIYLFYNGrUKI0aNUrp6enq37+/1q5dq9TU1Gbb7Kq4CwwAgCuQ1WrV1q1bdfDgQVVXV8vhcOjBBx/U8ePHlZycrO3bt2v//v165513NH36dNntdm3dulULFizQjh07VFZWpjVr1ujYsWO67rrrnG1+8skn2rdvn6qrq9XU1NTq/pw6dUqlpaXOhyIeOHBApaWlKisra4/hXxABCACAK9AjjzwiT09PDRkyRL1791ZZWZkiIiK0efNm2e12jR07VsOGDdPMmTMVFBQkDw8PBQQE6IMPPtDNN9+sH/zgB/rd736nJ598UuPHj5ck3XvvvbrmmmsUHx+v3r17a/Pmza3uz44dOxQbG6vY2FhJUmpqqmJjY5Went4u478Qi2EYhlvO3InV1tYqMDBQNTU1CggIcHd3AABu8s033+jAgQMaMGCAfH193d0d6Py/k4t5/+YKEAAAMB0CEAAAMB0CEAAAMB0CEAAAMB0CEAAAF8D9Qp1HW/0uCEAAALSgW7dukqT6ejd9+SnOcfZ3cfZ3c6l4EjQAAC3w9PRUUFCQqqqqJEndu3eXpRXfxo62ZxiG6uvrVVVVpaCgoMv+lvlOEYCWLl2qRYsWqaKiQtHR0VqyZIkSEhKarbt8+XK99NJL2rVrlyQpLi5OCxYscKk/d+5crVq1SuXl5fL29lZcXJzmz5+vxMTEDhkPAODKcfYbzc+GILhXUFBQm3zLvNsD0OrVq5Wamqpnn31WiYmJys3N1bhx47Rv3z716dPnnPqFhYVKTk7WyJEj5evrq+zsbI0dO1a7d+9WZGSkJOkHP/iB8vLyNHDgQJ0+fVpPPfWUxo4dqy+//FK9e/fu6CECALowi8Wivn37qk+fPhf11Q9oe926dbvsKz9nuf1J0ImJiRoxYoTy8vIkSQ6HQ1FRUZoxY4bS0tIueLzdbldwcLDy8vKUkpLSbJ2zT4bcsGGDbrzxxgu2yZOgAQDoerrMk6AbGxtVXFwsm83mLPPw8JDNZlNRUVGr2qivr1dTU5NCQkJaPMfzzz+vwMBARUdHN1unoaFBtbW1LhsAALhyuTUAVVdXy263KywszKU8LCxMFRUVrWpj9uzZioiIcAlRkvTmm2/K399fvr6+euqpp/Tuu+8qNDS02TYyMzMVGBjo3KKioi5tQAAAoEvo0rfBZ2VladWqVVq7du05X4j2k5/8RKWlpdqyZYtuuukmTZ48ucUFbHPmzFFNTY1zKy8v74juAwAAN3FrAAoNDZWnp6cqKytdyisrKy+4wjsnJ0dZWVlav369hg8ffs7+Hj16aNCgQfrhD3+oF154QV5eXnrhhReabcvHx0cBAQEuGwAAuHK5NQCdvUW9oKDAWeZwOFRQUKCkpKQWj1u4cKHmzZun/Px8xcfHt+pcDodDDQ0Nl91nAADQ9bn9NvjU1FRNnTpV8fHxSkhIUG5ururq6jR9+nRJUkpKiiIjI5WZmSlJys7OVnp6ul577TVZrVbnWiF/f3/5+/urrq5O8+fP12233aa+ffuqurpaS5cu1eHDh3XnnXe6bZwAAKDzcHsAmjJlio4dO6b09HRVVFQoJiZG+fn5zoXRZWVl8vD47kLVsmXL1NjYqEmTJrm0k5GRoblz58rT01N79+7VypUrVV1drV69emnEiBHatGmThg4d2qFjAwAAnZPbnwPUGfEcIAAAup4u8xwgAAAAdyAAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0yEAAQAA0+kUAWjp0qWyWq3y9fVVYmKitm3b1mLd5cuX68c//rGCg4MVHBwsm83mUr+pqUmzZ8/WsGHD1KNHD0VERCglJUVHjhzpiKEAAIAuwO0BaPXq1UpNTVVGRoZKSkoUHR2tcePGqaqqqtn6hYWFSk5O1saNG1VUVKSoqCiNHTtWhw8fliTV19erpKREjz32mEpKSrRmzRrt27dPt912W0cOCwAAdGIWwzAMd3YgMTFRI0aMUF5eniTJ4XAoKipKM2bMUFpa2gWPt9vtCg4OVl5enlJSUpqts337diUkJOjQoUPq16/fOfsbGhrU0NDgfF1bW6uoqCjV1NQoICDgEkcGAAA6Um1trQIDA1v1/u3WK0CNjY0qLi6WzWZzlnl4eMhms6moqKhVbdTX16upqUkhISEt1qmpqZHFYlFQUFCz+zMzMxUYGOjcoqKiLmocAACga3FrAKqurpbdbldYWJhLeVhYmCoqKlrVxuzZsxUREeESov7VN998o9mzZys5ObnFNDhnzhzV1NQ4t/Ly8osbCAAA6FK83N2By5GVlaVVq1apsLBQvr6+5+xvamrS5MmTZRiGli1b1mI7Pj4+8vHxac+uAgCATsStASg0NFSenp6qrKx0Ka+srFR4ePh5j83JyVFWVpY2bNig4cOHn7P/bPg5dOiQ3nvvPdbyAAAAJ7d+BObt7a24uDgVFBQ4yxwOhwoKCpSUlNTicQsXLtS8efOUn5+v+Pj4c/afDT9ffPGFNmzYoF69erVL/wEAQNfk9o/AUlNTNXXqVMXHxyshIUG5ubmqq6vT9OnTJUkpKSmKjIxUZmamJCk7O1vp6el67bXXZLVanWuF/P395e/vr6amJk2aNEklJSV68803ZbfbnXVCQkLk7e3tnoECAIBOw+0BaMqUKTp27JjS09NVUVGhmJgY5efnOxdGl5WVycPjuwtVy5YtU2NjoyZNmuTSTkZGhubOnavDhw9r3bp1kqSYmBiXOhs3btSYMWPadTwAAKDzc/tzgDqji3mOAAAA6By6zHOAAAAA3IEABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATIcABAAATKdTBKClS5fKarXK19dXiYmJ2rZtW4t1ly9frh//+McKDg5WcHCwbDbbOfXXrFmjsWPHqlevXrJYLCotLW3nEQAAgK7E7QFo9erVSk1NVUZGhkpKShQdHa1x48apqqqq2fqFhYVKTk7Wxo0bVVRUpKioKI0dO1aHDx921qmrq9OPfvQjZWdnd9QwAABAF2IxDMNwZwcSExM1YsQI5eXlSZIcDoeioqI0Y8YMpaWlXfB4u92u4OBg5eXlKSUlxWXfwYMHNWDAAH388ceKiYlpdZ9qa2sVGBiompoaBQQEXNR4AACAe1zM+7dbrwA1NjaquLhYNpvNWebh4SGbzaaioqJWtVFfX6+mpiaFhIRccj8aGhpUW1vrsgEAgCuXWwNQdXW17Ha7wsLCXMrDwsJUUVHRqjZmz56tiIgIlxB1sTIzMxUYGOjcoqKiLrktAADQ+bl9DdDlyMrK0qpVq7R27Vr5+vpecjtz5sxRTU2NcysvL2/DXgIAgM7Gy50nDw0NlaenpyorK13KKysrFR4eft5jc3JylJWVpQ0bNmj48OGX1Q8fHx/5+PhcVhsAAKDrcOsVIG9vb8XFxamgoMBZ5nA4VFBQoKSkpBaPW7hwoebNm6f8/HzFx8d3RFcBAMAVxK1XgCQpNTVVU6dOVXx8vBISEpSbm6u6ujpNnz5dkpSSkqLIyEhlZmZKkrKzs5Wenq7XXntNVqvVuVbI399f/v7+kqTjx4+rrKxMR44ckSTt27dPkhQeHn7BK0sAAODK5/YANGXKFB07dkzp6emqqKhQTEyM8vPznQujy8rK5OHx3YWqZcuWqbGxUZMmTXJpJyMjQ3PnzpUkrVu3zhmgJOmuu+46pw4AADAvtz8HqDPiOUAAAHQ9XeY5QAAAAO5AAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKZDAAIAAKbTZgGovLxc99xzT1s1BwAA0G7aLAAdP35cK1eubKvmAAAA2o1XayuuW7fuvPv/8Y9/XHInli5dqkWLFqmiokLR0dFasmSJEhISmq27fPlyvfTSS9q1a5ckKS4uTgsWLHCpbxiGMjIytHz5cp04cUKjRo3SsmXLNHjw4EvuIwAAuHK0OgDdfvvtslgsMgyjxToWi+WiO7B69Wqlpqbq2WefVWJionJzczVu3Djt27dPffr0Oad+YWGhkpOTNXLkSPn6+io7O1tjx47V7t27FRkZKUlauHChnn76aa1cuVIDBgzQY489pnHjxmnPnj3y9fW96D4CAIAri8U4X6L5F5GRkXrmmWc0YcKEZveXlpYqLi5Odrv9ojqQmJioESNGKC8vT5LkcDgUFRWlGTNmKC0t7YLH2+12BQcHKy8vTykpKTIMQxEREXr44Yf1yCOPSJJqamoUFhamFStW6K677jqnjYaGBjU0NDhf19bWKioqSjU1NQoICLio8QAAAPeora1VYGBgq96/W70GKC4uTsXFxS3uv9DVoeY0NjaquLhYNpvtuw55eMhms6moqKhVbdTX16upqUkhISGSpAMHDqiiosKlzcDAQCUmJrbYZmZmpgIDA51bVFTURY0DAAB0La0KQJ988olmzZqlkSNHtlhn0KBB2rhx40WdvLq6Wna7XWFhYS7lYWFhqqioaFUbs2fPVkREhDPwnD3uYtqcM2eOampqnFt5eflFjQMAAHQtrVoDFBsbq6NHj6pPnz4aOHCgtm/frl69ernU6dGjh0aPHt0unWxJVlaWVq1apcLCwsta2+Pj4yMfH5827BkAAOjMWnUFKCgoSAcOHJAkHTx4UA6Ho01OHhoaKk9PT1VWVrqUV1ZWKjw8/LzH5uTkKCsrS+vXr9fw4cOd5WePu5Q2AQCAObQqAE2cOFGjR4/WgAEDZLFYFB8fr4EDBza7XQxvb2/FxcWpoKDAWeZwOFRQUKCkpKQWj1u4cKHmzZun/Px8xcfHu+wbMGCAwsPDXdqsra3V1q1bz9smAAAwj1Z9BPb888/r5z//ub788ks99NBDuvfee9WzZ8826UBqaqqmTp2q+Ph4JSQkKDc3V3V1dZo+fbokKSUlRZGRkcrMzJQkZWdnKz09Xa+99pqsVqtzXY+/v7/8/f1lsVg0c+ZM/f73v9fgwYOdt8FHRETo9ttvb5M+AwCArq3VzwG66aabJEnFxcX67W9/22YBaMqUKTp27JjS09NVUVGhmJgY5efnOxcxl5WVycPjuwtVy5YtU2NjoyZNmuTSTkZGhubOnStJ+p//+R/V1dXpvvvu04kTJ/SjH/1I+fn5PAMIAABIuojnAJnJxTxHAAAAdA7t8hwgAACAKwUBCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmA4BCAAAmI7bA9DSpUtltVrl6+urxMREbdu2rcW6u3fv1sSJE2W1WmWxWJSbm3tOnZMnT2rmzJnq37+//Pz8NHLkSG3fvr0dRwAAALoatwag1atXKzU1VRkZGSopKVF0dLTGjRunqqqqZuvX19dr4MCBysrKUnh4eLN1fv3rX+vdd9/Vyy+/rE8//VRjx46VzWbT4cOH23MoAACgC7EYhmG46+SJiYkaMWKE8vLyJEkOh0NRUVGaMWOG0tLSznus1WrVzJkzNXPmTGfZ6dOn1bNnT/3tb3/TLbfc4iyPi4vT+PHj9fvf/75V/aqtrVVgYKBqamoUEBBw8QMDAAAd7mLev912BaixsVHFxcWy2WzfdcbDQzabTUVFRZfU5pkzZ2S32+Xr6+tS7ufnpw8//LDF4xoaGlRbW+uyAQCAK5fbAlB1dbXsdrvCwsJcysPCwlRRUXFJbfbs2VNJSUmaN2+ejhw5IrvdrldeeUVFRUU6evRoi8dlZmYqMDDQuUVFRV3S+QEAQNfg9kXQbe3ll1+WYRiKjIyUj4+Pnn76aSUnJ8vDo+WhzpkzRzU1Nc6tvLy8A3sMAAA6mpe7ThwaGipPT09VVla6lFdWVra4wLk1rr76ar3//vuqq6tTbW2t+vbtqylTpmjgwIEtHuPj4yMfH59LPicAAOha3HYFyNvbW3FxcSooKHCWORwOFRQUKCkp6bLb79Gjh/r27auvv/5a77zzjiZMmHDZbQIAgCuD264ASVJqaqqmTp2q+Ph4JSQkKDc3V3V1dZo+fbokKSUlRZGRkcrMzJT07cLpPXv2OH8+fPiwSktL5e/vr0GDBkmS3nnnHRmGoWuuuUZffvmlZs2apWuvvdbZJgAAgFsD0JQpU3Ts2DGlp6eroqJCMTExys/Pdy6MLisrc1m7c+TIEcXGxjpf5+TkKCcnR6NHj1ZhYaEkqaamRnPmzNFXX32lkJAQTZw4UfPnz1e3bt06dGwAAKDzcutzgDorngMEAEDX0yWeAwQAAOAuBCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6BCAAAGA6bg9AS5culdVqla+vrxITE7Vt27YW6+7evVsTJ06U1WqVxWJRbm7uOXXsdrsee+wxDRgwQH5+frr66qs1b948GYbRjqMAAABdiVsD0OrVq5WamqqMjAyVlJQoOjpa48aNU1VVVbP16+vrNXDgQGVlZSk8PLzZOtnZ2Vq2bJny8vL02WefKTs7WwsXLtSSJUvacygAAKALsRhuvDSSmJioESNGKC8vT5LkcDgUFRWlGTNmKC0t7bzHWq1WzZw5UzNnznQp/9nPfqawsDC98MILzrKJEyfKz89Pr7zySqv6VVtbq8DAQNXU1CggIODiBgUAANziYt6/3XYFqLGxUcXFxbLZbN91xsNDNptNRUVFl9zuyJEjVVBQoM8//1yStHPnTn344YcaP358i8c0NDSotrbWZQMAAFcuL3eduLq6Wna7XWFhYS7lYWFh2rt37yW3m5aWptraWl177bXy9PSU3W7X/Pnz9Ytf/KLFYzIzM/X4449f8jkBAEDX4vZF0G3tz3/+s1599VW99tprKikp0cqVK5WTk6OVK1e2eMycOXNUU1Pj3MrLyzuwxwAAoKO57QpQaGioPD09VVlZ6VJeWVnZ4gLn1pg1a5bS0tJ01113SZKGDRumQ4cOKTMzU1OnTm32GB8fH/n4+FzyOQEAQNfititA3t7eiouLU0FBgbPM4XCooKBASUlJl9xufX29PDxch+Xp6SmHw3HJbQIAgCuL264ASVJqaqqmTp2q+Ph4JSQkKDc3V3V1dZo+fbokKSUlRZGRkcrMzJT07cLpPXv2OH8+fPiwSktL5e/vr0GDBkmSbr31Vs2fP1/9+vXT0KFD9fHHH+sPf/iD7rnnHvcMEgAAdDpuvQ1ekvLy8rRo0SJVVFQoJiZGTz/9tBITEyVJY8aMkdVq1YoVKyRJBw8e1IABA85pY/To0SosLJQknTx5Uo899pjWrl2rqqoqRUREKDk5Wenp6fL29m5Vn7gNHgCArudi3r/dHoA6IwIQAABdT5d4DhAAAIC7EIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpEIAAAIDpdIoAtHTpUlmtVvn6+ioxMVHbtm1rse7u3bs1ceJEWa1WWSwW5ebmnlPn7L7vbw8++GA7jgIAAHQVbg9Aq1evVmpqqjIyMlRSUqLo6GiNGzdOVVVVzdavr6/XwIEDlZWVpfDw8GbrbN++XUePHnVu7777riTpzjvvbLdxAACArsNiGIbhzg4kJiZqxIgRysvLkyQ5HA5FRUVpxowZSktLO++xVqtVM2fO1MyZM89bb+bMmXrzzTf1xRdfyGKxXLBPtbW1CgwMVE1NjQICAlo9FgAA4D4X8/7t1itAjY2NKi4uls1mc5Z5eHjIZrOpqKiozc7xyiuv6J577mkx/DQ0NKi2ttZlAwAAVy63BqDq6mrZ7XaFhYW5lIeFhamioqJNzvHGG2/oxIkTmjZtWot1MjMzFRgY6NyioqLa5NwAAKBzcvsaoPb2wgsvaPz48YqIiGixzpw5c1RTU+PcysvLO7CHAACgo3m58+ShoaHy9PRUZWWlS3llZWWLC5wvxqFDh7RhwwatWbPmvPV8fHzk4+Nz2ecDAABdg1uvAHl7eysuLk4FBQXOMofDoYKCAiUlJV12+3/605/Up08f3XLLLZfdFgAAuHK49QqQJKWmpmrq1KmKj49XQkKCcnNzVVdXp+nTp0uSUlJSFBkZqczMTEnfLmres2eP8+fDhw+rtLRU/v7+GjRokLNdh8OhP/3pT5o6daq8vNw+TAAA0Im4PRlMmTJFx44dU3p6uioqKhQTE6P8/HznwuiysjJ5eHx3oerIkSOKjY11vs7JyVFOTo5Gjx6twsJCZ/mGDRtUVlame+65p8PGAgAAuga3PweoM+I5QAAAdD1d5jlAAAAA7kAAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApkMAAgAApuP2ALR06VJZrVb5+voqMTFR27Zta7Hu7t27NXHiRFmtVlksFuXm5jZb7/Dhw/rlL3+pXr16yc/PT8OGDdOOHTvaaQQAAKCrcWsAWr16tVJTU5WRkaGSkhJFR0dr3LhxqqqqarZ+fX29Bg4cqKysLIWHhzdb5+uvv9aoUaPUrVs3vf3229qzZ4+efPJJBQcHt+dQAABAF2IxDMNw18kTExM1YsQI5eXlSZIcDoeioqI0Y8YMpaWlnfdYq9WqmTNnaubMmS7laWlp2rx5szZt2nTJ/aqtrVVgYKBqamoUEBBwye0AAICOczHv3267AtTY2Kji4mLZbLbvOuPhIZvNpqKioktud926dYqPj9edd96pPn36KDY2VsuXLz/vMQ0NDaqtrXXZAADAlcttAai6ulp2u11hYWEu5WFhYaqoqLjkdv/xj39o2bJlGjx4sN555x3913/9lx566CGtXLmyxWMyMzMVGBjo3KKioi75/AAAoPNz+yLotuZwOHTDDTdowYIFio2N1X333ad7771Xzz77bIvHzJkzRzU1Nc6tvLy8A3sMAAA6mtsCUGhoqDw9PVVZWelSXllZ2eIC59bo27evhgwZ4lJ23XXXqaysrMVjfHx8FBAQ4LIBAIArl9sCkLe3t+Li4lRQUOAsczgcKigoUFJS0iW3O2rUKO3bt8+l7PPPP1f//v0vuU0AAHBl8XLnyVNTUzV16lTFx8crISFBubm5qqur0/Tp0yVJKSkpioyMVGZmpqRvF07v2bPH+fPhw4dVWloqf39/DRo0SJL03//93xo5cqQWLFigyZMna9u2bXr++ef1/PPPu2eQAACg03HrbfCSlJeXp0WLFqmiokIxMTF6+umnlZiYKEkaM2aMrFarVqxYIUk6ePCgBgwYcE4bo0ePVmFhofP1m2++qTlz5uiLL77QgAEDlJqaqnvvvbfVfeI2eAAAup6Lef92ewDqjAhAAAB0PV3iOUAAAADu4tY1QJ3V2YtiPBARAICu4+z7dms+3CIANePkyZOSxAMRAQDogk6ePKnAwMDz1mENUDMcDoeOHDminj17ymKxuLs7bldbW6uoqCiVl5ezJqodMc8dg3nuGMxzx2Guv2MYhk6ePKmIiAh5eJx/lQ9XgJrh4eGhq666yt3d6HR4SGTHYJ47BvPcMZjnjsNcf+tCV37OYhE0AAAwHQIQAAAwHQIQLsjHx0cZGRny8fFxd1euaMxzx2CeOwbz3HGY60vDImgAAGA6XAECAACmQwACAACmQwACAACmQwACAACmQwCCjh8/rl/84hcKCAhQUFCQfvWrX+nUqVPnPeabb77Rgw8+qF69esnf318TJ05UZWVls3X/+c9/6qqrrpLFYtGJEyfaYQRdQ3vM886dO5WcnKyoqCj5+fnpuuuu0+LFi9t7KJ3O0qVLZbVa5evrq8TERG3btu289f/yl7/o2muvla+vr4YNG6a33nrLZb9hGEpPT1ffvn3l5+cnm82mL774oj2H0CW05Tw3NTVp9uzZGjZsmHr06KGIiAilpKToyJEj7T2MTq+t/57/1f333y+LxaLc3Nw27nUXZMD0brrpJiM6Otr46KOPjE2bNhmDBg0ykpOTz3vM/fffb0RFRRkFBQXGjh07jB/+8IfGyJEjm607YcIEY/z48YYk4+uvv26HEXQN7THPL7zwgvHQQw8ZhYWFxv79+42XX37Z8PPzM5YsWdLew+k0Vq1aZXh7exsvvviisXv3buPee+81goKCjMrKymbrb9682fD09DQWLlxo7Nmzx/jd735ndOvWzfj000+ddbKysozAwEDjjTfeMHbu3GncdtttxoABA4zTp0931LA6nbae5xMnThg2m81YvXq1sXfvXqOoqMhISEgw4uLiOnJYnU57/D2ftWbNGiM6OtqIiIgwnnrqqXYeSedHADK5PXv2GJKM7du3O8vefvttw2KxGIcPH272mBMnThjdunUz/vKXvzjLPvvsM0OSUVRU5FL3mWeeMUaPHm0UFBSYOgC19zz/qwceeMD4yU9+0nad7+QSEhKMBx980PnabrcbERERRmZmZrP1J0+ebNxyyy0uZYmJicZ//ud/GoZhGA6HwwgPDzcWLVrk3H/ixAnDx8fH+N///d92GEHX0Nbz3Jxt27YZkoxDhw61Tae7oPaa56+++sqIjIw0du3aZfTv358AZBgGH4GZXFFRkYKCghQfH+8ss9ls8vDw0NatW5s9pri4WE1NTbLZbM6ya6+9Vv369VNRUZGzbM+ePXriiSf00ksvXfBL6a507TnP31dTU6OQkJC263wn1tjYqOLiYpc58vDwkM1ma3GOioqKXOpL0rhx45z1Dxw4oIqKCpc6gYGBSkxMPO+8X8naY56bU1NTI4vFoqCgoDbpd1fTXvPscDh09913a9asWRo6dGj7dL4LMve7ElRRUaE+ffq4lHl5eSkkJEQVFRUtHuPt7X3Of6TCwsKcxzQ0NCg5OVmLFi1Sv3792qXvXUl7zfP3bdmyRatXr9Z9993XJv3u7Kqrq2W32xUWFuZSfr45qqioOG/9s/+8mDavdO0xz9/3zTffaPbs2UpOTjbtF3q21zxnZ2fLy8tLDz30UNt3ugsjAF2h0tLSZLFYzrvt3bu33c4/Z84cXXfddfrlL3/ZbufoDNw9z/9q165dmjBhgjIyMjR27NgOOSfQFpqamjR58mQZhqFly5a5uztXlOLiYi1evFgrVqyQxWJxd3c6FS93dwDt4+GHH9a0adPOW2fgwIEKDw9XVVWVS/mZM2d0/PhxhYeHN3tceHi4GhsbdeLECZerE5WVlc5j3nvvPX366ad6/fXXJX17V40khYaG6tFHH9Xjjz9+iSPrXNw9z2ft2bNHN954o+677z797ne/u6SxdEWhoaHy9PQ85w7E5uborPDw8PPWP/vPyspK9e3b16VOTExMG/a+62iPeT7rbPg5dOiQ3nvvPdNe/ZHaZ543bdqkqqoqlyvxdrtdDz/8sHJzc3Xw4MG2HURX4u5FSHCvs4tzd+zY4Sx75513WrU49/XXX3eW7d2712Vx7pdffml8+umnzu3FF180JBlbtmxp8W6GK1l7zbNhGMauXbuMPn36GLNmzWq/AXRiCQkJxm9+8xvna7vdbkRGRp530ejPfvYzl7KkpKRzFkHn5OQ499fU1LAIuo3n2TAMo7Gx0bj99tuNoUOHGlVVVe3T8S6mree5urra5b/Fn376qREREWHMnj3b2Lt3b/sNpAsgAMG46aabjNjYWGPr1q3Ghx9+aAwePNjl9uyvvvrKuOaaa4ytW7c6y+6//36jX79+xnvvvWfs2LHDSEpKMpKSklo8x8aNG019F5hhtM88f/rpp0bv3r2NX/7yl8bRo0edm5neTFatWmX4+PgYK1asMPbs2WPcd999RlBQkFFRUWEYhmHcfffdRlpamrP+5s2bDS8vLyMnJ8f47LPPjIyMjGZvgw8KCjL+9re/GZ988okxYcIEboNv43lubGw0brvtNuOqq64ySktLXf5+Gxoa3DLGzqA9/p6/j7vAvkUAgvHPf/7TSE5ONvz9/Y2AgABj+vTpxsmTJ537Dxw4YEgyNm7c6Cw7ffq08cADDxjBwcFG9+7djTvuuMM4evRoi+cgALXPPGdkZBiSztn69+/fgSNzvyVLlhj9+vUzvL29jYSEBOOjjz5y7hs9erQxdepUl/p//vOfjR/84AeGt7e3MXToUOPvf/+7y36Hw2E89thjRlhYmOHj42PceOONxr59+zpiKJ1aW87z2b/35rZ//XfAjNr67/n7CEDfshjG/784AwAAwCS4CwwAAJgOAQgAAJgOAQgAAJgOAQgAAJgOAQgAAJgOAQgAAJgOAQgAAJgOAQgAAJgOAQgALsLcuXNN+6WowJWEAASg0zh27Ji8vb1VV1enpqYm9ejRQ2VlZe7uFoArEAEIQKdRVFSk6Oho9ejRQyUlJQoJCVG/fv3c3S0AVyACEIBOY8uWLRo1apQk6cMPP3T+fNaJEyf061//Wr1791ZAQIB++tOfaufOnc79Zz+eeu655xQVFaXu3btr8uTJqqmpcdZxOBx64okndNVVV8nHx0cxMTHKz893Oc9XX32l5ORkhYSEqEePHoqPj9fWrVtd6rz88suyWq0KDAzUXXfdpZMnTzr3vf766xo2bJj8/PzUq1cv2Ww21dXVtdk8Abh8Xu7uAABzKysr0/DhwyVJ9fX18vT01IoVK3T69GlZLBYFBQXpP/7jP/TMM8/ozjvvlJ+fn95++20FBgbqueee04033qjPP/9cISEhkqQvv/xSf/7zn/V///d/qq2t1a9+9Ss98MADevXVVyVJixcv1pNPPqnnnntOsbGxevHFF3Xbbbdp9+7dGjx4sE6dOqXRo0crMjJS69atU3h4uEpKSuRwOJx93r9/v9544w29+eab+vrrrzV58mRlZWVp/vz5Onr0qJKTk7Vw4ULdcccdOnnypDZt2iS+dxroZNz8bfQATK6pqck4cOCAsXPnTqNbt27Gzp07jS+//NLw9/c33n//fePAgQPGsWPHjE2bNhkBAQHGN99843L81VdfbTz33HOGYRhGRkaG4enpaXz11VfO/W+//bbh4eFhHD161DAMw4iIiDDmz5/v0saIESOMBx54wDAMw3juueeMnj17Gv/85z+b7W9GRobRvXt3o7a21lk2a9YsIzEx0TAMwyguLjYkGQcPHrzMmQHQnvgIDIBbeXl5yWq1au/evRoxYoSGDx+uiooKhYWF6d/+7d9ktVoVGhqqnTt36tSpU+rVq5f8/f2d24EDB7R//35ne/369VNkZKTzdVJSkhwOh/bt26fa2lodOXLknI/WRo0apc8++0ySVFpaqtjYWOcVpeZYrVb17NnT+bpv376qqqqSJEVHR+vGG2/UsGHDdOedd2r58uX6+uuv22SuALQdPgID4FZDhw7VoUOH1NTUJIfDIX9/f505c0ZnzpyRv7+/+vfvr927d+vUqVPq27evCgsLz2kjKCiozfrj5+d3wTrdunVzeW2xWJwfkXl6eurdd9/Vli1btH79ei1ZskSPPvqotm7dqgEDBrRZPwFcHq4AAXCrt956S6WlpQoPD9crr7yi0tJSXX/99crNzVVpaaneeustSdINN9ygiooKeXl5adCgQS5baGios72ysjIdOXLE+fqjjz6Sh4eHrrnmGgUEBCgiIkKbN2926cPmzZs1ZMgQSdLw4cNVWlqq48ePX/KYLBaLRo0apccff1wff/yxvL29tXbt2ktuD0DbIwABcKv+/fvL399flZWVmjBhgqKiorR7925NnDhRgwYNUv/+/SVJNptNSUlJuv3227V+/XodPHhQW7Zs0aOPPqodO3Y42/P19dXUqVO1c+dObdq0SQ899JAmT56s8PBwSdKsWbOUnZ2t1atXa9++fUpLS1Npaal++9vfSpKSk5MVHh6u22+/XZs3b9Y//vEP/fWvf1VRUVGrxrN161YtWLBAO3bsUFlZmdasWaNjx47puuuua+OZA3A5+AgMgNsVFhZqxIgR8vX11aZNm3TVVVepb9++LnUsFoveeustPfroo5o+fbqOHTum8PBw/du//ZvCwsKc9QYNGqSf//znuvnmm3X8+HH97Gc/0zPPPOPc/9BDD6mmpkYPP/ywqqqqNGTIEK1bt06DBw+WJHl7e2v9+vV6+OGHdfPNN+vMmTMaMmSIli5d2qqxBAQE6IMPPlBubq5qa2vVv39/Pfnkkxo/fnwbzBSAtmIxDO7NBHBlmDt3rt544w2Vlpa6uysAOjk+AgMAAKZDAAIAAKbDR2AAAMB0uAIEAABMhwAEAABMhwAEAABMhwAEAABMhwAEAABMhwAEAABMhwAEAABMhwAEAABM5/8BObS9d2aKcaYAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "## Plot accuracy\n",
    "\n",
    "plt.plot(np.arange(num_epochs), history_model[\"train_f1\"], label='train f1')\n",
    "plt.plot(np.arange(num_epochs), history_model[\"test_f1\"], label='test f1')\n",
    "plt.xlabel('#epochs')\n",
    "plt.ylabel('f1')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "b3f44965-b6e6-4932-a517-08239c27afb5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Baseline_Model(\n",
       "  (fc1): Linear(in_features=512, out_features=32, bias=True)\n",
       "  (fc2): Linear(in_features=32, out_features=5, bias=True)\n",
       "  (relu): ReLU()\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       ")"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e5ddf34-6152-4ffd-9fdf-e0ff7d64c1c6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "71b5f7d9-95af-4002-9b4e-f0a520d09dcd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch.save(model.state_dict(), f'models/Baseline_model/state_dict_epoch{i}_batch_size{CONFIG[\"batch_size\"]}.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d66a740e-4beb-46e0-8208-cf3d4da05782",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}