Toxic_Comments_using_LSTM.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Toxic Comments using LSTM.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/pranaya-mathur/Deep-Learning/blob/master/Toxic_Comments_using_LSTM.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "KFtrP32yIA-E",
        "colab_type": "code",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 91
        },
        "outputId": "613109b0-508f-4daf-c3c6-12c0ff560b52"
      },
      "cell_type": "code",
      "source": [
        "from google.colab import files\n",
        "files.upload()"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-f10fc0bf-c40e-48f8-a756-3cd1e46c4d90\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-f10fc0bf-c40e-48f8-a756-3cd1e46c4d90\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving kaggle.json to kaggle.json\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'kaggle.json': b'{\"username\":\"chinki86\",\"key\":\"2b37558d5b359235a655730a30d3156b\"}'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 1
        }
      ]
    },
    {
      "metadata": {
        "id": "CxpBIIPNIrge",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 442
        },
        "outputId": "ba15dc8c-ff3b-4eae-a16b-e1bf6291642a"
      },
      "cell_type": "code",
      "source": [
        "!ls -lha kaggle.json\n",
        "!mkdir -p ~/.kaggle\n",
        "!cp kaggle.json ~/.kaggle/\n",
        "\n",
        "!pip install kaggle\n",
        "\n",
        "!kaggle competitions download -c jigsaw-toxic-comment-classification-challenge"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-rw-r--r-- 1 root root 64 Apr 20 14:25 kaggle.json\n",
            "Requirement already satisfied: kaggle in /usr/local/lib/python3.6/dist-packages (1.5.3)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from kaggle) (1.22)\n",
            "Requirement already satisfied: six>=1.10 in /usr/local/lib/python3.6/dist-packages (from kaggle) (1.11.0)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.6/dist-packages (from kaggle) (2019.3.9)\n",
            "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.6/dist-packages (from kaggle) (2.5.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from kaggle) (2.18.4)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from kaggle) (4.28.1)\n",
            "Requirement already satisfied: python-slugify in /usr/local/lib/python3.6/dist-packages (from kaggle) (3.0.2)\n",
            "Requirement already satisfied: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->kaggle) (2.6)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->kaggle) (3.0.4)\n",
            "Requirement already satisfied: text-unidecode==1.2 in /usr/local/lib/python3.6/dist-packages (from python-slugify->kaggle) (1.2)\n",
            "Warning: Your Kaggle API key is readable by other users on this system! To fix this, you can run 'chmod 600 /root/.kaggle/kaggle.json'\n",
            "Downloading sample_submission.csv.zip to /content\n",
            "  0% 0.00/1.39M [00:00<?, ?B/s]\n",
            "100% 1.39M/1.39M [00:00<00:00, 91.3MB/s]\n",
            "Downloading test.csv.zip to /content\n",
            " 73% 17.0M/23.4M [00:00<00:00, 28.5MB/s]\n",
            "100% 23.4M/23.4M [00:00<00:00, 47.7MB/s]\n",
            "Downloading train.csv.zip to /content\n",
            " 38% 10.0M/26.3M [00:00<00:00, 25.8MB/s]\n",
            "100% 26.3M/26.3M [00:00<00:00, 66.6MB/s]\n",
            "Downloading test_labels.csv.zip to /content\n",
            "  0% 0.00/1.46M [00:00<?, ?B/s]\n",
            "100% 1.46M/1.46M [00:00<00:00, 207MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "TC0XAbCxKubt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 272
        },
        "outputId": "004179df-52f1-480a-9853-6fcef7850810"
      },
      "cell_type": "code",
      "source": [
        "# Loading Glove Vectors\n",
        "\n",
        "!wget http://nlp.stanford.edu/data/glove.6B.zip"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2019-04-20 14:34:05--  http://nlp.stanford.edu/data/glove.6B.zip\n",
            "Resolving nlp.stanford.edu (nlp.stanford.edu)... 171.64.67.140\n",
            "Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:80... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://nlp.stanford.edu/data/glove.6B.zip [following]\n",
            "--2019-04-20 14:34:05--  https://nlp.stanford.edu/data/glove.6B.zip\n",
            "Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 862182613 (822M) [application/zip]\n",
            "Saving to: ‘glove.6B.zip’\n",
            "\n",
            "glove.6B.zip        100%[===================>] 822.24M  90.6MB/s    in 9.1s    \n",
            "\n",
            "2019-04-20 14:34:15 (90.0 MB/s) - ‘glove.6B.zip’ saved [862182613/862182613]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "vdXsSOPsK5El",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import zipfile\n",
        "zip_ref = zipfile.ZipFile(\"glove.6B.zip\", 'r')\n",
        "zip_ref.extractall(\"glove vectors\")\n",
        "zip_ref.close()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "5NZdooDWK_vw",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "zip_ref = zipfile.ZipFile(\"train.csv.zip\", 'r')\n",
        "zip_ref.extractall(\"train_data\")\n",
        "zip_ref.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ZaOgX87yLONS",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "18b38af9-6812-4795-c65f-8e8b326c5fc3"
      },
      "cell_type": "code",
      "source": [
        "# Importing the required Libraries\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from keras.models import Model\n",
        "from keras.layers import Dense, Embedding, Input\n",
        "from keras.layers import LSTM, Bidirectional, GlobalMaxPool1D, Dropout\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.optimizers import Adam\n",
        "from sklearn.metrics import roc_auc_score"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "0HbmlMAnMqNr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "1077755f-e0f9-49d8-ee09-736c1aba6e99"
      },
      "cell_type": "code",
      "source": [
        "train_data = pd.read_csv(\"train_data/train.csv\")\n",
        "train_data.head()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>comment_text</th>\n",
              "      <th>toxic</th>\n",
              "      <th>severe_toxic</th>\n",
              "      <th>obscene</th>\n",
              "      <th>threat</th>\n",
              "      <th>insult</th>\n",
              "      <th>identity_hate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0000997932d777bf</td>\n",
              "      <td>Explanation\\nWhy the edits made under my usern...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>000103f0d9cfb60f</td>\n",
              "      <td>D'aww! He matches this background colour I'm s...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>000113f07ec002fd</td>\n",
              "      <td>Hey man, I'm really not trying to edit war. It...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0001b41b1c6bb37e</td>\n",
              "      <td>\"\\nMore\\nI can't make any real suggestions on ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0001d958c54c6e35</td>\n",
              "      <td>You, sir, are my hero. Any chance you remember...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 id                                       comment_text  toxic  \\\n",
              "0  0000997932d777bf  Explanation\\nWhy the edits made under my usern...      0   \n",
              "1  000103f0d9cfb60f  D'aww! He matches this background colour I'm s...      0   \n",
              "2  000113f07ec002fd  Hey man, I'm really not trying to edit war. It...      0   \n",
              "3  0001b41b1c6bb37e  \"\\nMore\\nI can't make any real suggestions on ...      0   \n",
              "4  0001d958c54c6e35  You, sir, are my hero. Any chance you remember...      0   \n",
              "\n",
              "   severe_toxic  obscene  threat  insult  identity_hate  \n",
              "0             0        0       0       0              0  \n",
              "1             0        0       0       0              0  \n",
              "2             0        0       0       0              0  \n",
              "3             0        0       0       0              0  \n",
              "4             0        0       0       0              0  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "metadata": {
        "id": "i9aUDGATNGCZ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "MAX_SEQUENCE_LENGTH = 100\n",
        "MAX_VOCAB_SIZE = 20000\n",
        "EMBEDDING_DIM = 300\n",
        "VALIDATION_SPLIT = 0.2\n",
        "BATCH_SIZE = 128\n",
        "EPOCHS = 5"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "_kBLtzghP4M3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "dd4138bc-0000-49b2-9587-2be7bbd6afd5"
      },
      "cell_type": "code",
      "source": [
        "word2vec = {}\n",
        "with open(\"glove vectors/glove.6B.300d.txt\") as f:\n",
        "  # is just a space-separated text file in the format:\n",
        "  # word vec[0] vec[1] vec[2] ...\n",
        "  for line in f:\n",
        "    values = line.split()\n",
        "    word = values[0]\n",
        "    vec = np.asarray(values[1:], dtype='float32')\n",
        "    word2vec[word] = vec\n",
        "print('Found %s word vectors.' % len(word2vec))"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 400000 word vectors.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "HeKN5RnSQiVY",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "sentences = train_data[\"comment_text\"].fillna(\"DUMMY_VALUE\").values\n",
        "possible_labels = [\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]\n",
        "targets = train_data[possible_labels].values"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "oEpFvBAXQpaZ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# convert the sentences (strings) into integers\n",
        "tokenizer = Tokenizer(num_words=MAX_VOCAB_SIZE)\n",
        "tokenizer.fit_on_texts(sentences)\n",
        "sequences = tokenizer.texts_to_sequences(sentences)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "gj0n3senQszo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "ef419fee-ff33-4fa0-feee-5437a3fbb822"
      },
      "cell_type": "code",
      "source": [
        "# get word -> integer mapping\n",
        "word2idx = tokenizer.word_index\n",
        "print('Found %s unique tokens.' % len(word2idx))"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 210337 unique tokens.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "vZknJ7Q2REKb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "6325f6ae-9668-4e9b-a0ba-a31d3c292551"
      },
      "cell_type": "code",
      "source": [
        "# pad sequences so that we get a N x T matrix\n",
        "data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)\n",
        "print('Shape of data tensor:', data.shape)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Shape of data tensor: (159571, 100)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "fSvjB1A-ReXI",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "num_words = min(MAX_VOCAB_SIZE, len(word2idx) + 1)\n",
        "embedding_matrix = np.zeros((num_words, EMBEDDING_DIM))\n",
        "for word, i in word2idx.items():\n",
        "  if i < MAX_VOCAB_SIZE:\n",
        "    embedding_vector = word2vec.get(word)\n",
        "    if embedding_vector is not None:\n",
        "      # words not found in embedding index will be all zeros.\n",
        "      embedding_matrix[i] = embedding_vector\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "P9XPp7pBRiIP",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# load pre-trained word embeddings into an Embedding layer\n",
        "# note that we set trainable = False so as to keep the embeddings fixed\n",
        "embedding_layer = Embedding(\n",
        "  num_words,\n",
        "  EMBEDDING_DIM,\n",
        "  weights=[embedding_matrix],\n",
        "  input_length=MAX_SEQUENCE_LENGTH,\n",
        "  trainable=False\n",
        ")\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "hV4puJHPRlbJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        },
        "outputId": "89e7ab2a-cea2-4778-8ce7-fb282ce1bef0"
      },
      "cell_type": "code",
      "source": [
        "print('Building model...')\n",
        "\n",
        "# create an LSTM network with a single LSTM\n",
        "input_ = Input(shape=(MAX_SEQUENCE_LENGTH,))\n",
        "x = embedding_layer(input_)\n",
        "# x = LSTM(15, return_sequences=True)(x)\n",
        "x = Bidirectional(LSTM(15, return_sequences=True))(x)\n",
        "x = GlobalMaxPool1D()(x)\n",
        "output = Dense(len(possible_labels), activation=\"sigmoid\")(x)\n",
        "\n",
        "model = Model(input_, output)\n",
        "model.compile(\n",
        "  loss='binary_crossentropy',\n",
        "  optimizer=Adam(lr=0.01),\n",
        "  metrics=['accuracy']\n",
        ")\n",
        "\n",
        "\n",
        "print('Training model...')\n",
        "r = model.fit(\n",
        "  data,\n",
        "  targets,\n",
        "  batch_size=BATCH_SIZE,\n",
        "  epochs=EPOCHS,\n",
        "  validation_split=VALIDATION_SPLIT\n",
        ")\n"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Building model...\n",
            "Training model...\n",
            "Train on 127656 samples, validate on 31915 samples\n",
            "Epoch 1/5\n",
            "127656/127656 [==============================] - 333s 3ms/step - loss: 0.0656 - acc: 0.9780 - val_loss: 0.0561 - val_acc: 0.9802\n",
            "Epoch 2/5\n",
            "127656/127656 [==============================] - 327s 3ms/step - loss: 0.0528 - acc: 0.9811 - val_loss: 0.0537 - val_acc: 0.9810\n",
            "Epoch 3/5\n",
            "127656/127656 [==============================] - 326s 3ms/step - loss: 0.0502 - acc: 0.9817 - val_loss: 0.0526 - val_acc: 0.9811\n",
            "Epoch 4/5\n",
            "127656/127656 [==============================] - 329s 3ms/step - loss: 0.0486 - acc: 0.9822 - val_loss: 0.0532 - val_acc: 0.9809\n",
            "Epoch 5/5\n",
            "127656/127656 [==============================] - 331s 3ms/step - loss: 0.0478 - acc: 0.9824 - val_loss: 0.0526 - val_acc: 0.9812\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "B4xyITuefNKl",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        },
        "outputId": "e1568fd4-8379-41d2-a3ac-5d93d5d11c78"
      },
      "cell_type": "code",
      "source": [
        "# plot some data\n",
        "plt.plot(r.history['loss'], label='loss')\n",
        "plt.plot(r.history['val_loss'], label='val_loss')\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAD8CAYAAABpcuN4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8VNX5+PHPk30lZGNLgLCEIJvI\nplYBcUVrpYoK1lawVb/WulSty/db+6311f7qVrWtfmuttVVrFQpacQOtoGBrKSGGfYtIIGHLRtgS\nsj2/P+4FhjBJBsjkTpLn/XrNKzP3nnvnuRdmnjnn3HOuqCrGGGNMU8K8DsAYY0xos0RhjDGmWZYo\njDHGNMsShTHGmGZZojDGGNMsSxTGGGOaZYnCGGNMsyxRGGOMaZYlCmOMMc2K8DqA1pCWlqZZWVle\nh2GMMe3K8uXLS1U1vaVyHSJRZGVlkZub63UYxhjTrohIYSDlrOnJGGNMsyxRGGOMaZYlCmOMMc3q\nEH0UxpjOqba2lqKiIqqrq70OJaTFxMSQmZlJZGTkSW1vicIY024VFRWRmJhIVlYWIuJ1OCFJVSkr\nK6OoqIh+/fqd1D6s6ckY025VV1eTmppqSaIZIkJqauop1bosURhj2jVLEi071XPUqRPFmu2VPDZ/\nPXY7WGOMaVqnThTLvirnd598yacbS7wOxRjTTiUkJHgdQtB16kTxrTP70jsllsfnb6ChwWoVxhjj\nT6dOFFERYdx7UQ5rd+zlnZXbvQ7HGNOOqSr33Xcfw4YNY/jw4cyaNQuAHTt2MGHCBEaOHMmwYcNY\nsmQJ9fX1zJw580jZp59+2uPomxfQ5bEiMhn4NRAOvKiqjzZaHw28AowGyoBpqrrFXTcC+D3QBWgA\nxqpqtYh8AvQEqtzdXKyqu5vbVzBccXovnv/0S3714UYuHdaTqIhOnTuNabd+9s4a1m7f26r7HNKr\nCz/9xtCAyr755pvk5+ezYsUKSktLGTt2LBMmTOCvf/0rl1xyCT/+8Y+pr6/n4MGD5OfnU1xczOrV\nqwHYs2dPq8bd2lr8VhSRcOA54FJgCHCdiAxpVOx7QIWqDgSeBh5zt40A/gLcqqpDgfOAWp/trlfV\nke5jd3P7CpawMOGByYPZWn6QN5ZtDeZbGWM6sM8++4zrrruO8PBwunfvzsSJE1m2bBljx47lT3/6\nEw8//DCrVq0iMTGR/v37s3nzZu644w7mz59Ply5dvA6/WYHUKMYBBaq6GUBE3gCmAGt9ykwBHnaf\nzwGeFed6rIuBlaq6AkBVywJ4P7/70iBemnReTjrj+qXwm483MXVUJvHRNg7RmPYm0F/+bW3ChAks\nXryY9957j5kzZ3LPPfdwww03sGLFChYsWMDzzz/P7Nmzeemll7wOtUmBtLNkANt8Xhe5y/yWUdU6\noBJIBQYBKiILRCRPRO5vtN2fRCRfRH4iRy/0bWpfQSMiPHjpYEr31/DSZ18F862MMR3U+PHjmTVr\nFvX19ZSUlLB48WLGjRtHYWEh3bt35+abb+amm24iLy+P0tJSGhoamDp1Kj//+c/Jy8vzOvxmBfun\ncwRwLjAWOAh8LCLLVfVjnGanYhFJBOYC38HpmwiIiNwC3ALQp0+fUw50VJ9kLh7Snd8v3sz1Z/Ul\nJT7qlPdpjOk8rrzySj7//HNOP/10RITHH3+cHj168PLLL/PEE08QGRlJQkICr7zyCsXFxdx44400\nNDQA8Mtf/tLj6JsnLbXoiMjZwMOqeon7+r8BVPWXPmUWuGU+d/sldgLpwDTgUlWd4Zb7CVCtqk80\neo+ZwBhVvb2pfTXX9DRmzBhtjRsXbdq1j0ueWcyN5/TjJ5c37oYxxoSadevWcdppp3kdRrvg71y5\nP9zHtLRtIE1Py4BsEeknIlHAdGBeozLzgBnu86uBhe4X+wJguIjEuV/6E4G1IhIhImluoJHA5cDq\nFvYVdNndE5k6KpNXPy+kqOJgW7ylMcaEvBYThdtPcDvOl/46YLaqrhGRR0TkCrfYH4FUESkA7gEe\ndLetAJ7CSTb5QJ6qvgdEAwtEZKW7vBj4Q3P7ait3XzQIBJ7+aFNbvq0xxoSsgPooVPV94P1Gy/7X\n53k1cE0T2/4F5xJZ32UHcMZJ+Cvf5L7aQq+uscw4uy8vfvYVt0zoT06PRK9CMcaYkGCjy/y47byB\nJERF8MSCDV6HYowxnrNE4UdyfBS3njeAf6zbRe6Wcq/DMcYYT1miaMKN52SRnhht05AbYzo9SxRN\niIuK4M4Lslm2pYKF63e3vIExxnRQliiaMX1sb7JS43h8/gbqbRpyY8wpau7eFVu2bGHYsGFtGE3g\nLFE0IzI8jHsvzmHDrn28nV/sdTjGGOMJm/2uBV8f3pPfL3amIf/6iJ5ER4R7HZIxxp8PHoSdq1p3\nnz2Gw6WPNrn6wQcfpHfv3vzgBz8A4OGHHyYiIoJFixZRUVFBbW0tP//5z5kyZcoJvW11dTXf//73\nyc3NJSIigqeeeopJkyaxZs0abrzxRmpqamhoaGDu3Ln06tWLa6+9lqKiIurr6/nJT37CtGnTTumw\nG7MaRQvCwoT7LxlM8Z4qXvu3TUNujDlq2rRpzJ49+8jr2bNnM2PGDN566y3y8vJYtGgR99577wlf\nEPPcc88hIqxatYrXX3+dGTNmUF1dzfPPP89dd91Ffn4+ubm5ZGZmMn/+fHr16sWKFStYvXo1kydP\nbu3DtBpFIMZnp/G1Aak8u6iAa8ZkkhgT6XVIxpjGmvnlHyxnnHEGu3fvZvv27ZSUlJCcnEyPHj24\n++67Wbx4MWFhYRQXF7Nr1y569OgR8H4/++wz7rjjDgAGDx5M37592bhxI2effTa/+MUvKCoq4qqr\nriI7O5vhw4dz77338sADD3D55Zczfvz4Vj9Oq1EEQMS5uVH5gRpeXGLTkBtjjrrmmmuYM2cOs2bN\nYtq0abz22muUlJSwfPly8vPz6d69O9XV1a3yXt/61reYN28esbGxXHbZZSxcuJBBgwaRl5fH8OHD\neeihh3jkkUda5b18WaII0Om9u3LZ8B68uGQzpfsPeR2OMSZETJs2jTfeeIM5c+ZwzTXXUFlZSbdu\n3YiMjGTRokUUFhae8D7Hjx/Pa6+9BsDGjRvZunUrOTk5bN68mf79+3PnnXcyZcoUVq5cyfbt24mL\ni+Pb3/429913X1DubWGJ4gTce3EO1XUNPLuwwOtQjDEhYujQoezbt4+MjAx69uzJ9ddfT25uLsOH\nD+eVV15h8ODBJ7zP2267jYaGBoYPH860adP485//THR0NLNnz2bYsGGMHDmS1atXc8MNN7Bq1SrG\njRvHyJEj+dnPfsZDDz3U6sfY4v0o2oPWuh9FIP77zZXMWV7Ex/ecR5/UuDZ5T2OMf3Y/isAF+34U\nxsddFwwiTISnPrIJA40xnYNd9XSCeiTFcOM5/fj94i+5ZcIAhvTq4nVIxph2ZNWqVXznO985Zll0\ndDRLly71KKKWWaI4Cd+fOIC/Li3kiQXr+dON47wOx5hOTVUREa/DCNjw4cPJz89v0/c81S4Ga3o6\nCUlxkdw2aSCLNpSwdHOZ1+EY02nFxMRQVlZmMzw3Q1UpKysjJibmpPdhNYqTNOPsLP70z694dP56\n3vz+19rVLxpjOorMzEyKioooKSnxOpSQFhMTQ2Zm5klvb4niJMVGhfPDCwfx32+u4sO1u7hkaOCj\nLo0xrSMyMpJ+/fp5HUaHF1DTk4hMFpENIlIgIg/6WR8tIrPc9UtFJMtn3QgR+VxE1ojIKhGJEZE4\nEXlPRNa7yx/1KT9TREpEJN993NQaBxoM14zOpH96PE8ssGnIjTEdV4uJQkTCgeeAS4EhwHUiMqRR\nse8BFao6EHgaeMzdNgL4C3Crqg4FzgNq3W2eVNXBwBnAOSJyqc/+ZqnqSPfx4kkfXZBFhIdx38U5\nFOzez9y8Iq/DMcaYoAikRjEOKFDVzapaA7wBNJ4zdwrwsvt8DnCBOI32FwMrVXUFgKqWqWq9qh5U\n1UXushogDzj5BjQPTR7Wg9N7d+WZjzZSXVvvdTjGGNPqAkkUGcA2n9dF7jK/ZVS1DqgEUoFBgIrI\nAhHJE5H7G+9cRLoC3wA+9lk8VURWisgcEekd8NF4wJkwMIftldW8+vmJz+lijDGhLtiXx0YA5wLX\nu3+vFJELDq90m6ZeB36jqpvdxe8AWao6AviIozWVY4jILSKSKyK5Xl/x8LUBaYzPTuO5TwrYW13b\n8gbGGNOOBJIoigHfX/WZ7jK/Zdwv/ySgDKf2sVhVS1X1IPA+MMpnuxeATar6zOEFbvPU4elZXwRG\n+wtKVV9Q1TGqOiY9PT2AwwiuByYPZs/BWl74dHPLhY0xph0JJFEsA7JFpJ+IRAHTgXmNyswDZrjP\nrwYWqjMCZgEw3L3KKQKYCKwFEJGf4ySUH/ruSER6+ry8Alh3YofkjWEZSXzj9F788bOv2L23deae\nN8aYUNBionD7HG7H+dJfB8xW1TUi8oiIXOEW+yOQKiIFwD3Ag+62FcBTOMkmH8hT1fdEJBP4Mc5V\nVHmNLoO9071kdgVwJzCzlY416O69aBC19Q38ZuEmr0MxxphWY9OMt7Kf/H01r/9nK/+4ZyJZafFe\nh2OMMU2yacY9cscFA4kMD+PJD20acmNMx2CJopV1S4zhe+f2492VO1hdXOl1OMYYc8osUQTBLRP7\nkxwXyWPz13sdijHGnDJLFEHQJSaSH0wayJJNpfyroNTrcIwx5pRYogiSb5/Vl15JMTw2f73NlW+M\nadcsUQRJTGQ4P7xoECuKKvlg9U6vwzHGmJNmiSKIpo7KJLtbAk8u2EBdfYPX4RhjzEmxRBFE4WHC\nfZfksLn0AH9bbtOQG2PaJ0sUQXbRkO6M7pvMM//YSFWNTUNujGl/LFEEmTMN+WB27T3En/+1xetw\njDHmhFmiaAPj+qUwKSed331SQOVBm4bcGNO+WKJoI/dPHsy+Q3X836cFXodijDEnxBJFGzmtZxe+\nOTKDP/9zCzsrbRpyY0z7YYmiDd1z0SAaVPn1xxu9DsUYYwJmiaIN9U6J4/oz+zI7t4gvS/Z7HY4x\nxgTEEkUbu/38gcREhPHkApuG3BjTPliiaGNpCdHcNL4/H6zeSf62PV6HY4wxLbJE4YGbJ/QnNT6K\nxz6wCQONMaHPEoUHEqIjuP38gXy+uYwlm2wacmNMaAsoUYjIZBHZICIFIvKgn/XRIjLLXb9URLJ8\n1o0Qkc9FZI2IrBKRGHf5aPd1gYj8RkTEXZ4iIh+JyCb3b3LrHGpo+daZfchMjuWx+etpaLBahTEm\ndLWYKEQkHHgOuBQYAlwnIkMaFfseUKGqA4GngcfcbSOAvwC3qupQ4Dzg8NDk3wE3A9nuY7K7/EHg\nY1XNBj52X3c40RHh3HvxINZs38u7q3Z4HY4xxjQpkBrFOKBAVTerag3wBjClUZkpwMvu8znABW4N\n4WJgpaquAFDVMlWtF5GeQBdV/bc6jfSvAN/0s6+XfZZ3OFecnsHgHon86sMN1NTZNOTGmNAUSKLI\nALb5vC5yl/kto6p1QCWQCgwCVEQWiEieiNzvU9533m3ffXZX1cM/sXcC3f0FJSK3iEiuiOSWlJQE\ncBihJzxMuH9yDoVlB5mVu63lDYwxxgPB7syOAM4Frnf/XikiFwS6sVvb8NuAr6ovqOoYVR2Tnp7e\nKsF6YVJON8ZlpfCbjzdxsKbO63CMMeY4gSSKYqC3z+tMd5nfMm6/RBJQhlNTWKyqpap6EHgfGOWW\nz2xin7vcpincv7tP5IDaGxHhgUsHU7LvEC999pXX4RhjzHECSRTLgGwR6SciUcB0YF6jMvOAGe7z\nq4GFbm1gATBcROLcBDIRWOs2Le0VkbPcvowbgLf97GuGz/IOa3TfZC4a0p3ff7qZigM1XodjjDHH\naDFRuH0Ot+N86a8DZqvqGhF5RESucIv9EUgVkQLgHtwrlVS1AngKJ9nkA3mq+p67zW3Ai0AB8CXw\ngbv8UeAiEdkEXOi+7vDuuySHAzV1PLfIpiE3xoQW6Qgjg8eMGaO5ubleh3HKfvS3FcxbsZ1FPzqP\njK6xXodjjOngRGS5qo5pqZyNzA4hd180CIBnPrJpyI0xocMSRQjJ6BrLDWf1ZW5eEZt27fM6HGOM\nASxRhJwfTBpIfFQEj9s05MaYEGGJIsQkx0dxy4T+fLR2F8sLy70OxxhjLFGEou+N70daQjSPfbDB\npiE3xnjOEkUIiouK4K4LBvKfLeV8sqF9Tk9ijOk4LFGEqOnj+tA3Nc6mITfGeM4SRYiKDA/j3otz\nWL9zH2+vaDxjijHGtB1LFCHs8uE9GdqrC7/6cCOH6uq9DscY00lZoghhYWHC/ZMHU1RRxetLt3od\njjGmk7JEEeImZKdxdv9UfruwgP2HbBpyY0zbs0QR4g5PQ152oIYXl2z2OhxjTCdkiaIdGNm7K5cO\n68EfFm+mdP8hr8MxxnQylijaiR9dkkN1XQPPLrRpyI0xbcsSRTsxID2Ba0Zn8trSQraVH/Q6HGNM\nJ2KJoh354YWDCBPhaZuG3BjThixRtCM9kmKYeU4Wb+UXs27HXq/DMcZ0EpYo2pnbJg4kMTqCJ2wa\ncmNMGwkoUYjIZBHZICIFIvKgn/XRIjLLXb9URLLc5VkiUiUi+e7jeXd5os+yfBEpFZFn3HUzRaTE\nZ91NrXe47V9SXCTfP28gC9fv5j9f2TTkxpjgazFRiEg48BxwKTAEuE5EhjQq9j2gQlUHAk8Dj/ms\n+1JVR7qPWwFUdZ/PspFAIfCmzzazfNa/ePKH1zHN/FoW3btE89j89TYNuTEm6AKpUYwDClR1s6rW\nAG8AUxqVmQK87D6fA1wgIhJIACIyCOgGLAksZBMbFc5dFwxieWEF/1i32+twjDEdXCCJIgPY5vO6\nyF3mt4yq1gGVQKq7rp+IfCEin4rIeD/7n45Tg/D9aTxVRFaKyBwR6R3IgXQ2147JpH9aPE8sWE+9\nTUNujAmiYHdm7wD6qOoZwD3AX0WkS6My04HXfV6/A2Sp6gjgI47WVI4hIreISK6I5JaUdL6b+0SE\nh/GjS3LYuGs/b+YVeR2OMaYDCyRRFAO+v+oz3WV+y4hIBJAElKnqIVUtA1DV5cCXwKDDG4nI6UCE\nuw63XJmqHp6n4kVgtL+gVPUFVR2jqmPS09MDOIyO59JhPRiRmcTTH22kutamITfGBEcgiWIZkC0i\n/UQkCqcGMK9RmXnADPf51cBCVVURSXc7wxGR/kA24Duz3XUcW5tARHr6vLwCWBfowXQ2IsIDkwez\nvbKav/y70OtwjDEdVERLBVS1TkRuBxYA4cBLqrpGRB4BclV1HvBH4FURKQDKcZIJwATgERGpBRqA\nW1XV95rOa4HLGr3lnSJyBVDn7mvmSR9dJ3DOwDTGZ6fx3KICrh3bmy4xkV6HZIzpYKQjXF45ZswY\nzc3N9ToMz6wuruTy337GHecP5N6Lc7wOxxjTTojIclUd01I5G5ndAQzLSOLyET15cclX7N5X7XU4\nxpgOxhJFB/Gji3OorW/gtx/bNOTGmNZliaKDyEqLZ9rY3rz+n60Ulh3wOhxjTAdiiaIDueuCbCLD\nw/jVhzYNuTGm9Vii6EC6dYnhu+dmMW/FdlYXV3odjjGmg7BE0cH818QBdI2L5HGbhtwY00osUXQw\nXWIi+cF5A1m8sYR/fVnqdTjGmA7AEkUH9J2z+9IzKYbH5m+waciNMafMEkUHFBMZzt0XDmLFtj0s\nWLPT63CMMe1c504UtdVQV+N1FEFx1agMBnZL4PEFG6irb/A6HGNMO9a5E0X+X+DJbHj7dvhyIdTX\neR1Rq4kID+O+S3LYXHKAOcttGnJjzMnr3Imi+3AYdAms+Tu8eiX8KgfevRu+WgIN7X/a7ouHdGdU\nn648849NNg25Meakde5E0edMuOoFuG8TTPsL9JsAK96Aly+Hp4bABw/A1qXQ0D6bbg5PQ75zbzUv\n/2uL1+EYY9opmz22sZoDsHE+rH4TNn0E9YegSyYM/SYMuwp6jYLAbgceMmb+6T98sXUPi++fRFKs\nTUNujHEEOnusJYrmVO+FDR/Amjeh4GNoqIXkLBh6JQy9CnoMbxdJY+32vXz9t0u4deIAHpg82Otw\njDEhwhJFa6uqgPXvOTWNzZ+A1kNqtlPLGHoVdAvtL+AfvvEF89fs5NP7JtG9S4zX4RhjQoAlimA6\nUArr5jlJY8tngEK3IU7CGHYVpA5ou1gCtK38IOf/6hOuHt2bX1413OtwjDEhwBJFW9m3C9a+7TRP\nbf3cWdZjxNGaRnJfb+Ly4+F5a3j134V8dPcE+qcneB2OMcZjlii8UFkMa//u1DSK3XgyRjsJY+iV\nkJThaXgl+w4x8YlFTMrpxnPXj/I0FmOM91r1VqgiMllENohIgYg86Gd9tIjMctcvFZEsd3mWiFSJ\nSL77eN5nm0/cfR5e1625fbULSRlw9g/g5o/hrhVw4cNQXwsf/hieHgIvTYalLzi1EA+kJ0Zz0/j+\nvLdqByuL9ngSgzGm/WkxUYhIOPAccCkwBLhORIY0KvY9oEJVBwJPA4/5rPtSVUe6j1sbbXe9z7rd\nAeyr/UjOgnPvhluXwO3LYdJDUF0JH9wHTw2GP18OuS/BgbI2Devm8f1IiY/isfnr2/R9jTHtVyA1\ninFAgapuVtUa4A1gSqMyU4CX3edzgAtETvq60dbcV2hIGwgT74PbPofblsKE+2DfTmcU+JPZzqjw\nvFedK6uCLDEmktsnDeSfBWUs2VQS9PczxrR/gSSKDGCbz+sid5nfMqpaB1QCqe66fiLyhYh8KiLj\nG233J7fZ6Sc+yaC5fbV/3QbDpP+B25fBfy2Bc+6Esi9h3u3wRDb8dRqsmOWM4QiS68/qQ0bXWB6f\nv4GGhvbfR2WMCa6IIO9/B9BHVctEZDTwdxEZqqp7cZqdikUkEZgLfAd4JdAdi8gtwC0Affr0CULo\nQSYCPUc4jwt+CtvznE7wNX93RoaHR0P2Rc7VU4MmQ1R8q711dEQ491w0iHv/toL3V+/g8hG9Wm3f\nxpiOJ5AaRTHQ2+d1prvMbxkRiQCSgDJVPaSqZQCquhz4Ehjkvi52/+4D/orTxNXkvhoHpaovqOoY\nVR2Tnp4ewGGEMBHn6qhLfgE/XAXf/RDG3AhFuTDnu/DEQPjbTFg7D2qrWuUtv3lGBjndE3lywQZq\nbRpyY0wzAkkUy4BsEeknIlHAdGBeozLzgBnu86uBhaqqIpLudoYjIv2BbGCziESISJq7PBK4HFjd\n3L5O7vDaobAwZ7LCSx+De9bCjHfh9OnOjLazv+Mkjbk3w4b5p3QvjfAw4f7JOWwpO8isZdta3sAY\n02kFNI5CRC4DngHCgZdU9Rci8giQq6rzRCQGeBU4AygHpqvqZhGZCjwC1AINwE9V9R0RiQcWA5Hu\nPv8B3KOq9U3tq7n4QmYcRTDV18GWxU7z1Lp3oHoPxCTB4G/AsCuh30QIP7EJ/1SVa3//OVvKDvLp\nfecRFxXslkhjTCixAXcdWV2NM9/Umjed+acO7YXYFBhyhTO4L+tcCAsPaFe5W8q5+vnPue+SHH4w\naWBw4zbGhJRAE4X9hGyPIqJg0MXOo7YaCv7hJI2Vf4Plf4b4bjBkCgybCr3PdJqzmjAmK4ULT+vG\n8598ybfG9SE5PqrtjsMY0y5YjaIjqTkImxa499L4EOqqIbGXM33IsKucDnM/Q1I27NzH5F8v5ubx\n/fmfy07zIHBjjBes6amzO7TP6fBe86ZT46ivga59jt5Lo+fpxySNe2ev4J2V2/nkR+fRq2ush4Eb\nY9qKJQpzVNUepy9jjXsvjYY6SOl/dFr0bkMo2lPF+U9+yjfP6MXjV5/udcTGmDZgicL4d7Dc514a\nS0AbIC0Hhl3F/5WM4Mk8ZcEPJ5DdPdHrSI0xQWaJwrRs/273XhpvQeG/AGW99mVd6oVc+e07IKWf\n1xEaY4LIEoU5MXu3w9q32fHP1+i5b5WzrNcZR++l0bV389sbY9odSxTmpBw4VMe0x2czPS6X6xOW\nIzvynRWZ45z+jNOu8PwGTMaY1mHjKMxJiY+O4NoLv8ZDbyeRcfmDTErb5zRNrXkL5j/oPOLSIH2w\nMxNu+mBIz4H00yA+ze/lt8aY9s1qFOY4NXUNXPjUp8RHR/DeHecSFuZ++ZdscC613b0OStY7rw/5\nTIcem+I/gSR0swRiTAiyGoU5aVERYdx78SDueiOfd1ZuZ8pIt6kpPcd5HKYK+3a4iWODmzzWw6q5\ncKjyaLmYrtDtNHf7wUcfiT0sgRjTDliNwvjV0KBc/tvP2H+ojn/cM5GoiIBur+5Qhf27jk8gu9c5\nkxkeFpPkU/PwSSBdelkCMaYNWI3CnJIwdxrymX9axuv/2cqMr2UFvrGIU1tI7AEDJh1drgoHSo5P\nIOvehTyfe1ZFdzlae0kf7DRfpedAUqYlEGM8YInCNGnioHTO6p/Cbxdu4urRmcRHn+J/FxGnvyKh\nG/SfeOy6A6U+fR9u/8fGBfDFX46WiUrwSR45Pgmkd7MTH5p2QtWZnywixn4QhBhrejLN+mJrBVf+\n37+4+8JB3HVhdtsHcKDMJ3n4JJH9u46WiYyH9EHHNl+l50DXvpZAvFZf6/wIOFDiPlp4XlcF4VHO\nlXXxaRCf7j7SmvibDpE2N9nJsqYn0yrO6JPM5KE9+MOSzXz7rD6kJkS3bQDxqRB/DmSdc+zyg+XH\nNl+VrHfmsVrx+tEyEbH+E0hyVsD36zCNqEJVRaMv+WYSgG+flK/wqGO/8NNznOcxXZ0JLX33UbYJ\n9rtJxJ+oBGfbOD9JJD7d/T/kPo9LPeEbfBmrUZgAFOzex8VPL2bm1/rxv98Y4nU4zauqgJKNx9dA\n9vrc5j0iBtKyj00g3U7rvAmktqrlL/z97t+Dpc6kkv7EphxfA0jo5ueLO83phzrR5qWaA43iavzX\nfX7Qfd1knMnHJo7jai0+r2O6duhaqY3MNq3qgTkreeuLYhb+aCKZyXFeh3Piqiv9J5BKn/uFh0c3\nSiA5bgLpB+HtqPJdXwdV5U18+fv5Yq3Z738/kfFNNP/4eR2XGlrnSNWpzRx37GX+z0NVBeDnu1DC\n/Td3HZNgfNZFxber/hVLFKaOV17NAAATt0lEQVRV7ais4rwnPuHrI3ry1LUjvQ6n9Rza55NAfK7G\n2rP1aJnwKEgdeHwCSenfNs0Yqm5zTFO/+Hcfu/xgOSf0pRef5twV8ZjXac6XXmfRbHIt9ZNc9/nf\nT0RME+c33U/TWBpEtHFTbiOt2kchIpOBXwPhwIuq+mij9dHAK8BooAyYpqpbRCQLWAdscIv+W1Vv\nFZE44G/AAKAeeEdVH3T3NRN4AjjcVvCsqr4YSJwmeHomxTLza1n8fvFmisqrmDo6g8uG9yQxpp23\n90YnQuZo5+Hr0H4o3Xhs7aN4uXNPj8PCIppIIAOc29U2p+5QE00nTSSD+kP+9xOTdPQLKS0b+n6t\n6RpAB29GOSXhEUevyAtEbfXRJi6//36lzuzMu9Y2/+8XnXQ0MftLMHG+tZgUz5pGW6xRiEg4sBG4\nCCgClgHXqepanzK3ASPcJDAduFJVp7mJ4l1VHdZon3HAmaq6SESigI+B/6eqH7iJYoyq3h7oQViN\nom1U19bz0j+/Ys7yIjaXHCAmMoxLhvbgqlGZnDswjfCw9lPlPmk1B6B007EJZPc6qNjCkV/xYRFO\nsjjccV6z3/nS8E0MviPXfYVHN/2LtHF7f1yq579ITQB8a4QHy5r4UeBTazlY6twn5jji0+Tlk1yG\nTYU+Z51UaK1ZoxgHFKjqZnfHbwBTgLU+ZaYAD7vP5wDPijTdUKeqB4FF7vMaEckDMgOIxXgoJjKc\n284byPcnDmBFUSVzlxcxb8V23s7fTvcu0XzzjAyuHpXZsW96FBUPvUY6D1+1VccnkF1rYMMHPr/6\n05xb0Pr9xe8+j05sV23cJgAiENPFeaQOaLl8Q4N7ZVlJ87WWnaud5z1PP+lEEfAhBFCjuBqYrKo3\nua+/g1MbuN2nzGq3TJH7+kvgTCABWINTI9kLPKSqSxrtvyuQB1yoqpvdGsUvgRJ3u7tVdRuNiMgt\nwC0Affr0GV1YWHjiR29O2aG6ehau283cvCI+2VBCXYMyIjOJqaMy+cbpvUiJb6EJxhhzalRP+sdF\nq3Vmn2Ki2AckqGqZiIwG/g4MVdW9brkI4B1ggao+4y5LBfar6iER+S+c/o7zm4vRmp5CQ+n+Q8zL\n387cvCLWbN9LZLgwKacbU0dnMimn24nNF2WMCbrWbHoqBnxvb5bJ0Y7mxmWK3C//JKBMnSx0CEBV\nl7sJZBBw+Fv9BWDT4SThlivz2e+LwOMBxGhCQFpCNN89tx/fPbcf63fuZe7yIv6ev50P1+4iOS6S\nK07vxdTRmQzPSKKZlkljTIgJJFEsA7JFpB9OQpgOfKtRmXnADOBz4GpgoaqqiKQD5apaLyL9gWzg\ncF/Hz3ESyk2+OxKRnqq6w315Bc5VU6adGdyjCz/++hAemDyYJQWlzF1exOvLtvHy54Vkd0tg6uhM\nrjwjg+5dYrwO1RjTgoDGUYjIZcAzOJfHvqSqvxCRR4BcVZ0nIjHAq8AZQDkw3e1vmAo8AtQCDcBP\nVfUdEckEtgHrcWscuJfBisgvcRJEnbuv76vq+ubis6an9qGyqpb3Vu5gbl4RywsrCBM4NzudqaMy\nuHhID2KjOuGoaGM8ZAPuTEjbUnqAN/OKmJtXTPGeKhKjI7hseE+mjs5kbFayNU0Z0wYsUZh2oaFB\nWfpVOXPzivhg1Q4O1NTTOyWWq87IZOqoTPqktsPpQoxpJyxRmHbnYE0d81fv5M28Yv75ZSmqMC4r\npeOMAjcmxFiiMO3a9j1VvPVFMXPzOvEocGOCzBKF6RBU9ZhR4JVVtZ1nFLgxQWaJwnQ4TY0Cv+qM\nDK4YmWGjwI05QZYoTIdmo8CNOXWWKEyncXgU+FtfbKd0/yEbBW5MgCxRmE6nrr6BJZtKmZtXxIdr\nd1FT12CjwI1phiUK06nZKHBjWmaJwhhX41HgCdERfN1GgRtjicKYxmwUuDHHskRhTDNsFLgxliiM\nCZiNAjedlSUKY06QqpK/bQ9z84p4Z8UOGwVuOjxLFMacAt9R4Is2lFDfoAzPSGLqKBsFbjoOSxTG\ntJLS/Yd4O387b9oocNPBWKIwJghsFLjpSCxRGBNENgrcdASWKIxpIzYK3LRXrZooRGQy8GsgHHhR\nVR9ttD4aeAUYDZQB01R1i4hkAeuADW7Rf6vqre42o4E/A7HA+8BdqqoikgLMArKALcC1qlrRXHyW\nKEyo+ModBf6mOwo8Piqc0VkpjO2bzJisFEb27mqJw4SMVksUIhIObAQuAoqAZcB1qrrWp8xtwAhV\nvVVEpgNXquo0N1G8q6rD/Oz3P8CdwFKcRPEbVf1ARB4HylX1URF5EEhW1Qeai9EShQk1h0eBv7dq\nO7lbKtiwax+qEBEmDMtIYmyWkzjG9E0mNSHa63BNJxVooogIYF/jgAJV3ezu+A1gCrDWp8wU4GH3\n+RzgWWmmV09EegJdVPXf7utXgG8CH7j7Os8t+jLwCdBsojAm1ISFCWcPSOXsAakAVB6sZfnWcpZt\nqSB3Szkv/6uQPyz5CoAB6fGMzUphTFYKY7OS6ZMSZ53iJqQEkigygG0+r4uAM5sqo6p1IlIJpLrr\n+onIF8Be4CFVXeKWL2q0zwz3eXdV3eE+3wl09xeUiNwC3ALQp0+fAA7DGO8kxUVy/uDunD/Y+e9c\nXVvP6uLKI4njg9U7eWOZ8zFLT4x2ahx9UxiblcJpPROJCLdLcI13AkkUp2IH0EdVy9w+ib+LyNBA\nN3b7LPy2janqC8AL4DQ9tUq0xrSRmMhwp+kpKwUYQEODUlCyn2VbysndUsGyLeW8v2onAHFR4Yzq\nk8yYrGTGuv0c8dHB/ugac1Qg/9uKgd4+rzPdZf7KFIlIBJAElKnTAXIIQFWXi8iXwCC3fGYT+9wl\nIj1VdYfbRLX7BI/JmHYnLEwY1D2RQd0Tuf7MvgDsqKwi161xLNtSwa8/3oQqhIcJQ3t1cWscyYzO\nSqZbol2Oa4InkESxDMgWkX44X+bTgW81KjMPmAF8DlwNLHRrA+k4HdP1ItIfyAY2q2q5iOwVkbNw\nOrNvAH7baF+Pun/fPqUjNKad6pkUyzdOj+Ubp/cCYG91LV9s3eMmjnL++p9CXvqn08+RlRp3pI9j\nTFYK/dPirZ/DtJpAL4+9DHgG5/LYl1T1FyLyCJCrqvNEJAZ4FTgDKAemq+pmEZkKPALUAg3AT1X1\nHXefYzh6eewHwB1uckkFZgN9gEKcy2PLm4vPrnoynVFNXQNrtlceaarKLayg/EANAKnxUYzum+x2\nkicztFeSTTVijmMD7ozpZFSVzaUHjjRV5W4pZ0vZQQBiIsMY2bvrkaurRvXpavfcMJYojDGwe181\ny7dUOImjsJw12/dS36CECQzu0YVx/Zwax5i+KfRIsn6OzsYShTHmOPsP1ZG/dY/bVFVOXuEeqmrr\nAeidEsvYvkfHcwxITyDMbtrUobXmgDtjTAeREB3BudlpnJudBkBtfQPrduw90lS1eFMpb37hXIDY\nNS6SMe7UI2OzkhmWkUR0hE0/0hlZjcIYc4SqUlh28Oh4jsJyNpccACAqIoyRmV2PjOcY1TeZpFjr\n52jPrOnJGNMqSvcfYnnh0fEcq4srqWtQRCCne+KRxDEmK4WMrrFeh2tOgCUKY0xQVNXUk7/NHc9R\nWEFeYQX7D9UB0Csp5pjxHDndE62fI4RZH4UxJihio8KPmfCwvkFZv3PvkfEcS78qY96K7QAkxkT4\n9HOkMCIziZhI6+dob6xGYYxpVapKUUUVy3zGc2zavR+AqPAwhmcmOc1VfVMY3TeZ5PgojyPuvKzp\nyRgTMioO1LC80Okcz91SwcqiPdTWO9892d0SjjRXje6bTGZyHOHWXNUmLFEYY0JWdW09K4sq3aur\nnOlH9lU7/RyR4UKPpBgyu8aRkRxLRtdYMpJjyXT/9kyKtelIWon1URhjQlZMZDjj+qUwrl8K4NwR\ncOPufXyxdQ9byw9SXFFF8Z4qPttUyq591fj+nhWB7okxxySRxskkLsq+2lqTnU1jjOfCwoTBPbow\nuEeX49bV1DWwo7KK4ooqivZUHUkixRVV5G/bw/urdlDXcGzLSEp8lJM8GieS5Fgyu8bRJTbCZtc9\nAZYojDEhLSoijL6p8fRNjfe7vr5B2b2v+kgCKfJJJJt27+OTjbuprm04ZpuE6Ihjkkhm8rEJJT0h\n2hKJD0sUxph2LTxM6Jnk9F34a2xXVcoP1BxJHoeTyeGEkrulnL1u/8hhURFhR2skfhJJjy4xner2\ntJYojDEdmoiQmhBNakI0IzK7+i2zr7r2uERyuKnr4/W7Kd1/6Jjy4WFCjy4xfpNIRtdYenWN7VDj\nRSxRGGM6vcSYSAb3iPTbRwLOVVrbGzVrHf679KtyduRX0aibhPTE6OM62X2TSXu6H4glCmOMaUFM\nZDj90xPon57gd31tfQM7K6uPSyLFe6pYu30vH63dRU3dsf0kSbGRx/eTdI0lM9m5LDg5LjJk+kks\nURhjzCmKDA+jd0ocvVPi/K5vaFBK9x867qqt4j1VFJYd4F8FpRyoqT9mm9jI8ONqIZnJhxNKHN0S\no9tsHq2AEoWITAZ+jXPP7BdV9dFG66OBV4DRQBkwTVW3+KzvA6wFHlbVJ0UkB5jls4v+wP+q6jMi\n8jBwM1DirvsfVX3/JI7NGGNCQliY0K1LDN26xDCqT/Jx61WVyqra45q2iioOUrynipVFe6g4WHvM\nNpHhTif+vRcPYsrIjKDG32KiEJFw4DngIqAIWCYi81R1rU+x7wEVqjpQRKYDjwHTfNY/BXxw+IWq\nbgBG+uy/GHjLp/zTqvrkyR2SMca0LyJC17gousZFMSwjyW+ZA4fqnH4SN5EcTiqp8dFBjy+QGsU4\noEBVNwOIyBvAFJwawmFTgIfd53OAZ0VEVFVF5JvAV8CBJvZ/AfClqhaeRPzGGNMpxEdHkN09kezu\niW3+3oFcCJwBbPN5XeQu81tGVeuASiBVRBKAB4CfNbP/6cDrjZbdLiIrReQlETm+nmaMMabNBHvE\nyMM4zUj7/a0UkSjgCuBvPot/BwzAaZraAfyqiW1vEZFcEcktKSnxV8QYY0wrCKTpqRjo7fM6013m\nr0yRiEQASTid2mcCV4vI40BXoEFEqlX1WXe7S4E8Vd11eEe+z0XkD8C7/oJS1ReAF8CZPTaA4zDG\nGHMSAkkUy4BsEemHkxCmA99qVGYeMAP4HLgaWKjO/OXjDxdwr2ba75MkAK6jUbOTiPRU1R3uyyuB\n1QEfjTHGmFbXYqJQ1ToRuR1YgHN57EuqukZEHgFyVXUe8EfgVREpAMpxkkmzRCQe50qq/2q06nER\nGQkosMXPemOMMW3IblxkjDGdVKA3Luo80x8aY4w5KZYojDHGNKtDND2JSAlwsgP20oDSVgyntVhc\nJ8biOnGhGpvFdWJOJa6+qpreUqEOkShOhYjkBtJG19YsrhNjcZ24UI3N4joxbRGXNT0ZY4xpliUK\nY4wxzbJE4Y7uDkEW14mxuE5cqMZmcZ2YoMfV6fsojDHGNM9qFMYYY5rVaRKFiEwWkQ0iUiAiD/pZ\nHy0is9z1S0UkK0TimikiJSKS7z5uaqO4XhKR3SLid64tcfzGjXuliIwKkbjOE5FKn/P1v20QU28R\nWSQia0VkjYjc5adMm5+vAOPy4nzFiMh/RGSFG9dxtyHw4vMYYFyefB7d9w4XkS9E5LiJUoN+vlS1\nwz9w5qj6EueWq1HACmBIozK3Ac+7z6cDs0IkrpnAsx6cswnAKGB1E+svw7lroQBnAUtDJK7zgHfb\n+Fz1BEa5zxOBjX7+Hdv8fAUYlxfnS4AE93kksBQ4q1EZLz6PgcTlyefRfe97gL/6+/cK9vnqLDWK\nI3fpU9Ua4PBd+nxNAV52n88BLhCRYN+5PJC4PKGqi3EmeGzKFOAVdfwb6CoiPUMgrjanqjtUNc99\nvg9Yx/E392rz8xVgXG3OPQeH71ET6T4ad5a2+ecxwLg8ISKZwNeBF5soEtTz1VkSxUnfpS8E4gKY\n6jZXzBGR3n7WeyHQ2L1wttt88IGIDG3LN3ar/Gfg/Br15en5aiYu8OB8uc0o+cBu4CNVbfJ8teHn\nMZC4wJvP4zPA/UBDE+uDer46S6Joz94BslR1BPARR381GP/ycKYlOB34LfD3tnpjcW79Oxf4oaru\nbav3bUkLcXlyvlS1XlVH4twIbZyIDGuL921JAHG1+edRRC4Hdqvq8mC/V1M6S6I4kbv0Icfepc/T\nuFS1TFUPuS9fBEYHOaZABXJO25yq7j3cfKCq7wORIpIW7PcVkUicL+PXVPVNP0U8OV8txeXV+fJ5\n/z3AImByo1VefB5bjMujz+M5wBUisgWnefp8EflLozJBPV+dJVEcuUufOPfpno5zVz5fh+/SB8fe\npc/TuBq1Y1+B084cCuYBN7hX85wFVOrROxN6RkR6HG6bFZFxOP/Hg/oF477fH4F1qvpUE8Xa/HwF\nEpdH5ytdRLq6z2NxbmC2vlGxNv88BhKXF59HVf1vVc1U1Syc74iFqvrtRsWCer4CuRVqu6dBuktf\nG8V1p4hcAdS5cc0MdlwAIvI6zhUxaSJSBPwUp3MPVX0eeB/nSp4C4CBwY4jEdTXwfRGpA6qA6W2Q\n8M8BvgOsctu3Af4H6OMTlxfnK5C4vDhfPYGXRSQcJzHNVtV3vf48BhiXJ59Hf9ryfNnIbGOMMc3q\nLE1PxhhjTpIlCmOMMc2yRGGMMaZZliiMMcY0yxKFMcaYZlmiMMYY0yxLFMYYY5plicIYY0yz/j86\nSeNzMYNpsAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "GOFaAPgUfPRC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        },
        "outputId": "eecb2b96-458a-4e12-9af4-b3faefc42d8b"
      },
      "cell_type": "code",
      "source": [
        "# accuracies\n",
        "plt.plot(r.history['acc'], label='acc')\n",
        "plt.plot(r.history['val_acc'], label='val_acc')\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD8CAYAAAB3u9PLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8VPW9//HXJ3tCQsgGBBIIS9iD\noEEBC0EUBVrFrWoXW3tbrFWr1trWLr9ra9urtdzWWr1Yar2V2tZ6bW21JSIKglZAg4JBCGEnCdlX\nQvaZ7++PcwiTkJAJJHNmks/z8ZgHM3POzHxmwnzf53zPd75HjDEopZRSQU4XoJRSyj9oICillAI0\nEJRSStk0EJRSSgEaCEoppWwaCEoppQANBKWUUjYNBKWUUoAGglJKKVuI0wX0RmJioklLS3O6DKWU\nCig7duyoMMYk9bReQAVCWloaOTk5TpehlFIBRUSOerOedhkppZQCNBCUUkrZNBCUUkoBAXYMoSut\nra0UFhbS1NTkdCl+KSIigpSUFEJDQ50uRSnl5wI+EAoLC4mJiSEtLQ0Rcbocv2KMobKyksLCQsaN\nG+d0OUopPxfwXUZNTU0kJCRoGHRBREhISNC9J6WUVwI+EAANg7PQz0Yp5a2A7zJSSqmBpqXNzfGa\nRgqqGzhW1UBBVSN3XjaBoRH9eyxQA0EppXzM7TaUnmiioKqRgqoGCqqtRr+guoHCqgZK6ppwe5zu\nPjRYuG72aIaO1EBQSqmAYoyhpqG1Q0NvNfyNFFY1UFjdSIvL3b6+CIyIiSA1PpK54xNIiY8iNS6S\n1PgoUuOjGDk0guCg/u/+1UDoA9deey0FBQU0NTVx7733cvvtt/Paa6/xve99D5fLRWJiIm+++Sb1\n9fV8/etfJycnBxHhoYce4oYbbnC6fKXUOWhoaaOw2t7Ctxv7giqri6ewupH65rYO6w+LCiU1Loop\nyTEsmTaivdEfEx/F6LhIwkOCHXonpw2oQPjRqx+z53hdnz7ntFFDeejq6Wdd59lnnyU+Pp7Gxkbm\nzJnDihUrWLlyJVu2bGHcuHFUVVUB8OMf/5jY2Fhyc3MBqK6u7tNalVJ9p9XlprimyerDr+7Y6BdW\nN1BR39Jh/YjQIFLjrC36ueMTSDm1hR8XRWp8JDH93P/fFwZUIDjliSee4OWXXwagoKCANWvWsHDh\nwvax//Hx8QC88cYbvPDCC+2Pi4uL832xSinA6scvr2/u2Ifvcb24trFDP35wkDB6WCSp8ZFcMXUE\nqfFRHRr9xOiwgB/VN6ACoact+f7w1ltv8cYbb7B161aioqJYtGgRs2bNIi8vz+e1KKU6qm3vx284\noz+/sLqR5jZ3h/WHx4STGh/FnLQ4UuNHkxoXRUp8JKlxUSTHRhASPCBG6ndrQAWCE2pra4mLiyMq\nKoq8vDy2bdtGU1MTW7Zs4fDhw+1dRvHx8SxZsoSnnnqKxx9/HLC6jHQvQalz19TqorBTQ39qmGZB\ndQMnmjr24w+NCCE1Por04TEsnjKcMfFRdl++tbUfEep8P76TNBDO09KlS3n66aeZOnUqkydPZu7c\nuSQlJbFmzRquv/563G43w4cPZ8OGDfzgBz/grrvuYsaMGQQHB/PQQw9x/fXXO/0WlPJbbS43xbVN\np7fyO43YKT/R3GH98JAgUuwDtZlpce399yl2335spP/34ztJA+E8hYeHk52d3eWyZcuWdbgdHR3N\nc88954uylAoIxpzqx2+0t/Q9Gv3qBo7XNOHy6MgPDhKSYyNIjYvisslJ7QdxU+1unaSY8IDvx3eS\nBoJSyifcbsOe4jo255fzwdHq9uGZja2uDuslRoeTGh/J7NQ4rrkg8nSjHxdF8rAIQgd4P76TNBCU\nUv2msr6Zdw5UsDm/nC35FVTUW108k0ZEMy5xCAsnJTHGYws/JS6KyLDB3Y/vJA0EpVSfaXO52VlQ\nw5b8cjbnl/NRUS3GQFxUKAsnJbEwPYkFkxIZHhPhdKmqCxoISqnzUlzb2B4Ab++v4ERTG0ECF46J\n4xtXTCJrUhIzRsf6ZOoFdX68CgQRWQr8CggGnjHGPNpp+VjgWSAJqAI+b4wptJc9BnwSa6rtDcC9\nQCTwf8AEwAW8aox5sC/ekFKqfzW1usg5Us3m/DI255eTX1oPQHJsBMtnJJM1OYlLJyQSG6UjegJN\nj4EgIsHAU8ASoBB4X0ReMcbs8VhtFbDWGPOciCwGHgFuFZH5wKXATHu9d4As4D1glTFmk4iEAW+K\nyDJjTNfDdZRSjjHGcKSygc37rADYdqiKxlYXYcFBXDwunk9flErW5CTSh0frCJ8A580ewsXAAWPM\nIQAReQFYAXgGwjTgfvv6JuDv9nUDRABhgAChQKkxpsFeD2NMi4h8AKSc31tRSvWV+uY2th6sbO8K\nOlbVAMC4xCHcPCeVrElJXDI+nqgw7XUeSLz5a44GCjxuFwKXdFpnF3A9VrfSdUCMiCQYY7aKyCag\nGCsQnjTG7PV8oIgMA662HzvgRUdHU19f73QZSnVgjGFv8Ql7NFA5OUeraHUZosKCmT8hkZULx5OV\nnsSYhCinS1X9qK/i/QHgSRG5DdgCFAEuEZkITOX01v8GEVlgjHkbQERCgD8DT5zaA+lMRG4HbgcY\nM2ZMH5WrlKo+2cLbByrYYodAmf2r36nJQ/nyJ8azcFIimWPjCQvRcf+DhTeBUASketxOse9rZ4w5\njrWHgIhEAzcYY2pEZCWwzRhTby/LBuYBb9sPXQPsN8Y83t2LG2PW2OuRmZlpulsPgOwHoSTXi7fU\nCyMzYNmj3S5+8MEHSU1N5a677gLghz/8ISEhIWzatInq6mpaW1v5yU9+wooVK3p8qfr6elasWNHl\n49auXcuqVasQEWbOnMkf/vAHSktLueOOOzh0yMrS1atXM3/+/D5402ogcrkNuwpr2LzP6gbaVViD\nMdY8/Z+YmEjWpCQWTkpixFAdEjpYeRMI7wPpIjIOKwhuAT7ruYKIJAJVxhg38F2sEUcAx4CVIvII\nVpdRFvC4/ZifALHAV/rgfTjm5ptv5r777msPhBdffJH169dzzz33MHToUCoqKpg7dy7XXHNNjwfc\nIiIiePnll8943J49e/jJT37Cu+++S2JiYvv5Fe655x6ysrJ4+eWXcblc2hWlzlBa18Rm+zjAO/sr\nqG1sJUjggtRh3Ht5OlmTkpiZMkyHhCrAi0AwxrSJyN3Aeqxhp88aYz4WkYeBHGPMK8Ai4BERMVhd\nRnfZD38JWAzkYh1gfs0Y86qIpADfB/KAD+yG8kljzDPn9W7OsiXfX2bPnk1ZWRnHjx+nvLycuLg4\nRo4cyTe+8Q22bNlCUFAQRUVFlJaWMnLkyLM+lzGG733ve2c8buPGjXz6058mMTEROH1+hY0bN7J2\n7VoAgoODiY2N7d83q/xec5uLHUeq20Mgr+QEYE3rfOW0EWRNTuITExMZFhXmcKXKH3l1DMEYsw5Y\n1+m+//S4/hJW49/5cS7gq13cX4i1xzAgfPrTn+all16ipKSEm2++mT/+8Y+Ul5ezY8cOQkNDSUtL\no6mpqcfnOdfHqcHtaOXJ9oPB7x6spKHFRWiwMCctnu8um8LCSUlMGRmjQ0JVj3TMWB+4+eabWbly\nJRUVFWzevJkXX3yR4cOHExoayqZNmzh69KhXz1NbW9vl4xYvXsx1113H/fffT0JCQvv5FS6//HJW\nr17Nfffd195lpHsJA19DSxvbDlW2Hws4UmkNCR0TH8WNF6WQNSmJueMTGBKuX2/VO/o/pg9Mnz6d\nEydOMHr0aJKTk/nc5z7H1VdfTUZGBpmZmUyZMsWr5+nucdOnT+f73/8+WVlZBAcHM3v2bH7/+9/z\nq1/9ittvv53f/e53BAcHs3r1aubNm9efb1U5wBjDvtIT7b8JeP9wNS0uN5GhwcyfkMCXLh1H1qQk\n0hKHOF2qCnBizNkH7viTzMxMk5OT0+G+vXv3MnXqVIcqCgz6GQWe2oZWe5ZQ69fBpXXWkNApI2NY\nOCmJrElJZKbFER6iM4OqnonIDmNMZk/r6R6CUn7A5TbkFtXa3UBl7CyowW2sUz4uSLcCYMGkRJJj\nI50uVQ1gGggOyM3N5dZbb+1wX3h4ONu3b3eoIuWEshNNbMmvsIeEllPd0IoIzEwZxt2LrSGhF6TE\nDvgTuyv/MSACwRgTUCMoMjIy2Llzp09eK5C6BAe6ljY3O45Ws2V/OZv3lbOnuA6wzhC2eMrpIaHx\nQ3RIqHJGwAdCREQElZWVJCQkBFQo+IIxhsrKSiIi9JenTimoamj/TcC7Byo42eIiJEjITIvj20sn\nkzUpiakjhxKkPwxTfiDgAyElJYXCwkLKy8udLsUvRUREkJKiE8n6SmOLi22HrSGhW/LLOVRxEoCU\nuEiunT2arElJzJuQQEyEnitA+Z+AD4TQ0FDGjRvndBlqEDtScZI39payOb+c7YeraGlzExEaxNzx\nCdw6byxZk5IYlzhE92CV3wv4QFDKCQfK6snOLWbd7hL22scC0odH84W5Y8manMSctHgiQnVIqAos\nGghKecEYw/6yetblFrMut7j9tJGZY+P4f5+axlXTR5ASp+cKUIFNA0Gpbpw6aUz2bisEDpafRAQu\nTovnR9dM56rpIxkZqwfs1cChgaCUB2MMu4vqWLe7mOzcYo5UNhAkMHd8ArddOo6rpo9geIyGgBqY\nNBDUoGeMYVdhrX1MoJiCqkaCg4T5ExL4atYErpw2goTocKfLVKrfaSCoQcntNnxYUM263BJe211C\nUU0jocHCpRMT+fridJZMHUGc/kBMDTIaCGrQcLkNOUeqyN5dQvbuYkrrmgkLDmLhpETuXzKJK6aO\nIDZKfx+gBi8NBDWgtbncvHekiuzcEl77uITyE82EhwSxaHISyzOSWTxluP5ITCmbBoIacFpdbrYd\nqmRdbgmvf1xC5ckWIkKDWDxlOMtmJHPZlOFE68ljlDqDfivUgNDS5ubfByvIzi3m9T2l1DS0EhUW\nzOVTR7B8xkiyJicRFab/3ZU6G/2GqIDV3Obinf0VrMstYcOeEuqa2ogOD+GKqcNZnpHMwklJ+mth\npXpBA0EFlKZWF5vzy8nOLebNvWWcaG5jaEQIS6aNZHnGSD6RnqhnEVPqHGkgKL/X2OLirX1l/Cu3\nmI15ZTS0uBgWFcryjGSWZYxk/oREwkL0JDJKnS8NBOWXTja3sTGvjOzdxWzKK6ex1UXCkDCunT2a\n5TOSuWR8PKF6JjGl+pQGgvIbdU2tbNxbxrrcYjbnl9Pc5iYpJpwbL0phWcZILk6L19NJKtWPNBCU\no2obWtmwt5Ts3GLe3l9Bi8vNyKERfObiMSzPSOaisXEE69nElPIJDQTlc9UnW9iwp5R1u4v594EK\nWl2GUbER3DpvLMszkpmdOkxPKamUAzQQlE9U1Dfz+selZO8u5t2DlbjchtT4SP7j0nEsy0jmgpRY\nPaOYUg7TQFD9puxEE+s/LmXdR8VsP1yJ20BaQhRfXTie5RnJTB81VENAKT+igaD6VEltE6/ttk4t\n+f6RKoyB8UlDuOuyiSybkczU5BgNAaW80dYMpbvh+E44/iEsfRTCo/v1JTUQ1HkrqmkkO7eY7N0l\n7DhaDcDkETHce3k6yzOSSR8erSGg1Nm0tUD5XqvhP3Up3QPuVmt5ZDzMvRNGTOvXMjQQ1DkpqGqw\nTy1Zws6CGgCmJg/lgSsnsXRGMhOH9++WjFIBy9UG5XmdGv/d4GqxlkfEwqjZMP9u699RsyE2FXyw\nUaWBoLx2pOKkfWrJEnKLagHIGB3Lt5dOZtmMZMYlDnG4QqX8jNsFFfkdG/+SXGhrspaHD4XkC+CS\nO043/nFpPmn8u6KBoM7qQFm9fWrJEvYW1wEwK3UY31s+hWUzkkmNj3K4QqX8hNsNlQdON/zFO6F4\nF7Q2WMvDoq3Gf85XPBr/cRDkPz+21EBQZ6iob+b5bUdZl1tMfmk9ABeNjeMHn5zKsoxkRg+LdLhC\npRzmdkP1YY8tf7vxbzlhLQ+JtBr/C78Io2ZZjX/CRAjy74kXNRBUB8YY7vjDDnYcq2ZOWjw/vHoa\nS2ckMzI2wunSnNXaBA0V0FAJJzv921Bh9QvHpsCwMacvQ0dDsH7FAp4xUH3k9Fb/8Q/h+C5otrpN\nCYmAkRlwwS2nt/wTJwXk3z7wKlb96o29ZeQcreYn187g83PHOl1O/zAGmus8GvRTjXuF/W+Vx3X7\ndkt9188lQdYIkKAQqC8FjMeyYCsUho2BYakdw6I9MPT0nX7FGKgtOL3Vf2oPoMkaOEFwGIyYDhk3\nnG78k6YMmL+jBoJq53IbHnstj/GJQ7h5TqrT5XjP1QaN1R6NuL3VfrKy+636U8P5OguJgKhEiIqH\nIYnWbv4Q+3ZUon09EaISrOsRw073Abc1Q20h1ByzGpWaY6cvh7dA3XE6BkaQR2DYl1iP4IhNGTAN\njV8yxvqbFO/seNC3odJaHhQCw6fBtBWnG//h0yAkzNm6+5FXgSAiS4FfAcHAM8aYRzstHws8CyQB\nVcDnjTGF9rLHgE8CQcAG4F5jjBGRnwJfAOKMMTpG0Q/89YNC9pfVs/pzFzo7tXRrY6et9kqPRr6L\nrfrGGjo0tJ7CY2FIgtWIDxtj9ee2N+z2/aeWRyVA2JBzH+EREg4JE6xLV9paoK7QIyg8QuPw21BX\nxBmBETPqzD2LU3sbQ1MGdOPU506UdNzqP/4hnCyzlkkwDJ8Kk5d5NP7TIXRwdZX2GAgiEgw8BSwB\nCoH3ReQVY8wej9VWAWuNMc+JyGLgEeBWEZkPXArMtNd7B8gC3gJeBZ4E9vfRe1HnoanVxS835HNB\n6jCWzhjZd09sDDTVntmId9cXf7ISWk92/VwSbDfi9tb5iOmnr7c37AmnG/zIeP9qMEPCIH68delK\nW4sVCp57Fqf2No7+G3JfBOP2eIDA0K4CY4wGRn35mVv+J4qtZRIEiZNh4hWnG/+RMyBUB0t4s4dw\nMXDAGHMIQEReAFYAnoEwDbjfvr4J+Lt93QARQBggQChQCmCM2WY/3/m9A9Un1m49QnFtE7+4adbZ\n/yauNmis6tSId9UXX3n6vm67ZyI9ttQTIDG9Y3dNhy35hI7dMwNRSBjEj7MuXXG1dh0YNcfg6FbI\n/b8zAyMmufvAiE2x9moCXUNVx4b/+E5rTwwAsf5fjVsIyfZon5EZ/T4FRKDyJhBGAwUetwuBSzqt\nswu4Hqtb6TogRkQSjDFbRWQTUIwVCE8aY/b2pkARuR24HWDMmDG9eajyUm1DK09tOsiiyUnMGx8P\nB96AI//uequ+sbr7J4qIPd2Ix6XB6AvP7HP3/DdMf8jWK8Gh1ucal9b1cler1SfeVWAc2wa7/wrG\n5fEAgZiR3QTGWP8MjMZqa3inZ+Nfc/T08vgJMGbu6aGeI2dCxFDn6g0wfXVQ+QHgSRG5DdgCFAEu\nEZkITAVS7PU2iMgCY8zb3j6xMWYNsAYgMzOzm45idT5Wbz5IS1M9/5V6AJ66Gyr2WQfUTm2ZRyVY\nW1WdG3bP7pqoeD0A6rTgUIgba1264mqDE90ERsF22P23ToGBtYcR28UIqVOB0Z997E11HRv/4p1Q\ndej08mFjrUY/8z+sf5MvgMhh/VfPIOBNIBQBnkNOUuz72hljjmPtISAi0cANxpgaEVkJbDPG1NvL\nsoF5gNeBoPpXWdFhhm39L3KiNjHknTrrS3XdGph+3eDtfx6ogkNON+hdcbVZ/exnBMZRKHwf9vwd\n3G0dHxM98syD3R0Cw8t++eZ6KPmo43DPSo/Di7Gp1lb/7M/bjf8sayNE9SlvAuF9IF1ExmEFwS3A\nZz1XEJFEoMoY4wa+izXiCOAYsFJEHsHqMsoCHu+j2tX5KNoB21aTkPs3Voqb5rRlsPDrMGaeY/Oo\nKIcFh9iNeirWWJBO3K4uAuOo9W9RTjeBMaKLYbVjISzKmtPn1NZ/+T7aR1jFjLIa/Zk32wd9Z1l7\noqrf9RgIxpg2EbkbWI817PRZY8zHIvIwkGOMeQVYBDwiIgary+gu++EvAYuBXKy/9mvGmFehfTjq\nZ4EoESnEGs76w758c6oTVxvk/RO2rYaCbbhDo/l925XUz/oy9964xOnqlL8LCra2+mNTYOz8M5e3\nB0ZBF4HxAex55cwBBkOGW8eapl9nH/SdZR3XUI4QYwKnWz4zM9Pk5OQ4XUbgaaqFD9bC9jVQe8w6\nKHnJHdy7bxpvHmpi87cWkRDtZwcP1cDjdlm/Bag5Bs0nrKGeMcm6R+oDIrLDGJPZ03r6S+WBrPIg\nbP8N7PyjNfXC2E/A0kdg8jJ2FNTxj7+/yzeXTNIwUL4RFAyxo62L8ksaCAONMXDkbatbaF+2NVoo\n40aY+zXrgDHWBHY/y84jMTqcLy/oZsy7UmrQ0UAYKNqaIfclKwhKc60hoQu/Zc29HjOiw6qb9pXx\n3pEqfnztDKLC9L+AUsqirUGgqy+DnGfh/WfgZLk1+dY1v4aMm7ocI+5yG36WvY+0hChuCaQJ7JRS\n/U4DIVCV5MK2p635bVwtkH6V1S00ftFZD9L9/cMi9pWe4MnPznZ2AjullN/RQAgkbjfsXw9bn7KO\nE4RGwYVfsM7Hmpje48ObWl38YkM+M1NiWT4j2QcFK6UCiQZCIGiuh51/gu2rrZ/uDx0NV/wILvoi\nRMZ5/TTPbztKUU0jP79xJkFBOtRPKdWRBoI/qzkG762BHWut0/WlzIHFP4Cp1/R63qC6plae3HSA\nBemJzJ+ov/pUSp1JA8HfGAMF78G2/4G9r1r3TVsBc++E1Dnn/LS/2XyQmoZWvrN0Sh8VqpQaaDQQ\n/IWrFfb8wwqCoh3WVNLz74aLb7emCjgPpXVN/O6dw6yYNYoZo2P7qGCl1ECjgeC0hirY8Xt477fW\n1MQJE2H5Kpj12T47X8Cv3tyPy2345pLJffJ8SqmBSQPBKeX51kHinX+GtkZruOjVj8PEJX16VrCD\n5fX85f0Cbp07ljEJUX32vEqpgUcDwZeMgYMbrV8TH9gAweEw8ybr9wMjpvfLS65av4+IkCDuXjyx\nX55fKTVwaCD4QmsjfPQXKwjK86wpfy/7vnWmp36c5/3DY9Vk7y7hvivSSdQJ7JRSPdBA6E91xdaU\nEjnPWiemH5kB1z4NM67v93PVGmN4NDuPxOgwvrJgfL++llJqYNBA6A/HP4St/wMf/82aA37KJ61h\no2Pn+2zu97fyy9l+uIqHV0wnOlz/zEqpnmlL0VfcLsj7lzVs9NhWCIuGOSvhkq9CvG+nmHa7remt\nx8RHccucbs6fq5RSnWggnK+mWvjwedj+tPXL4mFj4Kr/sk4GHuHMmP9/7Coir+QET3xmNmEhOoGd\nUso7GgjnquqQdUrKD5+HlhMwZj5c+VOreygo2LGymttcrFqfz4zRQ/lUhk5gp5TyngZCbxgDR/9t\njRbK+5fV8M+4wRo2Omq209UB8MdtxyiqaeTRGzJ0AjulVK9oIHijrRl2/806PlDyEUTGw4JvWmcj\nG+o/W+F1Ta38euN+PjExkQXpSU6Xo5QKMBoIZ1NfDjv+1xo6Wl8KSVPg6l/BzJshNNLp6s7w2y2H\nqNYJ7JRS50gDoSulH1t7Ax/9H7iarekk5n4NJiz22bDR3io70cQzbx/mUzOTyUjRCeyUUr2ngXCK\n221NJ7H1KTi8GUIiYfbn4JKvQdIkp6vr0RNv7qfV5eaBK3UCO6XUudFAaK6HXX+2DhRXHYSYUXD5\nQ3DRbRAV73R1XjlccZI/v1fAZy8eQ1pi38yQqpQafAZvINQUWGcj++A567cEoy6EG35nnYyml2cj\nc9qq1/cRHhLE1y/XCeyUUudu8AVCwfuw7SnY8wpgrNNRzr0TUi/22+MDZ7OroIZ/fVTMPZenMzwm\nwulylFIBbHAEgqsV9r5izS9UlAPhsTDvTutsZMMCd2oHYww/ey2P+CFhrFzg2+kxlFIDz+AIhOev\nh8NbIH48LPu5dTay8Ginqzpvb++v4N2DlTx09TRiIgKrm0sp5X8GRyBc8jWrWyj9qj49G5mT3G5r\neuvU+Eg+e0ng7uUopfzH4AiEKcudrqDPvfrRcfYU1/H4zbMID3Fu7iSl1MAxMDaXB5mWNjerXt/H\n1OShXHPBKKfLUUoNEBoIAehP249SUNXIg8um6AR2Sqk+o4EQYOqb2/j1xgPMG5/AwvT+Ox+zUmrw\n0UAIML/dcojKky08uGwKEoC/m1BK+S8NhABSfqKZ3759iE9mJHNB6jCny1FKDTBeBYKILBWRfSJy\nQEQe7GL5WBF5U0Q+EpG3RCTFY9ljIvKxiOwVkSfE3qwVkYtEJNd+zvb7Vfd+vXE/zW1uvnml/0+2\np5QKPD0GgogEA08By4BpwGdEZFqn1VYBa40xM4GHgUfsx84HLgVmAjOAOUCW/ZjVwEog3b4sPd83\nM5AdrTzJn7Yf45Y5qYxPCvwf1Sml/I83ewgXAweMMYeMMS3AC8CKTutMAzba1zd5LDdABBAGhAOh\nQKmIJANDjTHbjDEGWAtce17vZIBb9Xo+ocFB3Ht5utOlKKUGKG8CYTRQ4HG70L7P0y7gevv6dUCM\niCQYY7ZiBUSxfVlvjNlrP76wh+dUttzCWl7ddZwvf2Icw4fqBHZKqf7RVweVHwCyRORDrC6hIsAl\nIhOBqUAKVoO/WEQW9OaJReR2EckRkZzy8vI+KjewPLY+j7ioUG7PGu90KUqpAcybQCgCUj1up9j3\ntTPGHDfGXG+MmQ18376vBmtvYZsxpt4YUw9kA/Psx6ec7Tk9nnuNMSbTGJOZlDT4Thz/9v5y3t5f\nwd2L0xmqE9gppfqRN4HwPpAuIuNEJAy4BXjFcwURSRSRU8/1XeBZ+/oxrD2HEBEJxdp72GuMKQbq\nRGSuPbroC8A/+uD9DChutzW99ehhkXx+rk5gp5TqXz0GgjGmDbgbWA/sBV40xnwsIg+LyDX2aouA\nfSKSD4wAfmrf/xJwEMjFOs6wyxjzqr3sTuAZ4IC9TnafvKMB5F+5xewuquObV07SCeyUUv1OrEE+\ngSEzM9Pk5OQ4XYZPtLS5WfLLzUSGBvOvexYQrHMWKaXOkYjsMMZk9rSe/lLZT73w/jGOVjbwnaVT\nNAyUUj6hgeCHTja38cSb+7nOw9oPAAAOrElEQVRkXDyLJg++A+lKKWdoIPihZ94+TEW9TmCnlPIt\nDQQ/U1HfzJotB1k6fSSzx8Q5XY5SahDRQPAzT248QFObm28tnex0KUqpQUYDwY8cq2zgj9uPclNm\nKhN0AjullI9pIPiR/96wj+Ag4b4rdAI7pZTvaSD4id1Ftfxj53H+49JxjNAJ7JRSDtBA8BOPrd/H\nsKhQvpo1welSlFKDlAaCH3j3QAVb8su5a9FEYiN1AjullDM0EBxmjOHR1/IYFRvBrfPGOl2OUmoQ\n00Bw2LrcEj4qrOX+KycTEaoT2CmlnKOB4KBWl5ufr89j8ogYrputJ4xTSjlLA8FBf3m/gCOVDXx7\n6WSdwE4p5TgNBIecbG7j8Tf2c3FaPIunDHe6HKWU0kBwyrPvHKaivpnv6AR2Sik/oYHggKqTLfxm\nyyGunDaCi8bqBHZKKf+ggeCAJzceoKGljW/rBHZKKT+igeBjBVUNPL/tKJ++KJWJw2OcLkcppdpp\nIPjYLzfkIwL3LdEJ7JRS/kUDwYf2HK/j5Z1FfOnScSTHRjpdjlJKdaCB4EOPrc8jJjyEr+kEdkop\nP6SB4CNbD1by1r5y7rpsIrFROoGdUsr/aCD4wKkJ7JJjI/ji/DSny1FKqS5pIPjA+o9L2FVQwzeu\nmKQT2Cml/JYGQj9rc7l57LV9pA+P5voLdQI7pZT/0kDoZy/mFHKo4iTfXjqFkGD9uJVS/ktbqH7U\n2OLi8TfyyRwbxxVTdQI7pZR/00DoR8/++zBlJ3QCO6VUYNBA6CfVJ1t4+q2DXDF1BHPS4p0uRyml\neqSB0E/+560DnNQJ7JRSAUQDoR8UVjfw3LtHueHCFCaN0AnslFKBQQOhH/xyw34Q+MaSSU6XopRS\nXtNA6GN5JXX87cNCbpufxqhhOoGdUipwaCD0sZ+/to/o8BDuXKQT2CmlAosGQh9673AVb+aVceei\niQyLCnO6HKWU6hWvAkFElorIPhE5ICIPdrF8rIi8KSIfichbIpJi33+ZiOz0uDSJyLX2ssUi8oGI\n7BaR50QkpG/fmm8ZY3g0ey8jhoZzm05gp5QKQD0GgogEA08By4BpwGdEZFqn1VYBa40xM4GHgUcA\njDGbjDGzjDGzgMVAA/C6iAQBzwG3GGNmAEeBL/bRe3LE63tK+eCYNYFdZJhOYKeUCjze7CFcDBww\nxhwyxrQALwArOq0zDdhoX9/UxXKAG4FsY0wDkAC0GGPy7WUbgBt6W7y/sCawy2NC0hBuvCjF6XKU\nUuqceBMIo4ECj9uF9n2edgHX29evA2JEJKHTOrcAf7avVwAhIpJp374RSPW2aH/z1w8KOVh+km9d\npRPYKaUCV1+1Xg8AWSLyIZAFFAGuUwtFJBnIANYDGGMMVkD8UkTeA054ru9JRG4XkRwRySkvL++j\ncvtOY4uLX27Yz+wxw7hq+giny1FKqXPmzYHcIjpuvafY97UzxhzH3kMQkWjgBmNMjccqNwEvG2Na\nPR6zFVhgP+ZKoMtfcRlj1gBrADIzM40X9frU7989QkldE7+6ZZZOYKeUCmje7CG8D6SLyDgRCcPa\nsn/FcwURSbQPFAN8F3i203N8htPdRaceM9z+Nxz4DvB078t3Vk1DC6vfOsDiKcO5ZHznHjKllAos\nPQaCMaYNuBuru2cv8KIx5mMReVhErrFXWwTsE5F8YATw01OPF5E0rD2MzZ2e+lsishf4CHjVGLOR\nALP6rYOcaNYJ7JRSA4NY3fmBITMz0+Tk5DhdBgDHaxpZtOotrp45iv++6QKny1FKqW6JyA5jTGZP\n6+mQmHP0+Bv5YOAbS9KdLkUppfqEBsI5yC89wUs7CvnCvLGkxEU5XY5SSvUJDYRz8Nhr+xgSFsJd\nl010uhSllOozGgi9lHOkijf2lnLHognEDdEJ7JRSA4cGQi9YE9jlMTwmnC9dmuZ0OUop1ac0EHrh\njb1l5Byt5t4r0okKC+jJWZVS6gwaCF5yuQ2PvZbH+MQh3JQZsNMuKaVUtzQQvPTXDwrZX1bPt66a\nTKhOYKeUGoC0ZfNCU6uLX27I54LUYSydMdLpcpRSql9oIHhh7dYjFNc28eDSKTqBnVJqwNJA6EFt\nQytPbTrIoslJzJugE9gppQYuDYQerN58kLqmVr591RSnS1FKqX6lgXAWJbVN/O+/D3PtrNFMGzXU\n6XKUUqpfaSCcxeNv5GMM3L+ky3P3KKXUgKKB0I0DZSd4MaeAz80dQ2q8TmCnlBr4NBC68fP1+4gK\nC+FuncBOKTVIaCB0YcfRatZ/XMpXF44nITrc6XKUUsonNBA6Mcbws+w8EqPD+fKCcU6Xo5RSPqOB\n0MmmfWW8d6RKJ7BTSg06GggeXG7Dz7L3kZYQxS1zdAI7pdTgooHg4eUPi9hXeoIHdAI7pdQgpK2e\n7dQEdjNTYlk+I9npcpRSyuc0EGzPbztKUU0jDy6dQlCQTmCnlBp8NBCAuqZWntx0gAXpicyfmOh0\nOUop5QgNBOA3mw9S09DKd5bqBHZKqcFr0AdCaV0Tv3vnMCtmjWLG6Finy1FKKccM+kB4/I39uNyG\nby6Z7HQpSinlqEEdCAfL660J7C4Zy5gEncBOKTW4DepAWLV+HxEhQdy9WCewU0qpQRsIHx6rJnt3\nCSsXjidRJ7BTSqnBGQjGGB7NziMxOoyvLBjvdDlKKeUXBmUgvJVfzvbDVdxzeTrR4TqBnVJKwSAM\nBLfbmt56THwUt8wZ43Q5SinlNwZdIPxjVxF5JdYEdmEhg+7tK6VUtwZVi9jc5mLV+nxmjB7KpzJ0\nAjullPI0qALh+W3HKKpp5Ds6gZ1SSp1h0ARCXVMrT27czycmJrIgPcnpcpRSyu94FQgislRE9onI\nARF5sIvlY0XkTRH5SETeEpEU+/7LRGSnx6VJRK61l10uIh/Y978jIv3667DfbjlEtU5gp5RS3eox\nEEQkGHgKWAZMAz4jItM6rbYKWGuMmQk8DDwCYIzZZIyZZYyZBSwGGoDX7cesBj5nL/sT8IM+eD9d\nKqtr4pm3D/OpmclkpOgEdkop1RVv9hAuBg4YYw4ZY1qAF4AVndaZBmy0r2/qYjnAjUC2MabBvm2A\nofb1WOB4bwrvjSc27qfV5eaBK3UCO6WU6o43gTAaKPC4XWjf52kXcL19/TogRkQSOq1zC/Bnj9tf\nAdaJSCFwK/Cot0X3VkpcFCsXjictcUh/vYRSSgW8vjqo/ACQJSIfAllAEeA6tVBEkoEMYL3HY74B\nLDfGpAD/C/yiqycWkdtFJEdEcsrLy8+puDuyJuixA6WU6oE3gVAEpHrcTrHva2eMOW6Mud4YMxv4\nvn1fjccqNwEvG2NaAUQkCbjAGLPdXv4XYH5XL26MWWOMyTTGZCYl6eggpZTqL94EwvtAuoiME5Ew\nrK6fVzxXEJFEETn1XN8Fnu30HJ+hY3dRNRArIpPs20uAvb0tXimlVN/pcWY3Y0ybiNyN1d0TDDxr\njPlYRB4GcowxrwCLgEdExABbgLtOPV5E0rD2MDZ3es6VwF9FxI0VEP/RV29KKaVU74kxxukavJaZ\nmWlycnKcLkMppQKKiOwwxmT2tN6g+aWyUkqps9NAUEopBWggKKWUsmkgKKWUAgLsoLKIlANHz/Hh\niUBFH5bTV7Su3tG6ekfr6p2BWtdYY0yPP+QKqEA4HyKS481Rdl/TunpH6+odrat3Bntd2mWklFIK\n0EBQSillG0yBsMbpArqhdfWO1tU7WlfvDOq6Bs0xBKWUUmc3mPYQlFJKncWACwQvzv8cLiJ/sZdv\ntyff84e6bhORco/zT3/FBzU9KyJlIrK7m+UiIk/YNX8kIhf2d01e1rVIRGo9Pqv/9FFdqSKySUT2\niMjHInJvF+v4/DPzsi6ff2YiEiEi74nILruuH3Wxjs+/j17W5fPvo8drB4vIhyLyzy6W9e/nZYwZ\nMBes2VgPAuOBMKwzuU3rtM6dwNP29VuAv/hJXbcBT/r481oIXAjs7mb5ciAbEGAusN1P6loE/NOB\n/1/JwIX29Rggv4u/o88/My/r8vlnZn8G0fb1UGA7MLfTOk58H72py+ffR4/Xvh/rPPNn/L36+/Ma\naHsI3pz/eQXwnH39JeByERE/qMvnjDFbgKqzrLICWGss24Bh9tnvnK7LEcaYYmPMB/b1E1jn8Oh8\nOlmff2Ze1uVz9mdQb98MtS+dD1r6/PvoZV2OEJEU4JPAM92s0q+f10ALBG/O/9y+jjGmDagFOp//\n2Ym6AG6wuxleEpHULpb7mrd1O2GevcufLSLTff3i9q76bKytS0+OfmZnqQsc+Mzs7o+dQBmwwZw+\nS+IpTnwfvakLnPk+Pg58G3B3s7xfP6+BFgiB7FUgzRgzE9jA6a0AdaYPsH6KfwHwa+DvvnxxEYkG\n/grcZ4yp8+Vrn00PdTnymRljXMaYWVin3r1YRGb44nV74kVdPv8+isingDJjzI7+fq3uDLRA6PH8\nz57riEgIEAtUOl2XMabSGNNs33wGuKifa/KGN5+nzxlj6k7t8htj1gGhIpLoi9cWkVCsRvePxpi/\ndbGKI59ZT3U5+ZnZr1kDbAKWdlrkxPexx7oc+j5eClwjIkewupUXi8jzndbp189roAVCj+d/tm9/\n0b5+I7DR2EdonKyrUz/zNfjHOaZfAb5gj5yZC9QaY4qdLkpERp7qNxWRi7H+H/d7I2K/5u+AvcaY\nX3Szms8/M2/qcuIzE5EkERlmX4/EOnd6XqfVfP599KYuJ76PxpjvGmNSjDFpWG3ERmPM5zut1q+f\nV4/nVA4kxrvzP/8O+IOIHMA6cHmLn9R1j4hcA7TZdd3W33WJyJ+xRp8kikgh8BDWATaMMU8D67BG\nzRwAGoAv9XdNXtZ1I/A1EWkDGoFbfBDqYG3B3Qrk2v3PAN8DxnjU5sRn5k1dTnxmycBzIhKMFUAv\nGmP+6fT30cu6fP597I4vPy/9pbJSSilg4HUZKaWUOkcaCEoppQANBKWUUjYNBKWUUoAGglJKKZsG\nglJKKUADQSmllE0DQSmlFAD/H64pKJUrRrG0AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "te4U5_VrfUdp",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "p = model.predict(data)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "5bvk--egfYvs",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "c0ac07dd-a22e-49cc-b2ce-66a0be600974"
      },
      "cell_type": "code",
      "source": [
        "aucs = []\n",
        "for j in range(6):\n",
        "    auc = roc_auc_score(targets[:,j], p[:,j])\n",
        "    aucs.append(auc)\n",
        "print(np.mean(aucs))"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.9808841020370122\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}