visualization/2.attention-visualization-luong.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "/usr/local/lib/python3.5/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from utils import *\n",
    "import tensorflow as tf\n",
    "from sklearn.cross_validation import train_test_split\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['negative', 'positive']\n",
      "10662\n",
      "10662\n"
     ]
    }
   ],
   "source": [
    "trainset = sklearn.datasets.load_files(container_path = 'data', encoding = 'UTF-8')\n",
    "trainset.data, trainset.target = separate_dataset(trainset,1.0)\n",
    "print (trainset.target_names)\n",
    "print (len(trainset.data))\n",
    "print (len(trainset.target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X, test_X, train_Y, test_Y = train_test_split(trainset.data, trainset.target, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 20332\n",
      "Most common words [('film', 1453), ('movie', 1270), ('one', 727), ('like', 721), ('story', 477), ('much', 386)]\n",
      "Sample data [543, 2605, 3331, 18780, 36, 7289, 218, 150, 19, 4121] ['rock', 'destined', '21st', 'centurys', 'new', 'conan', 'hes', 'going', 'make', 'splash']\n"
     ]
    }
   ],
   "source": [
    "concat = ' '.join(trainset.data).split()\n",
    "vocabulary_size = len(list(set(concat)))\n",
    "data, count, dictionary, rev_dictionary = build_dataset(concat, vocabulary_size)\n",
    "print('vocab from size: %d'%(vocabulary_size))\n",
    "print('Most common words', count[4:10])\n",
    "print('Sample data', data[:10], [rev_dictionary[i] for i in data[:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = dictionary['GO']\n",
    "PAD = dictionary['PAD']\n",
    "EOS = dictionary['EOS']\n",
    "UNK = dictionary['UNK']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 128\n",
    "num_layers = 2\n",
    "embedded_size = 128\n",
    "dimension_output = len(trainset.target_names)\n",
    "learning_rate = 1e-3\n",
    "maxlen = 50\n",
    "batch_size = 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model:\n",
    "    def __init__(self, size_layer, num_layers, embedded_size,\n",
    "                 dict_size, dimension_output, learning_rate):\n",
    "        \n",
    "        def cells(reuse=False):\n",
    "            return tf.nn.rnn_cell.LSTMCell(size_layer,initializer=tf.orthogonal_initializer(),reuse=reuse)\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None])\n",
    "        batch_size = tf.shape(self.X)[0]\n",
    "        encoder_embeddings = tf.Variable(tf.random_uniform([dict_size, embedded_size], -1, 1))\n",
    "        encoder_embedded = tf.nn.embedding_lookup(encoder_embeddings, self.X)\n",
    "        outputs, last_state = tf.nn.dynamic_rnn(tf.nn.rnn_cell.MultiRNNCell([cells() for _ in range(num_layers)]), \n",
    "                                                encoder_embedded, dtype = tf.float32)\n",
    "        attention_mechanism = tf.contrib.seq2seq.LuongAttention(num_units = size_layer, \n",
    "                                                                    memory = outputs)\n",
    "        rnn_cells = tf.contrib.seq2seq.AttentionWrapper(cell = tf.nn.rnn_cell.MultiRNNCell([cells() for _ in range(num_layers)]), \n",
    "                                                        attention_mechanism = attention_mechanism,\n",
    "                                                        attention_layer_size = size_layer,\n",
    "                                                       alignment_history=True)\n",
    "        decoder_outputs, decoder_last_state = tf.nn.dynamic_rnn(rnn_cells, encoder_embedded, \n",
    "                                                                initial_state = rnn_cells.zero_state(batch_size, tf.float32).clone(cell_state=last_state),\n",
    "                                                                dtype = tf.float32)\n",
    "        self.alignments = tf.transpose(decoder_last_state.alignment_history.stack(),[1,2,0])\n",
    "        W = tf.get_variable('w',shape=(size_layer, dimension_output),initializer=tf.orthogonal_initializer())\n",
    "        b = tf.get_variable('b',shape=(dimension_output),initializer=tf.zeros_initializer())\n",
    "        self.logits = tf.matmul(outputs[:, -1], W) + b\n",
    "        self.cost = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits = self.logits, labels = self.Y))\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        correct_pred = tf.equal(tf.argmax(self.logits, 1,output_type=tf.int32), self.Y)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(size_layer,num_layers,embedded_size,len(dictionary),dimension_output,learning_rate)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.000000, current acc: 0.595703\n",
      "time taken: 2.8624331951141357\n",
      "epoch: 0, training loss: 0.684345, training acc: 0.543797, valid loss: 0.664340, valid acc: 0.595703\n",
      "\n",
      "epoch: 1, pass acc: 0.595703, current acc: 0.662109\n",
      "time taken: 2.77538800239563\n",
      "epoch: 1, training loss: 0.585240, training acc: 0.692235, valid loss: 0.626291, valid acc: 0.662109\n",
      "\n",
      "epoch: 2, pass acc: 0.662109, current acc: 0.675293\n",
      "time taken: 2.777212381362915\n",
      "epoch: 2, training loss: 0.431262, training acc: 0.807765, valid loss: 0.638474, valid acc: 0.675293\n",
      "\n",
      "epoch: 3, pass acc: 0.675293, current acc: 0.688477\n",
      "time taken: 2.793844699859619\n",
      "epoch: 3, training loss: 0.293421, training acc: 0.875710, valid loss: 0.705397, valid acc: 0.688477\n",
      "\n",
      "epoch: 4, pass acc: 0.688477, current acc: 0.698242\n",
      "time taken: 2.8104021549224854\n",
      "epoch: 4, training loss: 0.185434, training acc: 0.929569, valid loss: 1.045832, valid acc: 0.698242\n",
      "\n",
      "time taken: 2.790931463241577\n",
      "epoch: 5, training loss: 0.123517, training acc: 0.955492, valid loss: 1.152864, valid acc: 0.695801\n",
      "\n",
      "epoch: 6, pass acc: 0.698242, current acc: 0.701172\n",
      "time taken: 2.7840640544891357\n",
      "epoch: 6, training loss: 0.065245, training acc: 0.980824, valid loss: 1.655402, valid acc: 0.701172\n",
      "\n",
      "epoch: 7, pass acc: 0.701172, current acc: 0.706055\n",
      "time taken: 2.7694666385650635\n",
      "epoch: 7, training loss: 0.033158, training acc: 0.989228, valid loss: 1.710513, valid acc: 0.706055\n",
      "\n",
      "epoch: 8, pass acc: 0.706055, current acc: 0.711426\n",
      "time taken: 2.7587289810180664\n",
      "epoch: 8, training loss: 0.018099, training acc: 0.994437, valid loss: 1.959445, valid acc: 0.711426\n",
      "\n",
      "time taken: 2.773940324783325\n",
      "epoch: 9, training loss: 0.014265, training acc: 0.995739, valid loss: 2.054330, valid acc: 0.694336\n",
      "\n",
      "time taken: 2.797513723373413\n",
      "epoch: 10, training loss: 0.013101, training acc: 0.997041, valid loss: 2.234490, valid acc: 0.700195\n",
      "\n",
      "time taken: 2.780038833618164\n",
      "epoch: 11, training loss: 0.005318, training acc: 0.998580, valid loss: 2.492469, valid acc: 0.707520\n",
      "\n",
      "time taken: 2.7738423347473145\n",
      "epoch: 12, training loss: 0.002818, training acc: 0.999171, valid loss: 2.571098, valid acc: 0.699219\n",
      "\n",
      "time taken: 2.7855167388916016\n",
      "epoch: 13, training loss: 0.002345, training acc: 0.999527, valid loss: 2.692470, valid acc: 0.690430\n",
      "\n",
      "break epoch:14\n",
      "\n"
     ]
    }
   ],
   "source": [
    "EARLY_STOPPING, CURRENT_CHECKPOINT, CURRENT_ACC, EPOCH = 5, 0, 0, 0\n",
    "while True:\n",
    "    lasttime = time.time()\n",
    "    if CURRENT_CHECKPOINT == EARLY_STOPPING:\n",
    "        print('break epoch:%d\\n'%(EPOCH))\n",
    "        break\n",
    "        \n",
    "    train_acc, train_loss, test_acc, test_loss = 0, 0, 0, 0\n",
    "    for i in range(0, (len(train_X) // batch_size) * batch_size, batch_size):\n",
    "        batch_x = str_idx(train_X[i:i+batch_size],dictionary,maxlen)\n",
    "        acc, loss, _ = sess.run([model.accuracy, model.cost, model.optimizer], \n",
    "                           feed_dict = {model.X : batch_x, model.Y : train_Y[i:i+batch_size]})\n",
    "        train_loss += loss\n",
    "        train_acc += acc\n",
    "    \n",
    "    for i in range(0, (len(test_X) // batch_size) * batch_size, batch_size):\n",
    "        batch_x = str_idx(test_X[i:i+batch_size],dictionary,maxlen)\n",
    "        acc, loss = sess.run([model.accuracy, model.cost], \n",
    "                           feed_dict = {model.X : batch_x, model.Y : test_Y[i:i+batch_size]})\n",
    "        test_loss += loss\n",
    "        test_acc += acc\n",
    "    \n",
    "    train_loss /= (len(train_X) // batch_size)\n",
    "    train_acc /= (len(train_X) // batch_size)\n",
    "    test_loss /= (len(test_X) // batch_size)\n",
    "    test_acc /= (len(test_X) // batch_size)\n",
    "    \n",
    "    if test_acc > CURRENT_ACC:\n",
    "        print('epoch: %d, pass acc: %f, current acc: %f'%(EPOCH,CURRENT_ACC, test_acc))\n",
    "        CURRENT_ACC = test_acc\n",
    "        CURRENT_CHECKPOINT = 0\n",
    "    else:\n",
    "        CURRENT_CHECKPOINT += 1\n",
    "        \n",
    "    print('time taken:', time.time()-lasttime)\n",
    "    print('epoch: %d, training loss: %f, training acc: %f, valid loss: %f, valid acc: %f\\n'%(EPOCH,train_loss,\n",
    "                                                                                          train_acc,test_loss,\n",
    "                                                                                          test_acc))\n",
    "    EPOCH += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "heatmap=sess.run(model.alignments,feed_dict={model.X:str_idx(test_X[1:2],dictionary,len(test_X[1].split()))})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA04AAAI/CAYAAABAqYlRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAIABJREFUeJzs3X24bWVdL/zvChVJdpi4NYUUTfsZmG4FKd8lzdSro5kYKr5mT8fMt3yosDyF5lG0Yzy+9faooYlGmgbHTDTfMkLFjggi/s6jRgqZbtEUUlFY8/ljDo7L1Z57Dth7rTXXXp8P17yYY8x7jHGPuXE7f9f3vu+xNJlMAgAAwGzft9EdAAAAWHQKJwAAgDkUTgAAAHMonAAAAOZQOAEAAMyhcAIAAJjjehvdgUX2nS9/dp9Zq/2AW957o7uwV13v+/bb6C7sNTf9/h/Y6C7sVQfst/9Gd2GvuWr5qo3uwl71+cu/vNFd2Guuv9++9X9fBx+wbaO7sNdcf2nf+fs5SZazz/wUyKWXX7bRXdir9qXfAknyrW99bmmj+zDGev4+vv5Nb7tQ34nECQAAYA6FEwAAwBz71lgHAABg7SxfvdE92DASJwAAgDkkTgAAwDiT5Y3uwYaROAEAAMwhcQIAAMZZljgBAAAwg8QJAAAYZWKOEwAAALNInAAAgHHMcQIAAGAWiRMAADCOOU4AAADMonACAACYw1A9AABgnOWrN7oHG0biBAAAMMc+WzhV1Y2r6qkb3Q8AANhnTJbX77Vg9tnCKcmNkyicAACAPbYwc5yq6rFJnpHkBkk+nGnR87Ukf5TkIUm+kOS3krwkya2SPKu7z6yqJyZ5eJKDkhyS5A3d/bwkJyf5kao6L8m7k9w8yVu7+6+H652W5C+7+4x1u0kAANjMPAB3Y1XVjyU5Lsk9u3tHkquTHJ/kRkne291HJLk8yQuS/HSmhdLzV5zi6CSPSHKnJI+sqqOSnJjkM929o7t/PclrkjxxuN5BSe6R5G/W/u4AAIDNbiEKpyT3T3JkknOHhOj+SW6b5NtJ3jm0uSDJB7r7O8P7w1Yc/+7uvqy7v5nkrUnutfoC3f2BJLevqu1JHp3kr7r7qjW6HwAA2OdMJsvr9lo0izJUbynJ67r7OSt3VtUJ3T0ZNpeTXJkk3b1cVSv7Psn3Wr19jdcneWySRyV50h73GgAA2BIWpXB6T5IzquqU7v5SVd0kybZrcfxPD8d8M8nPJfnFTIf2rT7HqUk+kuTfuvuTe95tAADYQsxx2lhDEfPcJO+qqvMzXczhFtfiFB9J8ldJzs90CN5Hu/uyJGdX1Seq6veH63wxyUVJ/myv3gAAALBPW5TEKd19epLTV+0+cMXnJ61qf+CKzUu6++d2cc7HrNyuqu9Pcvskb9rT/gIAwJazgHOP1stCJE7roaoekGna9Iru/tpG9wcAANg8FiZxuq66+9RM5y7Na/d3SW691v0BAIB91vLVG92DDbNlEicAAIDratMnTgAAwDoxxwkAAIBZFE4AAABzGKoHAACM4wG4AAAAzCJxAgAAxrE4BAAAALNInAAAgHHMcQIAAGAWiRMAADDKZHL1Rndhw0icAAAA5pA4AQAA41hVDwAAgFkkTgAAwDhbeFU9hdNu/Ptk+0Z3Ya/5wqWf2uguAADApqVwAgAAxjHHCQAAgFkkTgAAwDjLnuMEAADADAonAACAOQzVAwAAxrE4BAAAALNInAAAgHG28ANwJU4AAABzSJwAAIBxzHECAABgFokTAAAwjjlOAAAAzCJxAgAAxpE4AQAAMIvECQAAGGUyuXqju7Bh9pnEqaqOqqqXb3Q/AACAfc+mS5yqar/u/k+lbnd/NMlHN6BLAACwNWzhOU5rXjhV1WOTPCPJDZJ8OMkLk/xdkrsn+UqSDyT5ve5+1y7aPrW7r66qK5L8SZIHJPnVqroyycuS3CjJlUnun+TIJCd0989W1X2Hz5NkkuQ+3X15Vf16kl9Isn+St3X37671/QMAAJvfmg7Vq6ofS3Jcknt2944kVye5b5IXJ/mjJP93kk8ORdOu2h4/nOpGST7c3XdO8pEkpyd55rD9gCTfXHXpE5L86nCeeyf5ZlU9MMntkxydZEeSI6vqPmt06wAAsO+ZLK/fa8GsdeJ0TRJ0blUlyQFJvtTdJ1XVI5M8JdMiZmbb4bOrk/zV8L6SfKG7z02S7v56kgzHXOPsJH9QVacleWt3XzIUTg9M8rGhzYGZFlJ/vzdvGAAAWHtV9aBMR5ntl+TV3X3yqs/3T/L6TGuMy5Ic190XV9VhSS5K0kPTD3X3U+Zdb60Lp6Ukr+vu56zcWVXfn+TQYfPAJJfPajv41q7mNc3S3SdX1d8keUiSs6vqZ4bzv6i7/+Q63AcAALAgqmq/JK9K8tNJLsk0fDmzuz+5otmTk3y1u29XVY/KdNTbccNnnxlGp4221qvqvSfJsVV1sySpqptU1a0z7fRpSX4nyf87p+1qneQWVXW3od22qvqeArCqfqS7L+juFyc5N8kdkpyV5Ber6sChzSHXXAsAABhheXn9Xrt3dJJPd/dnu/vbSf4iycNWtXlYktcN79+S5P5VtXRdb31NC6eh4ntukndV1flJ3p3ksCR3S/Li7j4tyber6kkz2t5iF+f8dqaV4iuq6uNDuxuuavasqvrEcJ7vJPnb7n5XkjcmOaeqLsj0y9u2128aAABYa4ck+fyK7UuGfbts091XJflakoOHz25TVR+rqg9U1b3HXHDNV9Xr7tMzXcxhpZ9c8fnPz2mb7j5w1fa5K88xeP/wSnc/fUZfXpbvrrYHAABcGwu4aMN18IUkt+ruy6rqyCR/XVVHXLN2wiz7zANwAQCALePSJD+8YvvQYd8u2wxTew5Kcll3X9ndlyVJd/9Tks8k+dF5F9x0D8AFAAA2yOI8APfcJLevqttkWiA9KsljVrU5M8kTkpyT5Ngk7+3uSVVtT/KV4Xmxt810pe3PzrugxAkAANhUhjlLT8t0AbiLkvxld19YVc+vqocOzV6T5OCq+nSSZyc5cdh/nyTnV9V5ma578JTu/sq8ay5NJpO9fR/7jJ07L/flAACw5rZv33adV3tbT98865Xr9vv4gJ952kJ9JxInAACAOcxxAgAAxlmcOU7rTuIEAAAwh8QJAAAYR+IEAADALBInAABgnInECQAAgBkkTgAAwDjmOAEAADCLwgkAAGAOQ/UAAIBxLA4BAADALBInAABgHItDAAAAMIvECQAAGMccJwAAAGaROAEAAOOY4wQAAMAsEicAAGAciRMAAACzSJwAAIBxJpON7sGGkTgBAADMIXECAADGMccJAACAWSROAADAOBInAAAAZpE4AQAA40wkTgAAAMygcAIAAJjDUD0AAGAci0MAAAAwy8IXTlX1jKq6qKourapXXstjH1pVJ8747Iq900MAANgiJpP1ey2YzTBU76lJHjC8jhp7UFVdr7vPTHLmWnUMAADYGha6cKqqP05y2yR/m+S1K/YfNmzfNMnOJE/q7s9V1alJvpXkLknOrqrzkxzV3U+rqtskeWOSA5OcsZ73AQAA+wRznBZTdz8lyb8mOSbJV1d89Iokr+vuOyU5LcnLV3x2aJJ7dPezV53uZUn+qLt/PMkX1q7XAADAvmahC6fduHum6VGS/HmSe6347M3dffUujrlnkjetOAYAALg2lpfX77VgNmvhtDv/sZvPFm+WGQAAsPA2a+H0j0keNbw/PskHRxxz9qpjAACAa2OyvH6vBbNZC6enJ3nSsPjD45I8c8Qxz0zyq1V1QZJD1rJzAADAvmVpsoBrpC+KnTsv9+UAALDmtm/ftrTRfRjjG3/6a+v2+/j7f/mUhfpONmviBAAAsG4W+jlOAADAAlnA1e7Wi8QJAABgDokTAAAwzgKudrdeJE4AAABzKJwAAADmMFQPAAAYZ3nrPq1H4gQAADCHxAkAABjHcuQAAADMInECAADGkTgBAAAwi8QJAAAYZ2JVPQAAAGaQOAEAAOOY4wQAAMAsEicAAGCcZXOcAAAAmEHiBAAAjDMxxwkAAIAZJE4AAMA45jgBAAAwi8IJAABgDkP1AACAUSYegAsAAMAsEicAAGAci0MAAAAwi8QJAAAYxwNwAQAAmEXiBAAAjGOOEwAAALNInAAAgHE8xwkAAIBZJE4AAMA45jjtXVV1alUdO7x/VlV9/4rP3lFVN94L19hRVQ8Z0e5+VfX2Pb0eAACwda1H4vSsJG9I8o0k6e65xc5IO5IcleQde+l8AADA7mzh5zjtUeFUVYcleXt333HYPiHJgSs+f0aSWyZ5X1V9ubuPqaqLMy14DkzyziQfSnKPJOcm+bMkz0tysyTHd/dHquroJC9LcsMk30zypCT/nOT5SQ6oqnsleVGStyd5RZI7Jrl+kpO6+4wVffm+JJ3kHt29c9j+30nu3t079+R7AAAA9m1rujhEd788yb8mOaa7j9lFk9sleWmSOwyvxyS5V5ITkvzW0OZTSe7d3XdJ8jtJXtjd3x7en97dO7r79CS/neS93X10kmOS/H5V3WhFX5YzTb6OH3Y9IMnHFU0AADDS8mT9Xgtmo1fV++fuvmAoai5M8p7uniS5IMlhQ5uDkry5qj6R5JQkR8w41wOTnFhV5yV5f6YJ1a1WtXltkscP738x04QLAABgt/Z0jtNV+d7i64bX8vgrV7xfXrG9nO/27feSvK+7Hz4MDXz/jHMtJXlEd/fKnVV182ved/fnq+qLVfVTSY7Od9MnAACAmfY0cfpikptV1cFVtX+Sn91Fm8uTbNuDaxyU5NLh/RN3c96zkjy9qpaSpKruMuN8r850yN6bu/vqPegXAABsKZPl5XV7LZo9Kpy6+zuZLtLwkSTvznQ+0mp/muSdVfW+63iZlyR5UVV9LN+bkL0vyeFVdV5VHZdpMnX9JOdX1YXD9q6cmenCFIbpAQAAoyxNJos38WotVdVRSU7p7nvPa7tz5+Vb68sBAGBDbN++bWmj+zDGFb/58+v2+/jAF791ob6T9XiO08KoqhOT/ErMbQIAAK6FLVU4dffJSU7e6H4AAMCmtIDLhK+XjV6OHAAAYOFtqcQJAADYA5PFW+1uvUicAAAA5pA4AQAA45jjBAAAwCwSJwAAYJSJxAkAAIBZJE4AAMA4EicAAABmkTgBAADjLC/Oc5yq6kFJXpZkvySv7u6TV32+f5LXJzkyyWVJjuvui1d8fqskn0xyUnf/j3nXkzgBAACbSlXtl+RVSR6c5PAkj66qw1c1e3KSr3b37ZKckuTFqz7/gyR/O/aaCicAAGCzOTrJp7v7s9397SR/keRhq9o8LMnrhvdvSXL/qlpKkqr6uST/nOTCsRdUOAEAAOMsT9bvtXuHJPn8iu1Lhn27bNPdVyX5WpKDq+rAJL+Z5HnX5tYVTgAAwFZyUpJTuvuKa3OQxSEAAIBxFmc58kuT/PCK7UOHfbtqc0lVXS/JQZkuEvETSY6tqpckuXGS5ar6Vne/cncXVDgBAACbzblJbl9Vt8m0QHpUksesanNmkickOSfJsUne292TJPe+pkFVnZTkinlFU2KoHgAAMNJkMlm31+4Mc5aeluSsJBcl+cvuvrCqnl9VDx2avSbTOU2fTvLsJCfuyb0vzevUVrZz5+W+HAAA1tz27duWNroPY3z9v/7Muv0+/oE/OWuhvhND9QAAgHEWZ47TujNUDwAAYA6JEwAAMI7ECQAAgFkkTgAAwCgTiRMAAACzSJwAAIBxJE4AAADMInECAADGWd7oDmwciRMAAMAcCicAAIA5DNUDAABGsRw5AAAAM0mcAACAcSROAAAAzLJPF05V9eqqOnwX+59YVa/ciD4BAMCmtbyOrwWzTw/V6+5f2ug+AAAAm99CFk5VdXKSz3f3q4btk5JcleSYJD+Y5PpJntvdZ1TVYUnemeSfktw1yYVJHt/d36iq9yc5obs/WlVPSvKcJP+e5ONJrlzXmwIAgE3OqnqL5/Qkv7Bi+xeSvC7Jw7v7rpkWUC+tqqXh80ryh939Y0m+nuSpK09WVbdI8rwk90xyryT/afgeAADALAtZOHX3x5LcrKpuWVV3TvLVJP+W5IVVdX6Sv0tySJKbD4d8vrvPHt6/IdPiaKWfSPL+7t7Z3d/OtDADAACujS08x2khC6fBm5Mcm+S4TAud45NsT3Jkd+9I8sUkNxzars4Mt26GCAAA7HWLXDidnuRRmRZPb05yUJIvdfd3quqYJLde0fZWVXX34f1jkvzDqnN9OMl9q+rgqrp+kkeubdcBAGDfM1merNtr0Sxs4dTdFybZluTS7v5CktOSHFVVFyR5fJJPrWye5Fer6qJMF4/4o1Xn+kKSk5Kck+TsJBet+Q0AAAD7jKXJZPGquWtjWFXv7d19x7197p07L9/cXw4AAJvC9u3blua32nhfedh91+338U3O+MBCfScLmzgBAAAsioV8jtO10d0XJ9nraRMAAPC9Jgu42t16kTgBAADMoXACAACYY9MP1QMAANaJoXoAAADMInECAABGsTgEAAAAM0mcAACAcSROAAAAzCJxAgAARjHHCQAAgJkkTgAAwCgSJwAAAGaSOAEAAKNInAAAAJhJ4gQAAIwzWdroHmwYiRMAAMAcEicAAGAUc5wAAACYSeEEAAAwh6F6AADAKJNli0MAAAAwg8QJAAAYxeIQAAAAzCRxAgAARpl4AC4AAACzSJwAAIBRzHECAABgJokTAAAwiuc4AQAAMJPECQAAGGUy2egebByJEwAAwBwSJwAAYBRznDaZqnpoVZ240f0AAAC2hk2XOFXV9br7zCRnbnRfAABgK9nKidNCFk5V9fgkJySZJDk/ydVJvpXkLknOrqrzkxzV3U+rqlOTfHP47GZJfjHJ45PcPcmHu/uJwzkfmOR5SfZP8pkkT+ruK9bxtgAAgE1q4YbqVdURSZ6b5Ke6+85Jnjl8dGiSe3T3s3dx2A9mWij9WqZJ1ClJjkjy41W1o6puOpzzAd191yQfTbKr8wAAAPwnC1c4JfmpJG/u7i8nSXd/Zdj/5u6+esYx/7O7J0kuSPLF7r6gu5eTXJjksCQ/meTwTNOq85I8Icmt1/AeAABgnzOZrN9r0SzkUL0Z/mM3n105/Ht5xftrtq+X6VC/d3f3o9eobwAAwD5sEROn9yZ5ZFUdnCRVdZO9cM4PJblnVd1uOOeNqupH98J5AQBgy5gsL63ba9EsXOHU3Rcm+e9JPlBVH0/yB3vhnDuTPDHJm4aFJc5Jcoc9PS8AALA1LE0WcQDhgti583JfDgAAa2779m2LF7Hswmfu+DPr9vv4Rz5x1kJ9JwuXOAEAACyazbQ4BAAAsIEmyxvdg40jcQIAAJhD4gQAAIyyPFmoaUfrSuIEAAAwh8QJAAAYZSJxAgAAYBaJEwAAMMpkWeIEAADADBInAABglMlko3uwcSROAAAAcyicAAAA5jBUDwAAGMXiEAAAAMwkcQIAAEZZ9gBcAAAAZpE4AQAAo0wkTgAAAMwicQIAAEbxAFwAAABmkjgBAACjWFUPAACAmSROAADAKFbVAwAAYCaJEwAAMIpV9QAAAJhJ4gQAAIyySKvqVdWDkrwsyX5JXt3dJ6/6fP8kr09yZJLLkhzX3RdX1dFJ/nRotpTkpO5+27zrSZwAAIBNpar2S/KqJA9OcniSR1fV4auaPTnJV7v7dklOSfLiYf8nkhzV3TuSPCjJn1TV3EBJ4gQAAIyyQKvqHZ3k09392SSpqr9I8rAkn1zR5mFJThrevyXJK6tqqbu/saLNDZOMmrklcQIAADabQ5J8fsX2JcO+Xbbp7quSfC3JwUlSVT9RVRcmuSDJU4bPd0vhBAAAbCnd/eHuPiLJ3ZI8p6puOO8YhRMAADDK8mRp3V5zXJrkh1dsHzrs22WbYQ7TQZkuEvF/dPdFSa5Icsd5F1Q4AQAAm825SW5fVbepqhskeVSSM1e1OTPJE4b3xyZ5b3dPhmOulyRVdeskd0hy8bwLKpwAAIBRJuv42p1hTtLTkpyV5KIkf9ndF1bV86vqoUOz1yQ5uKo+neTZSU4c9t8rycer6rwkb0vy1O7+8rx7X5ps5cf/zrFz5+W+HAAA1tz27dsWZrm63fnQLX9+3X4f/+S/vnWhvpM1TZyq6n5V9fZreczFVXXTteoTAABw3SzQHKd1tymG6lXVUlVtir4CAAD7nrkPwK2qw5K8M8mHktwj04lYf5bkeUluluT4JBcmeUWmq1FcP8lJ3X3GqvPcJMlrk9w2yTeS/HJ3n19VByd5U6brrJ+TZGnFdc9K8uEkRyZ5SFWdmOmSgQckeUt3/+7Q9uLhHA9OclWSX07yoiS3S/L73f3HVXVgkjOS/ODQx+eu7iMAADDbAj0Ad92NTXFul+Slma44cYckj8l0UtUJSX4ryW9nukrF0UmOSfL7VXWjVed4XpKPdfedhmNeP+z/3ST/MKyj/rYkt1pxzO2T/GF3H9Hd/5Lkt7v7qCR3SnLfqrrTiraf6+4dST6Y5NRMV874yeG6SfKtJA/v7rsOfXxpVW3dP3kAAGC0sYXTP3f3Bd29nGm69J7unmT6pN3DkjwwyYnDyhTvT3LDfG8BlEwLrT9Pku5+b6YrXPxAkvskecOw/2+SfHXFMf/S3R9asf0LVfW/knwsyRFJDl/x2TXLD16Q5MPdfXl370xyZVXdONMk64VVdX6Sv8s04br5yPsHAIAtb3kdX4tm7lC9wZUr3i+v2F4eznF1kkd0d688qKr2tDD5jxXnuk2mCdfduvurVXVqpgXa6j6u7N/KPh6fZHuSI7v7O8PwvrlPCAYAANhbCy6cleTp1wx9q6q77KLNBzMtXlJV90vy5e7+epK/z3ToX6rqwZnOQdqVH8i0kPraUJA9+Fr28aAkXxqKpmOS3PpaHg8AAFvaJEvr9lo0e6tw+r1MF1w4v6ouHLZXOynJkcNQuZPz3af4Pi/JfYbjfj7J53Z1ge7+eKZD9D6V5I1Jzr6WfTwtyVFVdUGSxw/nAQAAmMsDcHfDA3ABAFgPm+UBuO+/+SPX7ffx/b745oX6TjwbCQAAYI6xi0MAAABb3PICzj1aLxInAACAORROAAAAcxiqBwAAjLKIy4SvF4kTAADAHBInAABglOWN7sAGkjgBAADMIXECAABGMccJAACAmSROAADAKOY4AQAAMJPECQAAGEXiBAAAwEwSJwAAYBSr6gEAADCTxAkAABhleesGThInAACAeSROAADAKMvmOAEAADCLwgkAAGAOQ/UAAIBRJhvdgQ0kcQIAAJhD4gQAAIyyvNEd2EASJwAAgDkkTgAAwCjLS5YjBwAAYAaJEwAAMIpV9QAAAJhJ4gQAAIxiVT0AAABmkjgBAACjLG/dRfUkTgAAAPNInAAAgFGWs3Ujp02dOFXVFXM+P6yqHrNe/QEAAPZNm7pwGuGwJAonAADYCybr+Fo0+8RQvapaSvKSJA/O9Ht+QXefnuTkJD9WVecleV2SdyX5syQ3yLRofER3/38b02sAAGCz2CcKpyQ/n2RHkjsnuWmSc6vq75OcmOSE7v7ZJKmqVyR5WXefVlU3SLLfRnUYAADYPPaVwuleSd7U3Vcn+WJVfSDJ3ZJ8fVW7c5L8dlUdmuSt0iYAABjPcuRbRHe/MclDk3wzyTuq6qc2uEsAAMAmsK8UTh9MclxV7VdV25PcJ8lHklyeZNs1jarqtkk+290vT3JGkjttRGcBAGAzWl7H16LZV4bqvS3J3ZN8PNPFIX6ju/+tqi5LcnVVfTzJqUn2T/K4qvpOkn9L8sIN6i8AALCJLE0mi7jY32LYufNyXw4AAGtu+/Ztm2L20J8d8th1+338pEvfsFDfyb4yVA8AAGDN7CtD9QAAgDVmVT0AAABmkjgBAACjLOJqd+tF4gQAADCHxAkAABhF4gQAAMBMEicAAGCUiVX1AAAAmEXiBAAAjGKOEwAAADMpnAAAAOYwVA8AABjFUD0AAABmkjgBAACjTDa6AxtI4gQAADCHxAkAABhl2QNwAQAAmEXiBAAAjGJVPQAAAGaSOAEAAKNInAAAAJhJ4gQAAIziOU4AAADMJHECAABG8RwnAAAAZpI4AQAAo1hVDwAAgJkUTgAAAHMYqgcAAIxiOXIAAABmkjgBAACjLG/hzEniBAAAMIfECQAAGMVy5AAAAMy06RKnqnpTkiOS/Fl3n7LR/QEAgK1i685w2kSFU1VdL8lNk9ytu2+30f0BAAC2jnUvnKrqRkn+MsmhSfZL8ntJXpzkqO7+clUdleR/dPf9quqkJD+S5LZJPpdp0nRIVZ2X5OlJ7pDkl5PcIMmnkzyuu79RVTdP8sfDcUnyK939j1X12CTPGNp/OMlTu/vq9bhvAADY7MxxWl8PSvKv3X3n7r5jknfOaX94kgd096OTPDTJZ7p7R3d/MMlbu/tu3X3nJBclefJwzMuTfGDYf9ckF1bVjyU5Lsk9u3tHkquTHL/X7w4AANjnbMRQvQuSvLSqXpzk7d39waraXfszu/ubMz67Y1W9IMmNkxyY5Kxh/08leXySDInS16rqcUmOTHLucL0DknxpT28GAAC2iuWlje7Bxln3wqm7/3dV3TXJQ5K8oKrek+SqfDf9uuGqQ/5jN6c7NcnPdffHq+qJSe63m7ZLSV7X3c+5Lv0GAAC2ro2Y43TLJF/p7jdU1b8n+aUkF2eaBv1tkkdci9NtS/KFqrp+psPuLh32vyfJryT5f6pqv0zTqPckOaOqTunuL1XVTZJs6+5/2Rv3BQAA+7rlBVpXr6oelORlma6b8OruPnnV5/sneX2mdcZlSY7r7our6qeTnJzpugffTvLr3f3eedfbiDlOP57kI8MCD7+b5AVJnpfkZVX10UznHo313zJd5OHsJJ9asf+ZSY6pqguS/FOSw7v7k0mem+RdVXV+kncnucWe3gwAALC+hnDkVUkenOmaCI+uqsNXNXtykq8OK3KfkumCdEny5ST/pbt/PMkTkvz5mGsuTSaLUzUump07L/flAACw5rZv37YpZg/99mGPWbffx//94jfO/E6q6u5JTurunxm2n5Mk3f2iFW3OGtqcMzza6N+SbO/uyYo2S5mmUbfo7iuEtJ8VAAAb20lEQVR315+NSJwAAAD2xCFJPr9i+5Jh3y7bdPdVSb6W5OBVbR6R5H/NK5qSTfQAXAAAgL2lqo7IdPjeA8e0VzgBAACjLNADcC9N8sMrtg/NdxeKW93mkmGo3kGZDstLVR2a5G1JHt/dnxlzQYUTAACw2Zyb5PZVdZtMC6RHJXnMqjZnZrr4wzlJjk3y3u6eVNWNk/xNkhO7++yxF1Q4AQAAoyzKcuTdfVVVPS3JWZkuR/7a7r6wqp6f5KPdfWaS1yT586r6dJKvZFpcJcnTktwuye9U1e8M+x7Y3V/a3TWtqrcbVtUDAGA9bJZV9X7zsEev2+/jF1/8poX6TiROAADAKFs5VbAcOQAAwBwSJwAAYJQFWlVv3UmcAAAA5pA4AQAAoyzKqnobQeIEAAAwh8QJAAAYZevmTRInAACAuSROAADAKFbVAwAAYCaJEwAAMMpkC89ykjgBAADMoXACAACYw1A9AABgFItDAAAAMJPECQAAGGXZ4hAAAADMInECAABG2bp5k8QJAABgLokTAAAwijlOAAAAzCRxAgAARvEcJwAAAGaSOAEAAKNMzHECAABgFokTAAAwijlOAAAAzCRxAgAARjHHacFV1TOq6qKqurSqXnkdjr9fVb19LfoGAADs+zZL4vTUJA8YXkdtcF8AAIAtZuELp6r64yS3TfK3SV67Yv9/SfLcJDdIclmS47v7i1V13yQvG5pNktxneH9gVb0lyR2T/FOSx3b31s0aAQDgWrI4xALr7qck+dckxyT56oqP/iHJT3b3XZL8RZLfGPafkORXu3tHknsn+eaw/y5JnpXk8EwLsXuufe8BAIB9wcIXTrtxaJKzquqCJL+e5Ihh/9lJ/qCqnpHkxt191bD/I919SXcvJzkvyWHr3WEAANjMlieTdXstms1cOL0iySu7+8eT/NckN0yS7j45yS8lOSDJ2VV1h6H9lSuOvTqbYJgiAACwGDZz8XBQkkuH90+4ZmdV/Uh3X5Dkgqq6W5I7JPn3DegfAADsUxYvB1o/mzlxOinJm6vqn5J8ecX+Z1XVJ6rq/CTfyXRRCQAAgOtsabKA4wcXxc6dl/tyAABYc9u3b1va6D6M8ZhbP3zdfh+/8V/etlDfyWZOnAAAANbFZp7jBAAArKPJFp7lJHECAACYQ+IEAACMsrzRHdhAEicAAIA5JE4AAMAoy+Y4AQAAMIvECQAAGMWqegAAAMykcAIAAJjDUD0AAGAUy5EDAAAwk8QJAAAYZTKxOAQAAAAzSJwAAIBRPAAXAACAmSROAADAKFbVAwAAYCaJEwAAMMrEHCcAAABmkTgBAACjWFUPAACAmSROAADAKJOJxAkAAIAZJE4AAMAonuMEAADATBInAABgFM9xAgAAYCaFEwAAwByG6gEAAKN4AC4AAAAzSZwAAIBRPAAXAACAmSROAADAKOY4AQAAMNOmKpyq6orh37esqrcM73dU1UNWtLlfVd1jxfZJVXXC+vcWAAD2LZN1/GfRbMqhet39r0mOHTZ3JDkqyTuG7fsluSLJP65/zwAAgH3RpiycquqwJG9Pctckz09yQFXdK8mbkjwlydVV9dgkT1913I8keVWS7Um+keT/6u5PrWPXAQBg01q2qt7m1N3fTvI7SU7v7h3d/eIkf5zklGH7g6sO+dMkT+/uI5OckOQP17fHAADAZrQpE6froqoOTHKPJG+uqmt2779xPQIAgM1l6+ZNW6hwyjRd+/fu3rHRHQEAADaXTT1Ub3B5km272U6SdPfXk/xzVT0ySapqqaruvD5dBACAzW85k3V7LZp9oXB6X5LDq+q8qjouyf9M8vBh+96r2h6f5MlV9fEkFyZ52Dr3FQAA2ISWJlt4ZYx5du683JcDAMCa275929JG92GMux9yzLr9Pj7n0vct1HeyLyROAAAAa0rhBAAAMMdWWlUPAADYA1t5mo/ECQAAYA6JEwAAMMoiLhO+XiROAAAAc0icAACAUSYSJwAAAGaROAEAAKNYVQ8AAICZJE4AAMAoVtUDAABgJokTAAAwylae46RwAgAANp2qelCSlyXZL8mru/vkVZ/vn+T1SY5MclmS47r74qo6OMlbktwtyand/bQx1zNUDwAAGGU5k3V77U5V7ZfkVUkenOTwJI+uqsNXNXtykq929+2SnJLkxcP+byX5b0lOuDb3rnACAAA2m6OTfLq7P9vd307yF0ketqrNw5K8bnj/liT3r6ql7v6P7v6HTAuo0RROAADAKJN1/GeOQ5J8fsX2JcO+Xbbp7quSfC3Jwdf13hVOAAAAcyicAACAzebSJD+8YvvQYd8u21TV9ZIclOkiEdeJVfUAAIBRlhdnOfJzk9y+qm6TaYH0qCSPWdXmzCRPSHJOkmOTvLe7r/MNSJwAAIBNZZiz9LQkZyW5KMlfdveFVfX8qnro0Ow1SQ6uqk8neXaSE685vqouTvIHSZ5YVZfsYkW+/2RpKz/Eap6dOy/35QAAsOa2b9+2tNF9GOOIm//Euv0+vvCLH16o70TiBAAAMIc5TgAAwCgLNMdp3UmcAAAA5pA4AQAAo4x4MO0+S+IEAAAwh8QJAAAYxRwnAAAAZpI4AQAAo5jjBAAAwEwSJwAAYBRznAAAAJhJ4gQAAIxijtMCqaodVfWQFdsPraoT1+A6V+ztcwIAAPumRUycdiQ5Ksk7kqS7z0xy5ob2CAAA2NL2uHCqqscmeUaSGyT5cJKnJvlakj9K8pAkX0jyW0lekuRWSZ7V3WdW1Q2HNkcluSrJs5OcneT5SQ6oqnsleVGSA5Ic1d1Pq6rDkrw2yU2T7EzypO7+XFWdmuTrw7l+KMlvdPdbqurAJGck+cEk10/y3O4+Y0/vGQAAtqLJZHmju7Bh9mioXlX9WJLjktyzu3ckuTrJ8UlulOS93X1EksuTvCDJTyd5eKaFUZL8apJJd/94kkcned3Qn99Jcnp37+ju01dd8hVJXtfdd0pyWpKXr/jsFknuleRnk5w87PtWkod3912THJPkpVW1tCf3DAAAbD17Osfp/kmOTHJuVZ03bN82ybeTvHNoc0GSD3T3d4b3hw3775XkDUnS3Z9K8i9JfnTO9e6e5I3D+z8fznGNv+7u5e7+ZJKbD/uWkrywqs5P8ndJDlnxGQAAcC0sZ7Jur0Wzp0P1ljJNgJ6zcmdVndDd19ztcpIrk6S7l6tqreZVXbmqX8k0/dqe5Mju/k5VXZzkhmt0fQAAYB+1p4nTe5IcW1U3S5KquklV3XrksR/MtLBJVf1opvOfOtOhfdtmHPOPSR41vD9+OMfuHJTkS0PRdEySsX0DAABWmUwm6/ZaNHtUOA3D4p6b5F3DcLh3ZzrXaIw/TPJ9VXVBktOTPLG7r0zyviSHV9V5VXXcqmOenuRJw7Uel+SZc65xWpKjhms8PsmnRvYNAADg/1haxGpuUezcebkvBwCANbd9+7ZNsYDZoTe547r9Pr7kK59YqO9k4R6ACwAAsGgW8QG4AADAAtrKo9UkTgAAAHNInAAAgFGWJU4AAADMInECAABGmUTiBAAAwAwSJwAAYBSr6gEAADCTwgkAAGAOQ/UAAIBRli0OAQAAwCwSJwAAYBSLQwAAADCTxAkAABhlWeIEAADALBInAABgFHOcAAAAmEniBAAAjOI5TgAAAMwkcQIAAEYxxwkAAICZJE4AAMAonuMEAADATBInAABglIlV9QAAAJhF4QQAADCHoXoAAMAoFocAAABgJokTAAAwigfgAgAAMJPECQAAGMVy5AAAAMwkcQIAAEYxxwkAAICZJE4AAMAoEqc9UFVLVSW5AgAA9lnXKXGqqsOSnJXkw0mOTPKSqnpKkv2TfCbJk7r7iqo6OclDk1yV5F3dfcJw7GuT3DTJzqHt56rq1CRfT3JUkh9K8hvd/ZaqOjDJGUl+MMn1kzy3u88YzvO3Sf4hyT2SXJrkYd39zaq6XZI/TrI9ydVJHtndn6mqX0/yC0M/39bdv3td7h8AALairZs37VnidPskf5jkvkmenOQB3X3XJB9N8uyqOjjJw5Mc0d13SvKC4bhXJHndsO+0JC9fcc5bJLlXkp9NcvKw71tJHj6c+5gkL62qpRV9eFV3H5Hk35M8Yth/2rD/zpkWVV+oqgcO7Y9OsiPJkVV1nz24fwAAYIvYkzlO/9LdH6qqn01yeJKzqypJbpDknCRfy7ToeU1VvT3J24fj7p7k54f3f57kJSvO+dfdvZzkk1V182HfUpIXDkXOcpJDklzz2T9393nD+39KclhVbUtySHe/LUm6+1tJMhROD0zysaH9gZkWUn8/6wa3b9+2NOszAADYaq769qVb9vfxnhRO/zH8eynJu7v70asbVNXRSe6f5NgkT0vyU3POeeWK99f8oRyf6ZC7I7v7O1V1cZIb7qL91UkO2M25l5K8qLv/ZE4fAAAAvsfeWNThQ0nuOcwrSlXdqKp+dJibdFB3vyPJryW589D+H5M8anh/fJIPzjn/QUm+NBRNxyS59e4ad/flSS6pqp8b+rN/VX1/pnOyfnHoV6rqkKq62bW9WQAAYOvZ48Kpu3cmeWKSN1XV+ZkO07tDkm1J3j7s+4ckzx4OeXqSJw37H5fkmXMucVqSo6rqgiSPT/KpEd16XJJnDNf4xyQ/1N3vSvLGJOcM53rL0EcAAIDdWtrKa7EDAACM4flLAAAAcyicAAAA5lA4LbCqunFVPXWj+7G3VNVRVfXy+S3XtA/PqKqLqurSqnrltTz2oVV14ozPrtg7PdzluU+tqmOH988aFju55rN3VNWN98I1dlTVQ0a0u9/weIGFVFWvrqrDd7H/idf2z3st7e6/pUVzXf7Mq+riqrrpWvVpb5v3v9+qOqyqHrNe/VkLVfWmqjq/qn5to/syz578PT0cv1B/T13z31dV3bKq3jK8/56/c4c+32PF9klVdcL69/a628U9rcnfc2v5/7cwz54sR87au3GSp2b6oOFNo6r26+6rV+/v7o9m+oDkjfTUJP9/e+cdbVVxxeEPS6xIEox9GaLiVsACPIwUC4omGhPFGsWGJkaXoi6DxpYISlQk6kKWNRY0GIJd7GANIoKNIurPxBgrErEgCVYwf+x9fffed9t7914ezzXfWm+te86bM2efmT179szsc2ZA/DVUepGZrSRpIjCxXoJVyMnAOGAxgKSyg50K2RYvj/trlF+rIOlXrS1DOZYjXVqmxMbl7WKvvrZGJ+AQ/ANDbQozWwlYG+glabPWlqdCWmSnl3ckvYtvzwJNbe7OwH/xD1q1VXKeqTXsXBu3M4k2QBo41RAzOxQ4Ed8EeDpu/BcCVwJ7AvOAM/FNfzcGTpY00cyOBAbin17fEBgnaThwIbCpmc0EJuMb/94h6a64383ALZLurrHc5wMP45sVfwg8AZwnaVKhZ5S0JGaArsY7uuPN7HNgNLAGvt/WrkBPYKikvcxsp/g/wNfAjpIWmdmpwIHAKsCdks6p5tnynvMqYBPgAeD6rPOd4nht4H1gsKQ3zWwsvolzd3yD59lAg6QTzOxHuBO1JlBV+efJca+kbnE8NPLP/P9EYAPgMTNbIKl/7GvWEOkexLcH6AM8A9wADAfWAQZJmhF7q43G90L7FBgMvA6cC6xmZv2AC/ANq8cA3YCVgWHZemZmKwAC+kh6P45fBXrHlzab++wXAm9JujyOhwFfAf2B74UMZ0u6O8rpQXzT6x7AXOBwSYvN7HFcx541s8HAGcDHwCxy932rK2Z2ODAU1+3Z+D5zxXRpLF4X3fG6Ogr/gmhvYLqkIyPP3fH6XAV4DdfTimZes8qsqH7g5Vi0ziOf7+NtZRN88H6MpNlm1hEYj9uvacQ+fHHfh3Bb0RPYM2age+H77t2WaeOhy+OBPfC6PwbXxc2AUZKuiu0k7iZPJyopgwrLqR1un/fA626EpAm4Ld4ybPGNwCS8/L6DR27sJ+kftZKjiGxrALcAGwErAucBI3E9WmBmDcCfJO0c7WdTvJ7eBLoCG4b8Q/Av3x4T8v8TOCzaz7pAxk4CHCfpqWJ2v07PWcxO/xw4O2T4ALdp8wv1JfF7zVjd6YbbikMlterXsDI2Hrdb2TZ3PHAssCTKekjedZsCl+N7Wi4Gfi2pki8Ml5OnpT7LqpGmAW+rpwBTadqPrEajnetE8X72k8hrPeA0Sbc1p60XsDMXmdmx5NnK6Gd+ETJPkjS0VnJFPg/gX5HuA7wD7C3pU/Pteq7C628JcICk1+rp7yTqRwrVqxFmtiVwENBX0rZ44xiEDxweldQVWASMAHbDB0rnZmWxHbAfsDVwQHSCpwOvSdpW0qnAdfin3zGzDnjjvK8Ocu+Ed8hXAr8FXopBU7FnJJ5zuqRtgBnABOCkOB6AO4bZDAWOj3x2AD4Nx7BzlMW2QE8z25EaIelY4F3cGf8o619jgBslbY1//j47nHAjfHBwCrmMBq6UtBXeudQdSZcR8kvqXyDJZsDFuFO0BT5D3g8v6zMjzSvADpK6A38Azpf0RfyeELo2ATgL19vt8PIaFY5bRpal+MpXpv4HALNaMmgKJuAdSIYDcQd1oKQeIcPF4dgCGHCFpC3xzi0npNXM1scHBX2jDJqE79ULM+uKO3i7hP5ntlwopkvgnXBvfM+7icCluLO7VYS/rB15DojyeJbGLR4qpZx+lKzzYDjwQrSVM4Gb4vw5wJNh5+7EnawMnfG66irpDeAsSQ24rdvJzLbOSvtm2IQpwFh8dn77uC/44LOYTtSCfXHbk7Fbo0KXTgemRPu4FHdyR4esDcDbNZShGD8F3pW0TUyuPFgmfRdcXw7GncVMXzIFn4DrFfr5MnB0XHMZ8ESc7wHMLWP3a04JO/0ksH3Yrr8Bp8X5Jn1JnO+Or9B3wQdifeslc3MpYHNH4o71pVl1lM01wBBJPfHnrToKpUqf5Xjg6+j/DsZt9Qo07UeyKdXPro/bor3wSQpoflvvjJfLTrg+59jKmNwZCHQNGUbUQa7OwOVRdh/jPh2R7+XRrvoA8+rt7yTqRxo41Y7MisozMau3K26sv6Cxg5uDd0pfxu9OWddPlvSBpE+BO/DGmoOkJ4DOZvYD3FjdLumresgt6VpgLdxBGFoqbfxvCXB7/DZgnqRnQu5PCsg5FbgkVlG+G//fPf5eAJ7HnbvOVT5fJfSmMQTnL+SW/a1FZlb74rOEmWuWB16XNCcGNXOBR2KGNVvXOgC3mtmLNDrnhdgdOD3q+XF8hWrjvDTX4ysj4KskN7RUcEkvAOvEOwDb4A7Te8D5sTrzML6asW5c8pakqfF7HE3by4+BxyW9H05KfideT3bB9WYBgKQP43wxXQK4J6uu5ufVYyd88NAFX62aCRxBmc3AC1BOPyqp836Evkt6FOhoZmvhs/zj4vx95Dq8b0h6Ouv4QDN7Hm/nXckd1GbCeubgEzGLYjD+ufm7fO0orhO1oB8wXtISSfPx1fZeBdJNA840s98BPwy7XW/mALuZ2Ugz20HSwjLpJ5aQq5uZTTHf03AQjXZgF3zCjCiDhZS2+8uSjYCHQuZTaZS5UF8CMEPS26HvM8ntb9sMscLRB7fbM/HIjvVrkHU1Pks/Gtv7K8AbwOZl7leqn71L0lJJL9HYnpvb1jN2ppitXIgPeq4zs32JcPcay/W6pJnx+zmgk5m1BzaUdCeApM8kLab1/J1ElaRQvdrRDp+1OCP7pJkNVWN4wFIiXEjSUvPY8wz5IQTFQgpuAg4FfomHWVVLMblXxzsq8DCwRcXSBp+VcAqbIOlCM7sPDweYamY/ifwvkHR1C56jXvyvxP9qHfbxFbmTGas28/rsULSlWcdLaWzr5wGPSRoYoQWPF8mrHR5+pOyTEcoDgKS3zGy+me2Cz5pVOwt9K77CsB4+0BmEhzb0lPRlhHJlyqTS9rI8UUqXsusqvx5XwicmJsfqQUsppx9LKFPnLeSb544Q16H4+zYfRThMtp6XK4dSOrHMkPRXM5sO/Ay438x+EwPJet7zVTPrgdvMEWb2CLk2I78cSunbWGAfSbPMQ8V3LpG2lN1flowBLpGHiu0MDIOifQnk6s8S2q6/swLwcawK1ZJqfZZakl1XmdWb5rb1jL63o4itNA9V3xXvZ07AJwpqKVe+zq1WIu/l0d9JVEBacaodjwD7m9k64O8CmFlzZoR3i2tWA/bBZ9EWAe3z0o3Fww+IWZBqKSb3SHx5+Q/An8ukzUfA+mbWK9K1zze4ZrZpzH6PxN+32AKPUT4qZtgwsw0z96ozT+EDUXCjmB8mUYipedfUgvn4qktHM1sFDw/Ip5BONIcOeOw1RNhnkXwfAoZkQhDMrHuR/K7FZx5LraZUygS8TPfHB1EdgP9E59Sf3BWWjc2sd/w+BA/jyWY6HgbW0cxWBg6oUrbm8CgebtsRvnkvqFqeBvpGrDxmtoaZlZvhbS6V1PkUQt/DeV0g6RPg73g9YGZ74KGHhVgLd3AWxoBsj2bKWEonasEU4CAzWzFW9nfEQ49z2oeZbQL8Sx4+ezcedlhXzGwDYLGkccAoPJTu3/iqATSGBVVCezxcaGVy7dcjwHFxvxUjJLzavq1WZNuuIzIni/QlbYF8m1vQtkf7et3MDgB/Dy9W5aulmnrNtgOb4yvTonT/1Nx+tqVtvaCtDL+ig6T78ZDoTBnWVS5Ji4C3zWyfkGeVmJhuLX8nUSVp4FQjYhBzNjAplnAn07zl9Bl4qNtsPATvWUkf4DNoL5rZqLjPfDwmvcVhURXI3QkPTxkp6WbgCzMbXOkzRmjUQcAYM5sV6fJnik6O55oNfAk8IGkSvmQ+LcIxbqO6QUKlDAEGhyyH0fhOSilOwj+CMQdfqq+aCIc4F9eFyfj7SPlcAzxoZo+18DYXAReY2QvkzsA+BnQxs5lmdhC+MrUyMNvM5sZxISbiK5JV66OkuXh9vyNpHj5wb4gyPpzc8hBe/i/jTvqVeXnNw2ekp+GD3Jerla9S4jn+CDwR+n9JDfJ8Hx/ojg89nUbtHcRK6nwYHos/G4/5zziww4Ed47p98Q8SNEHSLDw05RW8rU8tlK4EpXSiFtyJ2+BZ+AD4NEnvxbklZjbL/HPeBwIvmocCdaPxXa96shUwI+55Dv6OxnBgtJk9i89wV8rv8cmFqeSW4UlA/yjf54AuNejbasUwPFztOWBB1vkmfUkryNYS8m3uPcDAON4hL+0g4OiwJ3OBvau9eZX1egWwQujJBOBISZ8XeKZsmtvPtqitl7CV7YF749yTNL4juizkOgw4Me7xFLBeK/o7iSpp9/XXbSHC5duNeahEg6QTKki7Oh5r3KOCGPdEoq6Yf8TkUkn5HX0979mJrK8PJhKJRCKRSCwL0opTG8LMBuAz52PSoCnR2ph/Vvp2/JPfiUQikUgkEt9q0opTIpFIJBKJRCKRSJQhrTglEolEIpFIJBKJRBnSwCmRSCQSiUQikUgkypAGTolEIpFIJBKJRCJRhjRwSiQSiUQikUgkEokypIFTIpFIJBKJRCKRSJQhDZwSiUQikUgkEolEogz/B5+JWdfJyGnJAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fb24a227eb8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(15,10))\n",
    "sns.heatmap(heatmap[:,0,:],\n",
    "           xticklabels=test_X[1].split(),yticklabels=test_X[1].split())\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}