notebooks/example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FAH GPU performance analysis example\n",
    "\n",
    "**DISCLAIMER: the GPU benchmark data in this repo is experimental and not final. For now, the code and data presented here is intended as an example of parsing performance data from fah-core logs, and of the types of analysis that might be useful to automate intelligent assignment of work units to specific devices.**\n",
    "\n",
    "Here we analyze performance data (`ns_per_day`) parsed from log files created by `fah-core` for the project `PROJ17101`. This project is intended to be a benchmark suite and contains 15 runs representative of the range of workloads run on FAH today."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import os\n",
    "import altair as alt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from fah_log_parser import ParseError, parse, science_log\n",
    "from fah_log_parser.util.pandas import parse_project_logs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parsing a single `science.log` file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "log = parse(science_log, \"../data/PROJ17102/RUN0/CLONE0/results0/science.log\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'linux2 4.19.76-linuxkit'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log.core_header.platform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "platform, device = log.get_active_device()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PlatformInfo(profile='FULL_PROFILE', version='OpenCL 1.2 CUDA 11.0.228', name='NVIDIA CUDA', vendor='NVIDIA Corporation')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "platform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Device(name='GeForce GTX 750 Ti', vendor='NVIDIA Corporation', version='OpenCL 1.2 CUDA', driver_version='450.66')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "174.551"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log.core_log.average_perf_ns_day"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parsing all logs in a project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 5994 entries, 0 to 5993\n",
      "Data columns (total 13 columns):\n",
      " #   Column                 Non-Null Count  Dtype  \n",
      "---  ------                 --------------  -----  \n",
      " 0   run                    5994 non-null   int64  \n",
      " 1   clone                  5994 non-null   int64  \n",
      " 2   gen                    5994 non-null   int64  \n",
      " 3   os                     5994 non-null   object \n",
      " 4   platform_name          5994 non-null   object \n",
      " 5   platform_vendor        5994 non-null   object \n",
      " 6   platform_version       5994 non-null   object \n",
      " 7   device_name            5994 non-null   object \n",
      " 8   device_vendor          5994 non-null   object \n",
      " 9   device_version         5994 non-null   object \n",
      " 10  device_driver_version  14 non-null     object \n",
      " 11  cuda_enabled           5994 non-null   bool   \n",
      " 12  perf_ns_per_day        5903 non-null   float64\n",
      "dtypes: bool(1), float64(1), int64(3), object(8)\n",
      "memory usage: 567.9+ KB\n"
     ]
    }
   ],
   "source": [
    "if os.path.exists(\"checkpoint\"):\n",
    "    data = pd.read_feather(\"checkpoint\")\n",
    "else:\n",
    "    data = parse_project_logs(\"../data/PROJ17101/\", sample=10000)\n",
    "    data.reset_index.to_feather(\"checkpoint\")\n",
    "    \n",
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<div id=\"altair-viz-fe07bd347d0747bfaa07e255227ee357\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "  (function(spec, embedOpt){\n",
       "    let outputDiv = document.currentScript.previousElementSibling;\n",
       "    if (outputDiv.id !== \"altair-viz-fe07bd347d0747bfaa07e255227ee357\") {\n",
       "      outputDiv = document.getElementById(\"altair-viz-fe07bd347d0747bfaa07e255227ee357\");\n",
       "    }\n",
       "    const paths = {\n",
       "      \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
       "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
       "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm//vega-lite@4.8.1?noext\",\n",
       "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm//vega-embed@6?noext\",\n",
       "    };\n",
       "\n",
       "    function loadScript(lib) {\n",
       "      return new Promise(function(resolve, reject) {\n",
       "        var s = document.createElement('script');\n",
       "        s.src = paths[lib];\n",
       "        s.async = true;\n",
       "        s.onload = () => resolve(paths[lib]);\n",
       "        s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
       "        document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "      });\n",
       "    }\n",
       "\n",
       "    function showError(err) {\n",
       "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
       "      throw err;\n",
       "    }\n",
       "\n",
       "    function displayChart(vegaEmbed) {\n",
       "      vegaEmbed(outputDiv, spec, embedOpt)\n",
       "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
       "    }\n",
       "\n",
       "    if(typeof define === \"function\" && define.amd) {\n",
       "      requirejs.config({paths});\n",
       "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
       "    } else if (typeof vegaEmbed === \"function\") {\n",
       "      displayChart(vegaEmbed);\n",
       "    } else {\n",
       "      loadScript(\"vega\")\n",
       "        .then(() => loadScript(\"vega-lite\"))\n",
       "        .then(() => loadScript(\"vega-embed\"))\n",
       "        .catch(showError)\n",
       "        .then(() => displayChart(vegaEmbed));\n",
       "    }\n",
       "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300}}, \"data\": {\"name\": \"data-d18b8d2ed7e7e999a5587dc416eacac5\"}, \"mark\": \"bar\", \"encoding\": {\"x\": {\"type\": \"quantitative\", \"aggregate\": \"mean\", \"field\": \"z_score_by_run\"}, \"y\": {\"type\": \"nominal\", \"field\": \"device_name\", \"sort\": \"-x\"}}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v4.8.1.json\", \"datasets\": {\"data-d18b8d2ed7e7e999a5587dc416eacac5\": [{\"device_name\": \"A100-SXM4-40GB\", \"z_score_by_run\": 2.145196017829584}, {\"device_name\": \"Baffin\", \"z_score_by_run\": -1.139362234187642}, {\"device_name\": \"Capeverde\", \"z_score_by_run\": -1.2949761460022684}, {\"device_name\": \"Ellesmere\", \"z_score_by_run\": -0.8271660875155432}, {\"device_name\": \"GRID V100D-8C\", \"z_score_by_run\": -0.7026657223897183}, {\"device_name\": \"GeForce GT 1030\", \"z_score_by_run\": -1.1455901555150905}, {\"device_name\": \"GeForce GTX 1050\", \"z_score_by_run\": -1.0507238229689162}, {\"device_name\": \"GeForce GTX 1050 Ti\", \"z_score_by_run\": -0.9333977807661279}, {\"device_name\": \"GeForce GTX 1060\", \"z_score_by_run\": -0.747159113284701}, {\"device_name\": \"GeForce GTX 1060 3GB\", \"z_score_by_run\": -0.7082118456628782}, {\"device_name\": \"GeForce GTX 1060 6GB\", \"z_score_by_run\": -0.7032884903088997}, {\"device_name\": \"GeForce GTX 1070\", \"z_score_by_run\": -0.4453830778300902}, {\"device_name\": \"GeForce GTX 1070 Ti\", \"z_score_by_run\": -0.3708319629499746}, {\"device_name\": \"GeForce GTX 1080\", \"z_score_by_run\": -0.34156477362776444}, {\"device_name\": \"GeForce GTX 1080 Ti\", \"z_score_by_run\": 0.032718531092282295}, {\"device_name\": \"GeForce GTX 1650\", \"z_score_by_run\": -0.8136764431789371}, {\"device_name\": \"GeForce GTX 1650 SUPER\", \"z_score_by_run\": -0.6152462900142383}, {\"device_name\": \"GeForce GTX 1660\", \"z_score_by_run\": -0.5177562946603633}, {\"device_name\": \"GeForce GTX 1660 SUPER\", \"z_score_by_run\": -0.4551191680081121}, {\"device_name\": \"GeForce GTX 1660 Ti\", \"z_score_by_run\": -0.3975934151209225}, {\"device_name\": \"GeForce GTX 750 Ti\", \"z_score_by_run\": -1.0808565030066177}, {\"device_name\": \"GeForce GTX 950\", \"z_score_by_run\": -0.985348758420641}, {\"device_name\": \"GeForce GTX 960\", \"z_score_by_run\": -0.9290434642992594}, {\"device_name\": \"GeForce GTX 970\", \"z_score_by_run\": -0.7320794399257827}, {\"device_name\": \"GeForce GTX 980\", \"z_score_by_run\": -0.6259570098179309}, {\"device_name\": \"GeForce GTX 980 Ti\", \"z_score_by_run\": -0.3671029828166222}, {\"device_name\": \"GeForce RTX 2060\", \"z_score_by_run\": -0.21334760797276087}, {\"device_name\": \"GeForce RTX 2060 SUPER\", \"z_score_by_run\": -0.016913762393887012}, {\"device_name\": \"GeForce RTX 2070\", \"z_score_by_run\": 0.018311490937226973}, {\"device_name\": \"GeForce RTX 2070 SUPER\", \"z_score_by_run\": 0.19429679723885587}, {\"device_name\": \"GeForce RTX 2080\", \"z_score_by_run\": 0.2716797202890408}, {\"device_name\": \"GeForce RTX 2080 SUPER\", \"z_score_by_run\": 0.4122777811879788}, {\"device_name\": \"GeForce RTX 2080 Ti\", \"z_score_by_run\": 0.7415944454372394}, {\"device_name\": \"Hainan\", \"z_score_by_run\": -1.3181769553186478}, {\"device_name\": \"Hawaii\", \"z_score_by_run\": -0.8523494897604232}, {\"device_name\": \"Quadro P4000\", \"z_score_by_run\": -0.48787732249603927}, {\"device_name\": \"Quadro RTX 6000\", \"z_score_by_run\": 0.8669239472377674}, {\"device_name\": \"Quadro RTX 8000\", \"z_score_by_run\": 0.8940214751252451}, {\"device_name\": \"TITAN V\", \"z_score_by_run\": 0.5426901833253283}, {\"device_name\": \"Tahiti\", \"z_score_by_run\": -0.9187383198385272}, {\"device_name\": \"Tesla K80\", \"z_score_by_run\": -0.8251577840464822}, {\"device_name\": \"Tesla M60\", \"z_score_by_run\": -0.679794747781278}, {\"device_name\": \"Tesla P100-PCIE-12GB\", \"z_score_by_run\": -0.046336566433891356}, {\"device_name\": \"Tesla P100-PCIE-16GB\", \"z_score_by_run\": 0.0843297592318133}, {\"device_name\": \"Tesla T4\", \"z_score_by_run\": -0.22255959950712037}, {\"device_name\": \"Tesla V100-PCIE-16GB\", \"z_score_by_run\": 0.5359655831078726}, {\"device_name\": \"Tesla V100-PCIE-32GB\", \"z_score_by_run\": 0.8350797030782945}, {\"device_name\": \"Tesla V100-SXM2-16GB\", \"z_score_by_run\": 1.143981722938211}, {\"device_name\": \"Tesla V100-SXM2-32GB\", \"z_score_by_run\": 1.1600875511388955}, {\"device_name\": \"Tesla V100-SXM3-32GB-H\", \"z_score_by_run\": 1.3737505408382924}, {\"device_name\": \"Tonga\", \"z_score_by_run\": -0.9360538245195453}, {\"device_name\": \"gfx1010\", \"z_score_by_run\": -0.36858894825280625}, {\"device_name\": \"gfx900\", \"z_score_by_run\": -0.3902427103654098}, {\"device_name\": \"gfx902\", \"z_score_by_run\": -1.1507768432817191}, {\"device_name\": \"gfx906\", \"z_score_by_run\": -0.10566153881447786}]}}, {\"mode\": \"vega-lite\"});\n",
       "</script>"
      ],
      "text/plain": [
       "alt.Chart(...)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = (data\n",
    " .set_index(['run', 'clone', 'gen', 'device_name'])\n",
    " ['perf_ns_per_day']\n",
    " .groupby('device_name').filter(lambda x: len(x) > 10)\n",
    " .groupby('run').transform(lambda x: (x - x.mean()) / x.std()).rename('z_score_by_run')\n",
    " .groupby('device_name').mean()\n",
    ").reset_index()\n",
    "\n",
    "alt.Chart(df).mark_bar().encode(\n",
    "    x=alt.X(\"z_score_by_run:Q\", aggregate='mean'),\n",
    "    y=alt.Y(\"device_name:N\", sort='-x')\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<div id=\"altair-viz-4be1edaf4b574b37bb7ca1d054cdf541\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "  (function(spec, embedOpt){\n",
       "    let outputDiv = document.currentScript.previousElementSibling;\n",
       "    if (outputDiv.id !== \"altair-viz-4be1edaf4b574b37bb7ca1d054cdf541\") {\n",
       "      outputDiv = document.getElementById(\"altair-viz-4be1edaf4b574b37bb7ca1d054cdf541\");\n",
       "    }\n",
       "    const paths = {\n",
       "      \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
       "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
       "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm//vega-lite@4.8.1?noext\",\n",
       "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm//vega-embed@6?noext\",\n",
       "    };\n",
       "\n",
       "    function loadScript(lib) {\n",
       "      return new Promise(function(resolve, reject) {\n",
       "        var s = document.createElement('script');\n",
       "        s.src = paths[lib];\n",
       "        s.async = true;\n",
       "        s.onload = () => resolve(paths[lib]);\n",
       "        s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
       "        document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "      });\n",
       "    }\n",
       "\n",
       "    function showError(err) {\n",
       "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
       "      throw err;\n",
       "    }\n",
       "\n",
       "    function displayChart(vegaEmbed) {\n",
       "      vegaEmbed(outputDiv, spec, embedOpt)\n",
       "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
       "    }\n",
       "\n",
       "    if(typeof define === \"function\" && define.amd) {\n",
       "      requirejs.config({paths});\n",
       "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
       "    } else if (typeof vegaEmbed === \"function\") {\n",
       "      displayChart(vegaEmbed);\n",
       "    } else {\n",
       "      loadScript(\"vega\")\n",
       "        .then(() => loadScript(\"vega-lite\"))\n",
       "        .then(() => loadScript(\"vega-embed\"))\n",
       "        .catch(showError)\n",
       "        .then(() => displayChart(vegaEmbed));\n",
       "    }\n",
       "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300}}, \"data\": {\"name\": \"data-23ed9abc5c3d56e09f5878349066aa75\"}, \"mark\": {\"type\": \"point\", \"filled\": true}, \"encoding\": {\"color\": {\"type\": \"nominal\", \"field\": \"run\"}, \"x\": {\"type\": \"quantitative\", \"field\": \"mean_ns_per_day\", \"scale\": {\"type\": \"log\"}}, \"y\": {\"type\": \"ordinal\", \"field\": \"device_name\", \"sort\": \"-x\"}}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v4.8.1.json\", \"datasets\": {\"data-23ed9abc5c3d56e09f5878349066aa75\": [{\"run\": 0, \"device_name\": \"A100-SXM4-40GB\", \"mean_ns_per_day\": 1096.1145000000001}, {\"run\": 0, \"device_name\": \"GeForce GTX 1080 Ti\", \"mean_ns_per_day\": 577.326888888889}, {\"run\": 0, \"device_name\": \"GeForce RTX 2080 Ti\", \"mean_ns_per_day\": 855.5646666666667}, {\"run\": 0, \"device_name\": \"Hainan\", \"mean_ns_per_day\": 29.992668260869564}, {\"run\": 0, \"device_name\": \"Tesla M60\", \"mean_ns_per_day\": 329.34882352941173}, {\"run\": 0, \"device_name\": \"Tesla V100-SXM2-16GB\", \"mean_ns_per_day\": 899.6408478260867}, {\"run\": 1, \"device_name\": \"A100-SXM4-40GB\", \"mean_ns_per_day\": 1092.5248387096774}, {\"run\": 1, \"device_name\": \"GeForce GTX 1080 Ti\", \"mean_ns_per_day\": 608.9757142857144}, {\"run\": 1, \"device_name\": \"GeForce RTX 2080 Ti\", \"mean_ns_per_day\": 953.0132916666666}, {\"run\": 1, \"device_name\": \"Hainan\", \"mean_ns_per_day\": 28.61730066666667}, {\"run\": 1, \"device_name\": \"Tesla M60\", \"mean_ns_per_day\": 329.6642727272727}, {\"run\": 1, \"device_name\": \"Tesla V100-SXM2-16GB\", \"mean_ns_per_day\": 901.7158292682925}, {\"run\": 2, \"device_name\": \"A100-SXM4-40GB\", \"mean_ns_per_day\": 2226.186}, {\"run\": 2, \"device_name\": \"GeForce GTX 1080 Ti\", \"mean_ns_per_day\": 1116.248875}, {\"run\": 2, \"device_name\": \"GeForce RTX 2080 Ti\", \"mean_ns_per_day\": 1425.9}, {\"run\": 2, \"device_name\": \"Hainan\", \"mean_ns_per_day\": 44.620892000000005}, {\"run\": 2, \"device_name\": \"Tesla M60\", \"mean_ns_per_day\": 626.779090909091}, {\"run\": 2, \"device_name\": \"Tesla V100-SXM2-16GB\", \"mean_ns_per_day\": 1974.3854545454556}, {\"run\": 3, \"device_name\": \"A100-SXM4-40GB\", \"mean_ns_per_day\": 2974.365217391304}, {\"run\": 3, \"device_name\": \"GeForce GTX 1080 Ti\", \"mean_ns_per_day\": 1456.571928571428}, {\"run\": 3, \"device_name\": \"GeForce RTX 2080 Ti\", \"mean_ns_per_day\": 2215.055263157894}, {\"run\": 3, \"device_name\": \"Hainan\", \"mean_ns_per_day\": 66.89308636363637}, {\"run\": 3, \"device_name\": \"Tesla M60\", \"mean_ns_per_day\": 814.0830000000001}, {\"run\": 3, \"device_name\": \"Tesla V100-SXM2-16GB\", \"mean_ns_per_day\": 2300.2917073170725}, {\"run\": 4, \"device_name\": \"A100-SXM4-40GB\", \"mean_ns_per_day\": 553.8044545454545}, {\"run\": 4, \"device_name\": \"GeForce GTX 1080 Ti\", \"mean_ns_per_day\": 178.23370588235298}, {\"run\": 4, \"device_name\": \"GeForce RTX 2080 Ti\", \"mean_ns_per_day\": 330.4836666666666}, {\"run\": 4, \"device_name\": \"Hainan\", \"mean_ns_per_day\": 6.395293928571429}, {\"run\": 4, \"device_name\": \"Tesla M60\", \"mean_ns_per_day\": 79.38350000000001}, {\"run\": 4, \"device_name\": \"Tesla V100-SXM2-16GB\", \"mean_ns_per_day\": 386.2433095238096}, {\"run\": 5, \"device_name\": \"A100-SXM4-40GB\", \"mean_ns_per_day\": 443.70761290322565}, {\"run\": 5, \"device_name\": \"GeForce GTX 1080 Ti\", \"mean_ns_per_day\": 148.50517647058825}, {\"run\": 5, \"device_name\": \"GeForce RTX 2080 Ti\", \"mean_ns_per_day\": 268.4251428571428}, {\"run\": 5, \"device_name\": \"Hainan\", \"mean_ns_per_day\": 5.8385196}, {\"run\": 5, \"device_name\": \"Tesla M60\", \"mean_ns_per_day\": 68.17677272727273}, {\"run\": 5, \"device_name\": \"Tesla V100-SXM2-16GB\", \"mean_ns_per_day\": 296.00283673469386}, {\"run\": 6, \"device_name\": \"A100-SXM4-40GB\", \"mean_ns_per_day\": 804.5451333333333}, {\"run\": 6, \"device_name\": \"GeForce GTX 1080 Ti\", \"mean_ns_per_day\": 330.7946400000001}, {\"run\": 6, \"device_name\": \"GeForce RTX 2080 Ti\", \"mean_ns_per_day\": 396.4988347826087}, {\"run\": 6, \"device_name\": \"Hainan\", \"mean_ns_per_day\": 10.290648421052634}, {\"run\": 6, \"device_name\": \"Tesla M60\", \"mean_ns_per_day\": 148.8842857142857}, {\"run\": 6, \"device_name\": \"Tesla V100-SXM2-16GB\", \"mean_ns_per_day\": 656.9780869565217}, {\"run\": 7, \"device_name\": \"A100-SXM4-40GB\", \"mean_ns_per_day\": 1096.898285714286}, {\"run\": 7, \"device_name\": \"GeForce GTX 1080 Ti\", \"mean_ns_per_day\": 363.2566111111111}, {\"run\": 7, \"device_name\": \"GeForce RTX 2080 Ti\", \"mean_ns_per_day\": 662.7508181818183}, {\"run\": 7, \"device_name\": \"Hainan\", \"mean_ns_per_day\": 12.5167665}, {\"run\": 7, \"device_name\": \"Tesla M60\", \"mean_ns_per_day\": 157.19946153846155}, {\"run\": 7, \"device_name\": \"Tesla V100-SXM2-16GB\", \"mean_ns_per_day\": 771.7125909090909}, {\"run\": 8, \"device_name\": \"A100-SXM4-40GB\", \"mean_ns_per_day\": 374.1055925925926}, {\"run\": 8, \"device_name\": \"GeForce GTX 1080 Ti\", \"mean_ns_per_day\": 135.3826}, {\"run\": 8, \"device_name\": \"GeForce RTX 2080 Ti\", \"mean_ns_per_day\": 217.8575}, {\"run\": 8, \"device_name\": \"Hainan\", \"mean_ns_per_day\": 5.4632056521739125}, {\"run\": 8, \"device_name\": \"Tesla M60\", \"mean_ns_per_day\": 57.35707500000001}, {\"run\": 8, \"device_name\": \"Tesla V100-SXM2-16GB\", \"mean_ns_per_day\": 269.9177738095238}, {\"run\": 9, \"device_name\": \"A100-SXM4-40GB\", \"mean_ns_per_day\": 1017.5256999999999}, {\"run\": 9, \"device_name\": \"GeForce GTX 1080 Ti\", \"mean_ns_per_day\": 642.2585333333334}, {\"run\": 9, \"device_name\": \"GeForce RTX 2080 Ti\", \"mean_ns_per_day\": 693.5590000000001}, {\"run\": 9, \"device_name\": \"Hainan\", \"mean_ns_per_day\": 54.00292842105264}, {\"run\": 9, \"device_name\": \"Tesla M60\", \"mean_ns_per_day\": 470.8969166666666}, {\"run\": 9, \"device_name\": \"Tesla V100-SXM2-16GB\", \"mean_ns_per_day\": 749.5004186046511}, {\"run\": 10, \"device_name\": \"A100-SXM4-40GB\", \"mean_ns_per_day\": 140.7019687499999}, {\"run\": 10, \"device_name\": \"GeForce GTX 1080 Ti\", \"mean_ns_per_day\": 36.1036}, {\"run\": 10, \"device_name\": \"GeForce RTX 2080 Ti\", \"mean_ns_per_day\": 66.36708333333334}, {\"run\": 10, \"device_name\": \"Hainan\", \"mean_ns_per_day\": 1.7345724999999999}, {\"run\": 10, \"device_name\": \"Tesla M60\", \"mean_ns_per_day\": 15.531512499999998}, {\"run\": 10, \"device_name\": \"Tesla V100-SXM2-16GB\", \"mean_ns_per_day\": 81.81513157894736}, {\"run\": 11, \"device_name\": \"A100-SXM4-40GB\", \"mean_ns_per_day\": 112.23034615384611}, {\"run\": 11, \"device_name\": \"GeForce GTX 1080 Ti\", \"mean_ns_per_day\": 31.867242105263156}, {\"run\": 11, \"device_name\": \"GeForce RTX 2080 Ti\", \"mean_ns_per_day\": 53.596707692307696}, {\"run\": 11, \"device_name\": \"Hainan\", \"mean_ns_per_day\": 1.3319934999999998}, {\"run\": 11, \"device_name\": \"Tesla M60\", \"mean_ns_per_day\": 12.083495238095237}, {\"run\": 11, \"device_name\": \"Tesla V100-SXM2-16GB\", \"mean_ns_per_day\": 63.92900869565219}, {\"run\": 12, \"device_name\": \"A100-SXM4-40GB\", \"mean_ns_per_day\": 1352.196666666667}, {\"run\": 12, \"device_name\": \"GeForce GTX 1080 Ti\", \"mean_ns_per_day\": 534.857090909091}, {\"run\": 12, \"device_name\": \"GeForce RTX 2080 Ti\", \"mean_ns_per_day\": 722.3207499999999}, {\"run\": 12, \"device_name\": \"Hainan\", \"mean_ns_per_day\": 18.139843750000004}, {\"run\": 12, \"device_name\": \"Tesla M60\", \"mean_ns_per_day\": 217.7441875}, {\"run\": 12, \"device_name\": \"Tesla V100-SXM2-16GB\", \"mean_ns_per_day\": 925.5799523809524}, {\"run\": 13, \"device_name\": \"A100-SXM4-40GB\", \"mean_ns_per_day\": 563.8571818181819}, {\"run\": 13, \"device_name\": \"GeForce GTX 1080 Ti\", \"mean_ns_per_day\": 166.404}, {\"run\": 13, \"device_name\": \"GeForce RTX 2080 Ti\", \"mean_ns_per_day\": 276.12611764705883}, {\"run\": 13, \"device_name\": \"Hainan\", \"mean_ns_per_day\": 6.686109166666667}, {\"run\": 13, \"device_name\": \"Tesla M60\", \"mean_ns_per_day\": 66.29112}, {\"run\": 13, \"device_name\": \"Tesla V100-SXM2-16GB\", \"mean_ns_per_day\": 354.2783478260868}, {\"run\": 14, \"device_name\": \"A100-SXM4-40GB\", \"mean_ns_per_day\": 891.29976}, {\"run\": 14, \"device_name\": \"GeForce GTX 1080 Ti\", \"mean_ns_per_day\": 299.088}, {\"run\": 14, \"device_name\": \"GeForce RTX 2080 Ti\", \"mean_ns_per_day\": 458.8765714285714}, {\"run\": 14, \"device_name\": \"Hainan\", \"mean_ns_per_day\": 12.010407333333331}, {\"run\": 14, \"device_name\": \"Tesla M60\", \"mean_ns_per_day\": 117.40576470588236}, {\"run\": 14, \"device_name\": \"Tesla V100-SXM2-16GB\", \"mean_ns_per_day\": 601.1317560975612}]}}, {\"mode\": \"vega-lite\"});\n",
       "</script>"
      ],
      "text/plain": [
       "alt.Chart(...)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = (data\n",
    " .set_index(['run', 'clone', 'gen', 'device_name'])\n",
    " ['perf_ns_per_day']\n",
    " .groupby(['run', 'device_name']).filter(lambda x: len(x) > 10)\n",
    " .groupby('device_name').filter(lambda x: len(x.groupby('run')) == 15)\n",
    " .groupby(['run', 'device_name']).mean()\n",
    " .rename('mean_ns_per_day')\n",
    ").reset_index()\n",
    "\n",
    "alt.Chart(df).mark_point(filled=True).encode(\n",
    "    x=alt.X('mean_ns_per_day:Q', scale=alt.Scale(type='log')),\n",
    "    y=alt.Y('device_name:O', sort='-x'),\n",
    "    color='run:N',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<seaborn.matrix.ClusterGrid at 0x7faee83de1c0>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsgAAALICAYAAABiqwZ2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAA5u0lEQVR4nO3deZhlV1k37N9DQugwRpA5kUQGmYQQinkGRQQkgKggIiAaRWRQUIL4STugUXgFUcQ3Am9Ao0wyRGQ0EBQZiySQhDAHIRFkngIZ+/n+OLthWVR1dyVddbqq7vu66qqz195rP2tXVTq/WrX2PtXdAQAAZi4z7wEAAMC+REAGAICBgAwAAAMBGQAABgIyAAAMBGQAABgIyAAAMBCQAQBgICADAMBg/9UcfOGXPrVPve3eZX/wh2veYwAAYHNZVUDOjovXaBgAALBvWF1AvviiNRoGAADsG1YVkFtAZg8sLCwcnWTbvMfBhnLe4uLiMfMeBAAkq55BvnCNhsEms21xcXH7vAfBxrGwsLB93mMAgJ0ssQAAgIElFgAAMLDEAgAABpZYAADAQEAGAIDB6tYg77DEAgCAzc0MMgAADNykBwAAAzPIAAAwWF1AvkhABgBgc1vlG4VYYgEAwOZmiQUAAAwEZAAAGAjIAAAwEJABAGAgIAMAwEBABgCAgecgAwDAYJUzyBev0TAAAGDfYIkFAAAMzCADAMDAGmQAABisKiC3GWQAADY5a5ABAGCwyiUWZpABANjcNv1NelV1UJIXJrl5kk7yS9397rkOCgCAfdZWmEH+yyRv6u6HVNUBSS4/7wEBALDv2tQzyFV1lSR3TfKoJOnuC5JcMM8xAQCwb1vdUyw23gzyYUm+mOT/VdUtk3wgyRO7+9z5DgsAgH3VZVZ19MUX71MfVXVUVS0OH0ctGfH+SY5I8oLuvlWSc5McvXe+dAAAbEYbeg1ydx+b5NhdHHJ2krO7+73T9qsiIAMAsAubeg1yd3++qj5bVT/S3R9Ncq8kH573uAAA2Hdt9jXISfL4JMdPT7D4VJJHz3k8AADswzb1DHKSdPepSRbmPQ4AADaGVc4g71ircQAAwD5hQ9+kBwAAe9vqAvKOXqNhAADAvmF1SywutsQCAIDNbZVLLARkAAA2NzfpAQDAYJUB2RpkAAA2t1UusRCQAQDY3MwgAwDAQEAGAIDBKgPyWg0DAAD2DQIyAAAMVhWQdwjIAABscqt8J71aq3EAAMA+YZUzyAIyAACb2+oCshlk+D4LCwtHJ9k273FscIcuLCxsn/cgNrDzFhcXj5n3IAA2CzPIcOltW1xc3D7vQbB1+eUCYO8ygwwAAAMBGQAABqsMyJdZq3EAAMA+YVUB+WIzyAAAbHJmkAEAYGANMgAADFa3xGKHGWQAADa3VQZkM8gAAGxuq1tisQEDclXtl2QxyTndff95jwcAgH3bVlhi8cQkZya58rwHAgDAvm91Abk31gxyVR2c5H5Jnpnkt+Y8HAAANoANPYNcVUclOWpoOra7jx22n5vkd5JcaT3HBQDAxrW6gJx9awZ5CsPHLrevqu6f5Avd/YGquvt6jgsAgI1rVQH5oo21xOJOSR5QVfdNsi3JlavqH7r7F+Y8LgAA9mEbegZ5V7r7aUmeliTTDPJThGMAAHZn0wZkAAC4JLZEQO7uk5KcNOdhAACwAaxuDXJtzIAMAAB7apUzyADzsbCwcHRmN9zy/Q5dWFjYPu9B7MPOW1xcPGbegwA2jtUFZDPIwPxsW1xc3D7vQbDx+OUBWK1VLrFYq2EAAMC+YUvcpAcAAHvKDDIAAAxWuQZ5rYYBAAD7htXNIK/VKAAAYB9hBhkAAAarCsg71moUAACwjzCDDAAAA2uQAQBgYAYZAAAGq3yjEAAA2NxWucSi12ocAACwTzCDDAAAg1W+1bQZZAAANjczyAAAMFhlQDaDDADA5uYmPQAAGFhiAQAAA0ssAABgICADsEcWFhaOTrJt3uO4BA5dWFjYPu9BXELnLS4uHjPvQcBWYw0yAHtq2+Li4vZ5D2Ir2cDBHjY0M8gAADDY1AG5qg5J8tIk10zSSY7t7r+c76gAANiXrS4g98YKyEkuSvLk7j65qq6U5ANV9dbu/vC8BwYAwL5plTPIO9ZqHGuiuz+X5HPT629W1ZlJrptEQAYAYFkb+ia9qjoqyVFD07HdfewKxx6a5FZJ3rsOQwMAYIPa0GuQpzC8bCAeVdUVk/xzkid19zfWfGAAAGxYq1yDvLGWWCRJVV02s3B8fHe/et7jAQBg37ahZ5B3p6oqyYuSnNndfzHv8QAAsO/b7DPId0ryiCSnVdWpU9vvdvcb5jckAAD2ZZt6Brm735mk5j0OAAA2js0+gwwAAKuyqZ+DDABrYWFh4egk29ah1KELCwvb16HOeYuLi8esQx3YEDb7O+kBwFrYtri4uH3eg9hb1imEw4axyjcKMYMMAMDmZg0yAAAMVhWQdwjIAABscmaQIZf6hptLcxONG2MAYB/jKRYwM5cbbtwYA6zGGj49Yy2flmEigA3HDDIAbBwb7ukZJgLYiARkgE1kjZ/Pu1azjGYYgX2KgAysyjq+QcJS6/WGCaONGNzMMAJcSgIyy3LTGruw4QLYJSW4AWxNqwzIF6/VONj3uGkNANiSzCADAMBgdQF5h4AMAMDmtqqA3Om1GgfAmrkUa+qtp99D7lsANhNLLICtYN3X1G/B9fTuWwA2DUssYAsy2wcAKzODDFuT2T4AWIEZZAAAGJhBBgB2ybIstppVBeQdAjIAbEWWZbGlrC4gW2IBAMAmV92ebbxZXfkKPzyXb+63Lzx/HmXzhh+4y1zqXv1y31n3mtszn7d9v0xqLnXPn9P1XjSnv5rdfL+D5lL33Dl9nS+f/da95oPPu2jdaybJOy93SVcpXDqHXDiXsrloPv9k5FHn/MOcKrNZXGbeAwAAgH2JgAwAAAMBGQAABgIyAAAMBGQAABgIyAAAMBCQAQBgICADAMBAQAYAgIGADAAAAwEZAAAGAjIAAAwEZAAAGAjIAAAwEJABAGAgIAMAwEBABgCAgYAMAAADARkAAAYCMgAADARkAAAYCMgAADAQkAEAYCAgAwDAQEAGAICBgAwAAAMBGQAABgIyAAAMBGQAABgIyAAAMBCQAQBgICADAMBAQAYAgIGADAAAAwEZAAAGAjIAAAwEZAAAGAjIAAAwEJABAGAgIAMAwEBABgC2lKr61pLtR1XVX++mzwOq6ui1HRn7iv3nPQAAgH1dd5+Q5IR5j4P1YQYZAGBSVT9VVe+tqlOq6t+q6ppT+3dnmavquKp6XlW9q6o+VVUPmdqvWFUnVtXJVXVaVR05tR9aVWdW1d9V1RlV9ZaqOnB+V8nuCMgAwFZzYFWduvMjyR8O+96Z5PbdfaskL0vyOyuc49pJ7pzk/kmOmdrOS/Kg7j4iyT2S/J+qqmnfDZM8v7tvluRrSX56L14Pe9klWmKxsLBwdJJte3ksAACXWlUdleSooenY7j522P5Odx8+HP+oJAvT5sFJXl5V105yQJKzVijz2u7ekeTDO2eZk1SSP6mquybZkeS6SXbuO6u7T51efyDJoau/MtbLJV2DvG1xcXH73hwIe9+Vr/DDz5j3GABgvU1h+NjdHri8v0ryF919QlXdPcn2FY47f3i9c5b44UmunuTW3X1hVX0635tQHI+/OIklFvswSywAAL7nKknOmV4/8hL0/cIUju+R5Hp7dWSsGwEZAOB7tid5ZVV9IMmXVtn3+CQLVXVakl9M8pG9PDbWice8AQBbSndfccn2cUmOm16/LsnrlukzHvOo5c7X3V9KcocVyt58OP7Zl2zkrBczyAAAMBCQAQBgICADAMBAQAYAgIGADAAAAwEZAAAGAjIAAAwEZAAAGAjIAAAwEJABAGAgIAMAwEBABgCAgYAMAAADARkAAAYCMgAADARkAAAYCMgAADAQkAEAYCAgAwDAQEAGAICBgAwAAAMBGQAABgIyAAAMBGQAABgIyAAAMBCQAQBgICADAMBAQAYAgIGADAAAAwEZAAAGAjIAAAwEZAAAGAjIAAAwEJABAGAgIAMAwEBABgCAgYAMAAADARkAAAYCMgAADPaf9wBYO0+6+h3mUvebuXgude/y/x08l7q50pXXveSfPu29614zSc67YL+51K2az+/y37loPv9E3ujmn5tL3b5oLmVz/jfW/+fqqj/zw+teM0lucdpZc6m737WuMpe6sFGZQQYAgIGADAAAAwEZAAAGAjIAAAwEZAAAGAjIAAAwEJABAGAgIAMAwEBABgCAgYAMAAADARkAAAYCMgAADARkAAAYCMgAADAQkAEAYCAgAwDAQEAGAICBgAwAAAMBGQAABgIyAAAMBGQAABgIyAAAMBCQAQBgICADAMBAQAYAgIGADAAAAwEZAAAGAjIAAAwEZAAAGAjIAAAwEJABAGAgIAMAwEBABgCAgYAMAAADARkAAAYCMgAADARkAAAYCMgAADAQkAEAYCAgAwDAQEAGALaMqrpaVZ06fXy+qs4Ztg/Yg/6Pqqq/XkW9u1dVV9UvD22HT21PGdoeX1UfqaozqurPh/anVdUnquqjVfUTq7lWLrn95z0AAID10t1fTnJ4klTV9iTf6u5nr3HZ05P8bJIXTtsPS/LBnTur6h5Jjkxyy+4+v6quMbXfNMlDk9wsyXWS/FtV3ai7L17j8W55ZpABgC2tqm5dVe+oqg9U1Zur6tpT+xOq6sNV9aGqetky/X6qqt5bVadU1b9V1TVXKPFfSbZV1TWrqpLcJ8kbh/2PTXJMd5+fJN39han9yCQv6+7zu/usJJ9Ictu9c9XsihnkS2FhYeHoJNvmPY6V3Dc/Mu8hAMC6q6qjkhw1NB3b3ceudHiSv0pyZHd/sap+Lskzk/xSkqOTHDbN6h60TN93Jrl9d+9cQvE7SZ68Qp1XJfmZJKckOTnJ+cO+GyW5S1U9M8l5SZ7S3e9Pct0k7xmOO3tqY40JyJfOtsXFxe3zHsRKfv/Qhz9j3mMAgPU2heGVAvFSl0ty8yRvnU3uZr8kn5v2fSjJ8VX12iSvXabvwUlePs04H5DkrF3UeUWSlye5cZJ/SnLHYd/+Sa6a5PZJbpPkFVX1w3s4ftaAJRYAwFZWSc7o7sOnjx/t7ntP++6X5PlJjkjy/qpaOrH4V0n+urt/NMmvZhd/Ve7uzye5MMmPJzlxye6zk7y6Z96XZEeSH0xyTpJDhuMOntpYYwIyALCVnZ/k6lV1hySpqstW1c2q6jJJDunutyd5apKrJLnikr5XyfcC6yP3oNbvJ3nqMjfZvTbJPab6N8psNvpLSU5I8tCqulxVHZbkhknet8rr4xKwxAIA2Mp2JHlIkudV1VUyy0bPTfKxJP8wtVWS53X316ZlGDttT/LKqvpqkrclOWxXhbr7XSvsenGSF1fV6UkuSPLI7u4kZ1TVK5J8OMlFSR7nCRbrQ0AGALak7t4+bN51mUPuvEyf45IcN71+XZLX7abGSUlO2lXt7r4gyS+s0P+Zmd00yDqyxAIAAAYCMgAADARkAAAYCMgAADAQkAEAYCAgAwDAQEAGAICBgAwAAAMBGQAABgIyAAAMBGQAABgIyAAAMBCQAQBgICADAMBAQAYAgIGADAAAAwEZAAAGAjIAAAwEZAAAGAjIAAAwEJABAGAgIAMAwEBABgCAgYAMAAADARkAAAYCMgAADARkAAAYCMgAADAQkAEAYCAgAwDAQEAGAICBgAwAAAMBGQAABgIyAAAMBGQAABgIyAAAMBCQAQBgICADAMBg/3kPgLVz+Hk9l7rnX2a/udTN9W4wl7J15aute81r3eLEda+ZJH3RXMpmx3lzqnthzaXugbe59lzq9oUXzqXugeevf90d//2F7PejN173upe97bZ1r5kk2W8+/y7X/nP6/wFcSmaQAdhy5hGOgY1DQAYAgIGADAAAAwEZAAAGAjIAAAwEZAAAGAjIAAAwEJABAGAgIAMAwEBABgCAgYAMAAADARkAAAYCMgAADARkAAAYCMgAADAQkAEAYCAgAwDAQEAGAICBgAwAAAMBGQAABgIyAAAMBGQAABgIyAAAMBCQAQBgICADAMBAQAYAgIGADAAAAwEZAAAGAjIAAAwEZAAAGAjIAAAwEJABAGAgIAMAwEBABgCAgYAMAAADARkAAAYCMgAADARkAAAYCMgAADDY/xL2O29hYWH73hzIBnXovAcAAMDedYkC8uLi4jF7eyAbkV8SAAA2H0ssAIAtpaquWVX/WFWfqqoPVNW7q+pBu+lzaFV9p6pOHT4OWKfx3rCqXl9Vn5zG+/aqumtVvWYaxyeq6uvDuO40HXfX4RxvqaqfWebcvzH176r6waG9qup5074PVdURw74/r6ozqurM6Zia2m9dVadNfb7bPvR7+jDGi4fXT6iqF1bVTdfmK7h6l3SJBQDAhjOFttcmeUl3//zUdr0kD9iD7p/s7sMvQc39uvvi1fab+m5L8q9JntLdJ0xtN0+y0N0PmrbvPu2//9Dv15P8XVXdOslDkuzo7lcuU+I/k7w+yUlL2n8yyQ2nj9sleUGS21XVHZPcKcktpuPemeRuU/8XJPmVJO9N8oYk90nyxp0n7O5nJnnmNL5vXZKv5XoxgwwAbCX3THJBd//tzobu/q/u/qtkFmar6llV9f5p5vRXd3WyqrpXVZ0yzZy+uKouN7V/uqr+rKpOTvIzVXWfqjq5qj5YVSdOx1xh6vO+6RxHLlPi4UnevTMcT+M9vbuP29W4uvu9Sd6dZHuSP0nyGyscd0p3f3qZXUcmeWnPvCfJQVV17SSdZFuSA5JcLsllk/zPtO/K3f2e7u4kL03ywF2NcVRVJ1XVwp4ev9bMIO8jFhYWjs7sB26v+d3caG+eDgA2hKo6KslRQ9Ox3X3s9PpmSU7eRffHJPl6d99mCrv/WVVvySwYXr+qTp2O+88kT05yXJJ7dffHquqlSR6b5LnTMV/u7iOq6upTzbt291lVddVp/9OTvK27f6mqDkryvqr6t+4+dxjP7sa7K09L8tkkz+3uT6yy73WnvjudneS63f3uqnp7ks8lqSR/3d1nTuH27KXHX8Jxz52AvO/Ytri4uH1vnvDV1/r5Z+zN8wHARjCF4WN3e2CSqnp+kjtnNqt8myT3TnKLqnrIdMhVMltm8LEsWWJRVbdMclZ3f2xqekmSx+V7Afnl0+fbJ/n37j5rGt9XpvZ7J3lAVT1l2t6W5IeSnLmL8b5m53i6+8G7uby7Jvl6kpvv5rg9VlU3SHKTJAdPTW+tqrsk+c7eqrEvsMQCANhKzkjy3RvOuvtxSe6V5OpTUyV5fHcfPn0c1t1vuYS1zt3N/kry00OtH+rupeF46XgflORRSa6aXaiqKyT588yWlFyjqu67yrGfk+SQYfvgqe1BSd7T3d/q7m9ltsb4DtO+g5c5fkMSkAGAreRtSbZV1WOHtssPr9+c5LFVddkkqaobTWFzOR9Ncug0q5okj0jyjmWOe0+Su1bVYdM5d4bbNyd5/PAUiFst0/cfk9ypqsabCC+/zHFL/X6SV3T3R5L8epLnTDf87akTkvzi9DSL22e27ORzST6T5G5Vtf/0NbpbkjOnfd+oqttP1/OLSV63inr7FAEZANgyphvIHphZyDurqt6X2dKIp06HvDDJh5OcXFWnJ/m/WWFJanefl+TRSV5ZVacl2ZHkb5c57ouZrYl+dVV9MN9bevFHmd3k9qGqOmPaXtr3O0nun+TXavZYuncn+b0kf7zSNVbVzTKb6X3mdI5TMgvjT13m2CdU1dmZzfh+qKpeOO16Q5JPJflEkr/LLGQnyauSfDLJaUk+mOSD3f0v075fz+zr94npmO8+wWKjsQYZANhSptnOh66wb0eS350+Rsuu5e3uE5N838xvdx+6ZPuNWRIYp/C7y6dkTMd9JMmKSyS6+6QMj2nr7jOS/32nfnc/YYW+z0vyvGXaO7P11EvbL15pzN29mD1c79zdV1yyffc96bdezCADAMBAQAYAgIGADAAAAwEZAAAGAjIAAAwEZAAAGAjIAAAwEJABAGAgIAMAwEBABgBg06mZX6iq35+2f6iqbrsnfQVkAAA2o79JcockD5u2v5nk+XvScf+1GhEAAMzR7br7iKo6JUm6+6tVdcCedDSDDADAZnRhVe2XpJOkqq6eZMeedBSQAQDYjJ6X5DVJrlFVz0zyziR/sicdLbEAAGDT6e7jq+oDSe6VpJI8sLvP3JO+AjIAAJvV/yT5j8wy74FVdUR3n7y7TgIyAACbTlX9UZJHJflkpnXI0+d77q6vgAwAwGb0s0mu390XrLajm/QAANiMTk9y0CXpaAYZAIDN6E+TnFJVpyc5f2djdz9gdx0FZAAANqOXJPmzJKdlD59/vJOADADAZvTt7n7eJekoIAMAsBn9R1X9aZIT8r+XWHjMGwAAW9Ktps+3H9o85g0AgK2pu+9xSfsKyAAAbEpVdb8kN0uybWdbd//h7vp5DjIAAJtOVf1tkp9L8vgkleRnklxvT/oKyAAAbEZ37O5fTPLV7v6DJHdIcqM96SggAwCwGX1n+vztqrpOkguTXHtPOlqDDADAZvT6qjooybOSnJzZEyxeuCcdBWQAADad7v6j6eU/V9Xrk2zr7q/vSV8BGQCATamq7pjk0EyZt6rS3S/dXT8BGQCATaeq/j7J9ZOcmuTiqbmTCMgAAGxJC0lu2t292o6eYgEAwGZ0epJrXZKOZpABANiMfjDJh6vqfUnO39nY3Q/YXUcBGQCAzWj7Je0oIAMAsOl09zt2tb+q3t3dd1hun4C8id3pJufMpe4F35rPj9V+P3zEXOrWlX9w3Wtuu/ct1r3mXH3jW3Mp2985by51L3PHu8ylbr72pfnUvewB86m77fLrXvIyV77autdMkj53jx79ClvNtpV2uEkPgK1nDuEY2Oes+HQLARkAAAYCMgAAW1GttENABgBgU6qq61XVj02vD6yqKw27H7FSPwEZAIBNp6p+JcmrkvzfqengJK/dub+7T1+pr4AMAMBm9Lgkd0ryjSTp7o8nucaedBSQAQDYjM7v7gt2blTV/tnFkytGAjIAAJvRO6rqd5McWFU/nuSVSf5lTzoKyAAAbEZHJ/liktOS/GqSNyT5vT3p6J30AADYjA5M8uLu/rskqar9prZv766jGWQAADajEzMLxDsdmOTf9qSjgAwAwGa0rbu/tXNjer1H7zMvIAMAsBmdW1VH7Nyoqlsn+c6edLQGGQCAzehJSV5ZVf+d2dtKXyvJz+1JRwEZAIBNp7vfX1U3TvIjU9NHu/vCPekrIAMAsGlU1T27+21V9eAlu25UVenuV+/uHAIyAACbyd2SvC3JTy2zr5MIyAAAbB3d/Yzp5S9398WX5ByeYgEAwGZ0VlUdW1X3qqpaTUcBGQCAzejGmb0xyOMyC8t/XVV33pOOAjIAAJtOd3+7u1/R3Q9OcqskV07yjj3pKyADALApVdXdqupvknwgybYkP7sn/dykBwDAplNVn05ySpJXJPnt7j53T/sKyAAAbEa36O5vXJKOllgAALAZXauqTqyq05Okqm5RVb+3Jx0FZAAANqO/S/K0JBcmSXd/KMlD96SjgAwAwGZ0+e5+35K2i/ako4AMAMBm9KWqun5mby+dqnpIks/tSUc36QEAsBk9LsmxSW5cVeckOSvJw/eko4AMAMCmUVW/NWy+IcnbM1s1cW6Sn07yF7s7h4AMAMBmcqXp848kuU2S1yWpJI9IsnRN8rIEZAAANo3u/oMkqap/T3JEd39z2t6e5F/35Bxu0gMAYDO6ZpILhu0LprbdMoN86Zy3sLCwfS+d69C9dB4AAJKXJnlfVb1m2n5gkuP2pKOAfCksLi4es7fOtReDNgDAltfdz6yqNya5y9T06O4+ZU/6CsgAAGxK3X1ykpNX288aZAAAGAjIAAAwEJABAGAgIAMAwEBABgCAgYAMAGxJVfXAquqquvHQ9qaq+lpVvX7JsYdV1Xur6hNV9fKqOmBqv9y0/Ylp/6Er1Lr9tP/Uqjpzele3VNVvVdWLh+MeXlX/Or3ePo3vBsP+J01tC0vOf0JVnb6b692vqk4Zr22l65r2/UJVfaiqzqiqD1bVC6vqoGnfSVX10eF6jtpV7Y1GQAYAtqqHJXnn9HmnZyV5xDLH/lmS53T3DZJ8NcljpvbHJPnq1P6c6bjlvCTJUd19eJKbJ3nF1P68JEdU1Z2m8PnHSR4/9DstyUOH7Z9JcsZ44qp6cJJvrXiV3/PEJGfuyXVV1X2S/GaSn+zumyU5Ism78r/fie7h0/XcKcmfjeF6oxOQAYAtp6qumOTOmQXC7wbQ7j4xyTeXHFtJ7pnkVVPTSzJ7V7YkOXLazrT/XtPxS10jyeemGhd394en1xcl+fUkz0/y50le3N2fGvq9dqqRqrp+kq8n+dKS6/itzIL1rq734CT3S/LCPbyupyd5SnefM4z5xd390WVOf8Uk5ya5eFdj2Ei8UcgGsLCwcHSSbavt969XusIajAYA9m3Tn/vHP/kf293HLjnsyCRv6u6PVdWXq+rW3f2BFU55tSRfm8Jskpyd5LrT6+sm+WwyC7tV9fXp+C8tOcdzkny0qk5K8qYkL+nu86Z+76qqM5P8WJKbLOn3jSSfraqbT2N+eZJHD/v/KMn/SfLtFca+03OT/E6SK+3hdd0su3+DjeOr6vwkN0zypO4WkFlX2xYXF7evttP/3ONuz1iDsQDAPm0Kw0sD8VIPS/KX0+uXTdsrBeS9MaY/rKrjk9w7yc9P9e6efHcWeCHJZZNcPbOgOnpZZrPcP5HkXpkCclUdnuT63f2bK619no67f5IvdPcHquruqx17Vf1okr/PLFz/bne/fNr18O5erKqrJ3lXVb2pu/9rteffF1liAQBsKVV11cyWFrywqj6d5LeT/OwKSyOS5MtJDqqqnROLByc5Z3p9TpJDpvPun+QqSb5cVf9vuoHtDTtP0t2f7O4XZBZyb1lVV5t2/UGSf0jyzMxmmpd6fWbroj/T3d8Y2u+QZGG6hncmudF089whU+1Tq+rXMlsj/IDpuJcluWdV/cNuruuMzNYdp7tPm9YavzHJgUsH191fzGy2+XYrfP02HAEZANhqHpLk77v7et19aHcfkuSsJHdZ7uDu7iRvn/olySOTvG56fcK0vfO8b+uZR3f34d193ySpqvsNAfyGma3X/do0O3u/zG6WOzbJoVX140vqfzvJUzML0GP7C7r7Ot19aGbrqT/W3Xfv7s9OtQ/v7r/t7qd198HTcQ+dxvgLu7muP03y7Gnt8k7fF46na7t8klsl+eRy+zciSywAgK3mYfn+p038c5KHVdUzk9w4yRWr6uwkj+nuN2cWUF9WVX+c5JQkL5r6vSjJ31fVJ5J8Jf/7iROjRyR5TlV9O8lFSR6eZEeSFyT5zZ3rkavqsUleOi2f+K7uftmluN5dWfa6uvsN09KJN1bVfkm+luT0JG8e+h5fVd9Jcrkkx+1iDfeGIyADAFtKd99jmbbn7abPp5Lcdpn28zJ79Nruaq4UnO+85LjFJDedNrevcK67L9P26cweH7e7cZyU5KRhe9nrmva9JN97Qsdux7CZWGIBAAADARkAAAYCMgAADARkAAAYCMgAADAQkAEAYCAgAwDAQEAGAICBgAwAAAMBGQAABgIyAAAMBGQAABgIyAAAMBCQAQBgICADAMBAQAYAgIGADAAAAwEZAAAGAjIAAAwEZAAAGAjIAAAwEJABAGAgIAMAwEBABgCAgYAMAAADARkAAAYCMgAADARkAAAYCMgAADAQkAEAYCAgAwDAQEAGAICBgAwAAAMBGQAABgIyAAAMBGQAABgIyAAAMBCQAQBgsP+8B8DaOecTB82l7rkXXnYuda/+hbPmUncev2Xu+Mx/z6Fq0ueeP5e62bFjPmW/OZ/rPeCwz8yl7o7Pnj2XunW5A+ZT95rXXPea/ZUvrnvNJMmBB86n7gHb5lMXLiUzyABsOfMIx8DGISADAMBAQAYAgIGADAAAAwEZAAAGAjIAAAwEZAAAGAjIAAAwEJABAGAgIAMAwEBABgCAgYAMAAADARkAAAYCMgAADARkAAAYCMgAADAQkAEAYCAgAwDAQEAGAICBgAwAAAMBGQAABgIyAAAM9p/3APiu8xYWFravsO/QdRwHAMCWJiDvIxYXF49Zad8ugjMAAHuZJRYAADAQkAEAYCAgAwDAQEAGAICBgAwAAAMBGQAABgIyAAAMBGQAABgIyAAAMBCQAQBgICADAMBAQAYAgIGADAAAAwEZAAAGAjIAAAwEZAAAGAjIAAAwEJABgC2lqq5ZVf9YVZ+qqg9U1bur6kG76XNoVX2nqk4dPg5Yh7GOdT9cVS+dxr9zDJ+vqnOG7etV1VlVddWp/w9M24cuOe8hVfX26ZxnVNUTh31Xraq3VtXHp88/MLVfpar+pao+OPV59NDnkdPxH6+qRy5zHa+ZxveJqvr6MN47VtW71uwLeAntP+8BAACsl6qqJK9N8pLu/vmp7XpJHrAH3T/Z3Ydfgpr7dffFq+23tG5V7ZfkrUl+bOc4qmp7km9197OHei9IckySo6bPx3b3p5ec86IkT+7uk6vqSkk+UFVv7e4PJzk6yYndfUxVHT1tPzXJ45J8uLt/qqqunuSjVXV8kismeUaShSQ9neuE7v7qzmLd/aBpbHdP8pTuvv8wljteiq/NmjCDDABsJfdMckF3/+3Ohu7+r+7+q2QWZqvqWVX1/qr6UFX96q5OVlX3qqpTquq0qnpxVV1uav90Vf1ZVZ2c5Geq6j5VdfI0+3ridMwVpj7vm85x5K5qTSH7fUmuu5trfE6S21fVk5LcOcmzlx7Q3Z/r7pOn199McuZw3iOTvGR6/ZIkD9zZLcmVpl8yrpjkK5kF7Z9I8tbu/soUit+a5D67GeN3VdW39vTY9WIGeYNbWFg4Osm25fYdu9v/fgBg86mqozKbPd3p2O4+dnp9syQn76L7Y5J8vbtvM4Xd/6yqt2QWDq9fVadOx/1nkicnOS7Jvbr7Y1X10iSPTfLc6Zgvd/cR02zryUnu2t3fXf6Q5OlJ3tbdv1RVByV5X1X9W3efu8J1bUtyuyRPXG7/Tt19YVX9dpI3Jbl3d1+4q+On5Re3SvLeqema3f256fXnk1xzev3XSU5I8t9JrpTk57p7R1VdN8lnh1Oend2H+H2agLzxbVtcXNy+3I6TDznyGes8FgCYuykMH7vbA5NU1fMzm2W9oLtvk+TeSW5RVQ+ZDrlKkhsm+ViWLLGoqlsmOau7PzY1vSSzZQjPnbZfPn2+fZJ/7+6zpvF9ZWq/d5IHVNVTpu1tSX4os9nc0c5gfliSf+3uD+3Bpf1kks8luXlmM7orXf8Vk/xzkid19zeW7u/urqqeNn8iyamZzcJfP8lbq+o/9mAsG44lFgDAVnJGkiN2bnT345LcK8nVp6ZK8vjuPnz6OKy733IJay07EzyoJD891Pqh7l4ajpPvBfPrJ7l1Ve1yvXRVHZ7kxzML5r9ZVdde4bjLZhaOj+/uVw+7/mdnn+nzF6b2Ryd5dc98IslZSW6c5Jwkhwz9D57aNiwBGQDYSt6WZFtVPXZou/zw+s1JHjuFx1TVjarqCiuc66NJDq2qG0zbj0jyjmWOe0+Su1bVYdM5dy6xeHOSx09relNVt9rVwLv7S5ndMPe0lY6ZzvWCzGaEP5PkWVlmDfJ03IuSnNndf7Fk9wlJdj6J4pFJXje9/kxmv0ykqq6Z5EeSfGq6jntPT8z4gcxmxt+8q2vZ1wnIAMCW0d2d2U1nd5sef/a+zJZGPHU65IVJPpzk5Ko6Pcn/zQpLUrv7vMxmVV9ZVacl2ZHkb5c57ouZrYl+dVV9MN9bevFHSS6b5ENVdca0vTuvTXL5qrrLCvt/Jclnunvnsoq/SXKTqrrbkuPulFmgv+fwyLX7TvuOSfLjVfXxJD82be8c7x2naz0xyVO7+0vTkpE/SvL+6eMPh2UkG5I1yADAljLdgPbQFfbtSPK708fo65mt5116/ImZ3eC2tP3QJdtvTPLGJW3fSbLLp2RMj2e7+bDdSW45bG9fcvz/Wn89PfniiCzR3e/MbInHcjW/nGmmeEn7f2c2O7xcnxcnefGurmU67qQkJy1pu+Lu+q03M8gAADAQkAEAYCAgAwDAQEAGAICBgAwAAAMBGQAABgIyAAAMBGQAABgIyAAAMBCQAQBgICADAMBAQAYAgIGADAAAAwEZAAAGAjIAAAwEZAAAGAjIAAAwEJABAGAgIAMAwEBABgCAgYAMAAADARkAAAYCMgAADARkAAAYCMgAADAQkAEAYCAgAwDAQEAGAICBgAwAAAMBGQAABgIyAAAMBGQAABgIyAAAMBCQAQBgICADAMBAQAYAgIGADAAAAwEZAAAG+897AKydA7ddOJe6O7rmUrcOutZc6uYy6/97Zh24bd1rJkntv99c6va5582l7mWuNJeyybYD51L2Mte9zlzq9oVz+Lfqsgesf80kucKcfqh6x3zqXnzRfOrCpSQgbwznLSwsbF9h36HrOA4AgE1PQN4AFhcXj1lp3y6CMwAAl4A1yAAAMBCQAQBgICADAMBAQAYAgIGADAAAAwEZAAAGAjIAAAwEZAAAGAjIAAAwEJABAGAgIAMAwEBABgCAgYAMAAADARkAAAYCMgAADARkAAAYCMgAADAQkAEAYCAgAwDAQEAGAICBgAwAAAMBGQAABgIyAAAMBGQAABgIyAAAMBCQAQBgICADAMBAQAYAgIGADAAAAwEZAAAGAjIAAAwEZAAAGAjIAAAwEJABAGAgIAMAwEBABgCAgYAMAAADARkAAAYCMgAADARkAGDLqKqrVdWp08fnq+qcYfuAPej/qKr66z2sdfmq+nJVXXlJ+2ur6ueq6sZV9e6qOr+qnrLkmPtU1Uer6hNVdfTQflhVvXdqf/lKY66qX6qq06rqQ1V1elUdObWfUFW/OBz3d1X129Prk6rqM1VVS8b6ren14dN4z5jO+3O7uPY3VdXXqur1S9qrqp5ZVR+rqjOr6glLrvl9VfWR6fvx8qr6oWnfcVV11tT+kap6xq6+9peWgAwAbBnd/eXuPry7D0/yt0mes3O7uy/Yy7W+neTNSR60s62qrpLkzkn+JclXkjwhybPHflW1X5LnJ/nJJDdN8rCquum0+8+mMd8gyVeTPGZp3ao6OMnTk9y5u2+R5PZJPjTtfkKSP6iqg6rqjklul+Q5Q/evJbnTdJ6Dklx72PftJL/Y3TdLcp8kz52OWc6zkjximfZHJTkkyY27+yZJXjbVunmSv0ryyO6+8fT9OT7JoUPf357aD0/yyKo6bIXal5qADABsaVV166p6R1V9oKreXFXXntqfUFUfnmZLX7ZMv5+aZnNPqap/q6prLnP6f0ry0GH7QUne3N3f7u4vdPf7k1y4pM9tk3yiuz81hfaXJTlymtm9Z5JXTce9JMkDl6l5jSTfTPKtJOnub3X3WdPrTyc5NsmfJ3lBkt/o7ouGvi8bxvvgJK/euaO7P9bdH59e/3eSLyS5+jL1090nTmNY6rFJ/rC7d0zHfWFqf2qSP+nuM4dznNDd/77MObZNn89drvbesP9anZh1c97CwsL25Xb8fa6xzkMBgPmrqqOSHDU0Hdvdx650eGYzl0d29xenZQPPTPJLSY5Oclh3n7/CTOk7k9y+u7uqfjnJ7yR58pJj3pzkhVV1te7+cmbhc3dLNK6b5LPD9tmZzfReLcnXhkB79nTsUh9M8j9JzqqqE5O8urv/Zdj/7CSfTPIfywTQE5P83TSL/dDMvo7/39ICVXXbJAdM51mN6yf5uap6UJIvJnnCFLpvliUz6ct4VlX9XpIbJHneEK73OgF5g1tcXDxmpX1n3vC+a7o+BwD2RVMYXikQL3W5JDdP8tZp6e1+ST437ftQkuOr6rVJXrtM34OTvHyacT4gyVnLjOWCqjohyUOq6p+T3Cqz0LxmuvviqrpPktskuVeS51TVrbt7+3TILTJbRXDjqrrMztncycWZBf+HJjmwuz89LElOkkzX+/eZLYfYkdW5XJLzunuhqh6c5MVJ7rLk/FfLLKhfPrNfbnYG59/u7ldV1RWTnFhVd+zud62y/h6xxAIA2MoqyRnDOuQf7e57T/vul9la4COSvL+qlk4s/lWSv+7uH03yq/nen/6X2rnM4iFJXtfdS5dULHVOZut0dzp4avtykoOGcRyc5Jyq2m+40fAPk6Rn3tfdfzrV/ukkqarLJPmbJL+Q5OOZLXlY6mVJnpfkFUt3TDcc/muSp3f3e6a22w31H7Cbazs731u28ZrMwnqSnJHZ1/m768Qz+yXniktP0N3fSnJSZmu514SADABsZecnuXpV3SFJquqyVXWzKUge0t1vz2x97FXy/WHtKpkF1yR55C5qnJTkhkkel1lY3p33J7nh9MSKAzILuCd0dyd5e2ZBe2fN13X3xUPA//2quk5VHTGc7/Ak/zW9/tUkH+/uk5L8VpKnVtXSdcT/keRPl451Gstrkry0u3eug053v3eof8Juru21Se4xvb5bko9Nr/88ydOr6ibDsZdf7gTTLwi3y+qXd+wxSywAgK1sR2aB83nTEyb2T/LczILbP0xtldma168tWW6wPckrq+qrSd6WZNmnKnT3jqp6VZKfTfKOne1Vda0ki0munGRHVT0pyU27+xtV9RuZLcXYL8mLu/uMqdtTk7ysqv44ySlJXrRMycsmeXZVXSfJeZmt9f21qrrG1P/207j+u6qem1k4ffQw3s7y64F/Nsldk1ytqh41tT2qu09demBV/UeSGye5YlWdneQx3f3mJMdktmzlNzO7ifCXp5qnVdUTk7x0mqX+UpLPJBmXi+5cg3xAZkswXp01UrOvAZvRmTe871y+ued+Z7ePkVwTP/pvvzmXurXtCute88KXPmf3B61J4d39VXBt9LnnzafuBRft/qA1sP9dbjeXuvn2mt0Qvks9h5+rutKVd3/QWrjCleZTd9XLRDe2A4/8ndr9UbAySywAAGAgIAMAwEBABgCAgYAMAAADARkAAAYCMgAADARkAAAYCMgAADAQkAEAYCAgAwDAQEAGAICBgAwAAAMBGQAABgIyAAAMBGQAABgIyAAAMBCQAQBgICADAMBAQAYAgIGADAAAAwEZAAAGAjIAAAwEZAAAGAjIAAAwEJABAGAgIAMAwEBABgCAgYAMAAADARkAAAYCMgAADARkAAAYCMgAADAQkAEAYCAgAwDAQEAGAICBgAwAAAMBGQAABgIyAAAMBGQAABhUd897DOyDquqo7j5WXXXV3Tg11VVXXdg7zCCzkqPUVVfdDVdTXXXVhb1AQAYAgIGADAAAAwGZlcxrzZe66m6WulvpWtVVd7PVZYtzkx4AAAzMIAMAwEBABgCAgYBMqurFVfWFqjp9aLtqVb21qj4+ff6Bdar7M1V1RlXtqKqFvV1zmTEcUlVvr6oPT3WfuNY1p7rbqup9VfXBqe4frEfdqfZ+VXVKVb1+jess9/39o6r6UFWdWlVvqarrrPEYPl1Vp031Ftey1pK6vzl9X0+vqn+qqm3rUPNHpuvc+fGNqnrSWtedah9UVa+qqo9U1ZlVdYc1qvN9P1PDvidXVVfVD65F7aHOE6fv6xlr/fVd4b+hZ01f5w9V1Wuq6qA1HsN9quqjVfWJqjp6LWsNNVf8PsN6EZBJkuOS3GdJ29FJTuzuGyY5cdpej7qnJ3lwkn9fg3rLuSjJk7v7pklun+RxVXXTdah7fpJ7dvctkxye5D5Vdft1qJskT0xy5jrUOS7f//19VnfforsPT/L6JL+/DuO4R3cf3t1r/gtXklTVdZM8IclCd988yX5JHrrWdbv7o9N1Hp7k1km+neQ1a1138pdJ3tTdN05yy6zdz9dx+f6fqVTVIUnuneQza1R3Z52bJ/mVJLfN7DrvX1U3WMOSx+X7r/etSW7e3bdI8rEkT1ur4lW1X5LnJ/nJJDdN8rB1+vfxuCzzfYb1JCCT7v73JF9Z0nxkkpdMr1+S5IHrUbe7z+zuj+7tWrsYw+e6++Tp9Tcz+x/7ddehbnf3t6bNy04fa37HbFUdnOR+SV641rVW+P5+Y9i8Qtbhmudk/yQHVtX+SS6f5L/Xuf69knyyu/9rrQtV1VWS3DXJi5Kkuy/o7q+tRa0V/q1Kkuck+Z2s/c/TTZK8t7u/3d0XJXlHZr/Qr4kV/ht6y1Q7Sd6T5OC1qp/ZLwKf6O5PdfcFSV6W2f8b1tQuvs+wbgRkVnLN7v7c9PrzSa45z8Gsh6o6NMmtkrx3nertV1WnJvlCkrd293rUfW5mQWLHOtRaVlU9s6o+m+ThWfsZ5E7ylqr6QFWtyztydfc5SZ6d2Wzm55J8vbvfsh61Bw9N8k/rVOuwJF9M8v+mpTsvrKorrFPtVNWRSc7p7g+uQ7nTk9ylqq5WVZdPct8kh6xD3ZX8UpI3ruH5r5vks8P22VmHCQTYFwjI7FbPngW4WWf6kiRVdcUk/5zkSUtmOddMd188/Tn84CS3nf58u2aq6v5JvtDdH1jLOrvT3U/v7kOSHJ/kN9a43J27+4jM/kT8uKq66xrXy7Re/8jMguN1klyhqn5hresO9Q9I8oAkr1ynkvsnOSLJC7r7VknOzdosyfo+U0j93azPUp1095lJ/izJW5K8KcmpSS5ej9pLVdXTM1sidvw86sNmJyCzkv+pqmsnyfT5C3Mez5qpqstmFo6P7+5Xr3f96c/Rb8/ar7m7U5IHVNWnM/tT6T2r6h/WuOauHJ/kp9eywDSbm+7+QmbrcW+7lvUmP5bkrO7+YndfmOTVSe64DnV3+skkJ3f3/6xTvbOTnD38BeRVmQXm9XD9zH4R+eD0c31wkpOr6lprVbC7X9Tdt+7uuyb5ambrgNdVVT0qyf2TPLzX9s0Mzsn/niE/eGqDTU9AZiUnJHnk9PqRSV43x7GsmaqqzNZOntndf7GOda++8+7zqjowyY8n+cha1uzup3X3wd19aGZ/gn9bd6/bzGaSVNUNh80js4bXXFVXqKor7Xyd2U1c63FX/GeS3L6qLj/9fN0r63NT5E4Py/otr0h3fz7JZ6vqR6ameyX58DrVPq27r9Hdh04/12cnOWIa05qoqmtMn38os/XH/7hWtVaof5/Mlkk9oLu/vcbl3p/khlV12PSXiYdm9v8G2PT2n/cAmL+q+qckd0/yg1V1dpJnJDkmySuq6jFJ/ivJz65T3a8k+askV0/yr1V1anf/xN6uPbhTkkckOW1aD5wkv9vdb1jDmkly7SQvme4Sv0ySV3T3mj52bb2t8P297xSkdmT2c/VraziEayZ5zSyjZv8k/9jdb1rDekmS7n5vVb0qycmZ/Qn8lKzT2+VOvwj8eJJfXY96g8cnOX4KUZ9K8ui1KLLcz1R3v2gtau3CP1fV1ZJcmORxa3VDYrLif0NPS3K5JG+dfrbf091r8t9Rd19UVb+R5M2ZPY3lxd19xlrUGu0j32e2OG81DQAAA0ssAABgICADAMBAQAYAgIGADAAAAwEZAAAGAjIAAAwEZIC9oGb8mwqwCfjHHOASqqpDq+qjVfXSzN6l7+Jh30Oq6rjp9XFV9byqeldVfaqqHjKnIQOwBwRkgEvnhkn+prtvluTcXRx37SR3TnL/zN6pEoB9lIAMcOn8V3e/Zw+Oe2137+juD2f2NtgA7KMEZIBLZ5w17uH1tiXHnT+8rrUbDgCXloAMsPf8T1XdZLpZ70HzHgwAl4yADLD3HJ3k9UneleRzcx4LAJdQdffujwIAgC3CDDIAAAwEZAAAGAjIAAAwEJABAGAgIAMAwEBABgCAgYAMAACD/x/76hgOVHhVugAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x720 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.clustermap(\n",
    "    data\n",
    "    .groupby(['run', 'device_name']).filter(lambda x: len(x) > 10)\n",
    "    .groupby('device_name').filter(lambda x: len(x.groupby('run')) == 15)\n",
    "    .groupby(['run', 'device_name'])\n",
    "    ['perf_ns_per_day']\n",
    "    .mean()\n",
    "    .pipe(lambda x: np.log(x))\n",
    "    .unstack('run'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}