viz-4b-anomaly-detection-annotation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Home](https://walterra.github.io/jupyter2kibana/)\n",
    "\n",
    "*To make sure the Vega charts render correctly, view the notebook not from the Github repo but the published website here: \n",
    "https://walterra.github.io/jupyter2kibana/viz-4b-anomaly-detection-annotation.html*\n",
    "\n",
    "# viz-4b-anomaly-detection-annotation.ipynb\n",
    "\n",
    "This notebook adds user created annotations as an additional layer to the visualization. It's a great example of how VEGA is able to fetch data from different sources/indices, something not easily doable with Kibana's own visualizations.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import altair as alt\n",
    "import eland as ed\n",
    "from elasticsearch import Elasticsearch\n",
    "import elastic_transport\n",
    "import logging\n",
    "import json\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import urllib3\n",
    "import warnings\n",
    "\n",
    "alt.data_transformers.disable_max_rows()\n",
    "logging.getLogger(\"elastic_transport\").setLevel(logging.ERROR)\n",
    "\n",
    "# Suppress insecure SSL connection warnings\n",
    "# In dev environments with the `verify_certs=False`\n",
    "# you might want to reduce those warnings.\n",
    "urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)\n",
    "urllib3.disable_warnings(elastic_transport.SecurityWarning)\n",
    "\n",
    "# For rendering the notebook to HTML hide all warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "annotation_index_name = '.ml-annotations-read'\n",
    "index_name = '.ml-anomalies-fq_single_count_15m'\n",
    "vis_name = 'ml-anomalies-fq'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(975, 73)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('config.json') as config_file:\n",
    "  es_config = json.load(config_file)\n",
    "\n",
    "# First instantiate an 'Elasticsearch' instance with the supplied config\n",
    "es = Elasticsearch(\n",
    "    hosts=[es_config['es_client']],\n",
    "    basic_auth=[\n",
    "        es_config['user'],\n",
    "        es_config['password']\n",
    "    ],\n",
    "    # Only in development environments with self signed certificates fall back to use `verify_certs=False`\n",
    "    verify_certs=False\n",
    ")\n",
    "\n",
    "ed_df = ed.DataFrame(es, index_name)\n",
    "ed_df_number = ed_df.select_dtypes(include=np.number)\n",
    "ed_df_number.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<div id=\"altair-viz-f3bb48a981f344dd96d889b89df317ab\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "  var VEGA_DEBUG = (typeof VEGA_DEBUG == \"undefined\") ? {} : VEGA_DEBUG;\n",
       "  (function(spec, embedOpt){\n",
       "    let outputDiv = document.currentScript.previousElementSibling;\n",
       "    if (outputDiv.id !== \"altair-viz-f3bb48a981f344dd96d889b89df317ab\") {\n",
       "      outputDiv = document.getElementById(\"altair-viz-f3bb48a981f344dd96d889b89df317ab\");\n",
       "    }\n",
       "    const paths = {\n",
       "      \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
       "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
       "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm//vega-lite@4.17.0?noext\",\n",
       "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm//vega-embed@6?noext\",\n",
       "    };\n",
       "\n",
       "    function maybeLoadScript(lib, version) {\n",
       "      var key = `${lib.replace(\"-\", \"\")}_version`;\n",
       "      return (VEGA_DEBUG[key] == version) ?\n",
       "        Promise.resolve(paths[lib]) :\n",
       "        new Promise(function(resolve, reject) {\n",
       "          var s = document.createElement('script');\n",
       "          document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "          s.async = true;\n",
       "          s.onload = () => {\n",
       "            VEGA_DEBUG[key] = version;\n",
       "            return resolve(paths[lib]);\n",
       "          };\n",
       "          s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
       "          s.src = paths[lib];\n",
       "        });\n",
       "    }\n",
       "\n",
       "    function showError(err) {\n",
       "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
       "      throw err;\n",
       "    }\n",
       "\n",
       "    function displayChart(vegaEmbed) {\n",
       "      vegaEmbed(outputDiv, spec, embedOpt)\n",
       "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
       "    }\n",
       "\n",
       "    if(typeof define === \"function\" && define.amd) {\n",
       "      requirejs.config({paths});\n",
       "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
       "    } else {\n",
       "      maybeLoadScript(\"vega\", \"5\")\n",
       "        .then(() => maybeLoadScript(\"vega-lite\", \"4.17.0\"))\n",
       "        .then(() => maybeLoadScript(\"vega-embed\", \"6\"))\n",
       "        .catch(showError)\n",
       "        .then(() => displayChart(vegaEmbed));\n",
       "    }\n",
       "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300}}, \"data\": {\"url\": \"https://walterra.github.io/jupyter2kibana/data/fq_single_count_15m_annotations.json\", \"format\": {\"property\": \"hits.hits\", \"type\": \"json\"}, \"name\": \"ml-annotations\"}, \"mark\": {\"type\": \"rect\", \"color\": \"#369\", \"opacity\": 0.15}, \"encoding\": {\"tooltip\": [{\"field\": \"annotation\", \"type\": \"nominal\"}], \"x\": {\"field\": \"timestamp\", \"type\": \"temporal\"}, \"x2\": {\"field\": \"end_timestamp\"}, \"y\": {\"field\": \"y\", \"type\": \"quantitative\"}, \"y2\": {\"field\": \"y2\"}}, \"transform\": [{\"calculate\": \"datum._source.annotation\", \"as\": \"annotation\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.end_timestamp\", \"as\": \"end_timestamp\"}, {\"calculate\": \"datum._source.job_id\", \"as\": \"job_id\"}, {\"filter\": \"(datum.job_id === 'fq_single_count_15m')\"}, {\"calculate\": \"350\", \"as\": \"y\"}, {\"calculate\": \"0\", \"as\": \"y2\"}], \"$schema\": \"https://vega.github.io/schema/vega-lite/v4.17.0.json\"}, {\"mode\": \"vega-lite\"});\n",
       "</script>"
      ],
      "text/plain": [
       "alt.Chart(...)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Note: To create the Vega spec using Altair we reference ES via URL first. This will only work\n",
    "# for non-secured ES instances. If your ES instance runs using SSL and/or authentication the chart\n",
    "# in this cell will render empty. You can still save the visualization in Kibana correctly in the\n",
    "# next cell because there the URL gets replaced with an Elasticsearch query\n",
    "# to be used via the Kibana Vega plugin.\n",
    "\n",
    "# WARNING:\n",
    "# Do the following approach using a proxy only for demo purposes in a development environment.\n",
    "# It will expose a secured ES instance unsecured!\n",
    "# To make this work for demo purposes run the nodejs based proxy in a separate terminal like this:\n",
    "# NODE_TLS_REJECT_UNAUTHORIZED='0' node proxy\n",
    "\n",
    "# URL as ES endpoint\n",
    "# url_annotation = 'http://localhost:9220/'+annotation_index_name+'/_search?size=10000'\n",
    "\n",
    "# URL static fallback\n",
    "url_annotation = 'https://walterra.github.io/jupyter2kibana/data/fq_single_count_15m_annotations.json'\n",
    "\n",
    "url_data_annotation = alt.Data(\n",
    "    url=url_annotation,\n",
    "    format=alt.DataFormat(property='hits.hits',type='json'),\n",
    "    name='ml-annotations'\n",
    ")\n",
    "\n",
    "fields_annotation = [\n",
    "    'annotation',\n",
    "    'timestamp',\n",
    "    'end_timestamp',\n",
    "    'job_id',\n",
    "]\n",
    "\n",
    "rename_dict_annotation = dict((a, 'datum._source.'+a) for a in fields_annotation)\n",
    "\n",
    "base_annotation = alt.Chart(url_data_annotation).transform_calculate(**rename_dict_annotation)\n",
    "\n",
    "annotations = base_annotation.transform_filter(\n",
    "    alt.datum.job_id == \"fq_single_count_15m\"\n",
    ").transform_calculate(\n",
    "    y=\"350\",\n",
    "    y2=\"0\"\n",
    ").mark_rect(\n",
    "    color=\"#369\",\n",
    "    opacity=0.15,\n",
    ").encode(\n",
    "    alt.X('timestamp:T'),\n",
    "    alt.X2('end_timestamp:T'),\n",
    "    alt.Y('y:Q'),\n",
    "    alt.Y2('y2:Q'),\n",
    "    tooltip=['annotation:N'],\n",
    ")\n",
    "\n",
    "annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<div id=\"altair-viz-523780ddd33e4230ac8c4204f21e1ab8\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "  var VEGA_DEBUG = (typeof VEGA_DEBUG == \"undefined\") ? {} : VEGA_DEBUG;\n",
       "  (function(spec, embedOpt){\n",
       "    let outputDiv = document.currentScript.previousElementSibling;\n",
       "    if (outputDiv.id !== \"altair-viz-523780ddd33e4230ac8c4204f21e1ab8\") {\n",
       "      outputDiv = document.getElementById(\"altair-viz-523780ddd33e4230ac8c4204f21e1ab8\");\n",
       "    }\n",
       "    const paths = {\n",
       "      \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
       "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
       "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm//vega-lite@4.17.0?noext\",\n",
       "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm//vega-embed@6?noext\",\n",
       "    };\n",
       "\n",
       "    function maybeLoadScript(lib, version) {\n",
       "      var key = `${lib.replace(\"-\", \"\")}_version`;\n",
       "      return (VEGA_DEBUG[key] == version) ?\n",
       "        Promise.resolve(paths[lib]) :\n",
       "        new Promise(function(resolve, reject) {\n",
       "          var s = document.createElement('script');\n",
       "          document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "          s.async = true;\n",
       "          s.onload = () => {\n",
       "            VEGA_DEBUG[key] = version;\n",
       "            return resolve(paths[lib]);\n",
       "          };\n",
       "          s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
       "          s.src = paths[lib];\n",
       "        });\n",
       "    }\n",
       "\n",
       "    function showError(err) {\n",
       "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
       "      throw err;\n",
       "    }\n",
       "\n",
       "    function displayChart(vegaEmbed) {\n",
       "      vegaEmbed(outputDiv, spec, embedOpt)\n",
       "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
       "    }\n",
       "\n",
       "    if(typeof define === \"function\" && define.amd) {\n",
       "      requirejs.config({paths});\n",
       "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
       "    } else {\n",
       "      maybeLoadScript(\"vega\", \"5\")\n",
       "        .then(() => maybeLoadScript(\"vega-lite\", \"4.17.0\"))\n",
       "        .then(() => maybeLoadScript(\"vega-embed\", \"6\"))\n",
       "        .catch(showError)\n",
       "        .then(() => displayChart(vegaEmbed));\n",
       "    }\n",
       "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300}}, \"concat\": [{\"mark\": \"bar\", \"encoding\": {\"tooltip\": [{\"bin\": true, \"field\": \"value\", \"title\": \"x\", \"type\": \"quantitative\"}, {\"aggregate\": \"count\", \"title\": \"y\", \"type\": \"quantitative\"}], \"x\": {\"bin\": true, \"field\": \"value\", \"title\": \"\", \"type\": \"quantitative\"}, \"y\": {\"aggregate\": \"count\", \"title\": \"\", \"type\": \"quantitative\"}}, \"height\": 100, \"title\": \"actual\", \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"fold\": [\"anomaly_score\", \"actual\", \"typical\", \"event_count\", \"model_lower\", \"model_median\", \"model_upper\", \"timestamp\", \"result_type\"], \"as\": [\"attribute\", \"value\"]}, {\"filter\": \"(datum.attribute === 'actual')\"}], \"width\": 100}, {\"mark\": \"bar\", \"encoding\": {\"tooltip\": [{\"bin\": true, \"field\": \"value\", \"title\": \"x\", \"type\": \"quantitative\"}, {\"aggregate\": \"count\", \"title\": \"y\", \"type\": \"quantitative\"}], \"x\": {\"bin\": true, \"field\": \"value\", \"title\": \"\", \"type\": \"quantitative\"}, \"y\": {\"aggregate\": \"count\", \"title\": \"\", \"type\": \"quantitative\"}}, \"height\": 100, \"title\": \"anomaly_score\", \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"fold\": [\"anomaly_score\", \"actual\", \"typical\", \"event_count\", \"model_lower\", \"model_median\", \"model_upper\", \"timestamp\", \"result_type\"], \"as\": [\"attribute\", \"value\"]}, {\"filter\": \"(datum.attribute === 'anomaly_score')\"}], \"width\": 100}, {\"mark\": \"bar\", \"encoding\": {\"tooltip\": [{\"bin\": true, \"field\": \"value\", \"title\": \"x\", \"type\": \"quantitative\"}, {\"aggregate\": \"count\", \"title\": \"y\", \"type\": \"quantitative\"}], \"x\": {\"bin\": true, \"field\": \"value\", \"title\": \"\", \"type\": \"quantitative\"}, \"y\": {\"aggregate\": \"count\", \"title\": \"\", \"type\": \"quantitative\"}}, \"height\": 100, \"title\": \"event_count\", \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"fold\": [\"anomaly_score\", \"actual\", \"typical\", \"event_count\", \"model_lower\", \"model_median\", \"model_upper\", \"timestamp\", \"result_type\"], \"as\": [\"attribute\", \"value\"]}, {\"filter\": \"(datum.attribute === 'event_count')\"}], \"width\": 100}, {\"mark\": \"bar\", \"encoding\": {\"tooltip\": [{\"bin\": true, \"field\": \"value\", \"title\": \"x\", \"type\": \"quantitative\"}, {\"aggregate\": \"count\", \"title\": \"y\", \"type\": \"quantitative\"}], \"x\": {\"bin\": true, \"field\": \"value\", \"title\": \"\", \"type\": \"quantitative\"}, \"y\": {\"aggregate\": \"count\", \"title\": \"\", \"type\": \"quantitative\"}}, \"height\": 100, \"title\": \"model_lower\", \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"fold\": [\"anomaly_score\", \"actual\", \"typical\", \"event_count\", \"model_lower\", \"model_median\", \"model_upper\", \"timestamp\", \"result_type\"], \"as\": [\"attribute\", \"value\"]}, {\"filter\": \"(datum.attribute === 'model_lower')\"}], \"width\": 100}, {\"mark\": \"bar\", \"encoding\": {\"tooltip\": [{\"bin\": true, \"field\": \"value\", \"title\": \"x\", \"type\": \"quantitative\"}, {\"aggregate\": \"count\", \"title\": \"y\", \"type\": \"quantitative\"}], \"x\": {\"bin\": true, \"field\": \"value\", \"title\": \"\", \"type\": \"quantitative\"}, \"y\": {\"aggregate\": \"count\", \"title\": \"\", \"type\": \"quantitative\"}}, \"height\": 100, \"title\": \"model_median\", \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"fold\": [\"anomaly_score\", \"actual\", \"typical\", \"event_count\", \"model_lower\", \"model_median\", \"model_upper\", \"timestamp\", \"result_type\"], \"as\": [\"attribute\", \"value\"]}, {\"filter\": \"(datum.attribute === 'model_median')\"}], \"width\": 100}, {\"mark\": \"bar\", \"encoding\": {\"tooltip\": [{\"bin\": true, \"field\": \"value\", \"title\": \"x\", \"type\": \"quantitative\"}, {\"aggregate\": \"count\", \"title\": \"y\", \"type\": \"quantitative\"}], \"x\": {\"bin\": true, \"field\": \"value\", \"title\": \"\", \"type\": \"quantitative\"}, \"y\": {\"aggregate\": \"count\", \"title\": \"\", \"type\": \"quantitative\"}}, \"height\": 100, \"title\": \"model_upper\", \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"fold\": [\"anomaly_score\", \"actual\", \"typical\", \"event_count\", \"model_lower\", \"model_median\", \"model_upper\", \"timestamp\", \"result_type\"], \"as\": [\"attribute\", \"value\"]}, {\"filter\": \"(datum.attribute === 'model_upper')\"}], \"width\": 100}, {\"mark\": \"bar\", \"encoding\": {\"tooltip\": [{\"bin\": true, \"field\": \"value\", \"title\": \"x\", \"type\": \"quantitative\"}, {\"aggregate\": \"count\", \"title\": \"y\", \"type\": \"quantitative\"}], \"x\": {\"bin\": true, \"field\": \"value\", \"title\": \"\", \"type\": \"quantitative\"}, \"y\": {\"aggregate\": \"count\", \"title\": \"\", \"type\": \"quantitative\"}}, \"height\": 100, \"title\": \"result_type\", \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"fold\": [\"anomaly_score\", \"actual\", \"typical\", \"event_count\", \"model_lower\", \"model_median\", \"model_upper\", \"timestamp\", \"result_type\"], \"as\": [\"attribute\", \"value\"]}, {\"filter\": \"(datum.attribute === 'result_type')\"}], \"width\": 100}, {\"mark\": \"bar\", \"encoding\": {\"tooltip\": [{\"bin\": true, \"field\": \"value\", \"title\": \"x\", \"type\": \"quantitative\"}, {\"aggregate\": \"count\", \"title\": \"y\", \"type\": \"quantitative\"}], \"x\": {\"bin\": true, \"field\": \"value\", \"title\": \"\", \"type\": \"quantitative\"}, \"y\": {\"aggregate\": \"count\", \"title\": \"\", \"type\": \"quantitative\"}}, \"height\": 100, \"title\": \"timestamp\", \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"fold\": [\"anomaly_score\", \"actual\", \"typical\", \"event_count\", \"model_lower\", \"model_median\", \"model_upper\", \"timestamp\", \"result_type\"], \"as\": [\"attribute\", \"value\"]}, {\"filter\": \"(datum.attribute === 'timestamp')\"}], \"width\": 100}, {\"mark\": \"bar\", \"encoding\": {\"tooltip\": [{\"bin\": true, \"field\": \"value\", \"title\": \"x\", \"type\": \"quantitative\"}, {\"aggregate\": \"count\", \"title\": \"y\", \"type\": \"quantitative\"}], \"x\": {\"bin\": true, \"field\": \"value\", \"title\": \"\", \"type\": \"quantitative\"}, \"y\": {\"aggregate\": \"count\", \"title\": \"\", \"type\": \"quantitative\"}}, \"height\": 100, \"title\": \"typical\", \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"fold\": [\"anomaly_score\", \"actual\", \"typical\", \"event_count\", \"model_lower\", \"model_median\", \"model_upper\", \"timestamp\", \"result_type\"], \"as\": [\"attribute\", \"value\"]}, {\"filter\": \"(datum.attribute === 'typical')\"}], \"width\": 100}], \"columns\": 4, \"data\": {\"url\": \"https://walterra.github.io/jupyter2kibana/data/fq_single_count_15m.json\", \"format\": {\"property\": \"hits.hits\", \"type\": \"json\"}, \"name\": \"ml-anomalies\"}, \"resolve\": {\"axis\": {\"x\": \"independent\", \"y\": \"independent\"}, \"scale\": {\"x\": \"independent\", \"y\": \"independent\"}}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v4.17.0.json\"}, {\"mode\": \"vega-lite\"});\n",
       "</script>"
      ],
      "text/plain": [
       "alt.ConcatChart(...)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# URL as ES endpoint\n",
    "# url = 'http://localhost:9220/'+index_name+'/_search?size=1000'\n",
    "\n",
    "# URL static fallback\n",
    "url = 'https://walterra.github.io/jupyter2kibana/data/fq_single_count_15m.json'\n",
    "\n",
    "url_data = alt.Data(\n",
    "    url=url,\n",
    "    format=alt.DataFormat(property='hits.hits',type='json'),\n",
    "    name='ml-anomalies'\n",
    ")\n",
    "\n",
    "fields = [\n",
    "    'anomaly_score',\n",
    "    'actual',\n",
    "    'typical',\n",
    "    'event_count',\n",
    "    'model_lower',\n",
    "    'model_median',\n",
    "    'model_upper',\n",
    "    'timestamp',\n",
    "    'result_type'\n",
    "]\n",
    "\n",
    "rename_dict = dict((a, 'datum._source.'+a) for a in fields)\n",
    "\n",
    "base = alt.Chart(url_data).transform_calculate(**rename_dict)\n",
    "\n",
    "url_chart = base.transform_fold(\n",
    "    fields,\n",
    "    as_=['attribute', 'value']\n",
    ").mark_bar().encode(\n",
    "    alt.X('value:Q', bin=True, title=''),\n",
    "    alt.Y('count()', title=''),\n",
    "    tooltip=[\n",
    "        alt.Tooltip('value:Q', bin=True, title='x'),\n",
    "        alt.Tooltip('count()', title='y')\n",
    "    ]\n",
    ").properties(\n",
    "    width=100,\n",
    "    height=100\n",
    ")\n",
    "\n",
    "url_charts = alt.ConcatChart(\n",
    "    concat=[\n",
    "      url_chart.transform_filter(alt.datum.attribute == attribute).properties(title=attribute)\n",
    "      for attribute in sorted(fields)\n",
    "    ],\n",
    "    columns=4\n",
    ").resolve_axis(\n",
    "    x='independent',\n",
    "    y='independent'\n",
    ").resolve_scale(\n",
    "    x='independent', \n",
    "    y='independent'\n",
    ")\n",
    "\n",
    "url_charts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<div id=\"altair-viz-f0aec9f7e1194f449ad92e5d3629734b\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "  var VEGA_DEBUG = (typeof VEGA_DEBUG == \"undefined\") ? {} : VEGA_DEBUG;\n",
       "  (function(spec, embedOpt){\n",
       "    let outputDiv = document.currentScript.previousElementSibling;\n",
       "    if (outputDiv.id !== \"altair-viz-f0aec9f7e1194f449ad92e5d3629734b\") {\n",
       "      outputDiv = document.getElementById(\"altair-viz-f0aec9f7e1194f449ad92e5d3629734b\");\n",
       "    }\n",
       "    const paths = {\n",
       "      \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
       "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
       "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm//vega-lite@4.17.0?noext\",\n",
       "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm//vega-embed@6?noext\",\n",
       "    };\n",
       "\n",
       "    function maybeLoadScript(lib, version) {\n",
       "      var key = `${lib.replace(\"-\", \"\")}_version`;\n",
       "      return (VEGA_DEBUG[key] == version) ?\n",
       "        Promise.resolve(paths[lib]) :\n",
       "        new Promise(function(resolve, reject) {\n",
       "          var s = document.createElement('script');\n",
       "          document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "          s.async = true;\n",
       "          s.onload = () => {\n",
       "            VEGA_DEBUG[key] = version;\n",
       "            return resolve(paths[lib]);\n",
       "          };\n",
       "          s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
       "          s.src = paths[lib];\n",
       "        });\n",
       "    }\n",
       "\n",
       "    function showError(err) {\n",
       "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
       "      throw err;\n",
       "    }\n",
       "\n",
       "    function displayChart(vegaEmbed) {\n",
       "      vegaEmbed(outputDiv, spec, embedOpt)\n",
       "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
       "    }\n",
       "\n",
       "    if(typeof define === \"function\" && define.amd) {\n",
       "      requirejs.config({paths});\n",
       "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
       "    } else {\n",
       "      maybeLoadScript(\"vega\", \"5\")\n",
       "        .then(() => maybeLoadScript(\"vega-lite\", \"4.17.0\"))\n",
       "        .then(() => maybeLoadScript(\"vega-embed\", \"6\"))\n",
       "        .catch(showError)\n",
       "        .then(() => displayChart(vegaEmbed));\n",
       "    }\n",
       "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300, \"strokeWidth\": 0}, \"axis\": {\"domainColor\": \"#ddd\", \"grid\": true, \"gridColor\": \"#eee\", \"labelBound\": true, \"labelColor\": \"gray\", \"tickColor\": \"#ddd\"}, \"title\": {\"anchor\": \"start\", \"color\": \"gray\", \"fontSize\": 14, \"fontWeight\": \"bold\"}}, \"layer\": [{\"data\": {\"url\": \"https://walterra.github.io/jupyter2kibana/data/fq_single_count_15m_annotations.json\", \"format\": {\"property\": \"hits.hits\", \"type\": \"json\"}, \"name\": \"ml-annotations\"}, \"mark\": {\"type\": \"rect\", \"color\": \"#369\", \"opacity\": 0.15}, \"encoding\": {\"tooltip\": [{\"field\": \"annotation\", \"type\": \"nominal\"}], \"x\": {\"field\": \"timestamp\", \"type\": \"temporal\"}, \"x2\": {\"field\": \"end_timestamp\"}, \"y\": {\"field\": \"y\", \"type\": \"quantitative\"}, \"y2\": {\"field\": \"y2\"}}, \"transform\": [{\"calculate\": \"datum._source.annotation\", \"as\": \"annotation\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.end_timestamp\", \"as\": \"end_timestamp\"}, {\"calculate\": \"datum._source.job_id\", \"as\": \"job_id\"}, {\"filter\": \"(datum.job_id === 'fq_single_count_15m')\"}, {\"calculate\": \"350\", \"as\": \"y\"}, {\"calculate\": \"0\", \"as\": \"y2\"}]}, {\"layer\": [{\"mark\": {\"type\": \"area\", \"color\": \"lightgray\", \"opacity\": 0.5}, \"encoding\": {\"x\": {\"axis\": {\"grid\": false}, \"field\": \"timestamp\", \"scale\": {\"nice\": false, \"zero\": false}, \"title\": \"\", \"type\": \"temporal\"}, \"y\": {\"aggregate\": \"mean\", \"field\": \"model_upper\", \"type\": \"quantitative\"}, \"y2\": {\"aggregate\": \"mean\", \"field\": \"model_lower\"}}, \"height\": 300, \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"filter\": \"(datum.result_type === 'model_plot')\"}], \"width\": 800}, {\"mark\": {\"type\": \"line\", \"color\": \"gray\", \"opacity\": 0.5, \"strokeJoin\": \"round\"}, \"encoding\": {\"x\": {\"axis\": {\"grid\": false}, \"field\": \"timestamp\", \"scale\": {\"nice\": false, \"zero\": false}, \"title\": \"\", \"type\": \"temporal\"}, \"y\": {\"aggregate\": \"mean\", \"field\": \"model_median\", \"title\": \"\", \"type\": \"quantitative\"}}, \"height\": 300, \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"filter\": \"(datum.result_type === 'model_plot')\"}], \"width\": 800}, {\"mark\": {\"type\": \"line\", \"color\": \"#1f77b4\", \"strokeJoin\": \"round\"}, \"encoding\": {\"x\": {\"axis\": {\"grid\": false}, \"field\": \"timestamp\", \"scale\": {\"nice\": false, \"zero\": false}, \"title\": \"\", \"type\": \"temporal\"}, \"y\": {\"aggregate\": \"mean\", \"field\": \"actual\", \"type\": \"quantitative\"}}, \"height\": 300, \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"filter\": \"(datum.result_type === 'model_plot')\"}], \"width\": 800}, {\"mark\": {\"type\": \"point\", \"color\": \"red\", \"size\": 60}, \"encoding\": {\"x\": {\"axis\": {\"grid\": false}, \"field\": \"timestamp\", \"scale\": {\"nice\": false, \"zero\": false}, \"title\": \"\", \"type\": \"temporal\"}, \"y\": {\"field\": \"mean_actual\", \"type\": \"quantitative\"}}, \"height\": 300, \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"aggregate\": [{\"op\": \"mean\", \"field\": \"actual\", \"as\": \"mean_actual\"}, {\"op\": \"mean\", \"field\": \"anomaly_score\", \"as\": \"mean_anomaly_score\"}], \"groupby\": [\"timestamp\"]}, {\"filter\": \"(datum.mean_anomaly_score > 0)\"}], \"width\": 800}, {\"mark\": {\"type\": \"circle\", \"opacity\": 0, \"size\": 100}, \"encoding\": {\"tooltip\": [{\"field\": \"mean_actual\", \"type\": \"quantitative\"}, {\"field\": \"mean_anomaly_score\", \"type\": \"quantitative\"}, {\"field\": \"timestamp\", \"type\": \"temporal\"}], \"x\": {\"axis\": {\"grid\": false}, \"field\": \"timestamp\", \"scale\": {\"nice\": false, \"zero\": false}, \"title\": \"\", \"type\": \"temporal\"}, \"y\": {\"field\": \"mean_actual\", \"type\": \"quantitative\"}}, \"height\": 300, \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"aggregate\": [{\"op\": \"mean\", \"field\": \"actual\", \"as\": \"mean_actual\"}, {\"op\": \"mean\", \"field\": \"anomaly_score\", \"as\": \"mean_anomaly_score\"}], \"groupby\": [\"timestamp\"]}], \"width\": 800}], \"data\": {\"url\": \"https://walterra.github.io/jupyter2kibana/data/fq_single_count_15m.json\", \"format\": {\"property\": \"hits.hits\", \"type\": \"json\"}, \"name\": \"ml-anomalies\"}}], \"$schema\": \"https://vega.github.io/schema/vega-lite/v4.17.0.json\"}, {\"mode\": \"vega-lite\"});\n",
       "</script>"
      ],
      "text/plain": [
       "alt.LayerChart(...)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# custom: ['#1f77b4', 'lightgray', 'gray']\n",
    "# ml:     ['#32a7c2', '#c8e6ef', '#bbd7df']\n",
    "\n",
    "colors = {'model_bounds':'lightgray','model_median':'gray','actual':'#1f77b4'}\n",
    "\n",
    "base = alt.Chart(url_data, width=800, height=300).transform_calculate(**rename_dict)\n",
    "\n",
    "model_bounds = base.mark_area(color=colors['model_bounds'], opacity=0.5).transform_filter(\n",
    "    alt.datum.result_type == 'model_plot'\n",
    ").encode(\n",
    "    alt.X('timestamp:T', scale=alt.Scale(zero=False,nice=False), title='', axis=alt.Axis(grid=False)),\n",
    "    alt.Y('mean(model_upper):Q'),\n",
    "    alt.Y2('mean(model_lower):Q'),\n",
    ")\n",
    "\n",
    "model_median = base.mark_line(color=colors['model_median'], opacity=0.5, strokeJoin='round').transform_filter(\n",
    "    alt.datum.result_type == 'model_plot'\n",
    ").encode(\n",
    "    alt.X('timestamp:T', scale=alt.Scale(zero=False,nice=False), title='', axis=alt.Axis(grid=False)),\n",
    "    alt.Y('mean(model_median):Q', title='')\n",
    ")\n",
    "\n",
    "actual = base.mark_line(color=colors['actual'], strokeJoin='round').transform_filter(\n",
    "    alt.datum.result_type == 'model_plot'\n",
    ").encode(\n",
    "    alt.X('timestamp:T', scale=alt.Scale(zero=False,nice=False), title='', axis=alt.Axis(grid=False)),\n",
    "    alt.Y('mean(actual):Q')\n",
    ")\n",
    "\n",
    "anomalies = base.mark_point(color='red', size=60).transform_aggregate(\n",
    "    mean_actual='mean(actual)',\n",
    "    mean_anomaly_score='mean(anomaly_score)',\n",
    "    groupby=[\"timestamp\"]\n",
    ").transform_filter(\n",
    "    alt.datum.mean_anomaly_score > 0\n",
    ").encode(\n",
    "    alt.X('timestamp:T', scale=alt.Scale(zero=False,nice=False), title='', axis=alt.Axis(grid=False)),\n",
    "    alt.Y('mean_actual:Q')\n",
    ")\n",
    "\n",
    "tooltip = base.mark_circle(opacity=0, size=100).transform_aggregate(\n",
    "    mean_actual='mean(actual)',\n",
    "    mean_anomaly_score='mean(anomaly_score)',\n",
    "    groupby=[\"timestamp\"]\n",
    ").encode(\n",
    "    alt.X('timestamp:T', scale=alt.Scale(zero=False,nice=False), title='', axis=alt.Axis(grid=False)),\n",
    "    alt.Y('mean_actual:Q'),\n",
    "    tooltip=['mean_actual:Q', 'mean_anomaly_score:Q', 'timestamp:T']\n",
    ")\n",
    "\n",
    "chart = (annotations + (model_bounds + model_median + actual + anomalies + tooltip)).configure_axis(\n",
    "    grid=True,\n",
    "    gridColor=\"#eee\",\n",
    "    domainColor=\"#ddd\",\n",
    "    tickColor=\"#ddd\",\n",
    "    labelColor=\"gray\",\n",
    "    labelBound=True,\n",
    ").configure_view(\n",
    "    strokeWidth=0\n",
    ").configure_title(\n",
    "    fontSize=14,\n",
    "    fontWeight='bold',\n",
    "    anchor='start',\n",
    "    color='gray'\n",
    ")\n",
    "\n",
    "chart"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'$schema': 'https://vega.github.io/schema/vega-lite/v4.17.0.json',\n",
       " 'config': {'axis': {'domainColor': '#ddd',\n",
       "   'grid': True,\n",
       "   'gridColor': '#eee',\n",
       "   'labelBound': True,\n",
       "   'labelColor': 'gray',\n",
       "   'tickColor': '#ddd'},\n",
       "  'title': {'anchor': 'start',\n",
       "   'color': 'gray',\n",
       "   'fontSize': 14,\n",
       "   'fontWeight': 'bold'},\n",
       "  'view': {'continuousHeight': 300, 'continuousWidth': 400, 'strokeWidth': 0}},\n",
       " 'layer': [{'data': {'format': {'property': 'hits.hits', 'type': 'json'},\n",
       "    'name': 'ml-annotations',\n",
       "    'url': 'https://walterra.github.io/jupyter2kibana/data/fq_single_count_15m_annotations.json'},\n",
       "   'encoding': {'tooltip': [{'field': 'annotation', 'type': 'nominal'}],\n",
       "    'x': {'field': 'timestamp', 'type': 'temporal'},\n",
       "    'x2': {'field': 'end_timestamp'},\n",
       "    'y': {'field': 'y', 'type': 'quantitative'},\n",
       "    'y2': {'field': 'y2'}},\n",
       "   'mark': {'color': '#369', 'opacity': 0.15, 'type': 'rect'},\n",
       "   'transform': [{'as': 'annotation', 'calculate': 'datum._source.annotation'},\n",
       "    {'as': 'timestamp', 'calculate': 'datum._source.timestamp'},\n",
       "    {'as': 'end_timestamp', 'calculate': 'datum._source.end_timestamp'},\n",
       "    {'as': 'job_id', 'calculate': 'datum._source.job_id'},\n",
       "    {'filter': \"(datum.job_id === 'fq_single_count_15m')\"},\n",
       "    {'as': 'y', 'calculate': '350'},\n",
       "    {'as': 'y2', 'calculate': '0'}]},\n",
       "  {'data': {'format': {'property': 'hits.hits', 'type': 'json'},\n",
       "    'name': 'ml-anomalies',\n",
       "    'url': 'https://walterra.github.io/jupyter2kibana/data/fq_single_count_15m.json'},\n",
       "   'layer': [{'encoding': {'x': {'axis': {'grid': False},\n",
       "       'field': 'timestamp',\n",
       "       'scale': {'nice': False, 'zero': False},\n",
       "       'title': '',\n",
       "       'type': 'temporal'},\n",
       "      'y': {'aggregate': 'mean',\n",
       "       'field': 'model_upper',\n",
       "       'type': 'quantitative'},\n",
       "      'y2': {'aggregate': 'mean', 'field': 'model_lower'}},\n",
       "     'height': 300,\n",
       "     'mark': {'color': 'lightgray', 'opacity': 0.5, 'type': 'area'},\n",
       "     'transform': [{'as': 'anomaly_score',\n",
       "       'calculate': 'datum._source.anomaly_score'},\n",
       "      {'as': 'actual', 'calculate': 'datum._source.actual'},\n",
       "      {'as': 'typical', 'calculate': 'datum._source.typical'},\n",
       "      {'as': 'event_count', 'calculate': 'datum._source.event_count'},\n",
       "      {'as': 'model_lower', 'calculate': 'datum._source.model_lower'},\n",
       "      {'as': 'model_median', 'calculate': 'datum._source.model_median'},\n",
       "      {'as': 'model_upper', 'calculate': 'datum._source.model_upper'},\n",
       "      {'as': 'timestamp', 'calculate': 'datum._source.timestamp'},\n",
       "      {'as': 'result_type', 'calculate': 'datum._source.result_type'},\n",
       "      {'filter': \"(datum.result_type === 'model_plot')\"}],\n",
       "     'width': 800},\n",
       "    {'encoding': {'x': {'axis': {'grid': False},\n",
       "       'field': 'timestamp',\n",
       "       'scale': {'nice': False, 'zero': False},\n",
       "       'title': '',\n",
       "       'type': 'temporal'},\n",
       "      'y': {'aggregate': 'mean',\n",
       "       'field': 'model_median',\n",
       "       'title': '',\n",
       "       'type': 'quantitative'}},\n",
       "     'height': 300,\n",
       "     'mark': {'color': 'gray',\n",
       "      'opacity': 0.5,\n",
       "      'strokeJoin': 'round',\n",
       "      'type': 'line'},\n",
       "     'transform': [{'as': 'anomaly_score',\n",
       "       'calculate': 'datum._source.anomaly_score'},\n",
       "      {'as': 'actual', 'calculate': 'datum._source.actual'},\n",
       "      {'as': 'typical', 'calculate': 'datum._source.typical'},\n",
       "      {'as': 'event_count', 'calculate': 'datum._source.event_count'},\n",
       "      {'as': 'model_lower', 'calculate': 'datum._source.model_lower'},\n",
       "      {'as': 'model_median', 'calculate': 'datum._source.model_median'},\n",
       "      {'as': 'model_upper', 'calculate': 'datum._source.model_upper'},\n",
       "      {'as': 'timestamp', 'calculate': 'datum._source.timestamp'},\n",
       "      {'as': 'result_type', 'calculate': 'datum._source.result_type'},\n",
       "      {'filter': \"(datum.result_type === 'model_plot')\"}],\n",
       "     'width': 800},\n",
       "    {'encoding': {'x': {'axis': {'grid': False},\n",
       "       'field': 'timestamp',\n",
       "       'scale': {'nice': False, 'zero': False},\n",
       "       'title': '',\n",
       "       'type': 'temporal'},\n",
       "      'y': {'aggregate': 'mean', 'field': 'actual', 'type': 'quantitative'}},\n",
       "     'height': 300,\n",
       "     'mark': {'color': '#1f77b4', 'strokeJoin': 'round', 'type': 'line'},\n",
       "     'transform': [{'as': 'anomaly_score',\n",
       "       'calculate': 'datum._source.anomaly_score'},\n",
       "      {'as': 'actual', 'calculate': 'datum._source.actual'},\n",
       "      {'as': 'typical', 'calculate': 'datum._source.typical'},\n",
       "      {'as': 'event_count', 'calculate': 'datum._source.event_count'},\n",
       "      {'as': 'model_lower', 'calculate': 'datum._source.model_lower'},\n",
       "      {'as': 'model_median', 'calculate': 'datum._source.model_median'},\n",
       "      {'as': 'model_upper', 'calculate': 'datum._source.model_upper'},\n",
       "      {'as': 'timestamp', 'calculate': 'datum._source.timestamp'},\n",
       "      {'as': 'result_type', 'calculate': 'datum._source.result_type'},\n",
       "      {'filter': \"(datum.result_type === 'model_plot')\"}],\n",
       "     'width': 800},\n",
       "    {'encoding': {'x': {'axis': {'grid': False},\n",
       "       'field': 'timestamp',\n",
       "       'scale': {'nice': False, 'zero': False},\n",
       "       'title': '',\n",
       "       'type': 'temporal'},\n",
       "      'y': {'field': 'mean_actual', 'type': 'quantitative'}},\n",
       "     'height': 300,\n",
       "     'mark': {'color': 'red', 'size': 60, 'type': 'point'},\n",
       "     'transform': [{'as': 'anomaly_score',\n",
       "       'calculate': 'datum._source.anomaly_score'},\n",
       "      {'as': 'actual', 'calculate': 'datum._source.actual'},\n",
       "      {'as': 'typical', 'calculate': 'datum._source.typical'},\n",
       "      {'as': 'event_count', 'calculate': 'datum._source.event_count'},\n",
       "      {'as': 'model_lower', 'calculate': 'datum._source.model_lower'},\n",
       "      {'as': 'model_median', 'calculate': 'datum._source.model_median'},\n",
       "      {'as': 'model_upper', 'calculate': 'datum._source.model_upper'},\n",
       "      {'as': 'timestamp', 'calculate': 'datum._source.timestamp'},\n",
       "      {'as': 'result_type', 'calculate': 'datum._source.result_type'},\n",
       "      {'aggregate': [{'as': 'mean_actual', 'field': 'actual', 'op': 'mean'},\n",
       "        {'as': 'mean_anomaly_score', 'field': 'anomaly_score', 'op': 'mean'}],\n",
       "       'groupby': ['timestamp']},\n",
       "      {'filter': '(datum.mean_anomaly_score > 0)'}],\n",
       "     'width': 800},\n",
       "    {'encoding': {'tooltip': [{'field': 'mean_actual', 'type': 'quantitative'},\n",
       "       {'field': 'mean_anomaly_score', 'type': 'quantitative'},\n",
       "       {'field': 'timestamp', 'type': 'temporal'}],\n",
       "      'x': {'axis': {'grid': False},\n",
       "       'field': 'timestamp',\n",
       "       'scale': {'nice': False, 'zero': False},\n",
       "       'title': '',\n",
       "       'type': 'temporal'},\n",
       "      'y': {'field': 'mean_actual', 'type': 'quantitative'}},\n",
       "     'height': 300,\n",
       "     'mark': {'opacity': 0, 'size': 100, 'type': 'circle'},\n",
       "     'transform': [{'as': 'anomaly_score',\n",
       "       'calculate': 'datum._source.anomaly_score'},\n",
       "      {'as': 'actual', 'calculate': 'datum._source.actual'},\n",
       "      {'as': 'typical', 'calculate': 'datum._source.typical'},\n",
       "      {'as': 'event_count', 'calculate': 'datum._source.event_count'},\n",
       "      {'as': 'model_lower', 'calculate': 'datum._source.model_lower'},\n",
       "      {'as': 'model_median', 'calculate': 'datum._source.model_median'},\n",
       "      {'as': 'model_upper', 'calculate': 'datum._source.model_upper'},\n",
       "      {'as': 'timestamp', 'calculate': 'datum._source.timestamp'},\n",
       "      {'as': 'result_type', 'calculate': 'datum._source.result_type'},\n",
       "      {'aggregate': [{'as': 'mean_actual', 'field': 'actual', 'op': 'mean'},\n",
       "        {'as': 'mean_anomaly_score', 'field': 'anomaly_score', 'op': 'mean'}],\n",
       "       'groupby': ['timestamp']}],\n",
       "     'width': 800}]}]}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "json.loads(chart.to_json())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [409]>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "import requests\n",
    "\n",
    "resultSize=10000\n",
    "visName='ml-anomalies-fq-annotations'\n",
    "\n",
    "chart_json = json.loads(chart.to_json())\n",
    "chart_json['layer'][0]['data']['url'] = {\n",
    "    \"%context%\": True,\n",
    "    \"%timefield%\": \"timestamp\",\n",
    "    \"index\": annotation_index_name,\n",
    "    \"body\": {\n",
    "        \"size\": resultSize\n",
    "    }\n",
    "}\n",
    "chart_json['layer'][1]['data']['url'] = {\n",
    "    \"%context%\": True,\n",
    "    \"%timefield%\": \"timestamp\",\n",
    "    \"index\": index_name,\n",
    "    \"body\": {\n",
    "        \"size\": resultSize\n",
    "    }\n",
    "}\n",
    "\n",
    "visState = {\n",
    "  \"type\": \"vega\",\n",
    "  \"aggs\": [],\n",
    "  \"params\": {\n",
    "    \"spec\": json.dumps(chart_json, sort_keys=True, indent=4, separators=(',', ': ')),\n",
    "  },\n",
    "  \"title\": visName\n",
    "}\n",
    "\n",
    "visSavedObject={\n",
    "    \"attributes\" : {\n",
    "      \"title\" : visName,\n",
    "      \"visState\" : json.dumps(visState, sort_keys=True, indent=4, separators=(',', ': ')),\n",
    "      \"uiStateJSON\" : \"{}\",\n",
    "      \"description\" : \"\",\n",
    "      \"version\" : 1,\n",
    "      \"kibanaSavedObjectMeta\" : {\n",
    "        \"searchSourceJSON\" : json.dumps({\n",
    "          \"query\": {\n",
    "            \"language\": \"kuery\",\n",
    "            \"query\": \"\"\n",
    "          },\n",
    "          \"filter\": []\n",
    "        }),\n",
    "      }\n",
    "    },\n",
    "}\n",
    "\n",
    "\n",
    "requests.post(\n",
    "    es_config['kibana_client'] + '/api/saved_objects/visualization/' + visName,\n",
    "    json=visSavedObject,\n",
    "    auth=(es_config['user'], es_config['password']),\n",
    "    headers={\"kbn-xsrf\":\"jupyter2kibana\"},\n",
    "    # Only in development environments with self signed certificates fall back to use `verify=False`\n",
    "    verify=False\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env-jupyter2kibana",
   "language": "python",
   "name": "env-jupyter2kibana"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}