viz-2b-cars-outlier.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Home](https://walterra.github.io/jupyter2kibana/)\n",
    "\n",
    "*To make sure the Vega charts render correctly, view the notebook not from the Github repo but the published website here: \n",
    "https://walterra.github.io/jupyter2kibana/viz-2b-cars-outlier.html*\n",
    "\n",
    "# viz-2b-cars-outlier.ipynb\n",
    "\n",
    "This notebook uses an index that was created using Elasticsearch's Machine Learning to detect outliers on the cars dataset. This index including ML's metadata is used to create a scatterplot matrix that highlights outliers. Additionally it includes a Vega based slider to adjust the threshold for highlighting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import altair as alt\n",
    "import eland as ed\n",
    "from elasticsearch import Elasticsearch\n",
    "import elastic_transport\n",
    "import logging\n",
    "import json\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import urllib3\n",
    "import warnings\n",
    "\n",
    "alt.data_transformers.disable_max_rows()\n",
    "logging.getLogger(\"elastic_transport\").setLevel(logging.ERROR)\n",
    "\n",
    "# Suppress insecure SSL connection warnings\n",
    "# In dev environments with the `verify_certs=False`\n",
    "# you might want to reduce those warnings.\n",
    "urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)\n",
    "urllib3.disable_warnings(elastic_transport.SecurityWarning)\n",
    "\n",
    "# For rendering the notebook to HTML hide all warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "index_name = 'cars_outlier'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Acceleration</th>\n",
       "      <th>Cylinders</th>\n",
       "      <th>Displacement</th>\n",
       "      <th>Horsepower</th>\n",
       "      <th>Miles_per_Gallon</th>\n",
       "      <th>Name</th>\n",
       "      <th>Origin</th>\n",
       "      <th>Weight_in_lbs</th>\n",
       "      <th>Year</th>\n",
       "      <th>ml.feature_influence</th>\n",
       "      <th>ml.feature_influence.feature_name</th>\n",
       "      <th>ml.feature_influence.influence</th>\n",
       "      <th>ml.outlier_score</th>\n",
       "      <th>ml__incremental_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12.0</td>\n",
       "      <td>8</td>\n",
       "      <td>307.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>chevrolet chevelle malibu</td>\n",
       "      <td>USA</td>\n",
       "      <td>3504</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.059161</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11.5</td>\n",
       "      <td>8</td>\n",
       "      <td>350.0</td>\n",
       "      <td>165.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>buick skylark 320</td>\n",
       "      <td>USA</td>\n",
       "      <td>3693</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.044278</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11.0</td>\n",
       "      <td>8</td>\n",
       "      <td>318.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>plymouth satellite</td>\n",
       "      <td>USA</td>\n",
       "      <td>3436</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.031178</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>12.0</td>\n",
       "      <td>8</td>\n",
       "      <td>304.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>amc rebel sst</td>\n",
       "      <td>USA</td>\n",
       "      <td>3433</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.033840</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10.5</td>\n",
       "      <td>8</td>\n",
       "      <td>302.0</td>\n",
       "      <td>140.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>ford torino</td>\n",
       "      <td>USA</td>\n",
       "      <td>3449</td>\n",
       "      <td>1970-01-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.042260</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "<p>5 rows × 14 columns</p>"
      ],
      "text/plain": [
       "   Acceleration  Cylinders  Displacement  Horsepower  Miles_per_Gallon  \\\n",
       "0          12.0          8         307.0       130.0              18.0   \n",
       "1          11.5          8         350.0       165.0              15.0   \n",
       "2          11.0          8         318.0       150.0              18.0   \n",
       "3          12.0          8         304.0       150.0              16.0   \n",
       "4          10.5          8         302.0       140.0              17.0   \n",
       "\n",
       "                        Name Origin  Weight_in_lbs       Year  \\\n",
       "0  chevrolet chevelle malibu    USA           3504 1970-01-01   \n",
       "1          buick skylark 320    USA           3693 1970-01-01   \n",
       "2         plymouth satellite    USA           3436 1970-01-01   \n",
       "3              amc rebel sst    USA           3433 1970-01-01   \n",
       "4                ford torino    USA           3449 1970-01-01   \n",
       "\n",
       "  ml.feature_influence ml.feature_influence.feature_name  \\\n",
       "0                  NaN                               NaN   \n",
       "1                  NaN                               NaN   \n",
       "2                  NaN                               NaN   \n",
       "3                  NaN                               NaN   \n",
       "4                  NaN                               NaN   \n",
       "\n",
       "   ml.feature_influence.influence  ml.outlier_score  ml__incremental_id  \n",
       "0                             NaN          0.059161                   0  \n",
       "1                             NaN          0.044278                   1  \n",
       "2                             NaN          0.031178                   2  \n",
       "3                             NaN          0.033840                   3  \n",
       "4                             NaN          0.042260                   4  \n",
       "\n",
       "[5 rows x 14 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('config.json') as config_file:\n",
    "  es_config = json.load(config_file)\n",
    "\n",
    "# First instantiate an 'Elasticsearch' instance with the supplied config\n",
    "es = Elasticsearch(\n",
    "    hosts=[es_config['es_client']],\n",
    "    basic_auth=[\n",
    "        es_config['user'],\n",
    "        es_config['password']\n",
    "    ],\n",
    "    # Only in development environments with self signed certificates fall back to use `verify_certs=False`\n",
    "    verify_certs=False\n",
    ")\n",
    "\n",
    "ed_df = ed.DataFrame(es, index_name)\n",
    "ed_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<div id=\"altair-viz-44d916b851fb4becac8bb39489dc24ab\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "  var VEGA_DEBUG = (typeof VEGA_DEBUG == \"undefined\") ? {} : VEGA_DEBUG;\n",
       "  (function(spec, embedOpt){\n",
       "    let outputDiv = document.currentScript.previousElementSibling;\n",
       "    if (outputDiv.id !== \"altair-viz-44d916b851fb4becac8bb39489dc24ab\") {\n",
       "      outputDiv = document.getElementById(\"altair-viz-44d916b851fb4becac8bb39489dc24ab\");\n",
       "    }\n",
       "    const paths = {\n",
       "      \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
       "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
       "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm//vega-lite@4.17.0?noext\",\n",
       "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm//vega-embed@6?noext\",\n",
       "    };\n",
       "\n",
       "    function maybeLoadScript(lib, version) {\n",
       "      var key = `${lib.replace(\"-\", \"\")}_version`;\n",
       "      return (VEGA_DEBUG[key] == version) ?\n",
       "        Promise.resolve(paths[lib]) :\n",
       "        new Promise(function(resolve, reject) {\n",
       "          var s = document.createElement('script');\n",
       "          document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "          s.async = true;\n",
       "          s.onload = () => {\n",
       "            VEGA_DEBUG[key] = version;\n",
       "            return resolve(paths[lib]);\n",
       "          };\n",
       "          s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
       "          s.src = paths[lib];\n",
       "        });\n",
       "    }\n",
       "\n",
       "    function showError(err) {\n",
       "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
       "      throw err;\n",
       "    }\n",
       "\n",
       "    function displayChart(vegaEmbed) {\n",
       "      vegaEmbed(outputDiv, spec, embedOpt)\n",
       "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
       "    }\n",
       "\n",
       "    if(typeof define === \"function\" && define.amd) {\n",
       "      requirejs.config({paths});\n",
       "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
       "    } else {\n",
       "      maybeLoadScript(\"vega\", \"5\")\n",
       "        .then(() => maybeLoadScript(\"vega-lite\", \"4.17.0\"))\n",
       "        .then(() => maybeLoadScript(\"vega-embed\", \"6\"))\n",
       "        .catch(showError)\n",
       "        .then(() => displayChart(vegaEmbed));\n",
       "    }\n",
       "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300}}, \"repeat\": {\"column\": [\"Miles_per_Gallon\", \"Acceleration\", \"Horsepower\"], \"row\": [\"Horsepower\", \"Acceleration\", \"Miles_per_Gallon\"]}, \"spec\": {\"data\": {\"url\": \"https://walterra.github.io/jupyter2kibana/data/cars_outlier.json\", \"format\": {\"property\": \"hits.hits\", \"type\": \"json\"}}, \"mark\": \"point\", \"encoding\": {\"color\": {\"condition\": {\"value\": \"red\", \"test\": \"(datum['ml.outlier_score'] >= SelectorName.cutoff)\"}, \"value\": \"gray\"}, \"opacity\": {\"condition\": {\"value\": 0.75, \"test\": \"(datum['ml.outlier_score'] >= SelectorName.cutoff)\"}, \"value\": 0.25}, \"size\": {\"condition\": {\"value\": 28, \"test\": \"(datum['ml.outlier_score'] >= SelectorName.cutoff)\"}, \"value\": 2}, \"tooltip\": [{\"field\": \"Name\", \"type\": \"nominal\"}, {\"field\": \"ml.outlier_score\", \"type\": \"quantitative\"}, {\"field\": \"Horsepower\", \"type\": \"quantitative\"}, {\"field\": \"Acceleration\", \"type\": \"quantitative\"}, {\"field\": \"Miles_per_Gallon\", \"type\": \"quantitative\"}], \"x\": {\"field\": {\"repeat\": \"column\"}, \"type\": \"quantitative\"}, \"y\": {\"field\": {\"repeat\": \"row\"}, \"type\": \"quantitative\"}}, \"height\": 150, \"selection\": {\"selector001\": {\"type\": \"interval\", \"bind\": \"scales\", \"encodings\": [\"x\", \"y\"]}, \"SelectorName\": {\"type\": \"single\", \"fields\": [\"cutoff\"], \"bind\": {\"input\": \"range\", \"max\": 1, \"min\": 0, \"name\": \"Outlier score Threshold:\", \"step\": 0.01}, \"init\": {\"cutoff\": 0.8}}}, \"transform\": [{\"calculate\": \"datum._source.Acceleration\", \"as\": \"Acceleration\"}, {\"calculate\": \"datum._source.Cylinders\", \"as\": \"Cylinders\"}, {\"calculate\": \"datum._source.Displacement\", \"as\": \"Displacement\"}, {\"calculate\": \"datum._source.Horsepower\", \"as\": \"Horsepower\"}, {\"calculate\": \"datum._source.Miles_per_Gallon\", \"as\": \"Miles_per_Gallon\"}, {\"calculate\": \"datum._source.Name\", \"as\": \"Name\"}, {\"calculate\": \"datum._source.Origin\", \"as\": \"Origin\"}, {\"calculate\": \"datum._source.Weight_in_lbs\", \"as\": \"Weight_in_lbs\"}, {\"calculate\": \"datum._source.Year\", \"as\": \"Year\"}, {\"calculate\": \"datum._source.ml.outlier_score\", \"as\": \"ml.outlier_score\"}], \"width\": 150}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v4.17.0.json\"}, {\"mode\": \"vega-lite\"});\n",
       "</script>"
      ],
      "text/plain": [
       "alt.RepeatChart(...)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Note: To create the Vega spec using Altair we reference ES via URL first. This will only work\n",
    "# for non-secured ES instances. If your ES instance runs using SSL and/or authentication the chart\n",
    "# in this cell will render empty. You can still save the visualization in Kibana correctly in the\n",
    "# next cell because there the URL gets replaced with an Elasticsearch query\n",
    "# to be used via the Kibana Vega plugin.\n",
    "\n",
    "# WARNING:\n",
    "# Do the following approach using a proxy only for demo purposes in a development environment.\n",
    "# It will expose a secured ES instance unsecured!\n",
    "# To make this work for demo purposes run the nodejs based proxy in a separate terminal like this:\n",
    "# NODE_TLS_REJECT_UNAUTHORIZED='0' node proxy\n",
    "\n",
    "# URL as ES endpoint\n",
    "# url = 'http://localhost:9220/'+index_name+'/_search?size=1000'\n",
    "\n",
    "# URL static fallback\n",
    "url = 'https://walterra.github.io/jupyter2kibana/data/cars_outlier.json'\n",
    "\n",
    "url_data = alt.Data(url=url, format=alt.DataFormat(property='hits.hits',type='json'))\n",
    "\n",
    "fields = ['Acceleration', 'Cylinders', 'Displacement', 'Horsepower',\n",
    "       'Miles_per_Gallon', 'Name', 'Origin', 'Weight_in_lbs', 'Year','ml.outlier_score']\n",
    "\n",
    "rename_dict = dict((a, 'datum._source.'+a) for a in fields)\n",
    "\n",
    "slider = alt.binding_range(min=0, max=1, step=.01, name='Outlier score Threshold:')\n",
    "selector = alt.selection_single(name=\"SelectorName\", fields=['cutoff'],\n",
    "                                bind=slider, init={'cutoff': .8})\n",
    "\n",
    "chart = alt.Chart(url_data).transform_calculate(**rename_dict).mark_point().encode(\n",
    "    alt.X(alt.repeat(\"column\"), type='quantitative'),\n",
    "    alt.Y(alt.repeat(\"row\"), type='quantitative'),\n",
    "    color=alt.condition(\n",
    "        alt.datum[\"ml.outlier_score\"] >= selector.cutoff,\n",
    "        alt.value('red'), alt.value('gray')\n",
    "    ),\n",
    "    opacity=alt.condition(\n",
    "        alt.datum[\"ml.outlier_score\"] >= selector.cutoff,\n",
    "        alt.value(.75), alt.value(.25)\n",
    "    ),\n",
    "    size=alt.condition(\n",
    "        alt.datum[\"ml.outlier_score\"] >= selector.cutoff,\n",
    "        alt.value(28), alt.value(2)\n",
    "    ),\n",
    "    tooltip=['Name:N', 'ml.outlier_score:Q', 'Horsepower:Q', 'Acceleration:Q', 'Miles_per_Gallon:Q']\n",
    ").properties(\n",
    "    width=150,\n",
    "    height=150\n",
    ").repeat(\n",
    "    row=['Horsepower', 'Acceleration', 'Miles_per_Gallon'],\n",
    "    column=['Miles_per_Gallon', 'Acceleration', 'Horsepower']\n",
    ").interactive().add_selection(\n",
    "    selector\n",
    ")\n",
    "\n",
    "chart"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [409]>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from kibana_vega_util import saveVegaVis\n",
    "\n",
    "saveVegaVis(\n",
    "    index_name,\n",
    "    'def-vega-cars-outlier-1',\n",
    "    chart,\n",
    "    resultSize=1000,\n",
    "    # Only in development environments with self signed certificates fall back to use `verify=False`\n",
    "    verify=False,\n",
    "    timeField=\"Year\"\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env-jupyter2kibana",
   "language": "python",
   "name": "env-jupyter2kibana"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}