Log Pipeline Extensions

elasticsearch-init/Dockerfile

@@ -0,0 +1,9 @@
+FROM appropriate/curl
+
+ENV ELASTICSEARCH_URI=elasticsearch:9200 \
+  ELASTICSEARCH_TIMEOUT=60
+
+COPY wait-for.sh upload.sh /
+RUN chmod +x /wait-for.sh /upload.sh
+
+ENTRYPOINT /wait-for.sh ${ELASTICSEARCH_URI} && /upload.sh

elasticsearch-init/README.md

@@ -0,0 +1,41 @@
+elasticsearch-init
+==================
+
+A container for loading the templates to ElasticSearch.
+
+Tags
+----
+
+**elasticsearch-init** uses simple [SemVer][3] tags as follows:
+
+* `0.0.1` - `latest`
+
+Usage
+-----
+
+**elasticsearch-init** leverages the [Docker volumes][1]. Each template,
+that image is supposed to upload to [ElasticSearch][2], is represented as single file
+mounted to ```/templates/``` directory inside the container.
+Another point, to keep in mind, is that template uploading requires also a **template name**.
+That bit is equal to the filename that holds the template. For instance:
+
+```docker run -v ./tpls/logs.json:/templates/logs -l elasticsearch monasca/elasticsearch-init```
+
+means that:
+
+* content of ```/template/logs``` will be uploaded to [ElasticSearch][2]
+* template name will be ```logs```
+
+Configuration
+-------------
+
+A number of environment variables can be passed to the container:
+
+| Variable                  | Default      | Description                       |
+|---------------------------|--------------|-----------------------------------|
+| `ELASTICSEARCH_URI`       | `elasticsearch:9200` | URI to connect to ES      |
+| `ELASTICSEARCH_TIMEOUT`   | `60`         | How long to wait for ElasticSearch connection |
+
+[1]: https://docs.docker.com/engine/tutorials/dockervolumes/
+[2]: https://hub.docker.com/_/elasticsearch/
+[3]: http://semver.org/

elasticsearch-init/build.yml

@@ -0,0 +1,5 @@
+repository: monasca/elasticsearch-init
+variants:
+  - tag: 0.0.1
+    aliases:
+      - :latest

elasticsearch-init/upload.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+TPL_DIR=/templates
+
+_get_tpl_name_from_file() {
+    local tpl=$1
+    local tpl_name=$(basename $tpl)
+    echo $tpl_name
+}
+
+if [ ! -d $TPL_DIR ]; then
+    echo "No templates mounted"
+    exit 0
+else
+    TPLS=$(ls $TPL_DIR)
+fi
+
+for template in $TPLS; do
+
+    echo "Handling template file $template"
+    tpl_name=`_get_tpl_name_from_file $template`
+
+    curl -XPUT --retry 2 --retry-delay 2 $ELASTICSEARCH_URI/_template/${tpl_name} -d @$TPL_DIR/$template
+
+done
+
+

elasticsearch-init/wait-for.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+: ${SLEEP_LENGTH:=2}
+
+wait_for() {
+  echo Waiting for $1 to listen on $2...
+  while ! nc -z $1 $2; do echo sleeping; sleep $SLEEP_LENGTH; done
+}
+
+for var in "$@"
+do
+  host=${var%:*}
+  port=${var#*:}
+  wait_for $host $port
+done

elasticsearch-templates/logs.template

@@ -0,0 +1,63 @@
+  {
+    "order" : 0,
+    "template" : "*",
+    "settings" : {
+      "index.refresh_interval" : "5s",
+      "index" : {
+        "analysis": {
+          "analyzer" : {
+            "no_token" : {
+              "type" : "custom",
+              "tokenizer" : "keyword",
+              "filter" : "lowercase"
+            }
+          }
+        }
+      }
+    },
+    "mappings" : {
+      "_default_" : {
+        "dynamic_templates" : [ {
+          "message_field" : {
+            "mapping" : {
+              "index" : "analyzed",
+              "omit_norms" : true,
+              "type" : "string"
+            },
+            "match_mapping_type" : "string",
+            "match" : "message"
+          }
+        }, {
+          "string_fields" : {
+            "mapping" : {
+              "analyzer" : "no_token",
+              "omit_norms" : true,
+              "type" : "string"
+            },
+            "match_mapping_type" : "string",
+            "match" : "*"
+          }
+        } ],
+        "properties" : {
+          "geoip" : {
+            "dynamic" : true,
+            "properties" : {
+              "location" : {
+                "type" : "geo_point"
+              }
+            },
+            "type" : "object"
+          },
+          "@version" : {
+            "index" : "not_analyzed",
+            "type" : "string"
+          }
+        },
+        "_all" : {
+          "enabled" : true,
+          "omit_norms" : true
+        }
+      }
+    },
+    "aliases" : { }
+  }

kibana/Dockerfile

@@ -0,0 +1,43 @@
+ARG KIBANA_VERSION="4.6.3"
+
+FROM node:4-alpine
+
+ARG KIBANA_PLUGIN_VER=""
+ARG KIBANA_PLUGIN_REPO=https://github.com/openstack/monasca-kibana-plugin.git
+ARG KIBANA_PLUGIN_BRANCH=master
+
+WORKDIR /mkp/
+
+ARG REBUILD_PLUGIN_DEPS=1
+RUN apk add --no-cache --virtual build-dep git rsync
+
+ARG REBUILD_PLUGIN_PLUGIN=1
+RUN git clone $KIBANA_PLUGIN_REPO --depth 1 --branch $KIBANA_PLUGIN_BRANCH monasca-kibana-plugin && \
+  cd monasca-kibana-plugin && \
+  npm install --quiet && \
+  npm run package --quiet && \
+  KIBANA_PLUGIN_VER=$(node -e "console.log(require('./package.json').version)") && \
+  mv target/monasca-kibana-plugin-${KIBANA_PLUGIN_VER}.tar.gz /monasca-kibana-plugin.tar.gz && \
+  cd / && \
+  rm -rf /mpk/monasca-kibana-plugin && \
+  apk del build-dep
+
+FROM kibana:${KIBANA_VERSION}
+
+ENV KEYSTONE_URI=keystone:5000 \
+    MONASCA_PLUGIN_ENABLED=False
+
+WORKDIR /
+
+ARG REBUILD_FILES=1
+COPY --from=0 /monasca-kibana-plugin.tar.gz .
+COPY wait-for plugin_config start /
+RUN chmod +x /wait-for /plugin_config /start
+
+ARG REBUILD_INSTALL_PLUGIN=1
+RUN /plugin_config && \
+  kibana plugin -r monasca-kibana-plugin && \
+  kibana plugin -i monasca-kibana-plugin -u file:///monasca-kibana-plugin.tar.gz && \
+  rm -rf /monasca-kibana-plugin.tar.gz
+
+CMD /start

kibana/README.md

@@ -0,0 +1,42 @@
+Kibana 4 with monasca-kibana-plugin
+===================================
+
+Image extends official [Kibana image][1] with [monasca-kibana-plugin][2].
+For more details about the aformentioned plugin, please visit plugin's
+[repository][2].
+
+Tags
+----
+
+Tags in this image match tags in the official [Kibana image][1]. 
+This is dictated by the fact that plugin should be build for specific
+Kibana version. Each tag includes also ``KIBANA_PLUGIN_BRANCH`` information
+detail. Final tag is represented as ``{KIBANA_VERSION}-{KIBANA_PLUGIN_BRANCH}``.
+That said, tagging presents as follows:
+
+* `4.6.3-master` [`4.6-master`, `4-master`] - corresponds to **Kibana 4.6.3**
+  [monasca-kibana-plugin][2] build from **master** branch.
+
+Configuration
+-------------
+
+| Variable                       |   Default       | Description                       |
+|--------------------------------|-----------------|-----------------------------------|
+| `KEYSTONE_URI`                 | `keystone:5000` | An URI to Keystone Admin Endpoint |
+| `MONASCA_PLUGIN_ENABLED`       | `False`         | Should the plugin be enabled or disabled |
+
+Usage
+-----
+
+To use this image, first you need to deploy [ElasticSearch][3] and [Keystone][4].
+
+    Keystone is required only if ``MONASCA_PLUGIN_ENABLED`` 
+    is set to ``True``
+
+Once prerequisites are met, image can ba launched, for example, with:
+```docker run -it --rm -l elasticsearch monasca/kibana:4.6.3-master``` 
+
+[1]: https://hub.docker.com/r/library/kibana 
+[2]: https://github.com/openstack/monasca-kibana-plugin
+[3]: https://hub.docker.com/r/library/elasticsearch
+[4]: https://github.com/monasca/monasca-docker/keystone

kibana/build.yml

@@ -0,0 +1,10 @@
+repository: monasca/kibana
+variants:
+  - tag: latest
+    aliases:
+      - :4.6.3-master
+      - :4.6-master
+      - :4-master
+    args:
+      KIBANA_VERSION: 4.6.3
+      KIBANA_PLUGIN_BRANCH: master

kibana/plugin_config

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+echo "#monasca-kibana-plugin
+monasca-kibana-plugin.auth_uri: http://${KEYSTONE_URI}
+monasca-kibana-plugin.enabled: False
+monasca-kibana-plugin.cookie.isSecure: False
+" >> /opt/kibana/config/kibana.yml

kibana/start

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+sed -e "
+    s|monasca-kibana-plugin.enabled:\sFalse|monasca-kibana-plugin.enabled: ${MONASCA_PLUGIN_ENABLED}|g;
+" -i /opt/kibana/config/kibana.yml
+
+if [ $MONASCA_PLUGIN_ENABLED == True ]; then
+    /wait-for $KEYSTONE_URI -- kibana
+else
+    kibana
+fi