diff --git a/elasticsearch-init/Dockerfile b/elasticsearch-init/Dockerfile new file mode 100644 index 000000000..02a42bc36 --- /dev/null +++ b/elasticsearch-init/Dockerfile @@ -0,0 +1,9 @@ +FROM appropriate/curl + +ENV ELASTICSEARCH_URI=elasticsearch:9200 \ + ELASTICSEARCH_TIMEOUT=60 + +COPY wait-for.sh upload.sh / +RUN chmod +x /wait-for.sh /upload.sh + +ENTRYPOINT /wait-for.sh ${ELASTICSEARCH_URI} && /upload.sh diff --git a/elasticsearch-init/README.md b/elasticsearch-init/README.md new file mode 100644 index 000000000..02175ee23 --- /dev/null +++ b/elasticsearch-init/README.md @@ -0,0 +1,41 @@ +elasticsearch-init +================== + +A container for loading the templates to ElasticSearch. + +Tags +---- + +**elasticsearch-init** uses simple [SemVer][3] tags as follows: + +* `0.0.1` - `latest` + +Usage +----- + +**elasticsearch-init** leverages the [Docker volumes][1]. Each template, +that image is supposed to upload to [ElasticSearch][2], is represented as single file +mounted to ```/templates/``` directory inside the container. +Another point, to keep in mind, is that template uploading requires also a **template name**. +That bit is equal to the filename that holds the template. For instance: + +```docker run -v ./tpls/logs.json:/templates/logs -l elasticsearch monasca/elasticsearch-init``` + +means that: + +* content of ```/template/logs``` will be uploaded to [ElasticSearch][2] +* template name will be ```logs``` + +Configuration +------------- + +A number of environment variables can be passed to the container: + +| Variable | Default | Description | +|---------------------------|--------------|-----------------------------------| +| `ELASTICSEARCH_URI` | `elasticsearch:9200` | URI to connect to ES | +| `ELASTICSEARCH_TIMEOUT` | `60` | How long to wait for ElasticSearch connection | + +[1]: https://docs.docker.com/engine/tutorials/dockervolumes/ +[2]: https://hub.docker.com/_/elasticsearch/ +[3]: http://semver.org/ diff --git a/elasticsearch-init/build.yml b/elasticsearch-init/build.yml new file mode 100644 index 000000000..a5fdc6353 --- /dev/null +++ b/elasticsearch-init/build.yml @@ -0,0 +1,5 @@ +repository: monasca/elasticsearch-init +variants: + - tag: 0.0.1 + aliases: + - :latest diff --git a/elasticsearch-init/upload.sh b/elasticsearch-init/upload.sh new file mode 100644 index 000000000..2114f2dd5 --- /dev/null +++ b/elasticsearch-init/upload.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +TPL_DIR=/templates + +_get_tpl_name_from_file() { + local tpl=$1 + local tpl_name=$(basename $tpl) + echo $tpl_name +} + +if [ ! -d $TPL_DIR ]; then + echo "No templates mounted" + exit 0 +else + TPLS=$(ls $TPL_DIR) +fi + +for template in $TPLS; do + + echo "Handling template file $template" + tpl_name=`_get_tpl_name_from_file $template` + + curl -XPUT --retry 2 --retry-delay 2 $ELASTICSEARCH_URI/_template/${tpl_name} -d @$TPL_DIR/$template + +done + + diff --git a/elasticsearch-init/wait-for.sh b/elasticsearch-init/wait-for.sh new file mode 100644 index 000000000..b2915da26 --- /dev/null +++ b/elasticsearch-init/wait-for.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +: ${SLEEP_LENGTH:=2} + +wait_for() { + echo Waiting for $1 to listen on $2... + while ! nc -z $1 $2; do echo sleeping; sleep $SLEEP_LENGTH; done +} + +for var in "$@" +do + host=${var%:*} + port=${var#*:} + wait_for $host $port +done diff --git a/elasticsearch-templates/logs.template b/elasticsearch-templates/logs.template new file mode 100644 index 000000000..833ccd725 --- /dev/null +++ b/elasticsearch-templates/logs.template @@ -0,0 +1,63 @@ + { + "order" : 0, + "template" : "*", + "settings" : { + "index.refresh_interval" : "5s", + "index" : { + "analysis": { + "analyzer" : { + "no_token" : { + "type" : "custom", + "tokenizer" : "keyword", + "filter" : "lowercase" + } + } + } + } + }, + "mappings" : { + "_default_" : { + "dynamic_templates" : [ { + "message_field" : { + "mapping" : { + "index" : "analyzed", + "omit_norms" : true, + "type" : "string" + }, + "match_mapping_type" : "string", + "match" : "message" + } + }, { + "string_fields" : { + "mapping" : { + "analyzer" : "no_token", + "omit_norms" : true, + "type" : "string" + }, + "match_mapping_type" : "string", + "match" : "*" + } + } ], + "properties" : { + "geoip" : { + "dynamic" : true, + "properties" : { + "location" : { + "type" : "geo_point" + } + }, + "type" : "object" + }, + "@version" : { + "index" : "not_analyzed", + "type" : "string" + } + }, + "_all" : { + "enabled" : true, + "omit_norms" : true + } + } + }, + "aliases" : { } + } diff --git a/kibana/Dockerfile b/kibana/Dockerfile new file mode 100644 index 000000000..a3e6b74a2 --- /dev/null +++ b/kibana/Dockerfile @@ -0,0 +1,43 @@ +ARG KIBANA_VERSION="4.6.3" + +FROM node:4-alpine + +ARG KIBANA_PLUGIN_VER="" +ARG KIBANA_PLUGIN_REPO=https://github.com/openstack/monasca-kibana-plugin.git +ARG KIBANA_PLUGIN_BRANCH=master + +WORKDIR /mkp/ + +ARG REBUILD_PLUGIN_DEPS=1 +RUN apk add --no-cache --virtual build-dep git rsync + +ARG REBUILD_PLUGIN_PLUGIN=1 +RUN git clone $KIBANA_PLUGIN_REPO --depth 1 --branch $KIBANA_PLUGIN_BRANCH monasca-kibana-plugin && \ + cd monasca-kibana-plugin && \ + npm install --quiet && \ + npm run package --quiet && \ + KIBANA_PLUGIN_VER=$(node -e "console.log(require('./package.json').version)") && \ + mv target/monasca-kibana-plugin-${KIBANA_PLUGIN_VER}.tar.gz /monasca-kibana-plugin.tar.gz && \ + cd / && \ + rm -rf /mpk/monasca-kibana-plugin && \ + apk del build-dep + +FROM kibana:${KIBANA_VERSION} + +ENV KEYSTONE_URI=keystone:5000 \ + MONASCA_PLUGIN_ENABLED=False + +WORKDIR / + +ARG REBUILD_FILES=1 +COPY --from=0 /monasca-kibana-plugin.tar.gz . +COPY wait-for plugin_config start / +RUN chmod +x /wait-for /plugin_config /start + +ARG REBUILD_INSTALL_PLUGIN=1 +RUN /plugin_config && \ + kibana plugin -r monasca-kibana-plugin && \ + kibana plugin -i monasca-kibana-plugin -u file:///monasca-kibana-plugin.tar.gz && \ + rm -rf /monasca-kibana-plugin.tar.gz + +CMD /start diff --git a/kibana/README.md b/kibana/README.md new file mode 100644 index 000000000..7b9ca93d3 --- /dev/null +++ b/kibana/README.md @@ -0,0 +1,42 @@ +Kibana 4 with monasca-kibana-plugin +=================================== + +Image extends official [Kibana image][1] with [monasca-kibana-plugin][2]. +For more details about the aformentioned plugin, please visit plugin's +[repository][2]. + +Tags +---- + +Tags in this image match tags in the official [Kibana image][1]. +This is dictated by the fact that plugin should be build for specific +Kibana version. Each tag includes also ``KIBANA_PLUGIN_BRANCH`` information +detail. Final tag is represented as ``{KIBANA_VERSION}-{KIBANA_PLUGIN_BRANCH}``. +That said, tagging presents as follows: + +* `4.6.3-master` [`4.6-master`, `4-master`] - corresponds to **Kibana 4.6.3** + [monasca-kibana-plugin][2] build from **master** branch. + +Configuration +------------- + +| Variable | Default | Description | +|--------------------------------|-----------------|-----------------------------------| +| `KEYSTONE_URI` | `keystone:5000` | An URI to Keystone Admin Endpoint | +| `MONASCA_PLUGIN_ENABLED` | `False` | Should the plugin be enabled or disabled | + +Usage +----- + +To use this image, first you need to deploy [ElasticSearch][3] and [Keystone][4]. + + Keystone is required only if ``MONASCA_PLUGIN_ENABLED`` + is set to ``True`` + +Once prerequisites are met, image can ba launched, for example, with: +```docker run -it --rm -l elasticsearch monasca/kibana:4.6.3-master``` + +[1]: https://hub.docker.com/r/library/kibana +[2]: https://github.com/openstack/monasca-kibana-plugin +[3]: https://hub.docker.com/r/library/elasticsearch +[4]: https://github.com/monasca/monasca-docker/keystone diff --git a/kibana/build.yml b/kibana/build.yml new file mode 100644 index 000000000..fe8154eef --- /dev/null +++ b/kibana/build.yml @@ -0,0 +1,10 @@ +repository: monasca/kibana +variants: + - tag: latest + aliases: + - :4.6.3-master + - :4.6-master + - :4-master + args: + KIBANA_VERSION: 4.6.3 + KIBANA_PLUGIN_BRANCH: master diff --git a/kibana/plugin_config b/kibana/plugin_config new file mode 100644 index 000000000..ada07da38 --- /dev/null +++ b/kibana/plugin_config @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +echo "#monasca-kibana-plugin +monasca-kibana-plugin.auth_uri: http://${KEYSTONE_URI} +monasca-kibana-plugin.enabled: False +monasca-kibana-plugin.cookie.isSecure: False +" >> /opt/kibana/config/kibana.yml diff --git a/kibana/start b/kibana/start new file mode 100644 index 000000000..16a1080f8 --- /dev/null +++ b/kibana/start @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +sed -e " + s|monasca-kibana-plugin.enabled:\sFalse|monasca-kibana-plugin.enabled: ${MONASCA_PLUGIN_ENABLED}|g; +" -i /opt/kibana/config/kibana.yml + +if [ $MONASCA_PLUGIN_ENABLED == True ]; then + /wait-for $KEYSTONE_URI -- kibana +else + kibana +fi diff --git a/kibana/wait-for b/kibana/wait-for new file mode 100644 index 000000000..8abfe51a2 --- /dev/null +++ b/kibana/wait-for @@ -0,0 +1,176 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +cmdname=$(basename $0) + +echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $TIMEOUT -gt 0 ]]; then + echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT" + else + echoerr "$cmdname: waiting for $HOST:$PORT without a timeout" + fi + start_ts=$(date +%s) + while : + do + if [[ $ISBUSY -eq 1 ]]; then + nc -z $HOST $PORT + result=$? + else + (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1 + result=$? + fi + if [[ $result -eq 0 ]]; then + end_ts=$(date +%s) + echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds" + break + fi + sleep 1 + done + return $result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $QUIET -eq 1 ]]; then + timeout $BUSYTIMEFLAG $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & + else + timeout $BUSYTIMEFLAG $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & + fi + PID=$! + trap "kill -INT -$PID" INT + wait $PID + RESULT=$? + if [[ $RESULT -ne 0 ]]; then + echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT" + fi + return $RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + hostport=(${1//:/ }) + HOST=${hostport[0]} + PORT=${hostport[1]} + shift 1 + ;; + --child) + CHILD=1 + shift 1 + ;; + -q | --quiet) + QUIET=1 + shift 1 + ;; + -s | --strict) + STRICT=1 + shift 1 + ;; + -h) + HOST="$2" + if [[ $HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + HOST="${1#*=}" + shift 1 + ;; + -p) + PORT="$2" + if [[ $PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + PORT="${1#*=}" + shift 1 + ;; + -t) + TIMEOUT="$2" + if [[ $TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + CLI="$@" + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$HOST" == "" || "$PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +TIMEOUT=${TIMEOUT:-15} +STRICT=${STRICT:-0} +CHILD=${CHILD:-0} +QUIET=${QUIET:-0} + +# check to see if timeout is from busybox? +TIMEOUT_PATH=$(realpath $(which timeout)) +if [[ $TIMEOUT_PATH =~ "busybox" ]]; then + ISBUSY=1 + BUSYTIMEFLAG="-t" +else + ISBUSY=0 + BUSYTIMEFLAG="" +fi + +if [[ $CHILD -gt 0 ]]; then + wait_for + RESULT=$? + exit $RESULT +else + if [[ $TIMEOUT -gt 0 ]]; then + wait_for_wrapper + RESULT=$? + else + wait_for + RESULT=$? + fi +fi + +if [[ $CLI != "" ]]; then + if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then + echoerr "$cmdname: strict mode, refusing to execute subprocess" + exit $RESULT + fi + exec $CLI +else + exit $RESULT +fi diff --git a/log-pipeline.yml b/log-pipeline.yml new file mode 100644 index 000000000..18fa3c5f7 --- /dev/null +++ b/log-pipeline.yml @@ -0,0 +1,74 @@ +version: '3' +services: + + log-metrics: + image: monasca/log-metrics:latest + depends_on: + - kafka + - zookeeper + - log-transformer + + log-persister: + image: monasca/log-persister:latest + depends_on: + - kafka + - zookeeper + - elasticsearch + - log-transformer + + log-transformer: + image: monasca/log-transformer:latest + depends_on: + - kafka + - zookeeper + - log-api + + elasticsearch: + image: elasticsearch:2-alpine + environment: + - cluster.name=monasca + - http.host=0.0.0.0 + - transport.host=127.0.0.1 + - bootstrap.memory_lock=true + - ES_JAVA_OPTS=-Xms1g -Xmx1g + ports: + - 9200:9200 + - 9300:9300 + + elasticsearch-init: + image: monasca/elasticsearch-init + volumes: + - ./elasticsearch-templates/logs.template:/templates/logs.json + depends_on: + - elasticsearch + + kafka-log-init: + image: monasca/kafka-init:0.0.1 + environment: + KAFKA_TOPIC_CONFIG: segment.ms=900000 # 15m + KAFKA_CREATE_TOPICS: "\ + log:4:1,\ + log-transformed:4:1" + depends_on: + - kafka + + kibana: + image: monasca/kibana:4 + environment: + SERVER_NAME: kibana + ELASTICSEARCH_PINGTIMEOUT: 1000 + depends_on: + - elasticsearch + - keystone + ports: + - 5601:5601 + + log-api: + image: monasca/log-api:master + depends_on: + - keystone + - zookeeper + - kafka + - memcached + ports: + - "5607:5607" diff --git a/monasca-log-api/Dockerfile b/monasca-log-api/Dockerfile new file mode 100644 index 000000000..efd03dccc --- /dev/null +++ b/monasca-log-api/Dockerfile @@ -0,0 +1,55 @@ +ARG PYTHON_VERSION +ARG TIMESTAMP_SLUG +FROM monasca/python:${PYTHON_VERSION}-${TIMESTAMP_SLUG} + +ARG LOG_API_REPO=https://github.com/openstack/monasca-log-api.git +ARG LOG_API_BRANCH=master +ARG CONSTRAINTS_BRANCH=master +ARG EXTRA_DEPENDENCIES="gunicorn python-memcached" + +# To force a rebuild, pass --build-arg REBUILD="$(DATE)" when running +# `docker build` +ARG REBUILD=1 + +ENV CONFIG_TEMPLATE=true \ + LOG_LEVEL_ROOT=WARN \ + LOG_LEVEL_CONSOLE=INFO \ + LOG_LEVEL_ACCESS=INFO \ + MONASCA_CONTAINER_LOG_API_PORT=5607 \ + KAFKA_URI=kafka:9092 \ + KAFKA_WAIT_FOR_TOPICS=log \ + MEMCACHED_URI=memcached:11211 \ + KEYSTONE_IDENTITY_URI=http://keystone:35357 \ + KEYSTONE_AUTH_URI=http://keystone:5000 \ + KEYSTONE_ADMIN_USER=admin \ + KEYSTONE_ADMIN_PASSWORD=secretadmin \ + KEYSTONE_ADMIN_TENANT=admin \ + KEYSTONE_ADMIN_DOMAIN=default \ + ADD_ACCESS_LOG=true \ + ACCESS_LOG_FORMAT="%(asctime)s [%(process)d] gunicorn.access [%(levelname)s] %(message)s" \ + ACCESS_LOG_FIELDS='%(h)s %(l)s %(u)s %(t)s %(r)s %(s)s %(b)s "%(f)s" "%(a)s" %(L)s' + +ARG REBUILD_PROJECT=1 +RUN /build.sh \ + -r "${LOG_API_REPO}" \ + -b "${LOG_API_BRANCH}" \ + -q "${CONSTRAINTS_BRANCH}" \ + -d "${EXTRA_DEPENDENCIES}" && \ + rm -rf /usr/local/etc/monasca + +ARG REBUILD_CONFIG=1 +COPY log-api* /etc/monasca/ +COPY template.py start.sh health-check.sh kafka_wait_for_topics.py / + +EXPOSE ${MONASCA_CONTAINER_LOG_API_PORT} + +HEALTHCHECK --interval=10s --timeout=5s \ + CMD /health-check.sh \ + $KEYSTONE_AUTH_URI \ + $KEYSTONE_ADMIN_USER \ + $KEYSTONE_ADMIN_PASSWORD \ + $KEYSTONE_ADMIN_TENANT \ + $KEYSTONE_ADMIN_DOMAIN \ + $MONASCA_CONTAINER_LOG_API_PORT + +CMD ["/start.sh"] diff --git a/monasca-log-api/README.md b/monasca-log-api/README.md new file mode 100644 index 000000000..41e77015e --- /dev/null +++ b/monasca-log-api/README.md @@ -0,0 +1,106 @@ +monasca-log-api +=============== + +This image contains containerized version of Monasca Log API. +For more details about monasca-log-api, please visit: + +* [monasca-log-api's documentation][1] +* [monasca-log-api's API reference][2] +* [monasca-log-api's repository][3] + +Tags +---- + +The images in this repository follow a few tagging conventions: + +* `latest`: refers to the latest stable Python point release, e.g. + `2.2.1` +* `2.2.1`, `2.2`, `2`: standard semver tags, based on git tags in the + [official repository][3] +* `ocata`: named versions following OpenStack release names + built from the tip of the `stable/RELEASENAME` branches in the repository +* `master`, `master-DATESTAMP`: unstable builds from the master branch, not +intended for general use + +Usage +----- + +In order to run monasca-log-api container, [Kafka][4] needs to be available. +Once it is, **monasca-log-api** can be launched using +```docker run -p 5607:5607 -l kafka monasca/log-api:latest``` . + +Configuration +------------- + +A number of environment variables can be passed to the container: + +| Variable | Default | Description | +|-------------------------------- |--------------|-----------------------------------| +| `LOG_LEVEL_ROOT` | `WARN` | The level of the root logger | +| `LOG_LEVEL_CONSOLE` | `INFO` | Minimum level for console output | +| `LOG_LEVEL_ACCESS` | `INFO` | Minimum level for access output | +| `MONASCA_CONTAINER_LOG_API_PORT` | `5607` | The Log API's HTTP port | +| `KAFKA_URI` | `kafka:9092` | The host and port for kafka | +| `KAFKA_WAIT_FOR_TOPICS` | `log` | Topics to wait on at startup | +| `KAFKA_WAIT_RETRIES` | `24` | # of kafka wait attempts | +| `KAFKA_WAIT_DELAY` | `5` | seconds to wait between attempts | +| `KEYSTONE_IDENTITY_URI` | `http://keystone:35357` | Keystone identity address | +| `KEYSTONE_AUTH_URI` | `http://keystone:5000` | Keystone auth address | +| `KEYSTONE_ADMIN_USER` | `admin` | Keystone admin account user | +| `KEYSTONE_ADMIN_PASSWORD` | `secretadmin` | Keystone admin account password | +| `KEYSTONE_ADMIN_TENANT` | `admin` | Keystone admin account tenant | +| `KEYSTONE_ADMIN_DOMAIN` | `default` | Keystone admin domain | +| `MEMCACHED_URI` | `memcached:11211` | A list of URIs pointing at memcached | +| `AUTHORIZED_ROLES` | `admin, domainuser, domainadmin, monasca-user` | Roles for admin Users | +| `AGENT_AUTHORIZED_ROLES` | `monasca-agent` | Roles for metric write only users | +| `GUNICORN_WORKERS` | `9` | number of API worker processes | +| `GUNICORN_WORKER_CLASS` | `gevent` | async worker class | +| `GUNICORN_WORKER_CONNECTIONS` | `2000` | no. connections for async worker | +| `GUNICORN_BACKLOG` | `1000` | gunicorn backlog size | +| `GUNICORN_TIMEOUT` | `1000` | gunicorn timeout | +| `ADD_ACCESS_LOG` | `true` | if true, produce an access log on stderr | +| `ACCESS_LOG_FORMAT` | `%(asctime)s [%(process)d] gunicorn.access [%(levelname)s] %(message)s` | Log format for access log | +| `ACCESS_LOG_FIELDS` | `%(h)s %(l)s %(u)s %(t)s %(r)s %(s)s %(b)s "%(f)s" "%(a)s" %(L)s` | Access log fields | + +If additional values need to be overridden, new config files or jinja2 templates +can be provided by mounting a replacement on top of the original template: + + * `/etc/monasca/log-api.conf.j2` + * `/etc/monasca/log-api-paste.ini.j2` + * `/etc/monasca/log-api-logging.conf.j2` + +If jinja2 formatting is not desired, the environment variable `CONFIG_TEMPLATE` +can be set to `false`. Note that the jinja2 template should still be overwritten +(rather than the target file without the `.j2` suffix) as it will be copied at +runtime. + +The config file sources are available [in the repository][5]. If necessary, the +generated config files can be viewed at runtime by running: + +```bash +docker exec -it some_container_id cat /etc/monasca/log-api.conf +docker exec -it some_container_id cat /etc/monasca/log-api-paste.ini +docker exec -it some_container_id cat /etc/monasca/log-api-logging.conf +``` + +Troubleshooting +------------- + +Container status can be checked by the following command (example): +```bash +docker ps --filter 'name=log-api' --format '{{.Names}}\t{{.Image}}\t{{.Status}}' +``` + +Result of health check can be get by the following command: +```bash +docker inspect --format '{{json .State.Health}}' log-api | python -m json.tool +``` + +Health check `ExitCode`s: + * 1: Monasca API error + +[1]: https://docs.openstack.org/monasca-log-api/latest/ +[2]: https://developer.openstack.org/api-ref/monitoring-log-api/ +[3]: https://githubm/openstack/monasca-log-api +[4]: https://github.com/monasca/monasca-docker/tree/master/kafka +[5]: https://github.com/monasca/monasca-docker/blob/master/monasca-log-api/ diff --git a/monasca-log-api/build.yml b/monasca-log-api/build.yml new file mode 100644 index 000000000..6541fa521 --- /dev/null +++ b/monasca-log-api/build.yml @@ -0,0 +1,46 @@ +repository: monasca/log-api +variants: + - tag: master + aliases: + - :master-{date}-{time} + args: + LOG_API_BRANCH: master + PYTHON_VERSION: '2' + TIMESTAMP_SLUG: 20170809-155207 + CONSTRAINTS_BRANCH: master + + - tag: 2.2.1 + aliases: + - :latest + - :2.2 + - :2 + args: + LOG_API_BRANCH: 2.2.1 + PYTHON_VERSION: '2' + TIMESTAMP_SLUG: 20170809-155207 + CONSTRAINTS_BRANCH: master + - tag: 2.1.0 + aliases: + - :2.1 + args: + LOG_API_BRANCH: 2.1.0 + CONSTRAINTS_BRANCH: master + PYTHON_VERSION: '2' + TIMESTAMP_SLUG: 20170809-155207 + - tag: 2.0.0 + aliases: + - :2.0 + args: + LOG_API_BRANCH: 2.0.0 + CONSTRAINTS_BRANCH: master + PYTHON_VERSION: '2' + TIMESTAMP_SLUG: 20170809-155207 + + - tag: ocata + aliases: + - :ocata-{date}-{time} + args: + LOG_API_BRANCH: stable/ocata + CONSTRAINTS_BRANCH: stable/ocata + PYTHON_VERSION: '2' + TIMESTAMP_SLUG: 20170809-155207 diff --git a/monasca-log-api/health-check.sh b/monasca-log-api/health-check.sh new file mode 100755 index 000000000..6f84d45c3 --- /dev/null +++ b/monasca-log-api/health-check.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +# Default values for executing it manually on the host +MONASCA_CONTAINER_LOG_API_PORT=${1:-5607} + +curl --include --silent --show-error --output - \ + http://localhost:${MONASCA_CONTAINER_LOG_API_PORT}/healthcheck 2>&1 | \ + awk ' + BEGIN {status_code="0"; body=""; output=""} + $1 ~ /^HTTP\// {status_line=$0; status_code=$2} + $1 ~ /^\{/ {body=$0} + {output=output $0 "\n"} + END { + if(status_code=="200") { + print status_line; + print body; + } else { + print output; + exit 2; + } + }' + +exit $? diff --git a/monasca-log-api/kafka_wait_for_topics.py b/monasca-log-api/kafka_wait_for_topics.py new file mode 100644 index 000000000..7d6a7942a --- /dev/null +++ b/monasca-log-api/kafka_wait_for_topics.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +# (C) Copyright 2017 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import print_function + +import os +import sys + +from pykafka import KafkaClient + +client = KafkaClient(hosts=os.environ.get('KAFKA_URI', 'kafka:9092')) + +required_topics = os.environ.get('KAFKA_WAIT_FOR_TOPICS', '').split(',') +print('Checking for available topics:', repr(required_topics)) +for req_topic in required_topics: + if req_topic in client.topics: + topic = client.topics[req_topic] + if len(topic.partitions) > 0: + print('Topic is ready:', req_topic) + else: + print('Topic has no partitions:', req_topic) + sys.exit(1) + else: + print('Topic not found:', req_topic) + sys.exit(1) diff --git a/monasca-log-api/log-api-gunicorn.conf.j2 b/monasca-log-api/log-api-gunicorn.conf.j2 new file mode 100644 index 000000000..58d9eeecc --- /dev/null +++ b/monasca-log-api/log-api-gunicorn.conf.j2 @@ -0,0 +1,13 @@ +bind = '0.0.0.0:{{ MONASCA_CONTAINER_LOG_API_PORT }}' +proc_name = 'monasca-log-api' + +backlog = {{ GUNICORN_BACKLOG|int }} +workers = {{ GUNICORN_WORKERS|int }} +worker_class = '{{ GUNICORN_WORKER_CLASS }}' +timeout = {{ GUNICORN_TIMEOUT|int }} + +{% if ADD_ACCESS_LOG %} +accesslog = '-' +access_log_format = '{{ ACCESS_LOG_FIELDS }}' +{% endif %} +capture_output = True diff --git a/monasca-log-api/log-api-logging.conf.j2 b/monasca-log-api/log-api-logging.conf.j2 new file mode 100644 index 000000000..76d60f160 --- /dev/null +++ b/monasca-log-api/log-api-logging.conf.j2 @@ -0,0 +1,47 @@ +[default] +disable_existing_loggers = 0 + +[loggers] +keys = root, gunicorn_access, kafka + +[handlers] +keys = console, gunicorn_access + +[formatters] +keys = context, gunicorn_access + +[logger_root] +level = {{ LOG_LEVEL_ROOT }} +handlers = console + +[logger_gunicorn_access] +level = {{ LOG_LEVEL_ACCESS }} +handlers = console +propagate = 0 +qualname = gunicorn.access + +[logger_kafka] +qualname = kafka +level = DEBUG +handlers = console +propagate = 0 + +[handler_console] +class = logging.StreamHandler +args = (sys.stdout,) +level = {{ LOG_LEVEL_CONSOLE }} +formatter = context + +[handler_gunicorn_access] +class = logging.StreamHandler +args = (sys.stdout,) +level = {{ LOG_LEVEL_ACCESS }} +formatter = gunicorn_access + +[formatter_context] +class = oslo_log.formatters.ContextFormatter + +[formatter_gunicorn_access] +class = logging.Formatter +format = {{ ACCESS_LOG_FORMAT }} +datefmt = %Y-%m-%d %H:%M:%S diff --git a/monasca-log-api/log-api-paste.ini.j2 b/monasca-log-api/log-api-paste.ini.j2 new file mode 100644 index 000000000..e3e69e096 --- /dev/null +++ b/monasca-log-api/log-api-paste.ini.j2 @@ -0,0 +1,53 @@ +[DEFAULT] +name = main + +[composite:main] +use = egg:Paste#urlmap +/: la_version +/healthcheck: la_healthcheck +/v2.0: la_api_v2 +/v3.0: la_api_v3 + +[pipeline:la_version] +pipeline = error_trap versionapp + +[pipeline:la_healthcheck] +pipeline = error_trap healthcheckapp + +[pipeline:la_api_v2] +pipeline = error_trap request_id auth roles api_v2_app + +[pipeline:la_api_v3] +pipeline = error_trap request_id auth roles api_v3_app + +[app:versionapp] +paste.app_factory = monasca_log_api.app.api:create_version_app + +[app:healthcheckapp] +paste.app_factory = monasca_log_api.app.api:create_healthcheck_app + +[app:api_v2_app] +paste.app_factory = monasca_log_api.app.api:create_api_app +set api_version=v2.0 + +[app:api_v3_app] +paste.app_factory = monasca_log_api.app.api:create_api_app +set api_version=v3.0 + +[filter:auth] +paste.filter_factory = keystonemiddleware.auth_token:filter_factory + +[filter:roles] +paste.filter_factory = monasca_log_api.middleware.role_middleware:RoleMiddleware.factory + +[filter:request_id] +paste.filter_factory = oslo_middleware.request_id:RequestId.factory + +[filter:debug] +paste.filter_factory = oslo_middleware.debug:Debug.factory + +[filter:error_trap] +paste.filter_factory = oslo_middleware.catch_errors:CatchErrors.factory + +[server:main] +use = egg:gunicorn#main diff --git a/monasca-log-api/log-api.conf.j2 b/monasca-log-api/log-api.conf.j2 new file mode 100644 index 000000000..4622e5687 --- /dev/null +++ b/monasca-log-api/log-api.conf.j2 @@ -0,0 +1,41 @@ +[DEFAULT] +log_config_append=/etc/monasca/log-api-logging.conf + +[monitoring] +statsd_host = {{ STATSD_HOST | default('127.0.0.1') }} +statsd_port = {{ STATSD_PORT | default(8125) }} +statsd_buffer = {{ STATSD_BUFFER | default(50) }} + +[service] +region = useast +max_log_size = 1048576 + +[roles_middleware] +path = /v2.0/log,/v3.0/logs +default_roles = {{ AUTHORIZED_ROLES | default('admin, domainuser, domainadmin, monasca-user') }} +agent_roles = {{ AGENT_AUTHORIZED_ROLES | default('monasca-agent') }} + +[log_publisher] +topics = log +kafka_url = {{ KAFKA_URI | default('kafka:9092') }} +max_message_size = 1048576 + +[kafka_healthcheck] +kafka_url = {{ KAFKA_URI | default('kafka:9092') }} +kafka_topics = log + +[keystone_authtoken] +auth_type = password +auth_url = {{ KEYSTONE_IDENTITY_URI }} +auth_uri = {{ KEYSTONE_AUTH_URI }} +username = {{ KEYSTONE_ADMIN_USER }} +password = {{ KEYSTONE_ADMIN_PASSWORD }} +user_domain_name = Default +project_name = {{ KEYSTONE_ADMIN_TENANT }} +project_domain_name = Default +service_token_roles_required = true +memcached_servers = {{ MEMCACHED_URI }} +insecure = false +cafile = +certfile = +keyfile = diff --git a/monasca-log-api/start.sh b/monasca-log-api/start.sh new file mode 100755 index 000000000..bbd28e170 --- /dev/null +++ b/monasca-log-api/start.sh @@ -0,0 +1,60 @@ +#!/bin/sh + +export GUNICORN_WORKERS=${GUNICORN_WORKERS:-1} +export GUNICORN_WORKER_CLASS=${GUNICORN_WORKER_CLASS:-"gevent"} +export GUNICORN_WORKER_CONNECTIONS=${GUNICORN_WORKER_CONNECTIONS:-2000} +export GUNICORN_TIMEOUT=${GUNICORN_TIMEOUT:-10} +export GUNICORN_BACKLOG=${GUNICORN_BACKLOG:-1000} + +KAFKA_WAIT_RETRIES=${KAFKA_WAIT_RETRIES:-"24"} +KAFKA_WAIT_DELAY=${KAFKA_WAIT_DELAY:-"5"} + +if [ -n "$KAFKA_WAIT_FOR_TOPICS" ]; then + echo "Waiting for Kafka topics to become available..." + success="false" + + for i in $(seq $KAFKA_WAIT_RETRIES); do + python /kafka_wait_for_topics.py + if [ $? -eq 0 ]; then + success="true" + break + else + echo "Kafka not yet ready (attempt $i of $KAFKA_WAIT_RETRIES)" + sleep "$KAFKA_WAIT_DELAY" + fi + done + + if [ "$success" != "true" ]; then + echo "Kafka failed to become ready, exiting..." + sleep 1 + exit 1 + fi +fi + +if [ "$CONFIG_TEMPLATE" = "true" ]; then + python template.py \ + /etc/monasca/log-api.conf.j2 \ + /etc/monasca/log-api.conf + + python template.py \ + /etc/monasca/log-api-paste.ini.j2 \ + /etc/monasca/log-api-paste.ini + + python template.py \ + /etc/monasca/log-api-logging.conf.j2 \ + /etc/monasca/log-api-logging.conf + + python template.py \ + /etc/monasca/log-api-gunicorn.conf.j2 \ + /etc/monasca/log-api-gunicorn.conf +else + cp /etc/monasca/log-api.conf.j2 /etc/monasca/log-api.conf + cp /etc/monasca/log-api-paste.ini.j2 /etc/monasca/log-api-paste.ini + cp /etc/monasca/log-api-logging.conf.j2 /etc/monasca/log-api-logging.conf + cp /etc/monasca/log-api-gunicorn.conf.j2 /etc/monasca/log-api-gunicorn.conf +fi + +export PYTHONIOENCODING=utf-8 +gunicorn \ + --config /etc/monasca/log-api-gunicorn.conf \ + --paste /etc/monasca/log-api-paste.ini diff --git a/monasca-log-api/template.py b/monasca-log-api/template.py new file mode 100755 index 000000000..e83ec2fcf --- /dev/null +++ b/monasca-log-api/template.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +# (C) Copyright 2017 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import print_function + +import os +import sys + +from jinja2 import Template + + +def main(): + if len(sys.argv) != 3: + print('Usage: {} [input] [output]'.format(sys.argv[0])) + sys.exit(1) + + in_path = sys.argv[1] + out_path = sys.argv[2] + + with open(in_path, 'r') as in_file, open(out_path, 'w') as out_file: + t = Template(in_file.read()) + out_file.write(t.render(os.environ)) + +if __name__ == '__main__': + main() diff --git a/monasca-log-metrics/Dockerfile b/monasca-log-metrics/Dockerfile new file mode 100644 index 000000000..bd84d9e74 --- /dev/null +++ b/monasca-log-metrics/Dockerfile @@ -0,0 +1,23 @@ +FROM logstash:2-alpine + +# To force a rebuild, pass --build-arg REBUILD="$(DATE)" when running +# `docker build` +ARG REBUILD=1 + +ENV CONFIG_TEMPLATE=true \ + KAFKA_URI=kafka:9092 \ + ZOOKEEPER_URI=zookeeper:2181 \ + KAFKA_WAIT_FOR_TOPICS=log-transformed,metrics + +ARG REBUILD_DEPENDENCIES=1 +RUN apk add --no-cache python py2-pip py2-jinja2 && \ + apk add --no-cache --virtual build-dep \ + python-dev make g++ linux-headers && \ + pip install pykafka && \ + apk del build-dep + +ARG REBUILD_CONFIG=1 +COPY log-metrics* /etc/monasca/ +COPY template.py start.sh kafka_wait_for_topics.py / + +CMD ["/start.sh"] diff --git a/monasca-log-metrics/README.md b/monasca-log-metrics/README.md new file mode 100644 index 000000000..181c72b7e --- /dev/null +++ b/monasca-log-metrics/README.md @@ -0,0 +1,40 @@ +monasca-log-metrics +------------------- + +**monasca-log-metrics** image contains [Logstash][1] configuration +to transform logs into metrics based on log's severity. + +Tags +---- + +**monasca-log-metrics** uses simple [SemVer][2] tags as follows: + +* `0.0.1` - `latest` + +Configuration +------------- + +| Variable | Default | Description | +|---------------------------|------------------|------------------------------------| +| `ZOOKEEPER_URI` | `zookeeper:2181` | An URI to Zookeeper server | +| `KAFKA_URI` | `kafka:9092` | The host and port for kafka | +| `KAFKA_WAIT_FOR_TOPICS` | `log-transformed,metrics` | Topics to wait on at startup | +| `KAFKA_WAIT_RETRIES` | `24` | # of kafka wait attempts | +| `KAFKA_WAIT_DELAY` | `5` | # seconds to wait between attempts | + +Usage +----- + +In order to run **monasca-log-metrics**: + +* [kafka][3] needs to be available +* [zookeeper][4] needs to be available +* `log-transformed` and `metrics` topics needs to be created + +After that, **monasca-log-metrics** can be run with: +```docker run -l zookeeper -l kafka monasca/log-metrics``` + +[1]: https://hub.docker.com/_/logstash/ +[2]: http://semver.org/ +[3]: https://github.com/monasca/monasca-docker/kafka +[4]: https://hub.docker.com/_/zookeeper diff --git a/monasca-log-metrics/build.yml b/monasca-log-metrics/build.yml new file mode 100644 index 000000000..a2bf9c183 --- /dev/null +++ b/monasca-log-metrics/build.yml @@ -0,0 +1,5 @@ +repository: monasca/log-metrics +variants: + - tag: 0.0.1 + aliases: + - :latest diff --git a/monasca-log-metrics/kafka_wait_for_topics.py b/monasca-log-metrics/kafka_wait_for_topics.py new file mode 100644 index 000000000..7d6a7942a --- /dev/null +++ b/monasca-log-metrics/kafka_wait_for_topics.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +# (C) Copyright 2017 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import print_function + +import os +import sys + +from pykafka import KafkaClient + +client = KafkaClient(hosts=os.environ.get('KAFKA_URI', 'kafka:9092')) + +required_topics = os.environ.get('KAFKA_WAIT_FOR_TOPICS', '').split(',') +print('Checking for available topics:', repr(required_topics)) +for req_topic in required_topics: + if req_topic in client.topics: + topic = client.topics[req_topic] + if len(topic.partitions) > 0: + print('Topic is ready:', req_topic) + else: + print('Topic has no partitions:', req_topic) + sys.exit(1) + else: + print('Topic not found:', req_topic) + sys.exit(1) diff --git a/monasca-log-metrics/log-metrics.conf.j2 b/monasca-log-metrics/log-metrics.conf.j2 new file mode 100644 index 000000000..f6a76a79c --- /dev/null +++ b/monasca-log-metrics/log-metrics.conf.j2 @@ -0,0 +1,65 @@ +input { + kafka { + zk_connect => "{{ ZOOKEEPER_URI }}" + topic_id => "log-transformed" + group_id => "log-metric" + consumer_id => "monasca_log_metrics" + consumer_threads => "1" + } +} + + +filter { + + # drop logs that have not set log level + if "level" not in [log] { + drop { periodic_flush => true } + } else { + ruby { + code => " + log_level = event['log']['level'].downcase + event['log']['level'] = log_level + " + } + } + + # drop logs with log level not in warning,error + if [log][level] not in [warning,error] { + drop { periodic_flush => true } + } + + ruby { + code => " + log_level = event['log']['level'].downcase + log_ts = Time.now.to_f * 1000.0 + + # metric name + metric_name = 'log.%s' % log_level + + # build metric + metric = {} + metric['name'] = metric_name + metric['timestamp'] = log_ts + metric['value'] = 1 + metric['dimensions'] = event['log']['dimensions'] + metric['value_meta'] = {} + + event['metric'] = metric.to_hash + " + } + + mutate { + remove_field => ["log", "@version", "@timestamp", "log_level_original", "tags"] + } + +} + + +output { + kafka { + bootstrap_servers => "{{ KAFKA_URI }}" + topic_id => "metrics" + client_id => "monasca_log_metrics" + compression_type => "none" + } +} diff --git a/monasca-log-metrics/start.sh b/monasca-log-metrics/start.sh new file mode 100755 index 000000000..cc157528c --- /dev/null +++ b/monasca-log-metrics/start.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +KAFKA_WAIT_RETRIES=${KAFKA_WAIT_RETRIES:-"24"} +KAFKA_WAIT_DELAY=${KAFKA_WAIT_DELAY:-"5"} + +if [ -n "$KAFKA_WAIT_FOR_TOPICS" ]; then + echo "Waiting for Kafka topics to become available..." + success="false" + + for i in $(seq $KAFKA_WAIT_RETRIES); do + python /kafka_wait_for_topics.py + if [ $? -eq 0 ]; then + success="true" + break + else + echo "Kafka not yet ready (attempt $i of $KAFKA_WAIT_RETRIES)" + sleep "$KAFKA_WAIT_DELAY" + fi + done + + if [ "$success" != "true" ]; then + echo "Kafka failed to become ready, exiting..." + sleep 1 + exit 1 + fi +fi + +if [ "$CONFIG_TEMPLATE" = "true" ]; then + python template.py \ + /etc/monasca/log-metrics.conf.j2 \ + /etc/monasca/log-metrics.conf +else + cp /etc/monasca/log-metrics.conf.j2 /etc/monasca/log-metrics.conf +fi + +logstash -f /etc/monasca/log-metrics.conf diff --git a/monasca-log-metrics/template.py b/monasca-log-metrics/template.py new file mode 100755 index 000000000..e83ec2fcf --- /dev/null +++ b/monasca-log-metrics/template.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +# (C) Copyright 2017 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import print_function + +import os +import sys + +from jinja2 import Template + + +def main(): + if len(sys.argv) != 3: + print('Usage: {} [input] [output]'.format(sys.argv[0])) + sys.exit(1) + + in_path = sys.argv[1] + out_path = sys.argv[2] + + with open(in_path, 'r') as in_file, open(out_path, 'w') as out_file: + t = Template(in_file.read()) + out_file.write(t.render(os.environ)) + +if __name__ == '__main__': + main() diff --git a/monasca-log-persister/Dockerfile b/monasca-log-persister/Dockerfile new file mode 100644 index 000000000..78e3522ca --- /dev/null +++ b/monasca-log-persister/Dockerfile @@ -0,0 +1,33 @@ +FROM logstash:2-alpine + +# To force a rebuild, pass --build-arg REBUILD="$(DATE)" when running +# `docker build` +ARG REBUILD=1 + +ENV CONFIG_TEMPLATE=true \ + ELASTICSEARCH_HOST=elasticsearch \ + ELASTICSEARCH_PORT=9200 \ + ELASTICSEARCH_TIMEOUT=60 \ + ELASTICSEARCH_FLUSH_SIZE=600 \ + ELASTICSEARCH_IDLE_FLUSH_TIME=60 \ + ELASTICSEARCH_INDEX="%{tenant}-%{index_date}" \ + ELASTICSEARCH_DOC_TYPE="logs" \ + ELASTICSEARCH_SNIFFING=true \ + ELASTICSEARCH_SNIFFING_DELAY=5 \ + ZOOKEEPER_URI=zookeeper:2181 \ + KAFKA_WAIT_FOR_TOPICS=log-transformed + +ARG REBUILD_DEPENDENCIES=1 +RUN apk add --no-cache python py2-pip py2-jinja2 && \ + apk add --no-cache --virtual build-dep \ + python-dev make g++ linux-headers && \ + pip install pykafka && \ + apk del build-dep + +ARG REBUILD_CONFIG=1 +COPY log-persister* /etc/monasca/ +COPY template.py wait-for start.sh kafka_wait_for_topics.py / + +RUN chmod +x /wait-for + +ENTRYPOINT /wait-for ${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT} --timeout=${ELASTICSEARCH_TIMEOUT} -- /start.sh diff --git a/monasca-log-persister/README.md b/monasca-log-persister/README.md new file mode 100644 index 000000000..b1010e4fd --- /dev/null +++ b/monasca-log-persister/README.md @@ -0,0 +1,54 @@ +monasca-log-persister +--------------------- + +**monasca-log-persister** image contains [Logstash][1] configuration +to save logs inside **log-db** (i.e. [ElasticSearch][5]) + +Tags +---- + +**monasca-log-persister** uses simple [SemVer][2] tags as follows: + +* `0.0.1` - `latest` + +Configuration +------------- + +| Variable | Default | Description | +|----------------------------|-------------------|-------------------------------------| +| `ZOOKEEPER_URI` | `zookeeper:2181` | An URI to Zookeeper server | +| `KAFKA_WAIT_FOR_TOPICS` | `log-transformed` | Topics to wait on at startup | +| `ELASTICSEARCH_HOST` | `elasticsearch` | Comma delimited ElasticSearch hosts | +| `ELASTICSEARCH_PORT` | `9200` | Port of ElasticSearch | +| `ELASTICSEARCH_TIMEOUT` | `60` | How long to wait for ElasticSearch | +| `ELASTICSEARCH_FLUSH_SIZE` | `600` | See [flush_size][6] | +| `ELASTICSEARCH_IDLE_FLUSH_TIME` | `60` | See [idle_flush_time][7] | +| `ELASTICSEARCH_INDEX` | `%{tenant}-%{index_date}` | See [index][8] | +| `ELASTICSEARCH_DOC_TYPE` | `logs` | See [document_type][9] | +| `ELASTICSEARCH_SNIFFING` | `true` | See [sniffing][10] | +| `ELASTICSEARCH_SNIFFING_DELAY` | `5` | See [sniffing_delay][11] | + +Usage +----- + +In order to run **monasca-log-persister**: + +* [kafka][3] needs to be available +* [zookeeper][4] needs to be available +* [elasticsearch][5] needs to be available +* `log-transformed` topic needs to be created + +After that, **monasca-log-persister** can be run with: +```docker run -l zookeeper -l kafka -l elasticsearch monasca/log-persister``` + +[1]: https://hub.docker.com/_/logstash/ +[2]: http://semver.org/ +[3]: https://github.comonasca/monasca-docker/kafka +[4]: https://hub.docker.com/_/zookeeper +[5]: https://hub.docker.com/_/elasticsearch +[6]: https://www.elastic.co/guide/en/logstash/2.4/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-flush_size +[7]: https://www.elastic.co/guide/en/logstash/2.4/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-idle_flush_time +[8]: https://www.elastic.co/guide/en/logstash/2.4/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-index +[9]: https://www.elastic.co/guide/en/logstash/2.4/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-document_type +[10]: https://www.elastic.co/guide/en/logstash/2.4/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-sniffing +[11]: https://www.elastic.co/guide/en/logstash/2.4/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-sniffing_delay diff --git a/monasca-log-persister/build.yml b/monasca-log-persister/build.yml new file mode 100644 index 000000000..2a56d92ae --- /dev/null +++ b/monasca-log-persister/build.yml @@ -0,0 +1,5 @@ +repository: monasca/log-persister +variants: + - tag: 0.0.1 + aliases: + - :latest diff --git a/monasca-log-persister/kafka_wait_for_topics.py b/monasca-log-persister/kafka_wait_for_topics.py new file mode 100644 index 000000000..7d6a7942a --- /dev/null +++ b/monasca-log-persister/kafka_wait_for_topics.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +# (C) Copyright 2017 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import print_function + +import os +import sys + +from pykafka import KafkaClient + +client = KafkaClient(hosts=os.environ.get('KAFKA_URI', 'kafka:9092')) + +required_topics = os.environ.get('KAFKA_WAIT_FOR_TOPICS', '').split(',') +print('Checking for available topics:', repr(required_topics)) +for req_topic in required_topics: + if req_topic in client.topics: + topic = client.topics[req_topic] + if len(topic.partitions) > 0: + print('Topic is ready:', req_topic) + else: + print('Topic has no partitions:', req_topic) + sys.exit(1) + else: + print('Topic not found:', req_topic) + sys.exit(1) diff --git a/monasca-log-persister/log-persister.conf.j2 b/monasca-log-persister/log-persister.conf.j2 new file mode 100644 index 000000000..332c38343 --- /dev/null +++ b/monasca-log-persister/log-persister.conf.j2 @@ -0,0 +1,60 @@ +input { + kafka { + zk_connect => "{{ ZOOKEEPER_URI }}" + topic_id => "log-transformed" + group_id => "log-persister" + consumer_id => "monasca_log_persister" + consumer_threads => "1" + } +} + +filter { + date { + match => ["[log][timestamp]", "UNIX"] + target => "@timestamp" + } + + date { + match => ["creation_time", "UNIX"] + target => "creation_time" + } + + grok { + match => { + "[@timestamp]" => "^(?\d{4}-\d{2}-\d{2})" + } + } + + if "dimensions" in [log] { + ruby { + code => " + fieldHash = event['log']['dimensions'] + fieldHash.each do |key, value| + event[key] = value + end + " + } + } + + mutate { + add_field => { + message => "%{[log][message]}" + log_level => "%{[log][level]}" + tenant => "%{[meta][tenantId]}" + region => "%{[meta][region]}" + } + remove_field => ["@version", "host", "type", "tags" ,"_index_date", "meta", "log"] + } +} + +output { + elasticsearch { + index => "{{ ELASTICSEARCH_INDEX }}" + document_type => "{{ ELASTICSEARCH_DOC_TYPE }}" + hosts => ["{{ ELASTICSEARCH_HOST }}"] + flush_size => {{ ELASTICSEARCH_FLUSH_SIZE | int }} + idle_flush_time => {{ ELASTICSEARCH_IDLE_FLUSH_TIME | int }} + sniffing => {{ ELASTICSEARCH_SNIFFING }} + sniffing_delay => {{ ELASTICSEARCH_SNIFFING_DELAY | int }} + } +} diff --git a/monasca-log-persister/start.sh b/monasca-log-persister/start.sh new file mode 100755 index 000000000..790fecc4a --- /dev/null +++ b/monasca-log-persister/start.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +KAFKA_WAIT_RETRIES=${KAFKA_WAIT_RETRIES:-"24"} +KAFKA_WAIT_DELAY=${KAFKA_WAIT_DELAY:-"5"} + +if [ -n "$KAFKA_WAIT_FOR_TOPICS" ]; then + echo "Waiting for Kafka topics to become available..." + success="false" + + for i in $(seq $KAFKA_WAIT_RETRIES); do + python /kafka_wait_for_topics.py + if [ $? -eq 0 ]; then + success="true" + break + else + echo "Kafka not yet ready (attempt $i of $KAFKA_WAIT_RETRIES)" + sleep "$KAFKA_WAIT_DELAY" + fi + done + + if [ "$success" != "true" ]; then + echo "Kafka failed to become ready, exiting..." + sleep 1 + exit 1 + fi +fi + +if [ "$CONFIG_TEMPLATE" = "true" ]; then + python template.py \ + /etc/monasca/log-persister.conf.j2 \ + /etc/monasca/log-persister.conf +else + cp /etc/monasca/log-persister.conf.j2 /etc/monasca/log-persister.conf +fi + +logstash -f /etc/monasca/log-persister.conf diff --git a/monasca-log-persister/template.py b/monasca-log-persister/template.py new file mode 100755 index 000000000..e83ec2fcf --- /dev/null +++ b/monasca-log-persister/template.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +# (C) Copyright 2017 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import print_function + +import os +import sys + +from jinja2 import Template + + +def main(): + if len(sys.argv) != 3: + print('Usage: {} [input] [output]'.format(sys.argv[0])) + sys.exit(1) + + in_path = sys.argv[1] + out_path = sys.argv[2] + + with open(in_path, 'r') as in_file, open(out_path, 'w') as out_file: + t = Template(in_file.read()) + out_file.write(t.render(os.environ)) + +if __name__ == '__main__': + main() diff --git a/monasca-log-persister/wait-for b/monasca-log-persister/wait-for new file mode 100644 index 000000000..401a6f179 --- /dev/null +++ b/monasca-log-persister/wait-for @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +cmdname=$(basename $0) + +echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $TIMEOUT -gt 0 ]]; then + echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT" + else + echoerr "$cmdname: waiting for $HOST:$PORT without a timeout" + fi + start_ts=$(date +%s) + while : + do + if [[ $ISBUSY -eq 1 ]]; then + nc -z $HOST $PORT + result=$? + else + (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1 + result=$? + fi + if [[ $result -eq 0 ]]; then + end_ts=$(date +%s) + echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds" + break + fi + sleep 1 + done + return $result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $QUIET -eq 1 ]]; then + timeout $BUSYTIMEFLAG $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & + else + timeout $BUSYTIMEFLAG $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & + fi + PID=$! + trap "kill -INT -$PID" INT + wait $PID + RESULT=$? + if [[ $RESULT -ne 0 ]]; then + echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT" + fi + return $RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + hostport=(${1//:/ }) + HOST=${hostport[0]} + PORT=${hostport[1]} + shift 1 + ;; + --child) + CHILD=1 + shift 1 + ;; + -q | --quiet) + QUIET=1 + shift 1 + ;; + -s | --strict) + STRICT=1 + shift 1 + ;; + -h) + HOST="$2" + if [[ $HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + HOST="${1#*=}" + shift 1 + ;; + -p) + PORT="$2" + if [[ $PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + PORT="${1#*=}" + shift 1 + ;; + -t) + TIMEOUT="$2" + if [[ $TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + CLI="$@" + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$HOST" == "" || "$PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +TIMEOUT=${TIMEOUT:-15} +STRICT=${STRICT:-0} +CHILD=${CHILD:-0} +QUIET=${QUIET:-0} + +# check to see if timeout is from busybox? +# check to see if timeout is from busybox? +TIMEOUT_PATH=$(realpath $(which timeout)) +if [[ $TIMEOUT_PATH =~ "busybox" ]]; then + ISBUSY=1 + BUSYTIMEFLAG="-t" +else + ISBUSY=0 + BUSYTIMEFLAG="" +fi + +if [[ $CHILD -gt 0 ]]; then + wait_for + RESULT=$? + exit $RESULT +else + if [[ $TIMEOUT -gt 0 ]]; then + wait_for_wrapper + RESULT=$? + else + wait_for + RESULT=$? + fi +fi + +if [[ $CLI != "" ]]; then + if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then + echoerr "$cmdname: strict mode, refusing to execute subprocess" + exit $RESULT + fi + exec $CLI +else + exit $RESULT +fi diff --git a/monasca-log-transformer/Dockerfile b/monasca-log-transformer/Dockerfile new file mode 100644 index 000000000..d468ad66b --- /dev/null +++ b/monasca-log-transformer/Dockerfile @@ -0,0 +1,23 @@ +FROM logstash:2-alpine + +# To force a rebuild, pass --build-arg REBUILD="$(DATE)" when running +# `docker build` +ARG REBUILD=1 + +ENV CONFIG_TEMPLATE=true \ + KAFKA_URI=kafka:9092 \ + ZOOKEEPER_URI=zookeeper:2181 \ + KAFKA_WAIT_FOR_TOPICS=log-transformed,log + +ARG REBUILD_DEPENDENCIES=1 +RUN apk add --no-cache python py2-pip py2-jinja2 && \ + apk add --no-cache --virtual build-dep \ + python-dev make g++ linux-headers && \ + pip install pykafka && \ + apk del build-dep + +ARG REBUILD_CONFIG=1 +COPY log-transformer* /etc/monasca/ +COPY template.py start.sh kafka_wait_for_topics.py / + +CMD ["/start.sh"] diff --git a/monasca-log-transformer/README.md b/monasca-log-transformer/README.md new file mode 100644 index 000000000..f53151963 --- /dev/null +++ b/monasca-log-transformer/README.md @@ -0,0 +1,40 @@ +monasca-log-transformer +----------------------- + +**monasca-log-transformer** image contains [Logstash][1] configuration +to detect log's severity. + +Tags +---- + +**monasca-log-transformer** uses simple [SemVer][2] tags as follows: + +* `0.0.1` - `latest` + +Configuration +------------- + +| Variable | Default | Description | +|---------------------------|------------------|------------------------------------| +| `ZOOKEEPER_URI` | `zookeeper:2181` | An URI to Zookeeper server | +| `KAFKA_URI` | `kafka:9092` | The host and port for kafka | +| `KAFKA_WAIT_FOR_TOPICS` | `log-transformed,log` | Topics to wait on at startup | +| `KAFKA_WAIT_RETRIES` | `24` | # of kafka wait attempts | +| `KAFKA_WAIT_DELAY` | `5` | # seconds to wait between attempts | + +Usage +----- + +In order to run **monasca-log-transformer**: + +* [kafka][3] needs to be available +* [zookeeper][4] needs to be available +* `log-transformed` and `log` topics needs to be created + +After that, **monasca-log-transformer** can be run with: +```docker run -l zookeeper -l kafka monasca/log-transformer``` + +[1]: https://hub.docker.com/_/logstash/ +[2]: http://semver.org/ +[3]: https://github.comonasca/monasca-docker/kafka +[4]: https://hub.docker.com/_/zookeeper diff --git a/monasca-log-transformer/build.yml b/monasca-log-transformer/build.yml new file mode 100644 index 000000000..214f8b49d --- /dev/null +++ b/monasca-log-transformer/build.yml @@ -0,0 +1,5 @@ +repository: monasca/log-transformer +variants: + - tag: 0.0.1 + aliases: + - :latest diff --git a/monasca-log-transformer/kafka_wait_for_topics.py b/monasca-log-transformer/kafka_wait_for_topics.py new file mode 100644 index 000000000..7d6a7942a --- /dev/null +++ b/monasca-log-transformer/kafka_wait_for_topics.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +# (C) Copyright 2017 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import print_function + +import os +import sys + +from pykafka import KafkaClient + +client = KafkaClient(hosts=os.environ.get('KAFKA_URI', 'kafka:9092')) + +required_topics = os.environ.get('KAFKA_WAIT_FOR_TOPICS', '').split(',') +print('Checking for available topics:', repr(required_topics)) +for req_topic in required_topics: + if req_topic in client.topics: + topic = client.topics[req_topic] + if len(topic.partitions) > 0: + print('Topic is ready:', req_topic) + else: + print('Topic has no partitions:', req_topic) + sys.exit(1) + else: + print('Topic not found:', req_topic) + sys.exit(1) diff --git a/monasca-log-transformer/log-transformer.conf.j2 b/monasca-log-transformer/log-transformer.conf.j2 new file mode 100644 index 000000000..cc8815441 --- /dev/null +++ b/monasca-log-transformer/log-transformer.conf.j2 @@ -0,0 +1,75 @@ +input { + kafka { + zk_connect => "{{ ZOOKEEPER_URI }}" + topic_id => "log" + group_id => "log-transformer" + consumer_id => "monasca_log_transformer" + consumer_threads => "1" + } +} + +filter { + ruby { + code => "event['message_tmp'] = event['log']['message'][0..49]" + } + grok { + match => { + "[message_tmp]" => "(?i)(?AUDIT|CRITICAL|DEBUG|INFO|TRACE|ERR(OR)?|WARN(ING)?)|\"level\":\s?(?\d{2})" + } + } + if ! [log_level] { + grok { + match => { + "[log][message]" => "(?i)(?AUDIT|CRITICAL|DEBUG|INFO|TRACE|ERR(OR)?|WARN(ING)?)|\"level\":\s?(?\d{2})" + } + } + } + ruby { + init => " + LOG_LEVELS_MAP = { + # SYSLOG + 'warn' => :Warning, + 'err' => :Error, + # Bunyan errcodes + '10' => :Trace, + '20' => :Debug, + '30' => :Info, + '40' => :Warning, + '50' => :Error, + '60' => :Fatal + } + " + code => " + if event['log_level'] + # keep original value + log_level = event['log_level'].downcase + if LOG_LEVELS_MAP.has_key?(log_level) + event['log_level_original'] = event['log_level'] + event['log_level'] = LOG_LEVELS_MAP[log_level] + else + event['log_level'] = log_level.capitalize + end + else + event['log_level'] = 'Unknown' + end + " + } + + mutate { + add_field => { + "[log][level]" => "%{log_level}" + } + + # remove temporary fields + remove_field => ["log_level", "message_tmp"] + } +} + +output { + kafka { + bootstrap_servers => "{{ KAFKA_URI }}" + topic_id => "log-transformed" + client_id => "monasca_log_transformer" + compression_type => "none" + } +} diff --git a/monasca-log-transformer/start.sh b/monasca-log-transformer/start.sh new file mode 100755 index 000000000..c12c7e6f4 --- /dev/null +++ b/monasca-log-transformer/start.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +KAFKA_WAIT_RETRIES=${KAFKA_WAIT_RETRIES:-"24"} +KAFKA_WAIT_DELAY=${KAFKA_WAIT_DELAY:-"5"} + +if [ -n "$KAFKA_WAIT_FOR_TOPICS" ]; then + echo "Waiting for Kafka topics to become available..." + success="false" + + for i in $(seq $KAFKA_WAIT_RETRIES); do + python /kafka_wait_for_topics.py + if [ $? -eq 0 ]; then + success="true" + break + else + echo "Kafka not yet ready (attempt $i of $KAFKA_WAIT_RETRIES)" + sleep "$KAFKA_WAIT_DELAY" + fi + done + + if [ "$success" != "true" ]; then + echo "Kafka failed to become ready, exiting..." + sleep 1 + exit 1 + fi +fi + +if [ "$CONFIG_TEMPLATE" = "true" ]; then + python template.py \ + /etc/monasca/log-transformer.conf.j2 \ + /etc/monasca/log-transformer.conf +else + cp /etc/monasca/log-transformer.conf.j2 /etc/monasca/log-transformer.conf +fi + +logstash -f /etc/monasca/log-transformer.conf diff --git a/monasca-log-transformer/template.py b/monasca-log-transformer/template.py new file mode 100755 index 000000000..e83ec2fcf --- /dev/null +++ b/monasca-log-transformer/template.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +# (C) Copyright 2017 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import print_function + +import os +import sys + +from jinja2 import Template + + +def main(): + if len(sys.argv) != 3: + print('Usage: {} [input] [output]'.format(sys.argv[0])) + sys.exit(1) + + in_path = sys.argv[1] + out_path = sys.argv[2] + + with open(in_path, 'r') as in_file, open(out_path, 'w') as out_file: + t = Template(in_file.read()) + out_file.write(t.render(os.environ)) + +if __name__ == '__main__': + main()