From d30c17d6aab09445715f6bf652214fff46171c30 Mon Sep 17 00:00:00 2001
From: Tim Buckley <timothyb89@gmail.com>
Date: Wed, 16 Aug 2017 10:50:01 -0600
Subject: [PATCH] Merge branch 'master' into issue_124

---
 .env                                   |  24 ++++
 README.md                              |  58 ++++++++
 ci.py                                  |  30 ++++-
 docker-compose.yml                     |  49 +++----
 docker_build_util.sh                   |  41 ------
 grafana-init/build.sh                  |  17 ---
 grafana/build.sh                       |  15 ---
 kafka-init/Dockerfile                  |  13 ++
 kafka-init/README.md                   |  89 +++++++++++++
 kafka-init/build.yml                   |   7 +
 {kafka => kafka-init}/create_topics.py |  45 +------
 kafka-init/wait-for.sh                 | 178 +++++++++++++++++++++++++
 kafka/Dockerfile                       |   2 +-
 kafka/README.md                        |  51 -------
 kafka/build.sh                         |  18 ---
 kafka/build.yml                        |   3 +-
 kafka/start.sh                         |   3 -
 keystone-init/README.md                |   6 +-
 keystone-init/build.yml                |   2 +-
 keystone-init/keystone_init.py         |  24 ++++
 monasca-agent-collector/Dockerfile     |   2 +-
 monasca-agent-forwarder/Dockerfile     |   2 +-
 monasca-api-python/build.sh            |  20 ---
 monasca-forwarder/build.sh             |  18 ---
 monasca-notification/build.sh          |  18 ---
 monasca-persister-python/build.sh      |  36 -----
 mysql-init/build.sh                    |  17 ---
 27 files changed, 456 insertions(+), 332 deletions(-)
 create mode 100644 .env
 delete mode 100644 docker_build_util.sh
 delete mode 100755 grafana-init/build.sh
 delete mode 100755 grafana/build.sh
 create mode 100644 kafka-init/Dockerfile
 create mode 100644 kafka-init/README.md
 create mode 100644 kafka-init/build.yml
 rename {kafka => kafka-init}/create_topics.py (80%)
 create mode 100644 kafka-init/wait-for.sh
 delete mode 100755 kafka/build.sh
 delete mode 100755 monasca-api-python/build.sh
 delete mode 100755 monasca-forwarder/build.sh
 delete mode 100755 monasca-notification/build.sh
 delete mode 100755 monasca-persister-python/build.sh
 delete mode 100755 mysql-init/build.sh

diff --git a/.env b/.env
new file mode 100644
index 000000000..823ff6790
--- /dev/null
+++ b/.env
@@ -0,0 +1,24 @@
+INFLUXDB_VERSION=1.3.3-alpine
+INFLUXDB_INIT_VERSION=latest
+
+MYSQL_VERSION=5.5
+MYSQL_INIT_VERSION=1.5.2
+
+MEMCACHED_VERSION=1.5.0-alpine
+CADVISOR_VERSION=v0.26.1
+ZOOKEEPER_VERSION=3.4
+
+MON_SIDECAR_VERSION=1.0.0
+MON_KEYSTONE_VERSION=1.1.1
+MON_KAFKA_VERSION=0.9.0.1-2.11-1.1.0
+MON_KAFKA_INIT_VERSION=0.0.1
+MON_ALARMS_VERSION=1.1.0
+MON_GRAFANA_VERSION=4.0.0-1.1.1
+MON_GRAFANA_INIT_VERSION=1.2.0
+
+MON_API_VERSION=master-20170803-134039
+MON_PERSISTER_VERSION=master-20170626-213856
+MON_THRESH_VERSION=master-20170726-195714
+MON_NOTIFICATION_VERSION=master-20170505-220138
+MON_AGENT_FORWARDER_VERSION=master-20170811-222309
+MON_AGENT_COLLECTOR_VERSION=master-20170811-222309
diff --git a/README.md b/README.md
index 9577fd9c3..22c430d24 100644
--- a/README.md
+++ b/README.md
@@ -55,6 +55,64 @@ A number of custom dependency containers are also here:
  * `kafka/`: Dockerfile for k8s-compatible kafka
  * `storm/`: Dockerfile for Storm container
 
+Version control
+---------------
+
+**monasca-docker** leverages the [environmental variable handling][3] of 
+[docker-compose][1]. The latest release, a.k.a. versions of the images that
+are used for *master* branch is controlled with the help of [.env](./.env) file.
+In overall the file contains all the images' versions and resembles the following
+code snippet:
+
+```sh
+INFLUXDB_VERSION=1.3.2-alpine
+INFLUXDB_INIT_VERSION=latest
+MYSQL_VERSION=5.5
+MYSQL_INIT_VERSION=1.5.1
+MEMCACHED_VERSION=1.5.0-alpine
+CADVISOR_VERSION=v0.26.1
+ZOOKEEPER_VERSION=3.4
+
+MON_SIDECAR_VERSION=1.0.0
+MON_KEYSTONE_VERSION=1.1.1
+MON_KAFKA_VERSION=0.9.0.1-2.11-1.0.2
+MON_ALARMS_VERSION=1.1.0
+MON_GRAFANA_VERSION=4.0.0-1.1.1
+MON_GRAFANA_INIT_VERSION=1.1.0
+
+MON_API_VERSION=master
+MON_PERSISTER_VERSION=master
+MON_THRESH_VERSION=latest
+MON_NOTIFICATION_VERSION=master
+MON_AGENT_FORWARDER_VERSION=master
+MON_AGENT_COLLECTOR_VERSION=master
+```
+
+This file is automatically picked by [docker-compose][1], hence there are no ```export```
+instructions. 
+
+In order to override certain images versions, for example to create OpenStack Ocata based deployment,
+you may want to create a file, similar to this one:
+
+```sh
+
+export INFLUXDB_VERSION=1.1.0-alpine
+export MON_API_VERSION=ocata
+export MON_PERSISTER_VERSION=ocata
+export MON_THRESH_VERSION=ocata
+export MON_NOTIFICATION_VERSION=ocata
+export MON_AGENT_FORWARDER_VERSION=ocata
+export MON_AGENT_COLLECTOR_VERSION=ocata
+
+```
+
+That file, called ```.ocata.env```, differs from [.env](./.env), as it contains ```export```
+instructions. That is mandatory approach, as sourced variables takes precedence over
+aformentioned environmental file. 
+
+    Alternative approach, that achieves the same result, is to override values
+    of certain variables directly in ```.env``` file.
 
 [1]: https://docs.docker.com/compose/
 [2]: https://github.com/hpcloud-mon/monasca-docker/blob/master/keystone/preload.yml
+[3]: https://docs.docker.com/compose/environment-variables/
diff --git a/ci.py b/ci.py
index 43895167f..abc9948cd 100644
--- a/ci.py
+++ b/ci.py
@@ -32,6 +32,7 @@
     'zookeeper': 'zookeeper',
     'influxdb': 'influxdb',
     'kafka': 'kafka',
+    'kafka-init': 'kafka-init',
     'monasca-thresh': 'thresh',
     'monasca-persister-python': 'monasca-persister',
     'mysql-init': 'mysql-init',
@@ -254,6 +255,7 @@ def handle_pull_request(files, modules, tags):
     run_docker_compose()
     wait_for_init_jobs()
     run_smoke_tests()
+    run_tempest_tests()
 
 
 def get_current_init_status(docker_id):
@@ -334,9 +336,11 @@ def kill(signal, frame):
 
 def wait_for_init_jobs():
     init_status_dict = {"mysql-init": False,
-                        "influxdb-init": False}
+                        "influxdb-init": False,
+                        "kafka-init": False}
     docker_id_dict = {"mysql-init": "",
-                      "influxdb-init": ""}
+                      "influxdb-init": "",
+                      "kafka-init": ""}
     amount_succeeded = 0
     for attempt in range(40):
         time.sleep(30)
@@ -445,6 +449,28 @@ def kill(signal, frame):
         sys.exit(p.returncode)
 
 
+def run_tempest_tests():
+    tempest_tests_run = ['docker', 'run', '-e', 'KEYSTONE_SERVER=keystone', '-e',
+                         'KEYSTONE_PORT=5000', '--net', 'monascadocker_default',
+                         'monasca/tempest-tests:latest']
+
+    p = subprocess.Popen(tempest_tests_run, stdin=subprocess.PIPE)
+
+    def kill(signal, frame):
+        p.kill()
+        print()
+        print('killed!')
+        sys.exit(1)
+
+    signal.signal(signal.SIGINT, kill)
+    if p.wait() != 0:
+        print('Tempest-tests failed, listing containers/logs.')
+        output_docker_logs()
+        output_docker_ps()
+        print('Exiting!')
+        sys.exit(p.returncode)
+
+
 def handle_other(files, modules, tags):
     print('Unsupported event type "%s", nothing to do.' % (
         os.environ.get('TRAVS_EVENT_TYPE')))
diff --git a/docker-compose.yml b/docker-compose.yml
index dffbcbe0b..4be2ed067 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -2,10 +2,14 @@ version: '3'
 services:
 
   memcached:
-    image: memcached:alpine
+    image: memcached:${MEMCACHED_VERSION}
 
   influxdb:
-    image: influxdb:latest
+    image: influxdb:${INFLUXDB_VERSION}
+  influxdb-init:
+    image: monasca/influxdb-init:${INFLUXDB_INIT_VERSION}
+    depends_on:
+      - influxdb
 
   # cadvisor will allow host metrics to be collected, but requires significant
   # access to the host system
@@ -13,7 +17,7 @@ services:
   # environment variable should be set to "false" in the `agent-collector`
   # block - however no metrics will be collected
   cadvisor:
-    image: google/cadvisor:v0.26.1
+    image: google/cadvisor:${CADVISOR_VERSION}
     volumes:
       - "/:/rootfs:ro"
       - "/var/run:/var/run:rw"
@@ -21,30 +25,29 @@ services:
       - "/var/lib/docker:/var/lib/docker:ro"
 
   agent-forwarder:
-    image: monasca/agent-forwarder:master
+    image: monasca/agent-forwarder:${MON_AGENT_FORWARDER_VERSION}
     environment:
       NON_LOCAL_TRAFFIC: "true"
 
   agent-collector:
-    image: monasca/agent-collector:master
+    image: monasca/agent-collector:${MON_AGENT_COLLECTOR_VERSION}
     environment:
       AGENT_HOSTNAME: "docker-host"
       FORWARDER_URL: "http://agent-forwarder:17123"
       CADVISOR: "true"
       CADVISOR_URL: "http://cadvisor:8080/"
 
-  influxdb-init:
-    image: monasca/influxdb-init:latest
-    depends_on:
-      - influxdb
-
   alarms:
-    image: monasca/alarms:1.1.0
+    image: monasca/alarms:${MON_ALARMS_VERSION}
 
   zookeeper:
-    image: zookeeper:3.4
+    image: zookeeper:${ZOOKEEPER_VERSION}
   kafka:
-    image: monasca/kafka:0.9.0.1-2.11-1.0.2
+    image: monasca/kafka:${MON_KAFKA_VERSION}
+    depends_on:
+      - zookeeper
+  kafka-init:
+    image: monasca/kafka-init:${MON_KAFKA_INIT_VERSION}
     environment:
       ZOOKEEPER_CONNECTION_STRING: "zookeeper:2181"
       KAFKA_DELETE_TOPIC_ENABLE: "true"
@@ -60,17 +63,17 @@ services:
       - zookeeper
 
   mysql:
-    image: mysql:5.5
+    image: mysql:${MYSQL_VERSION}
     environment:
       MYSQL_ROOT_PASSWORD: secretmysql
   mysql-init:
-    image: monasca/mysql-init:1.5.1
+    image: monasca/mysql-init:${MYSQL_INIT_VERSION}
     environment:
       MYSQL_INIT_DISABLE_REMOTE_ROOT: "false"
       MYSQL_INIT_RANDOM_PASSWORD: "false"
 
   keystone:
-    image: monasca/keystone:1.1.1
+    image: monasca/keystone:${MON_KEYSTONE_VERSION}
     environment:
       KEYSTONE_HOST: keystone
       KEYSTONE_PASSWORD: secretadmin
@@ -86,10 +89,10 @@ services:
       - "35357:35357"
 
   monasca-sidecar:
-    image: timothyb89/monasca-sidecar:1.0.0
+    image: timothyb89/monasca-sidecar:${MON_SIDECAR_VERSION}
 
   monasca:
-    image: monasca/api:master
+    image: monasca/api:${MON_API_VERSION}
     environment:
       SIDECAR_URL: http://monasca-sidecar:4888/v1/ingest
     depends_on:
@@ -104,7 +107,7 @@ services:
       - "8070:8070"
 
   monasca-persister:
-    image: monasca/persister:master
+    image: monasca/persister:${MON_PERSISTER_VERSION}
     depends_on:
       - monasca
       - influxdb
@@ -112,7 +115,7 @@ services:
       - kafka
 
   thresh:
-    image: monasca/thresh:latest
+    image: monasca/thresh:${MON_THRESH_VERSION}
     environment:
       NO_STORM_CLUSTER: "true"
       WORKER_MAX_HEAP_MB: "256"
@@ -121,7 +124,7 @@ services:
       - kafka
 
   monasca-notification:
-    image: monasca/notification:master
+    image: monasca/notification:${MON_NOTIFICATION_VERSION}
     environment:
       NF_PLUGINS: "webhook"
     depends_on:
@@ -131,7 +134,7 @@ services:
       - mysql
 
   grafana:
-    image: monasca/grafana:4.0.0-1.1.1
+    image: monasca/grafana:${MON_GRAFANA_VERSION}
     environment:
       GF_AUTH_BASIC_ENABLED: "false"
       GF_USERS_ALLOW_SIGN_UP: "true"
@@ -147,6 +150,6 @@ services:
       - monasca
 
   grafana-init:
-    image: monasca/grafana-init:1.1.0
+    image: monasca/grafana-init:${MON_GRAFANA_INIT_VERSION}
     depends_on:
       - grafana
diff --git a/docker_build_util.sh b/docker_build_util.sh
deleted file mode 100644
index 9012fecba..000000000
--- a/docker_build_util.sh
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/bin/bash
-
-# A small collection of functions for automating bulk docker image builds.
-
-# usage: build tag build_arg...
-# uses global $docker_repo
-function build() {
-  tag=$1
-  shift
-
-  args=()
-  for arg in "$@"; do
-    args+=("--build-arg $arg")
-  done
-
-  if [[ -n "$HTTP_PROXY" ]]; then
-    args+=("--build-arg HTTP_PROXY=$HTTP_PROXY")
-  fi
-
-  if [[ -n "$HTTPS_PROXY" ]]; then
-    args+=("--build-arg HTTPS_PROXY=$HTTPS_PROXY")
-  fi
-
-  # shellcheck disable=SC2068
-  docker build \
-    -t ${docker_repo:?}:$tag \
-    ${args[@]} \
-    .
-
-  docker push ${docker_repo:?}:$tag
-}
-
-# usage: retag [old tag] [new tag]
-# uses global $docker_repo
-function retag() {
-  old_tag=$1
-  new_tag=$2
-
-  docker tag ${docker_repo:?}:$old_tag $docker_repo:$new_tag
-  docker push ${docker_repo:?}:$new_tag
-}
diff --git a/grafana-init/build.sh b/grafana-init/build.sh
deleted file mode 100755
index 20862c082..000000000
--- a/grafana-init/build.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/bin/bash
-
-# A small script to build the  against several branches and
-# docker hub tags.
-
-set -e
-
-source "$(dirname "$(dirname "$(readlink -f $0)")")/docker_build_util.sh"
-
-# $docker_repo is referenced in docker_build_util.sh
-# shellcheck disable=SC2034
-docker_repo=${1:-"monasca/grafana-init"}
-
-build 1.1.0
-retag 1.1.0 1.1
-retag 1.1.0 1
-retag 1.1.0 latest
diff --git a/grafana/build.sh b/grafana/build.sh
deleted file mode 100755
index 89a4f95a1..000000000
--- a/grafana/build.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-
-# A small script to build the grafana against several branches and
-# docker hub tags.
-
-set -e
-
-source "$(dirname "$(dirname "$(readlink -f $0)")")/docker_build_util.sh"
-
-# $docker_repo is referenced in docker_build_util.sh
-# shellcheck disable=SC2034
-docker_repo=${1:-"monasca/grafana"}
-
-build 4.1.0-pre1-1.0.0
-retag 4.1.0-pre1-1.0.0 latest
diff --git a/kafka-init/Dockerfile b/kafka-init/Dockerfile
new file mode 100644
index 000000000..fdd217395
--- /dev/null
+++ b/kafka-init/Dockerfile
@@ -0,0 +1,13 @@
+ARG MON_KAFKA_VERSION=0.9.0.1-2.11-1.1.0
+FROM monasca/kafka:${MON_KAFKA_VERSION}
+
+ENV KAFKA_HOST=kafka:9092 \
+    KAFKA_TIMEOUT=60 \
+    ZOOKEEPER_CONNECTION_STRING=zookeeper:2181 \
+    KAFKA_CREATE_TOPICS="" \
+    KAFKA_TOPIC_CONFIG=""
+
+COPY create_topics.py wait-for.sh /
+RUN chmod +x /create_topics.py /wait-for.sh
+
+CMD /wait-for.sh $KAFKA_HOST --strict --timeout=$KAFKA_TIMEOUT -- /create_topics.py
diff --git a/kafka-init/README.md b/kafka-init/README.md
new file mode 100644
index 000000000..1e55ea92b
--- /dev/null
+++ b/kafka-init/README.md
@@ -0,0 +1,89 @@
+kafka-init
+==========
+
+**kafka-init** is responsible for creating the topics.
+
+Tags
+----
+
+Images in this repository are tagged as follows:
+
+* `0.0.1` - standard [SemVer][1]
+* `latest`: the latest version recommended to use with other Monasca
+  components.
+
+
+Usage
+-----
+
+**kafka-init** requires a running instance of [Kafka][2]. It can be launched using
+command similar to this ```docker run -d -p 9092:9092 -l zookeeper monasca/kafka```.
+**kafka-init** will wait for Kafka to become accessible.
+
+Configuration
+-------------
+
+Several parameters can be specified using environment variables:
+
+| Variable                      | Default          | Description                           |
+|-------------------------------|------------------|---------------------------------------|
+| `ZOOKEEPER_CONNECTION_STRING` | `zookeeper:2181` | Comma-separated list of ZK hosts      |
+| `KAFKA_HOST`                  | `kafka:9092`     | One of the Kafka brokers              |
+| `KAFKA_TIMEOUT`               | `60`      | How long to wait for Kafka to become available |
+| `KAFKA_CREATE_TOPICS`         | `unset`   | Topics to create on startup, see below       |
+| `KAFKA_TOPIC_CONFIG`          | `unset`   | Default config args for created topics       |
+| `KAFKA_CREATE_TOPICS_SCRIPT`  | `/kafka/bin/kafka-topics.sh` | Path to script that creates topics |
+
+Topic creation
+--------------
+
+This image will create topics automatically on first startup when
+`KAFKA_CREATE_TOPICS` is set:
+
+```
+docker run \
+    --name kafka-init \
+    --link zookeeper \
+    --link kafka \
+    -e KAFKA_CREATE_TOPICS="topic1:64:1,topic2:16:1"
+    monasca/kafka-init:latest
+```
+
+The variable should be set to a comma-separated list of topic strings. These
+each look like so:
+
+```
+[topic name]:partitions=[partitions]:replicas=[replicas]:[key]=[val]:[key]=[val]
+```
+
+In the above, `partitions` and `replicas` are required, and tokens surrounded by
+`[brackets]` should be replaced with the desired value. All other properties
+will be translated to `--config key=val` when running `kafka-topics.sh`.
+Example:
+
+```
+my_topic_name:partitions=3:replicas=1:segment.ms=900000
+```
+
+Partitions and replicas also support an index-based shorthand, so the following
+works as well:
+
+```
+[topic name]:[partitions]:[replicas]
+```
+
+As an example, this is a valid `KAFKA_CREATE_TOPICS` string for [Monasca][3]
+installations as used in the [docker-compose][4] environment:
+
+    metrics:64:1,alarm-state-transitions:12:1,alarm-notifications:12:1,retry-notifications:3:1,events:12:1,60-seconds-notifications:3:1
+
+If desired, default config parameters can be set using `KAFKA_TOPIC_CONFIG`.
+These use `key=value,key2=value2` syntax and will be translated into
+`--config key=value` arguments as described above. If a duplicate key is passed
+in a specific topic string, it will override the value specified here.
+
+
+[1]: http://semver.org/
+[2]: https://github.com/monasca/monasca-docker/tree/master/kafka
+[3]: https://hub.docker.com/r/monasca/api/
+[4]: https://github.com/monasca/monasca-docker/
diff --git a/kafka-init/build.yml b/kafka-init/build.yml
new file mode 100644
index 000000000..9ed50f17e
--- /dev/null
+++ b/kafka-init/build.yml
@@ -0,0 +1,7 @@
+repository: monasca/kafka-init
+variants:
+  - tag: 0.0.1
+    aliases:
+      - ':latest'
+    args:
+      MON_KAFKA_VERSION: "0.9.0.1-2.11-1.1.0"
diff --git a/kafka/create_topics.py b/kafka-init/create_topics.py
similarity index 80%
rename from kafka/create_topics.py
rename to kafka-init/create_topics.py
index fc42ac723..56f7807d6 100644
--- a/kafka/create_topics.py
+++ b/kafka-init/create_topics.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 # (C) Copyright 2017 Hewlett Packard Enterprise Development LP
+# Copyright 2017 Fujitsu LIMITED
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
@@ -25,14 +26,9 @@
 
 SCRIPT_PATH = os.environ.get('KAFKA_CREATE_TOPICS_SCRIPT',
                              '/kafka/bin/kafka-topics.sh')
-
 TOPIC_STRING = os.environ.get('KAFKA_CREATE_TOPICS', '')
 CONFIG_STRING = os.environ.get('KAFKA_TOPIC_CONFIG', '')
-
 ZOOKEEPER_CONNECTION_STRING = os.environ.get('ZOOKEEPER_CONNECTION_STRING')
-KAFKA_LISTEN_PORT = os.environ.get('KAFKA_LISTEN_PORT', '9092')
-KAFKA_WAIT_DELAY = os.environ.get('KAFKA_WAIT_DELAY', '5')
-KAFKA_WAIT_RETRIES = os.environ.get('KAFKA_WAIT_RETRIES', '24')
 
 logging.basicConfig(level=logging.DEBUG)
 logger = logging.getLogger(__name__)
@@ -60,26 +56,6 @@ def __str__(self):
         )
 
 
-def is_kafka_running():
-    p = subprocess.Popen(['netstat', '-nlt'],
-                         stdout=subprocess.PIPE,
-                         stderr=subprocess.PIPE,
-                         env=subprocess_env)
-
-    out, err = p.communicate()
-    if p.returncode != 0:
-        raise CaptureException(p.returncode, out, err)
-
-    lines = out.splitlines()[2:]
-    for line in lines:
-        tokens = line.split()
-        address = tokens[3]
-        if address.endswith(':%s' % KAFKA_LISTEN_PORT):
-            return True
-
-    return False
-
-
 def kafka_topics(verb, args=None):
     args = [
         SCRIPT_PATH,
@@ -194,25 +170,6 @@ def create_topics(default_config, existing_topics):
 
 
 def main():
-    retries = int(KAFKA_WAIT_RETRIES)
-    delay = float(KAFKA_WAIT_DELAY)
-
-    logging.info('Waiting for Kafka to start...')
-    ready = False
-    for i in range(retries):
-        if is_kafka_running():
-            logger.info('Kafka has started, continuing...')
-            ready = True
-            break
-        else:
-            logger.info('Kafka is not ready yet (attempt %d of %d)',
-                        i + 1, retries)
-            time.sleep(delay)
-
-    if not ready:
-        logger.error('Kafka did not become ready in time, giving up!')
-        sys.exit(1)
-
     default_config = get_default_config()
     logger.info('Default topic config: %r', default_config)
 
diff --git a/kafka-init/wait-for.sh b/kafka-init/wait-for.sh
new file mode 100644
index 000000000..4e9bc4f33
--- /dev/null
+++ b/kafka-init/wait-for.sh
@@ -0,0 +1,178 @@
+#!/usr/bin/env bash
+# Use this script to test if a given TCP host/port are available
+# https://github.com/vishnubob/wait-for-it/blob/master/wait-for-it.sh
+
+cmdname=$(basename $0)
+
+echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $TIMEOUT -gt 0 ]]; then
+        echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT"
+    else
+        echoerr "$cmdname: waiting for $HOST:$PORT without a timeout"
+    fi
+    start_ts=$(date +%s)
+    while :
+    do
+        if [[ $ISBUSY -eq 1 ]]; then
+            nc -z $HOST $PORT
+            result=$?
+        else
+            (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1
+            result=$?
+        fi
+        if [[ $result -eq 0 ]]; then
+            end_ts=$(date +%s)
+            echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $QUIET -eq 1 ]]; then
+        timeout $BUSYTIMEFLAG $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT &
+    else
+        timeout $BUSYTIMEFLAG $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT &
+    fi
+    PID=$!
+    trap "kill -INT -$PID" INT
+    wait $PID
+    RESULT=$?
+    if [[ $RESULT -ne 0 ]]; then
+        echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT"
+    fi
+    return $RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        hostport=(${1//:/ })
+        HOST=${hostport[0]}
+        PORT=${hostport[1]}
+        shift 1
+        ;;
+        --child)
+        CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        STRICT=1
+        shift 1
+        ;;
+        -h)
+        HOST="$2"
+        if [[ $HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        PORT="$2"
+        if [[ $PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        TIMEOUT="$2"
+        if [[ $TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        CLI="$@"
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$HOST" == "" || "$PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+TIMEOUT=${TIMEOUT:-15}
+STRICT=${STRICT:-0}
+CHILD=${CHILD:-0}
+QUIET=${QUIET:-0}
+
+# check to see if timeout is from busybox?
+# check to see if timeout is from busybox?
+TIMEOUT_PATH=$(realpath $(which timeout))
+if [[ $TIMEOUT_PATH =~ "busybox" ]]; then
+        ISBUSY=1
+        BUSYTIMEFLAG="-t"
+else
+        ISBUSY=0
+        BUSYTIMEFLAG=""
+fi
+
+if [[ $CHILD -gt 0 ]]; then
+    wait_for
+    RESULT=$?
+    exit $RESULT
+else
+    if [[ $TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        RESULT=$?
+    else
+        wait_for
+        RESULT=$?
+    fi
+fi
+
+if [[ $CLI != "" ]]; then
+    if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then
+        echoerr "$cmdname: strict mode, refusing to execute subprocess"
+        exit $RESULT
+    fi
+    exec $CLI
+else
+    exit $RESULT
+fi
diff --git a/kafka/Dockerfile b/kafka/Dockerfile
index 73e0de3c1..a66d6d90b 100644
--- a/kafka/Dockerfile
+++ b/kafka/Dockerfile
@@ -28,7 +28,7 @@ RUN set -x && \
   rm /kafka.tar.gz /kafka_mirror.py && \
   apk del build-dep
 
-COPY template.py start.sh create_topics.py heap.py /
+COPY template.py start.sh heap.py /
 COPY templates /templates
 
 ENV PATH /kafka/bin:$PATH
diff --git a/kafka/README.md b/kafka/README.md
index 4ecb41247..6290b3543 100644
--- a/kafka/README.md
+++ b/kafka/README.md
@@ -52,9 +52,6 @@ Several parameters can be specified using environment variables:
 | `KAFKA_HOSTNAME_FROM_IP`      | `true`  | If `true`, set advertised hostname to container IP  |
 | `KAFKA_ADVERTISED_HOST_NAME`  | from IP | If set, use value as advertised hostname       |
 | `KAFKA_BROKER_ID`             | `-1`    | Unique Kafka broker ID, `-1` for auto          |
-| `KAFKA_CREATE_TOPICS`         | unset   | Topics to create on startup, see below         |
-| `KAFKA_TOPIC_CONFIG`          | unset   | Default config args for created topics         |
-| `KAFKA_DELETE_TOPIC_ENABLE`   | `false` | Enable topic deletion                          |
 | `KAFKA_LISTEN_PORT`           | `9092`  | Port for Kafka to listen on                    |
 | `KAFKA_ADVERTISED_PORT`       | `$KAFKA_LISTEN_PORT` | Kafka port advertised to clients  |
 | `KAFKA_CONTROLLED_SHUTDOWN_ENABLE` | `true` | If `true`, enable [controlled shutdown][6] |
@@ -81,57 +78,9 @@ kafka_1                 | [2017-06-15 06:32:09,572] INFO logfile=server.log Load
 If `GC_LOG_ENABLED` is set to True, the JVM Garbage Collection log will be written within the
 container at /kafka/logs/kafkaServer-gc.log. It can't be redirected to stdout.
 
-### Topic creation
-
-This image will create topics automatically on first startup when
-`KAFKA_CREATE_TOPICS` is set:
-
-```
-docker run \
-    --name kafka \
-    --link zookeeper \
-    -e KAFKA_CREATE_TOPICS="topic1:64:1,topic2:16:1"
-    monasca/kafka:latest
-```
-
-The variable should be set to a comma-separated list of topic strings. These
-each look like so:
-
-```
-[topic name]:partitions=[partitions]:replicas=[replicas]:[key]=[val]:[key]=[val]
-```
-
-In the above, `partitions` and `replicas` are required, and tokens surrounded by
-`[brackets]` should be replaced with the desired value. All other properties
-will be translated to `--config key=val` when running `kafka-topics.sh`.
-Example:
-
-```
-my_topic_name:partitions=3:replicas=1:segment.ms=900000
-```
-
-Partitions and replicas also support an index-based shorthand, so the following
-works as well:
-
-```
-[topic name]:[partitions]:[replicas]
-```
-
-As an example, this is a valid `KAFKA_CREATE_TOPICS` string for [Monasca][8]
-installations as used in the [docker-compose][4] environment:
-
-    metrics:64:1,alarm-state-transitions:12:1,alarm-notifications:12:1,retry-notifications:3:1,events:12:1,60-seconds-notifications:3:1
-
-If desired, default config parameters can be set using `KAFKA_TOPIC_CONFIG`.
-These use `key=value,key2=value2` syntax and will be translated into
-`--config key=value` arguments as described above. If a duplicate key is passed
-in a specific topic string, it will override the value specified here.
-
 [1]: http://kafka.apache.org/
 [2]: https://github.com/hpcloud-mon/monasca-docker/blob/master/kafka/
 [3]: https://github.com/hpcloud-mon/monasca-docker/blob/master/kafka/Dockerfile
 [4]: https://github.com/hpcloud-mon/monasca-docker/
 [5]: https://hub.docker.com/r/library/zookeeper/
 [6]: https://kafka.apache.org/documentation/#basic_ops_restarting
-[7]: https://github.com/wurstmeister/kafka-docker
-[8]: https://hub.docker.com/r/monasca/api/
diff --git a/kafka/build.sh b/kafka/build.sh
deleted file mode 100755
index 7894211df..000000000
--- a/kafka/build.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash
-
-# A small script to build the kafka container against several kafka versions and
-# docker hub tags.
-
-set -e
-
-source "$(dirname "$(dirname "$(readlink -f $0)")")/docker_build_util.sh"
-
-# $docker_repo is referenced in docker_build_util.sh
-# shellcheck disable=SC2034
-docker_repo=${1:-"monasca/kafka"}
-
-# TODO something about this build script prevents the kafka tarball from being
-# downloaded successfully. Will need to build manually until solved.
-
-build 0.9.0.1-2.11
-retag 0.9.0.1-2.11 latest
diff --git a/kafka/build.yml b/kafka/build.yml
index 50c64d014..cd6508c8f 100644
--- a/kafka/build.yml
+++ b/kafka/build.yml
@@ -1,7 +1,6 @@
 repository: monasca/kafka
 variants:
-  - tag: 0.9.0.1-2.11-1.0.2
-    aliases: [':latest']
+  - tag: 0.9.0.1-2.11-1.1.0
     args:
       KAFKA_VERSION: "0.9.0.1"
       SCALA_VERSION: "2.11"
diff --git a/kafka/start.sh b/kafka/start.sh
index 4d779c5c7..74ca2ae27 100755
--- a/kafka/start.sh
+++ b/kafka/start.sh
@@ -94,9 +94,6 @@ if [ "$KAFKA_JMX" = "true" ]; then
   fi
 fi
 
-echo "Creating topics..."
-python /create_topics.py &
-
 if [ "$GC_LOG_ENABLED" != "true" ]; then
   # This turns off the GC logging in /kafka/bin/kafka-server-start.sh
   # It is a hack, but I could not find another way to do it
diff --git a/keystone-init/README.md b/keystone-init/README.md
index 3eae749bc..2c51f4124 100644
--- a/keystone-init/README.md
+++ b/keystone-init/README.md
@@ -174,9 +174,9 @@ containers:
 {{ include "keystone_env" .Values.my_pod.auth | indent(6) }}
 ```
 
-[1]: https://github.com/hpcloud-mon/monasca-docker/blob/master/mysql-init/
-[2]: https://github.com/hpcloud-mon/monasca-docker/blob/master/mysql-init/Dockerfile
-[3]: https://github.com/hpcloud-mon/monasca-docker/
+[1]: https://github.com/monasca/monasca-docker/blob/master/keystone-init/
+[2]: https://github.com/monasca/monasca-docker/blob/master/keystone-init/Dockerfile
+[3]: https://github.com/monasca/monasca-docker/
 [4]: https://github.com/monasca/monasca-helm/tree/master/keystone-init
 [5]: https://github.com/monasca/monasca-docker/blob/7a7a6032c29ebba3a33e6af29566fd26243cf3ba/keystone-init/keystone_init.py#L39
 [6]: https://github.com/monasca/monasca-helm/blob/master/keystone-init/templates/_keystone_env.tpl
diff --git a/keystone-init/build.yml b/keystone-init/build.yml
index aedff51fc..d87dd4d12 100644
--- a/keystone-init/build.yml
+++ b/keystone-init/build.yml
@@ -2,6 +2,6 @@ repository: monasca/keystone-init
 variants:
   - tag: latest
     aliases:
-      - :1.0.2
+      - :1.0.3
       - :1.0
       - :1
diff --git a/keystone-init/keystone_init.py b/keystone-init/keystone_init.py
index 3731f1b86..a7333d1b0 100644
--- a/keystone-init/keystone_init.py
+++ b/keystone-init/keystone_init.py
@@ -173,6 +173,26 @@ def get_or_create_domain(client, name=None):
     return domain
 
 
+def get_keystone_admin_url(client, domain):
+    """
+    :type client: keystoneclient.v3.client.Client
+    :type domain: keystoneclient.v3.domains.Domain
+    :return: str or None
+    """
+    if 'OS_ADMIN_URL' in os.environ:
+        return os.environ['OS_ADMIN_URL']
+
+    try:
+        keystone_service = client.services.list(type='identity')[0]
+        endpoints = client.endpoints.list(domain=domain,
+                                          service=keystone_service,
+                                          interface='admin')
+        return endpoints[0].url
+    except IndexError:
+        logging.warn('failed to detect keystone admin URL!', exc_info=True)
+        return None
+
+
 @retry()
 def get_or_create_project(client, domain, name):
     """
@@ -412,6 +432,7 @@ def load_domains(ks, domains):
 
     for name, options in domains.iteritems():
         domain = get_or_create_domain(ks, None if name == 'default' else name)
+        admin_url = get_keystone_admin_url(ks, domain)
 
         logging.info('creating projects...')
         for project in options.get('projects', []):
@@ -463,6 +484,9 @@ def load_domains(ks, domains):
                             'OS_USER_DOMAIN_NAME': domain.name
                         }
 
+                        if admin_url:
+                            fields.update({'OS_ADMIN_URL': admin_url})
+
                         if project:
                             fields.update({
                                 'OS_PROJECT_NAME': project.name,
diff --git a/monasca-agent-collector/Dockerfile b/monasca-agent-collector/Dockerfile
index 990c1d697..20afdb498 100644
--- a/monasca-agent-collector/Dockerfile
+++ b/monasca-agent-collector/Dockerfile
@@ -1,4 +1,4 @@
-FROM monasca/agent-base:master-20170706-222054
+FROM monasca/agent-base:master-20170811-213513
 
 ARG AGENT_USER="mon-agent"
 
diff --git a/monasca-agent-forwarder/Dockerfile b/monasca-agent-forwarder/Dockerfile
index 9a4f3dad8..69e838c14 100644
--- a/monasca-agent-forwarder/Dockerfile
+++ b/monasca-agent-forwarder/Dockerfile
@@ -1,4 +1,4 @@
-FROM monasca/agent-base:master-20170706-222054
+FROM monasca/agent-base:master-20170811-213513
 
 ARG AGENT_USER="mon-agent"
 
diff --git a/monasca-api-python/build.sh b/monasca-api-python/build.sh
deleted file mode 100755
index 99c91d93e..000000000
--- a/monasca-api-python/build.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash
-
-# A small script to build the Python API against several branches and
-# docker hub tags.
-
-set -e
-
-source "$(dirname "$(dirname "$(readlink -f $0)")")/docker_build_util.sh"
-
-# $docker_repo is referenced in docker_build_util.sh
-# shellcheck disable=SC2034
-docker_repo=${1:-"monasca/api"}
-
-build master API_BRANCH=master
-retag master master-python
-retag master latest
-retag master latest-python
-
-# TODO: build stable API branches and releases when one exists that will work
-# with the provided config files
diff --git a/monasca-forwarder/build.sh b/monasca-forwarder/build.sh
deleted file mode 100755
index 1678842ac..000000000
--- a/monasca-forwarder/build.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash
-
-# A small script to build the forwarder against several branches and docker hub
-# tags.
-
-set -e
-
-source "$(dirname "$(dirname "$(readlink -f $0)")")/docker_build_util.sh"
-
-# $docker_repo is referenced in docker_build_util.sh
-# shellcheck disable=SC2034
-docker_repo=${1:-"monasca/forwarder"}
-
-build master FORWARDER_BRANCH=master
-retag master latest
-
-# TODO: build stable branches and releases when one exists that includes
-# necessary patches
diff --git a/monasca-notification/build.sh b/monasca-notification/build.sh
deleted file mode 100755
index c51aee57e..000000000
--- a/monasca-notification/build.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash
-
-# A small script to build the notification engine against several branches and
-# docker hub tags.
-
-set -e
-
-source "$(dirname "$(dirname "$(readlink -f $0)")")/docker_build_util.sh"
-
-# $docker_repo is referenced in docker_build_util.sh
-# shellcheck disable=SC2034
-docker_repo=${1:-"monasca/notification"}
-
-build master NOTIFICATION_BRANCH=master
-retag master latest
-
-# TODO: build stable branches and releases when one exists that includes
-# necessary patches
diff --git a/monasca-persister-python/build.sh b/monasca-persister-python/build.sh
deleted file mode 100755
index 8ddbcdd6f..000000000
--- a/monasca-persister-python/build.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/bash
-
-# A small script to build the Python persister against several branches and
-# docker hub tags.
-
-set -e
-
-source "$(dirname "$(dirname "$(readlink -f $0)")")/docker_build_util.sh"
-
-# $docker_repo is referenced in docker_build_util.sh
-# shellcheck disable=SC2034
-docker_repo=${1:-"monasca/persister"}
-
-build master PERSISTER_BRANCH=master
-retag master master-python
-
-build 1.4.0 PERSISTER_BRANCH=1.4.0
-retag 1.4.0 1.4
-retag 1.4.0 1
-retag 1.4.0 1.4.0-python
-retag 1.4.0 1.4-python
-retag 1.4.0 1-python
-
-build 1.3.0 PERSISTER_BRANCH=1.3.0
-retag 1.3.0 1.3
-retag 1.3.0 1.3.0-python
-retag 1.3.0 1.3-python
-
-build mitaka PERSISTER_BRANCH="stable/mitaka"
-retag mitaka mitaka-python
-
-build newton PERSISTER_BRANCH="stable/newton"
-retag newton newton-python
-
-retag 1.4.0 latest
-retag 1.4.0 latest-python
diff --git a/mysql-init/build.sh b/mysql-init/build.sh
deleted file mode 100755
index fa05021a4..000000000
--- a/mysql-init/build.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/bin/bash
-
-# A small script to build the mysql-init container against several docker hub
-# tags.
-
-set -e
-
-source "$(dirname "$(dirname "$(readlink -f $0)")")/docker_build_util.sh"
-
-# $docker_repo is referenced in docker_build_util.sh
-# shellcheck disable=SC2034
-docker_repo=${1:-"monasca/mysql-init"}
-
-build 1.1.0
-retag 1.1.0 1.1
-retag 1.1.0 1
-retag 1.1.0 latest