Merge branch 'master' into fix_lint4

.env

@@ -19,7 +19,7 @@ MON_GRAFANA_INIT_VERSION=1.2.0
 MON_API_VERSION=master-20170803-134039
 MON_PERSISTER_VERSION=master-20170626-213856
 MON_THRESH_VERSION=master-20170726-195714
-MON_NOTIFICATION_VERSION=master-20170505-220138
+MON_NOTIFICATION_VERSION=master-20170908-065949
 MON_AGENT_FORWARDER_VERSION=master-20170811-222309
 MON_AGENT_COLLECTOR_VERSION=master-20170811-222309
 

README.md

@@ -169,7 +169,7 @@ environment:
 In `kibana`:
 ```yaml
 environment:
-  KEYSTONE_URI: http://192.168.10.6/identity
+  KEYSTONE_URI: 192.168.10.6/identity
 ```
 
 In `log-api`:

ci.py

@@ -157,6 +157,10 @@ def get_dirty_modules(dirty_files):
 
             dirty.add(mod)
 
+    if len(dirty) > 5:
+        print ('Max number of changed modules exceded. '
+               'Please break up the patch set until a maximum of 5 modules are changed.')
+        sys.exit(1)
     return list(dirty)
 
 

docker-compose.yml

@@ -31,6 +31,7 @@ services:
 
   agent-collector:
     image: monasca/agent-collector:${MON_AGENT_COLLECTOR_VERSION}
+    restart: on-failure
     environment:
       AGENT_HOSTNAME: "docker-host"
       FORWARDER_URL: "http://agent-forwarder:17123"

elasticsearch-init/upload.sh

@@ -4,8 +4,9 @@ TPL_DIR=/templates
 
 _get_tpl_name_from_file() {
     local tpl=$1
-    local tpl_name=$(basename $tpl)
-    echo $tpl_name
+    local tpl_name
+    tpl_name=$(basename "$tpl")
+    echo "$tpl_name"
 }
 
 if [ ! -d $TPL_DIR ]; then
@@ -18,10 +19,8 @@ fi
 for template in $TPLS; do
 
     echo "Handling template file $template"
-    tpl_name=`_get_tpl_name_from_file $template`
+    tpl_name=`_get_tpl_name_from_file "$template"`
 
-    curl -XPUT --retry 2 --retry-delay 2 $ELASTICSEARCH_URI/_template/${tpl_name} -d @$TPL_DIR/$template
+    curl -XPUT --retry 2 --retry-delay 2 "$ELASTICSEARCH_URI"/_template/"${tpl_name}" -d @$TPL_DIR/"$template"
 
 done
-
-

elasticsearch-init/wait-for.sh

@@ -1,15 +1,15 @@
 #!/bin/sh
 
-: ${SLEEP_LENGTH:=2}
+: "${SLEEP_LENGTH:=2}"
 
 wait_for() {
-  echo Waiting for $1 to listen on $2...
-  while ! nc -z $1 $2; do echo sleeping; sleep $SLEEP_LENGTH; done
+  echo "Waiting for $1 to listen on $2..."
+  while ! nc -z "$1" "$2"; do echo sleeping; sleep "$SLEEP_LENGTH"; done
 }
 
 for var in "$@"
 do
   host=${var%:*}
   port=${var#*:}
-  wait_for $host $port
+  wait_for "$host" "$port"
 done

grafana/Dockerfile

@@ -11,19 +11,22 @@ arg MONASCA_APP_BRANCH=master
 # `docker build`
 arg REBUILD=1
 
-env GOPATH=/go GOBIN=/go/bin
+env GOPATH=/go GOBIN=/go/bin \
+    GRAFANA_ADMIN_USER=admin \
+    GRAFANA_ADMIN_PASSWORD=admin
 
 run mkdir -p /var/lib/grafana && \
   mkdir -p $GOPATH/src/github.com/grafana/grafana && \
   cd $GOPATH/src/github.com/grafana/grafana/ && \
   apk add --no-cache --virtual build-dep \
-    nodejs go git musl-dev python make g++ && \
+    nodejs go git musl-dev make g++ && \
+  apk add --no-cache python py-jinja2 && \
   git init && \
   git remote add origin $GRAFANA_REPO && \
   git fetch origin $GRAFANA_BRANCH && \
   git reset --hard FETCH_HEAD && \
   go run build.go setup && go run build.go build && \
-  npm install -g yarn && yarn && \
+  npm install -g yarn@0.27 && yarn && \
   ./node_modules/.bin/grunt build --force && \
   cp -r ./conf /var/lib/grafana/conf && \
   cp -r ./public_gen /var/lib/grafana/public && \
@@ -51,10 +54,13 @@ run mkdir -p /var/lib/grafana && \
   rm -f /go/bin/govendor && \
   rm -rf /go/pkg
 
-copy grafana.ini /etc/grafana/grafana.ini
+copy grafana.ini.j2 /etc/grafana/grafana.ini.j2
+copy template.py start.sh /
+copy drilldown.js /var/lib/grafana/public/dashboards/drilldown.js
+run chmod +x /template.py /start.sh
 expose 3000
 
 healthcheck --interval=10s --timeout=5s \
   cmd wget -q http://localhost:3000 -O - > /dev/null
 
-cmd ["/go/bin/grafana-server", "-config", "/etc/grafana/grafana.ini", "-homepath", "/var/lib/grafana"]
+cmd ["/start.sh"]

grafana/README.md

@@ -37,9 +37,22 @@ properties:
 Note that in the [Kubernetes environment][6], the URL should instead be
 `http://monasca-api:8070/`.
 
+The dashboard for monasca-ui, which is opened by clicking on "Graph Metric"
+of the 'Alarm' tab, is available. You can use this dashboard outside of monasca-ui
+with the following link.
+`http://your-grafana-url:3000/dashboard/script/drilldown.js`
+Specify the metric name with the key "metric" and dimensions as additional
+parameters if necessary as below.
+`http://your-grafana-url:3000/dashboard/script/drilldown.js?metric=sample&dim1=val1`
+
 Configuration
 -------------
 
+| Variable                 | Default | Description                     |
+|--------------------------|---------|---------------------------------|
+| `GRAFANA_ADMIN_USER`     | `admin` | Grafana admin user name         |
+| `GRAFANA_ADMIN_PASSWORD` | `admin` | Grafana admin user password     |
+
 Grafana can be configured using [environment variables][7], though a
 configuration file can also be mounted into the image at
 `/etc/grafana/grafana.ini`. Plugins should be placed in

grafana/drilldown.js

@@ -0,0 +1,88 @@
+/* global _ */
+
+/*
+ * Complex scripted dashboard
+ * This script generates a dashboard object that Grafana can load. It also takes a number of user
+ * supplied URL parameters (in the ARGS variable)
+ *
+ * Return a dashboard object, or a function
+ *
+ * For async scripts, return a function, this function must take a single callback function as argument,
+ * call this callback function with the dashboard object (see scripted_async.js for an example)
+ */
+
+'use strict';
+
+// accessible variables in this scope
+var window, document, ARGS, $, jQuery, moment, kbn;
+
+// Setup variables
+var dashboard, timespan;
+
+// All url parameters are available via the ARGS object
+var ARGS;
+
+// Set a default timespan if one isn't specified
+timespan = '1d';
+
+// keys which should not be dimensions
+var exclusion_keys = [
+  'metric',
+  'type',
+  'slug',
+  'fullscreen',
+  'edit',
+  'panelId',
+  'from',
+  'to'
+];
+
+// Intialize a skeleton with nothing but a rows array and service object
+dashboard = {
+  rows : [],
+};
+
+// Set a title
+dashboard.title = 'Alarm drilldown';
+dashboard.time = {
+  from: "now-" + (ARGS.from || timespan),
+  to: "now"
+};
+
+var metricName = 'metricname';
+
+if(!_.isUndefined(ARGS.metric)) {
+  metricName = ARGS.metric;
+}
+
+// Set dimensions
+var dimensions = [];
+for (var key in ARGS) {
+  if (exclusion_keys.indexOf(key) == -1) {
+    dimensions.push({'key': key, 'value': ARGS[key]});
+  }
+}
+
+dashboard.rows.push({
+  title: 'Chart',
+  height: '300px',
+  panels: [
+    {
+      title: metricName,
+      type: 'graph',
+      span: 12,
+      fill: 1,
+      linewidth: 2,
+      targets: [
+        {
+          "metric": metricName,
+          "aggregator": "avg",
+          "period": 300,
+          "dimensions": dimensions
+        }
+      ]
+    }
+  ]
+});
+
+return dashboard;

grafana/grafana.ini → grafana/grafana.ini.j2

@@ -116,10 +116,10 @@ check_for_updates = true
 #################################### Security ####################################
 [security]
 # default admin user, created on startup
-;admin_user = admin
+admin_user = {{ GRAFANA_ADMIN_USER }}
 
 # default admin password, can be changed before first start of grafana,  or in profile settings
-;admin_password = admin
+admin_password = {{ GRAFANA_ADMIN_PASSWORD }}
 
 # used for signing
 ;secret_key = SW2YcwTIb9zpOOhoPsMm

grafana/start.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+python /template.py /etc/grafana/grafana.ini.j2 /etc/grafana/grafana.ini
+/go/bin/grafana-server -config /etc/grafana/grafana.ini -homepath /var/lib/grafana

grafana/template.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+# (C) Copyright 2017 Hewlett Packard Enterprise Development LP
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import print_function
+
+import os
+import sys
+
+from jinja2 import Template
+
+
+def main():
+    if len(sys.argv) != 3:
+        print('Usage: {} [input] [output]'.format(sys.argv[0]))
+        sys.exit(1)
+
+    in_path = sys.argv[1]
+    out_path = sys.argv[2]
+
+    with open(in_path, 'r') as in_file, open(out_path, 'w') as out_file:
+        t = Template(in_file.read())
+        out_file.write(t.render(os.environ))
+
+if __name__ == '__main__':
+    main()

kibana/Dockerfile

@@ -25,7 +25,9 @@ RUN git clone $KIBANA_PLUGIN_REPO --depth 1 --branch $KIBANA_PLUGIN_BRANCH monas
 FROM kibana:${KIBANA_VERSION}
 
 ENV KEYSTONE_URI=keystone:5000 \
-    MONASCA_PLUGIN_ENABLED=False
+    MONASCA_PLUGIN_ENABLED=False \
+    ELASTIC_SEARCH_URL=elasticsearch:9200 \
+    BASE_PATH=""
 
 WORKDIR /
 

kibana/README.md

@@ -24,6 +24,8 @@ Configuration
 |--------------------------------|-----------------|-----------------------------------|
 | `KEYSTONE_URI`                 | `keystone:5000` | An URI to Keystone Admin Endpoint |
 | `MONASCA_PLUGIN_ENABLED`       | `False`         | Should the plugin be enabled or disabled |
+| `ELASTIC_SEARCH_URL`           | `elasticsearch:9200` | An URL to Elasticsearch container |
+| `BASE_PATH`                    | `unset`              | Path to mount Kibana at if you are running behind a proxy |
 
 Usage
 -----

kibana/kibana.yml.j2

@@ -1,3 +1,5 @@
 monasca-kibana-plugin.auth_uri: http://{{ KEYSTONE_URI }}
 monasca-kibana-plugin.enabled: {{ MONASCA_PLUGIN_ENABLED }}
 monasca-kibana-plugin.cookie.isSecure: False
+elasticsearch.url: http://{{ ELASTIC_SEARCH_URL }}
+server.basePath: "{{ BASE_PATH }}"

monasca-aggregator/build.yml

@@ -1,5 +1,5 @@
 repository: monasca/aggregator
 variants:
-  - tag: 0.1.1
+  - tag: 0.1.2
     aliases:
       - :latest

monasca-python/build.sh

@@ -24,21 +24,21 @@ else
 
         if [ "z$extra_deps" != "z" ]; then
             for extra in $extra_deps; do
-                pip install --no-cache-dir $extra -c $constraints
+                pip install --no-cache-dir "$extra" -c "$constraints"
             done
         else
             echo "No extra dependencies"
         fi
 
         if [ "z$extras" != "z" ]; then
             for extra in $extras; do
-                pip install --no-cache-dir .[${extra}] -c $constraints
+                pip install --no-cache-dir .["${extra}"] -c "$constraints"
             done
         else
             echo "No extras"
         fi
 
-        pip install --no-cache-dir -r requirements.txt -c $constraints
+        pip install --no-cache-dir -r requirements.txt -c "$constraints"
         python setup.py install
     }
 fi
@@ -101,7 +101,7 @@ EXTRA_DEPS="${EXTRA_DEPS:-""}"
 
 echo "Installing APK Dependencies" && install_apk_deps
 mkdir -p /app
-cd /app
+cd /app || exit
 
 # actual build happens here
 cat << EOF
@@ -115,7 +115,7 @@ echo "Cloning ${REPO}@${BRANCH}" && clone "${REPO}" "${BRANCH}"
 echo "Installing ${REPO}" && install "${CONSTRAINTS}" "${EXTRAS}" "${EXTRA_DEPS}"
 # end of actual build
 
-cd -
+cd - || exit
 rm -rf /install.sh /clone.sh /install_apk_deps.sh /app /root/.cache/pip
 
 apk del build-dep

smoke-tests/start.sh

@@ -10,6 +10,7 @@ export OS_DOMAIN_NAME=${OS_DOMAIN_NAME:-"Default"}
 export OS_AUTH_URL=${OS_AUTH_URL:-"http://keystone:35357/v3"}
 export MONASCA_URL=${MONASCA_URL:-"http://monasca-api:8070"}
 
-export WEBHOOK_IP=$(ip route get 8.8.8.8 | awk 'NR==1 {print $NF}')
+export WEBHOOK_IP
+WEBHOOK_IP=$(ip route get 8.8.8.8 | awk 'NR==1 {print $NF}')
 
 /smoke-test

tempest-tests/start.sh

@@ -36,7 +36,7 @@ if [ "$MONASCA_WAIT_FOR_API" = "true" ]; then
   echo "Waiting for Monasca API to become available..."
   success="false"
 
-  for i in $(seq $MONASCA_API_WAIT_RETRIES); do
+  for i in $(seq "$MONASCA_API_WAIT_RETRIES"); do
     monasca --os-user-domain-name "${OS_DOMAIN_NAME}" --os-project-name "${OS_TENANT_NAME}" \
        --os-auth-url "${AUTH_URI_V3}" --os-username "${OS_USERNAME}" \
        --os-password "${OS_PASSWORD}" alarm-list --limit 1
@@ -60,7 +60,7 @@ python template.py \
   /etc/tempest/tempest.conf.j2 \
   /etc/tempest/tempest.conf
 
-cd /monasca-api
+cd /monasca-api || exit
 export OS_TEST_PATH=./monasca_tempest_tests/tests/api
 
 if [ ! -r .testrepository ]; then