Issue 674 new (#729)

* rebase and add elasticsearch monitoring dashboard and alert

* update dashboard panels

* fix errors

* double check about the missing point

* remove alerts from downloader dashboard

* update elasticsearch alerts

---------

Co-authored-by: PEIWEN JIN <[email protected]>
Co-authored-by: Maaike <[email protected]>

docker-compose.dev.yml

@@ -5,15 +5,27 @@ services:
       - ./wis2box-management/wis2box/wis2box.cron:/etc/cron.d/wis2box:ro
       - ./wis2box-management/wis2box:/app/wis2box
     command: ["wis2box", "pubsub" , "subscribe"]
+    depends_on:
+      - minio
+      - mosquitto
+      - wis2box-api
 
   wis2box-api:
     volumes:
       - ../wis2box-api/wis2box_api:/app/wis2box_api
+    depends_on:
+      - elasticsearch
 
   wis2box-auth:
     volumes:
       - ../wis2box-auth/wis2box_auth:/app/wis2box_auth
+    depends_on:
+      - wis2box-management
 
   elasticsearch:
     ports:
       - 9200:9200
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9200"]
+      interval: 5s
+      retries: 100

docker-compose.monitoring.yml

@@ -25,14 +25,12 @@ services:
     <<: *logging
     container_name: loki
     image: grafana/loki:2.4.1
+    restart: always
     command: -config.file=/etc/loki/loki-config.yml
     volumes:
       - loki-data:/loki
       - ./loki/loki-config.yml:/etc/loki/loki-config.yml
-    networks:
-      vpcbr: # this is the place where we assign the static ipv4 address
-        ipv4_address: 10.5.0.2
-      default:
+
   # mqtt_metrics_collector, listens to mqtt-broker
   mqtt_metrics_collector:
     <<: *logging
@@ -54,12 +52,14 @@ services:
     <<: *logging
     image: prom/prometheus:v2.37.0
     container_name: prometheus
+    restart: always
     volumes:
       - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
       - prometheus-data:/prometheus
     command:
       - '--config.file=/etc/prometheus/prometheus.yml'
       - '--storage.tsdb.retention.time=10d'
+
   # uncomment cadvisor to monitor containers
   # cadvisor:
   #  image: gcr.io/cadvisor/cadvisor:v0.45.0
@@ -77,12 +77,14 @@ services:
   grafana:
     <<: *logging
     container_name: grafana
+    restart: always
     env_file:
       - wis2box.env
     image: grafana/grafana-oss:9.0.3
     volumes:
       - ./grafana/dashboards:/etc/grafana/provisioning/dashboards
       - ./grafana/datasources:/etc/grafana/provisioning/datasources
+  # if alert want to link emails this place should mount local cofiguration to grafana
     environment:
       - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER:-admin}
       - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-admin}
@@ -100,14 +102,23 @@ services:
       - GF_SERVER_SERVE_FROM_SUB_PATH=true
     ports:
       - 3000:3000
+
+  # Elasticsearch Exporter for monitoring Elasticsearch metrics
+  elasticsearch-exporter:
+    <<: *logging
+    image: quay.io/prometheuscommunity/elasticsearch-exporter:latest
+    command:
+      - '--es.uri=http://elasticsearch:9200'
+      - '--es.indices_settings'
+    restart: always
+  elasticsearch:
+    <<: *logging
   wis2box-management:
     <<: *logging
   mosquitto:
     <<: *logging
   wis2box-api:
     <<: *logging
-  elasticsearch:
-    <<: *logging
   wis2box-auth:
     <<: *logging
   minio:
@@ -120,3 +131,4 @@ services:
 volumes:
   loki-data:
   prometheus-data:
+

docker-compose.yml

@@ -34,6 +34,7 @@ services:
     image: ghcr.io/wmo-im/wis2box-webapp:latest
     env_file:
       - wis2box.env
+    restart: always
 
   wis2box-api:
     container_name: wis2box-api
@@ -145,6 +146,7 @@ services:
 
   wis2downloader:
     container_name: wis2downloader
+    restart: always
     build: ./wis2downloader
     env_file:
       - wis2box.env
@@ -155,4 +157,4 @@ volumes:
   es-data:
   minio-data:
   auth-data:
-  htpasswd:
+  htpasswd: