separate config for power-users

docker_minimum/README.md

@@ -0,0 +1,62 @@
+# docker minimum readme
+
+This directory provides provided separate docker-compose files to start the minimum required wis2box-services without relying on the full wis2box-stack as managed by wis2box-ctl.py.
+
+It is intended to allow "power-users" to more easily re-use services and configuration in a customized wis2box environment.
+
+To start the minimum required services, run the following command:
+
+```bash
+
+docker compose -f docker-compose.minio.yml -f docker-compose.wis2box-management.yml -f docker-compose.mosquitto.yml -f docker-compose.elasticsearch.yml -f docker-compose.wis2box-api.yml -p wis2box-minimum up -d
+
+```
+
+This starts the stack and defines the project name as `wis2box-minimum`.
+
+You can see the environment-variables used by each wis2box-service in the 'environment' section of the docker-compose files.
+
+### wis2box-management
+
+This service uses an Ubuntu base image and installs the wis2box python classes defined in the wis2box-management directory of this repository. Within the container a set of commands are available to manage the wis2box. The container is listening to the internal message broker for messages on the topic wis2box/#. The storage service should be configured to send messages to this topic when objects are added to the storage to allow the wis2box-management to trigger the workflow defined by the data-mappings configuration for each dataset.
+
+Note that the command that keeps the container running is:
+
+```bash
+wis2box pubsub subscribe
+```
+
+The wis2box pubsub class defined in wis2box-management/wis2box/pubsub/ determines what actions are taken when a message is received on the internal message broker. 
+
+### mosquitto
+
+Mosquitto-service that provides the internal message broker for the WIS2Box.
+
+Note that in the standard wis2box-stack the internal broker is also used as the external broker.
+
+The wis2box-broker image is defined by the Dockerfile in the wis2box-broker directory of this repository.
+
+### MinIO
+
+MinIO is used to provide an http-compatible storage endpoint to serve the data and metadata published by wis2box. It is an S3-compatible object storage server. 
+
+Notifications are sent to the internal message broker when objects are added to the storage to trigger the workflow defined by the data-mappings configuration for each dataset.
+
+### wis2box-api
+
+This is a pygeoapi instance that provides access to the WIS2Box data and metadata. For the source-code see github repository: https://github.com/wmo-im/wis2box-api
+
+The backend for the wis2box-api is provided by elasticsearch.
+
+Note that wis2box-api service includes the eccodes library to allow the following conversion to bufr.
+
+Note that wis2box-api provides access to the collections 'stations' and 'discovery-metadata':
+- 'stations' is a collection of station metadata used when processing data using csv2bufr, synop2bufr and bufr2bufr
+- 'discovery-metadata' is a collection of datasets configured in the wis2box, with each dataset containing metadata that describes the dataset and the data-mappings that define the workflow to be triggered when data is added to the storage.
+
+
+### elasticsearch
+
+An elasticsearch instance that provides the backend for the wis2box-api.
+
+

docker_minimum/docker-compose.elasticsearch.yml

@@ -0,0 +1,23 @@
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.6.2
+    restart: always
+    environment:
+      - discovery.type=single-node
+      - discovery.seed_hosts=[]
+      - node.name=elasticsearch-01
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - cluster.name=es-wis2box
+      - xpack.security.enabled=false
+    mem_limit: 1.5g
+    memswap_limit: 1.5g
+    volumes:
+      - elasticsearch-data:/usr/share/elasticsearch/data:rw
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9200"]
+      interval: 5s
+      retries: 100
+
+volumes:
+  elasticsearch-data:

docker_minimum/docker-compose.minio.yml

@@ -0,0 +1,30 @@
+services:
+  minio:
+    image: minio/minio:RELEASE.2024-05-10T01-41-38Z
+    mem_limit: 512m
+    memswap_limit: 512m
+    restart: always
+    environment:
+      - MINIO_ROOT_USER=minio-admin
+      - MINIO_ROOT_PASSWORD=minio-password
+      - MINIO_PROMETHEUS_AUTH_TYPE=public
+      - MINIO_NOTIFY_MQTT_ENABLE_WIS2BOX=on
+      - MINIO_NOTIFY_MQTT_USERNAME_WIS2BOX=localbroker-admin
+      - MINIO_NOTIFY_MQTT_PASSWORD_WIS2BOX=localbroker-password
+      - MINIO_NOTIFY_MQTT_BROKER_WIS2BOX=tcp://mosquitto:1883
+      - MINIO_NOTIFY_MQTT_TOPIC_WIS2BOX=wis2box/storage
+      - MINIO_NOTIFY_MQTT_QOS_WIS2BOX=1
+    command: server --console-address ":9001" /data
+    volumes:
+      - minio-data:/data
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 5s
+      timeout: 1s
+      retries: 3
+    depends_on:
+      mosquitto:
+        condition: service_started
+
+volumes:
+  minio-data:

docker_minimum/docker-compose.mosquitto.yml

@@ -0,0 +1,11 @@
+services:
+  mosquitto:
+    build: 
+      context: ../wis2box-broker
+    restart: always
+    environment:
+      - WIS2BOX_BROKER_USERNAME=localbroker-admin
+      - WIS2BOX_BROKER_PASSWORD=localbroker-password
+      - WIS2BOX_BROKER_QUEUE_MAX=1000
+    ports:
+      - 7883:1883

docker_minimum/docker-compose.wis2box-api.yml

@@ -0,0 +1,27 @@
+services: 
+  wis2box-api:
+    image: ghcr.io/wmo-im/wis2box-api:latest
+    restart: always
+    environment:
+      - WIS2BOX_API_URL=http://localhost/oapi
+      - WIS2BOX_API_BACKEND_URL=http://elasticsearch:9200
+      - WIS2BOX_BASEMAP_URL=https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png
+      - WIS2BOX_BASEMAP_ATTRIBUTION="<a href="https://osm.org/copyright">OpenStreetMap</a> contributors"
+      - WIS2BOX_BROKER_USERNAME=localbroker-admin
+      - WIS2BOX_BROKER_PASSWORD=localbroker-password
+      - WIS2BOX_BROKER_HOST=mosquitto
+      - WIS2BOX_BROKER_PORT=1883
+      - WIS2BOX_LOGGING_LOGLEVEL=INFO
+      - WIS2BOX_DATADIR=/data/wis2box
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+    volumes:
+      - api-config:/data/wis2box/config/pygeoapi/:rw
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost/oapi/admin/resources"]
+      interval: 5s
+      retries: 100
+
+volumes:
+  api-config:

docker_minimum/docker-compose.wis2box-management.yml

@@ -0,0 +1,33 @@
+services:
+  wis2box-management:
+    mem_limit: 1g
+    memswap_limit: 1g
+    restart: always
+    build:
+      context: ../wis2box-management
+    environment:
+      - WIS2BOX_URL=http://www.mywis2data.com
+      - WIS2BOX_DOCKER_API_URL=http://wis2box-api:80/oapi
+      - WIS2BOX_API_BACKEND_URL=http://elasticsearch:9200
+      - MINIO_NOTIFY_MQTT_USERNAME_WIS2BOX=localbroker-admin
+      - MINIO_NOTIFY_MQTT_PASSWORD_WIS2BOX=localbroker-password
+      - WIS2BOX_BROKER_HOST=mosquitto
+      - WIS2BOX_BROKER_PORT=1883
+      - WIS2BOX_BROKER_USERNAME=localbroker-admin
+      - WIS2BOX_BROKER_PASSWORD=localbroker-password
+      - WIS2BOX_STORAGE_SOURCE=http://minio:9000
+      - WIS2BOX_STORAGE_USERNAME=minio-admin
+      - WIS2BOX_STORAGE_PASSWORD=minio-password
+      - WIS2BOX_BROKER_PUBLIC=mqtt://localbroker-admin:localbroker-password@mosquitto:1883
+      - WIS2BOX_DATADIR=/data/wis2box
+      - WIS2BOX_LOGGING_LOGLEVEL=INFO
+    volumes:
+      - ../tests/data:/data/wis2box:rw
+    depends_on:
+      minio:
+        condition: service_healthy
+      mosquitto:
+        condition: service_started
+      wis2box-api:
+        condition: service_healthy
+    command: ["wis2box", "pubsub" , "subscribe"]

wis2box-management/docker/entrypoint.sh

@@ -73,6 +73,17 @@ else
     htpasswd -bc /home/wis2box/.htpasswd/webapp $WIS2BOX_WEBAPP_USERNAME $WIS2BOX_WEBAPP_PASSWORD || true
 fi
 
+# check if WIS2BOX_AUTH_URL is set, if not skip running the remaining commands
+auth_url=${WIS2BOX_AUTH_URL}
+if [ -z "$auth_url" ]; then
+    echo "WIS2BOX_AUTH_URL is not set, skipping authentication setup"
+    echo "END /entrypoint.sh"
+    exec "$@"
+else
+    echo "WIS2BOX_AUTH_URL is set to $auth_url"
+    echo "Proceed to restrict processes/wis2box and collections/stations"
+fi
+
 # Check if the path is restricted and capture the output
 is_restricted=$(wis2box auth is-restricted-path --path processes/wis2box)
 if [ "$is_restricted" = "True" ]; then