From d4393b7c094bd4461995dbc5fa8f9163e05b8445 Mon Sep 17 00:00:00 2001 From: Maaike Date: Fri, 15 Mar 2024 09:39:23 +0100 Subject: [PATCH 1/3] initial version of separate config --- docker_compose/README.md | 45 +++++++++++++++++++ .../docker-compose.elasticsearch.yml | 21 +++++++++ docker_compose/docker-compose.web-proxy.yml | 9 ++++ docker_compose/docker-compose.wis2box-api.yml | 28 ++++++++++++ .../docker-compose.wis2box-auth.yml | 11 +++++ .../docker-compose.wis2box-broker.yml | 12 +++++ .../docker-compose.wis2box-management.yml | 31 +++++++++++++ .../docker-compose.wis2box-minio.yml | 28 ++++++++++++ docker_compose/docker-compose.wis2box-ui.yml | 9 ++++ .../docker-compose.wis2box-webapp.yml | 9 ++++ 10 files changed, 203 insertions(+) create mode 100644 docker_compose/README.md create mode 100644 docker_compose/docker-compose.elasticsearch.yml create mode 100644 docker_compose/docker-compose.web-proxy.yml create mode 100644 docker_compose/docker-compose.wis2box-api.yml create mode 100644 docker_compose/docker-compose.wis2box-auth.yml create mode 100644 docker_compose/docker-compose.wis2box-broker.yml create mode 100644 docker_compose/docker-compose.wis2box-management.yml create mode 100644 docker_compose/docker-compose.wis2box-minio.yml create mode 100644 docker_compose/docker-compose.wis2box-ui.yml create mode 100644 docker_compose/docker-compose.wis2box-webapp.yml diff --git a/docker_compose/README.md b/docker_compose/README.md new file mode 100644 index 00000000..e64e27b1 --- /dev/null +++ b/docker_compose/README.md @@ -0,0 +1,45 @@ +# docker compose readme + +## Docker Compose + +This directory contains separate docker-compose files for each of the services that are part of the project. + +The directory 'core' contains the main docker-compose file that enable the publication of WIS2 data and metadata notifications. + +The directory 'frontend' contains files for the web-based frontend services. + +## Core + +The following services are part of the core wis2box-services: + +### wis2box-management + +This is a set of python modules that provide the core functionality of the WIS2Box. The command that keeps the container running is: + +```bash +wis2box pubsub subscribe +``` + +This command subscribes to the internal WIS2Box message broker on the topic wis2box/#. + +By default the internal WIS2Box message broker also serve as the public-facing broker by setting = + +```bash +WIS2BOX_BROKER_PUBLIC=mqtt://${WIS2BOX_BROKER_USERNAME}:${WIS2BOX_BROKER_PASSWORD}@mosquitto:1883 +``` + +### wis2box-broker + +Mosquitto-service that provides the (internal) message broker for the WIS2Box. + +### wis2box-api + +This is a pygeoapi instance that provides access to the WIS2Box data and metadata. For the source-code see github repository: https://github.com/wmo-im/wis2box-api + +The default backend for the wis2box-api is provided by an elasticsearch container on the same (docker) network. + +### wis2box-elasticsearch + +An elasticsearch instance that provides the backend for the wis2box-api. + + diff --git a/docker_compose/docker-compose.elasticsearch.yml b/docker_compose/docker-compose.elasticsearch.yml new file mode 100644 index 00000000..275a394a --- /dev/null +++ b/docker_compose/docker-compose.elasticsearch.yml @@ -0,0 +1,21 @@ +services: + elasticsearch: + container_name: elasticsearch + image: docker.elastic.co/elasticsearch/elasticsearch:8.6.2 + restart: always + environment: + - discovery.type=single-node + - discovery.seed_hosts=[] + - node.name=elasticsearch-01 + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - cluster.name=es-wis2box + - xpack.security.enabled=false + mem_limit: 1.5g + memswap_limit: 1.5g + volumes: + - ${ELASTICSEARH_DATA}:/usr/share/elasticsearch/data:rw + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9200"] + interval: 5s + retries: 100 \ No newline at end of file diff --git a/docker_compose/docker-compose.web-proxy.yml b/docker_compose/docker-compose.web-proxy.yml new file mode 100644 index 00000000..a3337b72 --- /dev/null +++ b/docker_compose/docker-compose.web-proxy.yml @@ -0,0 +1,9 @@ +services: + web-proxy: + container_name: nginx + image: nginx:alpine + restart: always + volumes: + - ${NGINX_CONF}:/etc/nginx/conf.d/default.conf + ports: + - 80:80 \ No newline at end of file diff --git a/docker_compose/docker-compose.wis2box-api.yml b/docker_compose/docker-compose.wis2box-api.yml new file mode 100644 index 00000000..fbb4239a --- /dev/null +++ b/docker_compose/docker-compose.wis2box-api.yml @@ -0,0 +1,28 @@ +services: + wis2box-api: + container_name: wis2box-api + image: ghcr.io/wmo-im/wis2box-api:latest + restart: always + environment: + - WIS2BOX_API_URL=${WIS2BOX_API_URL} + - WIS2BOX_URL=${WIS2BOX_URL} + - WIS2BOX_API_BACKEND_URL=$(WIS2BOX_API_BACKEND_URL) + - WIS2BOX_BASEMAP_URL=${WIS2BOX_BASEMAP_URL} + - WIS2BOX_BASEMAP_ATTRIBUTION=${WIS2BOX_BASEMAP_ATTRIBUTION} + - WIS2BOX_BROKER_USERNAME=${WIS2BOX_BROKER_USERNAME} + - WIS2BOX_BROKER_PASSWORD=${WIS2BOX_BROKER_PASSWORD} + - WIS2BOX_BROKER_HOST=${WIS2BOX_BROKER_HOST} + - WIS2BOX_BROKER_PORT=${WIS2BOX_BROKER_PORT} + - WIS2BOX_STORAGE_SOURCE=${WIS2BOX_STORAGE_SOURCE} + depends_on: + elasticsearch: + condition: service_healthy + volumes: + - api-config:/data/wis2box/config/pygeoapi/:rw + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost/oapi/admin/resources"] + interval: 5s + retries: 100 + +volumes: + api-config: \ No newline at end of file diff --git a/docker_compose/docker-compose.wis2box-auth.yml b/docker_compose/docker-compose.wis2box-auth.yml new file mode 100644 index 00000000..600db034 --- /dev/null +++ b/docker_compose/docker-compose.wis2box-auth.yml @@ -0,0 +1,11 @@ +services: + wis2box-auth: + container_name: wis2box-auth + image: ghcr.io/wmo-im/wis2box-auth:latest + restart: always + env_file: + - wis2box.env + volumes: + - ${AUTH_DATA}:/data/wis2box:rw + depends_on: + - wis2box-management \ No newline at end of file diff --git a/docker_compose/docker-compose.wis2box-broker.yml b/docker_compose/docker-compose.wis2box-broker.yml new file mode 100644 index 00000000..71d5fc74 --- /dev/null +++ b/docker_compose/docker-compose.wis2box-broker.yml @@ -0,0 +1,12 @@ +services: + mosquitto: + image: ghcr.io/wmo-im/wis2box-broker:latest + restart: always + build: + context: ./wis2box-broker + environment: + - WIS2BOX_BROKER_USERNAME=${WIS2BOX_BROKER_USERNAME} + - WIS2BOX_BROKER_PASSWORD=${WIS2BOX_BROKER_PASSWORD} + + ports: + - 1883:1883 \ No newline at end of file diff --git a/docker_compose/docker-compose.wis2box-management.yml b/docker_compose/docker-compose.wis2box-management.yml new file mode 100644 index 00000000..b414b0d3 --- /dev/null +++ b/docker_compose/docker-compose.wis2box-management.yml @@ -0,0 +1,31 @@ +services: + wis2box-management: + container_name: wis2box-management + mem_limit: 1g + memswap_limit: 1g + restart: always + #image: ghcr.io/wmo-im/wis2box-management:1.0.beta1 + build: + context: ./wis2box-management + environment: + - WIS2BOX_API_URL=${WIS2BOX_API_URL} + - WIS2BOX_URL=${WIS2BOX_URL} + - WIS2BOX_API_BACKEND_URL=$(WIS2BOX_API_BACKEND_URL) + - WIS2BOX_BASEMAP_URL=${WIS2BOX_BASEMAP_URL} + - WIS2BOX_BASEMAP_ATTRIBUTION=${WIS2BOX_BASEMAP_ATTRIBUTION} + - WIS2BOX_BROKER_USERNAME=${WIS2BOX_BROKER_USERNAME} + - WIS2BOX_BROKER_PASSWORD=${WIS2BOX_BROKER_PASSWORD} + - WIS2BOX_BROKER_HOST=${WIS2BOX_BROKER_HOST} + - WIS2BOX_BROKER_PORT=${WIS2BOX_BROKER_PORT} + - WIS2BOX_STORAGE_SOURCE=${WIS2BOX_STORAGE_SOURCE} + + volumes: + - ${WIS2BOX_HOST_DATADIR}:/data/wis2box:rw + depends_on: + minio: + condition: service_healthy + mosquitto: + condition: service_started + wis2box-api: + condition: service_healthy + command: ["wis2box", "pubsub" , "subscribe"] \ No newline at end of file diff --git a/docker_compose/docker-compose.wis2box-minio.yml b/docker_compose/docker-compose.wis2box-minio.yml new file mode 100644 index 00000000..392c2ed7 --- /dev/null +++ b/docker_compose/docker-compose.wis2box-minio.yml @@ -0,0 +1,28 @@ +services: + minio: + container_name: wis2box-minio + image: minio/minio:RELEASE.2022-12-02T19-19-22Z.fips + mem_limit: 512m + memswap_limit: 512m + restart: always + environment: + - MINIO_ROOT_USER=${WIS2BOX_STORAGE_USERNAME} + - MINIO_ROOT_PASSWORD=${WIS2BOX_STORAGE_PASSWORD} + - MINIO_PROMETHEUS_AUTH_TYPE=public + - MINIO_NOTIFY_MQTT_ENABLE_WIS2BOX=on + - MINIO_NOTIFY_MQTT_USERNAME_WIS2BOX=${WIS2BOX_BROKER_USERNAME} + - MINIO_NOTIFY_MQTT_PASSWORD_WIS2BOX=${WIS2BOX_BROKER_PASSWORD} + - MINIO_NOTIFY_MQTT_BROKER_WIS2BOX=tcp://${WIS2BOX_BROKER_HOST}:${WIS2BOX_BROKER_PORT} + - MINIO_NOTIFY_MQTT_TOPIC_WIS2BOX=wis2box/storage + - MINIO_NOTIFY_MQTT_QOS_WIS2BOX=1 + command: server --console-address ":9001" /data + volumes: + - ${MINIO_DATA}:/data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 5s + timeout: 1s + retries: 3 + depends_on: + mosquitto: + condition: service_started \ No newline at end of file diff --git a/docker_compose/docker-compose.wis2box-ui.yml b/docker_compose/docker-compose.wis2box-ui.yml new file mode 100644 index 00000000..adbeb5f8 --- /dev/null +++ b/docker_compose/docker-compose.wis2box-ui.yml @@ -0,0 +1,9 @@ +services: + wis2box-ui: + container_name: wis2box-ui + image: ghcr.io/wmo-im/wis2box-ui:latest + environment: + - WIS2BOX_API_URL=${WIS2BOX_API_URL} + - WIS2BOX_URL=${WIS2BOX_URL} + - WIS2BOX_BASEMAP_URL=${WIS2BOX_BASEMAP_URL} + - WIS2BOX_BASEMAP_ATTRIBUTION=${WIS2BOX_BASEMAP_ATTRIBUTION} \ No newline at end of file diff --git a/docker_compose/docker-compose.wis2box-webapp.yml b/docker_compose/docker-compose.wis2box-webapp.yml new file mode 100644 index 00000000..9540cc8d --- /dev/null +++ b/docker_compose/docker-compose.wis2box-webapp.yml @@ -0,0 +1,9 @@ +services: + wis2box-webapp: + container_name: wis2box-webapp + image: ghcr.io/wmo-im/wis2box-webapp:latest + environment: + - WIS2BOX_API_URL=${WIS2BOX_API_URL} + - WIS2BOX_URL=${WIS2BOX_URL} + - WIS2BOX_BASEMAP_URL=${WIS2BOX_BASEMAP_URL} + - WIS2BOX_BASEMAP_ATTRIBUTION=${WIS2BOX_BASEMAP_ATTRIBUTION} \ No newline at end of file From 16e9d3f08448e999593c91e66e8ee6953a560aff Mon Sep 17 00:00:00 2001 From: Maaike Date: Tue, 14 May 2024 15:19:48 +0200 Subject: [PATCH 2/3] separate docker-services for power-users --- docker-compose.override.yml | 2 +- docker_compose/README.md | 45 -------------- docker_compose/docker-compose.web-proxy.yml | 9 --- docker_compose/docker-compose.wis2box-api.yml | 28 --------- .../docker-compose.wis2box-auth.yml | 11 ---- .../docker-compose.wis2box-broker.yml | 12 ---- .../docker-compose.wis2box-management.yml | 31 ---------- docker_compose/docker-compose.wis2box-ui.yml | 9 --- .../docker-compose.wis2box-webapp.yml | 9 --- docker_minimum/README.md | 62 +++++++++++++++++++ .../docker-compose.elasticsearch.yml | 8 ++- .../docker-compose.minio.yml | 20 +++--- docker_minimum/docker-compose.mosquitto.yml | 11 ++++ docker_minimum/docker-compose.wis2box-api.yml | 27 ++++++++ .../docker-compose.wis2box-management.yml | 33 ++++++++++ wis2box-management/docker/entrypoint.sh | 11 ++++ 16 files changed, 161 insertions(+), 167 deletions(-) delete mode 100644 docker_compose/README.md delete mode 100644 docker_compose/docker-compose.web-proxy.yml delete mode 100644 docker_compose/docker-compose.wis2box-api.yml delete mode 100644 docker_compose/docker-compose.wis2box-auth.yml delete mode 100644 docker_compose/docker-compose.wis2box-broker.yml delete mode 100644 docker_compose/docker-compose.wis2box-management.yml delete mode 100644 docker_compose/docker-compose.wis2box-ui.yml delete mode 100644 docker_compose/docker-compose.wis2box-webapp.yml create mode 100644 docker_minimum/README.md rename {docker_compose => docker_minimum}/docker-compose.elasticsearch.yml (82%) rename docker_compose/docker-compose.wis2box-minio.yml => docker_minimum/docker-compose.minio.yml (51%) create mode 100644 docker_minimum/docker-compose.mosquitto.yml create mode 100644 docker_minimum/docker-compose.wis2box-api.yml create mode 100644 docker_minimum/docker-compose.wis2box-management.yml diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 292c5ea7..415bdc13 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -1,7 +1,7 @@ services: web-proxy: ports: - - 80:80 + - 4480:80 wis2box-ui: ports: diff --git a/docker_compose/README.md b/docker_compose/README.md deleted file mode 100644 index e64e27b1..00000000 --- a/docker_compose/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# docker compose readme - -## Docker Compose - -This directory contains separate docker-compose files for each of the services that are part of the project. - -The directory 'core' contains the main docker-compose file that enable the publication of WIS2 data and metadata notifications. - -The directory 'frontend' contains files for the web-based frontend services. - -## Core - -The following services are part of the core wis2box-services: - -### wis2box-management - -This is a set of python modules that provide the core functionality of the WIS2Box. The command that keeps the container running is: - -```bash -wis2box pubsub subscribe -``` - -This command subscribes to the internal WIS2Box message broker on the topic wis2box/#. - -By default the internal WIS2Box message broker also serve as the public-facing broker by setting = - -```bash -WIS2BOX_BROKER_PUBLIC=mqtt://${WIS2BOX_BROKER_USERNAME}:${WIS2BOX_BROKER_PASSWORD}@mosquitto:1883 -``` - -### wis2box-broker - -Mosquitto-service that provides the (internal) message broker for the WIS2Box. - -### wis2box-api - -This is a pygeoapi instance that provides access to the WIS2Box data and metadata. For the source-code see github repository: https://github.com/wmo-im/wis2box-api - -The default backend for the wis2box-api is provided by an elasticsearch container on the same (docker) network. - -### wis2box-elasticsearch - -An elasticsearch instance that provides the backend for the wis2box-api. - - diff --git a/docker_compose/docker-compose.web-proxy.yml b/docker_compose/docker-compose.web-proxy.yml deleted file mode 100644 index a3337b72..00000000 --- a/docker_compose/docker-compose.web-proxy.yml +++ /dev/null @@ -1,9 +0,0 @@ -services: - web-proxy: - container_name: nginx - image: nginx:alpine - restart: always - volumes: - - ${NGINX_CONF}:/etc/nginx/conf.d/default.conf - ports: - - 80:80 \ No newline at end of file diff --git a/docker_compose/docker-compose.wis2box-api.yml b/docker_compose/docker-compose.wis2box-api.yml deleted file mode 100644 index fbb4239a..00000000 --- a/docker_compose/docker-compose.wis2box-api.yml +++ /dev/null @@ -1,28 +0,0 @@ -services: - wis2box-api: - container_name: wis2box-api - image: ghcr.io/wmo-im/wis2box-api:latest - restart: always - environment: - - WIS2BOX_API_URL=${WIS2BOX_API_URL} - - WIS2BOX_URL=${WIS2BOX_URL} - - WIS2BOX_API_BACKEND_URL=$(WIS2BOX_API_BACKEND_URL) - - WIS2BOX_BASEMAP_URL=${WIS2BOX_BASEMAP_URL} - - WIS2BOX_BASEMAP_ATTRIBUTION=${WIS2BOX_BASEMAP_ATTRIBUTION} - - WIS2BOX_BROKER_USERNAME=${WIS2BOX_BROKER_USERNAME} - - WIS2BOX_BROKER_PASSWORD=${WIS2BOX_BROKER_PASSWORD} - - WIS2BOX_BROKER_HOST=${WIS2BOX_BROKER_HOST} - - WIS2BOX_BROKER_PORT=${WIS2BOX_BROKER_PORT} - - WIS2BOX_STORAGE_SOURCE=${WIS2BOX_STORAGE_SOURCE} - depends_on: - elasticsearch: - condition: service_healthy - volumes: - - api-config:/data/wis2box/config/pygeoapi/:rw - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost/oapi/admin/resources"] - interval: 5s - retries: 100 - -volumes: - api-config: \ No newline at end of file diff --git a/docker_compose/docker-compose.wis2box-auth.yml b/docker_compose/docker-compose.wis2box-auth.yml deleted file mode 100644 index 600db034..00000000 --- a/docker_compose/docker-compose.wis2box-auth.yml +++ /dev/null @@ -1,11 +0,0 @@ -services: - wis2box-auth: - container_name: wis2box-auth - image: ghcr.io/wmo-im/wis2box-auth:latest - restart: always - env_file: - - wis2box.env - volumes: - - ${AUTH_DATA}:/data/wis2box:rw - depends_on: - - wis2box-management \ No newline at end of file diff --git a/docker_compose/docker-compose.wis2box-broker.yml b/docker_compose/docker-compose.wis2box-broker.yml deleted file mode 100644 index 71d5fc74..00000000 --- a/docker_compose/docker-compose.wis2box-broker.yml +++ /dev/null @@ -1,12 +0,0 @@ -services: - mosquitto: - image: ghcr.io/wmo-im/wis2box-broker:latest - restart: always - build: - context: ./wis2box-broker - environment: - - WIS2BOX_BROKER_USERNAME=${WIS2BOX_BROKER_USERNAME} - - WIS2BOX_BROKER_PASSWORD=${WIS2BOX_BROKER_PASSWORD} - - ports: - - 1883:1883 \ No newline at end of file diff --git a/docker_compose/docker-compose.wis2box-management.yml b/docker_compose/docker-compose.wis2box-management.yml deleted file mode 100644 index b414b0d3..00000000 --- a/docker_compose/docker-compose.wis2box-management.yml +++ /dev/null @@ -1,31 +0,0 @@ -services: - wis2box-management: - container_name: wis2box-management - mem_limit: 1g - memswap_limit: 1g - restart: always - #image: ghcr.io/wmo-im/wis2box-management:1.0.beta1 - build: - context: ./wis2box-management - environment: - - WIS2BOX_API_URL=${WIS2BOX_API_URL} - - WIS2BOX_URL=${WIS2BOX_URL} - - WIS2BOX_API_BACKEND_URL=$(WIS2BOX_API_BACKEND_URL) - - WIS2BOX_BASEMAP_URL=${WIS2BOX_BASEMAP_URL} - - WIS2BOX_BASEMAP_ATTRIBUTION=${WIS2BOX_BASEMAP_ATTRIBUTION} - - WIS2BOX_BROKER_USERNAME=${WIS2BOX_BROKER_USERNAME} - - WIS2BOX_BROKER_PASSWORD=${WIS2BOX_BROKER_PASSWORD} - - WIS2BOX_BROKER_HOST=${WIS2BOX_BROKER_HOST} - - WIS2BOX_BROKER_PORT=${WIS2BOX_BROKER_PORT} - - WIS2BOX_STORAGE_SOURCE=${WIS2BOX_STORAGE_SOURCE} - - volumes: - - ${WIS2BOX_HOST_DATADIR}:/data/wis2box:rw - depends_on: - minio: - condition: service_healthy - mosquitto: - condition: service_started - wis2box-api: - condition: service_healthy - command: ["wis2box", "pubsub" , "subscribe"] \ No newline at end of file diff --git a/docker_compose/docker-compose.wis2box-ui.yml b/docker_compose/docker-compose.wis2box-ui.yml deleted file mode 100644 index adbeb5f8..00000000 --- a/docker_compose/docker-compose.wis2box-ui.yml +++ /dev/null @@ -1,9 +0,0 @@ -services: - wis2box-ui: - container_name: wis2box-ui - image: ghcr.io/wmo-im/wis2box-ui:latest - environment: - - WIS2BOX_API_URL=${WIS2BOX_API_URL} - - WIS2BOX_URL=${WIS2BOX_URL} - - WIS2BOX_BASEMAP_URL=${WIS2BOX_BASEMAP_URL} - - WIS2BOX_BASEMAP_ATTRIBUTION=${WIS2BOX_BASEMAP_ATTRIBUTION} \ No newline at end of file diff --git a/docker_compose/docker-compose.wis2box-webapp.yml b/docker_compose/docker-compose.wis2box-webapp.yml deleted file mode 100644 index 9540cc8d..00000000 --- a/docker_compose/docker-compose.wis2box-webapp.yml +++ /dev/null @@ -1,9 +0,0 @@ -services: - wis2box-webapp: - container_name: wis2box-webapp - image: ghcr.io/wmo-im/wis2box-webapp:latest - environment: - - WIS2BOX_API_URL=${WIS2BOX_API_URL} - - WIS2BOX_URL=${WIS2BOX_URL} - - WIS2BOX_BASEMAP_URL=${WIS2BOX_BASEMAP_URL} - - WIS2BOX_BASEMAP_ATTRIBUTION=${WIS2BOX_BASEMAP_ATTRIBUTION} \ No newline at end of file diff --git a/docker_minimum/README.md b/docker_minimum/README.md new file mode 100644 index 00000000..7f2a0ad8 --- /dev/null +++ b/docker_minimum/README.md @@ -0,0 +1,62 @@ +# docker minimum readme + +This directory provides provided separate docker-compose files to start the minimum required wis2box-services without relying on the full wis2box-stack as managed by wis2box-ctl.py. + +It is intended to allow "power-users" to more easily re-use services and configuration in a customized wis2box environment. + +To start the minimum required services, run the following command: + +```bash + +docker compose -f docker-compose.minio.yml -f docker-compose.wis2box-management.yml -f docker-compose.mosquitto.yml -f docker-compose.elasticsearch.yml -f docker-compose.wis2box-api.yml -p wis2box-minimum up -d + +``` + +This starts the stack and defines the project name as `wis2box-minimum`. + +You can see the environment-variables used by each wis2box-service in the 'environment' section of the docker-compose files. + +### wis2box-management + +This service uses an Ubuntu base image and installs the wis2box python classes defined in the wis2box-management directory of this repository. Within the container a set of commands are available to manage the wis2box. The container is listening to the internal message broker for messages on the topic wis2box/#. The storage service should be configured to send messages to this topic when objects are added to the storage to allow the wis2box-management to trigger the workflow defined by the data-mappings configuration for each dataset. + +Note that the command that keeps the container running is: + +```bash +wis2box pubsub subscribe +``` + +The wis2box pubsub class defined in wis2box-management/wis2box/pubsub/ determines what actions are taken when a message is received on the internal message broker. + +### mosquitto + +Mosquitto-service that provides the internal message broker for the WIS2Box. + +Note that in the standard wis2box-stack the internal broker is also used as the external broker. + +The wis2box-broker image is defined by the Dockerfile in the wis2box-broker directory of this repository. + +### MinIO + +MinIO is used to provide an http-compatible storage endpoint to serve the data and metadata published by wis2box. It is an S3-compatible object storage server. + +Notifications are sent to the internal message broker when objects are added to the storage to trigger the workflow defined by the data-mappings configuration for each dataset. + +### wis2box-api + +This is a pygeoapi instance that provides access to the WIS2Box data and metadata. For the source-code see github repository: https://github.com/wmo-im/wis2box-api + +The backend for the wis2box-api is provided by elasticsearch. + +Note that wis2box-api service includes the eccodes library to allow the following conversion to bufr. + +Note that wis2box-api provides access to the collections 'stations' and 'discovery-metadata': +- 'stations' is a collection of station metadata used when processing data using csv2bufr, synop2bufr and bufr2bufr +- 'discovery-metadata' is a collection of datasets configured in the wis2box, with each dataset containing metadata that describes the dataset and the data-mappings that define the workflow to be triggered when data is added to the storage. + + +### elasticsearch + +An elasticsearch instance that provides the backend for the wis2box-api. + + diff --git a/docker_compose/docker-compose.elasticsearch.yml b/docker_minimum/docker-compose.elasticsearch.yml similarity index 82% rename from docker_compose/docker-compose.elasticsearch.yml rename to docker_minimum/docker-compose.elasticsearch.yml index 275a394a..28a47883 100644 --- a/docker_compose/docker-compose.elasticsearch.yml +++ b/docker_minimum/docker-compose.elasticsearch.yml @@ -1,6 +1,5 @@ services: elasticsearch: - container_name: elasticsearch image: docker.elastic.co/elasticsearch/elasticsearch:8.6.2 restart: always environment: @@ -14,8 +13,11 @@ services: mem_limit: 1.5g memswap_limit: 1.5g volumes: - - ${ELASTICSEARH_DATA}:/usr/share/elasticsearch/data:rw + - elasticsearch-data:/usr/share/elasticsearch/data:rw healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9200"] interval: 5s - retries: 100 \ No newline at end of file + retries: 100 + +volumes: + elasticsearch-data: \ No newline at end of file diff --git a/docker_compose/docker-compose.wis2box-minio.yml b/docker_minimum/docker-compose.minio.yml similarity index 51% rename from docker_compose/docker-compose.wis2box-minio.yml rename to docker_minimum/docker-compose.minio.yml index 392c2ed7..8ec3a20d 100644 --- a/docker_compose/docker-compose.wis2box-minio.yml +++ b/docker_minimum/docker-compose.minio.yml @@ -1,23 +1,22 @@ services: minio: - container_name: wis2box-minio - image: minio/minio:RELEASE.2022-12-02T19-19-22Z.fips + image: minio/minio:RELEASE.2024-05-10T01-41-38Z mem_limit: 512m memswap_limit: 512m restart: always environment: - - MINIO_ROOT_USER=${WIS2BOX_STORAGE_USERNAME} - - MINIO_ROOT_PASSWORD=${WIS2BOX_STORAGE_PASSWORD} + - MINIO_ROOT_USER=minio-admin + - MINIO_ROOT_PASSWORD=minio-password - MINIO_PROMETHEUS_AUTH_TYPE=public - MINIO_NOTIFY_MQTT_ENABLE_WIS2BOX=on - - MINIO_NOTIFY_MQTT_USERNAME_WIS2BOX=${WIS2BOX_BROKER_USERNAME} - - MINIO_NOTIFY_MQTT_PASSWORD_WIS2BOX=${WIS2BOX_BROKER_PASSWORD} - - MINIO_NOTIFY_MQTT_BROKER_WIS2BOX=tcp://${WIS2BOX_BROKER_HOST}:${WIS2BOX_BROKER_PORT} + - MINIO_NOTIFY_MQTT_USERNAME_WIS2BOX=localbroker-admin + - MINIO_NOTIFY_MQTT_PASSWORD_WIS2BOX=localbroker-password + - MINIO_NOTIFY_MQTT_BROKER_WIS2BOX=tcp://mosquitto:1883 - MINIO_NOTIFY_MQTT_TOPIC_WIS2BOX=wis2box/storage - MINIO_NOTIFY_MQTT_QOS_WIS2BOX=1 command: server --console-address ":9001" /data volumes: - - ${MINIO_DATA}:/data + - minio-data:/data healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] interval: 5s @@ -25,4 +24,7 @@ services: retries: 3 depends_on: mosquitto: - condition: service_started \ No newline at end of file + condition: service_started + +volumes: + minio-data: \ No newline at end of file diff --git a/docker_minimum/docker-compose.mosquitto.yml b/docker_minimum/docker-compose.mosquitto.yml new file mode 100644 index 00000000..4af22e8d --- /dev/null +++ b/docker_minimum/docker-compose.mosquitto.yml @@ -0,0 +1,11 @@ +services: + mosquitto: + build: + context: ../wis2box-broker + restart: always + environment: + - WIS2BOX_BROKER_USERNAME=localbroker-admin + - WIS2BOX_BROKER_PASSWORD=localbroker-password + - WIS2BOX_BROKER_QUEUE_MAX=1000 + ports: + - 7883:1883 \ No newline at end of file diff --git a/docker_minimum/docker-compose.wis2box-api.yml b/docker_minimum/docker-compose.wis2box-api.yml new file mode 100644 index 00000000..51e047ad --- /dev/null +++ b/docker_minimum/docker-compose.wis2box-api.yml @@ -0,0 +1,27 @@ +services: + wis2box-api: + image: ghcr.io/wmo-im/wis2box-api:latest + restart: always + environment: + - WIS2BOX_API_URL=http://localhost/oapi + - WIS2BOX_API_BACKEND_URL=http://elasticsearch:9200 + - WIS2BOX_BASEMAP_URL=https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png + - WIS2BOX_BASEMAP_ATTRIBUTION="OpenStreetMap contributors" + - WIS2BOX_BROKER_USERNAME=localbroker-admin + - WIS2BOX_BROKER_PASSWORD=localbroker-password + - WIS2BOX_BROKER_HOST=mosquitto + - WIS2BOX_BROKER_PORT=1883 + - WIS2BOX_LOGGING_LOGLEVEL=INFO + - WIS2BOX_DATADIR=/data/wis2box + depends_on: + elasticsearch: + condition: service_healthy + volumes: + - api-config:/data/wis2box/config/pygeoapi/:rw + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost/oapi/admin/resources"] + interval: 5s + retries: 100 + +volumes: + api-config: \ No newline at end of file diff --git a/docker_minimum/docker-compose.wis2box-management.yml b/docker_minimum/docker-compose.wis2box-management.yml new file mode 100644 index 00000000..a42a3e9c --- /dev/null +++ b/docker_minimum/docker-compose.wis2box-management.yml @@ -0,0 +1,33 @@ +services: + wis2box-management: + mem_limit: 1g + memswap_limit: 1g + restart: always + build: + context: ../wis2box-management + environment: + - WIS2BOX_URL=http://www.mywis2data.com + - WIS2BOX_DOCKER_API_URL=http://wis2box-api:80/oapi + - WIS2BOX_API_BACKEND_URL=http://elasticsearch:9200 + - MINIO_NOTIFY_MQTT_USERNAME_WIS2BOX=localbroker-admin + - MINIO_NOTIFY_MQTT_PASSWORD_WIS2BOX=localbroker-password + - WIS2BOX_BROKER_HOST=mosquitto + - WIS2BOX_BROKER_PORT=1883 + - WIS2BOX_BROKER_USERNAME=localbroker-admin + - WIS2BOX_BROKER_PASSWORD=localbroker-password + - WIS2BOX_STORAGE_SOURCE=http://minio:9000 + - WIS2BOX_STORAGE_USERNAME=minio-admin + - WIS2BOX_STORAGE_PASSWORD=minio-password + - WIS2BOX_BROKER_PUBLIC=mqtt://localbroker-admin:localbroker-password@mosquitto:1883 + - WIS2BOX_DATADIR=/data/wis2box + - WIS2BOX_LOGGING_LOGLEVEL=INFO + volumes: + - ../tests/data:/data/wis2box:rw + depends_on: + minio: + condition: service_healthy + mosquitto: + condition: service_started + wis2box-api: + condition: service_healthy + command: ["wis2box", "pubsub" , "subscribe"] \ No newline at end of file diff --git a/wis2box-management/docker/entrypoint.sh b/wis2box-management/docker/entrypoint.sh index c357cc9a..d77d9895 100755 --- a/wis2box-management/docker/entrypoint.sh +++ b/wis2box-management/docker/entrypoint.sh @@ -73,6 +73,17 @@ else htpasswd -bc /home/wis2box/.htpasswd/webapp $WIS2BOX_WEBAPP_USERNAME $WIS2BOX_WEBAPP_PASSWORD || true fi +# check if WIS2BOX_AUTH_URL is set, if not skip running the remaining commands +auth_url=${WIS2BOX_AUTH_URL} +if [ -z "$auth_url" ]; then + echo "WIS2BOX_AUTH_URL is not set, skipping authentication setup" + echo "END /entrypoint.sh" + exec "$@" +else + echo "WIS2BOX_AUTH_URL is set to $auth_url" + echo "Proceed to restrict processes/wis2box and collections/stations" +fi + # Check if the path is restricted and capture the output is_restricted=$(wis2box auth is-restricted-path --path processes/wis2box) if [ "$is_restricted" = "True" ]; then From 23edc5bcfd10281e9d0a52e841ed8cf151587396 Mon Sep 17 00:00:00 2001 From: Maaike Date: Wed, 15 May 2024 08:44:29 +0200 Subject: [PATCH 3/3] Update docker-compose.override.yml revert accidental local commit --- docker-compose.override.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 415bdc13..292c5ea7 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -1,7 +1,7 @@ services: web-proxy: ports: - - 4480:80 + - 80:80 wis2box-ui: ports: