Skip to content

add CAP test

add CAP test #4195

Workflow file for this run

name: test docker deployment ⚙️
on: [ push, pull_request ]
jobs:
main:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
name: setup Python
with:
python-version: "3.10"
- name: install testing requirements 📦
working-directory: tests
run: |
pip3 install -r requirements.txt
- name: cache schemas 📦
run: |
pywis-pubsub schema sync
pywcmp bundle sync
- name: display Docker and Python versions 📦
run: |
docker version
docker compose version
python3 -V
- name: setup wis2downloader
run: |
# make sure current user in docker group
sudo usermod -aG docker $(whoami)
# Add env parameters to tests/test.env
echo "" >> tests/test.env #no new line at end of test.env file
echo "DOCKER_GID=$(getent group docker | cut -d: -f3)" >> tests/test.env
# make sure data dir is in group docker and has 775 permissions
mkdir -p tests/data/downloads
chown -R $(whoami):docker tests/data/downloads
chmod -R 775 tests/data/downloads
- name: setup wis2box configuration, replace localhost with IP on host 📦
run: |
export IP=$(hostname -I | awk '{print $1}')
cp tests/test.env wis2box.env
sed -i "s/localhost/$IP/g" wis2box.env
cat wis2box.env
python3 wis2box-ctl.py config
- name: build wis2box
run: |
python3 wis2box-ctl.py build
python3 wis2box-ctl.py update
- name: start containers ⚙️
run: |
python3 wis2box-ctl.py start
python3 wis2box-ctl.py status -a
- name: show environment and check collections exist ⚙️
run: |
sleep 30
python3 wis2box-ctl.py execute wis2box environment show
curl http://localhost/oapi/collections/stations
curl http://localhost/oapi/collections/discovery-metadata
curl http://localhost/oapi/collections/messages
- name: downloader subscribe to topics
run: |
python3 wis2box-ctl.py execute wis2box auth add-token -p wis2downloader github123 -y
echo '{"topic": "origin/a/wis2/+/data/core/#"}' > test.json
curl -X POST http://localhost/wis2downloader/subscriptions -H "Content-Type: application/json" -H "Authorization: Bearer github123" -d @test.json
- name: add CAP data, cap-publish
env:
DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-cap.yml
DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:cap
run: |
python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
python3 tests/send_CAP_as_message.py
- name: add Malawi synop data (csv2bufr synop_bufr template) 🇲🇼
env:
TOPIC_HIERARCHY: origin/a/wis2/mw-mw_met_centre-test/data/core/weather/surface-based-observations/synop
TERRITORY: MWI
DISCOVERY_METADATA: /data/wis2box/metadata/discovery/mw-surface-weather-observations.yml
DISCOVERY_METADATA_ID: urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations
STATION_METADATA: /data/wis2box/metadata/station/malawi.csv
TEST_DATA: /data/wis2box/observations/malawi
TEST_DATA_UPDATE: /data/wis2box/observations/malawi_update
run: |
python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY
curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID
pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID
python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA
python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA_UPDATE
- name: add Italy synop data (bufr2bufr) 🇮🇹
env:
TOPIC_HIERARCHY: origin/a/wis2/it-meteoam/data/core/weather/surface-based-observations/synop
TERRITORY: ITA
DISCOVERY_METADATA: /data/wis2box/metadata/discovery/it-surface-weather-observations.yml
DISCOVERY_METADATA_ID: urn:wmo:md:it-meteoam:surface-weather-observations
STATION_METADATA: /data/wis2box/metadata/station/italy.csv
TEST_DATA: /data/wis2box/observations/italy
run: |
python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY
curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID
pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID
python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA
- name: add Algeria synop data (bufr2bufr) 🇩🇿
env:
TOPIC_HIERARCHY: origin/a/wis2/dz-meteoalgerie/data/core/weather/surface-based-observations/synop
TERRITORY: DZA
DISCOVERY_METADATA: /data/wis2box/metadata/discovery/dz-surface-weather-observations.yml
DISCOVERY_METADATA_ID: urn:wmo:md:dz-meteoalgerie:surface-weather-observations
STATION_METADATA: /data/wis2box/metadata/station/algeria.csv
TEST_DATA: /data/wis2box/observations/algeria
run: |
python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY
curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID
pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID
python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA
- name: add Romania synop data (synop2bufr and csv2bufr aws-template) 🇷🇴
env:
TOPIC_HIERARCHY: origin/a/wis2/ro-rnimh-test/data/core/weather/surface-based-observations/synop
TERRITORY: ROU
DISCOVERY_METADATA: /data/wis2box/metadata/discovery/ro-synoptic-weather-observations.yml
DISCOVERY_METADATA_ID: urn:wmo:md:ro-rnimh-test:synoptic-weather-observations
STATION_METADATA: /data/wis2box/metadata/station/romania.csv
TEST_DATA: /data/wis2box/observations/romania
run: |
python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY
curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID
pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID
python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA
- name: add Congo synop data (synop2bufr) 🇨🇩
env:
TOPIC_HIERARCHY: origin/a/wis2/cg-met/data/recommended/weather/surface-based-observations/synop
TERRITORY: COD
DISCOVERY_METADATA: /data/wis2box/metadata/discovery/cd-surface-weather-observations.yml
DISCOVERY_METADATA_ID: urn:wmo:md:cg-met:surface-weather-observations
STATION_METADATA: /data/wis2box/metadata/station/congo.csv
TEST_DATA: /data/wis2box/observations/congo
run: |
python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY
curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID
#pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID # uncomment once wis2box improves support for recommended data
python3 wis2box-ctl.py execute wis2box auth add-token --metadata-id $DISCOVERY_METADATA_ID -p token123 -y
python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA
- name: add example hourly ship data (bufr2bufr) WMO
env:
TOPIC_HIERARCHY: origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship-hourly
DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-ship-hourly.yml
DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:surface-weather-observations:ship-hourly
STATION_METADATA: /data/wis2box/metadata/station/wmo-test-ship-hourly.csv
TEST_DATA: /data/wis2box/observations/wmo/ship-hourly
run: |
python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY
curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID
pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID
python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA
- name: add example drifting buoy data (bufr2bufr) WMO
env:
TOPIC_HIERARCHY: origin/a/wis2/int-wmo-test/data/core/ocean/surface-based-observations/drifting-buoys
DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-drifting-buoys.yml
DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:surface-weather-observations:drifting-buoys
STATION_METADATA: /data/wis2box/metadata/station/wmo-test-drifting-buoys.csv
TEST_DATA: /data/wis2box/observations/wmo/drifting-buoys
run: |
python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY
curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID
pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID
python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA
- name: add example wind profile data (bufr2bufr) WMO
env:
TOPIC_HIERARCHY: origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/wind-profile
DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-wind-profile.yml
DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:surface-weather-observations:wind-profile
STATION_METADATA: /data/wis2box/metadata/station/wmo-test-wind-profile.csv
TEST_DATA: /data/wis2box/observations/wmo/wind-profile
run: |
python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY
curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID
pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID
python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA
- name: add China GRIB2 data (universal pipeline) 🇨🇳
env:
DISCOVERY_METADATA: /data/wis2box/metadata/discovery/cn-grapes-geps-global.yml
DISCOVERY_METADATA_ID: urn:wmo:md:cn-cma:grapes-geps-global
TEST_DATA: /data/wis2box/observations/china
run: |
python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID
pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID
python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA
- name: sleep 30 seconds then run integration tests ⚙️
run: |
sleep 30
docker logs wis2box-management
pytest -s tests/integration
- name: run flake8 ⚙️
run: |
find . -type f -name "*.py" | xargs flake8
- name: failed tests 🚩
if: ${{ failure() }}
run: |
docker logs -n1000 wis2box-management
docker logs -n1000 wis2box-api