CAP Editor Integration

docker-compose.override.yml

@@ -17,4 +17,4 @@ services:
   mosquitto:
     ports:
       - 1883:1883
-      - 8884:8884
+      - 8884:8884

docs/source/reference/running/data-pipeline-plugins.rst

@@ -98,6 +98,27 @@ A typical BUFR4 plugin workflow definition would be defined as follows:
          notify: true  # trigger GeoJSON publishing for API and UI
          file-pattern: '^.*\.bin$'
 
+``wis2box.data.cap_message.CAPMessageData``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This plugin takes the incoming XML file, then validates it against the
+`CAP v1.2 schema <https://docs.oasis-open.org/emergency/cap/v1.2/CAP-v1.2-os.html>`_
+and verifies the digital signature before publishing.
+
+The validation is performed using the `capvalidator <https://github.com/wmo-im/capvalidator>`_
+package.
+
+A typical CAP message plugin workflow definition would be defined as follows:
+
+.. code-block:: yaml
+
+   xml:
+       - plugin: wis2box.data.cap_message.CAPMessageData
+         notify: true
+          buckets:
+            - ${WIS2BOX_STORAGE_INCOMING}
+         file-pattern: '^.*\.xml$'
+
 ``wis2box.data.universal.UniversalData``
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

tests/data/CAP/sc_example.xml

@@ -0,0 +1,71 @@
+<?xml version='1.0' encoding='utf-8'?>
+<?xml-stylesheet type="text/xsl" href="https://www.meteo.sc/cap-alert-style.xsl"?>
+<alert xmlns="urn:oasis:names:tc:emergency:cap:1.2">
+    <identifier>urn:oid:2.49.0.0.690.0.2024.5.19.13.18.0</identifier>
+    <sender>[email protected]</sender>
+    <sent>2024-05-19T17:18:00+04:00</sent>
+    <status>Actual</status>
+    <msgType>Alert</msgType>
+    <scope>Public</scope>
+    <info>
+        <language>en</language>
+        <category>Met</category>
+        <event>Strong Winds</event>
+        <urgency>Immediate</urgency>
+        <severity>Moderate</severity>
+        <certainty>Observed</certainty>
+        <audience>General Public</audience>
+        <effective>2024-05-19T17:30:00+04:00</effective>
+        <onset>2024-05-19T17:30:00+04:00</onset>
+        <expires>2024-05-19T23:30:00+04:00</expires>
+        <senderName>Seychelles Meteorological Authority</senderName>
+        <headline>Moderate to strong south-easterly winds over Aldabra area</headline>
+        <description>Moderate to strong south-easterly winds over Aldabra area associated with a severe tropical storm Ialy on 19th May 2024 from 5pm to 11pm</description>
+        <instruction>Beware of strong south-easterly winds of 40km/hr gusting to 60km/hr causing rough seas. Mariners are advised to take extra precautions when navigating these areas</instruction>
+        <web>https://www.meteo.sc/alerts/severe-tropical-storm-ialy-is-expected-to-cause-moderate-to-strong-south-easterly-winds-over-aldabra-area-on-19th-may-2024/</web>
+        <contact>[email protected]</contact>
+        <area>
+            <areaDesc>Aldabra area</areaDesc>
+            <polygon>-9.186965,46.078276 -9.336746,45.936002 -9.57065,45.983427 -9.748309,46.116216 -9.897844,46.249006 -10.000609,46.42922 -9.953902,46.694799 -9.692216,46.770679 -9.467752,46.770679 -9.327387,46.742224 -9.186965,46.694799 -9.140146,46.533555 -9.121416,46.362825 -9.186965,46.078276</polygon>
+        </area>
+    </info>
+    <ds:Signature xmlns:ds="http://www.w3.org/2000/09/xmldsig#">
+        <ds:SignedInfo>
+            <ds:CanonicalizationMethod Algorithm="http://www.w3.org/2006/12/xml-c14n11"/>
+            <ds:SignatureMethod Algorithm="http://www.w3.org/2001/04/xmldsig-more#ecdsa-sha256"/>
+            <ds:Reference URI="">
+                <ds:Transforms>
+                    <ds:Transform Algorithm="http://www.w3.org/2000/09/xmldsig#enveloped-signature"/>
+                    <ds:Transform Algorithm="http://www.w3.org/2006/12/xml-c14n11"/>
+                </ds:Transforms>
+                <ds:DigestMethod Algorithm="http://www.w3.org/2001/04/xmlenc#sha256"/>
+                <ds:DigestValue>s9RKcAph3khDLX4nOQQDZ7c23uVZqCDJziRZznnh3nA=</ds:DigestValue>
+            </ds:Reference>
+        </ds:SignedInfo>
+        <ds:SignatureValue>liScT3sHEpNO1TXEDY1nMC5FoBnQruioH/xkU1rNRKfK8Y3y4/lRz64ueJfoWGYB/N3NWLkMhQ6V0X4lHGPyfA==</ds:SignatureValue>
+        <ds:KeyInfo>
+            <ds:X509Data>
+                <ds:X509Certificate>MIIDeTCCAv6gAwIBAgISA7FZkY9qfwdIeZgOZUlggQr+MAoGCCqGSM49BAMDMDIx
+CzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1MZXQncyBFbmNyeXB0MQswCQYDVQQDEwJF
+NTAeFw0yNDA3MDIxMTIzNDlaFw0yNDA5MzAxMTIzNDhaMBcxFTATBgNVBAMTDHd3
+dy5tZXRlby5zYzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABPnwJK7C7LIL7c3L
+DXDTbFlLSHx8VXbbSKFqkr0v8xjseuNjh8IXFm95mvkdk7q1S0SUYyYn3d8a0krt
+2qhGqCKjggINMIICCTAOBgNVHQ8BAf8EBAMCB4AwHQYDVR0lBBYwFAYIKwYBBQUH
+AwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFCQFyLRIaHFpqT8I
+nTKhv8oSaTW1MB8GA1UdIwQYMBaAFJ8rX888IU+dBLftKyzExnCL0tcNMFUGCCsG
+AQUFBwEBBEkwRzAhBggrBgEFBQcwAYYVaHR0cDovL2U1Lm8ubGVuY3Iub3JnMCIG
+CCsGAQUFBzAChhZodHRwOi8vZTUuaS5sZW5jci5vcmcvMBcGA1UdEQQQMA6CDHd3
+dy5tZXRlby5zYzATBgNVHSAEDDAKMAgGBmeBDAECATCCAQMGCisGAQQB1nkCBAIE
+gfQEgfEA7wB2AHb/iD8KtvuVUcJhzPWHujS0pM27KdxoQgqf5mdMWjp0AAABkHNm
+tr8AAAQDAEcwRQIhANX48FhFLRl8W0qsVh12vz2F92wr2aKId+AQ/0kvE+a0AiA/
+eB4KudtHm4LJL7VSVL7UvffuEPOsY+PvoongycjZpQB1AEiw42vapkc0D+VqAvqd
+MOscUgHLVt0sgdm7v6s52IRzAAABkHNmvkUAAAQDAEYwRAIgFtFOST10XUPf2BYT
+xBBvHVqU98eB2hwQtgVJ4hJP5RoCIF0wDotvI7r+kamXqgvee+/ig4NP2ZbqaLP6
+a2/T5cjnMAoGCCqGSM49BAMDA2kAMGYCMQCyLgDsI/yPYKkI1zM3zs0w7iI23MfZ
+BGuNKUUa7qHLR1O6eNnEmrSH24bdzXdacRoCMQCLW6bf0Y1mwuJN+jBCjTbyCe+F
+1ZEDJBb2AKxTZpWdVdtfRErY5BxHuACOm9SlXGE=
+</ds:X509Certificate>
+            </ds:X509Data>
+        </ds:KeyInfo>
+    </ds:Signature>
+</alert>

tests/data/metadata/discovery/int-wmo-test-cap.yml

@@ -0,0 +1,64 @@
+wis2box:
+    retention: P30D
+    topic_hierarchy: int-wmo-test/data/core/weather/advisories-warnings
+    centre_id: int-wmo-test
+    data_mappings:
+        plugins:
+            xml:
+                - plugin: wis2box.data.cap_message.CAPMessageData
+                  notify: true
+                  buckets:
+                      - ${WIS2BOX_STORAGE_INCOMING}
+                  file-pattern: '^.*\.xml$'
+
+mcf:
+    version: 1.0
+
+metadata:
+    identifier: urn:wmo:md:int_wmo_test:cap
+    hierarchylevel: dataset
+
+identification:
+    title: CAP Alerts test dataset
+    abstract: CAP Alerts test dataset
+    dates:
+        creation: 2023-03-26
+    keywords:
+        default:
+            keywords:
+                - CAP
+                - warnings
+                - alerts
+        wmo:
+            keywords:
+                - weather
+            keywords_type: themes
+            vocabulary:
+                name: Earth system disciplines as defined by the WMO Unified Data Policy, Resolution 1 (Cg-Ext(2021), Annex 1.
+                url: https://codes.wmo.int/topic-hierarchy/earth-system-discipline
+    extents:
+        spatial:
+            - bbox: [-180.0, -90.0, 180.0, 90.0]
+              crs: 4326
+        temporal:
+            - begin: 2024-07-02
+              end: null
+              resolution: P1H
+    wmo_data_policy: core
+
+contact:
+    host:
+        organization: WMO
+        url: https://wmo.int
+        individualname: Firstname Lastname
+        positionname: Position Name
+        phone: null
+        fax: null
+        address: null
+        city: null
+        administrativearea: null
+        postalcode: null
+        country: Switzerland
+        email: [email protected]
+        hoursofservice: 0700h - 1500h UTC
+        contactinstructions: email

tests/requirements.txt

@@ -1,4 +1,5 @@
 flake8
+paho-mqtt<2
 pytest
 pywis-pubsub
-requests
+requests

tests/test_publish_cap_message.py

@@ -0,0 +1,57 @@
+###############################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+###############################################################################
+
+import base64
+import json
+
+import paho.mqtt.publish as publish
+
+BROKER_USERNAME = 'wis2box'
+BROKER_PASSWORD = 'wis2box'
+BROKER_HOST = 'localhost'
+BROKER_PORT = '1883'
+
+filename = 'tests/data/cap/sc_example.xml'
+
+# create a message containing the CAP alert in
+#  the data field as base64 encoded bytes
+with open(filename, 'rb') as file:
+    data = base64.b64encode(file.read()).decode()
+
+msg = {
+    'metadata_id': 'urn:wmo:md:int_wmo_test:cap',
+    'data': data,
+    'filename': filename.split('/')[-1]
+}
+
+# publish notification on internal broker
+private_auth = {
+    'username': BROKER_USERNAME,
+    'password': BROKER_PASSWORD
+}
+
+publish.single(topic='wis2box/cap/publication',
+               payload=json.dumps(msg),
+               qos=1,
+               retain=False,
+               hostname=BROKER_HOST,
+               port=int(BROKER_PORT),
+               auth=private_auth)

wis2box-broker/entrypoint.sh

@@ -24,4 +24,15 @@ fi
 sed -i "s#_WIS2BOX_BROKER_QUEUE_MAX#$WIS2BOX_BROKER_QUEUE_MAX#" /mosquitto/config/mosquitto.conf
 sed -i "s#_WIS2BOX_BROKER_USERNAME#$WIS2BOX_BROKER_USERNAME#" /mosquitto/config/acl.conf
 
+for i in `env | grep -Ee "\<WIS2BOX_BROKER_USERNAME_[[:alnum:]]+"`; do
+    NAME_TAIL=`echo $i | awk -FWIS2BOX_BROKER_USERNAME_ '{print $2}' | awk -F= '{print $1}'`
+    username=WIS2BOX_BROKER_USERNAME_$NAME_TAIL
+    password=WIS2BOX_BROKER_PASSWORD_$NAME_TAIL
+    topic=WIS2BOX_BROKER_TOPIC_$NAME_TAIL
+    echo ${!username}, ${!password}
+    mosquitto_passwd -b /mosquitto/config/password.txt ${!username} ${!password}
+    echo "user ${!username}" >>  /mosquitto/config/acl.conf
+    echo "topic readwrite ${!topic}" >>  /mosquitto/config/acl.conf
+done
+
 /usr/sbin/mosquitto -c /mosquitto/config/mosquitto.conf

wis2box-management/Dockerfile

@@ -19,7 +19,7 @@
 #
 ###############################################################################
 
-FROM ubuntu:focal
+FROM ubuntu:jammy
 
 LABEL maintainer="[email protected]; [email protected]"
 
@@ -40,6 +40,8 @@ RUN apt-get update -y && apt-get install -y ${DEBIAN_PACKAGES} \
     https://github.com/wmo-cop/pyoscar/archive/refs/tags/0.7.0.zip \
     https://github.com/geopython/pygeometa/archive/refs/tags/0.16.0.zip \
     https://github.com/wmo-im/pywis-topics/archive/refs/tags/0.2.0.zip \
+    # install cap validator
+    && pip3 install --no-cache-dir capvalidator>=0.1.0-dev4 \
     # install shapely
     && pip3 install --no-cache-dir cython pygeos==0.13 \
     && pip3 install shapely \

wis2box-management/requirements.txt

@@ -9,3 +9,4 @@ pywis-pubsub
 pywis-topics
 PyYAML
 requests
+capvalidator>=0.1.0-dev4

wis2box-management/wis2box/data/cap_message.py

@@ -0,0 +1,95 @@
+###############################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+###############################################################################
+
+from datetime import datetime
+import logging
+from pathlib import Path
+from typing import Union
+
+from capvalidator import validate_cap_message, get_dates
+
+from wis2box.data.base import BaseAbstractData
+
+LOGGER = logging.getLogger(__name__)
+
+
+class CAPMessageData(BaseAbstractData):
+    """
+    DataPublish:
+
+    transform sets output_data to input_data
+    using metadata received by the plugin
+    """
+
+    def __init__(self, defs: dict) -> None:
+        """
+        CAPMessageData data initializer
+
+        :param def: `dict` object of resource mappings
+
+        :returns: `None`
+        """
+
+        super().__init__(defs)
+
+    def transform(self, input_data: Union[Path, bytes],
+                  filename: str = '') -> bool:
+        """
+        Transform input_data to output_data
+
+        :param input_data: input data
+        :param filename: filename of input data
+        :param _meta: metadata of input data
+
+        :returns: `bool` of result
+        """
+
+        suffix = filename.split('.')[-1]
+        rmk = filename.split('.')[0]
+        input_bytes = self.as_bytes(input_data)
+
+        # get the sent date from the CAP XML
+        sent_date = get_dates(input_bytes).sent
+        # convert isoformat to datetime
+        _meta = {}
+        _meta['data_date'] = datetime.fromisoformat(sent_date)
+        # add relative filepath to _meta
+        _meta['relative_filepath'] = self.get_local_filepath(_meta['data_date'])  # noqa
+
+        # validate the CAP XML string content using the capvalidator package
+        result = validate_cap_message(input_bytes, strict=False)
+        if not result.passed:
+            LOGGER.error(
+                f'Invalid CAP XML, not publishing. Reason: {result.message}')
+            return False
+
+        LOGGER.info(
+            f'CAP XML is valid, publishing to wis2box. {result.message}')
+
+        self.output_data[rmk] = {
+            suffix: input_bytes,
+            '_meta': _meta
+        }
+        return True
+
+    def get_local_filepath(self, date_):
+        yyyymmdd = date_.strftime('%Y-%m-%d')
+        return Path(yyyymmdd) / 'wis' / self.metadata_id