Squash commit message: set up a SMTP account to sent out feedback to …

…contributors

Feat: Woudc-Data-Registry

set up a SMTP account to sent out feedback to contributors

Added another click option for ops

Minor fixes to processing reports and future testing

fix flake8 errors

fixed a comment

default.env

@@ -41,3 +41,15 @@ export WDR_UV_INDEX_FORMULA_LOOKUP=/path/to/data/uv-index-formula-lookup.csv
 
 # enable shell autocompletion - optional
 # eval "$(_WOUDC_DATA_REGISTRY_COMPLETE=source woudc-data-registry)"
+
+# Email configuration
+export WDR_EMAIL_HOST=host
+export WDR_EMAIL_PORT=port
+export WDR_EMAIL_SECURE=False
+export WDR_EMAIL_TEST=False
+export WDR_EMAIL_FROM_USERNAME=email
+export WDR_EMAIL_FROM_PASSWORD=<secret>
+export WDR_EMAIL_TO=email
+export WDR_EMAIL_CC=email
+export WDR_EMAIL_BCC=email
+export WDR_TEMPLATE_PATH=/path/to/etc/woudc-contributor-feedback.txt

etc/woudc-contributor-feedback.txt

@@ -0,0 +1,28 @@
+Dear WOUDC Data Contributor,
+
+This message is from WOUDC (World Ozone and Ultraviolet Radiation Data Centre) to inform you of the status of your recent data submission.
+Data submissions now require the content, data generation, platform, instrument and location metadata to exactly match our metadata information for the data to be properly identified and processed into the database.  
+
+Where possible, your submitted files have been manually repaired to contain the correct metadata information and resubmitted on your behalf.  You do not need to resubmit the files, but please update your procedures to use the correct metadata for all future submissions.  (Detailed data submission requirements are here: https://guide.woudc.org/en/#chapter-3-standard-data-format).
+
+If your station, platform, agency or contact information have been updated then please reply to this message with the updated information.
+Thank you for your support of WOUDC.
+
+$EMAIL_SUMMARY
+
+World Ozone and Ultraviolet Radiation Data Centre
+
+Meteorological Service of Canada
+Environment and Climate Change Canada
+4905 Dufferin Street
+Toronto, ON M3H 5T4
+Canada
+Email: [email protected]
+Website: https://woudc.org
+Data policy: https://woudc.org/about/data-policy.php
+Data submissions: ftp://ftp.woudc.org
+Data access: https://woudc.org/about/data-access.php
+Contributor guidebook: https://guide.woudc.org
+
+Dobson and Brewer software: http://www.o3soft.eu
+

woudc_data_registry/config.py

@@ -71,6 +71,16 @@
 WDR_ALIAS_CONFIG = os.getenv('WDR_ALIAS_CONFIG')
 WDR_EXTRA_CONFIG = os.getenv('WDR_EXTRA_CONFIG')
 WDR_UV_INDEX_FORMULA_LOOKUP = os.getenv('WDR_UV_INDEX_FORMULA_LOOKUP')
+WDR_EMAIL_HOST = os.getenv('WDR_EMAIL_HOST')
+WDR_EMAIL_PORT = os.getenv('WDR_EMAIL_PORT')
+WDR_EMAIL_SECURE = os.getenv('WDR_EMAIL_SECURE')
+WDR_EMAIL_TEST = os.getenv('WDR_EMAIL_TEST')
+WDR_EMAIL_FROM_USERNAME = os.getenv('WDR_EMAIL_FROM_USERNAME')
+WDR_EMAIL_FROM_PASSWORD = os.getenv('WDR_EMAIL_FROM_PASSWORD')
+WDR_EMAIL_TO = os.getenv('WDR_EMAIL_TO')
+WDR_EMAIL_CC = os.getenv('WDR_EMAIL_CC')
+WDR_EMAIL_BCC = os.getenv('WDR_EMAIL_BCC')
+WDR_TEMPLATE_PATH = os.getenv('WDR_TEMPLATE_PATH')
 
 if not WDR_SEARCH_INDEX_BASENAME:
     msg = 'WDR_SEARCH_INDEX_BASENAME was not set. \

woudc_data_registry/controller.py

@@ -52,7 +52,9 @@
                           MetadataValidationError)
 
 from woudc_data_registry import config
-from woudc_data_registry.util import is_text_file, read_file
+from woudc_data_registry.util import (is_text_file, read_file,
+                                      send_email)
+
 
 from woudc_data_registry.processing import Process
 
@@ -261,11 +263,75 @@ def generate_emails(ctx, working_dir):
     email_summary = EmailSummary(working_dir)
 
     contributors = registry.query_full_index(Contributor)
-    addresses = {model.acronym: model.email for model in contributors}
+    ctx.addresses = {model.acronym: model.email for model in contributors}
+
+    email_summary.write(ctx.addresses)
+
 
-    email_summary.write(addresses)
+@click.command()
+@click.pass_context
+@click.option('--test', is_flag=True, help="Enable the test flag.")
+@click.option('--ops', is_flag=True, help="Enable the ops flag.")
+@click.argument('failed_files', type=click.File('r'))
+def send_feedback(ctx, failed_files, test, ops):
+    """Send operating reports to contributors. """
+
+    LOGGER.debug("test: {} ops: {}".format(test, ops))
+    with open(config.WDR_TEMPLATE_PATH, 'r') as file:
+        message = file.read()
+
+    templates = failed_files.read().split('\n\n')
+    template_collection = [template.split('\n') for template in templates]
+
+    subject = 'WOUDC data processing report (contributor_acronym)'
+    host = config.WDR_EMAIL_HOST
+    port = config.WDR_EMAIL_PORT
+    from_email_address = config.WDR_EMAIL_FROM_USERNAME
+    cc_addresses = [config.WDR_EMAIL_CC]
+    bcc_addresses = [config.WDR_EMAIL_BCC]
+
+    LOGGER.info('Configs all set to send feedback to contributors')
+
+    for contributor in template_collection:
+        acronym = contributor[0].split(' ')[0].lower()
+        message = message.replace(
+            "$EMAIL_SUMMARY", "\n".join(contributor[1:]))
+        specific_subject = subject.replace('contributor_acronym', acronym)
+
+        if test:
+            to_email_addresses = config.WDR_EMAIL_TO.split(",")
+            subject = (
+                'TEST: WOUDC data processing report ({})'.format(acronym))
+            LOGGER.info(
+                'Sending Test data report to agency: %s with emails to: %s',
+                acronym, to_email_addresses
+            )
+            send_email(
+                message, subject, from_email_address,
+                to_email_addresses, host, port, cc_addresses,
+                bcc_addresses
+            )
+        elif ops:
+            to_email_addresses = [
+                email.strip() for email in contributor[0].split(' ')[1]
+                .translate(str.maketrans("", "", "()")).split(";")]
+            LOGGER.info(
+                'Sending data report to agency: %s with emails to: %s',
+                acronym, to_email_addresses
+            )
+            send_email(
+                message, specific_subject, from_email_address,
+                to_email_addresses, host, port, cc_addresses,
+                bcc_addresses
+            )
+        LOGGER.debug(
+            'Sent email to %s with emails to %s',
+            acronym, to_email_addresses
+        )
+    LOGGER.info('Processing Reports have been sent')
 
 
 data.add_command(ingest)
 data.add_command(verify)
 data.add_command(generate_emails, name='generate-emails')
+data.add_command(send_feedback, name='send-feedback')

woudc_data_registry/util.py

@@ -46,12 +46,107 @@
 from datetime import date, datetime, time
 import logging
 import io
+import smtplib
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
 
 LOGGER = logging.getLogger(__name__)
 
 RFC3339_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
 
 
+def send_email(message, subject, from_email_address, to_email_addresses,
+               host, port, cc_addresses=None, bcc_addresses=None, secure=False,
+               from_email_password=None):
+
+    """
+    Send email
+
+    :param message: body of the email
+    :param subject: subject of the email
+    :param from_email_address: email of the sender
+    :param to_email_addresses: list of emails of the receipients
+    :param host: host of SMTP server
+    :param cc_addresses: list of cc email addresses
+    :param port: port on SMTP server
+    :param secure: Turn on/off TLS
+    :param from_email_password: password of sender, if TLS is turned on
+    :returns: list of emailing statuses
+    """
+    try:
+        server = smtplib.SMTP(host, port)
+    except Exception as err:
+        msg = 'Unable to establish connection to {}:{}'.format(host, port)
+        LOGGER.critical(msg)
+        raise err
+
+    if all([secure, from_email_password is not None]):
+        try:
+            server.starttls()
+        except Exception as err:
+            msg = 'Unable to start TLS: {}'.format(err)
+        try:
+            server.login(from_email_address, from_email_password)
+        except Exception as err:
+            msg = 'Unable to login using username {}: {}'.format(
+                from_email_address, err)
+
+    send_statuses = []
+    cc = False
+    LOGGER.debug('cc: {}' .format(cc_addresses))
+    # cc
+    if all([
+            cc_addresses is not None,
+            cc_addresses != ['']
+            ]):
+        to_email_addresses += cc_addresses
+        cc = True
+    LOGGER.debug('bcc: {}' .format(bcc_addresses))
+    # bcc
+    if all([
+            bcc_addresses is not None,
+            bcc_addresses != ['']
+            ]):
+        to_email_addresses += bcc_addresses
+
+    LOGGER.debug('to_email: {}' .format(to_email_addresses))
+    if isinstance(to_email_addresses, str):
+        to_email_addresses = to_email_addresses.split(';')
+
+    # set up the message
+    msg = MIMEMultipart()
+    msg['From'] = from_email_address
+    msg['To'] = ', '.join(to_email_addresses)
+    if cc:
+        msg['Cc'] = ', '.join(cc_addresses)  # Add CC addresses if they exist
+    msg['Subject'] = subject
+    msg.attach(MIMEText(message, 'plain'))
+
+    # Convert the message to a string
+    text = msg.as_string()
+    LOGGER.debug('Message: {}' .format(text))
+
+    # send message
+    try:
+        LOGGER.debug(
+            'Sending report to {}'.format(to_email_addresses)
+            )
+        send_status = server.sendmail(
+            msg['From'], to_email_addresses + cc_addresses, text)
+        send_statuses.append(send_status)
+    except Exception as err:
+        error_msg = (
+            'Unable to send mail from: {} to {}: {}'.format(
+                msg['From'], msg['To'], err
+            )
+        )
+
+        LOGGER.error(error_msg)
+        raise err
+
+    server.quit()
+
+
 def point2geojsongeometry(x, y, z=None):
     """
     helper function to generate GeoJSON geometry of point