From 0810e091c20805e4e6e2ebbdec5fde6e0c8679b8 Mon Sep 17 00:00:00 2001 From: bedroesb Date: Fri, 29 Nov 2024 10:26:58 +0100 Subject: [PATCH 01/21] type pandas --- ena_upload/ena_upload.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ena_upload/ena_upload.py b/ena_upload/ena_upload.py index be66bed..40ab893 100755 --- a/ena_upload/ena_upload.py +++ b/ena_upload/ena_upload.py @@ -941,6 +941,7 @@ def main(): # SettingWithCopyWarning causes false positive # e.g at df.loc[:, 'file_checksum'] = md5 pd.options.mode.chained_assignment = None + df['file_checksum'] = df['file_checksum'].astype('string') df.loc[:, 'file_checksum'] = md5 print("done.") elif check_file_checksum(df): @@ -969,10 +970,12 @@ def main(): if pd.notna(row['scientific_name']) and pd.isna(row['taxon_id']): # retrieve taxon id using scientific name taxonID = get_taxon_id(row['scientific_name']) + df['taxon_id'] = df['taxon_id'].astype('string') df.loc[index, 'taxon_id'] = taxonID elif pd.notna(row['taxon_id']) and pd.isna(row['scientific_name']): # retrieve scientific name using taxon id scientificName = get_scientific_name(row['taxon_id']) + df['scientific_name'] = df['scientific_name'].astype('string') df.loc[index, 'scientific_name'] = scientificName elif pd.isna(row['taxon_id']) and pd.isna(row['scientific_name']): sys.exit( From b91b8c2e12d767e4ad6c61dc77d4e8a2d45a12c8 Mon Sep 17 00:00:00 2001 From: bedroesb Date: Fri, 29 Nov 2024 10:56:24 +0100 Subject: [PATCH 02/21] test out new workflow --- .github/workflows/python-app.yml | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 0792e14..2647d33 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -10,25 +10,29 @@ on: branches: [ master ] jobs: + build: - + strategy: + matrix: + version: [3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13] runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.8 - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: ${{ matrix.version }} - name: Install dependencies run: | python -m pip install --upgrade pip - pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Lint with flake8 + pip install . + - name: Test base functionality tool run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - + ena-upload-cli --help + - name: Create credentials file + run: | + echo "username: ${{ secrets.ENA_USERNAME }}" >> .secrets.yaml + echo "password: ${{ secrets.ENA_PASSWORD }}" >> .secrets.yml + - name: Test submission in --draft mode + run: | + ena-upload-cli --action add --draft --dev --center ${{ secrets.ENA_CENTER }} --data example_data/*gz --checklist ERC000033 --secret .secret.yml --xlsx example_tables/ENA_excel_example_ERC000033.xlsx From 70005ed72c6260ab5b740b4c4a81603c12e1f354 Mon Sep 17 00:00:00 2001 From: bedroesb Date: Fri, 29 Nov 2024 11:01:54 +0100 Subject: [PATCH 03/21] fix versions --- .github/workflows/python-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 2647d33..13a3b92 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -14,7 +14,7 @@ jobs: build: strategy: matrix: - version: [3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13] + version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From f2c5819b805eedd8a8aea5279fdde4d9d8a11071 Mon Sep 17 00:00:00 2001 From: bedroesb Date: Fri, 29 Nov 2024 11:04:26 +0100 Subject: [PATCH 04/21] drop support for python v3.8 --- .github/workflows/python-app.yml | 2 +- README.md | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 13a3b92..cedf2f0 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -14,7 +14,7 @@ jobs: build: strategy: matrix: - version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/README.md b/README.md index a2af5fc..dd7722b 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ After a successful submission, new tsv tables will be generated with the ENA acc ## Tool dependencies -* python 3.7+ including following packages: +* python 3.8+ including following packages: * Genshi * lxml * pandas diff --git a/setup.py b/setup.py index c50e215..ce11b92 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ classifiers=[ "Operating System :: OS Independent" ], - python_requires='>=3.7', + python_requires='>=3.8', entry_points={ 'console_scripts': ["ena-upload-cli=ena_upload.ena_upload:main"] }, From d39db40a557fcf53d37d3c9123732a733ae0ca36 Mon Sep 17 00:00:00 2001 From: bedroesb Date: Fri, 29 Nov 2024 11:14:07 +0100 Subject: [PATCH 05/21] add 3.13 support --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ce11b92..7b6a80f 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ from setuptools import setup +from setuptools import find_packages from ena_upload._version import __version__ with open("README.md", 'r') as f: @@ -15,7 +16,7 @@ author="Dilmurat Yusuf", author_email="bjoern.gruening@gmail.com", long_description_content_type='text/markdown', - packages=['ena_upload', 'ena_upload.json_parsing'], + packages= find_packages(), package_dir={'ena_upload': 'ena_upload'}, package_data={ 'ena_upload': ['templates/*.xml', 'templates/*.xsd', 'json_parsing/json_schemas/*.json'] From 5c5e503eece887df2379300b98db0fc4e328433d Mon Sep 17 00:00:00 2001 From: bedroesb Date: Fri, 29 Nov 2024 11:17:24 +0100 Subject: [PATCH 06/21] update lxml --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 636bdcf..58b6f70 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ genshi==0.7.* -lxml>=4.9.3, <= 5.0.0 +lxml==5.3.0 pandas>=2.0.3 , <= 3.0.0 # pyyaml set to v5.3.* to prevent problems with setuptools upon installation, as described here: https://github.com/yaml/pyyaml/issues/723#issuecomment-1638560401 pyyaml==5.3.* From 3a4751656a371744dd2b9595ae0b31446882c332 Mon Sep 17 00:00:00 2001 From: bedroesb Date: Fri, 29 Nov 2024 11:23:18 +0100 Subject: [PATCH 07/21] test for more operating systems --- .github/workflows/python-app.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index cedf2f0..c97ec98 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -14,8 +14,9 @@ jobs: build: strategy: matrix: + os: [ubuntu-latest, macos-latest, windows-latest] version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - name: Set up Python From bc72fb983315ba25256d1a380bfff1e57a5488f7 Mon Sep 17 00:00:00 2001 From: bedroesb Date: Fri, 29 Nov 2024 11:26:18 +0100 Subject: [PATCH 08/21] add windows support in tests --- .github/workflows/python-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index c97ec98..830b6bf 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -36,4 +36,4 @@ jobs: echo "password: ${{ secrets.ENA_PASSWORD }}" >> .secrets.yml - name: Test submission in --draft mode run: | - ena-upload-cli --action add --draft --dev --center ${{ secrets.ENA_CENTER }} --data example_data/*gz --checklist ERC000033 --secret .secret.yml --xlsx example_tables/ENA_excel_example_ERC000033.xlsx + ena-upload-cli --action add --draft --dev --center ${{ secrets.ENA_CENTER }} --data ENA_TEST1.R1.fastq.gz ENA_TEST2.R1.fastq.gz ENA_TEST2.R2.fastq.gz --checklist ERC000033 --secret .secret.yml --xlsx example_tables/ENA_excel_example_ERC000033.xlsx From 7eefb57effa5e04b481624d2d2a74ce80bc9405e Mon Sep 17 00:00:00 2001 From: bedroesb Date: Fri, 29 Nov 2024 11:27:53 +0100 Subject: [PATCH 09/21] add relative path --- .github/workflows/python-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 830b6bf..29b04ec 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -36,4 +36,4 @@ jobs: echo "password: ${{ secrets.ENA_PASSWORD }}" >> .secrets.yml - name: Test submission in --draft mode run: | - ena-upload-cli --action add --draft --dev --center ${{ secrets.ENA_CENTER }} --data ENA_TEST1.R1.fastq.gz ENA_TEST2.R1.fastq.gz ENA_TEST2.R2.fastq.gz --checklist ERC000033 --secret .secret.yml --xlsx example_tables/ENA_excel_example_ERC000033.xlsx + ena-upload-cli --action add --draft --dev --center ${{ secrets.ENA_CENTER }} --data example_data/ENA_TEST1.R1.fastq.gz example_data/ENA_TEST2.R1.fastq.gz example_data/ENA_TEST2.R2.fastq.gz --checklist ERC000033 --secret .secret.yml --xlsx example_tables/ENA_excel_example_ERC000033.xlsx From 0cd2c5d04483c72056fe33cbaead2c17cc0c81e5 Mon Sep 17 00:00:00 2001 From: bedroesb Date: Fri, 29 Nov 2024 11:33:15 +0100 Subject: [PATCH 10/21] update pyyaml --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 58b6f70..ec44469 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ genshi==0.7.* lxml==5.3.0 pandas>=2.0.3 , <= 3.0.0 # pyyaml set to v5.3.* to prevent problems with setuptools upon installation, as described here: https://github.com/yaml/pyyaml/issues/723#issuecomment-1638560401 -pyyaml==5.3.* +pyyaml==6.0.* requests>=2.31.0 , <= 3.0.0 openpyxl>=3.1.2 , <= 4.0.0 jsonschema>=4.19.1 From ed08d8956a317d46b6851312b41901abc6435410 Mon Sep 17 00:00:00 2001 From: bedroesb Date: Sat, 7 Dec 2024 13:20:51 +0100 Subject: [PATCH 11/21] dtype fix --- ena_upload/ena_upload.py | 2102 ++++++++++---------- ena_upload/templates/ENA_template_runs.xml | 80 +- 2 files changed, 1093 insertions(+), 1089 deletions(-) mode change 100755 => 100644 ena_upload/ena_upload.py mode change 100755 => 100644 ena_upload/templates/ENA_template_runs.xml diff --git a/ena_upload/ena_upload.py b/ena_upload/ena_upload.py old mode 100755 new mode 100644 index 7e9eb78..717f063 --- a/ena_upload/ena_upload.py +++ b/ena_upload/ena_upload.py @@ -1,1049 +1,1053 @@ -#! /usr/bin/env python -__authors__ = ["Dilmurat Yusuf", "Bert Droesbeke"] -__copyright__ = "Copyright 2020, Dilmurat Yusuf" -__maintainer__ = "Bert Droesbeke" -__email__ = "bedro@psb.vib-ugent.be" -__license__ = "MIT" - -import os -import sys -import argparse -import yaml -import hashlib -import ftplib -import requests -import json -import uuid -import numpy as np -import re -from genshi.template import TemplateLoader -from lxml import etree -import pandas as pd -import tempfile -from ena_upload._version import __version__ -from ena_upload.check_remote import remote_check -from ena_upload.json_parsing.ena_submission import EnaSubmission - - -SCHEMA_TYPES = ['study', 'experiment', 'run', 'sample'] - -STATUS_CHANGES = {'ADD': 'ADDED', 'MODIFY': 'MODIFIED', - 'CANCEL': 'CANCELLED', 'RELEASE': 'RELEASED'} - -class MyFTP_TLS(ftplib.FTP_TLS): - """Explicit FTPS, with shared TLS session""" - - def ntransfercmd(self, cmd, rest=None): - conn, size = ftplib.FTP.ntransfercmd(self, cmd, rest) - if self._prot_p: - conn = self.context.wrap_socket(conn, - server_hostname=self.host, - session=self.sock.session) - # fix reuse of ssl socket: - # https://stackoverflow.com/a/53456626/10971151 and - # https://stackoverflow.com/a/70830916/10971151 - def custom_unwrap(): - pass - conn.unwrap = custom_unwrap - return conn, size - - -def create_dataframe(schema_tables, action, dev, auto_action): - '''create pandas dataframe from the tables in schema_tables - and return schema_dataframe - - schema_tables - a dictionary with schema:table - schema -- run, experiment, sample, study - table -- file path - - schema_dataframe - a dictionary with schema:dataframe - schema -- run, experiment, sample, study - dataframe -- pandas dataframe - ''' - - schema_dataframe = {} - - for schema, table in schema_tables.items(): - df = pd.read_csv(table, sep='\t', comment='#', dtype=str, na_values=["NA", "Na", "na", "NaN"]) - df = df.dropna(how='all') - df = check_columns(df, schema, action, dev, auto_action) - schema_dataframe[schema] = df - - return schema_dataframe - - -def extract_targets(action, schema_dataframe): - ''' extract targeted rows in dataframe tagged by action and - return schema_targets - - action - ADD, MODIFY, CANCEL, RELEASE - - schema_dataframe/schema_targets - a dictionary with schema:dataframe - schema -- run, experiment, sample, study - dataframe -- pandas dataframe - ''' - schema_targets = {} - - for schema, dataframe in schema_dataframe.items(): - filtered = dataframe.query(f'status=="{action}"') - if not filtered.empty: - schema_targets[schema] = filtered - - return schema_targets - - -def check_columns(df, schema, action, dev, auto_action): - # checking for optional columns and if not present, adding them - print(f"Check if all required columns are present in the {schema} table.") - if schema == 'sample': - optional_columns = ['accession', 'submission_date', - 'status', 'scientific_name', 'taxon_id'] - elif schema == 'run': - optional_columns = ['accession', - 'submission_date', 'status', 'file_checksum'] - else: - optional_columns = ['accession', 'submission_date', 'status'] - - for header in optional_columns: - if not header in df.columns: - if header == 'status': - if auto_action: - for index, row in df.iterrows(): - remote_present = np.nan - try: - remote_present = remote_check( - schema, str(df['alias'][index]), dev) - - except Exception as e: - print(e) - print( - f"Something went wrong with detecting the ENA object {df['alias'][index]} on the servers of ENA. This object will be skipped.") - if remote_present and action == 'MODIFY': - df.at[index, header] = action - print( - f"\t'{df['alias'][index]}' gets '{action}' as action in the status column") - elif not remote_present and action in ['ADD', 'CANCEL', 'RELEASE']: - df.at[index, header] = action - print( - f"\t'{df['alias'][index]}' gets '{action}' as action in the status column") - else: - df.at[index, header] = np.nan - print( - f"\t'{df['alias'][index]}' gets skipped since it is already present at ENA") - - else: - # status column contain action keywords - # for xml rendering, keywords require uppercase - # according to scheme definition of submission - df[header] = str(action).upper() - else: - df[header] = np.nan - else: - if header == 'status': - df[header] = df[header].str.upper() - - return df - - -def check_filenames(file_paths, run_df): - """Compare data filenames from command line and from RUN table. - - :param file_paths: a dictionary of filename string and file_path string - :param df: dataframe built from RUN table - """ - - cmd_input = file_paths.keys() - table_input = run_df['file_name'].values - - # symmetric difference between two sets - difference = set(cmd_input) ^ set(table_input) - - if difference: - msg = f"different file names between command line and RUN table: {difference}" - sys.exit(msg) - - -def check_file_checksum(df): - '''Return True if 'file_checksum' series contains valid MD5 sums''' - - regex_valid_md5sum = re.compile('^[a-f0-9]{32}$') - - def _is_str_md5sum(x): - if pd.notna(x): - match = regex_valid_md5sum.fullmatch(x) - if match: - return True - else: - return False - - s = df.file_checksum.apply(_is_str_md5sum) - - return s.all() - - -def validate_xml(xsd, xml): - ''' - validate xml against xsd scheme - ''' - - xmlschema_doc = etree.parse(source=xsd) - xmlschema = etree.XMLSchema(xmlschema_doc) - - doc = etree.XML(xml) - - return xmlschema.assertValid(doc) - - -def generate_stream(schema, targets, Template, center, tool): - ''' generate stream from Template cache - - :param schema: ENA objects -- run, experiment, sample, study - :param targets: the pandas dataframe of the run targets - :param Template: Template cache genrated by TemplateLoader - in genshi - :param center: center name used to register ENA Webin - :param tool: dict of tool_name and tool_version , by default ena-upload-cli - - :return: stream - ''' - - if schema == 'run': - # These attributes are required for rendering - # the run xml templates - # Adding backwards compatibility for file_format - if 'file_format' in targets: - targets.rename(columns={'file_format': 'file_type'}, inplace=True) - file_attrib = ['file_name', 'file_type', 'file_checksum'] - other_attrib = ['alias', 'experiment_alias'] - run_groups = targets[other_attrib].groupby(targets['alias']) - run_groups = run_groups.experiment_alias.unique() - file_groups = targets[file_attrib].groupby(targets['alias']) - - # param in generate() determined by the setup in template - stream = Template.generate(run_groups=run_groups, - file_groups=file_groups, - center=center, - tool_name=tool['tool_name'], - tool_version=tool['tool_version']) - else: - stream = Template.generate( - df=targets, center=center, tool_name=tool['tool_name'], tool_version=tool['tool_version']) - - return stream - - -def construct_xml(schema, stream, xsd): - '''construct XML for ENA submission - - :param xsd: the schema definition in - ftp://ftp.sra.ebi.ac.uk/meta/xsd/ - - :return: the file name of XML for ENA submission - ''' - - xml_string = stream.render(method='xml', encoding='utf-8') - - validate_xml(xsd, xml_string) - - xml_file = os.path.join(tempfile.gettempdir(), - schema + '_' + str(uuid.uuid4()) + '.xml') - with open(xml_file, 'w') as fw: - fw.write(xml_string.decode("utf-8")) - - print(f'wrote {xml_file}') - - return xml_file - - -def actors(template_path, checklist): - ''':return: the filenames of schema definitions and templates - ''' - - def add_path(dic, path): - for schema, filename in dic.items(): - dic[schema] = f'{path}/{filename}' - return dic - - xsds = {'run': 'SRA.run.xsd', - 'experiment': 'SRA.experiment.xsd', - 'submission': 'SRA.submission.xsd', - 'sample': 'SRA.sample.xsd', - 'study': 'SRA.study.xsd'} - - templates = {'run': 'ENA_template_runs.xml', - 'experiment': 'ENA_template_experiments.xml', - 'submission': 'ENA_template_submission.xml', - 'sample': f'ENA_template_samples_{checklist}.xml', - 'study': 'ENA_template_studies.xml'} - - xsds = add_path(xsds, template_path) - - return xsds, templates - - -def run_construct(template_path, schema_targets, center, checklist, tool): - '''construct XMLs for schema in schema_targets - - :param schema_targets: dictionary of 'schema:targets' generated - by extract_targets() - :param loader: object of TemplateLoader in genshi - :param center: center name used to register ENA Webin - :param tool: dict of tool_name and tool_version , by default ena-upload-cli - :param checklist: parameter to select a specific sample checklist - - :return schema_xmls: dictionary of 'schema:filename' - ''' - - xsds, templates = actors(template_path, checklist) - - schema_xmls = {} - - loader = TemplateLoader(search_path=template_path) - for schema, targets in schema_targets.items(): - template = templates[schema] - Template = loader.load(template) - stream = generate_stream(schema, targets, Template, center, tool) - print(f"Constructing XML for '{schema}' schema") - schema_xmls[schema] = construct_xml(schema, stream, xsds[schema]) - - return schema_xmls - - -def construct_submission(template_path, action, submission_input, center, checklist, tool): - '''construct XML for submission - - :param action: action for submission - - :param submission_input: schema_xmls or schema_targets depending on action - ADD/MODIFY: schema_xmls - CANCEL/RELEASE: schema_targets - :param loader: object of TemplateLoader in genshi - :param center: center name used to register ENA Webin - :param tool: tool name, by default ena-upload-cli - :param checklist: parameter to select a specific sample checklist - - :return submission_xml: filename of submission XML - ''' - - print(f"Constructing XML for submission schema") - - xsds, templates = actors(template_path, checklist) - - template = templates['submission'] - loader = TemplateLoader(search_path=template_path) - Template = loader.load(template) - - stream = Template.generate(action=action, input=submission_input, - center=center, tool_name=tool['tool_name'], tool_version=tool['tool_version']) - submission_xml = construct_xml('submission', stream, xsds['submission']) - - return submission_xml - - -def get_md5(filepath): - '''calculate the MD5 hash of file - - :param filepath: file path - - :return: md5 hash - ''' - - md5sum = hashlib.md5() - - with open(filepath, "rb") as fr: - while True: - # the MD5 digest block - chunk = fr.read(128) - - if not chunk: - break - - md5sum.update(chunk) - - return md5sum.hexdigest() - - -def get_taxon_id(scientific_name): - """Get taxon ID for input scientific_name. - - :param scientific_name: scientific name of sample that distinguishes - its taxonomy - :return taxon_id: NCBI taxonomy identifier - """ - # endpoint for taxonomy id - url = 'http://www.ebi.ac.uk/ena/taxonomy/rest/scientific-name' - session = requests.Session() - session.trust_env = False - # url encoding: space -> %20 - scientific_name = '%20'.join(scientific_name.strip().split()) - r = session.get(f"{url}/{scientific_name}") - try: - taxon_id = r.json()[0]['taxId'] - return taxon_id - except ValueError: - msg = f'Oops, no taxon ID available for {scientific_name}. Is it a valid scientific name?' - sys.exit(msg) - - -def get_scientific_name(taxon_id): - """Get scientific name for input taxon_id. - - :param taxon_id: NCBI taxonomy identifier - :return scientific_name: scientific name of sample that distinguishes its taxonomy - """ - # endpoint for scientific name - url = 'http://www.ebi.ac.uk/ena/taxonomy/rest/tax-id' - session = requests.Session() - session.trust_env = False - r = session.get(f"{url}/{str(taxon_id).strip()}") - try: - taxon_id = r.json()['scientificName'] - return taxon_id - except ValueError: - msg = f'Oops, no scientific name available for {taxon_id}. Is it a valid taxon_id?' - sys.exit(msg) - - -def submit_data(file_paths, password, webin_id): - """Submit data to webin ftp server. - - :param file_paths: a dictionary of filename string and file_path string - :param args: the command-line arguments parsed by ArgumentParser - """ - ftp_host = "webin2.ebi.ac.uk" - - print("\nConnecting to ftp.webin2.ebi.ac.uk....") - try: - ftps = MyFTP_TLS(timeout=120) - ftps.context.set_ciphers('HIGH:!DH:!aNULL') - ftps.connect(ftp_host, port=21) - ftps.auth() - ftps.login(webin_id, password) - ftps.prot_p() - - except IOError as ioe: - print(ioe) - sys.exit("ERROR: could not connect to the ftp server.\ - Please check your login details.") - for filename, path in file_paths.items(): - print(f'uploading {path}') - try: - print(ftps.storbinary(f'STOR {filename}', open(path, 'rb'))) - except BaseException as err: - print(f"ERROR: {err}") - print("ERROR: If your connection times out at this stage, it probably is because of a firewall that is in place. FTP is used in passive mode and connection will be opened to one of the ports: 40000 and 50000.") - raise - print(ftps.quit()) - - -def columns_to_update(df): - ''' - returns the column names where contains the cells to update - used after processing the receipt xmls - ''' - return df[df.apply(lambda x: x == 'to_update')].dropna(axis=1, how='all').columns - - -def send_schemas(schema_xmls, url, webin_id, password): - '''submit compiled XML files to the given ENA server - return the receipt object after the submission. - - schema_xmls -- dictionary of schema and the corresponding XML file name - e.g. {'submission':'submission.xml', - 'study':'study.xml', - 'run':'run.xml', - 'sample':'sample.xml', - 'experiment':'experiment.xml'} - - url -- ENA servers - test: - https://wwwdev.ebi.ac.uk/ena/submit/drop-box/submit/?auth=ENA - production: - https://www.ebi.ac.uk/ena/submit/drop-box/submit/?auth=ENA - - webin_id -- ENA webin ID of user - password -- ENA password of user - ''' - - sources = [(schema.upper(), (source, open(source, 'rb'))) - for schema, source in schema_xmls.items()] - session = requests.Session() - session.trust_env = False - r = session.post(f"{url}", - auth=(webin_id, password), - files=sources) - return r - - -def process_receipt(receipt, action): - '''Process submission receipt from ENA. - - :param receipt: a string of XML - - :return schema_update: a dictionary - {schema:update} - schema: a string - 'study', 'sample', - 'run', 'experiment' - update: a dataframe with columns - 'alias', - 'accession', 'submission_date' - ''' - receipt_root = etree.fromstring(receipt) - - success = receipt_root.get('success') - - if success == 'true': - print('Submission was done successfully') - else: - errors = [] - for element in receipt_root.findall('MESSAGES/ERROR'): - error = element.text - errors.append(error) - errors = '\nOops:\n' + '\n'.join(errors) - sys.exit(errors) - - def make_update(update, ena_type): - update_list = [] - print(f"\n{ena_type.capitalize()} accession details:") - for element in update: - extract = (element.get('alias'), element.get( - 'accession'), receiptDate, STATUS_CHANGES[action]) - print("\t".join(extract)) - update_list.append(extract) - # used for labelling dataframe - labels = ['alias', 'accession', 'submission_date', 'status'] - df = pd.DataFrame.from_records(update_list, columns=labels) - return df - - receiptDate = receipt_root.get('receiptDate') - schema_update = {} # schema as key, dataframe as value - if action in ['ADD', 'MODIFY']: - study_update = receipt_root.findall('STUDY') - sample_update = receipt_root.findall('SAMPLE') - experiment_update = receipt_root.findall('EXPERIMENT') - run_update = receipt_root.findall('RUN') - - if study_update: - schema_update['study'] = make_update(study_update, 'study') - - if sample_update: - schema_update['sample'] = make_update(sample_update, 'sample') - - if experiment_update: - schema_update['experiment'] = make_update( - experiment_update, 'experiment') - - if run_update: - schema_update['run'] = make_update(run_update, 'run') - return schema_update - - # release does have the accession numbers that are released in the recipe - elif action == 'RELEASE': - receipt_info = {} - infoblocks = receipt_root.findall('MESSAGES/INFO') - for element in infoblocks: - match = re.search('(.+?) accession "(.+?)"', element.text) - if match and match.group(1) in receipt_info: - receipt_info[match.group(1)].append(match.group(2)) - elif match and match.group(1) not in receipt_info: - receipt_info[match.group(1)] = [match.group(2)] - for ena_type, accessions in receipt_info.items(): - print(f"\n{ena_type.capitalize()} accession details:") - update_list = [] - for accession in accessions: - extract = (accession, receiptDate, STATUS_CHANGES[action]) - update_list.append(extract) - print("\t".join(extract)) - - -def update_table(schema_dataframe, schema_targets, schema_update): - """Update schema_dataframe with info in schema_targets/update. - - :param schema_dataframe: a dictionary - {schema:dataframe} - :param_targets: a dictionary - {schema:targets} - :param schema_update: a dictionary - {schema:update} - - 'schema' -- a string - 'study', 'sample','run', 'experiment' - 'dataframe' -- a pandas dataframe created from the input tables - 'targets' -- a filtered dataframe with 'action' keywords - contains updated columns - md5sum and taxon_id - 'update' -- a dataframe with updated columns - 'alias', 'accession', - 'submission_date' - - :return schema_dataframe: a dictionary - {schema:dataframe} - dataframe -- updated accession, status, - submission_date, - md5sum, taxon_id - """ - - for schema in schema_update.keys(): - dataframe = schema_dataframe[schema] - targets = schema_targets[schema] - update = schema_update[schema] - - dataframe.set_index('alias', inplace=True) - targets.set_index('alias', inplace=True) - update.set_index('alias', inplace=True) - - for index in update.index: - dataframe.loc[index, 'accession'] = update.loc[index, 'accession'] - dataframe.loc[index, - 'submission_date'] = update.loc[index, 'submission_date'] - dataframe.loc[index, 'status'] = update.loc[index, 'status'] - - if schema == 'sample': - dataframe.loc[index, - 'taxon_id'] = targets.loc[index, 'taxon_id'] - elif schema == 'run': - # Since index is set to alias - # then targets of run can have multiple rows with - # identical index because each row has a file - # which is associated with a run - # and a run can have multiple files. - # The following assignment assumes 'targets' retain - # the original row order in 'dataframe' - # because targets was initially subset of 'dataframe'. - dataframe.loc[index, - 'file_checksum'] = targets.loc[index, 'file_checksum'] - - return schema_dataframe - - -def update_table_simple(schema_dataframe, schema_targets, action): - """Update schema_dataframe with info in schema_targets. - - :param schema_dataframe: a dictionary - {schema:dataframe} - :param_targets: a dictionary - {schema:targets} - - 'schema' -- a string - 'study', 'sample','run', 'experiment' - 'dataframe' -- a pandas dataframe created from the input tables - 'targets' -- a filtered dataframe with 'action' keywords - contains updated columns - md5sum and taxon_id - - :return schema_dataframe: a dictionary - {schema:dataframe} - dataframe -- updated status - """ - - for schema in schema_targets.keys(): - dataframe = schema_dataframe[schema] - targets = schema_targets[schema] - - dataframe.set_index('alias', inplace=True) - targets.set_index('alias', inplace=True) - - for index in targets.index: - dataframe.loc[index, 'status'] = STATUS_CHANGES[action] - - return schema_dataframe - - -def save_update(schema_tables, schema_dataframe): - """Write updated dataframe to tsv file. - - :param schema_tables_: a dictionary - {schema:file_path} - :param schema_dataframe_: a dictionary - {schema:dataframe} - :schema: a string - 'study', 'sample', 'run', 'experiment' - :file_path: a string - :dataframe: a dataframe - """ - - print('\nSaving updates in new tsv tables:') - for schema in schema_tables: - table = schema_tables[schema] - dataframe = schema_dataframe[schema] - - file_name, file_extension = os.path.splitext(table) - update_name = f'{file_name}_updated{file_extension}' - dataframe.to_csv(update_name, sep='\t') - print(f'{update_name}') - - -class SmartFormatter(argparse.HelpFormatter): - '''subclass the HelpFormatter and provide a special intro - for the options that should be handled "raw" ("R|rest of help"). - - :adapted from: Anthon's code at - https://stackoverflow.com/questions/3853722/python-argparse-how-to-insert-newline-in-the-help-text - ''' - - def _split_lines(self, text, width): - if text.startswith('R|'): - return text[2:].splitlines() - # this is the RawTextHelpFormatter._split_lines - return argparse.HelpFormatter._split_lines(self, text, width) - - -def process_args(): - '''parse command-line arguments - ''' - - parser = argparse.ArgumentParser(prog='ena-upoad-cli', - description='''The program makes submission - of data and respective metadata to European - Nucleotide Archive (ENA). The metadate - should be provided in separate tables - corresponding the ENA objects -- STUDY, - SAMPLE, EXPERIMENT and RUN.''', - formatter_class=SmartFormatter) - parser.add_argument('--version', action='version', - version='%(prog)s '+__version__) - - parser.add_argument('--action', - choices=['add', 'modify', 'cancel', 'release'], - required=True, - help='R| add: add an object to the archive\n' - ' modify: modify an object in the archive\n' - ' cancel: cancel a private object and its dependent objects\n' - ' release: release a private object immediately to public') - - parser.add_argument('--study', - help='table of STUDY object') - - parser.add_argument('--sample', - help='table of SAMPLE object') - - parser.add_argument('--experiment', - help='table of EXPERIMENT object') - - parser.add_argument('--run', - help='table of RUN object') - - parser.add_argument('--data', - nargs='*', - help='data for submission, this can be a list of files', - metavar='FILE') - - parser.add_argument('--center', - dest='center_name', - required=True, - help='specific to your Webin account') - - parser.add_argument('--checklist', help="specify the sample checklist with following pattern: ERC0000XX, Default: ERC000011", dest='checklist', - default='ERC000011') - - parser.add_argument('--xlsx', - help='filled in excel template with metadata') - - parser.add_argument('--isa_json', - help='BETA: ISA json describing describing the ENA objects') - - parser.add_argument('--isa_assay_stream', - nargs='*', - help='BETA: specify the assay stream(s) that holds the ENA information, this can be a list of assay streams') - - parser.add_argument('--auto_action', - action="store_true", - default=False, - help='BETA: detect automatically which action (add or modify) to apply when the action column is not given') - - parser.add_argument('--tool', - dest='tool_name', - default='ena-upload-cli', - help='specify the name of the tool this submission is done with. Default: ena-upload-cli') - - parser.add_argument('--tool_version', - dest='tool_version', - default=__version__, - help='specify the version of the tool this submission is done with') - - parser.add_argument('--no_data_upload', - default=False, - action="store_true", - help='indicate if no upload should be performed and you like to submit a RUN object (e.g. if uploaded was done separately).') - - parser.add_argument('--draft', - default=False, - action="store_true", - help='indicate if no submission should be performed') - - parser.add_argument('--secret', - required=True, - help='.secret.yml file containing the password and Webin ID of your ENA account') - - parser.add_argument( - '-d', '--dev', help="flag to use the dev/sandbox endpoint of ENA", action="store_true") - - args = parser.parse_args() - - # check if any table is given - tables = set([args.study, args.sample, args.experiment, args.run]) - if tables == {None} and not args.xlsx and not args.isa_json: - parser.error('Requires at least one table for submission') - - # check if .secret file exists - if args.secret: - if not os.path.isfile(args.secret): - msg = f"Oops, the file {args.secret} does not exist" - parser.error(msg) - - # check if xlsx file exists - if args.xlsx: - if not os.path.isfile(args.xlsx): - msg = f"Oops, the file {args.xlsx} does not exist" - parser.error(msg) - - # check if ISA json file exists - if args.isa_json: - if not os.path.isfile(args.isa_json): - msg = f"Oops, the file {args.isa_json} does not exist" - parser.error(msg) - if args.isa_assay_stream is None : - parser.error("--isa_json requires --isa_assay_stream") - - # check if data is given when adding a 'run' table - if (not args.no_data_upload and args.run and args.action.upper() not in ['RELEASE', 'CANCEL']) or (not args.no_data_upload and args.xlsx and args.action.upper() not in ['RELEASE', 'CANCEL']): - if args.data is None: - parser.error('Oops, requires data for submitting RUN object') - - else: - # validate if given data is file - for path in args.data: - if not os.path.isfile(path): - msg = f"Oops, the file {path} does not exist" - parser.error(msg) - - return args - - -def collect_tables(args): - '''collect the schema whose table is not None - - :param args: the command-line arguments parsed by ArgumentParser - :return schema_tables: a dictionary of schema string and file path string - ''' - - schema_tables = {'study': args.study, 'sample': args.sample, - 'experiment': args.experiment, 'run': args.run} - - schema_tables = {schema: table for schema, table in schema_tables.items() - if table is not None} - - return schema_tables - - -def update_date(date): - if pd.isnull(date) or isinstance(date, str): - return date - try: - return date.strftime('%Y-%m-%d') - except AttributeError: - return date - except Exception: - raise - - -def main(): - args = process_args() - action = args.action.upper() - center = args.center_name - tool = {'tool_name': args.tool_name, 'tool_version': args.tool_version} - dev = args.dev - checklist = args.checklist - secret = args.secret - draft = args.draft - xlsx = args.xlsx - isa_json_file = args.isa_json - isa_assay_stream = args.isa_assay_stream - auto_action = args.auto_action - - with open(secret, 'r') as secret_file: - credentials = yaml.load(secret_file, Loader=yaml.FullLoader) - - password = credentials['password'].strip() - webin_id = credentials['username'].strip() - - if not password or not webin_id: - print( - f"Oops, file {args.secret} does not contain a password or username") - secret_file.close() - - if xlsx: - # create dataframe from xlsx table - xl_workbook = pd.ExcelFile(xlsx) - schema_dataframe = {} # load the parsed data in a dict: sheet_name -> pandas_frame - schema_tables = {} - - for schema in SCHEMA_TYPES: - if schema in xl_workbook.book.sheetnames: - xl_sheet = xl_workbook.parse(schema, header=0, na_values=["NA", "Na", "na", "NaN"]) - elif f"ENA_{schema}" in xl_workbook.book.sheetnames: - xl_sheet = xl_workbook.parse(f"ENA_{schema}", header=0, na_values=["NA", "Na", "na", "NaN"]) - else: - sys.exit( - f"The sheet '{schema}' is not present in the excel sheet {xlsx}") - xl_sheet = xl_sheet.drop(0).dropna(how='all') - for column_name in list(xl_sheet.columns.values): - if 'date' in column_name: - xl_sheet[column_name] = xl_sheet[column_name].apply( - update_date) - - if True in xl_sheet.columns.duplicated(): - sys.exit("Duplicated columns found") - - xl_sheet = check_columns( - xl_sheet, schema, action, dev, auto_action) - schema_dataframe[schema] = xl_sheet - path = os.path.dirname(os.path.abspath(xlsx)) - schema_tables[schema] = f"{path}/ENA_template_{schema}.tsv" - elif isa_json_file: - # Read json file - with open(isa_json_file, 'r') as json_file: - isa_json = json.load(json_file) - - schema_tables = {} - schema_dataframe = {} - required_assays = [] - for stream in isa_assay_stream: - required_assays.append({"assay_stream": stream}) - submission = EnaSubmission.from_isa_json(isa_json, required_assays) - submission_dataframes = submission.generate_dataframes() - for schema, df in submission_dataframes.items(): - schema_dataframe[schema] = check_columns( - df, schema, action, dev, auto_action) - path = os.path.dirname(os.path.abspath(isa_json_file)) - schema_tables[schema] = f"{path}/ENA_template_{schema}.tsv" - - - else: - # collect the schema with table input from command-line - schema_tables = collect_tables(args) - - # create dataframe from table - schema_dataframe = create_dataframe( - schema_tables, action, dev, auto_action) - - # ? add a function to sanitize characters - # ? print 'validate table for specific action' - # ? print 'catch error' - - # extract rows tagged by action - # these rows are the targets for submission - schema_targets = extract_targets(action, schema_dataframe) - - if not schema_targets: - sys.exit( - f"There is no table submitted having at least one row with {action} as action in the status column.") - - if action == 'ADD': - # when adding run object - # update schema_targets wit md5 hash - # submit data - if 'run' in schema_targets: - # a dictionary of filename:file_path - df = schema_targets['run'] - file_paths = {} - if args.data: - for path in args.data: - file_paths[os.path.basename(path)] = os.path.abspath(path) - # check if file names identical between command line and table - # if not, system exits - check_filenames(file_paths, df) - - # generate MD5 sum if not supplied in table - if file_paths and not check_file_checksum(df): - print("No valid checksums found, generate now...", end=" ") - file_md5 = {filename: get_md5(path) for filename, path - in file_paths.items()} - - # update schema_targets wih md5 hash - md5 = df['file_name'].apply(lambda x: file_md5[x]).values - # SettingWithCopyWarning causes false positive - # e.g at df.loc[:, 'file_checksum'] = md5 - pd.options.mode.chained_assignment = None - df['file_checksum'] = df['file_checksum'].astype('string') - df.loc[:, 'file_checksum'] = md5 - print("done.") - elif check_file_checksum(df): - print("Valid checksums found", end=" ") - else: - sys.exit("No valid checksums found and no files given to generate checksum from. Please list the files using the --data option or specify the checksums in the run-table when the data is uploaded separately.") - - schema_targets['run'] = df - - # submit data to webin ftp server - if args.no_data_upload: - print( - "No files will be uploaded, remove `--no_data_upload' argument to perform upload.") - elif draft: - print( - "No files will be uploaded, remove `--draft' argument to perform upload.") - else: - submit_data(file_paths, password, webin_id) - - # when adding sample - # update schema_targets with taxon ids or scientific names - if 'sample' in schema_targets: - df = schema_targets['sample'] - print('Retrieving taxon IDs and scientific names if needed') - for index, row in df.iterrows(): - if pd.notna(row['scientific_name']) and pd.isna(row['taxon_id']): - # retrieve taxon id using scientific name - taxonID = get_taxon_id(row['scientific_name']) - df['taxon_id'] = df['taxon_id'].astype('string') - df.loc[index, 'taxon_id'] = taxonID - elif pd.notna(row['taxon_id']) and pd.isna(row['scientific_name']): - # retrieve scientific name using taxon id - scientificName = get_scientific_name(row['taxon_id']) - df['scientific_name'] = df['scientific_name'].astype('string') - df.loc[index, 'scientific_name'] = scientificName - elif pd.isna(row['taxon_id']) and pd.isna(row['scientific_name']): - sys.exit( - f"No taxon_id or scientific_name was given with sample {row['alias']}.") - print('Taxon IDs and scientific names are retrieved') - schema_targets['sample'] = df - - # ? need to add a place holder for setting up - base_path = os.path.abspath(os.path.dirname(__file__)) - template_path = os.path.join(base_path, 'templates') - if action in ['ADD', 'MODIFY']: - # when ADD/MODIFY, - # requires source XMLs for 'run', 'experiment', 'sample', 'experiment' - # schema_xmls record XMLs for all these schema and following 'submission' - schema_xmls = run_construct( - template_path, schema_targets, center, checklist, tool) - - submission_xml = construct_submission(template_path, action, - schema_xmls, center, checklist, tool) - - elif action in ['CANCEL', 'RELEASE']: - # when CANCEL/RELEASE, only accessions needed - # schema_xmls only used to record the following 'submission' - schema_xmls = {} - - submission_xml = construct_submission(template_path, action, - schema_targets, center, checklist, tool) - - else: - print(f"The action {action} is not supported.") - schema_xmls['submission'] = submission_xml - - if draft: - print("No submission will be performed, remove `--draft' argument to perform submission.") - else: - if dev: - url = 'https://wwwdev.ebi.ac.uk/ena/submit/drop-box/submit/?auth=ENA' - else: - url = 'https://www.ebi.ac.uk/ena/submit/drop-box/submit/?auth=ENA' - - print(f'\nSubmitting XMLs to ENA server: {url}') - receipt = send_schemas(schema_xmls, url, webin_id, password).text - print("Printing receipt to ./receipt.xml") - with open('receipt.xml', 'w') as fw: - fw.write(receipt) - try: - schema_update = process_receipt(receipt.encode("utf-8"), action) - except ValueError: - print("There was an ERROR during submission:") - sys.exit(receipt) - - if action in ['ADD', 'MODIFY'] and not draft: - schema_dataframe = update_table(schema_dataframe, - schema_targets, - schema_update) - else: - schema_dataframe = update_table_simple(schema_dataframe, - schema_targets, - action) - # save updates in new tables - save_update(schema_tables, schema_dataframe) - - -if __name__ == "__main__": - main() +#! /usr/bin/env python +__authors__ = ["Dilmurat Yusuf", "Bert Droesbeke"] +__copyright__ = "Copyright 2020, Dilmurat Yusuf" +__maintainer__ = "Bert Droesbeke" +__email__ = "bert.droesbeke@vib.be" +__license__ = "MIT" + +import os +import sys +import argparse +import yaml +import hashlib +import ftplib +import requests +import json +import uuid +import numpy as np +import re +from genshi.template import TemplateLoader +from lxml import etree +import pandas as pd +import tempfile +from ena_upload._version import __version__ +from ena_upload.check_remote import remote_check +from ena_upload.json_parsing.ena_submission import EnaSubmission + + +SCHEMA_TYPES = ['study', 'experiment', 'run', 'sample'] + +STATUS_CHANGES = {'ADD': 'ADDED', 'MODIFY': 'MODIFIED', + 'CANCEL': 'CANCELLED', 'RELEASE': 'RELEASED'} + +class MyFTP_TLS(ftplib.FTP_TLS): + """Explicit FTPS, with shared TLS session""" + + def ntransfercmd(self, cmd, rest=None): + conn, size = ftplib.FTP.ntransfercmd(self, cmd, rest) + if self._prot_p: + conn = self.context.wrap_socket(conn, + server_hostname=self.host, + session=self.sock.session) + # fix reuse of ssl socket: + # https://stackoverflow.com/a/53456626/10971151 and + # https://stackoverflow.com/a/70830916/10971151 + def custom_unwrap(): + pass + conn.unwrap = custom_unwrap + return conn, size + + +def create_dataframe(schema_tables, action, dev, auto_action): + '''create pandas dataframe from the tables in schema_tables + and return schema_dataframe + + schema_tables - a dictionary with schema:table + schema -- run, experiment, sample, study + table -- file path + + schema_dataframe - a dictionary with schema:dataframe + schema -- run, experiment, sample, study + dataframe -- pandas dataframe + ''' + + schema_dataframe = {} + + for schema, table in schema_tables.items(): + # Read with string dtype for specific columns + dtype_dict = {'scientific_name': 'str', 'file_checksum': 'str'} + df = pd.read_csv(table, sep='\t', comment='#', dtype=dtype_dict, na_values=["NA", "Na", "na", "NaN"]) + df = df.dropna(how='all') + df = check_columns(df, schema, action, dev, auto_action) + schema_dataframe[schema] = df + + return schema_dataframe + +def extract_targets(action, schema_dataframe): + ''' extract targeted rows in dataframe tagged by action and + return schema_targets + + action - ADD, MODIFY, CANCEL, RELEASE + + schema_dataframe/schema_targets - a dictionary with schema:dataframe + schema -- run, experiment, sample, study + dataframe -- pandas dataframe + ''' + schema_targets = {} + + for schema, dataframe in schema_dataframe.items(): + filtered = dataframe.query(f'status=="{action}"') + if not filtered.empty: + schema_targets[schema] = filtered + + return schema_targets + + +def check_columns(df, schema, action, dev, auto_action): + # checking for optional columns and if not present, adding them + print(f"Check if all required columns are present in the {schema} table.") + if schema == 'sample': + optional_columns = ['accession', 'submission_date', + 'status', 'scientific_name', 'taxon_id'] + # Ensure string dtype for scientific_name + if 'scientific_name' not in df.columns: + df['scientific_name'] = pd.Series(dtype='str') + elif schema == 'run': + optional_columns = ['accession', + 'submission_date', 'status', 'file_checksum'] + # Ensure string dtype for file_checksum + if 'file_checksum' not in df.columns: + df['file_checksum'] = pd.Series(dtype='str') + else: + optional_columns = ['accession', 'submission_date', 'status'] + + for header in optional_columns: + if header not in df.columns: + if header == 'status': + if auto_action: + for index, row in df.iterrows(): + remote_present = np.nan + try: + remote_present = remote_check( + schema, str(df['alias'][index]), dev) + + except Exception as e: + print(e) + print( + f"Something went wrong with detecting the ENA object {df['alias'][index]} on the servers of ENA. This object will be skipped.") + if remote_present and action == 'MODIFY': + df.at[index, header] = action + print( + f"\t'{df['alias'][index]}' gets '{action}' as action in the status column") + elif not remote_present and action in ['ADD', 'CANCEL', 'RELEASE']: + df.at[index, header] = action + print( + f"\t'{df['alias'][index]}' gets '{action}' as action in the status column") + else: + df.at[index, header] = np.nan + print( + f"\t'{df['alias'][index]}' gets skipped since it is already present at ENA") + + else: + # status column contain action keywords + # for xml rendering, keywords require uppercase + # according to scheme definition of submission + df[header] = str(action).upper() + else: + df[header] = pd.Series(dtype='str') # Initialize as string type + else: + if header == 'status': + df[header] = df[header].str.upper() + + return df + + +def check_filenames(file_paths, run_df): + """Compare data filenames from command line and from RUN table. + + :param file_paths: a dictionary of filename string and file_path string + :param df: dataframe built from RUN table + """ + + cmd_input = file_paths.keys() + table_input = run_df['file_name'].values + + # symmetric difference between two sets + difference = set(cmd_input) ^ set(table_input) + + if difference: + msg = f"different file names between command line and RUN table: {difference}" + sys.exit(msg) + + +def check_file_checksum(df): + '''Return True if 'file_checksum' series contains valid MD5 sums''' + + regex_valid_md5sum = re.compile('^[a-f0-9]{32}$') + + def _is_str_md5sum(x): + if pd.notna(x): + match = regex_valid_md5sum.fullmatch(x) + if match: + return True + else: + return False + + s = df.file_checksum.apply(_is_str_md5sum) + + return s.all() + + +def validate_xml(xsd, xml): + ''' + validate xml against xsd scheme + ''' + + xmlschema_doc = etree.parse(source=xsd) + xmlschema = etree.XMLSchema(xmlschema_doc) + + doc = etree.XML(xml) + + return xmlschema.assertValid(doc) + + +def generate_stream(schema, targets, Template, center, tool): + ''' generate stream from Template cache + + :param schema: ENA objects -- run, experiment, sample, study + :param targets: the pandas dataframe of the run targets + :param Template: Template cache genrated by TemplateLoader + in genshi + :param center: center name used to register ENA Webin + :param tool: dict of tool_name and tool_version , by default ena-upload-cli + + :return: stream + ''' + + if schema == 'run': + # These attributes are required for rendering + # the run xml templates + # Adding backwards compatibility for file_format + if 'file_format' in targets: + targets.rename(columns={'file_format': 'file_type'}, inplace=True) + file_attrib = ['file_name', 'file_type', 'file_checksum'] + other_attrib = ['alias', 'experiment_alias'] + run_groups = targets[other_attrib].groupby(targets['alias']) + run_groups = run_groups.experiment_alias.unique() + file_groups = targets[file_attrib].groupby(targets['alias']) + + # param in generate() determined by the setup in template + stream = Template.generate(run_groups=run_groups, + file_groups=file_groups, + center=center, + tool_name=tool['tool_name'], + tool_version=tool['tool_version']) + else: + stream = Template.generate( + df=targets, center=center, tool_name=tool['tool_name'], tool_version=tool['tool_version']) + + return stream + + +def construct_xml(schema, stream, xsd): + '''construct XML for ENA submission + + :param xsd: the schema definition in + ftp://ftp.sra.ebi.ac.uk/meta/xsd/ + + :return: the file name of XML for ENA submission + ''' + + xml_string = stream.render(method='xml', encoding='utf-8') + + validate_xml(xsd, xml_string) + + xml_file = os.path.join(tempfile.gettempdir(), + schema + '_' + str(uuid.uuid4()) + '.xml') + with open(xml_file, 'w') as fw: + fw.write(xml_string.decode("utf-8")) + + print(f'wrote {xml_file}') + + return xml_file + + +def actors(template_path, checklist): + ''':return: the filenames of schema definitions and templates + ''' + + def add_path(dic, path): + for schema, filename in dic.items(): + dic[schema] = f'{path}/{filename}' + return dic + + xsds = {'run': 'SRA.run.xsd', + 'experiment': 'SRA.experiment.xsd', + 'submission': 'SRA.submission.xsd', + 'sample': 'SRA.sample.xsd', + 'study': 'SRA.study.xsd'} + + templates = {'run': 'ENA_template_runs.xml', + 'experiment': 'ENA_template_experiments.xml', + 'submission': 'ENA_template_submission.xml', + 'sample': f'ENA_template_samples_{checklist}.xml', + 'study': 'ENA_template_studies.xml'} + + xsds = add_path(xsds, template_path) + + return xsds, templates + + +def run_construct(template_path, schema_targets, center, checklist, tool): + '''construct XMLs for schema in schema_targets + + :param schema_targets: dictionary of 'schema:targets' generated + by extract_targets() + :param loader: object of TemplateLoader in genshi + :param center: center name used to register ENA Webin + :param tool: dict of tool_name and tool_version , by default ena-upload-cli + :param checklist: parameter to select a specific sample checklist + + :return schema_xmls: dictionary of 'schema:filename' + ''' + + xsds, templates = actors(template_path, checklist) + + schema_xmls = {} + + loader = TemplateLoader(search_path=template_path) + for schema, targets in schema_targets.items(): + template = templates[schema] + Template = loader.load(template) + stream = generate_stream(schema, targets, Template, center, tool) + print(f"Constructing XML for '{schema}' schema") + schema_xmls[schema] = construct_xml(schema, stream, xsds[schema]) + + return schema_xmls + + +def construct_submission(template_path, action, submission_input, center, checklist, tool): + '''construct XML for submission + + :param action: action for submission - + :param submission_input: schema_xmls or schema_targets depending on action + ADD/MODIFY: schema_xmls + CANCEL/RELEASE: schema_targets + :param loader: object of TemplateLoader in genshi + :param center: center name used to register ENA Webin + :param tool: tool name, by default ena-upload-cli + :param checklist: parameter to select a specific sample checklist + + :return submission_xml: filename of submission XML + ''' + + print(f"Constructing XML for submission schema") + + xsds, templates = actors(template_path, checklist) + + template = templates['submission'] + loader = TemplateLoader(search_path=template_path) + Template = loader.load(template) + + stream = Template.generate(action=action, input=submission_input, + center=center, tool_name=tool['tool_name'], tool_version=tool['tool_version']) + submission_xml = construct_xml('submission', stream, xsds['submission']) + + return submission_xml + + +def get_md5(filepath): + '''calculate the MD5 hash of file + + :param filepath: file path + + :return: md5 hash + ''' + + md5sum = hashlib.md5() + + with open(filepath, "rb") as fr: + while True: + # the MD5 digest block + chunk = fr.read(128) + + if not chunk: + break + + md5sum.update(chunk) + + return md5sum.hexdigest() + + +def get_taxon_id(scientific_name): + """Get taxon ID for input scientific_name. + + :param scientific_name: scientific name of sample that distinguishes + its taxonomy + :return taxon_id: NCBI taxonomy identifier + """ + # endpoint for taxonomy id + url = 'http://www.ebi.ac.uk/ena/taxonomy/rest/scientific-name' + session = requests.Session() + session.trust_env = False + # url encoding: space -> %20 + scientific_name = '%20'.join(scientific_name.strip().split()) + r = session.get(f"{url}/{scientific_name}") + try: + taxon_id = r.json()[0]['taxId'] + return taxon_id + except ValueError: + msg = f'Oops, no taxon ID available for {scientific_name}. Is it a valid scientific name?' + sys.exit(msg) + + +def get_scientific_name(taxon_id): + """Get scientific name for input taxon_id. + + :param taxon_id: NCBI taxonomy identifier + :return scientific_name: scientific name of sample that distinguishes its taxonomy + """ + # endpoint for scientific name + url = 'http://www.ebi.ac.uk/ena/taxonomy/rest/tax-id' + session = requests.Session() + session.trust_env = False + r = session.get(f"{url}/{str(taxon_id).strip()}") + try: + taxon_id = r.json()['scientificName'] + return taxon_id + except ValueError: + msg = f'Oops, no scientific name available for {taxon_id}. Is it a valid taxon_id?' + sys.exit(msg) + + +def submit_data(file_paths, password, webin_id): + """Submit data to webin ftp server. + + :param file_paths: a dictionary of filename string and file_path string + :param args: the command-line arguments parsed by ArgumentParser + """ + ftp_host = "webin2.ebi.ac.uk" + + print("\nConnecting to ftp.webin2.ebi.ac.uk....") + try: + ftps = MyFTP_TLS(timeout=120) + ftps.context.set_ciphers('HIGH:!DH:!aNULL') + ftps.connect(ftp_host, port=21) + ftps.auth() + ftps.login(webin_id, password) + ftps.prot_p() + + except IOError as ioe: + print(ioe) + sys.exit("ERROR: could not connect to the ftp server.\ + Please check your login details.") + for filename, path in file_paths.items(): + print(f'uploading {path}') + try: + print(ftps.storbinary(f'STOR {filename}', open(path, 'rb'))) + except BaseException as err: + print(f"ERROR: {err}") + print("ERROR: If your connection times out at this stage, it probably is because of a firewall that is in place. FTP is used in passive mode and connection will be opened to one of the ports: 40000 and 50000.") + raise + print(ftps.quit()) + + +def columns_to_update(df): + ''' + returns the column names where contains the cells to update + used after processing the receipt xmls + ''' + return df[df.apply(lambda x: x == 'to_update')].dropna(axis=1, how='all').columns + + +def send_schemas(schema_xmls, url, webin_id, password): + '''submit compiled XML files to the given ENA server + return the receipt object after the submission. + + schema_xmls -- dictionary of schema and the corresponding XML file name + e.g. {'submission':'submission.xml', + 'study':'study.xml', + 'run':'run.xml', + 'sample':'sample.xml', + 'experiment':'experiment.xml'} + + url -- ENA servers + test: + https://wwwdev.ebi.ac.uk/ena/submit/drop-box/submit/?auth=ENA + production: + https://www.ebi.ac.uk/ena/submit/drop-box/submit/?auth=ENA + + webin_id -- ENA webin ID of user + password -- ENA password of user + ''' + + sources = [(schema.upper(), (source, open(source, 'rb'))) + for schema, source in schema_xmls.items()] + session = requests.Session() + session.trust_env = False + r = session.post(f"{url}", + auth=(webin_id, password), + files=sources) + return r + + +def process_receipt(receipt, action): + '''Process submission receipt from ENA. + + :param receipt: a string of XML + + :return schema_update: a dictionary - {schema:update} + schema: a string - 'study', 'sample', + 'run', 'experiment' + update: a dataframe with columns - 'alias', + 'accession', 'submission_date' + ''' + receipt_root = etree.fromstring(receipt) + + success = receipt_root.get('success') + + if success == 'true': + print('Submission was done successfully') + else: + errors = [] + for element in receipt_root.findall('MESSAGES/ERROR'): + error = element.text + errors.append(error) + errors = '\nOops:\n' + '\n'.join(errors) + sys.exit(errors) + + def make_update(update, ena_type): + update_list = [] + print(f"\n{ena_type.capitalize()} accession details:") + for element in update: + extract = (element.get('alias'), element.get( + 'accession'), receiptDate, STATUS_CHANGES[action]) + print("\t".join(extract)) + update_list.append(extract) + # used for labelling dataframe + labels = ['alias', 'accession', 'submission_date', 'status'] + df = pd.DataFrame.from_records(update_list, columns=labels) + return df + + receiptDate = receipt_root.get('receiptDate') + schema_update = {} # schema as key, dataframe as value + if action in ['ADD', 'MODIFY']: + study_update = receipt_root.findall('STUDY') + sample_update = receipt_root.findall('SAMPLE') + experiment_update = receipt_root.findall('EXPERIMENT') + run_update = receipt_root.findall('RUN') + + if study_update: + schema_update['study'] = make_update(study_update, 'study') + + if sample_update: + schema_update['sample'] = make_update(sample_update, 'sample') + + if experiment_update: + schema_update['experiment'] = make_update( + experiment_update, 'experiment') + + if run_update: + schema_update['run'] = make_update(run_update, 'run') + return schema_update + + # release does have the accession numbers that are released in the recipe + elif action == 'RELEASE': + receipt_info = {} + infoblocks = receipt_root.findall('MESSAGES/INFO') + for element in infoblocks: + match = re.search('(.+?) accession "(.+?)"', element.text) + if match and match.group(1) in receipt_info: + receipt_info[match.group(1)].append(match.group(2)) + elif match and match.group(1) not in receipt_info: + receipt_info[match.group(1)] = [match.group(2)] + for ena_type, accessions in receipt_info.items(): + print(f"\n{ena_type.capitalize()} accession details:") + update_list = [] + for accession in accessions: + extract = (accession, receiptDate, STATUS_CHANGES[action]) + update_list.append(extract) + print("\t".join(extract)) + + +def update_table(schema_dataframe, schema_targets, schema_update): + """Update schema_dataframe with info in schema_targets/update. + + :param schema_dataframe: a dictionary - {schema:dataframe} + :param_targets: a dictionary - {schema:targets} + :param schema_update: a dictionary - {schema:update} + + 'schema' -- a string - 'study', 'sample','run', 'experiment' + 'dataframe' -- a pandas dataframe created from the input tables + 'targets' -- a filtered dataframe with 'action' keywords + contains updated columns - md5sum and taxon_id + 'update' -- a dataframe with updated columns - 'alias', 'accession', + 'submission_date' + + :return schema_dataframe: a dictionary - {schema:dataframe} + dataframe -- updated accession, status, + submission_date, + md5sum, taxon_id + """ + + for schema in schema_update.keys(): + dataframe = schema_dataframe[schema] + targets = schema_targets[schema] + update = schema_update[schema] + + dataframe.set_index('alias', inplace=True) + targets.set_index('alias', inplace=True) + update.set_index('alias', inplace=True) + + for index in update.index: + dataframe.loc[index, 'accession'] = update.loc[index, 'accession'] + dataframe.loc[index, + 'submission_date'] = update.loc[index, 'submission_date'] + dataframe.loc[index, 'status'] = update.loc[index, 'status'] + + if schema == 'sample': + dataframe.loc[index, + 'taxon_id'] = targets.loc[index, 'taxon_id'] + elif schema == 'run': + # Since index is set to alias + # then targets of run can have multiple rows with + # identical index because each row has a file + # which is associated with a run + # and a run can have multiple files. + # The following assignment assumes 'targets' retain + # the original row order in 'dataframe' + # because targets was initially subset of 'dataframe'. + dataframe.loc[index, + 'file_checksum'] = targets.loc[index, 'file_checksum'] + + return schema_dataframe + + +def update_table_simple(schema_dataframe, schema_targets, action): + """Update schema_dataframe with info in schema_targets. + + :param schema_dataframe: a dictionary - {schema:dataframe} + :param_targets: a dictionary - {schema:targets} + + 'schema' -- a string - 'study', 'sample','run', 'experiment' + 'dataframe' -- a pandas dataframe created from the input tables + 'targets' -- a filtered dataframe with 'action' keywords + contains updated columns - md5sum and taxon_id + + :return schema_dataframe: a dictionary - {schema:dataframe} + dataframe -- updated status + """ + + for schema in schema_targets.keys(): + dataframe = schema_dataframe[schema] + targets = schema_targets[schema] + + dataframe.set_index('alias', inplace=True) + targets.set_index('alias', inplace=True) + + for index in targets.index: + dataframe.loc[index, 'status'] = STATUS_CHANGES[action] + + return schema_dataframe + + +def save_update(schema_tables, schema_dataframe): + """Write updated dataframe to tsv file. + + :param schema_tables_: a dictionary - {schema:file_path} + :param schema_dataframe_: a dictionary - {schema:dataframe} + :schema: a string - 'study', 'sample', 'run', 'experiment' + :file_path: a string + :dataframe: a dataframe + """ + + print('\nSaving updates in new tsv tables:') + for schema in schema_tables: + table = schema_tables[schema] + dataframe = schema_dataframe[schema] + + file_name, file_extension = os.path.splitext(table) + update_name = f'{file_name}_updated{file_extension}' + dataframe.to_csv(update_name, sep='\t') + print(f'{update_name}') + + +class SmartFormatter(argparse.HelpFormatter): + '''subclass the HelpFormatter and provide a special intro + for the options that should be handled "raw" ("R|rest of help"). + + :adapted from: Anthon's code at + https://stackoverflow.com/questions/3853722/python-argparse-how-to-insert-newline-in-the-help-text + ''' + + def _split_lines(self, text, width): + if text.startswith('R|'): + return text[2:].splitlines() + # this is the RawTextHelpFormatter._split_lines + return argparse.HelpFormatter._split_lines(self, text, width) + + +def process_args(): + '''parse command-line arguments + ''' + + parser = argparse.ArgumentParser(prog='ena-upoad-cli', + description='''The program makes submission + of data and respective metadata to European + Nucleotide Archive (ENA). The metadate + should be provided in separate tables + corresponding the ENA objects -- STUDY, + SAMPLE, EXPERIMENT and RUN.''', + formatter_class=SmartFormatter) + parser.add_argument('--version', action='version', + version='%(prog)s '+__version__) + + parser.add_argument('--action', + choices=['add', 'modify', 'cancel', 'release'], + required=True, + help='R| add: add an object to the archive\n' + ' modify: modify an object in the archive\n' + ' cancel: cancel a private object and its dependent objects\n' + ' release: release a private object immediately to public') + + parser.add_argument('--study', + help='table of STUDY object') + + parser.add_argument('--sample', + help='table of SAMPLE object') + + parser.add_argument('--experiment', + help='table of EXPERIMENT object') + + parser.add_argument('--run', + help='table of RUN object') + + parser.add_argument('--data', + nargs='*', + help='data for submission, this can be a list of files', + metavar='FILE') + + parser.add_argument('--center', + dest='center_name', + required=True, + help='specific to your Webin account') + + parser.add_argument('--checklist', help="specify the sample checklist with following pattern: ERC0000XX, Default: ERC000011", dest='checklist', + default='ERC000011') + + parser.add_argument('--xlsx', + help='filled in excel template with metadata') + + parser.add_argument('--isa_json', + help='BETA: ISA json describing describing the ENA objects') + + parser.add_argument('--isa_assay_stream', + nargs='*', + help='BETA: specify the assay stream(s) that holds the ENA information, this can be a list of assay streams') + + parser.add_argument('--auto_action', + action="store_true", + default=False, + help='BETA: detect automatically which action (add or modify) to apply when the action column is not given') + + parser.add_argument('--tool', + dest='tool_name', + default='ena-upload-cli', + help='specify the name of the tool this submission is done with. Default: ena-upload-cli') + + parser.add_argument('--tool_version', + dest='tool_version', + default=__version__, + help='specify the version of the tool this submission is done with') + + parser.add_argument('--no_data_upload', + default=False, + action="store_true", + help='indicate if no upload should be performed and you like to submit a RUN object (e.g. if uploaded was done separately).') + + parser.add_argument('--draft', + default=False, + action="store_true", + help='indicate if no submission should be performed') + + parser.add_argument('--secret', + required=True, + help='.secret.yml file containing the password and Webin ID of your ENA account') + + parser.add_argument( + '-d', '--dev', help="flag to use the dev/sandbox endpoint of ENA", action="store_true") + + args = parser.parse_args() + + # check if any table is given + tables = set([args.study, args.sample, args.experiment, args.run]) + if tables == {None} and not args.xlsx and not args.isa_json: + parser.error('Requires at least one table for submission') + + # check if .secret file exists + if args.secret: + if not os.path.isfile(args.secret): + msg = f"Oops, the file {args.secret} does not exist" + parser.error(msg) + + # check if xlsx file exists + if args.xlsx: + if not os.path.isfile(args.xlsx): + msg = f"Oops, the file {args.xlsx} does not exist" + parser.error(msg) + + # check if ISA json file exists + if args.isa_json: + if not os.path.isfile(args.isa_json): + msg = f"Oops, the file {args.isa_json} does not exist" + parser.error(msg) + if args.isa_assay_stream is None : + parser.error("--isa_json requires --isa_assay_stream") + + # check if data is given when adding a 'run' table + if (not args.no_data_upload and args.run and args.action.upper() not in ['RELEASE', 'CANCEL']) or (not args.no_data_upload and args.xlsx and args.action.upper() not in ['RELEASE', 'CANCEL']): + if args.data is None: + parser.error('Oops, requires data for submitting RUN object') + + else: + # validate if given data is file + for path in args.data: + if not os.path.isfile(path): + msg = f"Oops, the file {path} does not exist" + parser.error(msg) + + return args + + +def collect_tables(args): + '''collect the schema whose table is not None + + :param args: the command-line arguments parsed by ArgumentParser + :return schema_tables: a dictionary of schema string and file path string + ''' + + schema_tables = {'study': args.study, 'sample': args.sample, + 'experiment': args.experiment, 'run': args.run} + + schema_tables = {schema: table for schema, table in schema_tables.items() + if table is not None} + + return schema_tables + + +def update_date(date): + if pd.isnull(date) or isinstance(date, str): + return date + try: + return date.strftime('%Y-%m-%d') + except AttributeError: + return date + except Exception: + raise + + +def main(): + args = process_args() + action = args.action.upper() + center = args.center_name + tool = {'tool_name': args.tool_name, 'tool_version': args.tool_version} + dev = args.dev + checklist = args.checklist + secret = args.secret + draft = args.draft + xlsx = args.xlsx + isa_json_file = args.isa_json + isa_assay_stream = args.isa_assay_stream + auto_action = args.auto_action + + with open(secret, 'r') as secret_file: + credentials = yaml.load(secret_file, Loader=yaml.FullLoader) + + password = credentials['password'].strip() + webin_id = credentials['username'].strip() + + if not password or not webin_id: + print( + f"Oops, file {args.secret} does not contain a password or username") + secret_file.close() + + if xlsx: + # create dataframe from xlsx table + xl_workbook = pd.ExcelFile(xlsx) + schema_dataframe = {} # load the parsed data in a dict: sheet_name -> pandas_frame + schema_tables = {} + + for schema in SCHEMA_TYPES: + if schema in xl_workbook.book.sheetnames: + xl_sheet = xl_workbook.parse(schema, header=0, na_values=["NA", "Na", "na", "NaN"]) + elif f"ENA_{schema}" in xl_workbook.book.sheetnames: + xl_sheet = xl_workbook.parse(f"ENA_{schema}", header=0, na_values=["NA", "Na", "na", "NaN"]) + else: + sys.exit( + f"The sheet '{schema}' is not present in the excel sheet {xlsx}") + xl_sheet = xl_sheet.drop(0).dropna(how='all') + for column_name in list(xl_sheet.columns.values): + if 'date' in column_name: + xl_sheet[column_name] = xl_sheet[column_name].apply( + update_date) + + if True in xl_sheet.columns.duplicated(): + sys.exit("Duplicated columns found") + + xl_sheet = check_columns( + xl_sheet, schema, action, dev, auto_action) + schema_dataframe[schema] = xl_sheet + path = os.path.dirname(os.path.abspath(xlsx)) + schema_tables[schema] = f"{path}/ENA_template_{schema}.tsv" + elif isa_json_file: + # Read json file + with open(isa_json_file, 'r') as json_file: + isa_json = json.load(json_file) + + schema_tables = {} + schema_dataframe = {} + required_assays = [] + for stream in isa_assay_stream: + required_assays.append({"assay_stream": stream}) + submission = EnaSubmission.from_isa_json(isa_json, required_assays) + submission_dataframes = submission.generate_dataframes() + for schema, df in submission_dataframes.items(): + schema_dataframe[schema] = check_columns( + df, schema, action, dev, auto_action) + path = os.path.dirname(os.path.abspath(isa_json_file)) + schema_tables[schema] = f"{path}/ENA_template_{schema}.tsv" + + + else: + # collect the schema with table input from command-line + schema_tables = collect_tables(args) + + # create dataframe from table + schema_dataframe = create_dataframe( + schema_tables, action, dev, auto_action) + + # ? add a function to sanitize characters + # ? print 'validate table for specific action' + # ? print 'catch error' + + # extract rows tagged by action + # these rows are the targets for submission + schema_targets = extract_targets(action, schema_dataframe) + + if not schema_targets: + sys.exit( + f"There is no table submitted having at least one row with {action} as action in the status column.") + + if action == 'ADD': + # when adding run object + # update schema_targets wit md5 hash + # submit data + if 'run' in schema_targets: + # a dictionary of filename:file_path + df = schema_targets['run'] + file_paths = {} + if args.data: + for path in args.data: + file_paths[os.path.basename(path)] = os.path.abspath(path) + # check if file names identical between command line and table + # if not, system exits + check_filenames(file_paths, df) + + # generate MD5 sum if not supplied in table + if file_paths and not check_file_checksum(df): + print("No valid checksums found, generate now...", end=" ") + file_md5 = {filename: get_md5(path) for filename, path + in file_paths.items()} + + # update schema_targets wih md5 hash + md5 = df['file_name'].apply(lambda x: file_md5[x]).values + # SettingWithCopyWarning causes false positive + # e.g at df.loc[:, 'file_checksum'] = md5 + pd.options.mode.chained_assignment = None + df.loc[:, 'file_checksum'] = md5 + print("done.") + elif check_file_checksum(df): + print("Valid checksums found", end=" ") + else: + sys.exit("No valid checksums found and no files given to generate checksum from. Please list the files using the --data option or specify the checksums in the run-table when the data is uploaded separately.") + + schema_targets['run'] = df + + # submit data to webin ftp server + if args.no_data_upload: + print( + "No files will be uploaded, remove `--no_data_upload' argument to perform upload.") + elif draft: + print( + "No files will be uploaded, remove `--draft' argument to perform upload.") + else: + submit_data(file_paths, password, webin_id) + + # when adding sample + # update schema_targets with taxon ids or scientific names + if 'sample' in schema_targets: + df = schema_targets['sample'] + print('Retrieving taxon IDs and scientific names if needed') + for index, row in df.iterrows(): + if pd.notna(row['scientific_name']) and pd.isna(row['taxon_id']): + # retrieve taxon id using scientific name + taxonID = get_taxon_id(row['scientific_name']) + df.loc[index, 'taxon_id'] = taxonID + elif pd.notna(row['taxon_id']) and pd.isna(row['scientific_name']): + # retrieve scientific name using taxon id + scientificName = get_scientific_name(row['taxon_id']) + df.loc[index, 'scientific_name'] = scientificName + elif pd.isna(row['taxon_id']) and pd.isna(row['scientific_name']): + sys.exit( + f"No taxon_id or scientific_name was given with sample {row['alias']}.") + print('Taxon IDs and scientific names are retrieved') + schema_targets['sample'] = df + + # ? need to add a place holder for setting up + base_path = os.path.abspath(os.path.dirname(__file__)) + template_path = os.path.join(base_path, 'templates') + if action in ['ADD', 'MODIFY']: + # when ADD/MODIFY, + # requires source XMLs for 'run', 'experiment', 'sample', 'experiment' + # schema_xmls record XMLs for all these schema and following 'submission' + schema_xmls = run_construct( + template_path, schema_targets, center, checklist, tool) + + submission_xml = construct_submission(template_path, action, + schema_xmls, center, checklist, tool) + + elif action in ['CANCEL', 'RELEASE']: + # when CANCEL/RELEASE, only accessions needed + # schema_xmls only used to record the following 'submission' + schema_xmls = {} + + submission_xml = construct_submission(template_path, action, + schema_targets, center, checklist, tool) + + else: + print(f"The action {action} is not supported.") + schema_xmls['submission'] = submission_xml + + if draft: + print("No submission will be performed, remove `--draft' argument to perform submission.") + else: + if dev: + url = 'https://wwwdev.ebi.ac.uk/ena/submit/drop-box/submit/?auth=ENA' + else: + url = 'https://www.ebi.ac.uk/ena/submit/drop-box/submit/?auth=ENA' + + print(f'\nSubmitting XMLs to ENA server: {url}') + receipt = send_schemas(schema_xmls, url, webin_id, password).text + print("Printing receipt to ./receipt.xml") + with open('receipt.xml', 'w') as fw: + fw.write(receipt) + try: + schema_update = process_receipt(receipt.encode("utf-8"), action) + except ValueError: + print("There was an ERROR during submission:") + sys.exit(receipt) + + if action in ['ADD', 'MODIFY'] and not draft: + schema_dataframe = update_table(schema_dataframe, + schema_targets, + schema_update) + else: + schema_dataframe = update_table_simple(schema_dataframe, + schema_targets, + action) + # save updates in new tables + save_update(schema_tables, schema_dataframe) + + +if __name__ == "__main__": + main() diff --git a/ena_upload/templates/ENA_template_runs.xml b/ena_upload/templates/ENA_template_runs.xml old mode 100755 new mode 100644 index 134d017..583db0a --- a/ena_upload/templates/ENA_template_runs.xml +++ b/ena_upload/templates/ENA_template_runs.xml @@ -1,40 +1,40 @@ - - - - - - - - - - - - - - - - - - SUBMISSION_TOOL - ${tool_name} - - - SUBMISSION_TOOL_VERSION - ${tool_version} - - - - - + + + + + + + + + + + + + + + + + + SUBMISSION_TOOL + ${tool_name} + + + SUBMISSION_TOOL_VERSION + ${tool_version} + + + + + From 5d47b99d4342ec4151d7a7035cb29cd436377fa5 Mon Sep 17 00:00:00 2001 From: bedroesb Date: Sat, 7 Dec 2024 15:16:20 +0100 Subject: [PATCH 12/21] fix + paired end xlsx example --- ena_upload/ena_upload.py | 4 ++-- ena_upload/templates/ENA_template_runs.xml | 2 +- .../ENA_excel_example_ERC000033.xlsx | Bin 31335 -> 31363 bytes 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ena_upload/ena_upload.py b/ena_upload/ena_upload.py index 717f063..5b7dbe1 100644 --- a/ena_upload/ena_upload.py +++ b/ena_upload/ena_upload.py @@ -222,8 +222,8 @@ def generate_stream(schema, targets, Template, center, tool): targets.rename(columns={'file_format': 'file_type'}, inplace=True) file_attrib = ['file_name', 'file_type', 'file_checksum'] other_attrib = ['alias', 'experiment_alias'] - run_groups = targets[other_attrib].groupby(targets['alias']) - run_groups = run_groups.experiment_alias.unique() + # Create groups with alias as index + run_groups = targets[other_attrib].groupby('alias')['experiment_alias'].first().to_dict() file_groups = targets[file_attrib].groupby(targets['alias']) # param in generate() determined by the setup in template diff --git a/ena_upload/templates/ENA_template_runs.xml b/ena_upload/templates/ENA_template_runs.xml index 583db0a..15feae7 100644 --- a/ena_upload/templates/ENA_template_runs.xml +++ b/ena_upload/templates/ENA_template_runs.xml @@ -18,7 +18,7 @@ def mandatorytest(row, column, index): - + diff --git a/example_tables/ENA_excel_example_ERC000033.xlsx b/example_tables/ENA_excel_example_ERC000033.xlsx index 85ba4ea9c0e56ce506d67c2179bb0f6691074b17..82d5949417f48b0ad8adc59aa191815796d6c15c 100644 GIT binary patch delta 18765 zcmbTcbyQqWw7wfAz`{?bWOQu3!7^4FmlDH28$ffb_2(^b<05Z7k+% zg#Wy1!NGbBSaYr3Uj3QVyjc4NZcC&+A|}&6o!_ltC9YmyUcQ4O@G9@r=Cq+YU*NjQq6u;Ywj4l~oWXQqb`S=|Gz&0||7 zM9gc4*^rTAYmMG}%NiOPRO3dk%%$6=>~%c~Z` zdor@6rpO6}!*huG%_(sUS&-YlRc0qw2<3JB+XIi_OYu2fQ(aANuguM{$gF775orH+ zo;KY+q=>2lY;yJ zJNd}5370z9RLlo~RN`9GFQeDxeGS8c$W-D{PRGUlgS`ws4S(@gOc3I1gG)L6o8x5$ ztvcA7MI$3_G*D3vDgfO3S_(4{w-n_GJ*>z!xntd zr!xa&+~^$YW>;L&;r^`-UBNJ{W>+?ZFe!GOX4hq&aH**f9l@~dq5fRtDcoqlw8>R) zR8gx6lkj!FKc$jjSX7>Ir;;NLLG+FD8U5$&&2B7uvVPm0&9hY|G6`z?Gx{JZf@mu7 zFM=H#n`i7IkP?v>hnWBPm8J;Pq34ea9|T%maVhQpH4#I-?ai~bf3xhi=-+xDc&a1# zwHvTD3V&Q4@iFQ$>XI^ulQTPS-BY7CrGk%xgTi5QCyPsRhHlFnec~|SSdXQ{v4C*J zSF}P{J|QE+_Y4}PL{B}hH`Oo=Nzow$z-@fz7~8u#{dJlVjea*2;KN4YDPM5cc29fq zcwt9Do{WWs7!WJa?G5LqhXTY^_ub3=TC%K)(4|q9Hof(O>kJRdwgu=paw*;P#z*_==n<{G<5gnH=9HX>h#iQdrt*5dQS+wAz?-H=``&l=h|c?KKl>}rFc&c?R(Dz z^?EN&+M7zYUV+qf40qz;Lqfj)mxAjEiSqVeuKp-N)kp=QWTesnnoVSY2ld9Tbd$h4 z>*%~S^5HUy%SSNhXYmk1Dq&F-^`J%>txaJWA7pH?z7J+Y=Io1m{MgS=czc>0mOiWl zmr8Xj7*XftnrVAGGr@u^9Va(r3P&wvHo@R}iH>;bYT2Wv<7=f(xurrb=H17Eqdh0h z^h8D0i<$25lNBbwD#hF5F7bG!xQI+E8M8H6pKq{txo#r9DbWkRTc()Qo=;PI5{{8h$IF@c#Un+nR^MQOVL0 zt=6pU@~h}kFVa0%GX2K<7diLg(~Km+*4v81GBc5{z4}lvvCCc;S8z@rTs{|;GT$oYXI^Q|C zeQnrOynLE15HZQ~*wIus)0A5}v+{sFDwyadl;6*7+G_k1<1RI1M{0!K)UBP=5RoEI zljqb3wDc^t3#MhWXF7pBD;`#H&OE(fdHtrb`L0;Nkvt8I;OwA{!Q#x6*`mMe!u9H^ zk?PI^S|9A7X(B@!t?)@R-|eY-**RKyCjFa=Jl$2>&j3tEv89P!RLyv`xpD=7X@HF} z@ev_pQKPayg5cO#orhJI^v@nmv#Y0FRxOl6CTK=mJTRi7e`j*j2XlUFyld4`JuTSio;_CPkb)Ho7q(ta&^k*9Z3yV50vs0xmK6|VNJ48?k?v`hVNj$sxp30$?1y%jm@}=5u z-!;3(T=V-Ym%A~p-!1J23rA!bQ4Zgj>PAlJJCd4C{Tpk$Fn=eR3@LUuUUk=hVcof2 zq}L%RTYY0m{H#jViA~E>u2tST(7Amri=P>RXJSL@{C6zUv=xz-EJ1P$U4LIV&*a9X zZQTJFyXf7so^N53ImuAJso96$_UxVuYZlJUqxr8y#^RDXy!$PI_1>eEX&oMq0Ve0Y0K1LEp)32%0&3z| zCIy_uB_sSDoW=x_*@7z=Q~yZ^r+x#=+8=b*(S6CWZ>ewB6vhR93X_sZ9qrDzYgd&t zl+M3Nue4zPnqKDYl|VJ&<_5BYI`n%h>KHzu$kB#peP|V)I{?Xll9H*_)aNl?$VZz1 z@@Ssu3!cSfFQ)5da^)^AwSu)l>|+JrX&a-=mmyyo!>3G_%FjQE3tacq@W2!cOiiqy zu*5=Z`;=gdnU>`H?>66eoT!DhptCrh_9 zQTr3|w{2-c(j;4Jc|o>~76o@Vv~zzwHIjxHegvfkzMg7igno-sU$!r-LYdQURj zl!4sXBKyWv(s~;zb`9zj(VhRw(}ttG-~#K{y&g3Eko_GNB_o1;R2^PYhs~a5HvdPC zTviv#0h%qQlK?k(ytgG`2uiUzCDL$sL?vNpO4zqW9F{E?{7RiIrlPM7n*KiCaUAj^ z=WT6%T{t>IrX(fk6xY+2LO5s&Ih&ssr6g^lK&Oa9Y@$b}NI-02L#IeVY~n?S@-i_% z<(RMmNXWcX~-h})=$&dXz9|K)w0XsKY$_y2q43&^V+{8lM&Hupvzu20n^*1i8UmX+X-kur zPxg#RL>3D>|Fl?_MBe~vZJ8tH<-x~R^+!yWs4=9Es>vOne=Y9o)Zqu-2aJ+hbHRPtWg5{y6QZj+SkvyY7P76}*(LmsP`Ec1lHEmcF94Mxoo4ru=T! zJU*4vTTRhSvC0leHrhJsE6s0%zobs`TF*eua|E({d zV~LheSL?|Lyu8u3KGEK-dDr$4le^YJx7ixa*1lSUa8S;RS+!kpmS=7EZgA`gJuS*O zqtW=#B=x;c>IleiX<46~j>1BPnflBtPKCN-uowY zLebd;-xMbYbJjr>*wukx1~OW^GzmNtrv>+>gwr$1Fuxr~>T@v(ixvLZAK%e$ON=59 z+aKvp&!uRDyOljMur~{IiT-B8NC3Q%e5IU8g61Ory2e5s@o<8@vhy1sH4S016wLJ| z-(DfM@k^*N+PvJ+-&Qn~0q%JHH@+>nNsK5K=&LiNvm(T;6{kSI3)|($lsl0VW@Tc4 zC<+(`FB7iTBs!mJWYVxe`po98l53YGYuECo3x2TV#xxR;l|0b^cbDvBb3QRE@c03~ z@?EpUtbcQ2YE~&=9SD5_TKnDk1DFCoAr&9puuVKB-{<_!C_(B*!VJK_?W+IgFU*}OV*Qs6HQ z28eZ~vX8+l2_}~C zH^&<}1XBV@=yR^^KQ?hcQOc)a%7&M$a0Tq=CPrXsVeUGgKCf%w)4v~#_HqMW6VyUk_EHQ3PY; z;qd$XMS-)Y7X@V_3{~P%1mhdwX!-mNgR?Jsfe?XEeQxRZ<}u(R(ao>u?}CYqaLi*| zp9pPV*sJ9Kgl^UR`?3*X7Th6{`dN`i^{b$UkGMZ@Q?2KkA=M&0OnTbRsyCzoD$>_X z=8<>k<=pYP&V8}6W9N;i%1hpNVLKV32G&c&XZ>pP`}+f0T+Rp(cP!%bX<$NjiJ83? zH=FOXoqaN`jG(_M3T2(gM`A7EU^+8fv7IQPFAw^q#8+IFj71h`S=Rh@uY0@r{#<%x;?wUry?xg!JV~Dk$)EV#goMe4Gg@ z)~9irVY^-DHuC~aB{xODG>qw|6h{&w39fm6-m7@ah#k-oLMRs4X|cC;cx=E%^Owr-eh? zk1-49`iQ*H&kzrVk5-T=$eMf(mSXzCymQv&3^`2UbIrWE!3kT6fY-OU9XyDI+>!`V z6?BnX5FZhL)(%jg@82*`iZnIPP(^af#=jqeGJ!|{=JyE`5b8{YnQH!#!I5UIU%J%! zKjXB5^~a`jv>LA+&daXpjO{CfTcS)E`)W^y3WNszuXhicMoT@l$RjT97`WaW3x>E1 z@R5srsB%+z(!dAygHVN97J%;t2mtyN=MP&AH`Fk2o0mQpy1A`%mZgNmJ)l3Jl`(oFzdDl;cl z$wkhD;IQz#j~QyXk9s+?0<#hh8)%JzC$;T|oyTQdNIbQ3Tg?%etd$DHLtIG48wyfSgLoDW z9=E0U>dXNUY%DHkF{SIQrsU;8lvxLv92n)`TQ*u{UTp8#>F?vgQVYkLi}|pHxEy{N zwOS0FVfHxpW3+U%{sbRxZx@Z8dH}``#T7>^qhOn(GbjcBX+}U@d+(piqSSFoaD-Ln zeQ&(_sX~yHe=A*E{1zU6n(p97{$+~At2Da|*ZGT-vjCbDhjFn@w_dkp-3je+YG3AY~Ii8p(cZcklq(iyBoisI*gV=rO^flWc02_TaXbNB-9 zAGYU(?X)M0+p`!KJ7@X_7D)@`y0j}nhoj#!z-fHq@*}Ju@t)kll~Z! zw2xZsc|@iq>S*Kur|_iF!g}{gc<=^}=RWVXr(lzn!HM(Y8%>nTN$MaY+BAEz8mw_% zOiiRpK1@x-y@)}|(slGb%J6mOJxYsp(mhJKb=Gk)^{%ylW-Ig(Nyqw2VC1|n0nq*v z*O04D+@9h;BOG%yERP?&K_C-Y=u`!8Xmpo@JOEtd)~guQ@skpE!^FA*hGFfV$W1 zzYl02s6?(4^-+Y4_a1m?wqW^KCrqgL>Bt|I=+K{2)XSEx2-Et~uQIyJ-U|VX7<|uI z7vi(M!R2b=#bzbyg)mVPy-sxHt1P5UKFW4IkHkav@+L$&^dDh=+#QNcV(}h;56=uo z!WS|1>z_zAICBu=)Da8>J@eXOSJ-oFd`LpEms<$stRXfP-vaXdBGBvx)VCraX{s^R zl;gXs-uUoVElZKOxXtCi2h6Wn%?P64t5qbd8C$DQo}Es@2M<4&sm3n-v@Y`iD*v2U zqlCmW*lWWgolYCu5~FARHHxVk<5x6p^z1r4%Y^%6@%|a7<>qj2=Q}!uddZNLF^_s{ zmA#L8mh@m>ElAM)HPVH^(~ZE>Lv5olDjYy?YH-xui}QHFKRhjSqh!CbWUtI~GdUyH zPBU`;@Kv+sw%Yqg9>Yjm+hK-%nUu=@I;t9-(}l(niP{=t{eI&_k1*rKCQ~Qy-i^A$ zTG%VQq{M6cW(%irjXBI1e06d+ecHFt>MVFAmK|*jQ9I%-T=~+G4%O7dgOpQOECM0X zh^f>T(My6Gt{3-aRRVp_gOvIqz2QP)9cp)0$+)$SX**k>edpIB!A9)EBeJxe67PgV zdlfz6lxQ!gzHH?cCFYkL=IN<-_E(97_f9+{dI-r}z5gnk$u)4N^N63@fge1t7~x(J zSC+xe`RIYB876ku7sGgU(MWhGoeA_xDW>%uY1-#bY+aqQ8+aj-9}5KXz;`IuZnKOf z?mV*@QK^2Al1OGb9La%isfai6_+ic&wKVw1T4bMpIOdGxmi+XSAf=$tT)EWV!XZFF zZSH^&6RqV@kR|RMG)roF%tF-bZ(DCOa9BU?_8_Z|v9ev?@ub#%b{@FQvqT^Mn(&P7OvrA(g?;iDL`ix=M z!c$+Kj0^A+-G8`S{U|0gN}b>Gj!kq*&a1}|lp7F{3=d7qzcOi!Tg{VixPC)a$|Nwh zH#WekR$Du0_@UTTqJ(ugWzuL4=BYa177MDKvBsM+-f07WvWZhD{xzIbu81A{IG6OBW8ZLmSkSy@@=U7qI6y=*0 z4j1anKH;SD7w*rvL5oT^Y=fq=-#>9kJ^#IMce7o(Ol{C6o%k&Q5-?NWTglPYC7XD< zy}U0pcdjDZE$#P&m8WdM8cjGK_J&2GI-DPY#4)uQG^FqFrDJEm%?am3$jPk7D^rf{ zb|o6E)wdW4bO9mqfx^d~4U-j~i~bYv{D{~@jialO&DYvieD9NI1(KD136Isb!*QF6 z)LGkk={2Uk-i608wboU^y6e47oBGE6J^ev3ul)%qOGk^xPPGu@^v+&1VuxaABhP31Oo>zcxk^4y-u$^KFE&%44 z2i%&ISb4^77u~&Ko|Zu(bFK5iiYqo-LBWZbicuJ+Yo7LXW`mT=3J;=dI`1b|7baNO z+|;bYow*CGE}Sm5&$@R@9`49d2fJjc03Kt*%QIDRxJbYs)r&+aK=b8vi=I z4?o*>zwZH3?Iy`Um&ZG^3&V|L{eKp=*`EJaYl8N~YF!&rZ+Lp`mq!L|<}+J&zN3ck z8hY2hZ}1|#nZLqZvKeF{pHN?-)Qz%oJ~|fmh9rz^!Q@g&(`+o(A_S!Sf3E``^SXdw zChMSY4U@d|Z^tcWt54X1BY zP$P=&8SMa^{rT&i*V?M=y|fG(sdk@FEnFJdJ;gZr16w^6xG)zU zRgAhnIqK%cUi#JnLtDAho!Vj{yY`-C!5=%cyU;z>cQl<(wA)O$xY`|TwzGHaSb?S5 z*`(_B-|mhzU0u^C?o4eQFU(v#q_`9nxa(cO&HkFO@J8Y`zHZyaom~JfQaF`xp4xV4 z`K#=;0;0-B$-^-QiQLlg6BWY!qwt^*QEv8Rd{rpjjGsAOVb{l!f3n-EeVbY9stWNcw5Sw9 z)8Qg~_-kHCjMmo3i9viL@CmLeI1_!~SLI0eJ2sce3O&kCM&|l|?#&9A3y9H#1 z!k0^$20adh{!ZFoe466_4uqbb!b4&f(t{Vdq$$wf$@u4i{Sl@qV!5RM7od`gNGt6h zmnT{b_J5ZrDl?t+3n&nMJDo)XQ8J&&LV$E_OaCYJ-x+H`hW^H{{#^91$hDUnAWHm$ z23^*Q_y+~LY#d@d2%RAY5l_ZnHWd+18eX!J!HQUw0-fRC8UL_6(R{E!N}j01bk=l% z=$mOEiwdFynaO$$QHtd%>T7xPpVU8$#Y`3=M2T}W7wk`*2lqXq9)xZlgUBu8ua%0( zE$yF@CtCgwBMP3*nkx`RpU$HDUp7##qI!suKbHZ0-T!97jD1rSbd zgQW-w)prACobY-}5mLX+CHc{*nn%x&h;4sk!K3d z(M0sy9fP{XJ|CEm0Y*?MSUw4zaMz%1u`dMXV~A0Z1RhO7=iN1kg7w9~e2g#vx@53? zGP=gDK?1BV3Fc#hQIHHCO-6U!HOPSVWx;&RFz8ai@+s&^y9NcYz9N{9IYvPWcr*pQ zY1g0v)>j4dvBaQD1F6^xIv74p?6o%*PgkE)6W7hEBL= zFaYZtg8A5E6r_Pi)6jYM0E2N@-z3b(5rZxrET4|9v6uDCm`x}^GCAH|R~y<~3wQc{ zR3Md5kOXU@yRSBMxEAi>{it9np%97EME9@SP*^S8&HGWIR6=2rn2GM~+R%eqxQF+n z!l{HJB$E@}m$jjHwQ$exM@3QzKapTfb|chf`9vYV?el&=mBlXO%pl~9}nYpPqkE>xxt4u^D9 zJe5#_L}{v9wJub%4vv6yR3ep7k|buT+o&!SQU^yuIx3k;C`B?k)$Ldp>Q)CwPC6=; zN+?Z&HQgOh7aCdz2T+raN~aRakSI-eCz>i%PP}^O#|Oa?gTSlsC;b4i$Ab*={WT#t z-y!f?{7DHQ_C(NnzP}p;#~%W3$e)x1VowGc6!<4YaB?B==KM)1Aof(ydVzm41g94Q zZ_l5U24YVK85H`%AUInPcz6D!3=n%JXuZ(?4uS)34zK9PpOgh+&juM3`4gGrP@BVt z@h9bg*mFVaMgDx|IAZ4Tar{YnAohHaL9xH4InH-;_!R!60uXy4Xua6q%^b(y96pCX zsR+bg3^FM3hhi)GKz+b;P(6jWDp#K;yHr24NY}f6G&3l{(Jj#GAmoxp?og`drSvIQ za4a^z^81&DM!mbtzcebe+UdF2WD^fnSVBR zOCUjCFTQ!(*PSfQ{_z&7TQPUAogF1OJ_PJ9n)XIIblXT>&>K802ZYcb$F4P6Iv!Wc zgYM1~O}M!pqrFL&L64Ier=7OECvM)bw$z)%wjZuvF>7BId%AbJ9cj%q|fs>c)7F=>yIs_t7F%gi(_fBi8)=7 z%L`{hfsV9Hz^z#;O}Ad`f|n)S+xzUh-$d%pdE&{vyXQ0SbgA`~rILR6(w1h&@nWyp z(}MkK`O@+4h3owj$(<|fR~Y!m{c8QZo2460OE%uWU$%n}{hRuYBI9k)Tx-?eLIILY@8f&c)|tcY=YJ}ru82_&ILueCqTBzyGOC3ri~Yx+JXI=m zfVKk(PmQ-or{PcS`kh~4H8lEhyotQ@hohYZ}XBm|kRfHB*aO8$Rwb zL8To=g`Lc@tvCUi7Jk^9c)rQ4Hx>t@0uM8_bxiX%`fXKq3=ZEgue9(l6Z1jM^ao#S z-M`IChV>PwB)DB~Ztw4ph3gb;YT_Q1&;#z*$2T7)a!Qmszt-%DxLq8lnufClW+NefCtlr6eNI>L?V?6ED=X+%}(cfjI5(9r@)sNX_XPR*=w?mR# zMDJC8NSYF%JCuH}F5>tGQh3cJ1yI%IHAL}%8JKG#ZrLoX$ZOF!9NSG!B(A}~{^AxJ zU%H4sK2~jwKaW$fI%+K_=q=LaEPQ zjw<={331D^)PjFVlaSQM%#FF{OPF91Yrb!Svyy+ftk$5a6bhs{I5#pA1vri6^rnsT ze$wN=0SZtIway0Rave5dL-|}rT>H><<4ordGYX2p@3hgcHIvPC7f+VFBJnU;wY$Ij z2!}ggN;qL>q%oF|Xk2!S*I*kSAJfJ{8?|Y^Ct)p|M1|7j!7HCzv#ES?s}@o#8wE(b zaBmT)rB*?vJv^Rfw${Z6;HGM)AGQYNxX91s{Qaho^gFl)Hwp*udxE2hc(;2!OY#E? zWO;z0>_B{xO*Fj{?~G3FN8>>O1(?phlg+Fa6`W#GjF%i!J(PEXAQLbUHzi&%gi$>% z^<`CBX(S^nkmuWI)Kv=+ebI{1FIw*YKJU(8mFRkt@a69TSc})yfWjM4hsgucrt7y! z1Bs4tqfhWPP3CH&?Hf+lgyspwf!RXBtm09WAq8;MAu1JY#!H~-_)z#ky9@?L{^W&y zlwO{aEb^btYM9?5uyHAfRHGXC?iN%A`TgBBX2=X3!)l~**_>1M_+uB4u}q*;3fhyu zQ5VKk1L3Wh(Fh_?fP!KWEoHRFeW$7=sT-%l=%>(xQf|b%OO4b%j-*-@Lci+-FwJ8M zz1-{Ng%o`i%y}PgCgU{xFpnki7GHVP!})^fS2^%yTr}U)-P3F%7Aki%#nS58MkUaV zG6Zh+?Nk-?qKHFn{DO-->=G#IQq%vizV&qLqa86{pz%IGNff>i(3xRvLxE4X*Rg95(2Ep9AMNeb$vrpvNq+fy#B?RwjGOjD}OY zefQi$j%$=%fV`;Jpp;`#q-ZoVi=QqC*1Q@;V7ND!?~jZEcIQ76CBOf3eR+mZJgQR8 zFaZ}n^kyv>y|enyS&Yrzv+F0|5>9{z83!5$4{ z&D@6xG5w2wr;FbZ*PgKslhr!q>dK+4YODxITtk0DddPQKsIo$vXPsiVH^yJte4i`K zuq0@XvmL@eu9s@cwZd&qowsHxQ{)rjWr#!VmuCO{*{N9Vjnk+o{G$-k zFhf#-5IwT9Aa5t!{#4$VlRodwi7pm6iA{Oe`OhVEA79YSDo10=Q2MKN^O4{vy?luKcjKz)+HdDO*|| z#TxB5i()xVwsSSIuG~d;ti0+;B8alEJ+p(0nlvFb-|LNus@}Co zg1mh-L))Nz$R{!Lv~O=qb~TO!U!)7L-Ln1m*Pgy0O*bgHIv}tN^DmP#D1?;ZvM94Ay*%#b^oc^24U8v1xNP+cX<^l+mztB~qk2Ef} z$IF?S0isR5?s3|BH_#U?{c}PK|6)kMp)%mdHH_`#>70Fp^`U;r)o- zyLc2n9^E%Uc!_eWw z^X5sROGgS%=nl{KW)tx(Dx{XCY!?no>@bM3iecuCT860$M|J4z?hojj_5JBQk- zWUT`?KjziDbx*<^#kG@G7h&J0Z)LUU1q$SdwNF^o<$P-{=yWTb3sJhYmjYRrp_TI< zN3J%uLOnT|-rW*p92{n78W&EP$Ahrjd-p1G7v+W6-h_$q!of5x<2i{_yrn$@2EJ+0Y-U3MS~tVXVG%WC5rRi$z+FOv&_RXzTeSf&2)_3~7BStR zj(DJiZ~EBvxqXo-LN*6dBCZ5r+VB11{hawE2~*Xm$Sv_WIDKr zT$_5y)51GX)7}v|Lf#TM3T!#ZTUF4Ib1#HE3+%*+$VY(L)l9R()l8}0(B&~o;6!`^8MZ;%x(GjQ(aIseSyZUAZ6iuVS-A!L8+{4U=hP9S#dG_ zH{a@T##Z~XL)%x6hTgKHhAE{tl}k!^8TDi1eucAjXlWX`IvR}n)gPM$RS(SW z-8}+HE z*7l?2-NVKqoc4D*3=>o)Q^*4UTa(h+bDFYz?xfEewSD^ugruCIkvd^-#BUC-O5HYo zaK?B>4J?<_bpl^s*8)zrggu)6F&#PRg87BVbBH+CGtMPlma8+)ou?y3k#CA(S>TAN zMx6cKEbYKvzp&=8PUrT%ni3FVXd^Sxl99wz4Kss^`>`~2*Pu|@6xr#Rg#4!Yew}ZS z*n1A;{S^vrX78=fHs^keId8)XBXjal=p|evu0*%aVu8R?PUpMB1-Gw$qUK|IB!`r< z&gHUx-1vK6j5R6C(NlG1hOF&7tb%-tl59vGps44kN%HC{l@S_+3{f1%HocM)D{3ya z8~p182ohxom2sX8oI&iki^G)%p9xAUk9&vWTUnK8PI~3zNy-}URlv{$Ivh1WWjcu5 za3gkJE6}eLUyXQHZXx}Pj~&h5aJ~qpvTp5S)yH7qqZ(EwpN{)^Ax^`Z1t*-32YH`Q zojYcb)~Qt9xDoqR5>`16ffbT#nu*9+5cIN5(T<|OCcG<=}kcIz2{&Xr{;198B zc>~bYBa^wn!1!z%%b66b+@=&O_KAP3?%MOh58?5MWw;Do@mI0MQAaL_QJFqf8(@r^ zAOVa-cf|?w-Ew>MUE4`WSs#0R!p?uZT z7hUw@j3bMbFxqTzi~uJml=maw)Zg*XZC_y)V}1y494iv}Es36f zrFoP>IicmPiNbZk`qpsmBSWo(&(czo@GMliaOJ%59=LlYGWkv@0ULk{l!e_4=rEnG{df~`VdVuM4Hy-0HyeRTh<|?94#O} zpR2gy7Mz$*_eSs3I>S3qI=%5OGt_}{Gc}UJOW(e`2lMHgLEr2WCptl;C5HeUE7_I4 znBFGxYhL%6NeL$75`UmAGp%&)~W#fhOn_V+f!;y=Q?2dN{4k1 zm0CikU27ccNAXn7-x20>y^ni&C-rAHPrEf@gTF+y6YukV;~r~yrdnHTkYZJzu09?y zJ9u?IGw<7IG9NHS@J_r+&`h27$R)95c^=rkJ(jDTn7jq^c{q|q`qY`k|i8>zb6%lRmnMcZ!XL>a|JuSEi(erkK~ zDgqvHd&$UjiKY1NcIj+nONeGxw^`=SF&;SSg_(oCoP_!rUco=;xQz_VPitLx;8>W0 zm2&?us94l^Kni#1tbeWfV;$AvH7B5Mj2qnfqP$j{e$IAm1vVp4?^GG9i#M6kmZQOiAOuSjT zv8|oeh^lB&C!oq`?L>H2mY8YuSW{S@z-6;TmaKyILzQR;WBiw*@mwo^_A~oFzNE4N zemO*yQGL2W-dgDL+jc4Qs^n}+YMU?>eVxLaZ&ncLK0Ah62BtvfjkhUthqq0^+hy8V z#ui&L_RRZFsS1$qdkDH6=0oa+2_G81^rR%JV>FH-%K)*u=5lOcsSdxbtgBY(cZTpk zM!zF)Eyo}E zis^8x;$G6m!uR&AlQ3WTQlRWqJ87!`UrT&+M%P6%-Rg!rAb3;kT?-8~VZ*t>OeMUu zNB}%8>yetge!G*m%R+}tm_=fbi-PAY6ttJUiW_^)?gxdI5E#FLIhr0UxF{cbD$7l2 z=n%z~8!0>>b4X)2y9+&4!k|@B0{C0rDe;So2Pl^*E6|UM14{GV26~8YV@TzZSv@%4 zRIQ05zrK|vlV7k!j3Zd+4?eMvx8K+G(+1yu;n1G4VHjDy^AQNHw&*!#8J|@fMxt3b z`0BSxU4GR1$?$0|M5X`b4f-sD{(NAFs?am=i6oxPTPESaMWW^Qeu>K`$%&(Yg%2OFZ zx2%AY{v1yPIkFjLlOm7Kf)NaiZYel%_|}ibKP03yj=8`BB3k2h28iQO9jLn#xpZVh ze*`H74H#{DU~X6oaJpbi{t*psm%MY?m31jGtpGh5)PfYBW*s%9)$Fg;{#pghA6YM9MWlS}4;k(q`k8XkTG}DHA^p?ot zIa6+0BSh#Magh`v0SaiY-F{|Mm79F|q{t{~pQy|a6=qP-l@z$)C{+;`ao>eQ73yhc zHm?ujxiIY|;KkYhY<+tifeP2b_6eOq#h$4F|L$`xiC;+m)cQB&=sBEUD;XLi`?*|i8+_djdz~gJ zEoP`2f6Bm4rs3~HFV~twxD~ozwIwo4V&c#i9k>6Y*N)@ERKC?KnDflY56@ChS!+Xi zS32(4@u^-4N{%$5Z=RdxzzaeBFZc=NV_Ne!s6D{OY(Nptr8`bP%Qu-EkAk7(;5=NiZOHY;_0Gk4Ga zGZTs03j@cf;@4$De3c#58rKK=P$AuKib=qbuLaY$Gjmq<6vSPdEx4#yx;iKer#fLs z*x-)~J@)8{Dma($t-DOtzQ%wj%NNJDEO&CV9C2il=~Hxb>PQDW`LXX*De3y1phe#n zTx3j?Oj=%N&ePa4ZGZ!4GYxZeVg|I{vQSxn6oTxjDz9t$GJ}f3bl$$X?_?Zq>}#`b z^5Tn_Xd0(_JDtd=YYQWkZwp(sV4L^MKh;VgvQWXp75>IJ7frj&nn4WM{*d@&A3|)l zoN^8vbJOS9)+AnuTO_HSM&a8-gszQR&l$tj+MkE z=iR07s#&N=v2nRphN+zZ6wQ5PfO}Wbj1`n~yVIs) z3@t_>IQjDHY^Kr+X0*4YoH2qpE=gpZ2Z&d!;%PCGeY61uyOsnmqXCDr#WAo!wgxA`jKc zwmS~t!mm}Pwi_7)^eK4_Xy(_y1W|u2k35=|UX>7veM1PgJ}brLJq;j<)KD`xH4pU0 z_U*CAl|=a!6$_F}UqL931R2Z1;im*&To!HE{YvX2I)AH%ouE`WV5ChkZBggw#`HJe zTyTLnoTN?1K|UZ-qoKS=%}T~hKi&%TlUTS$N63Vi;@uNl^qU%cVg&H{0B^bY8lQcK#Lo%V4H*Qd z_#zzW7PS`eZ zCZOj^=1_v?SyLr$TLP#3WD=tt9-+_irLjc=(X0EdaY+`-#feDJ8f-#Ru!A~8_kOaP zXQ#UV-21~nnxx=_K~UQ4pWfn1l_l!+qh{sp(-)PcZ6L}fHPBA`&9{tWW!xy@0*g8< z>z4nqWgC>Zxfd>T`{;Os_e5Q*+OK7Qu1NjjdGjSK1z1RiTkL(jO7qG?zxOFk@0zVc z?S4#PTzr4QifXkhvKrfG{_tg8aAc;2T!Or{uo*wc*YuSKp4wbiXqfR*x>M=4Txy)_SS#SM#{nC>m0k)TV1O&M3)28lZ6n^h>D)hpSTfxlH*X37V5Ly{| zG+)_aiTtt$*UQ%@toZt{ZFTiQ{cD;c;+~sl_n#|1%Kt7gc#U+?JiCiG1mCg;A2dAC z-rmY{W2L-nV5iXJAMS@V`D-{i7l&LFd*&J%rZD5* z5&iXZCaE{AyxHr=YQKQd^r!j#_$H0|qumyDp+B8;-bKy&bLiHm=l_9ch_G|ir=57G zCCk9jIUjw#ah;ucb2x_S3iEaP^U>&LR;I^(L_muL3NeSs+y*-(~%YZOs{vGstDAau|PS( z#5ez#lX0MAh2d1T$0|1tNNb*MnYW;~cvfQ0Q8uqU<@EvcV#1d_IpevZgUp z%Tm_cMcm~6{=0UIztqb2^B-;h`FW9~>6BG*QyHf8`S%(mHdvov%0F6`}gAYsww>L?+aD0mxjzM`rmKDkts82x9Z;1lp~X#BrQ3&U}a8% z`s@AEgD!=ec&3ZEFgU+LC#TGt;^xW4o4j%!w;C8lHhazjH5OeEu_K|2DbX`mz31C-@#T zZ(8P<>KMT07P0coI>-5MqxZQ)cHHr>i(uxpa&pN2=3u|4^ugk$393z;ytj_O2$Nvn z$bQy&;w`n(s@JCdE5BIh{Ep+S*qU@DL1`VwN<$yP9+sK)yR`*kJ61dY(GQ6#+@+s$ zd$G5TTl9sz2Z9nuw{O`xk$>%_174H9WV_CvD?R0bVdNpRSGN!Uu4IoB`J5COvbopj z@s(QL*X!!eykpz-r+`0=!v)PG(>b0>a(EO?I0~K|BD(j}ess delta 18522 zcmbTdWmFx}(k2|71P@Md4-SEYLm&{G;O_43)`Ww*1b26L2=4A4+}+)V+&gpM``(#v z*81j0Z+WVo+Er(DuiDkS5BM5l_X+}8Mgj^N;~hLa{5vW&TIEV)G6?9ugK9PiHwXyx z>Iz6vNW4V2I=&33ckdh!-@W_q`qdv{XrTT{_zF)0_m5G`S9r4OC=^KOf6Zc;oMEcl z&=DX3rX-0W0`PiE^-kM!mIFIV$SfOg>q_~kp*Bdb#o3-@XZ@7Uls*6Qtl$2)9|h`^ zNmfs@Y|3i3rvc`fK}rABjeTyLX^eeeKPhC-i$k9>iD*(?(v6=R&3^If)u}qUPw|Ud z`^>kgxTWO^f6PS#v1qJ8t4F^2^ZKbqG8#LeZloz9@5FwuQ9awLF2JAelzNZWiTR3` zr@{79dg}D{RLRLIn*%T6<@gdGV2b+fh!%bF7I{CYqh8I7Cbo?4IWCjb1kzL24lKEsIK?ZKH`_^2TX zG*bHgMp7ZQNd)1*nw-DZ#mX^ACZn}z1#>7U{$g*&bKrK+{aq%HR;UGY77X8XoEk~H zW#i*7)m$4zCforx&gcHPO(04vhQ4@SgbD=P8aH8PBs-YP={K@MnKgUS|!ojiQh~HVqzz9V*3MIK-bD9jMfV z!XkF()LhXaeV)xO7$kP*I`9|~dJT5x4c1r@S!j(e7~i5oM9{K;&xB((=Pn89H9AC0 z(7|xuGF&hS#D=^w_e2>8PmecbefxXBBr-g~>w`U;Evh`h-?uho5rr8Dg#+_kJi2-| z^}Qw)fcCcq|E?SP^+ThK137lYtJ68P;MPAJ>G|E?v)T3^OrN>nKF$xTjV?f$&vHN3 zV`DtTfcJoRL_2x>_?GinvCOCtEIAY<6rmYQP^i(zfy91qgLZ>rvBTBERbHY`7g6sabidvuNM5>LOZ<7<-TcbO zLPCrPPZVRySMF8r^6H_{FG8sR?F+H4&oZ*BIOF;1!Sjav3wU&U_3-$2faHzS4)>3R z)w!rDhQ)uvD82LbW+MJGnfTLC35m`Iiuf-(E=B;?IED?EHf9RfFsA3N*T*x6QNW{( zxxh1w*?Q}JJ{iQ=f1-^+!{>|9#m9}g{)A2qj0N(WbZfJ2{J?YcLQS{^zk7NI{}zEu z#wEDDga~>K?(5$Op<=S}C1Tw1X$kzX+0CxrN{q6V8ktiEy@;`)PZT!)y0(+0iM!tcl%t9VMIVFhJF*V(u(eKbm{R4lD zN6zTgj5(6NJkyR=8_Yv$hwf*ZG!x)&t5s;(j@n}vALM(M@Q z+AbYudBxZLFjX6VES(!Wtm0x!*V@SkV=p(W0*kEAu$PRVe#^`9*02!OeQC1`4`?(^ zr?q7qOHvB8=e3}*PTgM6B$ykMqV0`b;ZLVScD`YDsux&Ze=uN{I)7!}Hs8H_fRke& zI#7I&Czl*$e*W6|Sb7cwrzF}BYYj9GZq~V<%Wn;)uBX6Dm8p^tuD;lWTPEh*^NXrN z0wW6p->(!xB38;b57(!It+clvIMCDE*(mg6#AX8*y!cY7R>PxN4kIQj;zX3z=$C~R z<1H6bF>HRtia3j58fa-c5Ea{HpA^VGw%}cV592zWDd0Doe$EeL=tb9p>1%zEjWT8&FIiPmytJN=ju%`3RmQc}H4f&X~ z{TxI*l&$0Zz5hfF!v&z&xhS*fN#A=emIB5_ZwISO#;fA$HkWRy9#e;#=+v)Ed)6C% z1(+%I=pkEGpPIHU8-C$J&f))V=+Z_&O`q>D=}zOeg=*vep-L=dj8x*B7?@2I ztzz7ma#ft&7*m@K{<(X8aA+-jaZ72ALRCE+%|)2{vr7E(Nx$)!FguUCCnrJjz%}+g z(Vl-@`UrLvO>v2O>WfOKt`fuJd(_%Wtq@$s#i0@_ttPIuF)_RIyN$2LREE)ryY+z| za5Y1QGJ7D1%KB1+;sc<3Mql7$B=F(joz&w5CEoIsfysa7s(<7qG2&0~|H?}M8-t6b zMN5j5eKrTG|LG$w^$Ge{j~@$ZN!>B2g@YD(pi9|Wc9<;{%&#Sp$6{X3;@*{E`xeNB ztvO%K)XKGj0>Cqgo_Cs9vvC9dA!b(Y*kzWLH0Ye=ggZA8r*o$tm&`Cq?bHH;M~tLJ1Cq1Oc;$UFRUlk z&IOt86}Q|pm1;~9PKx)Tn)MGvmT!trHchETbQh~V z!CNTirRV;o(~)u$kK9T?g=FQELBdiP%@wutkL?y}e?(6838~CbSl+bA$D7hPo_g9% zlT7>I#ZcRx;LhRcIu`UtVH8xDx@$Ti*Z2v+$F0VbiD&KO%}u4vJtWVM$+PjTbH7n; z1$mMvW9HZnr=`0%G~_Q&u4pVm4rb{Or|C8~A5Ub9@9QVV} zlhkpBr;SFnvEwaLL{U5nTaNi0URs^i92izB=xH4sigy~M@=qyu zQ$kx4OhuA!%FpXE9PMj77V>1Xjtm_St9M*Siw1O;m?a+EtsdR@)hO6?STNjJXbpa! zd~{tG{&S+7|Ebv>uq@09NyOlmsQcmg&6y=&;^vYlXira@DTm^I-W1&`0&7VC}~O)r0#s&o(Zx6o{3ewD9UHSNztC{49)EigHl-=%&1 z$k6ljvZCJ0GoSbS7A6=hLcda=KZR_SI{U1O+yBVPCHpk04|M8mbqXcv)a1r#h5{cq zrJ7NRn^7hEiH>iEE)z^@w9E1)L$w3ft{Hs$8GO+%L=Ze(3DLux9%mGiqU49pt{H?9 z3^cyCFd65Nepf_;qWP$R`96bGSTPEch6h{GAJ<9^?jYZN-!(EkzAfFl4E&W#>&dsJ z1(|5_d_!dGn8Xn}G8~)v<%Ua86plfZH!z2gL7X=*mykh{H!zQoL7F#4fQ~Ijn+_H@ z*u0nyaQT00@zyR_yoWUl%;Be|G+krcVZGFM~;W^uxRuyxRmYPXobWz}|zXnU`* zXaTK?GAEZE#IQ#zy&}|r!$fLXA0iqrBIKFY%{cBn{|jl7 z`|1J=`ZAT=1RfgLWtKu0r7s|a}Kil044iqv@}TIk8=Vk|sUmue%$<;rT#e2gN9il63^iZek=gA}|)KBFqK z8bOl^jqG{hja@uEg_5aHgvGy3|UNvL{^ zk>;Nj*57m#L(I$rGmseYr7A|3Sqb=t84=Afn)RWY%Ys*9==Kg(xWldq?LITB)+$wW zuoBiovFiKvidT&DJ=w?fwIbdzc-yHm#Fz1-u(sfBIcnLx5Df)_#U*uytey~erF-{w z8ag7B5jdMa?c0Pn#t#E5{)f4aFPgA0n@NM^K3_FI3}gD$!k+Gj9}tCU&fNkV+&B;0 zCM}^&U56C9k%IC!Unli2t;Vk@6BFs3`uIcBrcLk*e!~qocT$H z9&3wCUwKH}AJHO}buB**cH{#llA)gl?`fWVJBiudUwz&!&>^Pz6*zJ3L{eh(W#6;A zKh@5VUm^1cfk!0j&AgCN1bOgc&OaryESM)iv(G;{a0N59>+nu~8F5`9M4E1^Bd-7{ z408}hJTCrOG=>dcHiWD?i$T*Y>%14{lE7Mcq9K@sR=gLK8vTh^GA09rc&!+!)u^^+ ziwx@4=i)B4d%8F7311-_f!VYeb2(sS;A3{clUZi$NF)z?6lA`$;Kt=gevUYp4}G@T zqhq~piOD*j0`rcLIZE2yG^naAa0!002Jq|PwSO$v34%HrL~`24Jeq_6IfF-j`;6f0 z^drT&!AKD3Ah4_KVA28Z2ywku6}06#%E%Q#a-%IiBLq4@pDflfE!N*PJAgCiPd|a2 zGDaxJdD@-`tY2AEzg|8*h=kt)TqKhH7V}#(qJ>=J=%9|)EbI7 zt_o258H{DkCajLD$#a^C;1^EQ0nMnj$MNMVC%Di%b){(98jfM%RwcsvB9VD;nIw$Q zKuOK=$1f6SNn{vQ@v70?UpVC9GnXYM#xFQo1;C$ZoR1P0U)5#`@^#e(H1*#(PGw@E z!uIwkE$#`W96UzPsy9$zh_N)}SeBP|eix-@o_aTG`*K%E$uE%?a3c4WoRH1VZty;= zKr&rAQAzTdv@qGf1Fl@)u-dh5D@!+zOTO-T!f1Pzl9;%Jns~Fe)le?=8`2IvNqWRC zoOKI^Z;ceS%h&JvICX^TmrR6k;QjF}jn}tbCmj4>FRb}j=y&g^G5_;<|NE4Su>aj= zE~hTbE%jshosP*jc<|559mYfE;lqm=35p5|{;;Du0x1dCIgsNDrg^DFTOPGIeLBFH zain4Y>~7jv>t0yTClDEP%*KNSfi+_bD9!b(>&S!|o24nhxn^Ry4aM+-J^^I-abvh- z>C#h`Ttj_BjcU`&#JPjfYVq15lUZsJH})4L=ahza<@rrz#>L&$RzpTSech9D%ljhM6hods=N(QW{`?A8xfc~IVDBdaZ(}m6#9=^P@5_KJ_g!BVyHvGWEcU$+ zQM_riqPR6}w7O&NLAh{q3-A&ROn16nu5FoAJe&sB6C9G~O7w5JjgY^?TTq&;G z~Qvg(UWQRRQY^YbM$m3 zb+D$H-dKIA%X#Y4(b4mksIgk@D`?+HOqQ+Bnr+WTb>`kfQOViYXx|_U>&R-!b2KKs z&odL-z&Z_-pSQzvHIAQ6d2B`8#}B^*!@YO2%w{D=MI~Ygc&*Yd{cvnP8T@l($qUE{ z?JlE>HajPzr8!nt%gXCF|9D;(FLtKt%I?sVUl~0AKrmXii9uk6dg?Zc^F#FJD(N!P zGu58ZdG6BxGEj3&eRT8KZN-)ln7hQ@eec8z*r(adU8Y?MHK!qrezSXNcOi|wq}V0l z1h#m{cYmY+0b5Urmys#u94G_#2!HYIH{AS9H7?~%gt8eVHoE#(#{DLXNT77o|9BO+ zFV9XX7SW&-CN@o0rmP=*9cVtA;LE$w#71cv(cly&_J=G_yC-ssR}W7qJv1n2Uxhg* z=nFlqKdWt?aZe=WC~GOsWOrg2W>+834VQ%YiKoH+oXNung`h7j0N<$pCtTmjM1=r`HxU&uOCBr>JV zp3qH6A5Rh9O=*)$al9IDuE-~Cra>!r*AH#3#K+F}P)UHZv$i+`fBivGCfo+;P&V9# z;Bcb5G49%ALu1U1)wax(H;H4ec13@KI}r5|l>|8}{smXCCKst*{Bl6o%MWWEZEU~J zxblUW6d!QA5_xY$=f*kWy0`7Hi^fgp#u>(TKOs=UcwVchjp63-F@Bi*35q?i<_oGW z{sa*VA4}|mfq>Rq+=Oj!KLdUnOEpPz(~e-~TwiD(Rmj69Wn|5%;Z8I@-Vhs!0EY63 zPoo$Y%j?I7$fv|!`zL1^HQr8eYQ8@N`LEL!2!LUgae~yoCA=YY=9ZO)v$P2!l^OKs z4oQ6guvr*?S>vhU=RipuQKk!@1xEY?AHnBe0h{f&i*gTT6#BU%!G~m(sCE10tdd_e zqkNnc@tozBb3%s(W5d&CaOW``>9q)S|7WtcKgb8+hgvS zsZ`W&X;?qo(x&WkuLd(Mj@ejxG~0BW|2w{X2K>KtwQSowO8NdJJlNefN6PpoQ>nSa z&RZ!%w6B}CkD&s;TAs%d67HP)<(m8DzIu!g0Rm7y)j13apn6z6B$uFCc8)%5p~rT# zN8w$bmZ+j@17aychz`r66$}&Om6z-B4-+w(?gV4SmOVZ#+{W|~jqy{M2^YcC6lKD5wF6syf~9=C2SaVBmnM$I;D z07a#DCKU@+Hw`1~_|{rPNKW~8uN>P5wl0;%gY`6R^f7vI;ng00u$BqiFGy4+FRJ7Q ztkF8DZ0}5ez$>yavam@9yQvlxNgo(qt4lssb3FY98s}mZh&ILdV|qE#i&yn#=QhqB zf>g&8qGzgiL|+Nz<2GdYn775BpXl?rfXUsn33RG+x41uhmoCrlk}ByPB9GGRBfi$t zbd3_1hWGrEiSRQ6va)$04^?~KW$wAIT;jEfeQz%m7z^xj4@NeTFIb*q8Cj)%%w#ET z&vv1mD5&P^|6$rA+_jDRRbRMbJIhC|?}A!Jm|W>r`o&s%e<(Bg_v{@1p>y@k0`O^s zd)1b-54VZbW?A=gcYKPc{9Xz?LM9X^TrPT*`CA+)ylHkUDitxPiNI3j9qmwUS>{pB zS~d4?pj&GBaYgdVA|E30R?C0}6@FBWTsLTM8ER6RmLiNrpGQpSks6z}xT=X+%TcOF zz~C0sagXed*lQQ<3=oE-%8Ugom}p0{<(5u)DD=8Ufkg5>6@=mkT|a%bj+V~nIv4g# zi<{Xj16^zzTs)kdLtbuCr%N`8B<&eiU8}VneLvb7IXq6?uTKuxNqIu}yUXz(9LKVG zd$s0b{D0cz#H_l&04>X_@=Hp1z2}e5v^cBmd!D(sq6hD|6Ah}5C96!6FpO?SJ0Mw^ zY2>*bd^NyAJzPLw?AScbZ)|)8D z_f_-rwcHmb5m@-&n=xxU74Nv*lr9XvxL!nLmutQn+Rw0d0UZrLXLeki*booXQBc@- zl}D~UF0<6xi_a57%G%OUHtV0a*Nv9hFMCb`%Hjjk4EN96R?3W=)$bZ#xtI1B7#1&g z9G$)FmrMMc!>&f`_(G>vHEIt>x@KQAo=T7DAMWSQbNPmM2Qai(c`qAyX>aPBm(y;S z^O|wAQk>r^DETEI_FDJ($Wbf(H8n_c?2(cEFufP4ybt@fQM0bAei`4GN~AgLb(Tn@ zW+-!i)iLC}L1Q>R*~r6+x6OhN)J1f-eA$(MQh)w4ARpyNIPxiQw`DRBF44ey1m|gQ z%kwzn<#j(=Y{zZFzEfk_YL({USn*-NZL9d|AEMA`R`gr9{@>LLa*Cu7a{cgla6(>8u4fEaOibi{BJfjpMz4;cq5N=WEiOtiQBg9yEob zp*2=-o%X0YjJ&g%T|W}B@O1ZbB7y>XT_X5(-q@#Kw|q8LP82?pysY|#@Z>QTM|r(O z6H&RZ`al>%E59hT&k!$Ska5Imq)#;>!ixxq=;^w>rCnYwy;$0*Qw`5{u6BKxZ@6#9 zcD}PXy^V3@CEB49@w~m(;Xz$8Y+XQHzFyAg)8sjNF4Ck+n%V4f)_>LGYHkdLv*OgY z_DcCgC9=De`pk7=S^bIm<;=+A>W2Ks%*6UJ$JFH`^XYiP!+ipZj$|S)(Iq#Y{#YN7 zlEwVuF^r3Yet*f#IDtT$4<4tb4e1jI*hF*Xp=02sL#D)v>GJ8#8PFBbAN<=;=C^;w z`@#t`+MYm@Oo+wWpBDgwWapt`x2@|M4-S%&EMAGd~-Ku!V?-@i)1kN z4w4#Mdu9eb_t*v3ec>D$8&7D_h6h@zjcH?ZXk*)ts7@zPOoknD#}fvYP7%XC=h6DX zD!Sx~?~BLk=uI7hpP7Y0oY>NDZg3wW9-BW59Vyq4T$~i=&?^26l3MWu@=b=XQvG^y zynfbR!OXsKDl(b>UR#~REc>*7POPlWV!oVVASWptA|jKvJxD$%cW^JYF)>t`%1#g` zQ({^LJr)aDurzjs;si_#{++XhuEz_k32#jAK>#l9aj&efGDUwYe(+u%*P_J<|zfg8AqA%Q8a zZJdUpiv87g8`R{so5iknVQ}?r`At#B*OOF@u?EmBGU@RawvRd4F>$GM(dJ?e7kpMV zpfUwJA<)ayKV5@sk4i+<#-s6x2=4qmT2g8W6He1Mng^Jp?CQ`i7M9A(EB%>`S5atc zzLo*Y`ChTCQ~=p5GDc>55pU}acoea>M6#0sr})xfzk>M+1f&IH&CMghcw6KOjdie` z{~K_(L`;=ASj*NgfMh2J)=&rIPiGLbQP;3he?|Hr2NqNZL;eMT{|%w;e#OQ3H{F}n z@Y;VTc|S{NW|j&-n6`6>n~Ng7t@e%$UJi^3 zv_nr5|JQtc79jqL^q>4FJaS8l)AZMd~i$MTMTn;R!4*vLW z0PsH%?}4cQE}74F7q>(bmjOplXJ7!o@uo8jh@NI=Goab1#gP6f37*cFoXvn_D-cDZ zu|%Sgf%;zlj~@J=g8vHuym2e7ApT_ooHU*BwhAm;0q)Mk-(DxQ@}F}tBr{7SGnxNV z`2PZ+`~}VbmnIf%&oiD-FSStKNQV{R9z(`kA$bH@{Y(m-T^kp~t6c}I`u5{=4Rs?k zE|m+~Ow!!gFL+^-l4-uFtyDd;Ks#0?;?x>GS_X#Jbg)#a4BZ@}VyqhJg~uJ=lCVs| z4RsUKL-}RQE7Ffc;G^jKMGo1@z|)w_Y<&89oFSu5Ukg!ztnVj-VE!(7_^oMb~Qo=8$o4pCYXSj zO+h+I;6x+zY$K>5&IB_MvpHxj3EXIe-em+;#hG9MVzvb7B!d@>&^L{snm7}zK+M*l zwPf(U5jvDHR3~SG4T#wmq>}>1HAW{hh8pBdumdsMgVs{OoW|(ijG-nu6C6Oyjv$>> z0Q{o;zGr^H+X*=@RcbgD$z@k(Wubd*!P^CyGEGV{4Jlz)XJ?^%Z^7FQIWJ9WI1Q;` zSLbA*`)tA61DP^iN-`a3eplyaq5E#Z+Y31_U1~TT>26o&Wug1sqPI6PWrmbw1`^Jm z4(wt#{GzuHa$bhia0U{`o(}S2H|nA{;EPO|DJ7YSq_U@jwb+fb=OqnGmnT3?Fr$f2eO}*$HjGUJxHJpXiu&2YY*v+))9g0kuEhU+aG{2_< zTI}Xn^bSYP%a$6>M!MV6;alwfy67E=OqnAknS+G0zvlcE6&sRzUq^hgTk>uF804KC zDa2eP(S04c#cqW~?^tA`T&d7pB$Is|wZ(3YMelgzom?rzJfy&V9lgbFgGKK|WTQN( z&^)AqeI2vKZi_|lWaOPZDa3rF{(T+0#cqd1?^I-?e5ufUq`iF|x5aLcMelUvoqQ?8 z0wnkY9iPQ+-$n0CWTOJ9&;leL>I0qN#qQ8W?`-6q0x85oB+&z%=*8||i{81&Muk$L zg-9j`I!TM&DU06u$UB8nh($<&2Rd1c-8qZiF;UW$3dPpYNRp0O?1f6JAMg?mgskQ1 z+i+Q_q1$9@8&KO7sN9Opu8}f-7D`_k%^au;LdmD@9T&~8`@!aMO?*-+wnq5Nt}jtq z#rexVMJ#P)V~tsno`_kK?gd71ckL?+opg%qcUmZZq=I58fmq&Y5{cDSrpInQwQ)Kai|EqF1cHNLF5Y#oOSGo&P$`2$~5$ z5ox*QT<;oDoI)($?RwfWS*|Z$e|9rzp4nL|N@N5UXSR22$Kv0%hukN4JQtO?w@g>; zOLLRg$G28&!|~-wGvinMJB~I>SKr)QU%S^EcJB=@tyZ+_HpAmbdKzkvdN>|mPI6CM z37f7@ifvQ*oVyyBP*yFe7XMa?B`iZcKTZ~I+B!8Hmc7%PR(Cnhldn76TCE-;j&FBd z>Hw)Oj-K=4gwKLVRu{RvO6=Esheb?W;c(!XT9_CB_iXM z0oi($r>doP7ayK`x2FrM5N8yAuV)A2&3f(Tm!XpyuKRzS>*#L6;65Y2dl%92pA}H` zPuXw(^IWH2juNo6UjkhKudf&>e}<*Fj5+IKSS=rH$cBE1ldDpF7Av=AMSUMsASMVS zx^$7>rs@65gnj=x!{MZ-ZK`?k0#bzztGP{+E`#XRHQjiklvQbeBj30$KxB+mwVO+} z=OEvh$Ot?2G)=e;Lzt9Bvp+q}GRN~~EdBLV#C#|)2@AN%H#Su*TWxjG-Bj4=Bs{34 z+hdflw3D4IH@faD3&RMK%B8zJ>KT}rsAsgx3^tZ6Db9M@Ur0XvktVE|=~Zqp23#B` zvyXYbOcbR94qd-%&C#azrXyl^l=vQ14aDb(w14rx-uV>1_*N+xQj&Kl>Bw6CT%*1Jw7gFvk>z)lUF0*bsvHt}Pgc69 z|0~a)YQ7(h`PAnsmpW{e1#d^GPfymc(;@}hBpeVj60;<*PEt@az&+QUUF0*uHP~|- z`4e)KTyXzcZfNh+=iyPeFZDH5SZlB9r8qA$^fjux1+-X6Oe&X9@Qac^uw?k{WILQG zD`+Z>_MS>)V|y?a&fd-JmxfU?brc9>MpBx5EgmvV$Xdl=1GSiNv#(z%q~-oWY;t>G z+y^ikjQgE4BpNGA_u)knv%6*^BL4escXgD6X}`%hRmTwDv2zchVHOqNemdXiCe`^H+{$KXRgq=8K)L&T&@U87HDs#gfVXrIlc)Ry73?shl9LmrD^r2*IhCiuQ6{bFAZ3 zY7bTWbC~NUgy8k_IG;>x7Nq>fXZyGcjUs!yK__6s&b_Yv@$Mo}B5tNhK%+<$z~DR% zeo>-j^~5m7?9*KOaRRSyJ)=R{ad=|mP}{Gl3Uc|iMxcsU4{ddSklDH2Ry}>)#7#kk zp-z7N@vYb%e$=nVT&N9oracjaiZ*%{31$;>7E6)tGTrhsSiuYQ`%vBk(UGk{wWLXk zkZsb{luT%IxHWf0-kRJ#+W(VL$%Kcba zIMCL19H9w4!|S(qG>sSsi^n_kP)fatZew-&s1=F{{fA+`6p@(8R(o7&O5Mu*><<1l_#HaQ0$Q7ugQ@(*hYeMPq~ zJ|rFJN`+YIE&0K_z+gwQ9!bAw9$_Q0L~X}jzO15>Zs%p^U}xvyaOE%9YMn_*CM%<1 zx!9%G&)`v@8yg#(((cI$oJf&YQo2^2Qc95kxV%(t6_kqilm~d>{M06mQ#(fac5oBL z*<|&ds{{n}vuNBHJzP$xEj$}Y&j-}O?6y0bU-2MgRGm2puiWRz0`K&JNdh4ZcpdLn z_6~!6NmT!cQxx=44ba)$-r3#AVYV_O-us1bxhU8J4`FVWOfjy&^H9(AmIdEIcv;_E zjJ*Y>;sjjoj8b2`NQW5=$TZ^O!SIY)C+3EFwe>wAjK#&iT`ZuX&WTYz8b^2EY5 zDk*`ac8eE`>_<)Li--M>OZ3+%S*CykCr5x|2|xrY3AVy0n~## z;If!})mPoF7$gIr+d7VZ|FD^tw%N0W&00DkaETW|h4)(_MinmA=C+1ET?^&G_^15N zpC;;Y{G3`IcZ5+d7rsOPqu<5rILQ{LNh#y^M;=^5dWP z@7o0P?#pO?^;Rh~ewf(bZxg95zQpC={~%5-{9S_2Mb7{p2%uV?+S@)yGsf3+o5}Sg zYOhg83y|#ZgXlmEZ*G}-x%S{n89sa8^2h!=HdxvNcJatujnIM!DZgN>fpEllE@t4K z+NA6j{A}dT*W>?Fc=9lO;MkGs)xQz&tl1UUEJUvRl5pQh;!$>4Da(l`ni3{0DGq`O$?{4itXyledc_n z1(T-A+-_7w#^Z zg>HV;Hs*z}b4>LPe$%V}oz;g5nQ{<9cC-mzQAAqxxT|v+$b1FbPcgSik;18lJxZ)6 z78`w^uXw8J(FQLSU$0}kuu8E`fr6nws3uod+J7QZZNl1}`R=1Z1$)=i@S!xrkV8yQ zvK$3aw6^6`h=s?~aU1n@m#ufo#tu2e+{MjlV=_PvGs?JfN?rRq0v{WV5*h?%W(C^{ z%X_=eW00ZJ5kBIfGMBh&pSuRJL=hYO9o=zg32Pl0w10j~#JsCY)lIAI5p_zbdGronKACnv1PQe9D&4q+g03$wIfJ{W zHNZy(q&mY4*jl*Sl&1{v|N9Wv>%1akss|Ip2=>UcVe8V+{2x#c3l{6@lR*cj+qXdELjyg4nYdSF4Rl;&zggnFtVLlj1gmE!V z!uc0d(L_{VII58NgO4BHd^ui|g>h)Q)Hh)bPZdje5^0 z!bRk``*wpL-PzcAG+@*{J=A}UPGj38jR1v!+}2TN`-86^c`5mK7F z-I|NeyJO8U{CwibtgJ$r+X`T~rS>a@tTupB@bl|Tq&H2osS6>C<}?YsfN>X(!FBN% z&0uaP!yQZ}Hf_feRr+yg6pAqT!n|p`=fM1t$!@z02P5c_Nq!sl zu(UN`%AjgMC|2jK5|+CFU|{zd_!JCF1VoSUJ~~Pxc5y7Snbl!5ENOnK{D4CoOG+zG z<%e2F9ydDFtv8AU=@zaS_G~M*Oe4QwIsR@(=5R!K34$z6PdQ!*b^%JfHOWPe^ zS?%;ka7~&=47n^1o?DPZ;Rlh^EUJfJ>Buspf)8oTlm)3@hH94jfwH8X(dAvPZcus^ z7g$&Z@5*7axJo6G9*!}Di%*5`x|Z=IldNQ>lJo`ANxi_BLc*!|F!%Wc zJP04q;vLE4w2}bN!khQrH|cf0PLjyJLQW2M7Zk4;9`&{~61%M@7?yEaoaU3u3_t(L2EHT~D4BkiDx&p*H7XSQIoCW?q)j5XYx!clp%HjzL?*f#ud#UsZaW)v zCVuYD&iz#;{aro>%?8Pt>J;S~X@MU>!D&*{@IfiMM)@K9^eW2jd@XC<;*^^Gi|Ggn z_6_B)(jT3CYS+iVT}QGhsrBN7EE`^g`R9U!|ouv+GverIw!ZhV6JtE zn6mViBF_0z0zZ}FU$c_$_2K2G@*?Xe@tInsg;JvHVg%p!EK#h&ds*#zdsA5wJ(L#) z>wRJT69b6H5zssH&JEpnipt8B$9;h|G^?S0wF~*J9LH7YZu!*Gc62n(d_6Xs>y?sW z@OmV5b>em`KpqQJ47v?zXuk(e%6~?i%lNfnmfPKj1t1z&TBTMpJhfc-$2m? z)KAnm^y#IZbDUf!DZl9=+5q>9i{tYN{_yY>LW1YBM*dpwGK8p(m3p^SYE!DyL-t9hbX0y&toK zGXiblvgF9QykERwasmZIDT?&z+4}eI^!r_at(X#g@oKJig0dun;NhR|WlTPFwH#g` zcHhu;k}ZzT65;6ZXTvDob}4_Xy00)37KpYv!fai2Z+4I+V>t=5*j+yu2hXqZHD50! zls1^&hWl~rEtC2znp}i<^g=*gSXNbA#!q`Z9p7!A6Gk7D{mJoY8f-(|uwYC6d|MzI{U$?K;ZR4u*Q&|O<&zp9?Vagec)B5 zY58o<#7m-Ixj9tLa7p5JA@L;BzO<4-HLds4{^O0x`VVg2(PlQ!D3O_7q@Db6=qG&L zkQpqPJE3prnsZtv*cwMXikBxkE|43SPnhHe-|DtGZ(CMi8My1(&OPQTj_vWi{F^;N zfM(P9M-}s!_d^j)t}7`mP7MuphCmVVw003@Jy)p5?9h)YFT`mjiKKeFQ@PKu;zgl$ zaK;C3e-6-09e>4fK**+M3Ylds!lX4-r>rM@V7h?8v(7dfk)#2iI%(w_!N;%Q|`BDn2 z_mw}?smgirkZ-cs2s}hH&N@QN=`SlPDo}Qb*b>_X>SDmq@v!6b`z3vY;giJu2T<*l;y46zg@arXldIb z-Y3gFN|xiR=#!DY*KTae>s_@P@hNiMf~H(`R2%Rqf#rrV70==+K`*D2NRm1KN@7KW zqt?LMi=4v+M|LmONbTu+CWL;)XyDqHd1t$Oryg@)NCHXpIR?py3NipO24qADNsAMs zWrR)s(Z>Ae-Han2XX%ok_yjL|GdE{bY=qXxWh3Rvx*NcATj;eDlW4?*PKBO~b7dX} zxL?)6HFzGm5%C(t{4t`3)jTN(h;xZSF~%W5sWB#}gcXvlCw4cTg&Qf}o$svR1FaBX zL*1oK2wjFe$A}s;0(S`Lpp>6mKqsyW1tNOq(269)(e01CcYGyq{}8}9DQ7in*Hifb+q>U*MC}I{uxTNwrRqDC;V{0Cuq{e6UB4wWxlX4+nhVz@qpR(Pe{t zs-%vk{dVj8j^*VwJISUMMSP|IR_7nlvvZ_)InrE`UzyB_cPrl-mPUt{Q_(!e*R2-N zhxDR8CyN8hnd6+CTpfrfPE8(ii&4TUjekg$c`{Dj&RcuQg_=y-m&libK9(8UcLmD? z?5?TM1|xM`96ZOhuNNN6LKs`>*tU$!3>yEK7*MdtBi3UnahIY(N05f-NvO21&RG3( z)gkt)Gjcf$3Nr(lS?Rm7xdU-+k#ER`w4bVi*j^y8RNlM5XAzEcl!vX^_tD{H32o?C z@x*NBuVnbvnmIOFr54C+quV;#ly`8%OOQ#oBCd)hekI~py!`U?UlZq9ciBf>kKKLj z9Vidfkztssbq@mKIhaEASs$0_zl)8D-*-Hf#6&TY~{vKT0M#lvV}t+jmj! z(xz}ekW`U@NljBVf>6KYrM|+){sj4TZ{)~Z4%9Swx>1aPHDuf2)T-n)7r7zOmFu@w zzg>+7!m(N#QfpM0UdiH?-N7@n=I7Dmbcz;R02JfPfLh`wFRM*K?}(2w;NV~QA*I5t z<9q(ZdZw%{D%085(2n&~vn&tPQa+X$Y4bJ&4*iQ){iSwtb zW+QO$U4AjPkoBd;F&RG)wxWTd2T}+-9`c4eenUV={zF}bMpy1M^mBU|XNbBEE|5Ni zq3I*>LRc4?6Ci6`6I44R;O1(Y1FaY+X1h#ohI6ACjr%LHtR0&(2w%blU3VZqff_rf zQpqPIP+k1TBBvsTi;_+VQW2@L;Ln#+@e6w9@=sRU$eUlvszrUSw#BhEK(OfLS+{t@ zY})(HafZD~3NU^@=7X>G-`|Gh!vgq*ZyzUS!Q$2DdxD+Jq3?c$zefd+Xf*bt8g}AA z@KMe&3j3EXaYrFw)U{;i;shhzVV!VS!cz76;m}Z@x%HLLfzhW;JKlXhh~w(Yxs^_a zHf;H*VZWbtgP7RvZWqV?*<I(li<((n_&jEH@y^P}!Hh zLYs{RSrc(O;JD%ws^Dzk#5@_MYDuWWj0}x~jUp6ebOLKf@&Q!}DQCtoSWGtq>+C)A zgwFx942E-Dba7?J*521xQJ z0nyfAyJvh{ZaD5yCWy;_!E0>~T}eprnzLbDgjCr?gPSz0$6{La&&4^Q43CU6z;^d* zy;mT5Qi*cj zemm;EaM!0jGpdNAoXP!8ico} z$Mui?7DaD0+M!Y6{qmY?*KDn67(DKvccq@w~13b+V{ozmyMcq?T3 zlN)A!t-WgJs<`P;y2xwVkW!<^05MC{Q)*+vhaV2en<+tcSamQlgINW7c73;v#FK?< zB64LMIEbcaTCurp9Q@H3ad0*G3d|e`KlqiN9K6TkDwD_B9nVSu{tIE_2W$K6v3#NR z)Sq)n(thldIs}gjdnUaR+)VoVW>~aW&5=T})WJ(IAEfdSr`t!^V_{F7(DPY_WW+95 zf=!6uG)$LvHWpCJw2 zM20BW!rABb*EFaR*jzHw{i7t1VC;_Jm+ilgNAH_l_PM0LQ?Q^=)&tcEs!BUEsP!{{ ztv??WcMIl%p!X@J^l5|H2gimf?!=78HR)IsK7;l>cItz)_l|IbmXu!d5Y)Dm!Ep0^ zqrM|hg3qcnHk@Ou@We(B-?RFizE?nR#nwnsbP`dFTtP~00Hb%w7A%+w5v*j;zMT=O zhF0y}+%a~TtgScG?X%Z?G&MqIO6N=PsQUq*@4{ATam+j$)(^DKnq5Bh2u~jEWM68W zgWcyb4*gT|f3P|UJ$5MAAYS~BJxo+&E_>x37_#Vy%D;&Pr)||1DcZ18> zc`->wJ;%03y^9d_x?Gn0`@5J#&MUc^-OuM;F5W-u%=gl#ZylsBo)DR+;3DU7PsC}O zX4OgYf4ZM~Qa2@pDl59O8)VFjRgDR}mr-%8mnoN3;=H7%^G>ge$D&*f&1#rUjT?hK z+}BCR8oyppueUMjPxsV@#RhX8O6h3NNZn!l7=f zgUQ#L)}1}dZd<%;O{=ZO4h`9po_o!cgtp~MZzhuPhl<%sG|Y|Ge>|ML&mmI8ZKM3XpAU7^ubZftblnsF^SZV^-Qq~^Ije6hF)PMC9K z%AEkA`MoAmU#6PAarypzQ)GNeQ|$G$-yb>e@G1KqI2v|?tL=-#zn}d#%|T0JeK{vT zPh(_YxXlEQ3U=s`DwFRO>eYK{MmuWo-ZFXHvPITvOY~G#y_8eaCy5-~AJ4#RabErR z)~`3}UcO2w+1M^Hb+hiQM_i%_t&>mXMSr=ucPe+tkt1uL+=|S&D5w;*k^Pt8oc0MK zN0fwfaxUtr?pIOil9iAv%)5O@GRDB>sZma+G@I&!tDP4ce3z_S)pkI!ajJrMeLd?U z?!uk^|69G({k|XdE&np}cn15i?0F4=Mc&racf6azcrE2;)P?l-TZL>=-ETzIvRBxz zS17hQTpZ(n?Nnd;zwC=&&L$U{UwAHF=b&P=T1|iPrA=!zcec+xF}wR}&aal$GS3_B zdne6a{YD%V9xS_7%Kv0wV3-9A6X1z4+(<{vOcpP`0y#0}eerqdl=-1jQx4FnGoXWJ zCjT!rh3wF`C^LsJip!iJjGbjpydZV(P5zTP$}QzVB8|1$PyYaSTt+%FFh~F`MgXbF z3FYbvAT`LlGEvpUXiT1HDmr;jxgf-zi{(;`>n6V`Hv;QJ-ra?&?~VajpMoLl$Z1p%MUCL(nUxA)HOTQMfTCvGf VB7|>KCB=Aja(I;*+q6=U9sr?q%0K`B From 467d3ac1963f39807a4b88fc346081ee7f568afd Mon Sep 17 00:00:00 2001 From: bedroesb Date: Sat, 7 Dec 2024 15:22:00 +0100 Subject: [PATCH 13/21] upload artifacts --- .github/workflows/python-app.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 29b04ec..5a7b5e8 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -37,3 +37,7 @@ jobs: - name: Test submission in --draft mode run: | ena-upload-cli --action add --draft --dev --center ${{ secrets.ENA_CENTER }} --data example_data/ENA_TEST1.R1.fastq.gz example_data/ENA_TEST2.R1.fastq.gz example_data/ENA_TEST2.R2.fastq.gz --checklist ERC000033 --secret .secret.yml --xlsx example_tables/ENA_excel_example_ERC000033.xlsx + - uses: actions/upload-artifact@v4 + with: + name: Output XMLs for ${{ matrix.os }} Python ${{ matrix.os }} + path: /tmp/*.xml From 959650ea59b0e1b7ae7791910fa5ec54ba4578e5 Mon Sep 17 00:00:00 2001 From: bedroesb Date: Sat, 7 Dec 2024 15:24:58 +0100 Subject: [PATCH 14/21] better output name --- .github/workflows/python-app.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 5a7b5e8..38b91de 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -38,6 +38,7 @@ jobs: run: | ena-upload-cli --action add --draft --dev --center ${{ secrets.ENA_CENTER }} --data example_data/ENA_TEST1.R1.fastq.gz example_data/ENA_TEST2.R1.fastq.gz example_data/ENA_TEST2.R2.fastq.gz --checklist ERC000033 --secret .secret.yml --xlsx example_tables/ENA_excel_example_ERC000033.xlsx - uses: actions/upload-artifact@v4 + name: Output XMLs for ${{ matrix.os }} Python ${{ matrix.os }} with: - name: Output XMLs for ${{ matrix.os }} Python ${{ matrix.os }} + name: ena-upload-cli_output_${{ matrix.os }}_python_${{ matrix.os }} path: /tmp/*.xml From e9abdfbc4f431e74d71b45f7e655ff2f879afc23 Mon Sep 17 00:00:00 2001 From: bedroesb Date: Sat, 7 Dec 2024 15:26:42 +0100 Subject: [PATCH 15/21] include python version --- .github/workflows/python-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 38b91de..bc71a47 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -40,5 +40,5 @@ jobs: - uses: actions/upload-artifact@v4 name: Output XMLs for ${{ matrix.os }} Python ${{ matrix.os }} with: - name: ena-upload-cli_output_${{ matrix.os }}_python_${{ matrix.os }} + name: ena-upload-cli_output_${{ matrix.os }}_python_${{ matrix.version }} path: /tmp/*.xml From 39044bc413d3bd17ae091cdd0e8e0cf0ccadb7ed Mon Sep 17 00:00:00 2001 From: bedroesb Date: Sat, 7 Dec 2024 15:28:03 +0100 Subject: [PATCH 16/21] temp dir variable --- .github/workflows/python-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index bc71a47..c46a59d 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -41,4 +41,4 @@ jobs: name: Output XMLs for ${{ matrix.os }} Python ${{ matrix.os }} with: name: ena-upload-cli_output_${{ matrix.os }}_python_${{ matrix.version }} - path: /tmp/*.xml + path: ${{ runner.temp }}/*.xml From 8dbca2da9c274d7a55c32df93e39e1de54ebb5de Mon Sep 17 00:00:00 2001 From: bedroesb Date: Sat, 7 Dec 2024 15:37:14 +0100 Subject: [PATCH 17/21] detect better temp dir --- .github/workflows/python-app.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index c46a59d..67d8eb2 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -37,8 +37,11 @@ jobs: - name: Test submission in --draft mode run: | ena-upload-cli --action add --draft --dev --center ${{ secrets.ENA_CENTER }} --data example_data/ENA_TEST1.R1.fastq.gz example_data/ENA_TEST2.R1.fastq.gz example_data/ENA_TEST2.R2.fastq.gz --checklist ERC000033 --secret .secret.yml --xlsx example_tables/ENA_excel_example_ERC000033.xlsx + - name: Run Python to get temp directory + run: | + echo "TEMP_DIR=$(python -c 'import tempfile; print(tempfile.gettempdir())')" >> $GITHUB_ENV - uses: actions/upload-artifact@v4 name: Output XMLs for ${{ matrix.os }} Python ${{ matrix.os }} with: name: ena-upload-cli_output_${{ matrix.os }}_python_${{ matrix.version }} - path: ${{ runner.temp }}/*.xml + path: $TEMP_DIR/*.xml From 6af3dd100e3f3c709ab7230d1ccd84fb9dd40f3f Mon Sep 17 00:00:00 2001 From: bedroesb Date: Sat, 7 Dec 2024 15:42:43 +0100 Subject: [PATCH 18/21] different variable syntax --- .github/workflows/python-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 67d8eb2..0cf5682 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -44,4 +44,4 @@ jobs: name: Output XMLs for ${{ matrix.os }} Python ${{ matrix.os }} with: name: ena-upload-cli_output_${{ matrix.os }}_python_${{ matrix.version }} - path: $TEMP_DIR/*.xml + path: ${{ env.TEMP_DIR}}/*.xml From 875394b4516511f32425c9d295994966d1f02b7a Mon Sep 17 00:00:00 2001 From: bedroesb Date: Mon, 9 Dec 2024 16:48:48 +0100 Subject: [PATCH 19/21] less tests should also do --- .github/workflows/python-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 0cf5682..20f5e25 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -15,7 +15,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + version: ["3.8", "3.13"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 From d994b5d6b102dc8c308e5d1cefd8e1c57404acc9 Mon Sep 17 00:00:00 2001 From: bedroesb Date: Mon, 9 Dec 2024 16:55:42 +0100 Subject: [PATCH 20/21] add extra debug echo --- .github/workflows/python-app.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 20f5e25..d2df55e 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -40,6 +40,7 @@ jobs: - name: Run Python to get temp directory run: | echo "TEMP_DIR=$(python -c 'import tempfile; print(tempfile.gettempdir())')" >> $GITHUB_ENV + echo ${{ env.TEMP_DIR}} - uses: actions/upload-artifact@v4 name: Output XMLs for ${{ matrix.os }} Python ${{ matrix.os }} with: From 40265d5e4ad8835931872cc5e6f658010534c7dc Mon Sep 17 00:00:00 2001 From: bedroesb Date: Mon, 9 Dec 2024 17:02:14 +0100 Subject: [PATCH 21/21] artifacts seem to not work on windows --- .github/workflows/python-app.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index d2df55e..bf00b84 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -40,9 +40,8 @@ jobs: - name: Run Python to get temp directory run: | echo "TEMP_DIR=$(python -c 'import tempfile; print(tempfile.gettempdir())')" >> $GITHUB_ENV - echo ${{ env.TEMP_DIR}} - uses: actions/upload-artifact@v4 name: Output XMLs for ${{ matrix.os }} Python ${{ matrix.os }} with: name: ena-upload-cli_output_${{ matrix.os }}_python_${{ matrix.version }} - path: ${{ env.TEMP_DIR}}/*.xml + path: ${{ env.TEMP_DIR }}/*.xml