From 90301970de3da0a1be72253107a75790f5fa0d6a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 21 Jul 2023 16:17:05 +0000 Subject: [PATCH 1/6] Update coverage on Readme --- python/idsse_common/test/coverage.txt | 2 +- python/idsse_common/test/pytest.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/idsse_common/test/coverage.txt b/python/idsse_common/test/coverage.txt index 9e8c047e..90da9f2e 100644 --- a/python/idsse_common/test/coverage.txt +++ b/python/idsse_common/test/coverage.txt @@ -25,4 +25,4 @@ Name ----------------------------------------------------------------------------------------------------------------------------------- TOTAL 557 258 54% -============================== 58 passed in 0.46s ============================== +============================== 58 passed in 0.47s ============================== diff --git a/python/idsse_common/test/pytest.xml b/python/idsse_common/test/pytest.xml index 5b28ffd3..2134c5fc 100644 --- a/python/idsse_common/test/pytest.xml +++ b/python/idsse_common/test/pytest.xml @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file From 5cf40c79eb47807a2559f165a4ec3a234047650b Mon Sep 17 00:00:00 2001 From: Mackenzie Grimes - NOAA Affiliate Date: Fri, 28 Jul 2023 00:00:39 -0600 Subject: [PATCH 2/6] minor type hint corrections in aws_utils.py and utils.py --- python/idsse_common/idsse/common/aws_utils.py | 23 ++++++++++--------- python/idsse_common/idsse/common/config.py | 2 +- python/idsse_common/idsse/common/utils.py | 8 +++---- python/idsse_common/test/test_aws_utils.py | 6 ++--- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/python/idsse_common/idsse/common/aws_utils.py b/python/idsse_common/idsse/common/aws_utils.py index 851be68d..f5a3135b 100644 --- a/python/idsse_common/idsse/common/aws_utils.py +++ b/python/idsse_common/idsse/common/aws_utils.py @@ -12,7 +12,7 @@ import logging import os from datetime import datetime, timedelta, timezone -from typing import Sequence, Tuple +from typing import List, Tuple, Optional, Sequence from .path_builder import PathBuilder from .utils import TimeDelta, datetime_gen, exec_cmd @@ -43,7 +43,7 @@ def get_path(self, issue: datetime, valid: datetime) -> str: lead = TimeDelta(valid-issue) return self.path_builder.build_path(issue=issue, valid=valid, lead=lead) - def aws_ls(self, path: str, prepend_path: bool = True) -> Sequence[str]: + def aws_ls(self, path: str, prepend_path: bool = True) -> List[str]: """Execute an 'ls' on the AWS s3 bucket specified by path Args: @@ -85,13 +85,13 @@ def aws_cp(self, path: str, dest: str) -> bool: commands = ['aws', 's3', '--no-sign-request', 'cp', path, dest] exec_cmd(commands) return True - except: + except Exception: # pytest: disable=broad-exception-caught return False finally: pass - - def check_for(self, issue: datetime, valid: datetime) -> Tuple[datetime, str]: + + def check_for(self, issue: datetime, valid: datetime) -> Optional[Tuple[datetime, str]]: """Checks if an object passed issue/valid exists Args: @@ -112,9 +112,9 @@ def check_for(self, issue: datetime, valid: datetime) -> Tuple[datetime, str]: return None def get_issues(self, - num_issues: int = 1, - issue_start: datetime = None, - issue_end: datetime = datetime.now(timezone.utc) + num_issues: Optional[int] = 1, + issue_start: Optional[datetime] = None, + issue_end: datetime = datetime.utcnow() ) -> Sequence[datetime]: """Determine the available issue date/times @@ -148,8 +148,8 @@ def get_issues(self, def get_valids(self, issue: datetime, - valid_start: datetime = None, - valid_end: datetime = None) -> Sequence[Tuple[datetime, str]]: + valid_start: Optional[datetime] = None, + valid_end: Optional[datetime] = None) -> Optional[Sequence[Tuple[datetime, str]]]: """Get all objects consistent with the passed issue date/time and filter by valid range Args: @@ -164,7 +164,8 @@ def get_valids(self, object's location) and the object's location (path) """ if valid_start and valid_start == valid_end: - return [self.check_for(issue, valid_start)] + valids_and_filenames = self.check_for(issue, valid_start) + return [valids_and_filenames] if valids_and_filenames is not None else None dir_path = self.path_builder.build_dir(issue=issue) diff --git a/python/idsse_common/idsse/common/config.py b/python/idsse_common/idsse/common/config.py index a0a4f2a5..03311ead 100644 --- a/python/idsse_common/idsse/common/config.py +++ b/python/idsse_common/idsse/common/config.py @@ -49,7 +49,7 @@ def __init__(self, filepaths = glob.glob(config, recursive=recursive) if len(filepaths) == 0: raise FileNotFoundError - elif len(filepaths) == 1: + if len(filepaths) == 1: self._from_filepath(filepaths[0], keys) else: self._from_filepaths(filepaths, keys) diff --git a/python/idsse_common/idsse/common/utils.py b/python/idsse_common/idsse/common/utils.py index 9deb79d7..96b47b33 100644 --- a/python/idsse_common/idsse/common/utils.py +++ b/python/idsse_common/idsse/common/utils.py @@ -13,7 +13,7 @@ import logging from datetime import datetime, timedelta from subprocess import Popen, PIPE, TimeoutExpired -from typing import Sequence +from typing import Sequence, Optional, Generator, Any logger = logging.getLogger(__name__) @@ -70,7 +70,7 @@ def __delitem__(self, key): del self.__dict__[key] -def exec_cmd(commands: Sequence[str], timeout: int = None) -> Sequence[str]: +def exec_cmd(commands: Sequence[str], timeout: Optional[int] = None) -> Sequence[str]: """Execute the passed commands via a Popen call Args: @@ -145,8 +145,8 @@ def dict_copy_with(old_dict: dict, **kwargs) -> dict: def datetime_gen(dt_: datetime, time_delta: timedelta, - end_dt: datetime = None, - max_num: int = 100) -> datetime: + end_dt: Optional[datetime] = None, + max_num: int = 100) -> Generator[datetime, Any, None]: """Create a date/time sequence generator, given a starting date/time and a time stride Args: diff --git a/python/idsse_common/test/test_aws_utils.py b/python/idsse_common/test/test_aws_utils.py index b0615fc2..1fd19816 100644 --- a/python/idsse_common/test/test_aws_utils.py +++ b/python/idsse_common/test/test_aws_utils.py @@ -107,7 +107,7 @@ def test_aws_cp_retries_with_s3_command_line(aws_utils: AwsUtils, monkeypatch: M mock_exec_cmd_failure) copy_success = aws_utils.aws_cp('s3:/some/path', 's3:/new/path') - assert copy_success == True + assert copy_success assert mock_exec_cmd_failure.call_count == 2 @@ -118,7 +118,7 @@ def test_aws_cp_fails(aws_utils: AwsUtils, monkeypatch: MonkeyPatch): mock_exec_cmd_failure) copy_success = aws_utils.aws_cp('s3:/some/path', 's3:/new/path') - assert copy_success == False + assert not copy_success mock_exec_cmd_failure.call_count == 2 @@ -146,7 +146,7 @@ def test_get_issues(aws_utils: AwsUtils, mock_exec_cmd): def test_get_issues_returns_latest_issue_from_today_if_no_args_passed(aws_utils: AwsUtils, mock_exec_cmd): result = aws_utils.get_issues() assert len(result) == 1 - assert result[0].date() == datetime.now().date() + assert result[0].date() == datetime.utcnow().date() def test_get_valids_all(aws_utils: AwsUtils, mock_exec_cmd): From f1810d4ff891928bc566d110a3082670c56f7bbd Mon Sep 17 00:00:00 2001 From: Mackenzie Grimes - NOAA Affiliate Date: Fri, 28 Jul 2023 00:04:32 -0600 Subject: [PATCH 3/6] aws_utils clean up imports --- python/idsse_common/idsse/common/aws_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/idsse_common/idsse/common/aws_utils.py b/python/idsse_common/idsse/common/aws_utils.py index f5a3135b..7bb7499a 100644 --- a/python/idsse_common/idsse/common/aws_utils.py +++ b/python/idsse_common/idsse/common/aws_utils.py @@ -11,8 +11,8 @@ import logging import os -from datetime import datetime, timedelta, timezone -from typing import List, Tuple, Optional, Sequence +from datetime import datetime, timedelta +from typing import Sequence, Tuple, Optional from .path_builder import PathBuilder from .utils import TimeDelta, datetime_gen, exec_cmd @@ -43,7 +43,7 @@ def get_path(self, issue: datetime, valid: datetime) -> str: lead = TimeDelta(valid-issue) return self.path_builder.build_path(issue=issue, valid=valid, lead=lead) - def aws_ls(self, path: str, prepend_path: bool = True) -> List[str]: + def aws_ls(self, path: str, prepend_path: bool = True) -> Sequence[str]: """Execute an 'ls' on the AWS s3 bucket specified by path Args: From 5cb4da64be456620f7da6c641d50eebef8e36664 Mon Sep 17 00:00:00 2001 From: Mackenzie Grimes - NOAA Affiliate Date: Mon, 31 Jul 2023 11:41:29 -0600 Subject: [PATCH 4/6] update docstrings to match type hints --- python/idsse_common/idsse/common/aws_utils.py | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/python/idsse_common/idsse/common/aws_utils.py b/python/idsse_common/idsse/common/aws_utils.py index 7bb7499a..0131b8be 100644 --- a/python/idsse_common/idsse/common/aws_utils.py +++ b/python/idsse_common/idsse/common/aws_utils.py @@ -99,9 +99,9 @@ def check_for(self, issue: datetime, valid: datetime) -> Optional[Tuple[datetime valid (datetime): The valid date/time used to format the path to the object's location Returns: - Tuple[datetime, str]: If object exists the valid date/time (indicated by object's - location) and the object's location (path) is returned as a tuple, - else the tuple(None, None) is returned + Optional[Tuple[datetime, str]]: A tuple of the valid date/time (indicated by object's + location) and location (path) of a object, or None + if object does not exist """ lead = TimeDelta(valid-issue) filenames = self.aws_ls(self.get_path(issue, valid), prepend_path=False) @@ -112,21 +112,21 @@ def check_for(self, issue: datetime, valid: datetime) -> Optional[Tuple[datetime return None def get_issues(self, - num_issues: Optional[int] = 1, + num_issues: int = 1, issue_start: Optional[datetime] = None, issue_end: datetime = datetime.utcnow() ) -> Sequence[datetime]: """Determine the available issue date/times Args: - num_issues (int, optional): Maximum number of issue to return. Defaults to 1. + num_issues (int): Maximum number of issue to return. Defaults to 1. issue_start (datetime, optional): The oldest date/time to look for. Defaults to None. - issue_end (datetime, optional): The newest date/time to look for. Defaults to None. + issue_end (datetime): The newest date/time to look for. Defaults to now (UTC). Returns: - Sequence[Tuple[datetime, str]]: A sequence of issue date/times + Sequence[datetime]: A sequence of issue date/times """ - issues_found = [] + issues_found: Sequence[datetime] = [] if issue_start: datetimes = datetime_gen(issue_end, timedelta(hours=-1), issue_start, num_issues) else: @@ -149,7 +149,7 @@ def get_issues(self, def get_valids(self, issue: datetime, valid_start: Optional[datetime] = None, - valid_end: Optional[datetime] = None) -> Optional[Sequence[Tuple[datetime, str]]]: + valid_end: Optional[datetime] = None) -> Sequence[Tuple[datetime, str]]: """Get all objects consistent with the passed issue date/time and filter by valid range Args: @@ -161,11 +161,12 @@ def get_valids(self, Returns: Sequence[Tuple[datetime, str]]: A sequence of tuples with valid date/time (indicated by - object's location) and the object's location (path) + object's location) and the object's location (path). + Empty Sequence if no valids found for given time range. """ if valid_start and valid_start == valid_end: valids_and_filenames = self.check_for(issue, valid_start) - return [valids_and_filenames] if valids_and_filenames is not None else None + return [valids_and_filenames] if valids_and_filenames is not None else [] dir_path = self.path_builder.build_dir(issue=issue) From 27e67b69bb75e7fb505d016243cc5179f7a72963 Mon Sep 17 00:00:00 2001 From: Mackenzie Grimes - NOAA Affiliate Date: Mon, 31 Jul 2023 11:57:01 -0600 Subject: [PATCH 5/6] more type hint fixes --- python/idsse_common/idsse/common/aws_utils.py | 2 +- python/idsse_common/idsse/common/config.py | 4 ++-- python/idsse_common/idsse/common/log_util.py | 4 ++-- python/idsse_common/idsse/common/publish_confirm.py | 2 +- python/idsse_common/idsse/common/utils.py | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/python/idsse_common/idsse/common/aws_utils.py b/python/idsse_common/idsse/common/aws_utils.py index 0131b8be..aa5fb891 100644 --- a/python/idsse_common/idsse/common/aws_utils.py +++ b/python/idsse_common/idsse/common/aws_utils.py @@ -85,7 +85,7 @@ def aws_cp(self, path: str, dest: str) -> bool: commands = ['aws', 's3', '--no-sign-request', 'cp', path, dest] exec_cmd(commands) return True - except Exception: # pytest: disable=broad-exception-caught + except Exception: # pylint: disable=broad-exception-caught return False finally: pass diff --git a/python/idsse_common/idsse/common/config.py b/python/idsse_common/idsse/common/config.py index 03311ead..c0a749c9 100644 --- a/python/idsse_common/idsse/common/config.py +++ b/python/idsse_common/idsse/common/config.py @@ -13,7 +13,7 @@ import json import logging from inspect import signature -from typing import Self, Union, List +from typing import Self, Union, List, Optional logger = logging.getLogger(__name__) @@ -23,7 +23,7 @@ class Config: def __init__(self, config: Union[dict, List[dict], str], - keys: Union[list, str] = None, + keys: Optional[Union[list, str]] = None, recursive: bool = False, ignore_missing: bool = False) -> None: diff --git a/python/idsse_common/idsse/common/log_util.py b/python/idsse_common/idsse/common/log_util.py index d05d59d3..a50941c9 100644 --- a/python/idsse_common/idsse/common/log_util.py +++ b/python/idsse_common/idsse/common/log_util.py @@ -31,7 +31,7 @@ def set_corr_id_context_var(originator: str, key: uuid = None, issue_dt: Union[str, datetime] = None) -> None: if not key: key = uuid.uuid4() - + if issue_dt: if not isinstance(issue_dt, str): issue_dt = to_iso(issue_dt) @@ -72,7 +72,7 @@ def get_default_log_config(level, with_corr_id=True): format_str = '%(asctime)-15s %(name)-5s %(levelname)-8s %(corr_id)s %(module)s::%(funcName)s(line %(lineno)d) %(message)s' else: format_str = '%(asctime)-15s %(name)-5s %(levelname)-8s %(module)s::%(funcName)s(line %(lineno)d) %(message)s' - + return { 'version': 1, 'disable_existing_loggers': False, diff --git a/python/idsse_common/idsse/common/publish_confirm.py b/python/idsse_common/idsse/common/publish_confirm.py index 27eb1a24..160e8eef 100644 --- a/python/idsse_common/idsse/common/publish_confirm.py +++ b/python/idsse_common/idsse/common/publish_confirm.py @@ -6,9 +6,9 @@ import logging.config import json import time +from threading import Thread import pika -from threading import Thread from pika.exchange_type import ExchangeType from idsse.common.log_util import get_default_log_config, set_corr_id_context_var diff --git a/python/idsse_common/idsse/common/utils.py b/python/idsse_common/idsse/common/utils.py index 96b47b33..56c66b8c 100644 --- a/python/idsse_common/idsse/common/utils.py +++ b/python/idsse_common/idsse/common/utils.py @@ -96,8 +96,8 @@ def exec_cmd(commands: Sequence[str], timeout: Optional[int] = None) -> Sequence raise OSError(process.returncode, errs.decode()) try: ans = outs.decode().splitlines() - except Exception as e: # pylint: disable=broad-exception-caught - raise RuntimeError(e) from e + except Exception as exc: # pylint: disable=broad-exception-caught + raise RuntimeError(exc) from exc return ans From b87ae69caff59eea9544675de184d827a7576728 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 31 Jul 2023 21:56:36 +0000 Subject: [PATCH 6/6] Update coverage on Readme --- README.md | 2 +- python/idsse_common/test/coverage.txt | 6 +++--- python/idsse_common/test/pytest.xml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index fe99a55e..e3fe1f52 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Pytest](https://github.com/NOAA-GSL/idss-engine-commons/actions/workflows/run-tests.yml/badge.svg?branch=main)](https://github.com/NOAA-GSL/idss-engine-commons/actions/workflows/run-tests.yml) -Coverage
Coverage Report
FileStmtsMissCoverMissing
python/idsse_common/idsse/common
   __init__.py00100% 
   aws_utils.py7755 94%
   config.py7499 88%
   json_message.py222222 0%
   log_util.py383838 0%
   path_builder.py1271010 92%
   publish_confirm.py141141141 0%
   utils.py783333 58%
TOTAL55725854% 
+Coverage
Coverage Report
FileStmtsMissCoverMissing
python/idsse_common/idsse/common
   __init__.py00100% 
   aws_utils.py7866 92%
   config.py7499 88%
   json_message.py222222 0%
   log_util.py383838 0%
   path_builder.py1271010 92%
   publish_confirm.py141141141 0%
   utils.py783333 58%
TOTAL55825954% 
## Overview diff --git a/python/idsse_common/test/coverage.txt b/python/idsse_common/test/coverage.txt index 90da9f2e..29fd2126 100644 --- a/python/idsse_common/test/coverage.txt +++ b/python/idsse_common/test/coverage.txt @@ -15,7 +15,7 @@ test_utils.py .......... [100%] Name Stmts Miss Cover ----------------------------------------------------------------------------------------------------------------------------------- /home/runner/work/idss-engine-commons/idss-engine-commons/python/idsse_common/idsse/common/__init__.py 0 0 100% -/home/runner/work/idss-engine-commons/idss-engine-commons/python/idsse_common/idsse/common/aws_utils.py 77 5 94% +/home/runner/work/idss-engine-commons/idss-engine-commons/python/idsse_common/idsse/common/aws_utils.py 78 6 92% /home/runner/work/idss-engine-commons/idss-engine-commons/python/idsse_common/idsse/common/config.py 74 9 88% /home/runner/work/idss-engine-commons/idss-engine-commons/python/idsse_common/idsse/common/json_message.py 22 22 0% /home/runner/work/idss-engine-commons/idss-engine-commons/python/idsse_common/idsse/common/log_util.py 38 38 0% @@ -23,6 +23,6 @@ Name /home/runner/work/idss-engine-commons/idss-engine-commons/python/idsse_common/idsse/common/publish_confirm.py 141 141 0% /home/runner/work/idss-engine-commons/idss-engine-commons/python/idsse_common/idsse/common/utils.py 78 33 58% ----------------------------------------------------------------------------------------------------------------------------------- -TOTAL 557 258 54% +TOTAL 558 259 54% -============================== 58 passed in 0.47s ============================== +============================== 58 passed in 0.38s ============================== diff --git a/python/idsse_common/test/pytest.xml b/python/idsse_common/test/pytest.xml index 2134c5fc..f0fff8a2 100644 --- a/python/idsse_common/test/pytest.xml +++ b/python/idsse_common/test/pytest.xml @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file