diff --git a/README.md b/README.md index fe99a55e..e3fe1f52 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Pytest](https://github.com/NOAA-GSL/idss-engine-commons/actions/workflows/run-tests.yml/badge.svg?branch=main)](https://github.com/NOAA-GSL/idss-engine-commons/actions/workflows/run-tests.yml) -Coverage
Coverage Report
FileStmtsMissCoverMissing
python/idsse_common/idsse/common
   __init__.py00100% 
   aws_utils.py7755 94%
   config.py7499 88%
   json_message.py222222 0%
   log_util.py383838 0%
   path_builder.py1271010 92%
   publish_confirm.py141141141 0%
   utils.py783333 58%
TOTAL55725854% 
+Coverage
Coverage Report
FileStmtsMissCoverMissing
python/idsse_common/idsse/common
   __init__.py00100% 
   aws_utils.py7866 92%
   config.py7499 88%
   json_message.py222222 0%
   log_util.py383838 0%
   path_builder.py1271010 92%
   publish_confirm.py141141141 0%
   utils.py783333 58%
TOTAL55825954% 
## Overview diff --git a/python/idsse_common/idsse/common/aws_utils.py b/python/idsse_common/idsse/common/aws_utils.py index 851be68d..aa5fb891 100644 --- a/python/idsse_common/idsse/common/aws_utils.py +++ b/python/idsse_common/idsse/common/aws_utils.py @@ -11,8 +11,8 @@ import logging import os -from datetime import datetime, timedelta, timezone -from typing import Sequence, Tuple +from datetime import datetime, timedelta +from typing import Sequence, Tuple, Optional from .path_builder import PathBuilder from .utils import TimeDelta, datetime_gen, exec_cmd @@ -85,13 +85,13 @@ def aws_cp(self, path: str, dest: str) -> bool: commands = ['aws', 's3', '--no-sign-request', 'cp', path, dest] exec_cmd(commands) return True - except: + except Exception: # pylint: disable=broad-exception-caught return False finally: pass - - def check_for(self, issue: datetime, valid: datetime) -> Tuple[datetime, str]: + + def check_for(self, issue: datetime, valid: datetime) -> Optional[Tuple[datetime, str]]: """Checks if an object passed issue/valid exists Args: @@ -99,9 +99,9 @@ def check_for(self, issue: datetime, valid: datetime) -> Tuple[datetime, str]: valid (datetime): The valid date/time used to format the path to the object's location Returns: - Tuple[datetime, str]: If object exists the valid date/time (indicated by object's - location) and the object's location (path) is returned as a tuple, - else the tuple(None, None) is returned + Optional[Tuple[datetime, str]]: A tuple of the valid date/time (indicated by object's + location) and location (path) of a object, or None + if object does not exist """ lead = TimeDelta(valid-issue) filenames = self.aws_ls(self.get_path(issue, valid), prepend_path=False) @@ -113,20 +113,20 @@ def check_for(self, issue: datetime, valid: datetime) -> Tuple[datetime, str]: def get_issues(self, num_issues: int = 1, - issue_start: datetime = None, - issue_end: datetime = datetime.now(timezone.utc) + issue_start: Optional[datetime] = None, + issue_end: datetime = datetime.utcnow() ) -> Sequence[datetime]: """Determine the available issue date/times Args: - num_issues (int, optional): Maximum number of issue to return. Defaults to 1. + num_issues (int): Maximum number of issue to return. Defaults to 1. issue_start (datetime, optional): The oldest date/time to look for. Defaults to None. - issue_end (datetime, optional): The newest date/time to look for. Defaults to None. + issue_end (datetime): The newest date/time to look for. Defaults to now (UTC). Returns: - Sequence[Tuple[datetime, str]]: A sequence of issue date/times + Sequence[datetime]: A sequence of issue date/times """ - issues_found = [] + issues_found: Sequence[datetime] = [] if issue_start: datetimes = datetime_gen(issue_end, timedelta(hours=-1), issue_start, num_issues) else: @@ -148,8 +148,8 @@ def get_issues(self, def get_valids(self, issue: datetime, - valid_start: datetime = None, - valid_end: datetime = None) -> Sequence[Tuple[datetime, str]]: + valid_start: Optional[datetime] = None, + valid_end: Optional[datetime] = None) -> Sequence[Tuple[datetime, str]]: """Get all objects consistent with the passed issue date/time and filter by valid range Args: @@ -161,10 +161,12 @@ def get_valids(self, Returns: Sequence[Tuple[datetime, str]]: A sequence of tuples with valid date/time (indicated by - object's location) and the object's location (path) + object's location) and the object's location (path). + Empty Sequence if no valids found for given time range. """ if valid_start and valid_start == valid_end: - return [self.check_for(issue, valid_start)] + valids_and_filenames = self.check_for(issue, valid_start) + return [valids_and_filenames] if valids_and_filenames is not None else [] dir_path = self.path_builder.build_dir(issue=issue) diff --git a/python/idsse_common/idsse/common/config.py b/python/idsse_common/idsse/common/config.py index a0a4f2a5..c0a749c9 100644 --- a/python/idsse_common/idsse/common/config.py +++ b/python/idsse_common/idsse/common/config.py @@ -13,7 +13,7 @@ import json import logging from inspect import signature -from typing import Self, Union, List +from typing import Self, Union, List, Optional logger = logging.getLogger(__name__) @@ -23,7 +23,7 @@ class Config: def __init__(self, config: Union[dict, List[dict], str], - keys: Union[list, str] = None, + keys: Optional[Union[list, str]] = None, recursive: bool = False, ignore_missing: bool = False) -> None: @@ -49,7 +49,7 @@ def __init__(self, filepaths = glob.glob(config, recursive=recursive) if len(filepaths) == 0: raise FileNotFoundError - elif len(filepaths) == 1: + if len(filepaths) == 1: self._from_filepath(filepaths[0], keys) else: self._from_filepaths(filepaths, keys) diff --git a/python/idsse_common/idsse/common/log_util.py b/python/idsse_common/idsse/common/log_util.py index d05d59d3..a50941c9 100644 --- a/python/idsse_common/idsse/common/log_util.py +++ b/python/idsse_common/idsse/common/log_util.py @@ -31,7 +31,7 @@ def set_corr_id_context_var(originator: str, key: uuid = None, issue_dt: Union[str, datetime] = None) -> None: if not key: key = uuid.uuid4() - + if issue_dt: if not isinstance(issue_dt, str): issue_dt = to_iso(issue_dt) @@ -72,7 +72,7 @@ def get_default_log_config(level, with_corr_id=True): format_str = '%(asctime)-15s %(name)-5s %(levelname)-8s %(corr_id)s %(module)s::%(funcName)s(line %(lineno)d) %(message)s' else: format_str = '%(asctime)-15s %(name)-5s %(levelname)-8s %(module)s::%(funcName)s(line %(lineno)d) %(message)s' - + return { 'version': 1, 'disable_existing_loggers': False, diff --git a/python/idsse_common/idsse/common/publish_confirm.py b/python/idsse_common/idsse/common/publish_confirm.py index 27eb1a24..160e8eef 100644 --- a/python/idsse_common/idsse/common/publish_confirm.py +++ b/python/idsse_common/idsse/common/publish_confirm.py @@ -6,9 +6,9 @@ import logging.config import json import time +from threading import Thread import pika -from threading import Thread from pika.exchange_type import ExchangeType from idsse.common.log_util import get_default_log_config, set_corr_id_context_var diff --git a/python/idsse_common/idsse/common/utils.py b/python/idsse_common/idsse/common/utils.py index 9deb79d7..56c66b8c 100644 --- a/python/idsse_common/idsse/common/utils.py +++ b/python/idsse_common/idsse/common/utils.py @@ -13,7 +13,7 @@ import logging from datetime import datetime, timedelta from subprocess import Popen, PIPE, TimeoutExpired -from typing import Sequence +from typing import Sequence, Optional, Generator, Any logger = logging.getLogger(__name__) @@ -70,7 +70,7 @@ def __delitem__(self, key): del self.__dict__[key] -def exec_cmd(commands: Sequence[str], timeout: int = None) -> Sequence[str]: +def exec_cmd(commands: Sequence[str], timeout: Optional[int] = None) -> Sequence[str]: """Execute the passed commands via a Popen call Args: @@ -96,8 +96,8 @@ def exec_cmd(commands: Sequence[str], timeout: int = None) -> Sequence[str]: raise OSError(process.returncode, errs.decode()) try: ans = outs.decode().splitlines() - except Exception as e: # pylint: disable=broad-exception-caught - raise RuntimeError(e) from e + except Exception as exc: # pylint: disable=broad-exception-caught + raise RuntimeError(exc) from exc return ans @@ -145,8 +145,8 @@ def dict_copy_with(old_dict: dict, **kwargs) -> dict: def datetime_gen(dt_: datetime, time_delta: timedelta, - end_dt: datetime = None, - max_num: int = 100) -> datetime: + end_dt: Optional[datetime] = None, + max_num: int = 100) -> Generator[datetime, Any, None]: """Create a date/time sequence generator, given a starting date/time and a time stride Args: diff --git a/python/idsse_common/test/coverage.txt b/python/idsse_common/test/coverage.txt index 9e8c047e..29fd2126 100644 --- a/python/idsse_common/test/coverage.txt +++ b/python/idsse_common/test/coverage.txt @@ -15,7 +15,7 @@ test_utils.py .......... [100%] Name Stmts Miss Cover ----------------------------------------------------------------------------------------------------------------------------------- /home/runner/work/idss-engine-commons/idss-engine-commons/python/idsse_common/idsse/common/__init__.py 0 0 100% -/home/runner/work/idss-engine-commons/idss-engine-commons/python/idsse_common/idsse/common/aws_utils.py 77 5 94% +/home/runner/work/idss-engine-commons/idss-engine-commons/python/idsse_common/idsse/common/aws_utils.py 78 6 92% /home/runner/work/idss-engine-commons/idss-engine-commons/python/idsse_common/idsse/common/config.py 74 9 88% /home/runner/work/idss-engine-commons/idss-engine-commons/python/idsse_common/idsse/common/json_message.py 22 22 0% /home/runner/work/idss-engine-commons/idss-engine-commons/python/idsse_common/idsse/common/log_util.py 38 38 0% @@ -23,6 +23,6 @@ Name /home/runner/work/idss-engine-commons/idss-engine-commons/python/idsse_common/idsse/common/publish_confirm.py 141 141 0% /home/runner/work/idss-engine-commons/idss-engine-commons/python/idsse_common/idsse/common/utils.py 78 33 58% ----------------------------------------------------------------------------------------------------------------------------------- -TOTAL 557 258 54% +TOTAL 558 259 54% -============================== 58 passed in 0.46s ============================== +============================== 58 passed in 0.38s ============================== diff --git a/python/idsse_common/test/pytest.xml b/python/idsse_common/test/pytest.xml index 5b28ffd3..f0fff8a2 100644 --- a/python/idsse_common/test/pytest.xml +++ b/python/idsse_common/test/pytest.xml @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/python/idsse_common/test/test_aws_utils.py b/python/idsse_common/test/test_aws_utils.py index b0615fc2..1fd19816 100644 --- a/python/idsse_common/test/test_aws_utils.py +++ b/python/idsse_common/test/test_aws_utils.py @@ -107,7 +107,7 @@ def test_aws_cp_retries_with_s3_command_line(aws_utils: AwsUtils, monkeypatch: M mock_exec_cmd_failure) copy_success = aws_utils.aws_cp('s3:/some/path', 's3:/new/path') - assert copy_success == True + assert copy_success assert mock_exec_cmd_failure.call_count == 2 @@ -118,7 +118,7 @@ def test_aws_cp_fails(aws_utils: AwsUtils, monkeypatch: MonkeyPatch): mock_exec_cmd_failure) copy_success = aws_utils.aws_cp('s3:/some/path', 's3:/new/path') - assert copy_success == False + assert not copy_success mock_exec_cmd_failure.call_count == 2 @@ -146,7 +146,7 @@ def test_get_issues(aws_utils: AwsUtils, mock_exec_cmd): def test_get_issues_returns_latest_issue_from_today_if_no_args_passed(aws_utils: AwsUtils, mock_exec_cmd): result = aws_utils.get_issues() assert len(result) == 1 - assert result[0].date() == datetime.now().date() + assert result[0].date() == datetime.utcnow().date() def test_get_valids_all(aws_utils: AwsUtils, mock_exec_cmd):