From 5fc2f67cba277026a3d5bcec570cdd20c8bcdb09 Mon Sep 17 00:00:00 2001 From: Claas Date: Wed, 23 Oct 2024 23:08:48 +0200 Subject: [PATCH] resolving issues raised by ruff (work in progress) --- ruff.toml | 7 + src/farn/cli/farn.py | 20 +- src/farn/core/case.py | 783 ++++++++------- src/farn/core/parameter.py | 85 +- src/farn/farn.py | 1549 +++++++++++++++--------------- src/farn/run/batchProcess.py | 145 ++- src/farn/run/cli/batchProcess.py | 7 +- src/farn/run/subProcess.py | 52 +- src/farn/run/utils/threading.py | 117 ++- src/farn/sampling/sampling.py | 225 +++-- src/farn/utils/logging.py | 31 +- src/farn/utils/os.py | 2 +- tests/cli/test_farn_cli.py | 50 +- tests/conftest.py | 20 + tests/test_cases.py | 680 ++++++------- tests/test_farn.py | 49 +- tests/test_sampling.py | 140 +-- 17 files changed, 2013 insertions(+), 1949 deletions(-) diff --git a/ruff.toml b/ruff.toml index c5ed14bd..79708cc6 100644 --- a/ruff.toml +++ b/ruff.toml @@ -20,6 +20,13 @@ select = [ "ALL", ] ignore = [ + # Ruff lint rules temporarily ignored, but which should be reactivated and resolved in the future. + "D", # Missing docstrings <- @TODO: reactivate and resolve docstring issues @CLAROS, 2024-10-21 + "N999", # Invalid module name <- @TODO: reactivate and resolve @CLAROS, 2024-10-21 + "C901", # Function is too complex <- @TODO: reactivate and resolve print statements @CLAROS, 2024-10-21 + "PLR0911", # Too many return statements <- @TODO: reactivate and resolve @CLAROS, 2024-10-21 + "PLR0912", # Too many branches <- @TODO: reactivate and resolve @CLAROS, 2024-10-21 + "PLR0915", # Too many statements <- @TODO: reactivate and resolve @CLAROS, 2024-10-21 # Ruff lint rules considered as too strict and hence ignored "ANN101", # Missing type annotation for `self` argument in instance methods (NOTE: also listed as deprecated by Ruff) "ANN102", # Missing type annotation for `cls` argument in class methods (NOTE: also listed as deprecated by Ruff) diff --git a/src/farn/cli/farn.py b/src/farn/cli/farn.py index bca53c25..c2ed7ac5 100755 --- a/src/farn/cli/farn.py +++ b/src/farn/cli/farn.py @@ -6,7 +6,6 @@ import sys from argparse import ArgumentParser from pathlib import Path -from typing import Tuple, Union # Remove current directory from Python search path. # Only through this trick it is possible that the current CLI file 'farn.py' @@ -132,12 +131,11 @@ def _argparser() -> argparse.ArgumentParser: return parser -def main(): +def main() -> None: """Entry point for console script as configured in setup.cfg. Runs the command line interface and parses arguments and options entered on the console. """ - parser = _argparser() try: args = parser.parse_args() @@ -154,14 +152,14 @@ def main(): log_level_console = "ERROR" if args.quiet else log_level_console log_level_console = "DEBUG" if args.verbose else log_level_console # ..to file - log_file: Union[Path, None] = Path(args.log) if args.log else None + log_file: Path | None = Path(args.log) if args.log else None log_level_file: str = args.log_level configure_logging(log_level_console, log_file, log_level_file) farn_dict_file: Path = Path(args.farnDict) sample: bool = args.sample generate: bool = args.generate - command: Union[str, None] = args.execute + command: str | None = args.execute batch: bool = args.batch test: bool = args.test @@ -200,7 +198,7 @@ def main(): ) -def _generate_barnsley_fern(): +def _generate_barnsley_fern() -> None: """ easter egg: Barnsley fern. @@ -224,19 +222,19 @@ def _generate_barnsley_fern(): from PIL import Image from PIL.ImageDraw import ImageDraw - def t1(p: Tuple[float, float]) -> Tuple[float, float]: + def t1(p: tuple[float, float]) -> tuple[float, float]: """1%.""" return (0.0, 0.16 * p[1]) - def t2(p: Tuple[float, float]) -> Tuple[float, float]: + def t2(p: tuple[float, float]) -> tuple[float, float]: """85%.""" return (0.85 * p[0] + 0.04 * p[1], -0.04 * p[0] + 0.85 * p[1] + 1.6) - def t3(p: Tuple[float, float]) -> Tuple[float, float]: + def t3(p: tuple[float, float]) -> tuple[float, float]: """7%.""" return (0.2 * p[0] - 0.26 * p[1], 0.23 * p[0] + 0.22 * p[1] + 1.6) - def t4(p: Tuple[float, float]) -> Tuple[float, float]: + def t4(p: tuple[float, float]) -> tuple[float, float]: """7%.""" return (-0.15 * p[0] + 0.28 * p[1], 0.26 * p[0] + 0.24 * p[1] + 0.44) @@ -245,7 +243,7 @@ def t4(p: Tuple[float, float]) -> Tuple[float, float]: im = Image.new("RGBA", (x_size, x_size)) draw = ImageDraw(im) - p: Tuple[float, float] = (0, 0) + p: tuple[float, float] = (0, 0) end = 20000 ii = 0 scale = 100 diff --git a/src/farn/core/case.py b/src/farn/core/case.py index 4a3946ab..ed5d050a 100644 --- a/src/farn/core/case.py +++ b/src/farn/core/case.py @@ -1,395 +1,388 @@ -# pyright: reportUnknownMemberType=false -import logging -import re -from copy import deepcopy -from enum import IntEnum -from pathlib import Path -from typing import ( - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Sequence, - Set, - Union, -) - -import numpy as np -from dictIO.utils.path import relative_path -from numpy import ndarray -from pandas import DataFrame, Series - -from farn.core import Parameter - -__ALL__ = [ - "CaseStatus", - "Case", - "Cases", -] - -logger = logging.getLogger(__name__) - - -class CaseStatus(IntEnum): - """Enumeration class allowing an algorithm that processes cases, i.e. a simulator or case processor, - to indicate the state a case iscurrently in. - """ - - NONE = 0 - FAILURE = 1 - PREPARED = 10 - RUNNING = 20 - SUCCESS = 30 - - -class Case: - """Dataclass holding case attributes. - - Case holds all relevant attributes needed by farn to process cases, e.g. - - condition - - parameter names and associated values - - commands - - .. - """ - - def __init__( - self, - case: str = "", - layer: str = "", - level: int = 0, - no_of_samples: int = 0, - index: int = 0, - path: Union[Path, None] = None, - is_leaf: bool = False, - condition: Union[MutableMapping[str, str], None] = None, - parameters: Union[MutableSequence[Parameter], None] = None, - command_sets: Union[MutableMapping[str, List[str]], None] = None, - ): - self.case: Union[str, None] = case - self.layer: Union[str, None] = layer - self.level: int = level - self.no_of_samples: int = no_of_samples - self.index: int = index - self.path: Path = path or Path.cwd() - self.is_leaf: bool = is_leaf - self.condition: MutableMapping[str, str] = condition or {} - self.parameters: MutableSequence[Parameter] = parameters or [] - self.command_sets: MutableMapping[str, List[str]] = command_sets or {} - self.status: CaseStatus = CaseStatus.NONE - - @property - def is_valid(self) -> bool: - """Evaluates whether the case matches the configured filter expression. - - A case is considered valid if it fulfils the filter citeria configured in farnDict for the respective layer. - - Returns - ------- - bool - result of validity check. True indicates the case is valid, False not valid. - """ - - # Check whether the '_condition' element is defined. Without it, case is in any case considered valid. - if not self.condition: - return True - - # Check whether filter expression is defined. - # If filter expression is missing, condition cannot be evaluated but case is, by default, still considered valid. - filter_expression = self.condition["_filter"] if "_filter" in self.condition else None - if not filter_expression: - logger.warning( - f"Layer {self.layer}: _condition element found but no _filter element defined therein. " - f"As the filter expression is missing, the condition cannot be evalued. Case {self.case} is hence considered valid. " - ) - return True - - # Check whether optional argument '_action' is defined. Use default action, if not. - action = self.condition["_action"] if "_action" in self.condition else None - if not action: - logger.warning( - f"Layer {self.layer}: No _action defined in _condition element. Default action 'exclude' is used. " - ) - action = "exclude" - - # Check for formal errors that lead to invalidity - if not self.parameters: - logger.warning( - f"Layer {self.layer}, case {self.case} validity check: case {self.case} is invalid: " - f"A filter expression {filter_expression} is defined, but no parameters exist. " - ) - return False - for parameter in self.parameters: - if not parameter.name: - logger.warning( - f"Layer {self.layer}, case {self.case} validity check: case {self.case} is invalid: " - f"A filter expression {filter_expression} is defined, " - f"but at least one parameter name is missing. " - ) - return False - if not parameter.value: - logger.warning( - f"Layer {self.layer}, case {self.case} validity check: case {self.case} is invalid: " - f"A filter expression {filter_expression} is defined and parameter names exist, " - f"but parameter values are missing. " - f"Parameter name: {parameter.name} " - f"Parameter value: None " - ) - return False - - # transfer a white list of case properties to locals() for subsequent filtering - available_vars: Set[str] = set() - for attribute in dir(self): - try: - if attribute in [ - "case", - "layer", - "level", - "index", - "path" "is_leaf", - "no_of_samples", - "condition", - "command_sets", - ]: - locals()[attribute] = eval(f"self.{attribute}") - available_vars.add(attribute) - except Exception: - logger.exception( - f"Layer {self.layer}, case {self.case} validity check: case {self.case} is invalid: " - f"Reading case property '{attribute}' failed." - ) - return False - - # Read all parameter names and their associated values defined in current case, and assign them to local in-memory variables - for parameter in self.parameters: - if parameter.name and not re.match("^_", parameter.name): - try: - exec(f"{parameter.name} = {parameter.value}") - available_vars.add(parameter.name) - except Exception: - logger.exception( - f"Layer {self.layer}, case {self.case} validity check: case {self.case} is invalid: " - f"Reading parameter {parameter.name} with value {parameter.value} failed. " - ) - return False - - logger.debug( - f"Layer {self.layer}, available filter variables in current scope: {'{'+', '.join(available_vars)+'}'}" - ) - - # Evaluate filter expression - filter_expression_evaluates_to_true = False - try: - filter_expression_evaluates_to_true = eval(filter_expression) - except Exception: - # In case evaluation of the filter expression fails, processing will not stop. - # However, a warning will be logged and the respective case will be considered valid. - logger.warning( - f"Layer {self.layer}, case {self.case} evaluation of the filter expression failed:\n" - f"\tOne or more of the variables used in the filter expression are not defined or not accessible in the current layer.\n" - f"\t\tLayer: {self.layer}\n" - f"\t\tLevel: {self.level}\n" - f"\t\tCase: {self.case}\n" - f"\t\tFilter expression: {filter_expression}\n" - f"\t\tParameter names: {[parameter.name for parameter in self.parameters]}\n" - f"\t\tParameter values: {[parameter.value for parameter in self.parameters]} " - ) - - # Finally: Determine case validity based on filter expression and action - if action == "exclude": - if filter_expression_evaluates_to_true: - logger.debug( - f"Layer {self.layer}, case {self.case} validity check: case {self.case} is invalid:\n" - f"\tThe filter expression '{filter_expression}' evaluated to True.\n" - f"\tAction '{action}' performed. Case {self.case} excluded." - ) - return False - return True - if action == "include": - if filter_expression_evaluates_to_true: - logger.debug( - f"Layer {self.layer}, case {self.case} validity check: case {self.case} is valid:\n" - f"\tThe filter expression '{filter_expression}' evaluated to True.\n" - f"\tAction '{action}' performed. Case {self.case} included." - ) - return True - return False - - return True - - def add_parameters( - self, - parameters: Union[MutableSequence[Parameter], MutableMapping[str, str], None] = None, - ): - """Manually add extra parameters.""" - if isinstance(parameters, MutableSequence): - self.parameters.extend(parameters) - - elif isinstance(parameters, MutableMapping): - self.parameters.extend( - Parameter(parameter_name, parameter_value) for parameter_name, parameter_value in parameters.items() - ) - - else: - logger.error( - f"Layer {self.layer}, case {self.case} add_parameters failed:\n" - f"\tWrong input data format for additional parameters.\n" - ) - exit(1) - - return True - - def to_dict(self) -> Dict[str, Any]: - """Return a dict with all case attributes. - - Returns - ------- - Dict[str, Any] - dict with all case attributes - """ - return { - "case": self.case, - "layer": self.layer, - "level": self.level, - "index": self.index, - "path": self.path, - "is_leaf": self.is_leaf, - "no_of_samples": self.no_of_samples, - "condition": self.condition, - "parameters": {parameter.name: parameter.value for parameter in self.parameters or []}, - "commands": self.command_sets, - "status": self.status, - } - - def __str__(self): - return str(self.to_dict()) - - def __eq__(self, __o: object) -> bool: - return str(self) == str(__o) - - -class Cases(List[Case]): - """Container Class for Cases. - - Inherits from List[Case] and can hence be transparently used as a Python list type. - However, Cases extends its list base class by two convenience methods: - to_pandas() and to_numpy(), which turn the list of Case objects - into a pandas DataFrame or numpy ndarray, respectively. - """ - - def add_parameters( - self, - parameters: Union[MutableSequence[Parameter], MutableMapping[str, str], None] = None, - ): - """Manually add extra parameters.""" - _cases: List[Case] = deepcopy(self) - for case in _cases: - _ = case.add_parameters(parameters) - - return False - - def to_pandas( - self, - use_path_as_index: bool = True, - parameters_only: bool = False, - ) -> DataFrame: - """Return cases as a pandas Dataframe. - - Returns a DataFrame with case properties and case specific parameter values of all cases. - - Parameters - ---------- - use_path_as_index : bool, optional - turn path column into index column, by default True - parameters_only : bool, optional - reduce DataFrame to contain only the case's parameter values, by default False - - Returns - ------- - DataFrame - DataFrame with case properties and case specific parameter values of all cases. - """ - indices: List[int] = [] - - _cases: List[Case] = deepcopy(self) - for _index, case in enumerate(_cases): - indices.append(_index) - case.path = relative_path(Path.cwd(), case.path) - if case.parameters: - for parameter in case.parameters: - if not parameter.name: - parameter.name = "NA" - - series: Dict[str, Series] = { # pyright: ignore - "case": Series(data=None, dtype=np.dtype(str), name="case"), - "path": Series(data=None, dtype=np.dtype(str), name="path"), - } - - for _index, case in enumerate(_cases): - if case.case: - series["case"].loc[_index] = case.case # pyright: ignore - series["path"].loc[_index] = str(case.path) # pyright: ignore - if case.parameters: - for parameter in case.parameters: - if parameter.name not in series: - series[parameter.name] = Series( - data=None, - dtype=parameter.dtype, # pyright: ignore - name=parameter.name, - ) - if parameter.value is not None: - series[parameter.name].loc[_index] = parameter.value # pyright: ignore - - if parameters_only: - _ = series.pop("case") - if not use_path_as_index: - _ = series.pop("path") - - df_X = DataFrame(data=series) # noqa: N806 - - if use_path_as_index: - df_X.set_index("path", inplace=True) - - return df_X - - def to_numpy(self) -> ndarray[Any, Any]: - """Return parameter values of all cases as a 2-dimensional numpy array. - - Returns - ------- - ndarray[Any, Any] - 2-dimensional numpy array with case specific parameter values of all cases. - """ - df_X: DataFrame = self.to_pandas(parameters_only=True) # noqa: N806 - array: ndarray[Any, Any] = df_X.to_numpy() - return array - - def filter( - self, - levels: Union[int, Sequence[int]] = -1, - valid_only: bool = True, - ) -> "Cases": - """Return a sub-set of cases according to the passed in selection criteria. - - Parameters - ---------- - levels : Union[int, Sequence[int]], optional - return all cases of a distinct level, or a sequence of levels. - level=-1 returns the last level (the leaf cases), by default -1 - valid_only : bool, optional - return only valid cases, i.e cases which pass a filter expression - defined for the case's layer, by default True - - Returns - ------- - Cases - Cases object containing all cases that match the selection criteria. - """ - _levels: List[int] = [levels] if isinstance(levels, int) else list(levels) - filtered_cases: List[Case] - filtered_cases = [case for case in self if case.level in _levels or (case.is_leaf and -1 in _levels)] - - if valid_only: - filtered_cases = [case for case in filtered_cases if case.is_valid] - - return Cases(filtered_cases) +# pyright: reportUnknownMemberType=false +import logging +import re +from collections.abc import MutableMapping, MutableSequence, Sequence +from copy import deepcopy +from enum import IntEnum +from pathlib import Path +from typing import ( + Any, +) + +import numpy as np +from dictIO.utils.path import relative_path +from numpy import ndarray +from pandas import DataFrame, Series + +from farn.core import Parameter + +__ALL__ = [ + "CaseStatus", + "Case", + "Cases", +] + +logger = logging.getLogger(__name__) + + +class CaseStatus(IntEnum): + """Enumeration class allowing an algorithm that processes cases, i.e. a simulator or case processor, + to indicate the state a case iscurrently in. + """ + + NONE = 0 + FAILURE = 1 + PREPARED = 10 + RUNNING = 20 + SUCCESS = 30 + + +class Case: + """Dataclass holding case attributes. + + Case holds all relevant attributes needed by farn to process cases, e.g. + - condition + - parameter names and associated values + - commands + - .. + """ + + def __init__( + self, + case: str = "", + layer: str = "", + level: int = 0, + no_of_samples: int = 0, + index: int = 0, + path: Path | None = None, + is_leaf: bool = False, + condition: MutableMapping[str, str] | None = None, + parameters: MutableSequence[Parameter] | None = None, + command_sets: MutableMapping[str, list[str]] | None = None, + ): + self.case: str | None = case + self.layer: str | None = layer + self.level: int = level + self.no_of_samples: int = no_of_samples + self.index: int = index + self.path: Path = path or Path.cwd() + self.is_leaf: bool = is_leaf + self.condition: MutableMapping[str, str] = condition or {} + self.parameters: MutableSequence[Parameter] = parameters or [] + self.command_sets: MutableMapping[str, list[str]] = command_sets or {} + self.status: CaseStatus = CaseStatus.NONE + + @property + def is_valid(self) -> bool: + """Evaluates whether the case matches the configured filter expression. + + A case is considered valid if it fulfils the filter citeria configured in farnDict for the respective layer. + + Returns + ------- + bool + result of validity check. True indicates the case is valid, False not valid. + """ + # Check whether the '_condition' element is defined. Without it, case is in any case considered valid. + if not self.condition: + return True + + # Check whether filter expression is defined. + # If filter expression is missing, condition cannot be evaluated but case is, by default, still considered valid. + filter_expression = self.condition["_filter"] if "_filter" in self.condition else None + if not filter_expression: + logger.warning( + f"Layer {self.layer}: _condition element found but no _filter element defined therein. " + f"As the filter expression is missing, the condition cannot be evalued. Case {self.case} is hence considered valid. " + ) + return True + + # Check whether optional argument '_action' is defined. Use default action, if not. + action = self.condition["_action"] if "_action" in self.condition else None + if not action: + logger.warning( + f"Layer {self.layer}: No _action defined in _condition element. Default action 'exclude' is used. " + ) + action = "exclude" + + # Check for formal errors that lead to invalidity + if not self.parameters: + logger.warning( + f"Layer {self.layer}, case {self.case} validity check: case {self.case} is invalid: " + f"A filter expression {filter_expression} is defined, but no parameters exist. " + ) + return False + for parameter in self.parameters: + if not parameter.name: + logger.warning( + f"Layer {self.layer}, case {self.case} validity check: case {self.case} is invalid: " + f"A filter expression {filter_expression} is defined, " + f"but at least one parameter name is missing. " + ) + return False + if not parameter.value: + logger.warning( + f"Layer {self.layer}, case {self.case} validity check: case {self.case} is invalid: " + f"A filter expression {filter_expression} is defined and parameter names exist, " + f"but parameter values are missing. " + f"Parameter name: {parameter.name} " + f"Parameter value: None " + ) + return False + + # transfer a white list of case properties to locals() for subsequent filtering + available_vars: set[str] = set() + for attribute in dir(self): + try: + if attribute in [ + "case", + "layer", + "level", + "index", + "path" "is_leaf", + "no_of_samples", + "condition", + "command_sets", + ]: + locals()[attribute] = eval(f"self.{attribute}") + available_vars.add(attribute) + except Exception: + logger.exception( + f"Layer {self.layer}, case {self.case} validity check: case {self.case} is invalid: " + f"Reading case property '{attribute}' failed." + ) + return False + + # Read all parameter names and their associated values defined in current case, and assign them to local in-memory variables + for parameter in self.parameters: + if parameter.name and not re.match("^_", parameter.name): + try: + exec(f"{parameter.name} = {parameter.value}") + available_vars.add(parameter.name) + except Exception: + logger.exception( + f"Layer {self.layer}, case {self.case} validity check: case {self.case} is invalid: " + f"Reading parameter {parameter.name} with value {parameter.value} failed. " + ) + return False + + logger.debug( + f"Layer {self.layer}, available filter variables in current scope: {'{'+', '.join(available_vars)+'}'}" + ) + + # Evaluate filter expression + filter_expression_evaluates_to_true = False + try: + filter_expression_evaluates_to_true = eval(filter_expression) + except Exception: + # In case evaluation of the filter expression fails, processing will not stop. + # However, a warning will be logged and the respective case will be considered valid. + logger.warning( + f"Layer {self.layer}, case {self.case} evaluation of the filter expression failed:\n" + f"\tOne or more of the variables used in the filter expression are not defined or not accessible in the current layer.\n" + f"\t\tLayer: {self.layer}\n" + f"\t\tLevel: {self.level}\n" + f"\t\tCase: {self.case}\n" + f"\t\tFilter expression: {filter_expression}\n" + f"\t\tParameter names: {[parameter.name for parameter in self.parameters]}\n" + f"\t\tParameter values: {[parameter.value for parameter in self.parameters]} " + ) + + # Finally: Determine case validity based on filter expression and action + if action == "exclude": + if filter_expression_evaluates_to_true: + logger.debug( + f"Layer {self.layer}, case {self.case} validity check: case {self.case} is invalid:\n" + f"\tThe filter expression '{filter_expression}' evaluated to True.\n" + f"\tAction '{action}' performed. Case {self.case} excluded." + ) + return False + return True + if action == "include": + if filter_expression_evaluates_to_true: + logger.debug( + f"Layer {self.layer}, case {self.case} validity check: case {self.case} is valid:\n" + f"\tThe filter expression '{filter_expression}' evaluated to True.\n" + f"\tAction '{action}' performed. Case {self.case} included." + ) + return True + return False + + return True + + def add_parameters( + self, + parameters: MutableSequence[Parameter] | MutableMapping[str, str] | None = None, + ): + """Manually add extra parameters.""" + if isinstance(parameters, MutableSequence): + self.parameters.extend(parameters) + + elif isinstance(parameters, MutableMapping): + self.parameters.extend( + Parameter(parameter_name, parameter_value) for parameter_name, parameter_value in parameters.items() + ) + + else: + logger.error( + f"Layer {self.layer}, case {self.case} add_parameters failed:\n" + f"\tWrong input data format for additional parameters.\n" + ) + exit(1) + + return True + + def to_dict(self) -> dict[str, Any]: + """Return a dict with all case attributes. + + Returns + ------- + Dict[str, Any] + dict with all case attributes + """ + return { + "case": self.case, + "layer": self.layer, + "level": self.level, + "index": self.index, + "path": self.path, + "is_leaf": self.is_leaf, + "no_of_samples": self.no_of_samples, + "condition": self.condition, + "parameters": {parameter.name: parameter.value for parameter in self.parameters or []}, + "commands": self.command_sets, + "status": self.status, + } + + def __str__(self) -> None: + return str(self.to_dict()) + + def __eq__(self, __o: object) -> bool: + return str(self) == str(__o) + + +class Cases(list[Case]): + """Container Class for Cases. + + Inherits from List[Case] and can hence be transparently used as a Python list type. + However, Cases extends its list base class by two convenience methods: + to_pandas() and to_numpy(), which turn the list of Case objects + into a pandas DataFrame or numpy ndarray, respectively. + """ + + def add_parameters( + self, + parameters: MutableSequence[Parameter] | MutableMapping[str, str] | None = None, + ): + """Manually add extra parameters.""" + _cases: list[Case] = deepcopy(self) + for case in _cases: + _ = case.add_parameters(parameters) + + return False + + def to_pandas( + self, + use_path_as_index: bool = True, + parameters_only: bool = False, + ) -> DataFrame: + """Return cases as a pandas Dataframe. + + Returns a DataFrame with case properties and case specific parameter values of all cases. + + Parameters + ---------- + use_path_as_index : bool, optional + turn path column into index column, by default True + parameters_only : bool, optional + reduce DataFrame to contain only the case's parameter values, by default False + + Returns + ------- + DataFrame + DataFrame with case properties and case specific parameter values of all cases. + """ + indices: list[int] = [] + + _cases: list[Case] = deepcopy(self) + for _index, case in enumerate(_cases): + indices.append(_index) + case.path = relative_path(Path.cwd(), case.path) + if case.parameters: + for parameter in case.parameters: + if not parameter.name: + parameter.name = "NA" + + series: dict[str, Series] = { # pyright: ignore + "case": Series(data=None, dtype=np.dtype(str), name="case"), + "path": Series(data=None, dtype=np.dtype(str), name="path"), + } + + for _index, case in enumerate(_cases): + if case.case: + series["case"].loc[_index] = case.case # pyright: ignore + series["path"].loc[_index] = str(case.path) # pyright: ignore + if case.parameters: + for parameter in case.parameters: + if parameter.name not in series: + series[parameter.name] = Series( + data=None, + dtype=parameter.dtype, # pyright: ignore + name=parameter.name, + ) + if parameter.value is not None: + series[parameter.name].loc[_index] = parameter.value # pyright: ignore + + if parameters_only: + _ = series.pop("case") + if not use_path_as_index: + _ = series.pop("path") + + df_X = DataFrame(data=series) # noqa: N806 + + if use_path_as_index: + df_X.set_index("path", inplace=True) + + return df_X + + def to_numpy(self) -> ndarray[Any, Any]: + """Return parameter values of all cases as a 2-dimensional numpy array. + + Returns + ------- + ndarray[Any, Any] + 2-dimensional numpy array with case specific parameter values of all cases. + """ + df_X: DataFrame = self.to_pandas(parameters_only=True) # noqa: N806 + array: ndarray[Any, Any] = df_X.to_numpy() + return array + + def filter( + self, + levels: int | Sequence[int] = -1, + valid_only: bool = True, + ) -> "Cases": + """Return a sub-set of cases according to the passed in selection criteria. + + Parameters + ---------- + levels : Union[int, Sequence[int]], optional + return all cases of a distinct level, or a sequence of levels. + level=-1 returns the last level (the leaf cases), by default -1 + valid_only : bool, optional + return only valid cases, i.e cases which pass a filter expression + defined for the case's layer, by default True + + Returns + ------- + Cases + Cases object containing all cases that match the selection criteria. + """ + _levels: list[int] = [levels] if isinstance(levels, int) else list(levels) + filtered_cases: list[Case] + filtered_cases = [case for case in self if case.level in _levels or (case.is_leaf and -1 in _levels)] + + if valid_only: + filtered_cases = [case for case in filtered_cases if case.is_valid] + + return Cases(filtered_cases) diff --git a/src/farn/core/parameter.py b/src/farn/core/parameter.py index 7b84e5ee..52e0d576 100644 --- a/src/farn/core/parameter.py +++ b/src/farn/core/parameter.py @@ -1,43 +1,42 @@ -import logging -from typing import Type, Union - -import numpy as np - -__ALL__ = ["Parameter"] - -logger = logging.getLogger(__name__) - - -class Parameter: - """Dataclass holding the parameter attributes 'name' and 'value'.""" - - def __init__( - self, - name: str = "", - value: Union[float, int, bool, str, None] = None, - ): - self.name: str = name - self.value: Union[float, int, bool, str, None] = value - - @property - def type(self) -> Union[Type[float], Type[int], Type[bool], Type[str], None]: - """Returns the Python type of the parameter. - - Returns - ------- - Union[Type[float], Type[int], Type[bool], Type[str], None] - the Python type - """ - return None if self.value is None else type(self.value) - - @property - def dtype(self) -> Union[np.dtype[float], np.dtype[int], np.dtype[bool], np.dtype[str], None]: # type: ignore - """Returns the numpy dtype of the parameter. - - Returns - ------- - Union[np.dtype[float], np.dtype[int], np.dtype[bool], np.dtype[str], None] - the numpy dtype - """ - _type = self.type - return None if _type is None else np.dtype(_type) # type: ignore +import logging + +import numpy as np + +__ALL__ = ["Parameter"] + +logger = logging.getLogger(__name__) + + +class Parameter: + """Dataclass holding the parameter attributes 'name' and 'value'.""" + + def __init__( + self, + name: str = "", + value: float | int | bool | str | None = None, + ): + self.name: str = name + self.value: float | int | bool | str | None = value + + @property + def type(self) -> type[float] | type[int] | type[bool] | type[str] | None: + """Returns the Python type of the parameter. + + Returns + ------- + Union[Type[float], Type[int], Type[bool], Type[str], None] + the Python type + """ + return None if self.value is None else type(self.value) + + @property + def dtype(self) -> np.dtype[float] | np.dtype[int] | np.dtype[bool] | np.dtype[str] | None: # type: ignore + """Returns the numpy dtype of the parameter. + + Returns + ------- + Union[np.dtype[float], np.dtype[int], np.dtype[bool], np.dtype[str], None] + the numpy dtype + """ + _type = self.type + return None if _type is None else np.dtype(_type) # type: ignore diff --git a/src/farn/farn.py b/src/farn/farn.py index fc8f49a1..22c02324 100644 --- a/src/farn/farn.py +++ b/src/farn/farn.py @@ -1,779 +1,770 @@ -import logging -import os -import platform -import re -from copy import deepcopy -from pathlib import Path -from typing import ( - Any, - Dict, - List, - MutableMapping, - MutableSequence, - MutableSet, - Sequence, - Union, -) - -from dictIO import CppDict, DictReader, DictWriter, create_target_file_name -from dictIO.utils.strings import remove_quotes - -from farn.core import Case, Cases, Parameter -from farn.run.batchProcess import AsyncBatchProcessor -from farn.run.subProcess import execute_in_sub_process -from farn.utils.logging import plural -from farn.utils.os import append_system_variable - -__ALL__ = [ - "run_farn", - "create_samples", - "create_cases", - "create_case_folders", - "create_param_dict_files", - "create_case_list_files", - "execute_command_set", -] - -logger = logging.getLogger(__name__) - - -def run_farn( - farn_dict_file: Union[str, os.PathLike[str]], - sample: bool = False, - generate: bool = False, - command: Union[str, None] = None, - batch: bool = False, - test: bool = False, -) -> Cases: - """Run farn. - - Runs the sampling for all layers as configured in farn dict, - generates the corresponding case folder structure and - executes user-defined shell command sets in all case folders. - - Parameters - ---------- - farn_dict_file : Union[str, os.PathLike[str]] - farnDict file. Contains the farn configuration. - sample : bool, optional - if True, runs the sampling defined for each layer and saves the sampled farnDict file with prefix sampled., by default False - generate : bool, optional - if True, generates the folder structure that spawns all layers and cases defined in farnDict, by default False - command : Union[str, None], optional - executes the given command set in all case folders. The command set must be defined in the commands section of the applicable layer in farnDict., by default None - batch : bool, optional - if True, executes the given command set in batch mode, i.e. asynchronously, by default False - test : bool, optional - if True, runs only first case and returns, by default False - - Returns - ------- - Cases - List containing all valid leaf cases. - - Raises - ------ - FileNotFoundError - if farn_dict_file does not exist - """ - # sourcery skip: extract-method - - # Make sure farn_dict_file argument is of type Path. If not, cast it to Path type. - farn_dict_file = farn_dict_file if isinstance(farn_dict_file, Path) else Path(farn_dict_file) - - # Check whether farn dict file exists - if not farn_dict_file.exists(): - logger.error(f"run_farn: File {farn_dict_file} not found.") - raise FileNotFoundError(farn_dict_file) - - # Set up farn environment - farn_dirs: Dict[str, Path] = _set_up_farn_environment(farn_dict_file) - - # Read farn dict - farn_dict = DictReader.read(farn_dict_file, comments=False) - - # Run sampling and create the samples for all layers in farn dict - if sample: - create_samples(farn_dict) # run sampling - assert farn_dict.source_file is not None - farn_dict.source_file = create_target_file_name( # change filename to 'sampled.*' - farn_dict.source_file, - prefix="sampled.", # type: ignore - ) - logger.info(f"Save sampled farn dict {farn_dict.name}...") # 1 - DictWriter.write(farn_dict, mode="w") # save sampled.* farn dict file - logger.info(f"Saved sampled farn dict in {farn_dict.source_file}.") # 1 - - # Document CLI arguments of current farn call in the farn dict (for traceability) - farn_opts = { - "farnDict": farn_dict.name, - "sample": sample, - "generate": generate, - "execute": command, - "test": test, - } - farn_dict.update({"_farnOpts": farn_opts}) - - # Create all valid cases from the samples defined in farn dict. - cases = create_cases( - farn_dict=farn_dict, - case_dir=farn_dirs["CASEDIR"], - valid_only=True, - ) - - # Generate case folder structure - # and create a case-specific paramDict file in each case folder - if generate: - _ = create_case_folders(cases) - _ = create_param_dict_files(cases) - _ = create_case_list_files( - cases=cases, - target_dir=farn_dirs["ROOTDIR"], - ) - - # Execute a given command set in all case folders - if command: - _ = execute_command_set( - cases=cases, - command_set=command, - batch=batch, - test=test, - ) - - valid_leaf_cases: Cases = cases.filter(levels=-1, valid_only=True) - - logger.info("Successfully finished farn.\n") - - return valid_leaf_cases - - -def create_samples(farn_dict: CppDict): - """Run sampling and create the samples inside all layers of the passed in farn dict. - - Creates the _samples element in each layer and populates it with the discrete samples generated for the parameters defined and varied in the respective layer. - In case the _samples element already exists in a layer, it will be overwritten. - - Parameters - ---------- - farn_dict : CppDict - farn dict the samples shall be created in - """ - from farn.sampling.sampling import DiscreteSampling - - if "_layers" not in farn_dict: - logger.error(f"no '_layers' element in farn dict {farn_dict.name}. Sampling not possible.") - return - - def create_samples_in_layer( - level: int, - layer_name: str, - layer: MutableMapping[str, Any], - ): - """Run sampling and generate the samples in the passed in layer.""" - if "_sampling" not in layer: - logger.error("no '_sampling' element in layer") - return - if "_type" not in layer["_sampling"]: - logger.error("no '_type' element in sampling") - return - - # instantiate and parameterize the sampling object - sampling = DiscreteSampling() - sampling.set_sampling_type(sampling_type=layer["_sampling"]["_type"]) - sampling.set_sampling_parameters( - sampling_parameters=layer["_sampling"], - layer_name=layer_name, - ) - - # in case a _samples element already exists (e.g. from a former run) -> delete it - if "_samples" in layer: - del layer["_samples"] - - # generate the samples and write them into the _samples element of the layer - samples: Dict[str, List[Any]] = sampling.generate_samples() - layer["_samples"] = samples - - # if the layer does not have a _comment element yet: create a default comment - if "_comment" not in layer: - default_comment = f"level {level:2d}, layer {layer_name}" - layer["_comment"] = default_comment - - return - - logger.info(f"Run sampling of {farn_dict.name}...") - - for index, (key, value) in enumerate(farn_dict["_layers"].items()): - create_samples_in_layer( - level=index, - layer_name=key, - layer=value, - ) - - logger.info(f"Successfully ran sampling of {farn_dict.name}.") - - return - - -def create_cases( - farn_dict: MutableMapping[Any, Any], - case_dir: Path, - valid_only: bool = False, -) -> Cases: - """Create cases based on the layers, filter expressions and samples defined in the passed farn dict. - - Creates case objects for all cases derived by recursive permutation of layers and the case specific samples defined per layer. - create_cases() creates one distinct case object for each case, holding all case attributes (parameters) set to their case specific values. - - Optionally, only _valid_ cases can be returned, i.e. cases which fulfill the filter criteria configured for the respective layer. - Invalid cases then get excluded. - - Note: - The corresponding case folder structure is not yet created by create_cases(). - Creating the case folder structure is the responsibility of create_case_folder_structure(). - However, the case_dir argument is passed in to allow create_cases() to already document in each case object - its _intended_ case folder path. This information is then read and used in create_case_folder_structure() - to actually create the case folders. - - Parameters - ---------- - farn_dict : MutableMapping - farn dict. The farn dict must be sampled, e.g. samples must have been generated for all layers defined in the farn dict. - case_dir : Path - directory the case folder structure is (intended) to be generated in. - valid_only: bool - whether or not only valid cases shall be returned, i.e. cases which fulfill the filter criteria configured for the respective layer., by default False - - Returns - ------- - Cases - list of case objects representing all created cases. - """ - log_msg: str = "List all valid cases.." if valid_only else "List all cases.." - logger.info(log_msg) - - # Check default distributions - default_distribution: Dict[str, Any] = {} - - if "_always" in farn_dict: - default_distribution = farn_dict["_always"] - - # Check arguments. - if "_layers" not in farn_dict: - logger.error("create_cases: No '_layers' element contained in farn dict.") - return Cases() - - # Initialize cases list - cases: Cases = Cases() - number_of_invalid_cases: int = 0 - - # Create a local layers list that carries also the layers' name - # to ease sequential and indexed access to individual layers in create_next_level_cases() - layers: List[Dict[str, Any]] = [] - for layer_name, layer in farn_dict["_layers"].items(): - layer_copy: Dict[str, Any] = deepcopy(layer) - layer_copy["_name"] = layer_name - layers.append(layer_copy) - - def create_next_level_cases( - level: int = 0, - base_case: Union[Case, None] = None, - ): - nonlocal cases - nonlocal number_of_invalid_cases - nonlocal layers - - base_case = base_case or Case(path=Path.cwd()) - base_case.parameters = base_case.parameters or [] - - current_layer: Dict[str, Any] = layers[level] - # validity checks for current layer - if "_samples" not in current_layer: - logger.warning( - f"No _samples element found in layer {current_layer['_name']}.\n" - f"Creation of cases for level {level:2d} aborted. " - ) - return - if "_case_name" not in current_layer["_samples"]: - logger.warning( - f"The _samples element in layer {current_layer['_name']} is empty or does not have a _case_name element.\n" - f"Creation of cases for level {level:2d} aborted. " - ) - return - - current_layer_name: str = str(current_layer["_name"]) - current_layer_is_leaf: bool = level == len(layers) - 1 - - no_of_samples_in_current_layer: int = len(current_layer["_samples"]["_case_name"]) - samples_in_current_layer: MutableMapping[str, MutableSequence[float]] = { - param_name: param_values - for param_name, param_values in current_layer["_samples"].items() - if param_name != "_case_name" - } - - parameter_names_used_in_preceeding_layers: MutableSet[str] = { - parameter.name for parameter in base_case.parameters if parameter.name - } - - parameter_names_in_current_layer: MutableSequence[str] = [] - for parameter_name in list(samples_in_current_layer.keys()): - if parameter_name in parameter_names_used_in_preceeding_layers: - logger.warning( - f"The parameter {parameter_name} defined in layer {current_layer_name} had already been defined in a preceeding layer.\n" - f"The preceeding definition prevails. The samples for parameter {parameter_name} defined in layer {current_layer_name} are skipped. " - ) - else: - parameter_names_in_current_layer.append(parameter_name) - - user_variables_in_current_layer: MutableSequence[Parameter] = [] - for key, item in default_distribution.items(): - if not key.startswith("_"): - default_variable = Parameter(name=key, value=item) - user_variables_in_current_layer.append(default_variable) - - for key, item in current_layer.items(): - if not key.startswith("_"): - user_variable = Parameter(name=key, value=item) - if user_variable.name in parameter_names_used_in_preceeding_layers: - logger.warning( - f"The user variable {user_variable.name} defined in layer {current_layer_name} matches a parameter that\n" - f"had already been defined in a preceeding layer.\n" - f"The preceeding definition prevails. The user variable {user_variable.name} defined in layer {current_layer_name} is skipped. " - ) - elif user_variable.name in parameter_names_in_current_layer: - logger.warning( - f"The user variable {user_variable.name} defined in layer {current_layer_name} matches a parameter name defined in the same layer.\n" - f"The preceeding definition prevails. The user variable {user_variable.name} defined in layer {current_layer_name} is skipped. " - ) - else: - user_variables_in_current_layer.append(user_variable) - - condition_in_current_layer: Union[MutableMapping[str, str], None] = ( - current_layer["_condition"] if "_condition" in current_layer else None - ) - commands_in_current_layer: Union[MutableMapping[str, List[str]], None] = ( - current_layer["_commands"] if "_commands" in current_layer else None - ) - - for index, case_name in enumerate(current_layer["_samples"]["_case_name"]): - case_name = remove_quotes(case_name) - - case_parameters: MutableSequence[Parameter] = [ - parameter for parameter in base_case.parameters if parameter.name - ] - case_parameters.extend( - Parameter(parameter_name, samples_in_current_layer[parameter_name][index]) - for parameter_name in parameter_names_in_current_layer - ) - case_parameters.extend(user_variables_in_current_layer) - - case = Case( - case=case_name, - layer=current_layer_name, - level=level, - no_of_samples=no_of_samples_in_current_layer, - index=index, - path=base_case.path / case_name, - is_leaf=current_layer_is_leaf, - condition=condition_in_current_layer, - parameters=case_parameters, - command_sets=commands_in_current_layer, - ) - - if not valid_only or case.is_valid: - cases.append(case) - if not case.is_leaf: # Recursion for next level cases - create_next_level_cases( - level=level + 1, - base_case=case, - ) - else: - number_of_invalid_cases += 1 - - return - - # Commence recursive collection of cases among all layers - base_case = Case(path=case_dir) - create_next_level_cases(level=0, base_case=base_case) - - leaf_cases = [case for case in cases if case.is_leaf] - - log_msg = "" - if valid_only: - log_msg = ( - f"Successfully listed {len(leaf_cases)} valid case{plural(len(leaf_cases))}. " - f'{number_of_invalid_cases} invalid case{plural(number_of_invalid_cases)} {plural(number_of_invalid_cases, "were")} excluded.' - ) - else: - log_msg = f"Successfully listed {len(leaf_cases)} case{plural(len(leaf_cases))}. " - logger.info(log_msg) - - return cases - - -def create_case_folders(cases: MutableSequence[Case]) -> int: - """Create the case folder structure for the passed in cases. - - Parameters - ---------- - cases : MutableSequence[Case] - cases the case folders shall be created for. - - Returns - ------- - int - number of case folders created. - """ - - logger.info("Create case folder structure...") - number_of_case_folders_created: int = 0 - - for case in cases: - logger.debug(f"creating case folder {case.path}") # 1 - case.path.mkdir(parents=True, exist_ok=True) - number_of_case_folders_created += 1 - - logger.info(f"Successfully created {number_of_case_folders_created} case folders.") - - return number_of_case_folders_created - - -def create_param_dict_files(cases: MutableSequence[Case]) -> int: - """Create the case specific paramDict files in the case folders of the passed in cases. - - paramDict files contain the case specific parameters, meaning, via the paramDict files the case specific values - for all parameters get distributed to and persisted in the case folders. - - Parameters - ---------- - cases : MutableSequence[Case] - cases the paramDict file shall be created for - - Returns - ------- - int - number of paramDict files created - """ - - logger.info("Create case-specific paramDict files in all case folders...") - number_of_param_dicts_created: int = 0 - - for case in cases: - logger.debug(f"creating paramDict in {case.path}") # 1 - target_file = case.path / "paramDict" - param_dict = CppDict(target_file) - - for parameter in case.parameters or []: - if parameter.name and not re.match("^_", parameter.name): - param_dict[parameter.name] = parameter.value - - param_dict["_case"] = case.to_dict() - - DictWriter.write(param_dict, target_file, mode="w") - - if case.is_leaf: - number_of_param_dicts_created += 1 - - leaf_cases = [case for case in cases if case.is_leaf] - - logger.info( - f"Successfully created {number_of_param_dicts_created} " - f"paramDict file{plural(number_of_param_dicts_created)} " - f"in {len(leaf_cases)} case folder{plural(len(leaf_cases))}." - ) - - return number_of_param_dicts_created - - -def create_case_list_files( - cases: MutableSequence[Case], - target_dir: Union[Path, None] = None, - levels: Union[int, Sequence[int], None] = None, -) -> list[Path]: - """Create case list files for the specified nest levels. - - Case list files are simple text files containing a list of paths to all case folders that share a common nest level within the case folder structure. - I.e. a case list file created for level 0 contains the paths to all case folders on level 0. - A case list file for level 1 contains the paths to all case folders on level 1, and so on. - - These lists can be used i.e. in a batchProcess to execute shell commands - in all case folders of a specific nest level inside the case folder structure. - - Parameters - ---------- - cases : MutableSequence[Case] - cases the case list files shall be created for - target_dir : Path, optional - directory in which the case list files shall be created. If None, current working directory will be used., by default None - levels : Union[int, Sequence[int], None], optional - list of integers indicating the nest levels for which case list files shall be created. - If missing, by default a case list file for the deepest nest level (the leaf level) will becreated., by default None - - Returns - ------- - list[Path] - The case list files that have been created (returned as a list of Path objects) - """ - - _remove_old_case_list_files() - target_dir = target_dir or Path.cwd() - case_list_file_all_levels = target_dir / "caseList" - logger.info(f"Create case list file '{case_list_file_all_levels}', containing all case folders.") - - case_list_files_created: MutableSequence[Path] = [] - max_level: int = 0 - with case_list_file_all_levels.open(mode="w") as f: - for case in cases: - _ = f.write(f"{case.path.absolute()}\n") - max_level = max(max_level, case.level) - case_list_files_created.append(case_list_file_all_levels) - - levels = levels or max_level - levels = [levels] if isinstance(levels, int) else levels - - for level in levels: - case_list_file_for_level = target_dir / f"caseList_level_{level:02d}" - logger.info( - f"Create case list file '{case_list_file_for_level}', containing the case folders of level {level}." - ) - with case_list_file_for_level.open(mode="w") as f: - for case in (case for case in cases if case.level == level): - _ = f.write(f"{case.path.absolute()}\n") - case_list_files_created.append(case_list_file_for_level) - - case_list_files_created_log = "".join("\t" + path.name + "\n" for path in case_list_files_created) - case_list_files_created_log = case_list_files_created_log.removesuffix("\n") - logger.info(f"Successfully created following case list files:\n {case_list_files_created_log}") - - return case_list_files_created - - -def execute_command_set( - cases: MutableSequence[Case], - command_set: str, - batch: bool = True, - test: bool = False, -) -> int: - """Execute the given command set in the case folders of the passed in cases. - - Parameters - ---------- - cases : MutableSequence[Case] - cases for which the specified command set shall be executed. - command_set : str - name of the command set to be executed, as defined in farnDict - batch : bool, optional - if True, executes the given command set in batch mode, i.e. asynchronously, by default False - test : bool, optional - if True, executes command set in only first case folder where command set is defined, by default False - - Returns - ------- - int - number of case folders in which the command set has been executed - """ - - logger.info(f"Execute command set '{command_set}' in all layers where '{command_set}' is defined...") - - cases_registered: List[Case] = [] - number_of_cases_registered: int = 0 - reached_first_leaf: bool = False - if test: - logger.warning( - f"farn.py called with option --test: Only first case folder where command set '{command_set}' is defined will be executed." - ) - - for case in cases: - if not case.path.exists(): - logger.warning( - f"Path {case.path} does not exist. " - f"This most commonly happens if a filter expression was changed in between generating the folder structure (option --generate) \n" - f"and executing a command set (option --execute). " - f"If so, first generate the missing cases by calling farn with option --generate once again \n" - f"and then retry to execute the command set with option --execute." - ) - continue - if case.command_sets: - if command_set in case.command_sets: - cases_registered.append(case) - number_of_cases_registered += 1 - if case.is_leaf: - reached_first_leaf = True - else: - logger.debug(f"Command set '{command_set}' not defined in case {case.case}") - if test and reached_first_leaf: # if test and at least one execution - break - - number_of_cases_processed: int = 0 - - if batch: - cases_per_shell_command: Dict[str, List[Case]] = {} - for case in cases_registered: - if case.command_sets and command_set in case.command_sets: - shell_commands: List[str] = case.command_sets[command_set] - for shell_command in shell_commands: - if shell_command in cases_per_shell_command: - cases_per_shell_command[shell_command].append(case) - else: - cases_per_shell_command |= {shell_command: [case]} - for index, (shell_command, cases) in enumerate(cases_per_shell_command.items()): - case_list_file = Path.cwd() / f"caseList_for_command_{index}" - with case_list_file.open(mode="w") as f: - for case in cases: - _ = f.write(f"{case.path.absolute()}\n") - batch_processor = AsyncBatchProcessor(case_list_file, shell_command) - batch_processor.run() - else: - for case in cases_registered: - if case.command_sets and command_set in case.command_sets: - shell_commands = case.command_sets[command_set] - # logger.debug(f"Execute command set '{command_set}' in {case.path}") # commented out as a similar message gets logged in also subProcess - # Temporarily change cwd to case folder, to execute the shell commands from there - current_dir = Path.cwd() - os.chdir(case.path) - # Execute shell commands - _execute_shell_commands(shell_commands) - # Change back cwd to current folder - os.chdir(current_dir) - number_of_cases_processed += 1 - - # @TODO: This is only a temporary dummy. - # To be replaced by a smarter algorithm. - # CLAROS, 2022-08-16 - number_of_cases_processed = number_of_cases_registered - - if number_of_cases_processed > 0: - if test: - logger.info( - f"Test finished. Executed command set '{command_set}' in following case folder:\n" - f"\t {cases_registered[-1].path}" - ) - else: - logger.info( - f"Successfully executed command set '{command_set}' " - f"in {number_of_cases_registered} case folder{plural(number_of_cases_registered)}." - ) - - return number_of_cases_registered - - -def _set_up_farn_environment(farn_dict_file: Path) -> Dict[str, Path]: - """Read the '_environment' section from farn dict and sets up the farn environment accordingly. - - Reads the '_environment' section from farnDict and sets up the farn environment directories as configured therein. - If the '_environment' section or certain entries therein are missing in farn dict, default values will be used. - - Parameters - ---------- - farn_dict_file : Path - farnDict file - - Returns - ------- - Dict[str, str] - dict containing the environment directories set up for farn (matching the _environment section in farnDict) - """ - - logger.info("Set up farn environment...") - - # Set up farn environment. - # 1: Define default values for environment - # sourcery skip: merge-dict-assign - environment: Dict[str, str] = {} - environment["CASEDIR"] = "cases" - environment["DUMPDIR"] = "dump" - environment["LOGDIR"] = "logs" - environment["RESULTDIR"] = "results" - environment["TEMPLATEDIR"] = "template" - # 2: Overwrite default values with values defined in farn dict, if so - if environment_from_farn_dict := DictReader.read(farn_dict_file, scope=["_environment"]): - environment |= environment_from_farn_dict - else: - logger.warning( - f"Key '_environment' is missing in farn dict {farn_dict_file}. Using default values for farn environment." - ) - - # Read farn directories from environment - farn_dirs: Dict[str, Path] - farn_dirs = {k: Path.joinpath(Path.cwd(), v) for k, v in environment.items()} - farn_dirs["ROOTDIR"] = Path.cwd() - # Configure logging handler to write the farn log (use an additional handler, exclusively for farn) - _configure_additional_logging_handler_exclusively_for_farn(farn_dirs["LOGDIR"]) - - # Set up system environment variables for each farn directory - # This is necessary to enable shell commands defined in farnDict to point to them with i.e. %TEMPLATEDIR% - for key, item in farn_dirs.items(): - append_system_variable(key, str(item)) - - logger.info("Successfully set up farn environment.") - - return farn_dirs - - -def _configure_additional_logging_handler_exclusively_for_farn(log_dir: Path): - """Create an additional logging handler exclusively for the farn log. - - Parameters - ---------- - log_dir : Path - folder in which the log file will be created - """ - # Create log file - log_dir.mkdir(parents=True, exist_ok=True) - log_file = log_dir / "farn.log" - # Create logging file handler - file_handler = logging.FileHandler(str(log_file.absolute()), "a") - file_handler.name = str(log_file.absolute()) - file_handler.setLevel(logging.INFO) - file_formatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s", "%Y-%m-%d %H:%M:%S") - file_handler.setFormatter(file_formatter) - # Register file handler at root logger - root_logger = logging.getLogger() - file_handler_already_exists: bool = any(handler.name == file_handler.name for handler in root_logger.handlers) - if not file_handler_already_exists: - root_logger.addHandler(file_handler) - return - - -def _remove_old_case_list_files(): # sourcery skip: avoid-builtin-shadow - """Remove old case list files, if existing.""" - logger.info("Remove old case list files...") - - lists = [list for list in Path.cwd().rglob("*") if re.search("(path|queue)List", str(list))] - - for list in lists: - list = Path(list) - list.unlink() - - logger.info("Successfully removed old case list files.") - - return - - -def _sys_call(shell_commands: MutableSequence[str]): - """Fallback function until _execute_command is usable under linux.""" - - for shell_command in shell_commands: - _ = os.system(shell_command) - - return - - -def _execute_shell_commands(shell_commands: MutableSequence[str]): - """Execute a sequence of shell commands using subprocess. - - Parameters - ---------- - shell_commands : MutableSequence - list with shell commands to be executed - """ - - # @TODO: until the problem with vanishing '.'s on Linux systems is solved (e.g. in command "ln -s target ."), - # reroute the function call to _sys_call instead, as a workaround. - if platform.system() == "Linux": - _sys_call(shell_commands) - return - - for shell_command in shell_commands: - _ = execute_in_sub_process(shell_command) - - return +import logging +import os +import platform +import re +from collections.abc import MutableMapping, MutableSequence, MutableSet, Sequence +from copy import deepcopy +from pathlib import Path +from typing import ( + Any, +) + +from dictIO import CppDict, DictReader, DictWriter, create_target_file_name +from dictIO.utils.strings import remove_quotes + +from farn.core import Case, Cases, Parameter +from farn.run.batchProcess import AsyncBatchProcessor +from farn.run.subProcess import execute_in_sub_process +from farn.utils.logging import plural +from farn.utils.os import append_system_variable + +__ALL__ = [ + "run_farn", + "create_samples", + "create_cases", + "create_case_folders", + "create_param_dict_files", + "create_case_list_files", + "execute_command_set", +] + +logger = logging.getLogger(__name__) + + +def run_farn( + farn_dict_file: str | os.PathLike[str], + sample: bool = False, + generate: bool = False, + command: str | None = None, + batch: bool = False, + test: bool = False, +) -> Cases: + """Run farn. + + Runs the sampling for all layers as configured in farn dict, + generates the corresponding case folder structure and + executes user-defined shell command sets in all case folders. + + Parameters + ---------- + farn_dict_file : Union[str, os.PathLike[str]] + farnDict file. Contains the farn configuration. + sample : bool, optional + if True, runs the sampling defined for each layer and saves the sampled farnDict file with prefix sampled., by default False + generate : bool, optional + if True, generates the folder structure that spawns all layers and cases defined in farnDict, by default False + command : Union[str, None], optional + executes the given command set in all case folders. The command set must be defined in the commands section of the applicable layer in farnDict., by default None + batch : bool, optional + if True, executes the given command set in batch mode, i.e. asynchronously, by default False + test : bool, optional + if True, runs only first case and returns, by default False + + Returns + ------- + Cases + List containing all valid leaf cases. + + Raises + ------ + FileNotFoundError + if farn_dict_file does not exist + """ + # sourcery skip: extract-method + + # Make sure farn_dict_file argument is of type Path. If not, cast it to Path type. + farn_dict_file = farn_dict_file if isinstance(farn_dict_file, Path) else Path(farn_dict_file) + + # Check whether farn dict file exists + if not farn_dict_file.exists(): + logger.error(f"run_farn: File {farn_dict_file} not found.") + raise FileNotFoundError(farn_dict_file) + + # Set up farn environment + farn_dirs: dict[str, Path] = _set_up_farn_environment(farn_dict_file) + + # Read farn dict + farn_dict = DictReader.read(farn_dict_file, comments=False) + + # Run sampling and create the samples for all layers in farn dict + if sample: + create_samples(farn_dict) # run sampling + assert farn_dict.source_file is not None + farn_dict.source_file = create_target_file_name( # change filename to 'sampled.*' + farn_dict.source_file, + prefix="sampled.", # type: ignore + ) + logger.info(f"Save sampled farn dict {farn_dict.name}...") # 1 + DictWriter.write(farn_dict, mode="w") # save sampled.* farn dict file + logger.info(f"Saved sampled farn dict in {farn_dict.source_file}.") # 1 + + # Document CLI arguments of current farn call in the farn dict (for traceability) + farn_opts = { + "farnDict": farn_dict.name, + "sample": sample, + "generate": generate, + "execute": command, + "test": test, + } + farn_dict.update({"_farnOpts": farn_opts}) + + # Create all valid cases from the samples defined in farn dict. + cases = create_cases( + farn_dict=farn_dict, + case_dir=farn_dirs["CASEDIR"], + valid_only=True, + ) + + # Generate case folder structure + # and create a case-specific paramDict file in each case folder + if generate: + _ = create_case_folders(cases) + _ = create_param_dict_files(cases) + _ = create_case_list_files( + cases=cases, + target_dir=farn_dirs["ROOTDIR"], + ) + + # Execute a given command set in all case folders + if command: + _ = execute_command_set( + cases=cases, + command_set=command, + batch=batch, + test=test, + ) + + valid_leaf_cases: Cases = cases.filter(levels=-1, valid_only=True) + + logger.info("Successfully finished farn.\n") + + return valid_leaf_cases + + +def create_samples(farn_dict: CppDict) -> None: + """Run sampling and create the samples inside all layers of the passed in farn dict. + + Creates the _samples element in each layer and populates it with the discrete samples generated for the parameters defined and varied in the respective layer. + In case the _samples element already exists in a layer, it will be overwritten. + + Parameters + ---------- + farn_dict : CppDict + farn dict the samples shall be created in + """ + from farn.sampling.sampling import DiscreteSampling + + if "_layers" not in farn_dict: + logger.error(f"no '_layers' element in farn dict {farn_dict.name}. Sampling not possible.") + return + + def create_samples_in_layer( + level: int, + layer_name: str, + layer: MutableMapping[str, Any], + ) -> None: + """Run sampling and generate the samples in the passed in layer.""" + if "_sampling" not in layer: + logger.error("no '_sampling' element in layer") + return + if "_type" not in layer["_sampling"]: + logger.error("no '_type' element in sampling") + return + + # instantiate and parameterize the sampling object + sampling = DiscreteSampling() + sampling.set_sampling_type(sampling_type=layer["_sampling"]["_type"]) + sampling.set_sampling_parameters( + sampling_parameters=layer["_sampling"], + layer_name=layer_name, + ) + + # in case a _samples element already exists (e.g. from a former run) -> delete it + if "_samples" in layer: + del layer["_samples"] + + # generate the samples and write them into the _samples element of the layer + samples: dict[str, list[Any]] = sampling.generate_samples() + layer["_samples"] = samples + + # if the layer does not have a _comment element yet: create a default comment + if "_comment" not in layer: + default_comment = f"level {level:2d}, layer {layer_name}" + layer["_comment"] = default_comment + + return + + logger.info(f"Run sampling of {farn_dict.name}...") + + for index, (key, value) in enumerate(farn_dict["_layers"].items()): + create_samples_in_layer( + level=index, + layer_name=key, + layer=value, + ) + + logger.info(f"Successfully ran sampling of {farn_dict.name}.") + + return + + +def create_cases( + farn_dict: MutableMapping[Any, Any], + case_dir: Path, + valid_only: bool = False, +) -> Cases: + """Create cases based on the layers, filter expressions and samples defined in the passed farn dict. + + Creates case objects for all cases derived by recursive permutation of layers and the case specific samples defined per layer. + create_cases() creates one distinct case object for each case, holding all case attributes (parameters) set to their case specific values. + + Optionally, only _valid_ cases can be returned, i.e. cases which fulfill the filter criteria configured for the respective layer. + Invalid cases then get excluded. + + Note: + The corresponding case folder structure is not yet created by create_cases(). + Creating the case folder structure is the responsibility of create_case_folder_structure(). + However, the case_dir argument is passed in to allow create_cases() to already document in each case object + its _intended_ case folder path. This information is then read and used in create_case_folder_structure() + to actually create the case folders. + + Parameters + ---------- + farn_dict : MutableMapping + farn dict. The farn dict must be sampled, e.g. samples must have been generated for all layers defined in the farn dict. + case_dir : Path + directory the case folder structure is (intended) to be generated in. + valid_only: bool + whether or not only valid cases shall be returned, i.e. cases which fulfill the filter criteria configured for the respective layer., by default False + + Returns + ------- + Cases + list of case objects representing all created cases. + """ + log_msg: str = "List all valid cases.." if valid_only else "List all cases.." + logger.info(log_msg) + + # Check default distributions + default_distribution: dict[str, Any] = {} + + if "_always" in farn_dict: + default_distribution = farn_dict["_always"] + + # Check arguments. + if "_layers" not in farn_dict: + logger.error("create_cases: No '_layers' element contained in farn dict.") + return Cases() + + # Initialize cases list + cases: Cases = Cases() + number_of_invalid_cases: int = 0 + + # Create a local layers list that carries also the layers' name + # to ease sequential and indexed access to individual layers in create_next_level_cases() + layers: list[dict[str, Any]] = [] + for layer_name, layer in farn_dict["_layers"].items(): + layer_copy: dict[str, Any] = deepcopy(layer) + layer_copy["_name"] = layer_name + layers.append(layer_copy) + + def create_next_level_cases( + level: int = 0, + base_case: Case | None = None, + ): + nonlocal cases + nonlocal number_of_invalid_cases + nonlocal layers + + base_case = base_case or Case(path=Path.cwd()) + base_case.parameters = base_case.parameters or [] + + current_layer: dict[str, Any] = layers[level] + # validity checks for current layer + if "_samples" not in current_layer: + logger.warning( + f"No _samples element found in layer {current_layer['_name']}.\n" + f"Creation of cases for level {level:2d} aborted. " + ) + return + if "_case_name" not in current_layer["_samples"]: + logger.warning( + f"The _samples element in layer {current_layer['_name']} is empty or does not have a _case_name element.\n" + f"Creation of cases for level {level:2d} aborted. " + ) + return + + current_layer_name: str = str(current_layer["_name"]) + current_layer_is_leaf: bool = level == len(layers) - 1 + + no_of_samples_in_current_layer: int = len(current_layer["_samples"]["_case_name"]) + samples_in_current_layer: MutableMapping[str, MutableSequence[float]] = { + param_name: param_values + for param_name, param_values in current_layer["_samples"].items() + if param_name != "_case_name" + } + + parameter_names_used_in_preceeding_layers: MutableSet[str] = { + parameter.name for parameter in base_case.parameters if parameter.name + } + + parameter_names_in_current_layer: MutableSequence[str] = [] + for parameter_name in list(samples_in_current_layer.keys()): + if parameter_name in parameter_names_used_in_preceeding_layers: + logger.warning( + f"The parameter {parameter_name} defined in layer {current_layer_name} had already been defined in a preceeding layer.\n" + f"The preceeding definition prevails. The samples for parameter {parameter_name} defined in layer {current_layer_name} are skipped. " + ) + else: + parameter_names_in_current_layer.append(parameter_name) + + user_variables_in_current_layer: MutableSequence[Parameter] = [] + for key, item in default_distribution.items(): + if not key.startswith("_"): + default_variable = Parameter(name=key, value=item) + user_variables_in_current_layer.append(default_variable) + + for key, item in current_layer.items(): + if not key.startswith("_"): + user_variable = Parameter(name=key, value=item) + if user_variable.name in parameter_names_used_in_preceeding_layers: + logger.warning( + f"The user variable {user_variable.name} defined in layer {current_layer_name} matches a parameter that\n" + f"had already been defined in a preceeding layer.\n" + f"The preceeding definition prevails. The user variable {user_variable.name} defined in layer {current_layer_name} is skipped. " + ) + elif user_variable.name in parameter_names_in_current_layer: + logger.warning( + f"The user variable {user_variable.name} defined in layer {current_layer_name} matches a parameter name defined in the same layer.\n" + f"The preceeding definition prevails. The user variable {user_variable.name} defined in layer {current_layer_name} is skipped. " + ) + else: + user_variables_in_current_layer.append(user_variable) + + condition_in_current_layer: MutableMapping[str, str] | None = ( + current_layer["_condition"] if "_condition" in current_layer else None + ) + commands_in_current_layer: MutableMapping[str, list[str]] | None = ( + current_layer["_commands"] if "_commands" in current_layer else None + ) + + for index, case_name in enumerate(current_layer["_samples"]["_case_name"]): + case_name = remove_quotes(case_name) + + case_parameters: MutableSequence[Parameter] = [ + parameter for parameter in base_case.parameters if parameter.name + ] + case_parameters.extend( + Parameter(parameter_name, samples_in_current_layer[parameter_name][index]) + for parameter_name in parameter_names_in_current_layer + ) + case_parameters.extend(user_variables_in_current_layer) + + case = Case( + case=case_name, + layer=current_layer_name, + level=level, + no_of_samples=no_of_samples_in_current_layer, + index=index, + path=base_case.path / case_name, + is_leaf=current_layer_is_leaf, + condition=condition_in_current_layer, + parameters=case_parameters, + command_sets=commands_in_current_layer, + ) + + if not valid_only or case.is_valid: + cases.append(case) + if not case.is_leaf: # Recursion for next level cases + create_next_level_cases( + level=level + 1, + base_case=case, + ) + else: + number_of_invalid_cases += 1 + + return + + # Commence recursive collection of cases among all layers + base_case = Case(path=case_dir) + create_next_level_cases(level=0, base_case=base_case) + + leaf_cases = [case for case in cases if case.is_leaf] + + log_msg = "" + if valid_only: + log_msg = ( + f"Successfully listed {len(leaf_cases)} valid case{plural(len(leaf_cases))}. " + f'{number_of_invalid_cases} invalid case{plural(number_of_invalid_cases)} {plural(number_of_invalid_cases, "were")} excluded.' + ) + else: + log_msg = f"Successfully listed {len(leaf_cases)} case{plural(len(leaf_cases))}. " + logger.info(log_msg) + + return cases + + +def create_case_folders(cases: MutableSequence[Case]) -> int: + """Create the case folder structure for the passed in cases. + + Parameters + ---------- + cases : MutableSequence[Case] + cases the case folders shall be created for. + + Returns + ------- + int + number of case folders created. + """ + logger.info("Create case folder structure...") + number_of_case_folders_created: int = 0 + + for case in cases: + logger.debug(f"creating case folder {case.path}") # 1 + case.path.mkdir(parents=True, exist_ok=True) + number_of_case_folders_created += 1 + + logger.info(f"Successfully created {number_of_case_folders_created} case folders.") + + return number_of_case_folders_created + + +def create_param_dict_files(cases: MutableSequence[Case]) -> int: + """Create the case specific paramDict files in the case folders of the passed in cases. + + paramDict files contain the case specific parameters, meaning, via the paramDict files the case specific values + for all parameters get distributed to and persisted in the case folders. + + Parameters + ---------- + cases : MutableSequence[Case] + cases the paramDict file shall be created for + + Returns + ------- + int + number of paramDict files created + """ + logger.info("Create case-specific paramDict files in all case folders...") + number_of_param_dicts_created: int = 0 + + for case in cases: + logger.debug(f"creating paramDict in {case.path}") # 1 + target_file = case.path / "paramDict" + param_dict = CppDict(target_file) + + for parameter in case.parameters or []: + if parameter.name and not re.match("^_", parameter.name): + param_dict[parameter.name] = parameter.value + + param_dict["_case"] = case.to_dict() + + DictWriter.write(param_dict, target_file, mode="w") + + if case.is_leaf: + number_of_param_dicts_created += 1 + + leaf_cases = [case for case in cases if case.is_leaf] + + logger.info( + f"Successfully created {number_of_param_dicts_created} " + f"paramDict file{plural(number_of_param_dicts_created)} " + f"in {len(leaf_cases)} case folder{plural(len(leaf_cases))}." + ) + + return number_of_param_dicts_created + + +def create_case_list_files( + cases: MutableSequence[Case], + target_dir: Path | None = None, + levels: int | Sequence[int] | None = None, +) -> list[Path]: + """Create case list files for the specified nest levels. + + Case list files are simple text files containing a list of paths to all case folders that share a common nest level within the case folder structure. + I.e. a case list file created for level 0 contains the paths to all case folders on level 0. + A case list file for level 1 contains the paths to all case folders on level 1, and so on. + + These lists can be used i.e. in a batchProcess to execute shell commands + in all case folders of a specific nest level inside the case folder structure. + + Parameters + ---------- + cases : MutableSequence[Case] + cases the case list files shall be created for + target_dir : Path, optional + directory in which the case list files shall be created. + If None, current working directory will be used., by default None + levels : Union[int, Sequence[int], None], optional + list of integers indicating the nest levels for which case list files shall be created. + If missing, by default a case list file for the deepest nest level (the leaf level) + will becreated., by default None + + Returns + ------- + list[Path] + The case list files that have been created (returned as a list of Path objects) + """ + _remove_old_case_list_files() + target_dir = target_dir or Path.cwd() + case_list_file_all_levels = target_dir / "caseList" + logger.info(f"Create case list file '{case_list_file_all_levels}', containing all case folders.") + + case_list_files_created: MutableSequence[Path] = [] + max_level: int = 0 + with case_list_file_all_levels.open(mode="w") as f: + for case in cases: + _ = f.write(f"{case.path.absolute()}\n") + max_level = max(max_level, case.level) + case_list_files_created.append(case_list_file_all_levels) + + levels = levels or max_level + levels = [levels] if isinstance(levels, int) else levels + + for level in levels: + case_list_file_for_level = target_dir / f"caseList_level_{level:02d}" + logger.info( + f"Create case list file '{case_list_file_for_level}', containing the case folders of level {level}." + ) + with case_list_file_for_level.open(mode="w") as f: + for case in (case for case in cases if case.level == level): + _ = f.write(f"{case.path.absolute()}\n") + case_list_files_created.append(case_list_file_for_level) + + case_list_files_created_log = "".join("\t" + path.name + "\n" for path in case_list_files_created) + case_list_files_created_log = case_list_files_created_log.removesuffix("\n") + logger.info(f"Successfully created following case list files:\n {case_list_files_created_log}") + + return case_list_files_created + + +def execute_command_set( + cases: MutableSequence[Case], + command_set: str, + *, + batch: bool = True, + test: bool = False, +) -> int: + """Execute the given command set in the case folders of the passed in cases. + + Parameters + ---------- + cases : MutableSequence[Case] + cases for which the specified command set shall be executed. + command_set : str + name of the command set to be executed, as defined in farnDict + batch : bool, optional + if True, executes the given command set in batch mode, i.e. asynchronously, by default False + test : bool, optional + if True, executes command set in only first case folder where command set is defined, by default False + + Returns + ------- + int + number of case folders in which the command set has been executed + """ + logger.info(f"Execute command set '{command_set}' in all layers where '{command_set}' is defined...") + + cases_registered: list[Case] = [] + number_of_cases_registered: int = 0 + reached_first_leaf: bool = False + if test: + logger.warning( + "farn.py called with option --test: " + f"Only first case folder where command set '{command_set}' is defined will be executed." + ) + + for case in cases: + if not case.path.exists(): + logger.warning( + f"Path {case.path} does not exist. " + "This most commonly happens if a filter expression was changed in between " + "generating the folder structure (option --generate) \n" + "and executing a command set (option --execute). " + "If so, first generate the missing cases by calling farn with option --generate once again \n" + "and then retry to execute the command set with option --execute." + ) + continue + if case.command_sets: + if command_set in case.command_sets: + cases_registered.append(case) + number_of_cases_registered += 1 + if case.is_leaf: + reached_first_leaf = True + else: + logger.debug(f"Command set '{command_set}' not defined in case {case.case}") + if test and reached_first_leaf: # if test and at least one execution + break + + number_of_cases_processed: int = 0 + + if batch: + cases_per_shell_command: dict[str, list[Case]] = {} + for case in cases_registered: + if case.command_sets and command_set in case.command_sets: + shell_commands: list[str] = case.command_sets[command_set] + for shell_command in shell_commands: + if shell_command in cases_per_shell_command: + cases_per_shell_command[shell_command].append(case) + else: + cases_per_shell_command |= {shell_command: [case]} + for index, (shell_command, _cases) in enumerate(cases_per_shell_command.items()): + case_list_file = Path.cwd() / f"caseList_for_command_{index}" + with case_list_file.open(mode="w") as f: + for case in _cases: + _ = f.write(f"{case.path.absolute()}\n") + batch_processor = AsyncBatchProcessor(case_list_file, shell_command) + batch_processor.run() + else: + for case in cases_registered: + if case.command_sets and command_set in case.command_sets: + shell_commands = case.command_sets[command_set] + # Temporarily change cwd to case folder, to execute the shell commands from there + current_dir = Path.cwd() + os.chdir(case.path) + # Execute shell commands + _execute_shell_commands(shell_commands) + # Change back cwd to current folder + os.chdir(current_dir) + number_of_cases_processed += 1 + + # @TODO: This is only a temporary dummy. + # To be replaced by a smarter algorithm. + # CLAROS, 2022-08-16 + number_of_cases_processed = number_of_cases_registered + + if number_of_cases_processed > 0: + if test: + logger.info( + f"Test finished. Executed command set '{command_set}' in following case folder:\n" + f"\t {cases_registered[-1].path}" + ) + else: + logger.info( + f"Successfully executed command set '{command_set}' " + f"in {number_of_cases_registered} case folder{plural(number_of_cases_registered)}." + ) + + return number_of_cases_registered + + +def _set_up_farn_environment(farn_dict_file: Path) -> dict[str, Path]: + """Read the '_environment' section from farn dict and sets up the farn environment accordingly. + + Reads the '_environment' section from farnDict and sets up the farn environment directories as configured therein. + If the '_environment' section or certain entries therein are missing in farn dict, default values will be used. + + Parameters + ---------- + farn_dict_file : Path + farnDict file + + Returns + ------- + Dict[str, str] + dict containing the environment directories set up for farn (matching the _environment section in farnDict) + """ + logger.info("Set up farn environment...") + + # Set up farn environment. + # 1: Define default values for environment + # sourcery skip: merge-dict-assign + environment: dict[str, str] = {} + environment["CASEDIR"] = "cases" + environment["DUMPDIR"] = "dump" + environment["LOGDIR"] = "logs" + environment["RESULTDIR"] = "results" + environment["TEMPLATEDIR"] = "template" + # 2: Overwrite default values with values defined in farn dict, if so + if environment_from_farn_dict := DictReader.read(farn_dict_file, scope=["_environment"]): + environment |= environment_from_farn_dict + else: + logger.warning( + f"Key '_environment' is missing in farn dict {farn_dict_file}. Using default values for farn environment." + ) + + # Read farn directories from environment + farn_dirs: dict[str, Path] + farn_dirs = {k: Path.joinpath(Path.cwd(), v) for k, v in environment.items()} + farn_dirs["ROOTDIR"] = Path.cwd() + # Configure logging handler to write the farn log (use an additional handler, exclusively for farn) + _configure_additional_logging_handler_exclusively_for_farn(farn_dirs["LOGDIR"]) + + # Set up system environment variables for each farn directory + # This is necessary to enable shell commands defined in farnDict to point to them with i.e. %TEMPLATEDIR% + for key, item in farn_dirs.items(): + append_system_variable(key, str(item)) + + logger.info("Successfully set up farn environment.") + + return farn_dirs + + +def _configure_additional_logging_handler_exclusively_for_farn(log_dir: Path) -> None: + """Create an additional logging handler exclusively for the farn log. + + Parameters + ---------- + log_dir : Path + folder in which the log file will be created + """ + # Create log file + log_dir.mkdir(parents=True, exist_ok=True) + log_file = log_dir / "farn.log" + # Create logging file handler + file_handler = logging.FileHandler(str(log_file.absolute()), "a") + file_handler.name = str(log_file.absolute()) + file_handler.setLevel(logging.INFO) + file_formatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s", "%Y-%m-%d %H:%M:%S") + file_handler.setFormatter(file_formatter) + # Register file handler at root logger + root_logger = logging.getLogger() + file_handler_already_exists: bool = any(handler.name == file_handler.name for handler in root_logger.handlers) + if not file_handler_already_exists: + root_logger.addHandler(file_handler) + return + + +def _remove_old_case_list_files() -> None: # sourcery skip: avoid-builtin-shadow + """Remove old case list files, if existing.""" + logger.info("Remove old case list files...") + + case_list_files = [file for file in Path.cwd().rglob("*") if re.search("(path|queue)List", str(file))] + + for _file in case_list_files: + file = Path(_file) + file.unlink() + + logger.info("Successfully removed old case list files.") + + return + + +def _sys_call(shell_commands: MutableSequence[str]) -> None: + """Fallback function until _execute_command is usable under linux.""" + for shell_command in shell_commands: + _ = os.system(shell_command) # noqa: S605 + + return + + +def _execute_shell_commands(shell_commands: MutableSequence[str]) -> None: + """Execute a sequence of shell commands using subprocess. + + Parameters + ---------- + shell_commands : MutableSequence + list with shell commands to be executed + """ + # @TODO: until the problem with vanishing '.'s on Linux systems is solved (e.g. in command "ln -s target ."), + # reroute the function call to _sys_call instead, as a workaround. + if platform.system() == "Linux": + _sys_call(shell_commands) + return + + for shell_command in shell_commands: + _ = execute_in_sub_process(shell_command) + + return diff --git a/src/farn/run/batchProcess.py b/src/farn/run/batchProcess.py index 88c22f9e..8aa19822 100644 --- a/src/farn/run/batchProcess.py +++ b/src/farn/run/batchProcess.py @@ -1,74 +1,71 @@ -import logging -from pathlib import Path - -from psutil import cpu_count - -from farn.run.subProcess import execute_in_sub_process -from farn.run.utils.threading import JobQueue, Worker - -logger = logging.getLogger(__name__) - - -class AsyncBatchProcessor: - """Batch processor for asynchroneous execution of a shell command in multiple case folders.""" - - def __init__( - self, - case_list_file: Path, - command: str, - timeout: int = 3600, - max_number_of_cpus: int = 0, - ): - """Instantiate an asynchroneous batch processor - to execute a shell command in multiple case folders. - - Parameters - ---------- - case_list_file : Path - the file containing the list of case folders the shell command shall be executed in - command : str - the shell command to be executed - timeout : int, optional - time out in seconds, by default 3600 - max_number_of_cpus : int, optional - number of cpus to be used, by default 0 - """ - self.case_list_file: Path = case_list_file - self.command: str = command - self.timeout: int = timeout - self.max_number_of_cpus: int = max_number_of_cpus - - def run(self): - """Run the shell command in all case folders.""" - - # Check whether caselist file exists - if not self.case_list_file.is_file(): - logger.error(f"AsyncBatchProcessor: File {self.case_list_file} not found.") - return - - # Read the case list and fill job queue - cases = [] - with open(self.case_list_file, "r") as f: - cases = f.readlines() - - jobs = JobQueue() - - for index, path in enumerate(cases): - path = path.strip() - jobs.put(execute_in_sub_process, self.command, path, self.timeout) - logger.info("Job %g queued in %s" % (index, path)) # 1 - - number_of_cpus = cpu_count() - if self.max_number_of_cpus: - number_of_cpus = min(number_of_cpus, int(self.max_number_of_cpus)) - - # Create worker threads that execute the jobs - # (threadPool being a simple list of threads, nothing sophisticated) - thread_pool = [Worker(jobs) for _ in range(number_of_cpus)] - - logger.info(f"AsyncBatchProcessor: started {len(thread_pool):2d} worker threads.") - - # Wait until all jobs are done - jobs.join() - - # exit(0) +import logging +from pathlib import Path + +from psutil import cpu_count + +from farn.run.subProcess import execute_in_sub_process +from farn.run.utils.threading import JobQueue, Worker + +logger = logging.getLogger(__name__) + + +class AsyncBatchProcessor: + """Batch processor for asynchroneous execution of a shell command in multiple case folders.""" + + def __init__( + self, + case_list_file: Path, + command: str, + timeout: int = 3600, + max_number_of_cpus: int = 0, + ) -> None: + """Instantiate an asynchroneous batch processor + to execute a shell command in multiple case folders. + + Parameters + ---------- + case_list_file : Path + the file containing the list of case folders the shell command shall be executed in + command : str + the shell command to be executed + timeout : int, optional + time out in seconds, by default 3600 + max_number_of_cpus : int, optional + number of cpus to be used, by default 0 + """ + self.case_list_file: Path = case_list_file + self.command: str = command + self.timeout: int = timeout + self.max_number_of_cpus: int = max_number_of_cpus + + def run(self) -> None: + """Run the shell command in all case folders.""" + # Check whether caselist file exists + if not self.case_list_file.is_file(): + logger.error(f"AsyncBatchProcessor: File {self.case_list_file} not found.") + return + + # Read the case list and fill job queue + cases = [] + with Path.open(self.case_list_file) as f: + cases = f.readlines() + + jobs = JobQueue() + + for index, _path in enumerate(cases): + path = _path.strip() + jobs.put_callable(execute_in_sub_process, self.command, path, self.timeout) + logger.info(f"Job {index:g} queued in {path}") # 1 + + number_of_cpus = cpu_count() + if self.max_number_of_cpus: + number_of_cpus = min(number_of_cpus, int(self.max_number_of_cpus)) + + # Create worker threads that execute the jobs + # (threadPool being a simple list of threads, nothing sophisticated) + thread_pool = [Worker(jobs) for _ in range(number_of_cpus)] + + logger.info(f"AsyncBatchProcessor: started {len(thread_pool):2d} worker threads.") + + # Wait until all jobs are done + jobs.join() diff --git a/src/farn/run/cli/batchProcess.py b/src/farn/run/cli/batchProcess.py index 7f038f47..3b3d693d 100644 --- a/src/farn/run/cli/batchProcess.py +++ b/src/farn/run/cli/batchProcess.py @@ -1,11 +1,9 @@ #!/usr/bin/env python -# coding: utf-8 import argparse import logging from argparse import ArgumentParser from pathlib import Path -from typing import Union from farn.run.batchProcess import AsyncBatchProcessor from farn.utils.logging import configure_logging @@ -104,12 +102,11 @@ def _argparser() -> argparse.ArgumentParser: return parser -def main(): +def main() -> None: """Entry point for console script as configured in setup.cfg. Runs the command line interface and parses arguments and options entered on the console. """ - parser = _argparser() args = parser.parse_args() @@ -120,7 +117,7 @@ def main(): log_level_console = "ERROR" if args.quiet else log_level_console log_level_console = "DEBUG" if args.verbose else log_level_console # ..to file - log_file: Union[Path, None] = Path(args.log) if args.log else None + log_file: Path | None = Path(args.log) if args.log else None log_level_file: str = args.log_level configure_logging(log_level_console, log_file, log_level_file) diff --git a/src/farn/run/subProcess.py b/src/farn/run/subProcess.py index e215486b..2479dc10 100644 --- a/src/farn/run/subProcess.py +++ b/src/farn/run/subProcess.py @@ -3,7 +3,6 @@ import subprocess as sub from pathlib import Path from threading import Lock -from typing import Union from psutil import Process @@ -13,12 +12,16 @@ lock = Lock() -def execute_in_sub_process(command: str, path: Union[Path, None] = None, timeout: Union[int, None] = 7200): # 1h ->2h +def execute_in_sub_process( + command: str, + path: Path | None = None, + timeout: int | None = 7200, # 2 hours +) -> None: """Create a subprocess with cwd = path and executes the given shell command. - The subprocess runs asyncroneous. The calling thread waits until the subprocess returns or until timeout is exceeded. + The subprocess runs asyncroneous. The calling thread waits until the subprocess returns + or until timeout is exceeded. If the subprocess has not returned after [timeout] seconds, the subprocess gets killed. """ - path = path or Path.cwd() # Configure and start subprocess in workDir (this part shall be atomic, hence secured by lock) @@ -27,19 +30,30 @@ def execute_in_sub_process(command: str, path: Union[Path, None] = None, timeout args = re.split(r"\s+", command.strip()) - sub_process = sub.Popen(args, stdout=sub.PIPE, stderr=sub.PIPE, shell=True, cwd=f"{path}") - - if len(command) > 18: - cmd_string = '"' + "".join(list(command)[:11]) + ".." + "".join(list(command)[-3:]) + '"' + sub_process = sub.Popen( # noqa: S602 + args, + stdout=sub.PIPE, + stderr=sub.PIPE, + shell=True, + cwd=f"{path}", + ) + + log_string: str + # NOTE: 18 as max string length is chosen arbitrarily. + # Purpose simply is to limit the length of the log string + # to what can practically be displayed in one line in a log console. + max_log_string_length: int = 18 + if len(command) > max_log_string_length: + log_string = '"' + "".join(list(command)[:11]) + ".." + "".join(list(command)[-3:]) + '"' else: - cmd_string = f'"{command}"' + log_string = f'"{command}"' - logger.info("Execute {:18} in {:}".format(cmd_string, path)) + logger.info(f"Execute {log_string:18} in {path}") logger.debug(f"(timout: {timeout}, pid: %{sub_process.pid})") # Wait for subprocess to finish - stdout = bytes() - stderr = bytes() + stdout = b"" + stderr = b"" try: stdout, stderr = sub_process.communicate(timeout=timeout) except sub.TimeoutExpired: @@ -50,7 +64,7 @@ def execute_in_sub_process(command: str, path: Union[Path, None] = None, timeout for child in parent.children(recursive=True): # raise exeption w/o termination child.kill() parent.kill() - except Exception: + except Exception: # noqa: BLE001 logger.warning(f"Process {sub_process.pid} non-existent. Perhaps previously terminated?") _log_subprocess_output(command, path, stdout, stderr) @@ -58,7 +72,7 @@ def execute_in_sub_process(command: str, path: Union[Path, None] = None, timeout return (stdout, stderr) -def _log_subprocess_output(command: str, path: Path, stdout: bytes, stderr: bytes): +def _log_subprocess_output(command: str, path: Path, stdout: bytes, stderr: bytes) -> None: if out := str(stdout, encoding="utf-8"): _log_subprocess_log(command, path, out) @@ -66,12 +80,12 @@ def _log_subprocess_output(command: str, path: Path, stdout: bytes, stderr: byte _log_subprocess_log(command, path, err) -def _log_subprocess_log(command: str, path: Path, log: str): - if re.search("error", log, re.I): +def _log_subprocess_log(command: str, path: Path, log: str) -> None: + if re.search("error", log, re.IGNORECASE): logger.error(f"during execution of {command} in {path}\n{log}") - elif re.search("warning", log, re.I): + elif re.search("warning", log, re.IGNORECASE): logger.warning(f"from execution of {command} in {path}\n{log}") - elif re.search("info", log, re.I): + elif re.search("info", log, re.IGNORECASE): logger.info(f"from execution of {command} in {path}\n{log}") - elif re.search("debug", log, re.I): + elif re.search("debug", log, re.IGNORECASE): logger.debug(f"from execution of {command} in {path}\n{log}") diff --git a/src/farn/run/utils/threading.py b/src/farn/run/utils/threading.py index b589590a..4f5c6eff 100644 --- a/src/farn/run/utils/threading.py +++ b/src/farn/run/utils/threading.py @@ -1,53 +1,64 @@ -import logging -from queue import Queue -from threading import Thread -from typing import Any, Mapping, Sequence, Tuple - -logger = logging.getLogger(__name__) - - -class JobQueue(Queue[Tuple[Any, Sequence[Any], Mapping[str, Any]]]): - """JobQueue extends threading.Queue, overriding its 'put' method to accept a generic list of arguments.""" - - def put(self, func: Any, *args: Any, **kwargs: Any): # pyright: ignore - """Put a callable object (function) in the JobQueue. - - Additional positional and keyword arguments provided with *args and *kwargs - will be passed forward to the called function. - - Parameters - ---------- - func : Any - the callable object (function) - """ - super().put(item=(func, args, kwargs)) - - -class Worker(Thread): - """Worker thread executing jobs from a job queue.""" - - # Override constructor of Thread class - def __init__(self, job_queue: JobQueue): - """Instantiate a Worker and bind it to the passed in JobQueue instance. - - Parameters - ---------- - job_queue : JobQueue - the JobQueue this Worker shall be bound to - """ - Thread.__init__(self) # invoke base class constructor - self.job_queue: JobQueue = job_queue - self.daemon: bool = True - self.start() - - # Override run() method of Thread class - def run(self): - """Run the next job from the JobQueue this Worker is bound to.""" - while True: - try: - func, args, kwargs = self.job_queue.get() - func(*args, **kwargs) - except Exception: - logger.exception("Worker: Exeption raised in worker thread.") - finally: - self.job_queue.task_done() +import logging +from collections.abc import Callable, Mapping, Sequence +from queue import Queue +from threading import Thread +from typing import Any + +logger = logging.getLogger(__name__) + + +class JobQueue(Queue[tuple[Any, Sequence[Any], Mapping[str, Any]]]): + """Queue for jobs to be executed by worker threads. + + JobQueue extends `threading.Queue`. + It provides an additional `put_callable()` method, allowing to put + a callable with a generic list of arguments in the queue. + """ + + def put_callable( + self, + func: Callable[..., Any], + *args: Any, # noqa: ANN401 + **kwargs: Any, # noqa: ANN401 + ) -> None: + """Put a callable object (function) in the JobQueue. + + Additional positional and keyword arguments provided with *args and *kwargs + will be passed on to the called function. + + Parameters + ---------- + func : Any + the callable object (function) + """ + super().put(item=(func, args, kwargs)) + + +class Worker(Thread): + """Worker thread executing jobs from a job queue.""" + + # Override constructor of Thread class + def __init__(self, job_queue: JobQueue) -> None: + """Instantiate a Worker and bind it to the passed in JobQueue instance. + + Parameters + ---------- + job_queue : JobQueue + the JobQueue this Worker shall be bound to + """ + Thread.__init__(self) # invoke base class constructor + self.job_queue: JobQueue = job_queue + self.daemon: bool = True + self.start() + + # Override run() method of Thread class + def run(self) -> None: + """Run the next job from the JobQueue this Worker is bound to.""" + while True: + try: + func, args, kwargs = self.job_queue.get() + func(*args, **kwargs) + except Exception: # noqa: PERF203 + logger.exception("Worker: Exeption raised in worker thread.") + finally: + self.job_queue.task_done() diff --git a/src/farn/sampling/sampling.py b/src/farn/sampling/sampling.py index 4e07e88a..3794fd3a 100644 --- a/src/farn/sampling/sampling.py +++ b/src/farn/sampling/sampling.py @@ -1,16 +1,11 @@ import logging import math -from typing import Any, Dict, Generator, Iterable, List, Mapping, Sequence, Union +from collections.abc import Generator, Iterable, Mapping, Sequence +from typing import Any import numpy as np from numpy import ndarray -# from typing import TYPE_CHECKING -# if TYPE_CHECKING: -# import numpy.typing as npt -# from numpy import integer, ndarray, number -# from scipy._lib._util import DecimalNumber, GeneratorType, IntNumber, SeedType - logger = logging.getLogger(__name__) @@ -19,22 +14,25 @@ class DiscreteSampling: i.e. of all variables defined in the given layer. """ - def __init__(self, seed: Union[int, None] = None): + def __init__( + self, + seed: int | None = None, + ) -> None: self.layer_name: str = "" self.sampling_parameters: Mapping[str, Any] = {} - self.fields: List[str] = [] - self.values: List[Sequence[Any]] = [] + self.fields: list[str] = [] + self.values: list[Sequence[Any]] = [] self.ranges: Sequence[Sequence[Any]] = [] - self.bounding_box: List[List[float]] = [] - self.minVals: List[Any] = [] - self.maxVals: List[Any] = [] - self.case_names: List[str] = [] + self.bounding_box: list[list[float]] = [] + self.minVals: list[Any] = [] + self.maxVals: list[Any] = [] + self.case_names: list[str] = [] self.mean: Sequence[float] = [] - self.std: Union[float, Sequence[float], Sequence[Sequence[float]]] = 0.0 + self.std: float | Sequence[float] | Sequence[Sequence[float]] = 0.0 self.onset: int = 0 - self.sampling_type: str = str() - self.known_sampling_types: Dict[str, Dict[str, List[str]]] = {} + self.sampling_type: str = "" + self.known_sampling_types: dict[str, dict[str, list[str]]] = {} self._set_up_known_sampling_types() self.number_of_fields: int = 0 @@ -46,9 +44,9 @@ def __init__(self, seed: Union[int, None] = None): self.minIterationDepth: int self.maxIterationDepth: int self.include_bounding_box: bool = False - self.seed: Union[int, None] = seed + self.seed: int | None = seed - def _set_up_known_sampling_types(self): + def _set_up_known_sampling_types(self) -> None: self.known_sampling_types = { "fixed": { "required_args": [ @@ -100,8 +98,8 @@ def _set_up_known_sampling_types(self): "_names", "_ranges", "_numberOfSamples", - "_distributionName", # uniform|normal|exp... better to have a dedicated name in known_sampling_types - "_distributionParameters", # mu|sigma|skew|camber not applicsble for uniform + "_distributionName", # uniform|normal|exp... + "_distributionParameters", # mu|sigma|skew|camber not applicable for uniform "_includeBoundingBox", # required ] }, @@ -118,7 +116,7 @@ def _set_up_known_sampling_types(self): }, } - def set_sampling_type(self, sampling_type: str): + def set_sampling_type(self, sampling_type: str) -> None: """Set the sampling type. Valid values: @@ -141,18 +139,17 @@ def set_sampling_parameters( self, sampling_parameters: Mapping[str, Any], layer_name: str = "", - ): + ) -> None: """Set the sampling parameters. The passed-in sampling parameters will be validated. Upon successful validation, the sampling is configured using the provided parameters. """ - self.layer_name = layer_name self.sampling_parameters = sampling_parameters # check if all required arguments are provided - # todo: check argument types + # TODO @CLAROS: check argument types for kwarg in self.known_sampling_types[self.sampling_type]["required_args"]: if kwarg not in self.sampling_parameters: msg: str = ( @@ -178,11 +175,12 @@ def set_sampling_parameters( self.number_of_samples = 0 self.number_of_bb_samples = 0 - def generate_samples(self) -> Dict[str, List[Any]]: + def generate_samples(self) -> dict[str, list[Any]]: """Return a dict with all generated samples for the layer this sampling is run on. The first element in the returned dict contains the case names generated. - All following elements (second to last) contain the values sampled for each variable defined in the layer this sampling is run on. + All following elements (second to last) contain the values sampled + for each variable defined in the layer this sampling is run on. I.e. "names": (case_name_1, case_name_2, .., case_name_n) "variable_1": (value_1, value_2, .., value_n) @@ -194,8 +192,7 @@ def generate_samples(self) -> Dict[str, List[Any]]: Dict[str, List[Any]] the dict with all generated samples """ - - samples: Dict[str, List[Any]] = {} + samples: dict[str, list[Any]] = {} if self.sampling_type == "fixed": samples = self._generate_samples_using_fixed_sampling() @@ -223,25 +220,28 @@ def generate_samples(self) -> Dict[str, List[Any]]: return samples - def _generate_samples_using_fixed_sampling(self) -> Dict[str, List[Any]]: + def _generate_samples_using_fixed_sampling(self) -> dict[str, list[Any]]: _ = self._check_length_matches_number_of_names("_values") - samples: Dict[str, List[Any]] = {} + samples: dict[str, list[Any]] = {} # Assert that the values per parameter are provided as a list for item in self.sampling_parameters["_values"]: if not isinstance(item, Sequence): msg: str = "_values: The values per parameter need to be provided as a list of values." logger.error(msg) - raise ValueError(msg) + raise TypeError(msg) # Assert that the number of values per parameter is the same for all parameters - number_of_values_per_parameter: List[int] = [len(item) for item in self.sampling_parameters["_values"]] + number_of_values_per_parameter: list[int] = [len(item) for item in self.sampling_parameters["_values"]] all_parameters_have_same_number_of_values: bool = all( number_of_values == number_of_values_per_parameter[0] # (breakline) for number_of_values in number_of_values_per_parameter ) if not all_parameters_have_same_number_of_values: - msg: str = "_values: The number of values per parameter need to be the same for all parameters. However, they are different." + msg: str = ( + "_values: The number of values per parameter need to be the same for all parameters. " + "However, they are different." + ) logger.error(msg) raise ValueError(msg) @@ -256,9 +256,9 @@ def _generate_samples_using_fixed_sampling(self) -> Dict[str, List[Any]]: return samples - def _generate_samples_using_linspace_sampling(self) -> Dict[str, List[Any]]: + def _generate_samples_using_linspace_sampling(self) -> dict[str, list[Any]]: _ = self._check_length_matches_number_of_names("_ranges") - samples: Dict[str, List[Any]] = self._generate_samples_dict() + samples: dict[str, list[Any]] = self._generate_samples_dict() self.minVals = [x[0] for x in self.ranges] self.maxVals = [x[1] for x in self.ranges] @@ -273,15 +273,15 @@ def _generate_samples_using_linspace_sampling(self) -> Dict[str, List[Any]]: return samples - def _generate_samples_using_uniform_lhs_sampling(self) -> Dict[str, List[Any]]: + def _generate_samples_using_uniform_lhs_sampling(self) -> dict[str, list[Any]]: _ = self._check_length_matches_number_of_names("_ranges") - samples: Dict[str, List[Any]] = self._generate_samples_dict() + samples: dict[str, list[Any]] = self._generate_samples_dict() values: ndarray[Any, Any] = self._generate_values_using_uniform_lhs_sampling() self._write_values_into_samples_dict(values, samples) return samples - def _generate_samples_using_normal_lhs_sampling(self) -> Dict[str, List[Any]]: + def _generate_samples_using_normal_lhs_sampling(self) -> dict[str, list[Any]]: """LHS using gaussian normal distributions. required input arguments: @@ -294,39 +294,42 @@ def _generate_samples_using_normal_lhs_sampling(self) -> Dict[str, List[Any]]: if isinstance(self.sampling_parameters["_sigma"], Sequence): _ = self._check_length_matches_number_of_names("_sigma") - samples: Dict[str, List[Any]] = self._generate_samples_dict() + samples: dict[str, list[Any]] = self._generate_samples_dict() self.mean = self.sampling_parameters["_mu"] self.std = self.sampling_parameters["_sigma"] values: ndarray[Any, Any] = self._generate_values_using_normal_lhs_sampling() - # Clipping (optional. Clipping will only be performed if sampling parameter "_ranges" is defined.) + # Clipping + # (optional. Clipping will only be performed if sampling parameter "_ranges" is defined.) # NOTE: In current implementation, sampled values exceeding a parameters valid range # are not discarded but reset to the respective range upper or lower bound. # If real clipping shall be implemented, it would require discarding exceeding values. - # As the number of values that would need to be discarded is different for each individual parameter (dimension), + # As the number of values that would need to be discarded is different + # for each individual parameter (dimension), # the necessary clipping logic will quickly become complex. - # Hence the somewhat simpler approach for now, where exceeding values simply get reset to the range bounderies. + # Hence the somewhat simpler approach for now, where exceeding values + # simply get reset to the range bounderies. if self.ranges: - range_lower_bounds: ndarray[Any, Any] = np.array([range[0] for range in self.ranges]) - range_upper_bounds: ndarray[Any, Any] = np.array([range[1] for range in self.ranges]) + range_lower_bounds: ndarray[Any, Any] = np.array([r[0] for r in self.ranges]) + range_upper_bounds: ndarray[Any, Any] = np.array([r[1] for r in self.ranges]) values = np.clip(values, range_lower_bounds, range_upper_bounds) self._write_values_into_samples_dict(values, samples) return samples - def _generate_samples_using_sobol_sampling(self) -> Dict[str, List[Any]]: + def _generate_samples_using_sobol_sampling(self) -> dict[str, list[Any]]: _ = self._check_length_matches_number_of_names("_ranges") self.onset = int(self.sampling_parameters["_onset"]) - samples: Dict[str, List[Any]] = self._generate_samples_dict() + samples: dict[str, list[Any]] = self._generate_samples_dict() values: ndarray[Any, Any] = self._generate_values_using_sobol_sampling() self._write_values_into_samples_dict(values, samples) return samples - def _generate_samples_using_arbitrary_sampling(self) -> Dict[str, List[Any]]: + def _generate_samples_using_arbitrary_sampling(self) -> dict[str, list[Any]]: """ Purpose: To perform a sampling based on the pre-drawn sample. Pre-requisite: @@ -338,7 +341,7 @@ def _generate_samples_using_arbitrary_sampling(self) -> Dict[str, List[Any]]: """ _ = self._check_length_matches_number_of_names("_ranges") - samples: Dict[str, List[Any]] = self._generate_samples_dict() + samples: dict[str, list[Any]] = self._generate_samples_dict() self.minVals = [x[0] for x in self.ranges] self.maxVals = [x[1] for x in self.ranges] @@ -350,9 +353,8 @@ def _generate_samples_using_arbitrary_sampling(self) -> Dict[str, List[Any]]: distribution_name = self.sampling_parameters["_distributionName"] distribution_parameters = self.sampling_parameters["_distributionParameters"] - eval_command = f"scipy.stats.{distribution_name[index]}" - - dist = eval(eval_command) # check this need! + _eval_command = f"scipy.stats.{distribution_name[index]}" + dist = eval(_eval_command) # noqa: S307 samples[self.fields[index]] = dist.rvs( *distribution_parameters[index], @@ -363,9 +365,9 @@ def _generate_samples_using_arbitrary_sampling(self) -> Dict[str, List[Any]]: return samples - def _generate_samples_using_hilbert_sampling(self) -> Dict[str, List[Any]]: + def _generate_samples_using_hilbert_sampling(self) -> dict[str, list[Any]]: _ = self._check_length_matches_number_of_names("_ranges") - samples: Dict[str, List[Any]] = self._generate_samples_dict() + samples: dict[str, list[Any]] = self._generate_samples_dict() # Depending on implementation self.minIterationDepth = 3 self.maxIterationDepth = 15 @@ -375,8 +377,8 @@ def _generate_samples_using_hilbert_sampling(self) -> Dict[str, List[Any]]: return samples - def _generate_samples_dict(self) -> Dict[str, List[Any]]: - samples_dict: Dict[str, List[Any]] = {} + def _generate_samples_dict(self) -> dict[str, list[Any]]: + samples_dict: dict[str, list[Any]] = {} self._determine_number_of_samples() self._generate_case_names(samples_dict) return samples_dict @@ -386,19 +388,19 @@ def _generate_values_using_uniform_lhs_sampling(self) -> ndarray[Any, Any]: from pyDOE3 import lhs from scipy.stats import uniform - lhs_distribution: Union[ndarray[Any, Any], None] = lhs( + lhs_distribution: ndarray[Any, Any] | None = lhs( n=self.number_of_fields, samples=self.number_of_samples - self.number_of_bb_samples, criterion="corr", random_state=self.seed, ) - _range_lower_bounds: ndarray[Any, Any] = np.array([range[0] for range in self.ranges]) - _range_upper_bounds: ndarray[Any, Any] = np.array([range[1] for range in self.ranges]) + _range_lower_bounds: ndarray[Any, Any] = np.array([r[0] for r in self.ranges]) + _range_upper_bounds: ndarray[Any, Any] = np.array([r[1] for r in self.ranges]) loc: ndarray[Any, Any] = _range_lower_bounds scale: ndarray[Any, Any] = _range_upper_bounds - _range_lower_bounds - sample_set: ndarray[Any, Any] = uniform(loc=loc, scale=scale).ppf(lhs_distribution) # pyright: ignore + sample_set: ndarray[Any, Any] = uniform(loc=loc, scale=scale).ppf(lhs_distribution) # pyright: ignore[reportUnknownMemberType] return sample_set @@ -407,22 +409,24 @@ def _generate_values_using_normal_lhs_sampling(self) -> ndarray[Any, Any]: from pyDOE3 import lhs from scipy.stats import norm - lhs_distribution: Union[ndarray[Any, Any], None] = lhs( + lhs_distribution: ndarray[Any, Any] | None = lhs( n=self.number_of_fields, samples=self.number_of_samples - self.number_of_bb_samples, criterion="corr", random_state=self.seed, ) - # criterion: a string that tells lhs how to sample the points (default: None, which simply randomizes the points within the intervals) + # criterion: a string that tells lhs how to sample the points + # (default: None, which simply randomizes the points within the intervals) # - center|c: center the points within the sampling intervals - # - maximin|m: maximize the minimum distance between points, but place the point in a randomized location within its interval + # - maximin|m: maximize the minimum distance between points, but place the point + # in a randomized location within its interval # - centermaximin|cm: same as “maximin”, but centered within the intervals # - correlation|corr: minimize the maximum correlation coefficient # std of type scalar (scale) or vector (stretch, scale), no rotation _std: ndarray[Any, Any] = np.array(self.std) - sample_set: ndarray[Any, Any] = norm(loc=self.mean, scale=_std).ppf(lhs_distribution) # type: ignore + sample_set: ndarray[Any, Any] = norm(loc=self.mean, scale=_std).ppf(lhs_distribution) # pyright: ignore[reportUnknownMemberType] return sample_set @@ -437,16 +441,20 @@ def _generate_values_using_sobol_sampling(self) -> ndarray[Any, Any]: ) if self.onset > 0: - _ = sobol_engine.fast_forward(n=self.onset) # type: ignore + _ = sobol_engine.fast_forward(n=self.onset) # pyright: ignore[reportUnknownMemberType] - points: ndarray[Any, Any] = sobol_engine.random( # type: ignore + points: ndarray[Any, Any] = sobol_engine.random( # pyright: ignore[reportUnknownMemberType] n=self.number_of_samples - self.number_of_bb_samples, ) # Upscale points from unit hypercube to bounds - range_lower_bounds: ndarray[Any, Any] = np.array([range[0] for range in self.ranges]) - range_upper_bounds: ndarray[Any, Any] = np.array([range[1] for range in self.ranges]) - sample_set: ndarray[Any, Any] = qmc.scale(points, range_lower_bounds, range_upper_bounds) # type: ignore + range_lower_bounds: ndarray[Any, Any] = np.array([r[0] for r in self.ranges]) + range_upper_bounds: ndarray[Any, Any] = np.array([r[1] for r in self.ranges]) + sample_set: ndarray[Any, Any] = qmc.scale( # pyright: ignore[reportUnknownMemberType] + points, + range_lower_bounds, + range_upper_bounds, + ) return sample_set @@ -461,32 +469,41 @@ def _generate_values_using_hilbert_sampling(self) -> ndarray[Any, Any]: from scipy.stats import qmc + msg: str try: from decimal import Decimal - except ImportError as e: - msg: str = "no module named Decimal" + except ImportError: + msg = "no module named Decimal" logger.exception(msg) - raise e + raise try: from hilbertcurve.hilbertcurve import HilbertCurve - except ImportError as e: - msg: str = "no module named HilbertCurve" + except ImportError: + msg = "no module named HilbertCurve" logger.exception(msg) - raise e + raise number_of_continuous_samples: int = self.number_of_samples - self.number_of_bb_samples - if "_iterationDepth" in self.sampling_parameters.keys(): + if "_iterationDepth" in self.sampling_parameters: if not isinstance(self.sampling_parameters["_iterationDepth"], int): - msg: str = f'_iterationDepth was not given as integer: {self.sampling_parameters["_iterationDepth"]}.' + msg = f'_iterationDepth was not given as integer: {self.sampling_parameters["_iterationDepth"]}.' logger.error(msg) raise ValueError(msg) if self.sampling_parameters["_iterationDepth"] > self.maxIterationDepth: - msg: str = f'_iterationDepth {self.sampling_parameters["_iterationDepth"]} given in farnDict is beyond the limit of {self.maxIterationDepth}...\n\t\tsetting to {self.maxIterationDepth}' + msg = ( + f'_iterationDepth {self.sampling_parameters["_iterationDepth"]} ' + f'given in farnDict is beyond the limit of {self.maxIterationDepth}...\n' + f'\t\tsetting to {self.maxIterationDepth}' + ) logger.warning(msg) self.iteration_depth = self.maxIterationDepth elif self.sampling_parameters["_iterationDepth"] < self.minIterationDepth: - msg: str = f'_iterationDepth {self.sampling_parameters["_iterationDepth"]} given in farnDict is below the limit of {self.minIterationDepth}...\n\t\tsetting to {self.minIterationDepth}' + msg = ( + f'_iterationDepth {self.sampling_parameters["_iterationDepth"]} ' + f'given in farnDict is below the limit of {self.minIterationDepth}...\n' + f'\t\tsetting to {self.minIterationDepth}' + ) logger.warning(msg) self.iteration_depth = self.minIterationDepth else: @@ -497,11 +514,13 @@ def _generate_values_using_hilbert_sampling(self) -> ndarray[Any, Any]: hc = HilbertCurve(self.iteration_depth, self.number_of_fields, n_procs=0) # -1: all threads logger.info( - f"The number of hilbert points is {hc.max_h}, the number of continuous samples is {number_of_continuous_samples}" + f"The number of hilbert points is {hc.max_h}, " + f"the number of continuous samples is {number_of_continuous_samples}" ) if hc.max_h <= int(10.0 * number_of_continuous_samples): logger.warning( - 'Try to set or increase "_iterationDepth" gradually to achieve a number of hilbert points of about 10-times higher than "_numberOfSamples".' + 'Try to set or increase "_iterationDepth" gradually to achieve ' + 'a number of hilbert points of about 10-times higher than "_numberOfSamples".' ) distribution = np.array( @@ -513,7 +532,7 @@ def _generate_values_using_hilbert_sampling(self) -> ndarray[Any, Any]: _points: Iterable[Iterable[float]] = [] interpolation_hits = 0 - for hpt, dst, idst in zip(hilbert_points, distribution, int_distribution): + for hpt, dst, idst in zip(hilbert_points, distribution, int_distribution, strict=False): if dst == idst: _points.append(hpt) else: @@ -525,10 +544,11 @@ def _generate_values_using_hilbert_sampling(self) -> ndarray[Any, Any]: # find the index where both discrete points are different and interpolate that index # and create the new real-valued point - point: Iterable[float] = [] - for i, j in zip(pt_from_dst, pt_from_dst_nn): + point: list[float] = [] + for i, j in zip(pt_from_dst, pt_from_dst_nn, strict=False): if i != j: - # non-matching index found, i.e. points are in the same dimension and need to be interpolated alongside + # non-matching index found, e.g. + # points are in the same dimension and need to be interpolated alongside smaller_index = min(i, j) fraction, _ = modf(dst) # add the component to real-valued vector @@ -541,17 +561,26 @@ def _generate_values_using_hilbert_sampling(self) -> ndarray[Any, Any]: points: ndarray[Any, Any] = np.array(_points) # Downscale points from hilbert space to unit hypercube [0,1)*d - points = qmc.scale(points, points.min(axis=0), points.max(axis=0), reverse=True) # type: ignore + points = qmc.scale( # pyright: ignore[reportUnknownMemberType] + points, + points.min(axis=0), + points.max(axis=0), + reverse=True, + ) # Upscale points from unit hypercube to bounds - range_lower_bounds: ndarray[Any, Any] = np.array([range[0] for range in self.ranges]) - range_upper_bounds: ndarray[Any, Any] = np.array([range[1] for range in self.ranges]) - sample_set: ndarray[Any, Any] = qmc.scale(points, range_lower_bounds, range_upper_bounds) # type: ignore + range_lower_bounds: ndarray[Any, Any] = np.array([r[0] for r in self.ranges]) + range_upper_bounds: ndarray[Any, Any] = np.array([r[1] for r in self.ranges]) + sample_set: ndarray[Any, Any] = qmc.scale( # pyright: ignore[reportUnknownMemberType] + points, + range_lower_bounds, + range_upper_bounds, + ) return sample_set - def _determine_number_of_samples(self): - if "_includeBoundingBox" in self.sampling_parameters.keys() and isinstance( + def _determine_number_of_samples(self) -> None: + if "_includeBoundingBox" in self.sampling_parameters and isinstance( self.sampling_parameters["_includeBoundingBox"], bool ): self.include_bounding_box = self.sampling_parameters["_includeBoundingBox"] @@ -563,8 +592,8 @@ def _determine_number_of_samples(self): def _generate_case_names( self, - samples: Dict[str, List[Any]], - ): + samples: dict[str, list[Any]], + ) -> None: self.case_names = [ f'{self.layer_name}_{format(i, "0%i" % self.leading_zeros)}' for i in range(self.number_of_samples) ] @@ -583,15 +612,15 @@ def _check_length_matches_number_of_names( def _check_consistency_of_ranges(self, ranges: Sequence[Sequence[Any]]) -> bool: for item in ranges: - if len(item) != 2: + if len(item) != 2: # noqa: PLR2004 logger.error("The structure of min and max values in _ranges is inconsistent.") return False return self._check_length_matches_number_of_names("_ranges") - def _create_bounding_box(self): + def _create_bounding_box(self) -> None: import itertools - tmp: List[Union[float, Sequence[float], Sequence[Any]]] = [] + tmp: list[float | Sequence[float] | Sequence[Any]] = [] if len(self.sampling_parameters["_ranges"]) == 1: # (only one single parameter defined) tmp = list(self.sampling_parameters["_ranges"][0]) @@ -613,7 +642,7 @@ def _create_bounding_box(self): self.bounding_box.append([item]) return - def _write_values_into_samples_dict(self, values: ndarray[Any, Any], samples: Dict[str, List[Any]]): + def _write_values_into_samples_dict(self, values: ndarray[Any, Any], samples: dict[str, list[Any]]) -> None: if self.include_bounding_box is True: self._create_bounding_box() values = np.concatenate((np.array(self.bounding_box), values), axis=0) @@ -624,7 +653,7 @@ def _write_values_into_samples_dict(self, values: ndarray[Any, Any], samples: Di def _flatten(self, iterable: Sequence[Any]) -> Generator[Any, Any, Any]: """Flattens sequence... happens why?.""" for element in iterable: - if isinstance(element, Sequence) and not isinstance(element, (str, bytes)): + if isinstance(element, Sequence) and not isinstance(element, str | bytes): yield from self._flatten(element) else: yield element diff --git a/src/farn/utils/logging.py b/src/farn/utils/logging.py index 7a1d4f72..c6c48755 100644 --- a/src/farn/utils/logging.py +++ b/src/farn/utils/logging.py @@ -1,7 +1,8 @@ +"""Functions to configure logging for the application.""" + import logging import sys from pathlib import Path -from typing import Union __all__ = ["configure_logging"] @@ -10,33 +11,39 @@ def configure_logging( log_level_console: str = "WARNING", - log_file: Union[Path, None] = None, + log_file: Path | None = None, log_level_file: str = "WARNING", -): # sourcery skip: extract-duplicate-method, extract-method - """Configure logging and set levels for log output to console and file. +) -> None: + """Configure logging for the application, allowing for both console and file logging. + + Sets the log levels and formats for the output, ensuring that logs are captured as specified. Parameters ---------- log_level_console : str, optional log level for console output, by default "WARNING" - log_file : Union[Path, None], optional - log file to be used (optional), by default None + log_file : Path | None, optional + log file to be used. If None, file logging is disabled. by default None log_level_file : str, optional log level for file output, by default "WARNING" Raises ------ - ValueError + TypeError if an invalid value for log_level_console or log_level_file is passed - """ + Examples + -------- + configure_logging(log_level_console="INFO", log_file=Path("app.log"), log_level_file="DEBUG") + """ + # sourcery skip: extract-duplicate-method, extract-method log_level_console_numeric = getattr(logging, log_level_console.upper(), None) if not isinstance(log_level_console_numeric, int): - raise ValueError(f"Invalid log level to console: {log_level_console_numeric}") + raise TypeError(f"Invalid log level to console: {log_level_console_numeric}") log_level_file_numeric = getattr(logging, log_level_file.upper(), None) if not isinstance(log_level_file_numeric, int): - raise ValueError(f"Invalid log level to file: {log_level_file_numeric}") + raise TypeError(f"Invalid log level to file: {log_level_file_numeric}") root_logger = logging.getLogger() root_logger.setLevel(logging.DEBUG) @@ -51,6 +58,7 @@ def configure_logging( if not log_file.parent.exists(): log_file.parent.mkdir(parents=True, exist_ok=True) file_handler = logging.FileHandler(str(log_file.absolute()), "a") + print(f"Logging to: {log_file.absolute()}") # noqa: T201 file_handler.setLevel(log_level_file_numeric) file_formatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s", "%Y-%m-%d %H:%M:%S") file_handler.setFormatter(file_formatter) @@ -65,7 +73,8 @@ def plural(count: int, string: str = "") -> str: Parameters ---------- count : int - used to determine whether singular or plural form of string shall be returned. Could i.e. be length of an iterable. If count > 1, plural form will be returned, else singular. + used to determine whether singular or plural form of string shall be returned. + Could i.e. be length of an iterable. If count > 1, plural form will be returned, else singular. string : str, optional the string to be returned in its singular or plural form, by default '' diff --git a/src/farn/utils/os.py b/src/farn/utils/os.py index c8891b06..6c9f4aab 100644 --- a/src/farn/utils/os.py +++ b/src/farn/utils/os.py @@ -3,7 +3,7 @@ __all__ = ["append_system_variable"] -def append_system_variable(variable: str, value: str): +def append_system_variable(variable: str, value: str) -> None: """Append system variable depending on system.""" os.environ[variable] = value return diff --git a/tests/cli/test_farn_cli.py b/tests/cli/test_farn_cli.py index 8ae20dd1..683c3c6f 100644 --- a/tests/cli/test_farn_cli.py +++ b/tests/cli/test_farn_cli.py @@ -3,10 +3,8 @@ from argparse import ArgumentError from dataclasses import dataclass, field from pathlib import Path -from typing import List, Union import pytest -from pytest import MonkeyPatch from farn.cli import farn from farn.cli.farn import _argparser, main @@ -19,12 +17,12 @@ class CliArgs: # Expected default values for the CLI arguments when farn gets called via the commandline quiet: bool = False verbose: bool = False - log: Union[str, None] = None + log: str | None = None log_level: str = field(default_factory=lambda: "WARNING") - farnDict: Union[str, None] = field(default_factory=lambda: "test_farnDict") # noqa: N815 + farnDict: str | None = field(default_factory=lambda: "test_farnDict") # noqa: N815 sample: bool = False generate: bool = False - execute: Union[str, None] = None + execute: str | None = None test: bool = False @@ -58,14 +56,14 @@ class CliArgs: ], ) def test_cli( - inputs: List[str], - expected: Union[CliArgs, type], - monkeypatch: MonkeyPatch, + inputs: list[str], + expected: CliArgs | type, + monkeypatch: pytest.MonkeyPatch, ): # sourcery skip: no-conditionals-in-tests # sourcery skip: no-loop-in-tests # Prepare - monkeypatch.setattr(sys, "argv", ["farn"] + inputs) + monkeypatch.setattr(sys, "argv", ["farn", *inputs]) parser = _argparser() # Execute if isinstance(expected, CliArgs): @@ -80,7 +78,7 @@ def test_cli( with pytest.raises((exception, SystemExit)): args = parser.parse_args() else: - raise AssertionError() + raise AssertionError # *****Ensure the CLI correctly configures logging************************************************* @@ -91,7 +89,7 @@ class ConfigureLoggingArgs: # Values that main() is expected to pass to ConfigureLogging() by default when configuring the logging # Note: 'INFO' deviates from standard 'WARNING', but was decided intentionally for farn log_level_console: str = field(default_factory=lambda: "INFO") - log_file: Union[Path, None] = None + log_file: Path | None = None log_level_file: str = field(default_factory=lambda: "WARNING") @@ -121,19 +119,19 @@ class ConfigureLoggingArgs: ], ) def test_logging_configuration( - inputs: List[str], - expected: Union[ConfigureLoggingArgs, type], - monkeypatch: MonkeyPatch, + inputs: list[str], + expected: ConfigureLoggingArgs | type, + monkeypatch: pytest.MonkeyPatch, ): # sourcery skip: no-conditionals-in-tests # sourcery skip: no-loop-in-tests # Prepare - monkeypatch.setattr(sys, "argv", ["farn"] + inputs) + monkeypatch.setattr(sys, "argv", ["farn", *inputs]) args: ConfigureLoggingArgs = ConfigureLoggingArgs() def fake_configure_logging( log_level_console: str, - log_file: Union[Path, None], + log_file: Path | None, log_level_file: str, ): args.log_level_console = log_level_console @@ -142,9 +140,10 @@ def fake_configure_logging( def fake_run_farn( farn_dict_file: Path, + *, sample: bool, generate: bool, - command: Union[str, None], + command: str | None, batch: bool, test: bool, ): @@ -165,7 +164,7 @@ def fake_run_farn( with pytest.raises((exception, SystemExit)): main() else: - raise AssertionError() + raise AssertionError # *****Ensure the CLI correctly invokes the API**************************************************** @@ -177,7 +176,7 @@ class ApiArgs: farn_dict_file: Path = field(default_factory=lambda: Path("test_farnDict")) sample: bool = False generate: bool = False - command: Union[str, None] = None + command: str | None = None batch: bool = False test: bool = False @@ -205,21 +204,22 @@ class ApiArgs: ], ) def test_api_invokation( - inputs: List[str], - expected: Union[ApiArgs, type], - monkeypatch: MonkeyPatch, + inputs: list[str], + expected: ApiArgs | type, + monkeypatch: pytest.MonkeyPatch, ): # sourcery skip: no-conditionals-in-tests # sourcery skip: no-loop-in-tests # Prepare - monkeypatch.setattr(sys, "argv", ["farn"] + inputs) + monkeypatch.setattr(sys, "argv", ["farn", *inputs]) args: ApiArgs = ApiArgs() def fake_run_farn( farn_dict_file: Path, + *, sample: bool = False, generate: bool = False, - command: Union[str, None] = None, + command: str | None = None, batch: bool = False, test: bool = False, ): @@ -244,4 +244,4 @@ def fake_run_farn( with pytest.raises((exception, SystemExit)): main() else: - raise AssertionError() + raise AssertionError diff --git a/tests/conftest.py b/tests/conftest.py index eec787b5..22e97d5a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,11 +8,19 @@ @pytest.fixture(scope="package", autouse=True) def chdir() -> None: + """ + Fixture that changes the current working directory to the 'test_working_directory' folder. + This fixture is automatically used for the entire package. + """ os.chdir(Path(__file__).parent.absolute() / "test_dicts") @pytest.fixture(scope="package", autouse=True) def test_dir() -> Path: + """ + Fixture that returns the absolute path of the directory containing the current file. + This fixture is automatically used for the entire package. + """ return Path(__file__).parent.absolute() @@ -39,12 +47,19 @@ def test_dir() -> Path: @pytest.fixture(autouse=True) def default_setup_and_teardown(): + """ + Fixture that performs setup and teardown actions before and after each test function. + It removes the output directories and files specified in 'output_dirs' and 'output_files' lists. + """ _remove_output_dirs_and_files() yield _remove_output_dirs_and_files() def _remove_output_dirs_and_files() -> None: + """ + Helper function that removes the output directories and files specified in 'output_dirs' and 'output_files' lists. + """ for folder in output_dirs: rmtree(folder, ignore_errors=True) for pattern in output_files: @@ -55,10 +70,15 @@ def _remove_output_dirs_and_files() -> None: @pytest.fixture(autouse=True) def setup_logging(caplog: pytest.LogCaptureFixture) -> None: + """ + Fixture that sets up logging for each test function. + It sets the log level to 'INFO' and clears the log capture. + """ caplog.set_level("INFO") caplog.clear() @pytest.fixture(autouse=True) def logger() -> logging.Logger: + """Fixture that returns the logger object.""" return logging.getLogger() diff --git a/tests/test_cases.py b/tests/test_cases.py index d4b94552..54c9df28 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -1,338 +1,342 @@ -# pyright: reportUnknownMemberType=false -# pyright: reportArgumentType=false - -from copy import deepcopy -from pathlib import Path -from typing import Any, List, Tuple - -import numpy as np -from dictIO import CppDict, DictReader -from dictIO.utils.path import relative_path -from numpy import ndarray -from pandas import DataFrame - -from farn import create_cases, create_samples -from farn.core import Case, Cases, Parameter - - -def test_cases(): - # Prepare - case_1, case_2, case_3 = _create_cases() - case_list_assert: List[Case] = [case_1, case_2, case_3] - # Execute - cases: Cases = Cases([case_1, case_2, case_3]) - cases_by_append: Cases = Cases() - cases_by_append.append(case_1) - cases_by_append.append(case_2) - cases_by_append.append(case_3) - cases_by_extend: Cases = Cases() - cases_by_extend.extend([case_1, case_2, case_3]) - # Assert - assert len(cases) == 3 - assert len(cases_by_append) == 3 - assert len(cases_by_extend) == 3 - _assert_type_and_equality(cases, case_list_assert) - _assert_type_and_equality(cases_by_append, case_list_assert) - _assert_type_and_equality(cases_by_extend, case_list_assert) - _assert_sequence(cases, case_1, case_2, case_3) - _assert_sequence(cases_by_append, case_1, case_2, case_3) - _assert_sequence(cases_by_extend, case_1, case_2, case_3) - assert len(cases[0].parameters) == 1 - assert len(cases[1].parameters) == 2 - assert len(cases[2].parameters) == 3 - assert cases[2].parameters[0].name == "param_1" # type: ignore - assert cases[2].parameters[1].name == "param_2" # type: ignore - assert cases[2].parameters[2].name == "param_3" # type: ignore - assert cases[2].parameters[0].value == 31.1 # type: ignore - assert cases[2].parameters[1].value == 32.2 # type: ignore - assert cases[2].parameters[2].value == 33.3 # type: ignore - assert cases[0].case == "case_1" - assert cases[1].case == "case_2" - assert cases[2].case == "case_3" - - -def _assert_type_and_equality(cases: Cases, case_list_assert: List[Case]): - assert cases == case_list_assert - assert isinstance(cases, List) - assert isinstance(cases, Cases) - - -def _assert_sequence(cases: Cases, case_assert_1: Case, case_assert_2: Case, case_assert_3: Case): - assert cases[0] is case_assert_1 - assert cases[1] is case_assert_2 - assert cases[2] is case_assert_3 - - -def test_to_pandas_range_index(): - # Prepare - case_1, case_2, case_3 = _create_cases() - cases: Cases = Cases([case_1, case_2, case_3]) - df_assert: DataFrame = _create_dataframe(use_path_as_index=False, parameters_only=False) - # Execute - df: DataFrame = cases.to_pandas(use_path_as_index=False) - # Assert - assert df.shape == df_assert.shape - assert df.shape == (3, 5) - assert df.equals(df_assert) - - -def test_to_pandas_range_index_parameters_only(): - # Prepare - case_1, case_2, case_3 = _create_cases() - cases: Cases = Cases([case_1, case_2, case_3]) - df_assert: DataFrame = _create_dataframe(use_path_as_index=False, parameters_only=True) - # Execute - df: DataFrame = cases.to_pandas(use_path_as_index=False, parameters_only=True) - # Assert - assert df.shape == df_assert.shape - assert df.shape == (3, 3) - assert df.equals(df_assert) - - -def test_to_pandas_path_index(): - # Prepare - case_1, case_2, case_3 = _create_cases() - cases: Cases = Cases([case_1, case_2, case_3]) - df_assert: DataFrame = _create_dataframe(use_path_as_index=True, parameters_only=False) - # Execute - df: DataFrame = cases.to_pandas() - # Assert - assert df.shape == df_assert.shape - assert df.shape == (3, 4) - assert df.equals(df_assert) - - -def test_to_pandas_path_index_parameters_only(): - # Prepare - case_1, case_2, case_3 = _create_cases() - cases: Cases = Cases([case_1, case_2, case_3]) - df_assert: DataFrame = _create_dataframe(use_path_as_index=True, parameters_only=True) - # Execute - df: DataFrame = cases.to_pandas(parameters_only=True) - # Assert - assert df.shape == df_assert.shape - assert df.shape == (3, 3) - assert df.equals(df_assert) - - -def test_to_numpy(): - # Prepare - case_1, case_2, case_3 = _create_cases() - cases: Cases = Cases([case_1, case_2, case_3]) - array_assert: ndarray[Any, Any] = _create_ndarray() - # Execute - array: ndarray[Any, Any] = cases.to_numpy() - # Assert - assert array.shape == array_assert.shape - assert array.shape == (3, 3) - assert str(array) == str(array_assert) - - -def _create_cases() -> Tuple[Case, Case, Case]: - parameter_11 = Parameter("param_1", 11.1) - parameter_12 = Parameter("param_2", 12.2) # noqa: F841 - parameter_13 = Parameter("param_3", 13.3) # noqa: F841 - case_1: Case = Case(case="case_1", parameters=[parameter_11]) - parameter_21 = Parameter("param_1", 21.1) - parameter_22 = Parameter("param_2", 22.2) - parameter_23 = Parameter("param_3", 23.3) # noqa: F841 - case_2: Case = Case(case="case_2", parameters=[parameter_21, parameter_22]) - parameter_31 = Parameter("param_1", 31.1) - parameter_32 = Parameter("param_2", 32.2) - parameter_33 = Parameter("param_3", 33.3) - case_3: Case = Case(case="case_3", parameters=[parameter_31, parameter_32, parameter_33]) - return (case_1, case_2, case_3) - - -def _create_dataframe(use_path_as_index: bool, parameters_only: bool) -> DataFrame: - cwd: Path = Path.cwd() - path: str = str(relative_path(cwd, cwd)) - index: List[int] = [0, 1, 2] - columns: List[str] = ["case", "path", "param_1", "param_2", "param_3"] - values: List[List[Any]] - values = [ - ["case_1", path, 11.1, None, None], - ["case_2", path, 21.1, 22.2, None], - ["case_3", path, 31.1, 32.2, 33.3], - ] - df: DataFrame = DataFrame(data=values, index=index, columns=columns) - if parameters_only: - df.drop(["case"], axis=1, inplace=True) - if not use_path_as_index: - df.drop(["path"], axis=1, inplace=True) - if use_path_as_index: - df.set_index("path", inplace=True) - return df - - -def _create_ndarray() -> ndarray[Any, Any]: - array: ndarray[Any, Any] = np.array( - [ - [11.1, np.nan, np.nan], - [21.1, 22.2, np.nan], - [31.1, 32.2, 33.3], - ] - ) - return array - - -def test_filter_all(): - # Prepare - farn_dict_file = Path("test_farnDict_exclude_filtering") - farn_dict: CppDict = DictReader.read(farn_dict_file, comments=False) - create_samples(farn_dict) - case_dir: Path = Path.cwd() - cases: Cases = create_cases(farn_dict, case_dir, valid_only=False) - cases_not_modified_assert: Cases = deepcopy(cases) - cases_all_assert: Cases = deepcopy(cases) - # Execute - cases_all: Cases = cases.filter([0, 1], valid_only=False) - # Assert - assert isinstance(cases_all, Cases) - assert len(cases_all) == len(cases_all_assert) - assert cases_all == cases_all_assert - assert cases == cases_not_modified_assert - - -def test_filter_level_0(): - # Prepare - farn_dict_file = Path("test_farnDict_exclude_filtering") - farn_dict: CppDict = DictReader.read(farn_dict_file, comments=False) - create_samples(farn_dict) - case_dir: Path = Path.cwd() - cases: Cases = create_cases(farn_dict, case_dir, valid_only=False) - cases_not_modified_assert: Cases = deepcopy(cases) - cases_filtered_assert: Cases = Cases([case for case in cases if case.level == 0]) - # Execute - cases_filtered: Cases = cases.filter(0, valid_only=False) - # Assert - assert isinstance(cases_filtered, Cases) - assert len(cases_filtered) == len(cases_filtered_assert) - assert cases_filtered == cases_filtered_assert - assert cases == cases_not_modified_assert - - -def test_filter_level_1(): - # Prepare - farn_dict_file = Path("test_farnDict_exclude_filtering") - farn_dict: CppDict = DictReader.read(farn_dict_file, comments=False) - create_samples(farn_dict) - case_dir: Path = Path.cwd() - cases: Cases = create_cases(farn_dict, case_dir, valid_only=False) - cases_not_modified_assert: Cases = deepcopy(cases) - cases_filtered_assert: Cases = Cases([case for case in cases if case.level == 1]) - # Execute - cases_filtered: Cases = cases.filter(1, valid_only=False) - # Assert - assert isinstance(cases_filtered, Cases) - assert len(cases_filtered) == len(cases_filtered_assert) - assert cases_filtered == cases_filtered_assert - assert cases == cases_not_modified_assert - - -def test_filter_level_minus_1(): - # Prepare - farn_dict_file = Path("test_farnDict_exclude_filtering") - farn_dict: CppDict = DictReader.read(farn_dict_file, comments=False) - create_samples(farn_dict) - case_dir: Path = Path.cwd() - cases: Cases = create_cases(farn_dict, case_dir, valid_only=False) - cases_not_modified_assert: Cases = deepcopy(cases) - cases_filtered_assert: Cases = Cases([case for case in cases if case.is_leaf]) - # Execute - cases_filtered: Cases = cases.filter(-1, valid_only=False) - # Assert - assert isinstance(cases_filtered, Cases) - assert len(cases_filtered) == len(cases_filtered_assert) - assert cases_filtered == cases_filtered_assert - assert cases == cases_not_modified_assert - - -def test_filter_all_valid_only(): - # Prepare - farn_dict_file = Path("test_farnDict_exclude_filtering") - farn_dict: CppDict = DictReader.read(farn_dict_file, comments=False) - create_samples(farn_dict) - case_dir: Path = Path.cwd() - cases: Cases = create_cases(farn_dict, case_dir, valid_only=False) - cases_not_modified_assert: Cases = deepcopy(cases) - cases_all_assert: Cases = Cases([case for case in cases if case.is_valid]) - # Execute - cases_all: Cases = cases.filter([0, 1], valid_only=True) - # Assert - assert isinstance(cases_all, Cases) - assert len(cases_all) == len(cases_all_assert) - assert cases_all == cases_all_assert - assert cases == cases_not_modified_assert - - -def test_filter_level_0_valid_only(): - # Prepare - farn_dict_file = Path("test_farnDict_exclude_filtering") - farn_dict: CppDict = DictReader.read(farn_dict_file, comments=False) - create_samples(farn_dict) - case_dir: Path = Path.cwd() - cases: Cases = create_cases(farn_dict, case_dir, valid_only=False) - cases_not_modified_assert: Cases = deepcopy(cases) - cases_filtered_assert: Cases = Cases([case for case in cases if case.level == 0 and case.is_valid]) - # Execute - cases_filtered: Cases = cases.filter(0, valid_only=True) - # Assert - assert isinstance(cases_filtered, Cases) - assert len(cases_filtered) == len(cases_filtered_assert) - assert cases_filtered == cases_filtered_assert - assert cases == cases_not_modified_assert - - -def test_filter_level_1_valid_only(): - # Prepare - farn_dict_file = Path("test_farnDict_exclude_filtering") - farn_dict: CppDict = DictReader.read(farn_dict_file, comments=False) - create_samples(farn_dict) - case_dir: Path = Path.cwd() - cases: Cases = create_cases(farn_dict, case_dir, valid_only=False) - cases_not_modified_assert: Cases = deepcopy(cases) - cases_filtered_assert: Cases = Cases([case for case in cases if case.level == 1 and case.is_valid]) - # Execute - cases_filtered: Cases = cases.filter(1, valid_only=True) - # Assert - assert isinstance(cases_filtered, Cases) - assert len(cases_filtered) == len(cases_filtered_assert) - assert cases_filtered == cases_filtered_assert - assert cases == cases_not_modified_assert - - -def test_filter_level_minus_1_valid_only(): - # Prepare - farn_dict_file = Path("test_farnDict_exclude_filtering") - farn_dict: CppDict = DictReader.read(farn_dict_file, comments=False) - create_samples(farn_dict) - case_dir: Path = Path.cwd() - cases: Cases = create_cases(farn_dict, case_dir, valid_only=False) - cases_not_modified_assert: Cases = deepcopy(cases) - cases_filtered_assert: Cases = Cases([case for case in cases if case.is_leaf and case.is_valid]) - # Execute - cases_filtered: Cases = cases.filter(-1, valid_only=True) - # Assert - assert isinstance(cases_filtered, Cases) - assert len(cases_filtered) == len(cases_filtered_assert) - assert cases_filtered == cases_filtered_assert - assert cases == cases_not_modified_assert - - -def test_filter_default_arguments(): - # Prepare - farn_dict_file = Path("test_farnDict_exclude_filtering") - farn_dict: CppDict = DictReader.read(farn_dict_file, comments=False) - create_samples(farn_dict) - case_dir: Path = Path.cwd() - cases: Cases = create_cases(farn_dict, case_dir, valid_only=False) - cases_not_modified_assert: Cases = deepcopy(cases) - cases_filtered_assert: Cases = Cases([case for case in cases if case.is_leaf and case.is_valid]) - # Execute - cases_filtered: Cases = cases.filter() - # Assert - assert isinstance(cases_filtered, Cases) - assert len(cases_filtered) == len(cases_filtered_assert) - assert cases_filtered == cases_filtered_assert - assert cases == cases_not_modified_assert +# pyright: reportUnknownMemberType=false +# pyright: reportArgumentType=false + +from copy import deepcopy +from pathlib import Path +from typing import Any + +import numpy as np +from dictIO import CppDict, DictReader +from dictIO.utils.path import relative_path +from numpy import ndarray +from pandas import DataFrame + +from farn import create_cases, create_samples +from farn.core import Case, Cases, Parameter + + +def test_cases() -> None: + # Prepare + case_1, case_2, case_3 = _create_cases() + case_list_assert: list[Case] = [case_1, case_2, case_3] + # Execute + cases: Cases = Cases([case_1, case_2, case_3]) + cases_by_append: Cases = Cases() + cases_by_append.append(case_1) + cases_by_append.append(case_2) + cases_by_append.append(case_3) + cases_by_extend: Cases = Cases() + cases_by_extend.extend([case_1, case_2, case_3]) + # Assert + assert len(cases) == 3 + assert len(cases_by_append) == 3 + assert len(cases_by_extend) == 3 + _assert_type_and_equality(cases, case_list_assert) + _assert_type_and_equality(cases_by_append, case_list_assert) + _assert_type_and_equality(cases_by_extend, case_list_assert) + _assert_sequence(cases, case_1, case_2, case_3) + _assert_sequence(cases_by_append, case_1, case_2, case_3) + _assert_sequence(cases_by_extend, case_1, case_2, case_3) + assert len(cases[0].parameters) == 1 + assert len(cases[1].parameters) == 2 + assert len(cases[2].parameters) == 3 + assert cases[2].parameters[0].name == "param_1" + assert cases[2].parameters[1].name == "param_2" + assert cases[2].parameters[2].name == "param_3" + assert cases[2].parameters[0].value == 31.1 + assert cases[2].parameters[1].value == 32.2 + assert cases[2].parameters[2].value == 33.3 + assert cases[0].case == "case_1" + assert cases[1].case == "case_2" + assert cases[2].case == "case_3" + + +def _assert_type_and_equality(cases: Cases, case_list_assert: list[Case]) -> None: + assert cases == case_list_assert + assert isinstance(cases, list) + assert isinstance(cases, Cases) + + +def _assert_sequence(cases: Cases, case_assert_1: Case, case_assert_2: Case, case_assert_3: Case) -> None: + assert cases[0] is case_assert_1 + assert cases[1] is case_assert_2 + assert cases[2] is case_assert_3 + + +def test_to_pandas_range_index() -> None: + # Prepare + case_1, case_2, case_3 = _create_cases() + cases: Cases = Cases([case_1, case_2, case_3]) + df_assert: DataFrame = _create_dataframe(use_path_as_index=False, parameters_only=False) + # Execute + df: DataFrame = cases.to_pandas(use_path_as_index=False) + # Assert + assert df.shape == df_assert.shape + assert df.shape == (3, 5) + assert df.equals(df_assert) + + +def test_to_pandas_range_index_parameters_only() -> None: + # Prepare + case_1, case_2, case_3 = _create_cases() + cases: Cases = Cases([case_1, case_2, case_3]) + df_assert: DataFrame = _create_dataframe(use_path_as_index=False, parameters_only=True) + # Execute + df: DataFrame = cases.to_pandas(use_path_as_index=False, parameters_only=True) + # Assert + assert df.shape == df_assert.shape + assert df.shape == (3, 3) + assert df.equals(df_assert) + + +def test_to_pandas_path_index() -> None: + # Prepare + case_1, case_2, case_3 = _create_cases() + cases: Cases = Cases([case_1, case_2, case_3]) + df_assert: DataFrame = _create_dataframe(use_path_as_index=True, parameters_only=False) + # Execute + df: DataFrame = cases.to_pandas() + # Assert + assert df.shape == df_assert.shape + assert df.shape == (3, 4) + assert df.equals(df_assert) + + +def test_to_pandas_path_index_parameters_only() -> None: + # Prepare + case_1, case_2, case_3 = _create_cases() + cases: Cases = Cases([case_1, case_2, case_3]) + df_assert: DataFrame = _create_dataframe(use_path_as_index=True, parameters_only=True) + # Execute + df: DataFrame = cases.to_pandas(parameters_only=True) + # Assert + assert df.shape == df_assert.shape + assert df.shape == (3, 3) + assert df.equals(df_assert) + + +def test_to_numpy() -> None: + # Prepare + case_1, case_2, case_3 = _create_cases() + cases: Cases = Cases([case_1, case_2, case_3]) + array_assert: ndarray[Any, Any] = _create_ndarray() + # Execute + array: ndarray[Any, Any] = cases.to_numpy() + # Assert + assert array.shape == array_assert.shape + assert array.shape == (3, 3) + assert str(array) == str(array_assert) + + +def _create_cases() -> tuple[Case, Case, Case]: + parameter_11 = Parameter("param_1", 11.1) + parameter_12 = Parameter("param_2", 12.2) # noqa: F841 + parameter_13 = Parameter("param_3", 13.3) # noqa: F841 + case_1: Case = Case(case="case_1", parameters=[parameter_11]) + parameter_21 = Parameter("param_1", 21.1) + parameter_22 = Parameter("param_2", 22.2) + parameter_23 = Parameter("param_3", 23.3) # noqa: F841 + case_2: Case = Case(case="case_2", parameters=[parameter_21, parameter_22]) + parameter_31 = Parameter("param_1", 31.1) + parameter_32 = Parameter("param_2", 32.2) + parameter_33 = Parameter("param_3", 33.3) + case_3: Case = Case(case="case_3", parameters=[parameter_31, parameter_32, parameter_33]) + return (case_1, case_2, case_3) + + +def _create_dataframe( + *, + use_path_as_index: bool, + parameters_only: bool, +) -> DataFrame: + cwd: Path = Path.cwd() + path: str = str(relative_path(cwd, cwd)) + index: list[int] = [0, 1, 2] + columns: list[str] = ["case", "path", "param_1", "param_2", "param_3"] + values: list[list[Any]] + values = [ + ["case_1", path, 11.1, None, None], + ["case_2", path, 21.1, 22.2, None], + ["case_3", path, 31.1, 32.2, 33.3], + ] + data: DataFrame = DataFrame(data=values, index=index, columns=columns) + if parameters_only: + data = data.drop(["case"], axis=1) + if not use_path_as_index: + data = data.drop(["path"], axis=1) + if use_path_as_index: + data = data.set_index("path") + return data + + +def _create_ndarray() -> ndarray[Any, Any]: + array: ndarray[Any, Any] = np.array( + [ + [11.1, np.nan, np.nan], + [21.1, 22.2, np.nan], + [31.1, 32.2, 33.3], + ] + ) + return array + + +def test_filter_all() -> None: + # Prepare + farn_dict_file = Path("test_farnDict_exclude_filtering") + farn_dict: CppDict = DictReader.read(farn_dict_file, comments=False) + create_samples(farn_dict) + case_dir: Path = Path.cwd() + cases: Cases = create_cases(farn_dict, case_dir, valid_only=False) + cases_not_modified_assert: Cases = deepcopy(cases) + cases_all_assert: Cases = deepcopy(cases) + # Execute + cases_all: Cases = cases.filter([0, 1], valid_only=False) + # Assert + assert isinstance(cases_all, Cases) + assert len(cases_all) == len(cases_all_assert) + assert cases_all == cases_all_assert + assert cases == cases_not_modified_assert + + +def test_filter_level_0() -> None: + # Prepare + farn_dict_file = Path("test_farnDict_exclude_filtering") + farn_dict: CppDict = DictReader.read(farn_dict_file, comments=False) + create_samples(farn_dict) + case_dir: Path = Path.cwd() + cases: Cases = create_cases(farn_dict, case_dir, valid_only=False) + cases_not_modified_assert: Cases = deepcopy(cases) + cases_filtered_assert: Cases = Cases([case for case in cases if case.level == 0]) + # Execute + cases_filtered: Cases = cases.filter(0, valid_only=False) + # Assert + assert isinstance(cases_filtered, Cases) + assert len(cases_filtered) == len(cases_filtered_assert) + assert cases_filtered == cases_filtered_assert + assert cases == cases_not_modified_assert + + +def test_filter_level_1() -> None: + # Prepare + farn_dict_file = Path("test_farnDict_exclude_filtering") + farn_dict: CppDict = DictReader.read(farn_dict_file, comments=False) + create_samples(farn_dict) + case_dir: Path = Path.cwd() + cases: Cases = create_cases(farn_dict, case_dir, valid_only=False) + cases_not_modified_assert: Cases = deepcopy(cases) + cases_filtered_assert: Cases = Cases([case for case in cases if case.level == 1]) + # Execute + cases_filtered: Cases = cases.filter(1, valid_only=False) + # Assert + assert isinstance(cases_filtered, Cases) + assert len(cases_filtered) == len(cases_filtered_assert) + assert cases_filtered == cases_filtered_assert + assert cases == cases_not_modified_assert + + +def test_filter_level_minus_1() -> None: + # Prepare + farn_dict_file = Path("test_farnDict_exclude_filtering") + farn_dict: CppDict = DictReader.read(farn_dict_file, comments=False) + create_samples(farn_dict) + case_dir: Path = Path.cwd() + cases: Cases = create_cases(farn_dict, case_dir, valid_only=False) + cases_not_modified_assert: Cases = deepcopy(cases) + cases_filtered_assert: Cases = Cases([case for case in cases if case.is_leaf]) + # Execute + cases_filtered: Cases = cases.filter(-1, valid_only=False) + # Assert + assert isinstance(cases_filtered, Cases) + assert len(cases_filtered) == len(cases_filtered_assert) + assert cases_filtered == cases_filtered_assert + assert cases == cases_not_modified_assert + + +def test_filter_all_valid_only() -> None: + # Prepare + farn_dict_file = Path("test_farnDict_exclude_filtering") + farn_dict: CppDict = DictReader.read(farn_dict_file, comments=False) + create_samples(farn_dict) + case_dir: Path = Path.cwd() + cases: Cases = create_cases(farn_dict, case_dir, valid_only=False) + cases_not_modified_assert: Cases = deepcopy(cases) + cases_all_assert: Cases = Cases([case for case in cases if case.is_valid]) + # Execute + cases_all: Cases = cases.filter([0, 1], valid_only=True) + # Assert + assert isinstance(cases_all, Cases) + assert len(cases_all) == len(cases_all_assert) + assert cases_all == cases_all_assert + assert cases == cases_not_modified_assert + + +def test_filter_level_0_valid_only() -> None: + # Prepare + farn_dict_file = Path("test_farnDict_exclude_filtering") + farn_dict: CppDict = DictReader.read(farn_dict_file, comments=False) + create_samples(farn_dict) + case_dir: Path = Path.cwd() + cases: Cases = create_cases(farn_dict, case_dir, valid_only=False) + cases_not_modified_assert: Cases = deepcopy(cases) + cases_filtered_assert: Cases = Cases([case for case in cases if case.level == 0 and case.is_valid]) + # Execute + cases_filtered: Cases = cases.filter(0, valid_only=True) + # Assert + assert isinstance(cases_filtered, Cases) + assert len(cases_filtered) == len(cases_filtered_assert) + assert cases_filtered == cases_filtered_assert + assert cases == cases_not_modified_assert + + +def test_filter_level_1_valid_only() -> None: + # Prepare + farn_dict_file = Path("test_farnDict_exclude_filtering") + farn_dict: CppDict = DictReader.read(farn_dict_file, comments=False) + create_samples(farn_dict) + case_dir: Path = Path.cwd() + cases: Cases = create_cases(farn_dict, case_dir, valid_only=False) + cases_not_modified_assert: Cases = deepcopy(cases) + cases_filtered_assert: Cases = Cases([case for case in cases if case.level == 1 and case.is_valid]) + # Execute + cases_filtered: Cases = cases.filter(1, valid_only=True) + # Assert + assert isinstance(cases_filtered, Cases) + assert len(cases_filtered) == len(cases_filtered_assert) + assert cases_filtered == cases_filtered_assert + assert cases == cases_not_modified_assert + + +def test_filter_level_minus_1_valid_only() -> None: + # Prepare + farn_dict_file = Path("test_farnDict_exclude_filtering") + farn_dict: CppDict = DictReader.read(farn_dict_file, comments=False) + create_samples(farn_dict) + case_dir: Path = Path.cwd() + cases: Cases = create_cases(farn_dict, case_dir, valid_only=False) + cases_not_modified_assert: Cases = deepcopy(cases) + cases_filtered_assert: Cases = Cases([case for case in cases if case.is_leaf and case.is_valid]) + # Execute + cases_filtered: Cases = cases.filter(-1, valid_only=True) + # Assert + assert isinstance(cases_filtered, Cases) + assert len(cases_filtered) == len(cases_filtered_assert) + assert cases_filtered == cases_filtered_assert + assert cases == cases_not_modified_assert + + +def test_filter_default_arguments() -> None: + # Prepare + farn_dict_file = Path("test_farnDict_exclude_filtering") + farn_dict: CppDict = DictReader.read(farn_dict_file, comments=False) + create_samples(farn_dict) + case_dir: Path = Path.cwd() + cases: Cases = create_cases(farn_dict, case_dir, valid_only=False) + cases_not_modified_assert: Cases = deepcopy(cases) + cases_filtered_assert: Cases = Cases([case for case in cases if case.is_leaf and case.is_valid]) + # Execute + cases_filtered: Cases = cases.filter() + # Assert + assert isinstance(cases_filtered, Cases) + assert len(cases_filtered) == len(cases_filtered_assert) + assert cases_filtered == cases_filtered_assert + assert cases == cases_not_modified_assert diff --git a/tests/test_farn.py b/tests/test_farn.py index f3d7a7d5..13c799f8 100644 --- a/tests/test_farn.py +++ b/tests/test_farn.py @@ -1,15 +1,14 @@ import os -import platform from pathlib import Path +import pytest from dictIO import CppDict, DictReader -from pytest import LogCaptureFixture from farn import create_cases, create_samples, run_farn from farn.core import Cases -def test_sample(): +def test_sample() -> None: # Prepare farn_dict_file = Path("test_farnDict_v4") sampled_file = Path(f"sampled.{farn_dict_file.name}") @@ -20,7 +19,7 @@ def test_sample(): assert sampled_file.exists() -def test_create_samples(): +def test_create_samples() -> None: # Prepare farn_dict_file = Path("test_farnDict_v4") farn_dict: CppDict = DictReader.read(farn_dict_file, comments=False) @@ -42,7 +41,7 @@ def test_create_samples(): assert len(farn_dict["_layers"]["mp"]) == len(sampled_farn_dict_assert["_layers"]["mp"]) -def test_create_cases(): +def test_create_cases() -> None: # Prepare farn_dict_file = Path("test_farnDict_no_filtering") farn_dict: CppDict = DictReader.read(farn_dict_file, comments=False) @@ -55,7 +54,7 @@ def test_create_cases(): assert len(cases) == 12 -def test_generate(caplog: LogCaptureFixture): +def test_generate(caplog: pytest.LogCaptureFixture) -> None: # Prepare farn_dict_file = Path("test_farnDict") sampled_file = Path(f"sampled.{farn_dict_file.name}") @@ -70,7 +69,7 @@ def test_generate(caplog: LogCaptureFixture): assert Path("cases/layer1_2").exists() -def test_regenerate(caplog: LogCaptureFixture): +def test_regenerate(caplog: pytest.LogCaptureFixture) -> None: # Prepare farn_dict_file = Path("test_farnDict") sampled_file = Path(f"sampled.{farn_dict_file.name}") @@ -86,7 +85,7 @@ def test_regenerate(caplog: LogCaptureFixture): assert Path("cases/layer1_2").exists() -def test_always_distribute_parameters(): +def test_always_distribute_parameters() -> None: # Prepare farn_dict_file = Path("test_farnDict_always_distribute") sampled_file = Path(f"sampled.{farn_dict_file.name}") @@ -108,7 +107,7 @@ def test_always_distribute_parameters(): # @TODO: There is nothing actually asserted in this test. -> Frank to check. # CLAROS, 2022-05-13 -def test_execute(caplog: LogCaptureFixture): +def test_execute(caplog: pytest.LogCaptureFixture) -> None: # sourcery skip: no-conditionals-in-tests # Prepare farn_dict_file = Path("test_farnDict") @@ -117,16 +116,12 @@ def test_execute(caplog: LogCaptureFixture): _ = run_farn(sampled_file, generate=True) caplog.clear() # Execute - if platform.system() == "Linux": - _ = os.system("farn.py sampled.test_farnDict -e testlinvar") - _ = os.system("farn.py sampled.test_farnDict -e printlinenv") - else: - _ = os.system(f"python -m farn.cli.farn {sampled_file.name} --execute testwinvar") - _ = os.system(f"python -m farn.cli.farn {sampled_file.name} --execute printwinenv") + _ = os.system(f"farn {sampled_file.name} --execute testwinvar") # noqa: S605 + _ = os.system(f"farn {sampled_file.name} --execute printwinenv") # noqa: S605 # Assert -def test_sample_logging_verbosity_default(caplog: LogCaptureFixture): +def test_sample_logging_verbosity_default(caplog: pytest.LogCaptureFixture) -> None: # Prepare farn_dict_file = Path("test_farnDict_no_filtering") # Execute @@ -136,7 +131,7 @@ def test_sample_logging_verbosity_default(caplog: LogCaptureFixture): assert "Successfully listed 10 valid cases. 0 invalid case was excluded." in out -def test_generate_logging_verbosity_default(caplog: LogCaptureFixture): +def test_generate_logging_verbosity_default(caplog: pytest.LogCaptureFixture) -> None: # Prepare farn_dict_file = Path("test_farnDict_no_filtering") sampled_file = Path(f"sampled.{farn_dict_file.name}") @@ -150,7 +145,7 @@ def test_generate_logging_verbosity_default(caplog: LogCaptureFixture): assert "creating case folder" not in out -def test_sample_failed_filtering(caplog: LogCaptureFixture): +def test_sample_failed_filtering(caplog: pytest.LogCaptureFixture) -> None: # Prepare farn_dict_file = Path("test_farnDict_failed_filtering") sampled_file = Path(f"sampled.{farn_dict_file.name}") @@ -163,7 +158,7 @@ def test_sample_failed_filtering(caplog: LogCaptureFixture): assert "evaluation of the filter expression failed" in out -def test_sample_exclude_filtering(caplog: LogCaptureFixture): +def test_sample_exclude_filtering(caplog: pytest.LogCaptureFixture) -> None: # Prepare farn_dict_file = Path("test_farnDict_exclude_filtering") caplog.set_level("DEBUG") @@ -176,7 +171,7 @@ def test_sample_exclude_filtering(caplog: LogCaptureFixture): assert "Action 'exclude' performed. Case lhsVariation_" in out -def test_sample_filtering_one_layer_filter_layer(caplog: LogCaptureFixture): +def test_sample_filtering_one_layer_filter_layer(caplog: pytest.LogCaptureFixture) -> None: # Prepare farn_dict_file = Path("test_farnDict_one_layer_filter_layer") # Execute @@ -186,7 +181,7 @@ def test_sample_filtering_one_layer_filter_layer(caplog: LogCaptureFixture): assert "Successfully listed 2 valid cases. 1 invalid case was excluded." in out -def test_generate_filtering_one_layer_filter_layer(caplog: LogCaptureFixture): +def test_generate_filtering_one_layer_filter_layer(caplog: pytest.LogCaptureFixture) -> None: # Prepare farn_dict_file = Path("test_farnDict_one_layer_filter_layer") sampled_file = Path(f"sampled.{farn_dict_file.name}") @@ -203,7 +198,7 @@ def test_generate_filtering_one_layer_filter_layer(caplog: LogCaptureFixture): assert "Successfully created 2 paramDict files in 2 case folders." in out -def test_sample_filtering_one_layer_filter_param(caplog: LogCaptureFixture): +def test_sample_filtering_one_layer_filter_param(caplog: pytest.LogCaptureFixture) -> None: # Prepare farn_dict_file = Path("test_farnDict_one_layer_filter_param") # Execute @@ -213,7 +208,7 @@ def test_sample_filtering_one_layer_filter_param(caplog: LogCaptureFixture): assert "Successfully listed 2 valid cases. 1 invalid case was excluded." in out -def test_generate_filtering_one_layer_filter_param(caplog: LogCaptureFixture): +def test_generate_filtering_one_layer_filter_param(caplog: pytest.LogCaptureFixture) -> None: # Prepare farn_dict_file = Path("test_farnDict_one_layer_filter_param") sampled_file = Path(f"sampled.{farn_dict_file.name}") @@ -227,7 +222,7 @@ def test_generate_filtering_one_layer_filter_param(caplog: LogCaptureFixture): assert "Successfully created 2 paramDict files in 2 case folders." in out -def test_sample_filtering_two_layers_filter_layer(caplog: LogCaptureFixture): +def test_sample_filtering_two_layers_filter_layer(caplog: pytest.LogCaptureFixture) -> None: # Prepare farn_dict_file = Path("test_farnDict_two_layers_filter_layer") # Execute @@ -237,7 +232,7 @@ def test_sample_filtering_two_layers_filter_layer(caplog: LogCaptureFixture): assert "Successfully listed 3 valid cases. 6 invalid cases were excluded." in out -def test_generate_filtering_two_layers_filter_layer(caplog: LogCaptureFixture): +def test_generate_filtering_two_layers_filter_layer(caplog: pytest.LogCaptureFixture) -> None: # Prepare farn_dict_file = Path("test_farnDict_two_layers_filter_layer") sampled_file = Path(f"sampled.{farn_dict_file.name}") @@ -254,7 +249,7 @@ def test_generate_filtering_two_layers_filter_layer(caplog: LogCaptureFixture): assert "Successfully created 3 paramDict files in 3 case folders." in out -def test_sample_filtering_two_layers_filter_param(caplog: LogCaptureFixture): +def test_sample_filtering_two_layers_filter_param(caplog: pytest.LogCaptureFixture) -> None: # Prepare farn_dict_file = Path("test_farnDict_two_layers_filter_param") # Execute @@ -264,7 +259,7 @@ def test_sample_filtering_two_layers_filter_param(caplog: LogCaptureFixture): assert "Successfully listed 3 valid cases. 6 invalid cases were excluded." in out -def test_generate_filtering_two_layers_filter_param(caplog: LogCaptureFixture): +def test_generate_filtering_two_layers_filter_param(caplog: pytest.LogCaptureFixture) -> None: # Prepare farn_dict_file = Path("test_farnDict_two_layers_filter_param") sampled_file = Path(f"sampled.{farn_dict_file.name}") diff --git a/tests/test_sampling.py b/tests/test_sampling.py index cb365340..9aecb50b 100644 --- a/tests/test_sampling.py +++ b/tests/test_sampling.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List +from typing import Any import numpy as np import pytest @@ -6,7 +6,7 @@ from farn.sampling.sampling import DiscreteSampling -def test_fixed_sampling_one_param(): +def test_fixed_sampling_one_param() -> None: # Prepare sampling: DiscreteSampling = DiscreteSampling() sampling.set_sampling_type(sampling_type="fixed") @@ -18,7 +18,7 @@ def test_fixed_sampling_one_param(): layer_name="layer0", ) # Execute - samples: Dict[str, List[Any]] = sampling.generate_samples() + samples: dict[str, list[Any]] = sampling.generate_samples() # Assert assert len(samples) == 2 assert samples.keys() == { @@ -31,7 +31,7 @@ def test_fixed_sampling_one_param(): assert samples["param1"] == [0.9, 1.3] -def test_fixed_sampling_two_params(): +def test_fixed_sampling_two_params() -> None: # Prepare sampling: DiscreteSampling = DiscreteSampling() sampling.set_sampling_type(sampling_type="fixed") @@ -43,7 +43,7 @@ def test_fixed_sampling_two_params(): layer_name="layer0", ) # Execute - samples: Dict[str, List[Any]] = sampling.generate_samples() + samples: dict[str, list[Any]] = sampling.generate_samples() # Assert assert len(samples) == 3 assert samples.keys() == { @@ -59,7 +59,7 @@ def test_fixed_sampling_two_params(): assert samples["param2"] == [-0.5, 2.7] -def test_fixed_sampling_raise_value_error_if_parameter_values_have_different_length(): +def test_fixed_sampling_raise_value_error_if_parameter_values_have_different_length() -> None: # Prepare sampling: DiscreteSampling = DiscreteSampling() sampling.set_sampling_type(sampling_type="fixed") @@ -75,7 +75,7 @@ def test_fixed_sampling_raise_value_error_if_parameter_values_have_different_len _ = sampling.generate_samples() -def test_linSpace_sampling_one_parameter(): +def test_linSpace_sampling_one_parameter() -> None: # Prepare sampling: DiscreteSampling = DiscreteSampling() sampling.set_sampling_type(sampling_type="linSpace") @@ -88,7 +88,7 @@ def test_linSpace_sampling_one_parameter(): layer_name="layer0", ) # Execute - samples: Dict[str, List[Any]] = sampling.generate_samples() + samples: dict[str, list[Any]] = sampling.generate_samples() # Assert assert len(samples) == 2 assert samples.keys() == { @@ -101,7 +101,7 @@ def test_linSpace_sampling_one_parameter(): assert np.allclose(samples["param1"], [0.5, 0.6, 0.7, 0.8, 0.9]) -def test_linSpace_sampling_two_parameters(): +def test_linSpace_sampling_two_parameters() -> None: # Prepare sampling: DiscreteSampling = DiscreteSampling() sampling.set_sampling_type(sampling_type="linSpace") @@ -114,7 +114,7 @@ def test_linSpace_sampling_two_parameters(): layer_name="layer0", ) # Execute - samples: Dict[str, List[Any]] = sampling.generate_samples() + samples: dict[str, list[Any]] = sampling.generate_samples() # Assert assert len(samples) == 3 assert samples.keys() == { @@ -130,7 +130,7 @@ def test_linSpace_sampling_two_parameters(): assert np.allclose(samples["param2"], [-0.3, -0.2, -0.1, 0.0, 0.1]) -def test_uniformLhs_sampling_three_parameters(): +def test_uniformLhs_sampling_three_parameters() -> None: # Prepare sampling: DiscreteSampling = DiscreteSampling(seed=42) sampling.set_sampling_type(sampling_type="uniformLhs") @@ -142,7 +142,7 @@ def test_uniformLhs_sampling_three_parameters(): }, layer_name="layer0", ) - case_names_expected: List[str] = [ + case_names_expected: list[str] = [ "layer0_00", "layer0_01", "layer0_02", @@ -164,7 +164,7 @@ def test_uniformLhs_sampling_three_parameters(): "layer0_18", "layer0_19", ] - param1_values_expected: List[float] = [ + param1_values_expected: list[float] = [ -8.401341515802963, -4.8165954901465655, -7.9419163878318, @@ -186,7 +186,7 @@ def test_uniformLhs_sampling_three_parameters(): 2.304613769173372, 5.662522284353983, ] - param2_values_expected: List[float] = [ + param2_values_expected: list[float] = [ 2.679549438315647, 2.1170926199511175, 0.9282423925179192, @@ -208,7 +208,7 @@ def test_uniformLhs_sampling_three_parameters(): 2.8323495297169674, 3.3113279911290454, ] - param3_values_expected: List[float] = [ + param3_values_expected: list[float] = [ 0.3636519092097309, 0.2183450418689097, 0.9333271545270508, @@ -232,7 +232,7 @@ def test_uniformLhs_sampling_three_parameters(): ] # Execute - samples: Dict[str, List[Any]] = sampling.generate_samples() + samples: dict[str, list[Any]] = sampling.generate_samples() # Assert assert len(samples) == 4 assert samples.keys() == { @@ -251,7 +251,7 @@ def test_uniformLhs_sampling_three_parameters(): assert np.allclose(samples["param3"], param3_values_expected) -def test_uniformLhs_sampling_three_parameters_including_bounding_box(): +def test_uniformLhs_sampling_three_parameters_including_bounding_box() -> None: # Prepare sampling: DiscreteSampling = DiscreteSampling(seed=42) sampling.set_sampling_type(sampling_type="uniformLhs") @@ -264,7 +264,7 @@ def test_uniformLhs_sampling_three_parameters_including_bounding_box(): }, layer_name="layer0", ) - case_names_expected: List[str] = [ + case_names_expected: list[str] = [ "layer0_00", "layer0_01", "layer0_02", @@ -294,7 +294,7 @@ def test_uniformLhs_sampling_three_parameters_including_bounding_box(): "layer0_26", "layer0_27", ] - param1_values_expected: List[float] = [ + param1_values_expected: list[float] = [ -10.0, -10.0, -10.0, @@ -324,7 +324,7 @@ def test_uniformLhs_sampling_three_parameters_including_bounding_box(): 2.304613769173372, 5.662522284353983, ] - param2_values_expected: List[float] = [ + param2_values_expected: list[float] = [ 0.0, 0.0, 3.5, @@ -354,7 +354,7 @@ def test_uniformLhs_sampling_three_parameters_including_bounding_box(): 2.8323495297169674, 3.3113279911290454, ] - param3_values_expected: List[float] = [ + param3_values_expected: list[float] = [ 0.0, 1.1, 0.0, @@ -386,7 +386,7 @@ def test_uniformLhs_sampling_three_parameters_including_bounding_box(): ] # Execute - samples: Dict[str, List[Any]] = sampling.generate_samples() + samples: dict[str, list[Any]] = sampling.generate_samples() # Assert assert len(samples) == 4 assert samples.keys() == { @@ -405,7 +405,7 @@ def test_uniformLhs_sampling_three_parameters_including_bounding_box(): assert np.allclose(samples["param3"], param3_values_expected) -def test_normalLhs_sampling_three_parameters(): +def test_normalLhs_sampling_three_parameters() -> None: # Prepare sampling: DiscreteSampling = DiscreteSampling(seed=42) sampling.set_sampling_type(sampling_type="normalLhs") @@ -419,7 +419,7 @@ def test_normalLhs_sampling_three_parameters(): }, layer_name="layer0", ) - case_names_expected: List[str] = [ + case_names_expected: list[str] = [ "layer0_00", "layer0_01", "layer0_02", @@ -441,7 +441,7 @@ def test_normalLhs_sampling_three_parameters(): "layer0_18", "layer0_19", ] - param1_values_expected: List[float] = [ + param1_values_expected: list[float] = [ -8.433137323079436, -3.8754339865832215, -7.591054033694368, @@ -463,7 +463,7 @@ def test_normalLhs_sampling_three_parameters(): 1.7578707049336253, 4.696767268691877, ] - param2_values_expected: List[float] = [ + param2_values_expected: list[float] = [ 3.126333090428874, 2.2554158395249084, 0.5580187332707216, @@ -485,7 +485,7 @@ def test_normalLhs_sampling_three_parameters(): 3.4127072609805866, 4.805396097326813, ] - param3_values_expected: List[float] = [ + param3_values_expected: list[float] = [ 0.2870336284886436, 0.04179554753128967, 1.1679596991386783, @@ -509,7 +509,7 @@ def test_normalLhs_sampling_three_parameters(): ] # Execute - samples: Dict[str, List[Any]] = sampling.generate_samples() + samples: dict[str, list[Any]] = sampling.generate_samples() # Assert assert len(samples) == 4 assert samples.keys() == { @@ -528,7 +528,7 @@ def test_normalLhs_sampling_three_parameters(): assert np.allclose(samples["param3"], param3_values_expected) -def test_normalLhs_sampling_three_parameters_with_clipping(): +def test_normalLhs_sampling_three_parameters_with_clipping() -> None: # Prepare sampling: DiscreteSampling = DiscreteSampling(seed=42) sampling.set_sampling_type(sampling_type="normalLhs") @@ -542,7 +542,7 @@ def test_normalLhs_sampling_three_parameters_with_clipping(): }, layer_name="layer0", ) - case_names_expected: List[str] = [ + case_names_expected: list[str] = [ "layer0_00", "layer0_01", "layer0_02", @@ -564,7 +564,7 @@ def test_normalLhs_sampling_three_parameters_with_clipping(): "layer0_18", "layer0_19", ] - param1_values_expected: List[float] = [ + param1_values_expected: list[float] = [ -8.433137323079436, -3.8754339865832215, -7.591054033694368, @@ -586,7 +586,7 @@ def test_normalLhs_sampling_three_parameters_with_clipping(): 1.7578707049336253, 4.696767268691877, ] - param2_values_expected: List[float] = [ + param2_values_expected: list[float] = [ 3.126333090428874, 2.2554158395249084, 0.5580187332707216, @@ -608,7 +608,7 @@ def test_normalLhs_sampling_three_parameters_with_clipping(): 3.4127072609805866, 3.5, # 4.805396097326813 ] - param3_values_expected: List[float] = [ + param3_values_expected: list[float] = [ 0.2870336284886436, 0.04179554753128967, 1.1, # 1.1679596991386783 @@ -632,7 +632,7 @@ def test_normalLhs_sampling_three_parameters_with_clipping(): ] # Execute - samples: Dict[str, List[Any]] = sampling.generate_samples() + samples: dict[str, list[Any]] = sampling.generate_samples() # Assert assert len(samples) == 4 assert samples.keys() == { @@ -651,7 +651,7 @@ def test_normalLhs_sampling_three_parameters_with_clipping(): assert np.allclose(samples["param3"], param3_values_expected) -def test_sobol_sampling_three_parameters(): +def test_sobol_sampling_three_parameters() -> None: # Prepare sampling: DiscreteSampling = DiscreteSampling(seed=42) sampling.set_sampling_type(sampling_type="sobol") @@ -664,7 +664,7 @@ def test_sobol_sampling_three_parameters(): }, layer_name="layer0", ) - case_names_expected: List[str] = [ + case_names_expected: list[str] = [ "layer0_00", "layer0_01", "layer0_02", @@ -686,7 +686,7 @@ def test_sobol_sampling_three_parameters(): "layer0_18", "layer0_19", ] - param1_values_expected: List[float] = [ + param1_values_expected: list[float] = [ -10.0, 0.0, 5.0, @@ -708,7 +708,7 @@ def test_sobol_sampling_three_parameters(): 6.875, -3.125, ] - param2_values_expected: List[float] = [ + param2_values_expected: list[float] = [ 0.0, 1.75, 0.875, @@ -730,7 +730,7 @@ def test_sobol_sampling_three_parameters(): 0.765625, 2.515625, ] - param3_values_expected: List[float] = [ + param3_values_expected: list[float] = [ 0.0, 0.55, 0.275, @@ -754,7 +754,7 @@ def test_sobol_sampling_three_parameters(): ] # Execute - samples: Dict[str, List[Any]] = sampling.generate_samples() + samples: dict[str, list[Any]] = sampling.generate_samples() # Assert assert len(samples) == 4 assert samples.keys() == { @@ -773,7 +773,7 @@ def test_sobol_sampling_three_parameters(): assert np.allclose(samples["param3"], param3_values_expected) -def test_sobol_sampling_three_parameters_with_onset(): +def test_sobol_sampling_three_parameters_with_onset() -> None: # Prepare sampling: DiscreteSampling = DiscreteSampling(seed=42) sampling.set_sampling_type(sampling_type="sobol") @@ -786,7 +786,7 @@ def test_sobol_sampling_three_parameters_with_onset(): }, layer_name="layer0", ) - case_names_expected: List[str] = [ + case_names_expected: list[str] = [ "layer0_00", "layer0_01", "layer0_02", @@ -808,7 +808,7 @@ def test_sobol_sampling_three_parameters_with_onset(): "layer0_18", "layer0_19", ] - param1_values_expected: List[float] = [ + param1_values_expected: list[float] = [ 8.75, -1.25, -3.75, @@ -831,7 +831,7 @@ def test_sobol_sampling_three_parameters_with_onset(): 5.625, ] - param2_values_expected: List[float] = [ + param2_values_expected: list[float] = [ 0.21875, 1.96875, 0.65625, @@ -854,7 +854,7 @@ def test_sobol_sampling_three_parameters_with_onset(): 2.734375, ] - param3_values_expected: List[float] = [ + param3_values_expected: list[float] = [ 0.7562500000000001, 0.20625000000000002, 0.34375, @@ -878,7 +878,7 @@ def test_sobol_sampling_three_parameters_with_onset(): ] # Execute - samples: Dict[str, List[Any]] = sampling.generate_samples() + samples: dict[str, list[Any]] = sampling.generate_samples() # Assert assert len(samples) == 4 assert samples.keys() == { @@ -900,7 +900,7 @@ def test_sobol_sampling_three_parameters_with_onset(): assert np.allclose(samples["param3"], param3_values_expected) -def test_sobol_sampling_three_parameters_including_bounding_box(): +def test_sobol_sampling_three_parameters_including_bounding_box() -> None: # Prepare sampling: DiscreteSampling = DiscreteSampling(seed=42) sampling.set_sampling_type(sampling_type="sobol") @@ -914,7 +914,7 @@ def test_sobol_sampling_three_parameters_including_bounding_box(): }, layer_name="layer0", ) - case_names_expected: List[str] = [ + case_names_expected: list[str] = [ "layer0_00", "layer0_01", "layer0_02", @@ -944,7 +944,7 @@ def test_sobol_sampling_three_parameters_including_bounding_box(): "layer0_26", "layer0_27", ] - param1_values_expected: List[float] = [ + param1_values_expected: list[float] = [ -10.0, -10.0, -10.0, @@ -974,7 +974,7 @@ def test_sobol_sampling_three_parameters_including_bounding_box(): 6.875, -3.125, ] - param2_values_expected: List[float] = [ + param2_values_expected: list[float] = [ 0.0, 0.0, 3.5, @@ -1004,7 +1004,7 @@ def test_sobol_sampling_three_parameters_including_bounding_box(): 0.765625, 2.515625, ] - param3_values_expected: List[float] = [ + param3_values_expected: list[float] = [ 0.0, 1.1, 0.0, @@ -1036,7 +1036,7 @@ def test_sobol_sampling_three_parameters_including_bounding_box(): ] # Execute - samples: Dict[str, List[Any]] = sampling.generate_samples() + samples: dict[str, list[Any]] = sampling.generate_samples() # Assert assert len(samples) == 4 assert samples.keys() == { @@ -1055,7 +1055,7 @@ def test_sobol_sampling_three_parameters_including_bounding_box(): assert np.allclose(samples["param3"], param3_values_expected) -def test_hilbertCurve_sampling_three_parameters(): +def test_hilbertCurve_sampling_three_parameters() -> None: # Prepare sampling: DiscreteSampling = DiscreteSampling() sampling.set_sampling_type(sampling_type="hilbertCurve") @@ -1068,7 +1068,7 @@ def test_hilbertCurve_sampling_three_parameters(): }, layer_name="layer0", ) - case_names_expected: List[str] = [ + case_names_expected: list[str] = [ "layer0_00", "layer0_01", "layer0_02", @@ -1090,7 +1090,7 @@ def test_hilbertCurve_sampling_three_parameters(): "layer0_18", "layer0_19", ] - param1_values_expected: List[float] = [ + param1_values_expected: list[float] = [ -10.0, -2.0625610948191593, -10.0, @@ -1112,7 +1112,7 @@ def test_hilbertCurve_sampling_three_parameters(): 2.0625610948191593, 10.0, ] - param2_values_expected: List[float] = [ + param2_values_expected: list[float] = [ 0.0, 0.7426904253928639, 1.773277701078467, @@ -1134,7 +1134,7 @@ def test_hilbertCurve_sampling_three_parameters(): 0.7426904253928639, 0.0, ] - param3_values_expected: List[float] = [ + param3_values_expected: list[float] = [ 0.0, 0.14691640866416344, 0.4258328173283269, @@ -1158,7 +1158,7 @@ def test_hilbertCurve_sampling_three_parameters(): ] # Execute - samples: Dict[str, List[Any]] = sampling.generate_samples() + samples: dict[str, list[Any]] = sampling.generate_samples() # Assert assert len(samples) == 4 assert samples.keys() == { @@ -1177,7 +1177,7 @@ def test_hilbertCurve_sampling_three_parameters(): assert np.allclose(samples["param3"], param3_values_expected) -def test_hilbertCurve_sampling_three_parameters_with_iteration_depth(): +def test_hilbertCurve_sampling_three_parameters_with_iteration_depth() -> None: # Prepare sampling: DiscreteSampling = DiscreteSampling() sampling.set_sampling_type(sampling_type="hilbertCurve") @@ -1190,7 +1190,7 @@ def test_hilbertCurve_sampling_three_parameters_with_iteration_depth(): }, layer_name="layer0", ) - case_names_expected: List[str] = [ + case_names_expected: list[str] = [ "layer0_00", "layer0_01", "layer0_02", @@ -1212,7 +1212,7 @@ def test_hilbertCurve_sampling_three_parameters_with_iteration_depth(): "layer0_18", "layer0_19", ] - param1_values_expected: List[float] = [ + param1_values_expected: list[float] = [ -10.0, -1.8845500848896517, -9.898132427843818, @@ -1234,7 +1234,7 @@ def test_hilbertCurve_sampling_three_parameters_with_iteration_depth(): 1.8845500848903836, 10.0, ] - param2_values_expected: List[float] = [ + param2_values_expected: list[float] = [ 0.0, 0.6597222222222319, 1.7152777777778028, @@ -1256,7 +1256,7 @@ def test_hilbertCurve_sampling_three_parameters_with_iteration_depth(): 0.6597222222222319, 0.0, ] - param3_values_expected: List[float] = [ + param3_values_expected: list[float] = [ 0.0, 0.12342519685039283, 0.4114173228346427, @@ -1280,7 +1280,7 @@ def test_hilbertCurve_sampling_three_parameters_with_iteration_depth(): ] # Execute - samples: Dict[str, List[Any]] = sampling.generate_samples() + samples: dict[str, list[Any]] = sampling.generate_samples() # Assert assert len(samples) == 4 assert samples.keys() == { @@ -1299,7 +1299,7 @@ def test_hilbertCurve_sampling_three_parameters_with_iteration_depth(): assert np.allclose(samples["param3"], param3_values_expected) -def test_hilbertCurve_sampling_three_parameters_including_bounding_box(): +def test_hilbertCurve_sampling_three_parameters_including_bounding_box() -> None: # Prepare sampling: DiscreteSampling = DiscreteSampling() sampling.set_sampling_type(sampling_type="hilbertCurve") @@ -1312,7 +1312,7 @@ def test_hilbertCurve_sampling_three_parameters_including_bounding_box(): }, layer_name="layer0", ) - case_names_expected: List[str] = [ + case_names_expected: list[str] = [ "layer0_00", "layer0_01", "layer0_02", @@ -1342,7 +1342,7 @@ def test_hilbertCurve_sampling_three_parameters_including_bounding_box(): "layer0_26", "layer0_27", ] - param1_values_expected: List[float] = [ + param1_values_expected: list[float] = [ -10.0, -10.0, -10.0, @@ -1372,7 +1372,7 @@ def test_hilbertCurve_sampling_three_parameters_including_bounding_box(): 2.0625610948191593, 10.0, ] - param2_values_expected: List[float] = [ + param2_values_expected: list[float] = [ 0.0, 0.0, 3.5, @@ -1402,7 +1402,7 @@ def test_hilbertCurve_sampling_three_parameters_including_bounding_box(): 0.7426904253928639, 0.0, ] - param3_values_expected: List[float] = [ + param3_values_expected: list[float] = [ 0.0, 1.1, 0.0, @@ -1434,7 +1434,7 @@ def test_hilbertCurve_sampling_three_parameters_including_bounding_box(): ] # Execute - samples: Dict[str, List[Any]] = sampling.generate_samples() + samples: dict[str, list[Any]] = sampling.generate_samples() # Assert assert len(samples) == 4 assert samples.keys() == {