From 828fab70092c1edbe5f00a6900cd6a9a61bc5370 Mon Sep 17 00:00:00 2001 From: Tyler Pauly Date: Fri, 4 Oct 2024 14:16:02 -0400 Subject: [PATCH 1/3] add logs to jwst datamodels --- jwst/stpipe/core.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/jwst/stpipe/core.py b/jwst/stpipe/core.py index 21efa00ef0..3e8fe562fe 100644 --- a/jwst/stpipe/core.py +++ b/jwst/stpipe/core.py @@ -4,16 +4,21 @@ from functools import wraps import logging import warnings +import time from stdatamodels.jwst.datamodels import JwstDataModel from stdatamodels.jwst import datamodels -from stpipe import crds_client -from stpipe import Step -from stpipe import Pipeline +from stpipe import crds_client, Step, Pipeline from .. import __version_commit__, __version__ from ..lib.suffix import remove_suffix +_LOG_FORMATTER = logging.Formatter( + "%(asctime)s.%(msecs)03d - %(name)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%dT%H:%M:%S", +) +_LOG_FORMATTER.converter = time.gmtime + log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) @@ -25,6 +30,8 @@ class JwstStep(Step): output_ext = string(default='.fits') # Output file type """ + _log_records_formatter = _LOG_FORMATTER + @classmethod def _datamodels_open(cls, init, **kwargs): return datamodels.open(init, **kwargs) @@ -92,6 +99,10 @@ def finalize_result(self, result, reference_files_used): if self.parent is None: log.info(f"Results used CRDS context: {result.meta.ref_file.crds.context_used}") + if hasattr(result, 'cal_logs'): + tmpdict = result.cal_logs.instance + tmpdict[self.class_alias] = '\n'.join(self._log_records) + result.cal_logs = tmpdict def remove_suffix(self, name): return remove_suffix(name) @@ -124,3 +135,12 @@ class JwstPipeline(Pipeline, JwstStep): def finalize_result(self, result, reference_files_used): if isinstance(result, JwstDataModel): log.info(f"Results used CRDS context: {crds_client.get_context_used(result.crds_observatory)}") + + if hasattr(result, 'cal_logs'): + tmpdict = result.cal_logs.instance + tmpdict[self.class_alias] = '\n'.join(self._log_records) + + for _, step in self.step_defs.items(): + if step.class_alias in tmpdict.keys(): + del tmpdict[step.class_alias] + result.cal_logs = tmpdict From 51837d109d49b769b4dddcd75dae01634a64772d Mon Sep 17 00:00:00 2001 From: Tyler Pauly Date: Fri, 4 Oct 2024 14:29:56 -0400 Subject: [PATCH 2/3] no init attr with no datamodels pr --- jwst/stpipe/core.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/jwst/stpipe/core.py b/jwst/stpipe/core.py index 3e8fe562fe..2cedc04f1e 100644 --- a/jwst/stpipe/core.py +++ b/jwst/stpipe/core.py @@ -101,8 +101,10 @@ def finalize_result(self, result, reference_files_used): if hasattr(result, 'cal_logs'): tmpdict = result.cal_logs.instance - tmpdict[self.class_alias] = '\n'.join(self._log_records) - result.cal_logs = tmpdict + else: + tmpdict = dict() + tmpdict[self.class_alias] = '\n'.join(self._log_records) + result.cal_logs = tmpdict def remove_suffix(self, name): return remove_suffix(name) @@ -138,9 +140,11 @@ def finalize_result(self, result, reference_files_used): if hasattr(result, 'cal_logs'): tmpdict = result.cal_logs.instance - tmpdict[self.class_alias] = '\n'.join(self._log_records) + else: + tmpdict = dict() + tmpdict[self.class_alias] = '\n'.join(self._log_records) - for _, step in self.step_defs.items(): - if step.class_alias in tmpdict.keys(): - del tmpdict[step.class_alias] - result.cal_logs = tmpdict + for _, step in self.step_defs.items(): + if step.class_alias in tmpdict.keys(): + del tmpdict[step.class_alias] + result.cal_logs = tmpdict From a6f56c991bfbe2331374bd4d18872478077b5fa1 Mon Sep 17 00:00:00 2001 From: Tyler Pauly Date: Mon, 7 Oct 2024 10:37:34 -0400 Subject: [PATCH 3/3] only log log if result is dm --- jwst/stpipe/core.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/jwst/stpipe/core.py b/jwst/stpipe/core.py index 2cedc04f1e..fa9aaae0c6 100644 --- a/jwst/stpipe/core.py +++ b/jwst/stpipe/core.py @@ -99,12 +99,12 @@ def finalize_result(self, result, reference_files_used): if self.parent is None: log.info(f"Results used CRDS context: {result.meta.ref_file.crds.context_used}") - if hasattr(result, 'cal_logs'): - tmpdict = result.cal_logs.instance - else: - tmpdict = dict() - tmpdict[self.class_alias] = '\n'.join(self._log_records) - result.cal_logs = tmpdict + if hasattr(result, 'cal_logs'): + tmpdict = result.cal_logs.instance + else: + tmpdict = dict() + tmpdict[self.class_alias] = '\n'.join(self._log_records) + result.cal_logs = tmpdict def remove_suffix(self, name): return remove_suffix(name)