JP-3773: Store logs from calibration pipeline in datamodel

jwst/stpipe/core.py

@@ -4,16 +4,21 @@
 from functools import wraps
 import logging
 import warnings
+import time
 
 from stdatamodels.jwst.datamodels import JwstDataModel
 from stdatamodels.jwst import datamodels
-from stpipe import crds_client
-from stpipe import Step
-from stpipe import Pipeline
+from stpipe import crds_client, Step, Pipeline
 
 from .. import __version_commit__, __version__
 from ..lib.suffix import remove_suffix
 
+_LOG_FORMATTER = logging.Formatter(
+    "%(asctime)s.%(msecs)03d - %(name)s - %(levelname)s - %(message)s",
+    datefmt="%Y-%m-%dT%H:%M:%S",
+)
+_LOG_FORMATTER.converter = time.gmtime
+
 
 log = logging.getLogger(__name__)
 log.setLevel(logging.DEBUG)
@@ -25,6 +30,8 @@ class JwstStep(Step):
     output_ext = string(default='.fits')  # Output file type
     """
 
+    _log_records_formatter = _LOG_FORMATTER
+
     @classmethod
     def _datamodels_open(cls, init, **kwargs):
         return datamodels.open(init, **kwargs)
@@ -92,6 +99,12 @@ def finalize_result(self, result, reference_files_used):
                 if self.parent is None:
                     log.info(f"Results used CRDS context: {result.meta.ref_file.crds.context_used}")
 
+            if hasattr(result, 'cal_logs'):
+                tmpdict = result.cal_logs.instance
+            else:
+                tmpdict = dict()
+            tmpdict[self.class_alias] = '\n'.join(self._log_records)
+            result.cal_logs = tmpdict
 
     def remove_suffix(self, name):
         return remove_suffix(name)
@@ -124,3 +137,14 @@ class JwstPipeline(Pipeline, JwstStep):
     def finalize_result(self, result, reference_files_used):
         if isinstance(result, JwstDataModel):
             log.info(f"Results used CRDS context: {crds_client.get_context_used(result.crds_observatory)}")
+
+            if hasattr(result, 'cal_logs'):
+                tmpdict = result.cal_logs.instance
+            else:
+                tmpdict = dict()
+            tmpdict[self.class_alias] = '\n'.join(self._log_records)
+
+            for _, step in self.step_defs.items():
+                if step.class_alias in tmpdict.keys():
+                    del tmpdict[step.class_alias]
+            result.cal_logs = tmpdict