move radiance bias correction staging to jedi class (NOAA-EMC#2868)

ush/python/pygfs/jedi/jedi.py

@@ -1,10 +1,13 @@
 #!/usr/bin/env python3
 
 import os
+import tarfile
 from logging import getLogger
+from pprint import pformat
 from typing import List, Dict, Any, Optional
 from jcb import render
 from wxflow import (AttrDict,
+                    FileHandler,
                     chdir, rm_p,
                     parse_j2yaml,
                     logit,
@@ -188,14 +191,10 @@ def get_obs_dict(self, task_config: AttrDict) -> Dict[str, Any]:
         return obs_dict
 
     @logit(logger)
-    def get_bias_dict(self, task_config: AttrDict) -> Dict[str, Any]:
+    def get_bias(self, task_config: AttrDict) -> Dict[str, Any]:
         """Compile a dictionary of observation files to copy
 
-        This method extracts 'observers' from the JEDI yaml and from that list, extracts a list of
-        observation bias correction files that are to be copied to the run directory
-        from the component directory.
-        TODO: COM_ATMOS_ANALYSIS_PREV is hardwired here and this method is not appropriate in
-        `analysis.py` and should be implemented in the component where this is applicable.
+        This method stages radiance bias correction files in the obs sub-diretory of the run directory
 
         Parameters
         ----------
@@ -204,8 +203,7 @@ def get_bias_dict(self, task_config: AttrDict) -> Dict[str, Any]:
 
         Returns
         ----------
-        bias_dict: Dict
-            a dictionary containing the list of observation bias files to copy for FileHandler
+        None
         """
 
         observations = find_value_in_nested_dict(self.config, 'observations')
@@ -218,14 +216,25 @@ def get_bias_dict(self, task_config: AttrDict) -> Dict[str, Any]:
                 basename = os.path.basename(obfile)
                 prefix = '.'.join(basename.split('.')[:-3])
                 bfile = f"{prefix}.rad_varbc_params.tar"
-                copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)])
+                radtar = os.path.join(obdir, bfile)
+                copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), radtar])
                 break
 
         bias_dict = {
             'mkdir': [os.path.join(task_config.DATA, 'bc')],
             'copy': copylist
         }
-        return bias_dict
+
+        # stage bias corrections
+        FileHandler(bias_dict).sync()
+        logger.debug(f"Bias correction files:\n{pformat(bias_dict)}")
+
+        # extract radiance bias correction files from tarball
+        radtar = os.path.join(obdir, bfile)
+        with tarfile.open(radtar, "r") as radbcor:
+            radbcor.extractall(path=os.path.join(task_config.DATA, 'obs'))
+            logger.info(f"Extract {radbcor.getnames()}")
+        radbcor.close()
 
 
 @logit(logger)

ush/python/pygfs/task/atm_analysis.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 import os
-import re
 import glob
 import gzip
 import tarfile
@@ -140,24 +139,7 @@ def initialize_analysis(self) -> None:
 
         # stage bias corrections
         logger.info(f"Staging list of bias correction files generated from JEDI config")
-        bias_dict = self.jedi.get_bias_dict(self.task_config)
-        FileHandler(bias_dict).sync()
-        logger.debug(f"Bias correction files:\n{pformat(bias_dict)}")
-
-        # Extract radiance bias correction files from tarball
-        for action, filelist in bias_dict.items():
-            if 'copy' in action:
-                for sublist in filelist:
-                    if len(sublist) != 2:
-                        raise Exception(
-                            f"List must be of the form ['src', 'dest'], not {sublist}")
-                    src = sublist[0]
-                    if re.search(".tar", src):
-                        radtar = src
-                        with tarfile.open(radtar, "r") as radbcor:
-                            radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs'))
-                            logger.info(f"Extract {radbcor.getnames()}")
-                        radbcor.close()
+        self.jedi.get_bias(self.task_config)
 
         # stage CRTM fix files
         logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}")

ush/python/pygfs/task/atmens_analysis.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 import os
-import re
 import glob
 import gzip
 import tarfile
@@ -139,24 +138,7 @@ def initialize_analysis(self) -> None:
 
         # stage bias corrections
         logger.info(f"Staging list of bias correction files generated from JEDI config")
-        bias_dict = self.jedi.get_bias_dict(self.task_config)
-        FileHandler(bias_dict).sync()
-        logger.debug(f"Bias correction files:\n{pformat(bias_dict)}")
-
-        # Extract radiance bias correction files from tarball
-        for action, filelist in bias_dict.items():
-            if 'copy' in action:
-                for sublist in filelist:
-                    if len(sublist) != 2:
-                        raise Exception(
-                            f"List must be of the form ['src', 'dest'], not {sublist}")
-                    src = sublist[0]
-                    if re.search(".tar", src):
-                        radtar = src
-                        with tarfile.open(radtar, "r") as radbcor:
-                            radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs'))
-                            logger.info(f"Extract {radbcor.getnames()}")
-                        radbcor.close()
+        self.jedi.get_bias(self.task_config)
 
         # stage CRTM fix files
         logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}")