Stage variational and ensemble DA job files with Jinja2-templated YAMLs

parm/config/gfs/config.atmanl

@@ -23,6 +23,9 @@
 
 export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2"
 export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2"
+export VAR_BKG_STAGING_YAML="${PARMgfs}/gdas/atm_var_bkg_staging.yaml.j2"
+export BERROR_STAGING_YAML="${PARMgfs}/gdas/atm_berror_"${STATICB_TYPE}"_staging.yaml.j2"
+export FV3ENS_STAGING_YAML="${PARMgfs}/gdas/atm_var_fv3ens_staging.yaml.j2"
 
 export layout_x_atmanl=@LAYOUT_X_ATMANL@
 export layout_y_atmanl=@LAYOUT_Y_ATMANL@

parm/config/gfs/config.atmensanl

@@ -12,6 +12,7 @@ export INTERP_METHOD='barycentric'
 
 export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2"
 export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2"
+export LGETKF_BKG_STAGING_YAML="${PARMgfs}/gdas/atm_lgetkf_bkg_staging.yaml.j2"
 
 export layout_x_atmensanl=@LAYOUT_X_ATMENSANL@
 export layout_y_atmensanl=@LAYOUT_Y_ATMENSANL@

parm/gdas/atm_berror_bump_staging.yaml.j2

@@ -0,0 +1,23 @@
+{% set src_dir = BERROR_DATA_DIR %}
+{% set dest_dir = DATA ~ '/berror' %}
+{% set ftype_list = ['cor_rh', 'cor_rv', 'stddev'] %}
+{% set nprocs = ntiles * layout_x * layout_y %}
+
+mkdir:
+- '{{ dest_dir }}'
+
+copy:
+{% for ftype in ftype_list %}
+{% set prefix = BERROR_DATE | to_fv3time ~ '.' ~ ftype %}
+{% set fname = prefix ~ '.coupler.res' %}
+- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}']
+{% for itile in range(1,7) %}
+{% set fname = prefix ~ '.fv_tracer.res.tile' ~  itile ~ '.nc' %}
+- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}']
+{% endfor %}
+{% endfor %}
+
+{% for nn in range(1,nprocs+1) %}
+{% set fname = 'nicas_aero_nicas_local_%06d-%06d.nc' | format(nprocs, nn) %}
+- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}']
+{% endfor %}

parm/gdas/atm_berror_gsibec_staging.yaml.j2

@@ -0,0 +1,10 @@
+{% set src_dir = HOMEgfs ~ '/fix/gdas/gsibec/' ~ CASE_ANL %}
+{% set dest_dir = DATA ~ '/berror' %}
+{% set fname_list = ['gfs_gsi_global.nml', 'gsi-coeffs-gfs-global.nc4'] %}
+
+mkdir:
+- '{{ dest_dir }}'
+copy:
+{% for fname in fname_list %}
+- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}']
+{% endfor %}

parm/gdas/atm_lgetkf_bkg_staging.yaml.j2

@@ -0,0 +1,27 @@
+{% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %}
+{% set prefix = current_cycle | to_fv3time %}
+
+mkdir:
+{% for imem in range(1,NMEM_ENS+1) %}
+{% set memchar = 'mem%03d' | format(imem) %}
+- '{{ DATA }}/bkg/{{ memchar }}'
+- '{{ DATA }}/anl/{{ memchar }}'
+{% endfor %}
+copy:
+{% for imem in range(1,NMEM_ENS+1) %}
+{% set memchar = 'mem%03d' | format(imem) %}
+{% set src_dir = COM_ATMOS_RESTART_TMPL | replace('${ROTDIR}', ROTDIR)
+                                        | replace('${RUN}', 'enkfgdas')
+                                        | replace('${YMD}', previous_cycle | to_YMD)
+                                        | replace('${HH}', previous_cycle | strftime('%H'))
+                                        | replace('${MEMDIR}', memchar) %}
+{% set dest_dir = DATA ~ '/bkg/' ~ memchar %}
+{% set fname = prefix ~ '.coupler.res' %}
+- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}']
+{% for ftype in ftype_list %}
+{% for itile in range(1,7) %}
+{% set fname = prefix ~ '.' ~ ftype ~ '.tile' ~ itile ~ '.nc' %}
+- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}']
+{% endfor %}
+{% endfor %}
+{% endfor %}

parm/gdas/atm_var_bkg_staging.yaml.j2

@@ -0,0 +1,16 @@
+{% set src_dir = COM_ATMOS_RESTART_PREV %}
+{% set dest_dir = DATA ~ '/bkg' %}
+{% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %}
+{% set prefix = current_cycle | to_fv3time %}
+
+mkdir:
+- '{{ dest_dir }}'
+copy:
+{% set fname = prefix ~ '.coupler.res' %}
+- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}']
+{% for ftype in ftype_list %}
+{% for itile in range(1,7) %}
+{% set fname = prefix ~ '.' ~ ftype ~ '.tile' ~ itile ~ '.nc' %}
+- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}']
+{% endfor %}
+{% endfor %}

parm/gdas/atm_var_fv3ens_staging.yaml.j2

@@ -0,0 +1,27 @@
+{% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %}
+{% set prefix = current_cycle | to_fv3time %}
+
+mkdir:
+{% for imem in range(1,NMEM_ENS+1) %}
+{% set memchar = 'mem%03d' | format(imem) %}
+{% set dest_dir = DATA ~ '/ens/' ~ memchar %}
+- '{{ dest_dir }}'
+{% endfor %}
+copy:
+{% for imem in range(1,NMEM_ENS+1) %}
+{% set memchar = 'mem%03d' | format(imem) %}
+{% set src_dir = COM_ATMOS_RESTART_TMPL | replace('${ROTDIR}', ROTDIR)
+                                        | replace('${RUN}', 'enkfgdas')
+                                        | replace('${YMD}', previous_cycle | to_YMD)
+                                        | replace('${HH}', previous_cycle | strftime('%H'))
+                                        | replace('${MEMDIR}', memchar) %}
+{% set dest_dir = DATA ~ '/ens/' ~ memchar %}
+{% set fname = prefix ~ '.coupler.res' %}
+- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}']
+{% for ftype in ftype_list %}
+{% for itile in range(1,7) %}
+{% set fname = prefix ~ '.' ~ ftype ~ '.tile' ~ itile ~ '.nc' %}
+- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}']
+{% endfor %}
+{% endfor %}
+{% endfor %}

ush/python/pygfs/task/analysis.py

@@ -194,44 +194,6 @@ def add_fv3_increments(self, inc_file_tmpl: str, bkg_file_tmpl: str, incvars: Li
                     except (AttributeError, RuntimeError):
                         pass  # checksum is missing, move on
 
-    @logit(logger)
-    def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]:
-        """Compile a dictionary of model background files to copy
-
-        This method is a placeholder for now... will be possibly made generic at a later date
-
-        Parameters
-        ----------
-        task_config: Dict
-            a dictionary containing all of the configuration needed for the task
-
-        Returns
-        ----------
-        bkg_dict: Dict
-            a dictionary containing the list of model background files to copy for FileHandler
-        """
-        bkg_dict = {'foo': 'bar'}
-        return bkg_dict
-
-    @logit(logger)
-    def get_berror_dict(self, config: Dict[str, Any]) -> Dict[str, List[str]]:
-        """Compile a dictionary of background error files to copy
-
-        This method is a placeholder for now... will be possibly made generic at a later date
-
-        Parameters
-        ----------
-        config: Dict
-            a dictionary containing all of the configuration needed
-
-        Returns
-        ----------
-        berror_dict: Dict
-            a dictionary containing the list of background error files to copy for FileHandler
-        """
-        berror_dict = {'foo': 'bar'}
-        return berror_dict
-
     @logit(logger)
     def link_jediexe(self) -> None:
         """Compile a dictionary of background error files to copy
@@ -258,68 +220,6 @@ def link_jediexe(self) -> None:
 
         return exe_dest
 
-    @staticmethod
-    @logit(logger)
-    def get_fv3ens_dict(config: Dict[str, Any]) -> Dict[str, Any]:
-        """Compile a dictionary of ensemble member restarts to copy
-
-        This method constructs a dictionary of ensemble FV3 restart files (coupler, core, tracer)
-        that are needed for global atmens DA and returns said dictionary for use by the FileHandler class.
-
-        Parameters
-        ----------
-        config: Dict
-            a dictionary containing all of the configuration needed
-
-        Returns
-        ----------
-        ens_dict: Dict
-            a dictionary containing the list of ensemble member restart files to copy for FileHandler
-        """
-        # NOTE for now this is FV3 restart files and just assumed to be fh006
-
-        # define template
-        template_res = config.COM_ATMOS_RESTART_TMPL
-        prev_cycle = config.previous_cycle
-        tmpl_res_dict = {
-            'ROTDIR': config.ROTDIR,
-            'RUN': config.RUN,
-            'YMD': to_YMD(prev_cycle),
-            'HH': prev_cycle.strftime('%H'),
-            'MEMDIR': None
-        }
-
-        # construct ensemble member file list
-        dirlist = []
-        enslist = []
-        for imem in range(1, config.NMEM_ENS + 1):
-            memchar = f"mem{imem:03d}"
-
-            # create directory path for ensemble member restart
-            dirlist.append(os.path.join(config.DATA, config.dirname, f'mem{imem:03d}'))
-
-            # get FV3 restart files, this will be a lot simpler when using history files
-            tmpl_res_dict['MEMDIR'] = memchar
-            rst_dir = Template.substitute_structure(template_res, TemplateConstants.DOLLAR_CURLY_BRACE, tmpl_res_dict.get)
-            run_dir = os.path.join(config.DATA, config.dirname, memchar)
-
-            # atmens DA needs coupler
-            basename = f'{to_fv3time(config.current_cycle)}.coupler.res'
-            enslist.append([os.path.join(rst_dir, basename), os.path.join(config.DATA, config.dirname, memchar, basename)])
-
-            # atmens DA needs core, srf_wnd, tracer, phy_data, sfc_data
-            for ftype in ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data']:
-                template = f'{to_fv3time(config.current_cycle)}.{ftype}.tile{{tilenum}}.nc'
-                for itile in range(1, config.ntiles + 1):
-                    basename = template.format(tilenum=itile)
-                    enslist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)])
-
-        ens_dict = {
-            'mkdir': dirlist,
-            'copy': enslist,
-        }
-        return ens_dict
-
     @staticmethod
     @logit(logger)
     def tgz_diags(statfile: str, diagdir: str) -> None: