From 732a874a2c6793296f136afb23545fab9869b181 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 4 Mar 2024 16:04:26 -0500 Subject: [PATCH] Reformat snowDA templates to jinja2 (#2371) # Description This PR: - replaces use of non-jinja2 templates in the yaml templates. Specifically `$( )` in favor of pure jinja2. - uses jinja2's built-in capability to include templates within templates, thereby allowing to assemble a completely rendered template before passing it to for e.g. yaml loader. - requires updates to `wxflow` and `gdasapp` - Changes in `wxflow` in `parse_j2yaml` are **not** backwards compatible Additionally, this PR: - renames `config.base.emc.dyn` to `config.base`. Resolves #2347 --- .gitignore | 8 +++- .gitmodules | 38 +++++++++---------- ci/cases/pr/C48mx500_3DVarAOWCDA.yaml | 1 + .../gefs/{config.base.emc.dyn => config.base} | 3 +- .../gfs/{config.base.emc.dyn => config.base} | 3 +- parm/config/gfs/config.prepsnowobs | 9 +++-- parm/config/gfs/config.snowanl | 14 ++----- parm/gdas/snow_jedi_fix.yaml | 7 ---- parm/gdas/snow_jedi_fix.yaml.j2 | 7 ++++ sorc/gdas.cd | 2 +- sorc/link_workflow.sh | 12 ++++++ sorc/wxflow | 2 +- ush/python/pygfs/task/analysis.py | 6 ++- ush/python/pygfs/task/snow_analysis.py | 11 +++--- workflow/setup_expt.py | 11 ++---- 15 files changed, 73 insertions(+), 61 deletions(-) rename parm/config/gefs/{config.base.emc.dyn => config.base} (99%) rename parm/config/gfs/{config.base.emc.dyn => config.base} (99%) delete mode 100644 parm/gdas/snow_jedi_fix.yaml create mode 100644 parm/gdas/snow_jedi_fix.yaml.j2 diff --git a/.gitignore b/.gitignore index 2935804ac4..c9e384aae0 100644 --- a/.gitignore +++ b/.gitignore @@ -43,8 +43,12 @@ fix/wave # Ignore parm file symlinks #-------------------------- -parm/config/config.base -parm/gldas +parm/gdas/aero +parm/gdas/atm +parm/gdas/io +parm/gdas/ioda +parm/gdas/snow +parm/gdas/soca parm/monitor parm/post/AEROSOL_LUTS.dat parm/post/nam_micro_lookup.dat diff --git a/.gitmodules b/.gitmodules index 3eb26fb0fe..5c9e569243 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,28 +1,28 @@ [submodule "sorc/ufs_model.fd"] - path = sorc/ufs_model.fd - url = https://github.com/ufs-community/ufs-weather-model - ignore = dirty + path = sorc/ufs_model.fd + url = https://github.com/ufs-community/ufs-weather-model + ignore = dirty [submodule "sorc/wxflow"] - path = sorc/wxflow - url = https://github.com/NOAA-EMC/wxflow + path = sorc/wxflow + url = https://github.com/NOAA-EMC/wxflow [submodule "sorc/gfs_utils.fd"] - path = sorc/gfs_utils.fd - url = https://github.com/NOAA-EMC/gfs-utils + path = sorc/gfs_utils.fd + url = https://github.com/NOAA-EMC/gfs-utils [submodule "sorc/ufs_utils.fd"] - path = sorc/ufs_utils.fd - url = https://github.com/ufs-community/UFS_UTILS.git + path = sorc/ufs_utils.fd + url = https://github.com/ufs-community/UFS_UTILS.git [submodule "sorc/verif-global.fd"] - path = sorc/verif-global.fd - url = https://github.com/NOAA-EMC/EMC_verif-global.git + path = sorc/verif-global.fd + url = https://github.com/NOAA-EMC/EMC_verif-global.git [submodule "sorc/gsi_enkf.fd"] - path = sorc/gsi_enkf.fd - url = https://github.com/NOAA-EMC/GSI.git + path = sorc/gsi_enkf.fd + url = https://github.com/NOAA-EMC/GSI.git [submodule "sorc/gdas.cd"] - path = sorc/gdas.cd - url = https://github.com/NOAA-EMC/GDASApp.git + path = sorc/gdas.cd + url = https://github.com/NOAA-EMC/GDASApp.git [submodule "sorc/gsi_utils.fd"] - path = sorc/gsi_utils.fd - url = https://github.com/NOAA-EMC/GSI-Utils.git + path = sorc/gsi_utils.fd + url = https://github.com/NOAA-EMC/GSI-Utils.git [submodule "sorc/gsi_monitor.fd"] - path = sorc/gsi_monitor.fd - url = https://github.com/NOAA-EMC/GSI-Monitor.git + path = sorc/gsi_monitor.fd + url = https://github.com/NOAA-EMC/GSI-Monitor.git diff --git a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml index b972d3a445..d9156e38f3 100644 --- a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml +++ b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml @@ -19,4 +19,5 @@ arguments: skip_ci_on_hosts: - orion + - hera - hercules diff --git a/parm/config/gefs/config.base.emc.dyn b/parm/config/gefs/config.base similarity index 99% rename from parm/config/gefs/config.base.emc.dyn rename to parm/config/gefs/config.base index 5ae47e3948..69d4ed94a4 100644 --- a/parm/config/gefs/config.base.emc.dyn +++ b/parm/config/gefs/config.base @@ -110,6 +110,7 @@ export RUN="gefs" # RUN is defined in the job-card (ecf); CDUMP is used at EMC # Get all the COM path templates source "${EXPDIR}/config.com" +# shellcheck disable=SC2016 export ERRSCRIPT=${ERRSCRIPT:-'eval [[ $err = 0 ]]'} export LOGSCRIPT=${LOGSCRIPT:-""} #export ERRSCRIPT=${ERRSCRIPT:-"err_chk"} @@ -210,7 +211,7 @@ export gfs_cyc=@gfs_cyc@ # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4: export FHMIN_GFS=0 export FHMIN=${FHMIN_GFS} export FHMAX_GFS=@FHMAX_GFS@ -export FHOUT_GFS=6 +export FHOUT_GFS=6 export FHMAX_HF_GFS=0 export FHOUT_HF_GFS=1 export FHOUT_OCNICE_GFS=6 diff --git a/parm/config/gfs/config.base.emc.dyn b/parm/config/gfs/config.base similarity index 99% rename from parm/config/gfs/config.base.emc.dyn rename to parm/config/gfs/config.base index 32284929c9..205153313e 100644 --- a/parm/config/gfs/config.base.emc.dyn +++ b/parm/config/gfs/config.base @@ -132,6 +132,7 @@ export RUN=${RUN:-${CDUMP:-"gfs"}} # RUN is defined in the job-card (ecf); CDUM # Get all the COM path templates source "${EXPDIR}/config.com" +# shellcheck disable=SC2016 export ERRSCRIPT=${ERRSCRIPT:-'eval [[ $err = 0 ]]'} export LOGSCRIPT=${LOGSCRIPT:-""} #export ERRSCRIPT=${ERRSCRIPT:-"err_chk"} @@ -247,7 +248,7 @@ export gfs_cyc=@gfs_cyc@ # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4: # GFS output and frequency export FHMIN_GFS=0 export FHMAX_GFS=@FHMAX_GFS@ -export FHOUT_GFS=3 +export FHOUT_GFS=3 export FHMAX_HF_GFS=0 export FHOUT_HF_GFS=1 export FHOUT_OCNICE_GFS=6 diff --git a/parm/config/gfs/config.prepsnowobs b/parm/config/gfs/config.prepsnowobs index 64eb8ba896..e2bfdd1905 100644 --- a/parm/config/gfs/config.prepsnowobs +++ b/parm/config/gfs/config.prepsnowobs @@ -8,11 +8,14 @@ echo "BEGIN: config.prepsnowobs" # Get task specific resources . "${EXPDIR}/config.resources" prepsnowobs -export GTS_OBS_LIST="${HOMEgfs}/sorc/gdas.cd/parm/snow/prep/prep_gts.yaml" +export GTS_OBS_LIST="${HOMEgfs}/parm/gdas/snow/prep/prep_gts.yaml.j2" +export IMS_OBS_LIST="${HOMEgfs}/parm/gdas/snow/prep/prep_ims.yaml.j2" + export BUFR2IODAX="${HOMEgfs}/exec/bufr2ioda.x" -export FIMS_NML_TMPL="${HOMEgfs}/sorc/gdas.cd/parm/snow/prep/fims.nml.j2" -export IMS_OBS_LIST="${HOMEgfs}/sorc/gdas.cd/parm/snow/prep/prep_ims.yaml" + export CALCFIMSEXE="${HOMEgfs}/exec/calcfIMS.exe" +export FIMS_NML_TMPL="${HOMEgfs}/parm/gdas/snow/prep/fims.nml.j2" + export IMS2IODACONV="${HOMEgfs}/ush/imsfv3_scf2ioda.py" echo "END: config.prepsnowobs" diff --git a/parm/config/gfs/config.snowanl b/parm/config/gfs/config.snowanl index 3303ce402b..d8554570d3 100644 --- a/parm/config/gfs/config.snowanl +++ b/parm/config/gfs/config.snowanl @@ -6,19 +6,13 @@ echo "BEGIN: config.snowanl" # Get task specific resources -. "${EXPDIR}/config.resources" snowanl +source "${EXPDIR}/config.resources" snowanl -obs_list_name=gdas_snow_gts_only.yaml -if [[ "${cyc}" = "18" ]]; then - obs_list_name=gdas_snow_prototype.yaml -fi - -export OBS_YAML_DIR=${HOMEgfs}/sorc/gdas.cd/parm/snow/obs/config/ -export OBS_LIST=${HOMEgfs}/sorc/gdas.cd/parm/snow/obs/lists/${obs_list_name} +export OBS_LIST="${HOMEgfs}/parm/gdas/snow/obs/lists/gdas_snow.yaml.j2" # Name of the JEDI executable and its yaml template export JEDIEXE="${HOMEgfs}/exec/fv3jedi_letkf.x" -export JEDIYAML="${HOMEgfs}/sorc/gdas.cd/parm/snow/letkfoi/letkfoi.yaml" +export JEDIYAML="${HOMEgfs}/parm/gdas/snow/letkfoi/letkfoi.yaml.j2" # Ensemble member properties export SNOWDEPTHVAR="snodl" @@ -26,7 +20,7 @@ export BESTDDEV="30." # Background Error Std. Dev. for LETKFOI # Name of the executable that applies increment to bkg and its namelist template export APPLY_INCR_EXE="${HOMEgfs}/exec/apply_incr.exe" -export APPLY_INCR_NML_TMPL="${HOMEgfs}/sorc/gdas.cd/parm/snow/letkfoi/apply_incr_nml.j2" +export APPLY_INCR_NML_TMPL="${HOMEgfs}/parm/gdas/snow/letkfoi/apply_incr_nml.j2" export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ diff --git a/parm/gdas/snow_jedi_fix.yaml b/parm/gdas/snow_jedi_fix.yaml deleted file mode 100644 index 3d1ca79f33..0000000000 --- a/parm/gdas/snow_jedi_fix.yaml +++ /dev/null @@ -1,7 +0,0 @@ -mkdir: -- $(DATA)/fv3jedi -copy: -- [$(HOMEgfs)/fix/gdas/fv3jedi/fv3files/akbk$(npz).nc4, $(DATA)/fv3jedi/akbk.nc4] -- [$(HOMEgfs)/fix/gdas/fv3jedi/fv3files/fmsmpp.nml, $(DATA)/fv3jedi/fmsmpp.nml] -- [$(HOMEgfs)/fix/gdas/fv3jedi/fv3files/field_table_gfdl, $(DATA)/fv3jedi/field_table] -- [$(HOMEgfs)/sorc/gdas.cd/parm/io/fv3jedi_fieldmetadata_restart.yaml, $(DATA)/fv3jedi/fv3jedi_fieldmetadata_restart.yaml] diff --git a/parm/gdas/snow_jedi_fix.yaml.j2 b/parm/gdas/snow_jedi_fix.yaml.j2 new file mode 100644 index 0000000000..4d820a82ba --- /dev/null +++ b/parm/gdas/snow_jedi_fix.yaml.j2 @@ -0,0 +1,7 @@ +mkdir: +- '{{ DATA }}/fv3jedi' +copy: +- ['{{ HOMEgfs }}/fix/gdas/fv3jedi/fv3files/akbk{{ npz }}.nc4', '{{ DATA }}/fv3jedi/akbk.nc4'] +- ['{{ HOMEgfs }}/fix/gdas/fv3jedi/fv3files/fmsmpp.nml', '{{ DATA }}/fv3jedi/fmsmpp.nml'] +- ['{{ HOMEgfs }}/fix/gdas/fv3jedi/fv3files/field_table_gfdl', '{{ DATA }}/fv3jedi/field_table'] +- ['{{ HOMEgfs }}/sorc/gdas.cd/parm/io/fv3jedi_fieldmetadata_restart.yaml', '{{ DATA }}/fv3jedi/fv3jedi_fieldmetadata_restart.yaml'] diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 992306f54e..37a28d114c 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 992306f54e0fa86043e5d4bffe9d81facc5869a0 +Subproject commit 37a28d114c9be6dccff890b33d742b03c22f07c1 diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 3e0ff4db64..97af110077 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -199,6 +199,18 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd" ]]; then done fi +#------------------------------ +#--add GDASApp parm directory +#------------------------------ +if [[ -d "${HOMEgfs}/sorc/gdas.cd" ]]; then + cd "${HOMEgfs}/parm/gdas" || exit 1 + declare -a gdasapp_comps=("aero" "atm" "io" "ioda" "snow" "soca") + for comp in "${gdasapp_comps[@]}"; do + [[ -d "${comp}" ]] && rm -rf "${comp}" + ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/parm/${comp}" . + done +fi + #------------------------------ #--add GDASApp files #------------------------------ diff --git a/sorc/wxflow b/sorc/wxflow index 528f5abb49..dd9ca24a5b 160000 --- a/sorc/wxflow +++ b/sorc/wxflow @@ -1 +1 @@ -Subproject commit 528f5abb49e80751f83ebd6eb0a87bc70012bb24 +Subproject commit dd9ca24a5bb14b75acde685e5fa23b300fd47770 diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index b562eeee4e..5709bc130e 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -25,6 +25,8 @@ class Analysis(Task): def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) self.config.ntiles = 6 + # Store location of GDASApp jinja2 templates + self.gdasapp_j2tmpl_dir = os.path.join(self.config.HOMEgfs, 'parm/gdas') def initialize(self) -> None: super().initialize() @@ -56,7 +58,7 @@ def get_obs_dict(self) -> Dict[str, Any]: a dictionary containing the list of observation files to copy for FileHandler """ logger.debug(f"OBS_LIST: {self.task_config['OBS_LIST']}") - obs_list_config = parse_j2yaml(self.task_config["OBS_LIST"], self.task_config) + obs_list_config = parse_j2yaml(self.task_config["OBS_LIST"], self.task_config, searchpath=self.gdasapp_j2tmpl_dir) logger.debug(f"obs_list_config: {obs_list_config}") # get observers from master dictionary observers = obs_list_config['observers'] @@ -88,7 +90,7 @@ def get_bias_dict(self) -> Dict[str, Any]: a dictionary containing the list of observation bias files to copy for FileHandler """ logger.debug(f"OBS_LIST: {self.task_config['OBS_LIST']}") - obs_list_config = parse_j2yaml(self.task_config["OBS_LIST"], self.task_config) + obs_list_config = parse_j2yaml(self.task_config["OBS_LIST"], self.task_config, searchpath=self.gdasapp_j2tmpl_dir) logger.debug(f"obs_list_config: {obs_list_config}") # get observers from master dictionary observers = obs_list_config['observers'] diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index 01c69dbc7b..9eee8314c3 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -11,7 +11,7 @@ FileHandler, to_fv3time, to_YMD, to_YMDH, to_timedelta, add_to_datetime, rm_p, - parse_j2yaml, parse_yamltmpl, save_as_yaml, + parse_j2yaml, save_as_yaml, Jinja, logit, Executable, @@ -99,7 +99,7 @@ def prepare_GTS(self) -> None: def _gtsbufr2iodax(exe, yaml_file): if not os.path.isfile(yaml_file): - logger.exception(f"{yaml_file} not found") + logger.exception(f"FATAL ERROR: {yaml_file} not found") raise FileNotFoundError(yaml_file) logger.info(f"Executing {exe}") @@ -260,9 +260,9 @@ def initialize(self) -> None: FileHandler({'mkdir': dirlist}).sync() # stage fix files - jedi_fix_list_path = os.path.join(self.task_config.HOMEgfs, 'parm', 'gdas', 'snow_jedi_fix.yaml') + jedi_fix_list_path = os.path.join(self.task_config.HOMEgfs, 'parm', 'gdas', 'snow_jedi_fix.yaml.j2') logger.info(f"Staging JEDI fix files from {jedi_fix_list_path}") - jedi_fix_list = parse_yamltmpl(jedi_fix_list_path, self.task_config) + jedi_fix_list = parse_j2yaml(jedi_fix_list_path, self.task_config) FileHandler(jedi_fix_list).sync() # stage backgrounds @@ -271,10 +271,9 @@ def initialize(self) -> None: # generate letkfoi YAML file logger.info(f"Generate JEDI LETKF YAML file: {self.task_config.jedi_yaml}") - letkfoi_yaml = parse_j2yaml(self.task_config.JEDIYAML, self.task_config) + letkfoi_yaml = parse_j2yaml(self.task_config.JEDIYAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir) save_as_yaml(letkfoi_yaml, self.task_config.jedi_yaml) logger.info(f"Wrote letkfoi YAML to: {self.task_config.jedi_yaml}") - # need output dir for diags and anl logger.info("Create empty output [anl, diags] directories to receive output from executable") newdirs = [ diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 3eeb584f46..90e890910f 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -252,12 +252,6 @@ def fill_EXPDIR(inputs): expdir = os.path.join(inputs.expdir, inputs.pslot) configs = glob.glob(f'{configdir}/config.*') - exclude_configs = ['base', 'base.emc.dyn', 'base.nco.static', 'fv3.nco.static'] - for exclude in exclude_configs: - try: - configs.remove(f'{configdir}/config.{exclude}') - except ValueError: - pass if len(configs) == 0: raise IOError(f'no config files found in {configdir}') for config in configs: @@ -295,7 +289,8 @@ def _update_defaults(dict_in: dict) -> dict: def edit_baseconfig(host, inputs, yaml_dict): """ - Parses and populates the templated `config.base.emc.dyn` to `config.base` + Parses and populates the templated `HOMEgfs/parm/config//config.base` + to `EXPDIR/pslot/config.base` """ tmpl_dict = { @@ -347,7 +342,7 @@ def edit_baseconfig(host, inputs, yaml_dict): except KeyError: pass - base_input = f'{inputs.configdir}/config.base.emc.dyn' + base_input = f'{inputs.configdir}/config.base' base_output = f'{inputs.expdir}/{inputs.pslot}/config.base' edit_config(base_input, base_output, tmpl_dict)