From 3a723e821164af337a084d26587fe59660aabcfc Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Wed, 23 Aug 2023 08:38:31 -0400 Subject: [PATCH 1/9] Initialize the branch to do the madis DA. --- parm/config/gfs/config.landanl | 2 +- parm/config/gfs/config.preplandobs | 3 +- ush/python/pygfs/task/land_analysis.py | 39 +++++++++++++++++++++----- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/parm/config/gfs/config.landanl b/parm/config/gfs/config.landanl index 942e4528fe..70ebae7529 100644 --- a/parm/config/gfs/config.landanl +++ b/parm/config/gfs/config.landanl @@ -8,7 +8,7 @@ echo "BEGIN: config.landanl" # Get task specific resources . "${EXPDIR}/config.resources" landanl -obs_list_name=gdas_land_adpsfc_only.yaml +obs_list_name=gdas_land_gts_only.yaml if [[ "${cyc}" = "18" ]]; then obs_list_name=gdas_land_prototype.yaml fi diff --git a/parm/config/gfs/config.preplandobs b/parm/config/gfs/config.preplandobs index 4e0b7f0d95..22f17964f0 100644 --- a/parm/config/gfs/config.preplandobs +++ b/parm/config/gfs/config.preplandobs @@ -10,7 +10,8 @@ echo "BEGIN: config.preplandobs" export GTS_OBS_LIST="${HOMEgfs}/sorc/gdas.cd/parm/land/prep/prep_gts.yaml" export BUFR2IODAX="${HOMEgfs}/exec/bufr2ioda.x" -export BUFR2IODAYAML="${HOMEgfs}/sorc/gdas.cd/test/testinput/bufr_adpsfc_snow.yaml" +export BUFRADPSFCYAML="${HOMEgfs}/sorc/gdas.cd/test/testinput/bufr_adpsfc_snow.yaml" +export BUFRSNOCVRYAML="${HOMEgfs}/sorc/gdas.cd/test/testinput/bufr_snocvr.yaml" export FIMS_NML_TMPL="${HOMEgfs}/sorc/gdas.cd/parm/land/prep/fims.nml.j2" export IMS_OBS_LIST="${HOMEgfs}/sorc/gdas.cd/parm/land/prep/prep_ims.yaml" export CALCFIMSEXE="${HOMEgfs}/exec/calcfIMS.exe" diff --git a/ush/python/pygfs/task/land_analysis.py b/ush/python/pygfs/task/land_analysis.py index 7ef74626ef..5efe89d0c7 100644 --- a/ush/python/pygfs/task/land_analysis.py +++ b/ush/python/pygfs/task/land_analysis.py @@ -87,12 +87,17 @@ def prepare_GTS(self) -> None: logger.info("Copying GTS obs for bufr2ioda.x") FileHandler(prep_gts_config.gtsbufr).sync() - # generate bufr2ioda YAML file - bufr2ioda_yaml = os.path.join(self.runtime_config.DATA, "bufr_adpsfc_snow.yaml") - logger.info(f"Generate BUFR2IODA YAML file: {bufr2ioda_yaml}") - temp_yaml = parse_j2yaml(self.task_config.BUFR2IODAYAML, self.task_config) - save_as_yaml(temp_yaml, bufr2ioda_yaml) - logger.info(f"Wrote bufr2ioda YAML to: {bufr2ioda_yaml}") + # generate bufr2ioda YAML files + adpsfc_yaml = os.path.join(self.runtime_config.DATA, "bufr_adpsfc_snow.yaml") + logger.info(f"Generate BUFR2IODA YAML file: {adpsfc_yaml}") + temp_yaml = parse_j2yaml(self.task_config.BUFRADPSFCYAML, self.task_config) + save_as_yaml(temp_yaml, adpsfc_yaml) + logger.info(f"Wrote bufr2ioda YAML to: {adpsfc_yaml}") + snocvr_yaml = os.path.join(self.runtime_config.DATA, "bufr_snocvr.yaml") + logger.info(f"Generate BUFR2IODA YAML file: {snocvr_yaml}") + temp_yaml = parse_j2yaml(self.task_config.BUFRSNOCVRYAML, self.task_config) + save_as_yaml(temp_yaml, snocvr_yaml) + logger.info(f"Wrote bufr2ioda YAML to: {snocvr_yaml}") logger.info("Link BUFR2IODAX into DATA/") exe_src = self.task_config.BUFR2IODAX @@ -105,7 +110,7 @@ def prepare_GTS(self) -> None: if os.path.isfile(f"{os.path.join(localconf.DATA, output_file)}"): rm_p(output_file) - # execute BUFR2IODAX to convert GTS bufr data into IODA format + # execute BUFR2IODAX to convert adpsfc bufr data into IODA format yaml_file = f"bufr_adpsfc_snow.yaml" if not os.path.isfile(f"{os.path.join(localconf.DATA, yaml_file)}"): logger.exception(f"{yaml_file} not found") @@ -113,6 +118,26 @@ def prepare_GTS(self) -> None: exe = Executable(self.task_config.BUFR2IODAX) exe.add_default_arg(os.path.join(localconf.DATA, f"{yaml_file}")) + logger.info(f"Executing {exe}") + try: + exe() + except OSError: + raise OSError(f"Failed to execute {exe}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exe}") + + output_file = f"{localconf.OPREFIX}snocvr_snow.nc4" + if os.path.isfile(f"{os.path.join(localconf.DATA, output_file)}"): + rm_p(output_file) + + # execute BUFR2IODAX to convert snocvr bufr data into IODA format + yaml_file = f"bufr_snocvr.yaml" + if not os.path.isfile(f"{os.path.join(localconf.DATA, yaml_file)}"): + logger.exception(f"{yaml_file} not found") + raise FileNotFoundError(f"{os.path.join(localconf.DATA, yaml_file)}") + exe = Executable(self.task_config.BUFR2IODAX) + exe.add_default_arg(os.path.join(localconf.DATA, f"{yaml_file}")) + logger.info(f"Executing {exe}") try: exe() From 7f44dd408b1ded032fa66f625de79fbff81f5f18 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Sat, 9 Sep 2023 11:48:47 -0400 Subject: [PATCH 2/9] Remove some repetitions. --- ush/python/pygfs/task/land_analysis.py | 53 +++++++++----------------- 1 file changed, 19 insertions(+), 34 deletions(-) diff --git a/ush/python/pygfs/task/land_analysis.py b/ush/python/pygfs/task/land_analysis.py index 5efe89d0c7..b5addeff1b 100644 --- a/ush/python/pygfs/task/land_analysis.py +++ b/ush/python/pygfs/task/land_analysis.py @@ -90,12 +90,12 @@ def prepare_GTS(self) -> None: # generate bufr2ioda YAML files adpsfc_yaml = os.path.join(self.runtime_config.DATA, "bufr_adpsfc_snow.yaml") logger.info(f"Generate BUFR2IODA YAML file: {adpsfc_yaml}") - temp_yaml = parse_j2yaml(self.task_config.BUFRADPSFCYAML, self.task_config) + temp_yaml = parse_j2yaml(self.task_config.BUFRADPSFCYAML, localconf) save_as_yaml(temp_yaml, adpsfc_yaml) logger.info(f"Wrote bufr2ioda YAML to: {adpsfc_yaml}") snocvr_yaml = os.path.join(self.runtime_config.DATA, "bufr_snocvr.yaml") logger.info(f"Generate BUFR2IODA YAML file: {snocvr_yaml}") - temp_yaml = parse_j2yaml(self.task_config.BUFRSNOCVRYAML, self.task_config) + temp_yaml = parse_j2yaml(self.task_config.BUFRSNOCVRYAML, localconf) save_as_yaml(temp_yaml, snocvr_yaml) logger.info(f"Wrote bufr2ioda YAML to: {snocvr_yaml}") @@ -106,45 +106,30 @@ def prepare_GTS(self) -> None: rm_p(exe_dest) os.symlink(exe_src, exe_dest) + # Create executable instance + exe = Executable(self.task_config.BUFR2IODAX) + def _gtsbufr2iodax(exe, yaml_file): + if not os.path.isfile(yaml_file): + logger.exception(f"{yaml_file} not found") + raise FileNotFoundError(yaml_file) + + logger.info(f"Executing {exe}") + try: + exe(yaml_file) + except OSError: + raise OSError(f"Failed to execute {exe} {yaml_file}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exe} {yaml_file}") + output_file = f"{localconf.OPREFIX}adpsfc_snow.nc4" if os.path.isfile(f"{os.path.join(localconf.DATA, output_file)}"): rm_p(output_file) # execute BUFR2IODAX to convert adpsfc bufr data into IODA format - yaml_file = f"bufr_adpsfc_snow.yaml" - if not os.path.isfile(f"{os.path.join(localconf.DATA, yaml_file)}"): - logger.exception(f"{yaml_file} not found") - raise FileNotFoundError(f"{os.path.join(localconf.DATA, yaml_file)}") - exe = Executable(self.task_config.BUFR2IODAX) - exe.add_default_arg(os.path.join(localconf.DATA, f"{yaml_file}")) - - logger.info(f"Executing {exe}") - try: - exe() - except OSError: - raise OSError(f"Failed to execute {exe}") - except Exception: - raise WorkflowException(f"An error occured during execution of {exe}") - - output_file = f"{localconf.OPREFIX}snocvr_snow.nc4" - if os.path.isfile(f"{os.path.join(localconf.DATA, output_file)}"): - rm_p(output_file) + _gtsbufr2iodax(exe, os.path.join(localconf.DATA, "bufr_adpsfc_snow.yaml")) # execute BUFR2IODAX to convert snocvr bufr data into IODA format - yaml_file = f"bufr_snocvr.yaml" - if not os.path.isfile(f"{os.path.join(localconf.DATA, yaml_file)}"): - logger.exception(f"{yaml_file} not found") - raise FileNotFoundError(f"{os.path.join(localconf.DATA, yaml_file)}") - exe = Executable(self.task_config.BUFR2IODAX) - exe.add_default_arg(os.path.join(localconf.DATA, f"{yaml_file}")) - - logger.info(f"Executing {exe}") - try: - exe() - except OSError: - raise OSError(f"Failed to execute {exe}") - except Exception: - raise WorkflowException(f"An error occured during execution of {exe}") + _gtsbufr2iodax(exe, os.path.join(localconf.DATA, "bufr_snocvr.yaml")) # Ensure the IODA snow depth GTS file is produced by the IODA converter # If so, copy to COM_OBS/ From 0273248d953f45ee693c28790145e5c68a870eb8 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Sat, 9 Sep 2023 11:54:36 -0400 Subject: [PATCH 3/9] Fix the python norm error. --- ush/python/pygfs/task/land_analysis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ush/python/pygfs/task/land_analysis.py b/ush/python/pygfs/task/land_analysis.py index b5addeff1b..9a4cb044e0 100644 --- a/ush/python/pygfs/task/land_analysis.py +++ b/ush/python/pygfs/task/land_analysis.py @@ -108,6 +108,7 @@ def prepare_GTS(self) -> None: # Create executable instance exe = Executable(self.task_config.BUFR2IODAX) + def _gtsbufr2iodax(exe, yaml_file): if not os.path.isfile(yaml_file): logger.exception(f"{yaml_file} not found") From 4d3e1056b48c54e82e2e31dddf8bf84e68e1b88b Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Sat, 9 Sep 2023 18:38:22 -0400 Subject: [PATCH 4/9] Simplify the code to generate bufr2ioda YAML files --- ush/python/pygfs/task/land_analysis.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/ush/python/pygfs/task/land_analysis.py b/ush/python/pygfs/task/land_analysis.py index 9a4cb044e0..a06cc84563 100644 --- a/ush/python/pygfs/task/land_analysis.py +++ b/ush/python/pygfs/task/land_analysis.py @@ -73,7 +73,7 @@ def prepare_GTS(self) -> None: # create a temporary dict of all keys needed in this method localconf = AttrDict() - keys = ['DATA', 'current_cycle', 'COM_OBS', 'COM_ATMOS_RESTART_PREV', + keys = ['HOMEgfs', 'DATA', 'current_cycle', 'COM_OBS', 'COM_ATMOS_RESTART_PREV', 'OPREFIX', 'CASE', 'ntiles'] for key in keys: localconf[key] = self.task_config[key] @@ -88,16 +88,12 @@ def prepare_GTS(self) -> None: FileHandler(prep_gts_config.gtsbufr).sync() # generate bufr2ioda YAML files - adpsfc_yaml = os.path.join(self.runtime_config.DATA, "bufr_adpsfc_snow.yaml") - logger.info(f"Generate BUFR2IODA YAML file: {adpsfc_yaml}") - temp_yaml = parse_j2yaml(self.task_config.BUFRADPSFCYAML, localconf) - save_as_yaml(temp_yaml, adpsfc_yaml) - logger.info(f"Wrote bufr2ioda YAML to: {adpsfc_yaml}") - snocvr_yaml = os.path.join(self.runtime_config.DATA, "bufr_snocvr.yaml") - logger.info(f"Generate BUFR2IODA YAML file: {snocvr_yaml}") - temp_yaml = parse_j2yaml(self.task_config.BUFRSNOCVRYAML, localconf) - save_as_yaml(temp_yaml, snocvr_yaml) - logger.info(f"Wrote bufr2ioda YAML to: {snocvr_yaml}") + for name in ["adpsfc", "snocvr"]: + gts_yaml = os.path.join(self.runtime_config.DATA, f"bufr_{name}_snow.yaml") + logger.info(f"Generate BUFR2IODA YAML file: {gts_yaml}") + temp_yaml = parse_j2yaml(prep_gts_config.bufr2ioda[name], localconf) + save_as_yaml(temp_yaml, gts_yaml) + logger.info(f"Wrote bufr2ioda YAML to: {gts_yaml}") logger.info("Link BUFR2IODAX into DATA/") exe_src = self.task_config.BUFR2IODAX @@ -130,7 +126,7 @@ def _gtsbufr2iodax(exe, yaml_file): _gtsbufr2iodax(exe, os.path.join(localconf.DATA, "bufr_adpsfc_snow.yaml")) # execute BUFR2IODAX to convert snocvr bufr data into IODA format - _gtsbufr2iodax(exe, os.path.join(localconf.DATA, "bufr_snocvr.yaml")) + _gtsbufr2iodax(exe, os.path.join(localconf.DATA, "bufr_snocvr_snow.yaml")) # Ensure the IODA snow depth GTS file is produced by the IODA converter # If so, copy to COM_OBS/ From 9bce693bcbe0b13cb4cb1458f2e21da48f71eaee Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Sat, 9 Sep 2023 18:44:24 -0400 Subject: [PATCH 5/9] Remove the obsolete BUFR2IODAYAML and BUFRADPSFCYAML. --- parm/config/gfs/config.preplandobs | 2 -- 1 file changed, 2 deletions(-) diff --git a/parm/config/gfs/config.preplandobs b/parm/config/gfs/config.preplandobs index 22f17964f0..20ae20b5ad 100644 --- a/parm/config/gfs/config.preplandobs +++ b/parm/config/gfs/config.preplandobs @@ -10,8 +10,6 @@ echo "BEGIN: config.preplandobs" export GTS_OBS_LIST="${HOMEgfs}/sorc/gdas.cd/parm/land/prep/prep_gts.yaml" export BUFR2IODAX="${HOMEgfs}/exec/bufr2ioda.x" -export BUFRADPSFCYAML="${HOMEgfs}/sorc/gdas.cd/test/testinput/bufr_adpsfc_snow.yaml" -export BUFRSNOCVRYAML="${HOMEgfs}/sorc/gdas.cd/test/testinput/bufr_snocvr.yaml" export FIMS_NML_TMPL="${HOMEgfs}/sorc/gdas.cd/parm/land/prep/fims.nml.j2" export IMS_OBS_LIST="${HOMEgfs}/sorc/gdas.cd/parm/land/prep/prep_ims.yaml" export CALCFIMSEXE="${HOMEgfs}/exec/calcfIMS.exe" From 60567dfa1fbde9900f77329ad63b362bc8108b6c Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Sun, 10 Sep 2023 20:30:24 -0400 Subject: [PATCH 6/9] Address reviewer's comments. --- ush/python/pygfs/task/land_analysis.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ush/python/pygfs/task/land_analysis.py b/ush/python/pygfs/task/land_analysis.py index a06cc84563..c10e1ca018 100644 --- a/ush/python/pygfs/task/land_analysis.py +++ b/ush/python/pygfs/task/land_analysis.py @@ -118,10 +118,6 @@ def _gtsbufr2iodax(exe, yaml_file): except Exception: raise WorkflowException(f"An error occured during execution of {exe} {yaml_file}") - output_file = f"{localconf.OPREFIX}adpsfc_snow.nc4" - if os.path.isfile(f"{os.path.join(localconf.DATA, output_file)}"): - rm_p(output_file) - # execute BUFR2IODAX to convert adpsfc bufr data into IODA format _gtsbufr2iodax(exe, os.path.join(localconf.DATA, "bufr_adpsfc_snow.yaml")) @@ -133,7 +129,7 @@ def _gtsbufr2iodax(exe, yaml_file): try: FileHandler(prep_gts_config.gtsioda).sync() except OSError as err: - logger.exception(f"{self.task_config.BUFR2IODAX} failed to produce {output_file}") + logger.exception(f"{self.task_config.BUFR2IODAX} failed to produce GTS ioda files") raise OSError(err) @logit(logger) From dc83591881540ed23472c1666397e234a9d64653 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Mon, 11 Sep 2023 18:12:14 -0400 Subject: [PATCH 7/9] Simplify the codes following the reviewer's suggestions. --- ush/python/pygfs/task/land_analysis.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/ush/python/pygfs/task/land_analysis.py b/ush/python/pygfs/task/land_analysis.py index c10e1ca018..aaeb8bad55 100644 --- a/ush/python/pygfs/task/land_analysis.py +++ b/ush/python/pygfs/task/land_analysis.py @@ -87,14 +87,6 @@ def prepare_GTS(self) -> None: logger.info("Copying GTS obs for bufr2ioda.x") FileHandler(prep_gts_config.gtsbufr).sync() - # generate bufr2ioda YAML files - for name in ["adpsfc", "snocvr"]: - gts_yaml = os.path.join(self.runtime_config.DATA, f"bufr_{name}_snow.yaml") - logger.info(f"Generate BUFR2IODA YAML file: {gts_yaml}") - temp_yaml = parse_j2yaml(prep_gts_config.bufr2ioda[name], localconf) - save_as_yaml(temp_yaml, gts_yaml) - logger.info(f"Wrote bufr2ioda YAML to: {gts_yaml}") - logger.info("Link BUFR2IODAX into DATA/") exe_src = self.task_config.BUFR2IODAX exe_dest = os.path.join(localconf.DATA, os.path.basename(exe_src)) @@ -118,11 +110,18 @@ def _gtsbufr2iodax(exe, yaml_file): except Exception: raise WorkflowException(f"An error occured during execution of {exe} {yaml_file}") - # execute BUFR2IODAX to convert adpsfc bufr data into IODA format - _gtsbufr2iodax(exe, os.path.join(localconf.DATA, "bufr_adpsfc_snow.yaml")) + # Loop over entries in prep_gts_config.bufr2ioda keys + # 1. generate bufr2ioda YAML files + # 2. execute bufr2ioda.x + for name in prep_gts_config.bufr2ioda.keys(): + gts_yaml = os.path.join(self.runtime_config.DATA, f"bufr_{name}_snow.yaml") + logger.info(f"Generate BUFR2IODA YAML file: {gts_yaml}") + temp_yaml = parse_j2yaml(prep_gts_config.bufr2ioda[name], localconf) + save_as_yaml(temp_yaml, gts_yaml) + logger.info(f"Wrote bufr2ioda YAML to: {gts_yaml}") - # execute BUFR2IODAX to convert snocvr bufr data into IODA format - _gtsbufr2iodax(exe, os.path.join(localconf.DATA, "bufr_snocvr_snow.yaml")) + # execute BUFR2IODAX to convert {name} bufr data into IODA format + _gtsbufr2iodax(exe, os.path.join(localconf.DATA, f"bufr_{name}_snow.yaml")) # Ensure the IODA snow depth GTS file is produced by the IODA converter # If so, copy to COM_OBS/ From 94a0ff8036bca03ecee11466504029e585436780 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Mon, 11 Sep 2023 18:24:44 -0400 Subject: [PATCH 8/9] Remove the unnecessary construction of gts_yaml. --- ush/python/pygfs/task/land_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/land_analysis.py b/ush/python/pygfs/task/land_analysis.py index aaeb8bad55..4bec65be5f 100644 --- a/ush/python/pygfs/task/land_analysis.py +++ b/ush/python/pygfs/task/land_analysis.py @@ -121,7 +121,7 @@ def _gtsbufr2iodax(exe, yaml_file): logger.info(f"Wrote bufr2ioda YAML to: {gts_yaml}") # execute BUFR2IODAX to convert {name} bufr data into IODA format - _gtsbufr2iodax(exe, os.path.join(localconf.DATA, f"bufr_{name}_snow.yaml")) + _gtsbufr2iodax(exe, gts_yaml) # Ensure the IODA snow depth GTS file is produced by the IODA converter # If so, copy to COM_OBS/ From 73fcc1c0b1b68da44a1ab1f7727bf3dd05b2c4b7 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Wed, 13 Sep 2023 09:51:21 -0400 Subject: [PATCH 9/9] Update the GDASApp's commit hash. --- Externals.cfg | 2 +- sorc/checkout.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 884f5deb31..325be174d2 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -57,7 +57,7 @@ protocol = git required = False [GDASApp] -hash = ac8fdb1 +hash = d347d22 local_path = sorc/gdas.cd repo_url = https://github.com/NOAA-EMC/GDASApp.git protocol = git diff --git a/sorc/checkout.sh b/sorc/checkout.sh index 76966387c5..c62d5b026d 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -160,7 +160,7 @@ if [[ ${checkout_gsi} == "YES" ]]; then fi if [[ ${checkout_gdas} == "YES" ]]; then - checkout "gdas.cd" "https://github.com/NOAA-EMC/GDASApp.git" "ac8fdb1"; errs=$((errs + $?)) + checkout "gdas.cd" "https://github.com/NOAA-EMC/GDASApp.git" "d347d22"; errs=$((errs + $?)) fi if [[ ${checkout_gsi} == "YES" || ${checkout_gdas} == "YES" ]]; then