Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add the capability to assimilate the MADIS snow depth data from GTS #1836

Merged
merged 13 commits into from
Sep 13, 2023
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion parm/config/gfs/config.landanl
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ echo "BEGIN: config.landanl"
# Get task specific resources
. "${EXPDIR}/config.resources" landanl

obs_list_name=gdas_land_adpsfc_only.yaml
obs_list_name=gdas_land_gts_only.yaml
if [[ "${cyc}" = "18" ]]; then
obs_list_name=gdas_land_prototype.yaml
fi
Expand Down
1 change: 0 additions & 1 deletion parm/config/gfs/config.preplandobs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ echo "BEGIN: config.preplandobs"

export GTS_OBS_LIST="${HOMEgfs}/sorc/gdas.cd/parm/land/prep/prep_gts.yaml"
export BUFR2IODAX="${HOMEgfs}/exec/bufr2ioda.x"
export BUFR2IODAYAML="${HOMEgfs}/sorc/gdas.cd/test/testinput/bufr_adpsfc_snow.yaml"
export FIMS_NML_TMPL="${HOMEgfs}/sorc/gdas.cd/parm/land/prep/fims.nml.j2"
export IMS_OBS_LIST="${HOMEgfs}/sorc/gdas.cd/parm/land/prep/prep_ims.yaml"
export CALCFIMSEXE="${HOMEgfs}/exec/calcfIMS.exe"
Expand Down
53 changes: 28 additions & 25 deletions ush/python/pygfs/task/land_analysis.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ def prepare_GTS(self) -> None:

# create a temporary dict of all keys needed in this method
localconf = AttrDict()
keys = ['DATA', 'current_cycle', 'COM_OBS', 'COM_ATMOS_RESTART_PREV',
keys = ['HOMEgfs', 'DATA', 'current_cycle', 'COM_OBS', 'COM_ATMOS_RESTART_PREV',
'OPREFIX', 'CASE', 'ntiles']
for key in keys:
localconf[key] = self.task_config[key]
Expand All @@ -87,12 +87,13 @@ def prepare_GTS(self) -> None:
logger.info("Copying GTS obs for bufr2ioda.x")
FileHandler(prep_gts_config.gtsbufr).sync()

# generate bufr2ioda YAML file
bufr2ioda_yaml = os.path.join(self.runtime_config.DATA, "bufr_adpsfc_snow.yaml")
logger.info(f"Generate BUFR2IODA YAML file: {bufr2ioda_yaml}")
temp_yaml = parse_j2yaml(self.task_config.BUFR2IODAYAML, self.task_config)
save_as_yaml(temp_yaml, bufr2ioda_yaml)
logger.info(f"Wrote bufr2ioda YAML to: {bufr2ioda_yaml}")
# generate bufr2ioda YAML files
for name in ["adpsfc", "snocvr"]:
gts_yaml = os.path.join(self.runtime_config.DATA, f"bufr_{name}_snow.yaml")
logger.info(f"Generate BUFR2IODA YAML file: {gts_yaml}")
temp_yaml = parse_j2yaml(prep_gts_config.bufr2ioda[name], localconf)
save_as_yaml(temp_yaml, gts_yaml)
logger.info(f"Wrote bufr2ioda YAML to: {gts_yaml}")
Copy link
Contributor

@aerorahul aerorahul Sep 11, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for updating this section @jiaruidong2017
Here is what I was suggesting.
In the attached patch, you will notice that I have removed any knowledge of what is being converted from bufr to ioda in the python script. The control comes directly from the yaml.

Would you be willing to give this a try?
diff.txt

Thanks

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks @aerorahul. I will try it and let you know then.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@aerorahul I just tested and it worked perfectly. The changes have been committed to the repo.


logger.info("Link BUFR2IODAX into DATA/")
exe_src = self.task_config.BUFR2IODAX
Expand All @@ -101,32 +102,34 @@ def prepare_GTS(self) -> None:
rm_p(exe_dest)
os.symlink(exe_src, exe_dest)

output_file = f"{localconf.OPREFIX}adpsfc_snow.nc4"
if os.path.isfile(f"{os.path.join(localconf.DATA, output_file)}"):
rm_p(output_file)

# execute BUFR2IODAX to convert GTS bufr data into IODA format
yaml_file = f"bufr_adpsfc_snow.yaml"
if not os.path.isfile(f"{os.path.join(localconf.DATA, yaml_file)}"):
logger.exception(f"{yaml_file} not found")
raise FileNotFoundError(f"{os.path.join(localconf.DATA, yaml_file)}")
# Create executable instance
exe = Executable(self.task_config.BUFR2IODAX)
exe.add_default_arg(os.path.join(localconf.DATA, f"{yaml_file}"))

logger.info(f"Executing {exe}")
try:
exe()
except OSError:
raise OSError(f"Failed to execute {exe}")
except Exception:
raise WorkflowException(f"An error occured during execution of {exe}")
def _gtsbufr2iodax(exe, yaml_file):
if not os.path.isfile(yaml_file):
logger.exception(f"{yaml_file} not found")
raise FileNotFoundError(yaml_file)

logger.info(f"Executing {exe}")
try:
exe(yaml_file)
except OSError:
raise OSError(f"Failed to execute {exe} {yaml_file}")
except Exception:
raise WorkflowException(f"An error occured during execution of {exe} {yaml_file}")

# execute BUFR2IODAX to convert adpsfc bufr data into IODA format
_gtsbufr2iodax(exe, os.path.join(localconf.DATA, "bufr_adpsfc_snow.yaml"))

# execute BUFR2IODAX to convert snocvr bufr data into IODA format
_gtsbufr2iodax(exe, os.path.join(localconf.DATA, "bufr_snocvr_snow.yaml"))

# Ensure the IODA snow depth GTS file is produced by the IODA converter
# If so, copy to COM_OBS/
try:
FileHandler(prep_gts_config.gtsioda).sync()
except OSError as err:
logger.exception(f"{self.task_config.BUFR2IODAX} failed to produce {output_file}")
logger.exception(f"{self.task_config.BUFR2IODAX} failed to produce GTS ioda files")
raise OSError(err)

@logit(logger)
Expand Down