From 2816c3b69d87425c538baa1215ba29719bdcde47 Mon Sep 17 00:00:00 2001 From: AntonMFernando-NOAA <167725623+AntonMFernando-NOAA@users.noreply.github.com> Date: Fri, 6 Sep 2024 23:32:59 -0400 Subject: [PATCH] Add an archive task to GEFS system to archive files locally (#2816) - This task is an extension of the empty arch job previously merged. - This feature adds an archive task to GEFS system to archive files locally. - This feature archives files in ensstat directory. Resolves #2698 Refs #832 #2772 --- jobs/rocoto/arch_test.sh | 3 -- parm/archive/gefs_arcdir.yaml.j2 | 38 +++++++++++++++++++ .../{arcdir.yaml.j2 => gfs_arcdir.yaml.j2} | 0 parm/config/gefs/yaml/defaults.yaml | 2 + scripts/exgdas_enkf_earc.py | 6 ++- scripts/exglobal_archive.py | 12 ++++-- ush/python/pygfs/task/archive.py | 8 ++-- workflow/rocoto/gefs_tasks.py | 2 +- 8 files changed, 57 insertions(+), 14 deletions(-) delete mode 100755 jobs/rocoto/arch_test.sh create mode 100644 parm/archive/gefs_arcdir.yaml.j2 rename parm/archive/{arcdir.yaml.j2 => gfs_arcdir.yaml.j2} (100%) diff --git a/jobs/rocoto/arch_test.sh b/jobs/rocoto/arch_test.sh deleted file mode 100755 index c723c842aa..0000000000 --- a/jobs/rocoto/arch_test.sh +++ /dev/null @@ -1,3 +0,0 @@ -#! /usr/bin/env bash -############################################################### -exit 0 diff --git a/parm/archive/gefs_arcdir.yaml.j2 b/parm/archive/gefs_arcdir.yaml.j2 new file mode 100644 index 0000000000..a59a0e1a8f --- /dev/null +++ b/parm/archive/gefs_arcdir.yaml.j2 @@ -0,0 +1,38 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set cycle_YMD = current_cycle | to_YMD %} +{% set head = RUN + ".t" + cycle_HH + "z." %} + +# Declare the GEFS_ARCH where atmos data will be sent +{% set GEFS_ARCH = ROTDIR ~ "/gefsarch" %} + +{% set file_set = [] %} + +{% set tmpl_dict = ({ '${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':cycle_YMD, + '${HH}':cycle_HH, + '${GRID}': '1p00', + '${MEMDIR}': 'ensstat' }) %} + +{% set COMIN_ATMOS_ENSSTAT_1p00 = COM_ATMOS_GRIB_GRID_TMPL | replace_tmpl(tmpl_dict) %} + +# Select ensstat files to copy to the arcdir +{% if RUN == "gefs" %} + {% set ensstat_files = [] %} + {% if path_exists(COMIN_ATMOS_ENSSTAT_1p00) %} + {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} + {% do ensstat_files.append([COMIN_ATMOS_ENSSTAT_1p00 ~ "/" ~ head ~ "mean.pres_." ~ + "1p00" ~ ".f" ~ '%03d'|format(fhr) ~ ".grib2", + GEFS_ARCH]) %} + {% endfor %} + {% endif %} +{% endif %} +{% set file_set = ensstat_files %} +# Actually write the yaml +mkdir: + - "{{ GEFS_ARCH }}" +copy: + {% for source_dest_pair in file_set %} + - {{ source_dest_pair }} + {% endfor %} diff --git a/parm/archive/arcdir.yaml.j2 b/parm/archive/gfs_arcdir.yaml.j2 similarity index 100% rename from parm/archive/arcdir.yaml.j2 rename to parm/archive/gfs_arcdir.yaml.j2 diff --git a/parm/config/gefs/yaml/defaults.yaml b/parm/config/gefs/yaml/defaults.yaml index e4666d1aba..5ecf690e18 100644 --- a/parm/config/gefs/yaml/defaults.yaml +++ b/parm/config/gefs/yaml/defaults.yaml @@ -14,3 +14,5 @@ base: FCST_BREAKPOINTS: "48" REPLAY_ICS: "NO" USE_OCN_PERTURB_FILES: "false" + HPSSARCH: "NO" + LOCALARCH: "NO" diff --git a/scripts/exgdas_enkf_earc.py b/scripts/exgdas_enkf_earc.py index a515ec9746..c724bdbd67 100755 --- a/scripts/exgdas_enkf_earc.py +++ b/scripts/exgdas_enkf_earc.py @@ -28,11 +28,13 @@ def main(): 'DOHYBVAR', 'DOIAU_ENKF', 'IAU_OFFSET', 'DOIAU', 'DO_CALC_INCREMENT', 'assim_freq', 'ARCH_CYC', 'ARCH_WARMICFREQ', 'ARCH_FCSTICFREQ', - 'IAUFHRS_ENKF'] + 'IAUFHRS_ENKF', 'NET'] archive_dict = AttrDict() for key in keys: - archive_dict[key] = archive.task_config[key] + archive_dict[key] = archive.task_config.get(key) + if archive_dict[key] is None: + print(f"Warning: key ({key}) not found in task_config!") # Also import all COMIN* directory and template variables for key in archive.task_config.keys(): diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index ec8154317f..793fa1c1ac 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -31,16 +31,20 @@ def main(): 'restart_interval_gdas', 'restart_interval_gfs', 'AERO_ANL_RUN', 'AERO_FCST_RUN', 'DOIBP_WAV', 'DO_JEDIOCNVAR', 'NMEM_ENS', 'DO_JEDIATMVAR', 'DO_VRFY_OCEANDA', 'FHMAX_FITS', - 'IAUFHRS', 'DO_FIT2OBS'] + 'IAUFHRS', 'DO_FIT2OBS', 'NET'] archive_dict = AttrDict() for key in keys: - archive_dict[key] = archive.task_config[key] + archive_dict[key] = archive.task_config.get(key) + if archive_dict[key] is None: + print(f"Warning: key ({key}) not found in task_config!") # Also import all COMIN* and COMOUT* directory and template variables for key in archive.task_config.keys(): - if key.startswith("COMIN_") or key.startswith("COMOUT_"): - archive_dict[key] = archive.task_config[key] + if key.startswith("COM_") or key.startswith("COMIN_") or key.startswith("COMOUT_"): + archive_dict[key] = archive.task_config.get(key) + if archive_dict[key] is None: + print(f"Warning: key ({key}) not found in task_config!") cwd = os.getcwd() diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 953a856192..14cd015601 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -63,9 +63,6 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str if not os.path.isdir(arch_dict.ROTDIR): raise FileNotFoundError(f"FATAL ERROR: The ROTDIR ({arch_dict.ROTDIR}) does not exist!") - if arch_dict.RUN == "gefs": - raise NotImplementedError("FATAL ERROR: Archiving is not yet set up for GEFS runs") - if arch_dict.RUN in ["gdas", "gfs"]: # Copy the cyclone track files and rename the experiments @@ -75,7 +72,7 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str archive_parm = os.path.join(arch_dict.PARMgfs, "archive") # Collect the dataset to archive locally - arcdir_j2yaml = os.path.join(archive_parm, "arcdir.yaml.j2") + arcdir_j2yaml = os.path.join(archive_parm, f"{arch_dict.NET}_arcdir.yaml.j2") # Add the glob.glob function for capturing log filenames # TODO remove this kludge once log filenames are explicit @@ -117,6 +114,9 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str self.tar_cmd = "" return arcdir_set, [] + if arch_dict.NET == "gefs": + raise NotImplementedError("GEFS archiving is not yet implemented!") + master_yaml = "master_" + arch_dict.RUN + ".yaml.j2" parsed_sets = parse_j2yaml(os.path.join(archive_parm, master_yaml), diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index d1ece0f096..70a39cea5a 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -555,7 +555,7 @@ def arch(self): 'envars': self.envars, 'cycledef': 'gefs', 'dependency': dependencies, - 'command': f'{self.HOMEgfs}/jobs/rocoto/arch_test.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/arch.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;'