From c679d94ae25462603c2a418c5d9f5e216e47e013 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 17 Apr 2024 12:06:20 -0400 Subject: [PATCH] Add rocoto `sh` tag, script to check netcdf file and apply this to check ocean output (#2484) This PR: - adds a Rocoto dependency tag that executes a shell command. The return code of the shell expression serves as a dependency check - adds a script that executes `ncdump` on a netCDF file. If the file is a valid netCDF file, the return code is 0, else it is non-zero - combines the above 2 to use as a dependency check for MOM6 output. If the model is still in the process of writing out the ocean output, the rocoto will execute the shell script and gather the return code. This PR also: - changes permissions on some `ush/` scripts that did not have executable permissions. Resolves #2328 --- ush/bash_utils.sh | 0 ush/check_netcdf.sh | 15 +++++++++++++++ ush/file_utils.sh | 0 ush/jjob_header.sh | 0 ush/preamble.sh | 0 workflow/rocoto/gefs_tasks.py | 8 +++++++- workflow/rocoto/gfs_tasks.py | 9 ++++++++- workflow/rocoto/rocoto.py | 29 ++++++++++++++++++++++++++++- 8 files changed, 58 insertions(+), 3 deletions(-) mode change 100644 => 100755 ush/bash_utils.sh create mode 100755 ush/check_netcdf.sh mode change 100644 => 100755 ush/file_utils.sh mode change 100644 => 100755 ush/jjob_header.sh mode change 100644 => 100755 ush/preamble.sh diff --git a/ush/bash_utils.sh b/ush/bash_utils.sh old mode 100644 new mode 100755 diff --git a/ush/check_netcdf.sh b/ush/check_netcdf.sh new file mode 100755 index 0000000000..dafdf6c0c6 --- /dev/null +++ b/ush/check_netcdf.sh @@ -0,0 +1,15 @@ +#! /usr/bin/env bash + +# shellcheck disable=SC2155,SC2312 +HOMEgfs=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )/.." && pwd -P) +declare -rx HOMEgfs + +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" 1>/dev/null 2>&1 + +ncfile=${1?} + +ncdump -h "${ncfile}" +rc=$? +# If there is no error, rc=0, else rc!=0 + +exit "${rc}" diff --git a/ush/file_utils.sh b/ush/file_utils.sh old mode 100644 new mode 100755 diff --git a/ush/jjob_header.sh b/ush/jjob_header.sh old mode 100644 new mode 100755 diff --git a/ush/preamble.sh b/ush/preamble.sh old mode 100644 new mode 100755 diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 5d1eda817b..6ee079fdfa 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -191,7 +191,13 @@ def _atmosoceaniceprod(self, component: str): data = f'{history_path}/{history_file_tmpl}' dep_dict = {'type': 'data', 'data': data, 'age': 120} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) + if component in ['ocean']: + command = f"{self.HOMEgfs}/ush/check_netcdf.sh {history_path}/{history_file_tmpl}" + dep_dict = {'type': 'sh', 'command': command} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps, dep_condition='and') + else: + dependencies = rocoto.create_dependency(dep=deps) postenvars = self.envars.copy() postenvar_dict = {'ENSMEM': '#member#', diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index db0e6af26d..8789a418f9 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1069,7 +1069,14 @@ def _atmosoceaniceprod(self, component: str): data = f'{history_path}/{history_file_tmpl}' dep_dict = {'type': 'data', 'data': data, 'age': 120} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) + if component in ['ocean']: + command = f"{self.HOMEgfs}/ush/check_netcdf.sh {history_path}/{history_file_tmpl}" + dep_dict = {'type': 'sh', 'command': command} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps, dep_condition='and') + else: + dependencies = rocoto.create_dependency(dep=deps) + cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump resources = self.get_resource(component_dict['config']) diff --git a/workflow/rocoto/rocoto.py b/workflow/rocoto/rocoto.py index 679c0952ed..0abb56cafb 100644 --- a/workflow/rocoto/rocoto.py +++ b/workflow/rocoto/rocoto.py @@ -8,6 +8,7 @@ ABOUT: Helper module to create tasks, metatasks, and dependencies for Rocoto + Rocoto documentation is available at https://christopherwharrop.github.io/rocoto ''' __all__ = ['create_task', @@ -182,7 +183,8 @@ def add_dependency(dep_dict: Dict[str, Any]) -> str: 'data': _add_data_tag, 'cycleexist': _add_cycle_tag, 'streq': _add_streq_tag, - 'strneq': _add_streq_tag} + 'strneq': _add_streq_tag, + 'sh': _add_sh_tag} dep_condition = dep_dict.get('condition', None) dep_type = dep_dict.get('type', None) @@ -333,6 +335,31 @@ def _add_streq_tag(dep_dict: Dict[str, Any]) -> str: return string +def _add_sh_tag(dep_dict: Dict[str, Any]) -> str: + """ + create a simple shell execution tag + :param: dep_dict: shell command to execute + :type dep_dict: dict + :return: Rocoto simple shell execution dependency + :rtype: str + """ + + shell = dep_dict.get('shell', '/bin/sh') + command = dep_dict.get('command', 'echo "Hello World"') + + if '@' in command: + offset_string_b = f'' + offset_string_e = '' + else: + offset_string_b = '' + offset_string_e = '' + cmd = f'{offset_string_b}{command}{offset_string_e}' + + string = f'{cmd}' + + return string + + def _traverse(o, tree_types=(list, tuple)): """ Traverse through a list of lists or tuples and yield the value