Add rocoto sh tag, script to check netcdf file and apply this to ch…

…eck ocean output (NOAA-EMC#2484)

This PR:
- adds a Rocoto dependency tag that executes a shell command. The return
code of the shell expression serves as a dependency check
- adds a script that executes `ncdump` on a netCDF file. If the file is
a valid netCDF file, the return code is 0, else it is non-zero
- combines the above 2 to use as a dependency check for MOM6 output. If
the model is still in the process of writing out the ocean output, the
rocoto will execute the shell script and gather the return code.

This PR also:
- changes permissions on some `ush/` scripts that did not have
executable permissions.

Resolves NOAA-EMC#2328

ush/check_netcdf.sh

@@ -0,0 +1,15 @@
+#! /usr/bin/env bash
+
+# shellcheck disable=SC2155,SC2312
+HOMEgfs=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )/.." && pwd -P)
+declare -rx HOMEgfs
+
+source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" 1>/dev/null 2>&1
+
+ncfile=${1?}
+
+ncdump -h "${ncfile}"
+rc=$?
+# If there is no error, rc=0, else rc!=0
+
+exit "${rc}"

workflow/rocoto/gefs_tasks.py

@@ -191,7 +191,13 @@ def _atmosoceaniceprod(self, component: str):
         data = f'{history_path}/{history_file_tmpl}'
         dep_dict = {'type': 'data', 'data': data, 'age': 120}
         deps.append(rocoto.add_dependency(dep_dict))
-        dependencies = rocoto.create_dependency(dep=deps)
+        if component in ['ocean']:
+            command = f"{self.HOMEgfs}/ush/check_netcdf.sh {history_path}/{history_file_tmpl}"
+            dep_dict = {'type': 'sh', 'command': command}
+            deps.append(rocoto.add_dependency(dep_dict))
+            dependencies = rocoto.create_dependency(dep=deps, dep_condition='and')
+        else:
+            dependencies = rocoto.create_dependency(dep=deps)
 
         postenvars = self.envars.copy()
         postenvar_dict = {'ENSMEM': '#member#',

workflow/rocoto/gfs_tasks.py

@@ -1069,7 +1069,14 @@ def _atmosoceaniceprod(self, component: str):
         data = f'{history_path}/{history_file_tmpl}'
         dep_dict = {'type': 'data', 'data': data, 'age': 120}
         deps.append(rocoto.add_dependency(dep_dict))
-        dependencies = rocoto.create_dependency(dep=deps)
+        if component in ['ocean']:
+            command = f"{self.HOMEgfs}/ush/check_netcdf.sh {history_path}/{history_file_tmpl}"
+            dep_dict = {'type': 'sh', 'command': command}
+            deps.append(rocoto.add_dependency(dep_dict))
+            dependencies = rocoto.create_dependency(dep=deps, dep_condition='and')
+        else:
+            dependencies = rocoto.create_dependency(dep=deps)
+
         cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump
         resources = self.get_resource(component_dict['config'])
 

workflow/rocoto/rocoto.py

@@ -8,6 +8,7 @@
 
     ABOUT:
         Helper module to create tasks, metatasks, and dependencies for Rocoto
+        Rocoto documentation is available at https://christopherwharrop.github.io/rocoto
 '''
 
 __all__ = ['create_task',
@@ -182,7 +183,8 @@ def add_dependency(dep_dict: Dict[str, Any]) -> str:
                'data': _add_data_tag,
                'cycleexist': _add_cycle_tag,
                'streq': _add_streq_tag,
-               'strneq': _add_streq_tag}
+               'strneq': _add_streq_tag,
+               'sh': _add_sh_tag}
 
     dep_condition = dep_dict.get('condition', None)
     dep_type = dep_dict.get('type', None)
@@ -333,6 +335,31 @@ def _add_streq_tag(dep_dict: Dict[str, Any]) -> str:
     return string
 
 
+def _add_sh_tag(dep_dict: Dict[str, Any]) -> str:
+    """
+    create a simple shell execution tag
+    :param: dep_dict: shell command to execute
+    :type dep_dict: dict
+    :return: Rocoto simple shell execution dependency
+    :rtype: str
+    """
+
+    shell = dep_dict.get('shell', '/bin/sh')
+    command = dep_dict.get('command', 'echo "Hello World"')
+
+    if '@' in command:
+        offset_string_b = f'<cyclestr>'
+        offset_string_e = '</cyclestr>'
+    else:
+        offset_string_b = ''
+        offset_string_e = ''
+    cmd = f'{offset_string_b}{command}{offset_string_e}'
+
+    string = f'<sh shell="{shell}">{cmd}</sh>'
+
+    return string
+
+
 def _traverse(o, tree_types=(list, tuple)):
     """
     Traverse through a list of lists or tuples and yield the value