Merge branch 'NOAA-EMC:develop' into selfhosted-Runner

.editorconfig

@@ -0,0 +1,29 @@
+# EditorConfig helps developers define and maintain consistent
+# coding styles between different editors and IDEs
+# editorconfig.org
+
+root = true
+
+
+[*]
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+indent_style = space
+indent_size = 2
+
+[*.py]
+indent_size = 4
+max_line_length = 88
+
+[*.txt]
+indent_style = tab
+indent_size = 4
+max_line_length = 79
+
+[*.{diff}]
+trim_trailing_whitespace = false
+
+[Makefile]
+indent_style = tab

.github/workflows/pynorms.yaml

@@ -21,4 +21,4 @@ jobs:
     - name: Run pycodestyle
       run: |
         cd $GITHUB_WORKSPACE/global-workflow
-        pycodestyle -v --config ./.pycodestyle --exclude='.git,.github' ./
+        pycodestyle -v --config ./.pycodestyle ./

.gitignore

@@ -142,6 +142,13 @@ ush/make_ntc_bull.pl
 ush/make_tif.sh
 ush/month_name.sh
 ush/imsfv3_scf2ioda.py
+
 # version files
 versions/build.ver
 versions/run.ver
+
+# wxflow checkout and symlinks
+sorc/wxflow
+ush/python/wxflow
+workflow/wxflow
+ci/scripts/wxflow

.pycodestyle

@@ -3,4 +3,4 @@ count = False
 ignore = E402,W504
 max-line-length = 160
 statistics = True
-exclude = Experimental
+exclude = Experimental,.git,.vscode,*.fd,*.cd

Externals.cfg

@@ -1,5 +1,12 @@
 # External sub-modules of global-workflow
 
+[wxflow]
+tag = 528f5ab
+local_path = sorc/wxflow
+repo_url = https://github.com/NOAA-EMC/wxflow.git
+protocol = git
+required = True
+
 [UFS]
 tag = bf60924
 local_path = sorc/ufs_model.fd

README.md

@@ -1,7 +1,6 @@
 [![Read The Docs Status](https://readthedocs.org/projects/global-workflow/badge/?badge=latest)](http://global-workflow.readthedocs.io/)
 [![shellnorms](https://github.com/NOAA-EMC/global-workflow/actions/workflows/linters.yaml/badge.svg)](https://github.com/NOAA-EMC/global-workflow/actions/workflows/linters.yaml)
 [![pynorms](https://github.com/NOAA-EMC/global-workflow/actions/workflows/pynorms.yaml/badge.svg)](https://github.com/NOAA-EMC/global-workflow/actions/workflows/pynorms.yaml)
-[![pytests](https://github.com/NOAA-EMC/global-workflow/actions/workflows/pytests.yaml/badge.svg)](https://github.com/NOAA-EMC/global-workflow/actions/workflows/pytests.yaml)
 
 ![Custom badge](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/emcbot/e35aa2904a54deae6bbb1fdc2d960c71/raw/hera.json)
 ![Custom badge](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/emcbot/e35aa2904a54deae6bbb1fdc2d960c71/raw/orion.json)

ci/scripts/clone-build_ci.sh

@@ -10,7 +10,7 @@ usage() {
   echo "Usage: $0 -p <PR#> -d <directory> -o <output> -h"
   echo
   echo "  -p  PR nunber to clone and build"
-  echo "  -d  Full path of <directory> of were to clone and build PR" 
+  echo "  -d  Full path of <directory> of were to clone and build PR"
   echo "  -o  Full path to output message file detailing results of CI tests"
   echo "  -h  display this message and quit"
   echo
@@ -35,8 +35,7 @@ while getopts "p:d:o:h" opt; do
     *)
       echo "Unrecognized option"
       usage
-      exit
-     ;; 
+     ;;
   esac
 done
 
@@ -56,8 +55,8 @@ if [[ "${pr_state}" != "OPEN" ]]; then
   title=$("${GH}" pr view "${PR}" --json title --jq '.title')
   echo "PR ${title} is no longer open, state is ${pr_state} ... quitting"
   exit 1
-fi  
- 
+fi
+
 # checkout pull request
 "${GH}" pr checkout "${PR}" --repo "${REPO_URL}"
 HOMEgfs="${PWD}"
@@ -81,7 +80,7 @@ echo "${commit}" > "../commit"
 # run checkout script
 cd sorc || exit 1
 set +e
-./checkout.sh -c -g -u &>> log.checkout
+./checkout.sh -c -g -u >> log.checkout 2>&1
 checkout_status=$?
 if [[ ${checkout_status} != 0 ]]; then
   {
@@ -101,7 +100,7 @@ fi
 source "${HOMEgfs}/ush/module-setup.sh"
 export BUILD_JOBS=8
 rm -rf log.build
-./build_all.sh  &>> log.build
+./build_all.sh  >> log.build 2>&1
 build_status=$?
 
 if [[ ${build_status} != 0 ]]; then

ci/scripts/create_experiment.py

@@ -20,18 +20,16 @@
 with an error code of 0 upon success.
 """
 
+from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
 from pathlib import Path
 
-from pygw.yaml_file import YAMLFile
-from pygw.logger import Logger
-from pygw.executable import Executable
-
-from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
+from wxflow import YAMLFile, Logger, logit, Executable
 
 
 logger = Logger(level='DEBUG', colored_log=True)
 
 
+@logit(logger)
 def input_args():
     """
     Method to collect user arguments for `create_experiment.py`

docs/source/clone.rst

@@ -11,35 +11,38 @@ Quick clone/build/link instructions (more detailed instructions below).
 .. note::
    Here we are making the assumption that you are using the workflow to run an experiment and so are working from the authoritative repository. If you are using a development branch then follow the instructions in :doc:`development.rst`. Once you do that you can follow the instructions here with the only difference being the repository/fork you are cloning from.
 
-For forecast-only (coupled or uncoupled):
+Clone the `global-workflow` and `cd` into the `sorc` directory:
 
 ::
 
-   git clone https://github.com/NOAA-EMC/global-workflow.git
+   git clone https://github.com/NOAA-EMC/global-workflow
    cd global-workflow/sorc
+
+For forecast-only (coupled or uncoupled) checkout the components:
+
+::
+
    ./checkout.sh
-   ./build_all.sh
-   ./link_workflow.sh
 
-For cycled (w/ data assimilation):
+For cycled (w/ data assimilation) use the `-g` option during checkout:
 
 ::
 
-   git clone https://github.com/NOAA-EMC/global-workflow.git
-   cd global-workflow/sorc
    ./checkout.sh -g
-   ./build_all.sh
-   ./link_workflow.sh
 
-For coupled cycling (include new UFSDA):
+For coupled cycling (include new UFSDA) use the `-gu` options during checkout:
 
 [Currently only available on Hera and Orion]
 
 ::
 
-   git clone https://github.com/NOAA-EMC/global-workflow.git
-   cd global-workflow/sorc
    ./checkout.sh -gu
+
+
+Build workflow components and link workflow artifacts such as executables, etc.
+
+::
+
    ./build_all.sh
    ./link_workflow.sh
 
@@ -52,24 +55,13 @@ Clone workflow and component repositories
 Workflow
 ********
 
-There are several ways to clone repositories from GitHub. Below we describe how to clone the global-workflow using either the ssh or https methods. **The ssh method is highly preferred and recommended.**
-
-ssh method (using a password protected SSH key):
-
-::
-
-   git clone [email protected]:NOAA-EMC/global-workflow.git
-
-.. note::
-   When using ssh methods you need to make sure that your GitHub account is configured for the computer from which you are accessing the repository (See `this link <https://docs.github.com/en/authentication/connecting-to-github-with-ssh/adding-a-new-ssh-key-to-your-github-account>`_)
-
-https method:
+There are several ways to clone repositories from GitHub. Below we describe how to clone the global-workflow using the `https` method.
 
 ::
 
-   git clone https://github.com/NOAA-EMC/global-workflow.git
+   git clone https://github.com/NOAA-EMC/global-workflow
 
-Check what you just cloned (by default you will have only the develop branch):
+Check what you just cloned (by default you will have only the `develop` branch):
 
 ::
 

jobs/rocoto/aeroanlfinal.sh

@@ -13,8 +13,8 @@ export jobid="${job}.$$"
 
 ###############################################################
 # setup python path for workflow utilities and tasks
-pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}"
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
 export PYTHONPATH
 ###############################################################
 # Execute the JJOB

jobs/rocoto/aeroanlinit.sh

@@ -13,8 +13,8 @@ export jobid="${job}.$$"
 
 ###############################################################
 # setup python path for workflow utilities and tasks
-pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}"
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
 export PYTHONPATH
 
 ###############################################################

jobs/rocoto/aeroanlrun.sh

@@ -13,8 +13,8 @@ export jobid="${job}.$$"
 
 ###############################################################
 # setup python path for workflow utilities and tasks
-pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}"
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
 export PYTHONPATH
 
 ###############################################################

jobs/rocoto/atmanlfinal.sh

@@ -13,8 +13,8 @@ export jobid="${job}.$$"
 
 ###############################################################
 # setup python path for workflow utilities and tasks
-pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}"
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
 export PYTHONPATH
 ###############################################################
 # Execute the JJOB

jobs/rocoto/atmanlinit.sh

@@ -13,8 +13,8 @@ export jobid="${job}.$$"
 
 ###############################################################
 # setup python path for workflow utilities and tasks
-pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}"
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
 export PYTHONPATH
 
 ###############################################################

jobs/rocoto/atmanlrun.sh

@@ -13,8 +13,8 @@ export jobid="${job}.$$"
 
 ###############################################################
 # setup python path for workflow utilities and tasks
-pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}"
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
 export PYTHONPATH
 
 ###############################################################

jobs/rocoto/atmensanlfinal.sh

@@ -13,8 +13,8 @@ export jobid="${job}.$$"
 
 ###############################################################
 # setup python path for workflow utilities and tasks
-pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}"
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
 export PYTHONPATH
 ###############################################################
 # Execute the JJOB

jobs/rocoto/atmensanlinit.sh

@@ -13,8 +13,8 @@ export jobid="${job}.$$"
 
 ###############################################################
 # setup python path for workflow utilities and tasks
-pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}"
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
 export PYTHONPATH
 
 ###############################################################

jobs/rocoto/atmensanlrun.sh

@@ -13,8 +13,8 @@ export jobid="${job}.$$"
 
 ###############################################################
 # setup python path for workflow utilities and tasks
-pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}"
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
 export PYTHONPATH
 
 ###############################################################

jobs/rocoto/fcst.sh

@@ -59,8 +59,8 @@ set_trace
 ###############################################################
 # exglobal_forecast.py requires the following in PYTHONPATH
 # This will be moved to a module load when ready
-pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src:${HOMEgfs}/ush/python/pygfs"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}"
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src:${HOMEgfs}/ush/python/pygfs"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
 export PYTHONPATH
 
 export job="fcst"

jobs/rocoto/landanl.sh

@@ -13,8 +13,8 @@ export jobid="${job}.$$"
 
 ###############################################################
 # setup python path for workflow utilities and tasks
-pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}"
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
 export PYTHONPATH
 
 ###############################################################

jobs/rocoto/preplandobs.sh

@@ -13,9 +13,9 @@ export jobid="${job}.$$"
 
 ###############################################################
 # setup python path for workflow utilities and tasks
-pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src"
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
 gdasappPATH="${HOMEgfs}/sorc/gdas.cd/iodaconv/src:${HOMEgfs}/sorc/gdas.cd/build/lib/python3.7/pyioda"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}:${gdasappPATH}"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}:${gdasappPATH}"
 export PYTHONPATH
 
 ###############################################################