Merge branch 'develop' into feature/ens-WCDA-ctest

AndrewEichmann-NOAA · Aug 21, 2024 · 16d805f · 16d805f
2 parents 20e48a1 + 1b18f2f
commit 16d805f
Show file tree

Hide file tree

Showing 125 changed files with 2,111 additions and 941 deletions.
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -33,6 +33,17 @@
 # Change characteristics
 - Is this a breaking change (a change in existing functionality)? YES/NO
 - Does this change require a documentation update? YES/NO
+- Does this change require an update to any of the following submodules? YES/NO (If YES, please add a link to any PRs that are pending.)
+  - [ ] EMC verif-global
+  - [ ] GDAS
+  - [ ] GFS-utils
+  - [ ] GSI
+  - [ ] GSI-monitor
+  - [ ] GSI-utils
+  - [ ] UFS-utils
+  - [ ] UFS-weather-model
+  - [ ] wxflow
+
 
 # How has this been tested?
 <!-- Please list any test you conducted, including the machine.

diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
diff --git a/ci/cases/gfsv17/ocnanal.yaml b/ci/cases/gfsv17/ocnanal.yaml
@@ -17,7 +17,7 @@ base:
   ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
 
 ocnanal:
-  SOCA_INPUT_FIX_DIR: {{ FIXgfs }}/gdas/soca/1440x1080x75/soca
+  SOCA_INPUT_FIX_DIR: {{ HOMEgfs }}/fix/gdas/soca/1440x1080x75/soca
   SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml
   SOCA_NINNER: 100
 

diff --git a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml
@@ -9,7 +9,7 @@ arguments:
   resdetocean: 5.0
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20240610
   idate: 2021032412
   edate: 2021032418
   nens: 0
@@ -19,5 +19,6 @@ arguments:
 
 skip_ci_on_hosts:
   - wcoss2
+  - gaea
   - orion
   - hercules
diff --git a/ci/cases/pr/C96C48_hybatmDA.yaml b/ci/cases/pr/C96C48_hybatmDA.yaml
@@ -10,7 +10,7 @@ arguments:
   resensatmos: 48
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610
   idate: 2021122018
   edate: 2021122106
   nens: 2

diff --git a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml
@@ -9,7 +9,7 @@ arguments:
   resensatmos: 48
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610
   idate: 2024022318
   edate: 2024022400
   nens: 2
@@ -19,6 +19,7 @@ arguments:
 
 skip_ci_on_hosts:
   - hera
+  - gaea
   - orion
   - hercules
 
diff --git a/ci/cases/pr/C96_atm3DVar.yaml b/ci/cases/pr/C96_atm3DVar.yaml
@@ -8,7 +8,7 @@ arguments:
   resdetatmos: 96
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610
   idate: 2021122018
   edate: 2021122106
   nens: 0

diff --git a/ci/cases/pr/C96_atm3DVar_extended.yaml b/ci/cases/pr/C96_atm3DVar_extended.yaml
@@ -8,7 +8,7 @@ arguments:
   resdetatmos: 96
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610
   idate: 2021122018
   edate: 2021122118
   nens: 0
@@ -18,5 +18,6 @@ arguments:
 
 skip_ci_on_hosts:
   - hera
+  - gaea
   - orion
   - hercules
diff --git a/ci/cases/pr/C96_atmaerosnowDA.yaml b/ci/cases/pr/C96_atmaerosnowDA.yaml
@@ -8,7 +8,7 @@ arguments:
   resdetatmos: 96
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610
   idate: 2021122012
   edate: 2021122100
   nens: 0
@@ -18,4 +18,5 @@ arguments:
 
 skip_ci_on_hosts:
   - orion
+  - gaea
   - hercules
diff --git a/ci/cases/yamls/gfs_extended_ci.yaml b/ci/cases/yamls/gfs_extended_ci.yaml
@@ -9,5 +9,6 @@ base:
   DO_AWIPS: "NO"
   DO_NPOESS: "YES"
   DO_GENESIS_FSU: "NO"
+  FCST_BREAKPOINTS: 192
   FHMAX_GFS: 384
   FHMAX_HF_GFS: 120
diff --git a/ci/platforms/config.gaea b/ci/platforms/config.gaea
@@ -0,0 +1,8 @@
+#!/usr/bin/bash
+
+export GFS_CI_ROOT=/gpfs/f5/epic/proj-shared/global/GFS_CI_ROOT
+export ICSDIR_ROOT=/gpfs/f5/epic/proj-shared/global/glopara/data/ICSDIR
+export STMP="/gpfs/f5/epic/scratch/${USER}"
+export SLURM_ACCOUNT=ufs-ard
+export max_concurrent_cases=5
+export max_concurrent_pr=4
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
@@ -21,7 +21,7 @@ REPO_URL=${REPO_URL:-"[email protected]:NOAA-EMC/global-workflow.git"}
 
 source "${HOMEgfs}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion | hercules | wcoss2)
+  hera | orion | hercules | wcoss2 | gaea)
    echo "Running Automated Testing on ${MACHINE_ID}"
    source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
    ;;
@@ -50,14 +50,14 @@ fi
 export GH
 
 rocotostat=$(command -v rocotostat)
-if [[ -z ${rocotostat+x} ]]; then
+if [[ -z ${rocotostat} ]]; then
   echo "rocotostat not found on system"
   exit 1
 else
   echo "rocotostat being used from ${rocotostat}"
 fi
 rocotocheck=$(command -v rocotocheck)
-if [[ -z ${rocotocheck+x} ]]; then
+if [[ -z ${rocotocheck} ]]; then
   echo "rocotocheck not found on system"
   exit 1
 else
@@ -70,7 +70,7 @@ pr_list=""
 if [[ -f "${pr_list_dbfile}" ]]; then
   pr_list=$("${HOMEgfs}/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --list Open Running) || true
 fi
-if [[ -z "${pr_list+x}" ]]; then
+if [[ -z "${pr_list}" ]]; then
   echo "no PRs open and ready to run cases on .. exiting"
   exit 0
 fi
@@ -124,7 +124,7 @@ for pr in ${pr_list}; do
 
   for pslot_dir in "${pr_dir}/RUNTESTS/EXPDIR/"*; do
     pslot=$(basename "${pslot_dir}") || true
-    if [[ -z "${pslot+x}" ]]; then
+    if [[ -z "${pslot}" ]]; then
       echo "No experiments found in ${pslot_dir} .. exiting"
       exit 0
     fi

diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
@@ -30,7 +30,7 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 
 source "${ROOT_DIR}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion | hercules | wcoss2)
+  hera | orion | hercules | wcoss2 | gaea)
     echo "Running Automated Testing on ${MACHINE_ID}"
     source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
     ;;
@@ -77,8 +77,9 @@ pr_list=$(${GH} pr list --repo "${REPO_URL}" --label "CI-${MACHINE_ID^}-Ready" -
 
 for pr in ${pr_list}; do
   pr_dir="${GFS_CI_ROOT}/PR/${pr}"
+  [[ ! -d ${pr_dir} ]] && mkdir -p "${pr_dir}"
   db_list=$("${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --add_pr "${pr}" --dbfile "${pr_list_dbfile}")
-  output_ci_single="${GFS_CI_ROOT}/PR/${pr}/output_single.log"
+  output_ci_single="${pr_dir}/output_single.log"
   #############################################################
   # Check if a Ready labeled PR has changed back from once set
   # and in that case completely kill the previose driver.sh cron
@@ -107,7 +108,9 @@ for pr in ${pr_list}; do
            echo -e "${pstree_out}" | grep -Pow "(?<=\()[0-9]+(?=\))" | xargs kill
         fi
       else
-        ssh "${driver_HOST}" 'pstree -A -p "${driver_PID}" | grep -Eow "[0-9]+" | xargs kill'
+        # Check if the driver is still running on the head node; if so, kill it and all child processes
+        #shellcheck disable=SC2029
+        ssh "${driver_HOST}" "pstree -A -p \"${driver_PID}\" | grep -Eow \"[0-9]+\" | xargs kill || echo \"Failed to kill process with PID: ${driver_PID}, it may not be valid.\""
       fi
       {
         echo "Driver PID: Requested termination of ${driver_PID} and children on ${driver_HOST}"
@@ -141,7 +144,7 @@ pr_list=""
 if [[ -f "${pr_list_dbfile}" ]]; then
   pr_list=$("${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --list Open Ready) || true
 fi
-if [[ -z "${pr_list+x}" ]]; then
+if [[ -z "${pr_list}" ]]; then
   echo "no PRs open and ready for checkout/build .. exiting"
   exit 0
 fi
@@ -155,7 +158,7 @@ fi
 for pr in ${pr_list}; do
   # Skip pr's that are currently Building for when overlapping driver scripts are being called from within cron
   pr_building=$("${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --display "${pr}" --dbfile "${pr_list_dbfile}" | grep Building) || true
-  if [[ -z "${pr_building+x}" ]]; then
+  if [[ -n "${pr_building}" ]]; then
       continue
   fi
   id=$("${GH}" pr view "${pr}" --repo "${REPO_URL}" --json id --jq '.id')

diff --git a/ci/scripts/driver_weekly.sh b/ci/scripts/driver_weekly.sh
@@ -38,7 +38,7 @@ export PS4='+ $(basename ${BASH_SOURCE[0]})[${LINENO}]'
 
 source "${ROOT_DIR}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion | hercules | wcoss2)
+  hera | orion | hercules | wcoss2 | gaea)
     echo "Running Automated Testing on ${MACHINE_ID}"
     source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
     ;;

diff --git a/ci/scripts/run_ci.sh b/ci/scripts/run_ci.sh
@@ -20,7 +20,7 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 
 source "${HOMEgfs}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion | hercules | wcoss2)
+  hera | orion | hercules | wcoss2 | gaea)
    echo "Running Automated Testing on ${MACHINE_ID}"
    source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
    ;;