Robust CI Restarts

ci/platforms/config.orion

@@ -2,7 +2,7 @@
 
 export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT
 export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
-export STMP="/work/noaa/stmp/${USER}"
+export STMP="/work2/noaa/stmp/${USER}"
 export SLURM_ACCOUNT=nems
 export max_concurrent_cases=5
 export max_concurrent_pr=4

ci/scripts/check_ci.sh

@@ -33,7 +33,7 @@ case ${MACHINE_ID} in
 esac
 set +x
 source "${ROOT_DIR}/ush/module-setup.sh"
-source "${ROOT_DIR}/ci/scripts/utils/ci_utils.h"
+source "${ROOT_DIR}/ci/scripts/utils/ci_utils.sh"
 module use "${ROOT_DIR}/modulefiles"
 module load "module_gwsetup.${MACHINE_ID}"
 module list

ci/scripts/driver.sh

@@ -89,14 +89,14 @@ for pr in ${pr_list}; do
     if [[ "${driver_PID}" -ne 0 ]]; then
       echo "Driver PID: ${driver_PID} no longer running this build having it killed"
       if [[ "${driver_HOST}" == "${host_name}"  ]]; then
-        pstree -A -p "${driver_PID}" | grep -Eow "[0-9]+" | xargs kill || true
-        sleep 30
+        #shellcheck disable=SC2312,SC2312
+        pstree -A -p "${driver_PID}" | grep -Pow "(?<=\()[0-9]+(?=\))" | xargs kill
       else
+        # shellcheck disable=SC2312
         ssh "${driver_HOST}" 'pstree -A -p "${driver_PID}" | grep -Eow "[0-9]+" | xargs kill'
-        sleep 30
       fi
       {
-        echo "Driver PID: ${driver_PID} on ${driver_HOST} is no longer running this test"
+        echo "Driver PID: Requested termination of ${driver_PID} and children on ${driver_HOST}"
         echo "Driver PID: has restarted as $$ on ${host_name}"
       } >> "${output_ci_single}"
     fi
@@ -105,8 +105,8 @@ for pr in ${pr_list}; do
     if [[ -z "${experiments}" ]]; then
        echo "No current experiments to cancel in PR: ${pr} on ${MACHINE_ID^}" >> "${output_ci_single}"
     else
-      for cases in ${experiments}; do
-        case_name=$(basename "${cases}")
+      for case in ${experiments}; do
+        case_name=$(basename "${case}")
         cancel_slurm_jobs "${case_name}"
         {
           echo "Canceled all jobs for experiment ${case_name} in PR:${pr} on ${MACHINE_ID^}"
@@ -115,8 +115,8 @@ for pr in ${pr_list}; do
     fi
     sed -i "1 i\`\`\`" "${output_ci_single}"
     "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}"
-    db_list=$("${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}")
-    db_list=$("${ROOT_DIR}/ci/scripts/pr_list_database.py" --add_pr "${pr}" --dbfile "${pr_list_dbfile}")
+    "${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
+    "${ROOT_DIR}/ci/scripts/pr_list_database.py" --add_pr "${pr}" --dbfile "${pr_list_dbfile}"
   fi
 done
 

ci/scripts/utils/ci_utils.sh

@@ -8,7 +8,7 @@ function cancel_slurm_jobs() {
 
   for job_id in ${job_ids}; do
     job_name=$(sacct -j "${job_id}" --format=JobName%100 | head -3 | tail -1 | sed -r 's/\s+//g') || true
-    if [[ "${job_name}" == *"${substring}"* ]]; then
+    if [[ "${job_name}" =~ ${substring} ]]; then
       echo "Canceling Slurm Job ${job_name} with: scancel ${job_id}"
       scancel "${job_id}"
     fi