Merge remote-tracking branch 'upstream/develop' into feature/build_je…

…di_exe * upstream/develop: Add nest capability (NOAA-EMC#2545) Update gfs_utils for Gaea (NOAA-EMC#2556) Updated GEMPAK version and APRUN launcher. (NOAA-EMC#2555) Utilize scale-dependent localization for atmospheric analysis (NOAA-EMC#2542) Remove implicit symlink names (NOAA-EMC#2527) Fixes sea ice archiving (NOAA-EMC#2541) Link ensemble analysis increment files to COMROOT for warm_start (NOAA-EMC#2553) Launch Multiple Platforms to Jenkins with polling (NOAA-EMC#2548)
danholdaway · May 3, 2024 · 66eab64 · 66eab64
2 parents 264f2fa + a005244
commit 66eab64
Show file tree

Hide file tree

Showing 76 changed files with 1,271 additions and 265 deletions.
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
@@ -21,31 +21,60 @@ pipeline {
              // Each Jenknis Node is connected to said machine via an JAVA agent via an ssh tunnel
              // no op 2
 
-        stage('Get Machine') {
+        stage('1. Get Machine') {
             agent { label 'built-in' }
             steps {
                 script {
-                    machine = 'none'
-                    for (label in pullRequest.labels) {
-                        echo "Label: ${label}"
-                        if ((label.matches('CI-Hera-Ready'))) {
-                            machine = 'hera'
-                        } else if ((label.matches('CI-Orion-Ready'))) {
-                            machine = 'orion'
-                        } else if ((label.matches('CI-Hercules-Ready'))) {
-                            machine = 'hercules'
+
+                    def causes = currentBuild.rawBuild.getCauses()
+                    def isSpawnedFromAnotherJob = causes.any { cause ->
+                        cause instanceof hudson.model.Cause.UpstreamCause
+                    }
+
+                    def run_nodes = []
+                    if (isSpawnedFromAnotherJob) {
+                        echo "machine being set to value passed to this spawned job"
+                        echo "passed machine: ${params.machine}"
+                        machine = params.machine
+                    } else {
+                        echo "This is parent job so getting list of nodes matching labels:"
+                        for (label in pullRequest.labels) {
+                            if (label.matches("CI-(.*?)-Ready")) {
+                                def Machine_name = label.split('-')[1].toString()
+                                jenkins.model.Jenkins.get().computers.each { c ->
+                                    if (c.node.selfLabel.name == "${Machine_name}-EMC") {
+                                        run_nodes.add(c.node.selfLabel.name)
+                                    }
+                                }
+                            }
                         }
-                    } // createing a second machine varible with first letter capital
-                      // because the first letter of the machine name is captitalized in the GitHub labels
-                    Machine = machine[0].toUpperCase() + machine.substring(1)
+                        // Spawning all the jobs on the nodes matching the labels
+                        if (run_nodes.size() > 1) {
+                            run_nodes.init().each { node ->
+                                def machine_name = node.split('-')[0].toLowerCase()
+                                echo "Spawning job on node: ${node} with machine name: ${machine_name}"
+                                build job: "/global-workflow/EMC-Global-Pipeline/PR-${env.CHANGE_ID}", parameters: [
+                                    string(name: 'machine', value: machine_name),
+                                    string(name: 'Node', value: node) ],
+                                    wait: false
+                            }
+                            machine = run_nodes.last().split('-')[0].toLowerCase()
+                            echo "Running parent job: ${machine}"
+                        } else {
+                            machine = run_nodes[0].split('-')[0].toLowerCase()
+                            echo "Running only the parent job: ${machine}"
+                        }
+                    }
                 }
             }
         }
 
-        stage('Get Common Workspace') {
+        stage('2. Get Common Workspace') {
             agent { label "${machine}-emc" }
             steps {
                 script {
+                    Machine = machine[0].toUpperCase() + machine.substring(1)
+                    echo "Getting Common Workspace for ${Machine}"
                     ws("${custom_workspace[machine]}/${env.CHANGE_ID}") {
                         properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                         HOME = "${WORKSPACE}"
@@ -57,7 +86,7 @@ pipeline {
             }
         }
 
-        stage('Build System') {
+        stage('3. Build System') {
             matrix {
                 agent { label "${machine}-emc" }
                 //options {
@@ -141,7 +170,7 @@ pipeline {
             }
         }
 
-        stage('Run Tests') {
+        stage('4. Run Tests') {
             failFast false 
             matrix {
                 agent { label "${machine}-emc" }
@@ -216,7 +245,7 @@ pipeline {
                                         STATUS = 'Failed'
                                         try {
                                             sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
-                                            sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} **FAILED** on ${Machine}\nin\\`${HOME}/RUNTESTS/${pslot}\\`" """)
+                                            sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} **FAILED** on ${Machine} in\n\\`${HOME}/RUNTESTS/${pslot}\\`" """)
                                         } catch (Exception e) {
                                             echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}"
                                         }
@@ -229,7 +258,7 @@ pipeline {
                 }
             }
         }
-        stage( 'FINALIZE' ) {
+        stage( '5. FINALIZE' ) {
             when { 
                 expression {
                    STATUS == 'Passed'

diff --git a/env/HERA.env b/env/HERA.env
@@ -288,15 +288,13 @@ elif [[ "${step}" = "gempak" ]]; then
     export CFP_MP="YES"
 
     if [[ ${CDUMP} == "gfs" ]]; then
-        npe_gempak=${npe_gempak_gfs}
         npe_node_gempak=${npe_node_gempak_gfs}
     fi
 
     nth_max=$((npe_node_max / npe_node_gempak))
 
     export NTHREADS_GEMPAK=${nth_gempak:-1}
     [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
-    export APRUN="${launcher} -n ${npe_gempak} ${mpmd_opt}"
 
 
 elif [[ "${step}" = "fit2obs" ]]; then

diff --git a/gempak/ush/gdas_ecmwf_meta_ver.sh b/gempak/ush/gdas_ecmwf_meta_ver.sh
@@ -24,7 +24,7 @@ fi
 
 export COMIN="gdas.${PDY}${cyc}"
 if [[ ! -L ${COMIN} ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 vergrid="F-GDAS | ${PDY:2}/0600"
 fcsthr="0600f006"
@@ -49,7 +49,7 @@ for area in ${areas}; do
         sdatenum=$(date --utc +%y%m%d -d "${PDY} ${cyc2} - ${fhr} hours")
 
         if [[ ! -L "ecmwf.20${sdatenum}" ]]; then
-            ln -sf "${COMINecmwf}/ecmwf.20${sdatenum}/gempak" "ecmwf.20${sdatenum}"
+            ${NLN} "${COMINecmwf}/ecmwf.20${sdatenum}/gempak" "ecmwf.20${sdatenum}"
         fi
         gdfile="ecmwf.20${sdatenum}/ecmwf_glob_20${sdatenum}12"
 

diff --git a/gempak/ush/gdas_meta_loop.sh b/gempak/ush/gdas_meta_loop.sh
@@ -13,7 +13,7 @@ device="nc | gdasloop.meta"
 #
 export COMIN="${RUN}.${PDY}${cyc}"
 if [[ ! -L "${COMIN}" ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 
 if [[ "${envir}" == "para" ]] ; then
@@ -41,7 +41,7 @@ for (( fhr=24; fhr<=144; fhr+=24 )); do
         YMD=${day} HH=${cyc} GRID=1p00 declare_from_tmpl "COM_ATMOS_GEMPAK_1p00_past:COM_ATMOS_GEMPAK_TMPL"
         export COMIN="${RUN}.${day}${cycle}"
         if [[ ! -L "${COMIN}" ]]; then
-            ln -sf "${COM_ATMOS_GEMPAK_1p00_past}" "${COMIN}"
+            ${NLN} "${COM_ATMOS_GEMPAK_1p00_past}" "${COMIN}"
         fi
         gdfile="${COMIN}/gdas_1p00_${day}${cycle}f000"
 

diff --git a/gempak/ush/gdas_meta_na.sh b/gempak/ush/gdas_meta_na.sh
@@ -13,7 +13,7 @@ device="nc | gdas.meta"
 #
 export COMIN="${RUN}.${PDY}${cyc}"
 if [[ ! -L "${COMIN}" ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 
 if [[ "${envir}" == "para" ]] ; then

diff --git a/gempak/ush/gdas_ukmet_meta_ver.sh b/gempak/ush/gdas_ukmet_meta_ver.sh
@@ -23,7 +23,7 @@ cp "${HOMEgfs}/gempak/fix/datatype.tbl" datatype.tbl
 # SET CURRENT CYCLE AS THE VERIFICATION GRIDDED FILE.
 export COMIN="gdas.${PDY}${cyc}"
 if [[ ! -L ${COMIN} ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 vergrid="F-GDAS | ${PDY:2}/0600"
 fcsthr="0600f006"
@@ -54,7 +54,7 @@ for area in ${areas}; do
         cyclenum=${stime:6}
 
         if [[ ! -L "ukmet.20${sdatenum}" ]]; then
-            ln -sf "${COMINukmet}/ukmet.20${sdatenum}/gempak" "ukmet.20${sdatenum}"
+            ${NLN} "${COMINukmet}/ukmet.20${sdatenum}/gempak" "ukmet.20${sdatenum}"
         fi
         gdfile="ukmet.20${sdatenum}/ukmet_20${sdatenum}${cyclenum}${dgdattim}"
 

diff --git a/gempak/ush/gfs_meta_ak.sh b/gempak/ush/gfs_meta_ak.sh
@@ -20,7 +20,7 @@ device="nc | gfs.meta.ak"
 #
 export COMIN="${RUN}.${PDY}${cyc}"
 if [[ ! -L ${COMIN} ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 
 fend=F216

diff --git a/gempak/ush/gfs_meta_bwx.sh b/gempak/ush/gfs_meta_bwx.sh
@@ -21,7 +21,7 @@ device="nc | ${metaname}"
 #
 export COMIN="${RUN}.${PDY}${cyc}"
 if [[ ! -L ${COMIN} ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 
 fend=F180

diff --git a/gempak/ush/gfs_meta_comp.sh b/gempak/ush/gfs_meta_comp.sh
@@ -29,14 +29,14 @@ for cycle in $(seq -f "%02g" -s ' ' 0  "${STEP_GFS}" "${cyc}"); do
     for file_in in "${gempak_dir}/gfs_1p00_${PDY}${cycle}f"*; do
         file_out="${COMIN}/$(basename "${file_in}")"
         if [[ ! -L "${file_out}" ]]; then
-            ln -sf "${file_in}" "${file_out}"
+            ${NLN} "${file_in}" "${file_out}"
         fi
     done
 done
 
 export HPCNAM="nam.${PDY}"
 if [[ ! -L ${HPCNAM} ]]; then
-    ln -sf "${COMINnam}/nam.${PDY}/gempak" "${HPCNAM}"
+    ${NLN} "${COMINnam}/nam.${PDY}/gempak" "${HPCNAM}"
 fi
 
 #
@@ -100,7 +100,7 @@ for gareas in US NP; do
         HPCGFS="${RUN}.${init_time}"
         if [[ ! -L ${HPCGFS} ]]; then
             YMD="${init_PDY}" HH="${init_cyc}" GRID="1p00" declare_from_tmpl source_dir:COM_ATMOS_GEMPAK_TMPL
-            ln -sf "${source_dir}" "${HPCGFS}"
+            ${NLN} "${source_dir}" "${HPCGFS}"
         fi
 
         if [[ ${init_PDY} == "${PDY}" ]]; then
@@ -230,7 +230,7 @@ EOF
         ukmet_cyc=${ukmet_date:8:2}
         export HPCUKMET=ukmet.${ukmet_PDY}
         if [[ ! -L "${HPCUKMET}" ]]; then
-            ln -sf "${COMINukmet}/ukmet.${ukmet_PDY}/gempak" "${HPCUKMET}"
+            ${NLN} "${COMINukmet}/ukmet.${ukmet_PDY}/gempak" "${HPCUKMET}"
         fi
         grid2="F-UKMETHPC | ${ukmet_PDY:2}/${ukmet_date}"
 

diff --git a/gempak/ush/gfs_meta_crb.sh b/gempak/ush/gfs_meta_crb.sh
@@ -23,7 +23,7 @@ device="nc | ${metaname}"
 #
 export COMIN="${RUN}.${PDY}${cyc}"
 if [[ ! -L ${COMIN} ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 
 # DEFINE YESTERDAY
@@ -254,13 +254,13 @@ if [[ ${cyc} == 00 ]] ; then
     HPCECMWF_m1=ecmwf.${PDY}
     export HPCUKMET=ukmet.${PDYm1}
     if [[ ! -L "${HPCECMWF}" ]]; then
-        ln -sf "${COMINecmwf}ecmwf.${PDY}/gempak" "${HPCECMWF}"
+        ${NLN} "${COMINecmwf}ecmwf.${PDY}/gempak" "${HPCECMWF}"
     fi
     if [[ ! -L "${HPCECMWF_m1}" ]]; then
-        ln -sf "${COMINecmwf}ecmwf.${PDYm1}/gempak" "${HPCECMWF_m1}"
+        ${NLN} "${COMINecmwf}ecmwf.${PDYm1}/gempak" "${HPCECMWF_m1}"
     fi
     if [[ ! -L "${HPCUKMET}" ]]; then
-        ln -sf "${COMINukmet}/ukmet.${PDYm1}/gempak" "${HPCUKMET}"
+        ${NLN} "${COMINukmet}/ukmet.${PDYm1}/gempak" "${HPCUKMET}"
     fi
 
     grid1="F-${MDL} | ${PDY:2}/${cyc}00"

diff --git a/gempak/ush/gfs_meta_hi.sh b/gempak/ush/gfs_meta_hi.sh
@@ -17,7 +17,7 @@ device="nc | mrfhi.meta"
 #
 export COMIN="${RUN}.${PDY}${cyc}"
 if [[ ! -L ${COMIN} ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 
 if [[ "${envir}" = "prod" ]] ; then

diff --git a/gempak/ush/gfs_meta_hur.sh b/gempak/ush/gfs_meta_hur.sh
@@ -23,7 +23,7 @@ device="nc | ${metaname}"
 #
 export COMIN="${RUN}.${PDY}${cyc}"
 if [[ ! -L ${COMIN} ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 
 #
@@ -307,13 +307,13 @@ if [[ ${cyc} -eq 00 ]] ; then
     HPCECMWF_m1=ecmwf.${PDY}
     export HPCUKMET=ukmet.${PDYm1}
     if [[ ! -L "${HPCECMWF}" ]]; then
-        ln -sf "${COMINecmwf}ecmwf.${PDY}/gempak" "${HPCECMWF}"
+        ${NLN} "${COMINecmwf}ecmwf.${PDY}/gempak" "${HPCECMWF}"
     fi
     if [[ ! -L "${HPCECMWF_m1}" ]]; then
         Ln -sf "${COMINecmwf}ecmwf.${PDYm1}/gempak" "${HPCECMWF_m1}"
     fi
     if [[ ! -L "${HPCUKMET}" ]]; then
-        ln -sf "${COMINukmet}/ukmet.${PDYm1}/gempak" "${HPCUKMET}"
+        ${NLN} "${COMINukmet}/ukmet.${PDYm1}/gempak" "${HPCUKMET}"
     fi
     grid1="F-${MDL} | ${PDY:2}/${cyc}00"
     grid2="${HPCECMWF_m1}/ecmwf_glob_${PDYm1}12"

diff --git a/gempak/ush/gfs_meta_mar_atl.sh b/gempak/ush/gfs_meta_mar_atl.sh
@@ -17,7 +17,7 @@ cp "${HOMEgfs}/gempak/fix/datatype.tbl" datatype.tbl
 #
 export COMIN="${RUN}.${PDY}${cyc}"
 if [[ ! -L ${COMIN} ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 
 mdl=gfs

diff --git a/gempak/ush/gfs_meta_mar_comp.sh b/gempak/ush/gfs_meta_mar_comp.sh
@@ -20,7 +20,7 @@ for cycle in $(seq -f "%02g" -s ' ' 0 "${STEP_GFS}" "${cyc}"); do
     for file_in in "${gempak_dir}/gfs_1p00_${PDY}${cycle}f"*; do
         file_out="${COMIN}/$(basename "${file_in}")"
         if [[ ! -L "${file_out}" ]]; then
-            ln -sf "${file_in}" "${file_out}"
+            ${NLN} "${file_in}" "${file_out}"
         fi
     done
 done
@@ -31,7 +31,7 @@ done
 #
 export HPCNAM="nam.${PDY}"
 if [[ ! -L ${HPCNAM} ]]; then
-    ln -sf "${COMINnam}/nam.${PDY}/gempak" "${HPCNAM}"
+    ${NLN} "${COMINnam}/nam.${PDY}/gempak" "${HPCNAM}"
 fi
 
 mdl=gfs
@@ -77,7 +77,7 @@ for garea in NAtl NPac; do
         HPCGFS="${RUN}.${init_time}"
         if [[ ! -L ${HPCGFS} ]]; then
             YMD="${init_PDY}" HH="${init_cyc}" GRID="1p00" declare_from_tmpl source_dir:COM_ATMOS_GEMPAK_TMPL
-            ln -sf "${source_dir}" "${HPCGFS}"
+            ${NLN} "${source_dir}" "${HPCGFS}"
         fi
 
         case ${cyc} in
@@ -222,7 +222,7 @@ EOF
 
         export HPCUKMET="ukmet.${ukmet_PDY}"
         if [[ ! -L "${HPCUKMET}" ]]; then
-            ln -sf "${COMINukmet}/ukmet.${ukmet_PDY}/gempak" "${HPCUKMET}"
+            ${NLN} "${COMINukmet}/ukmet.${ukmet_PDY}/gempak" "${HPCUKMET}"
         fi
         grid2="F-UKMETHPC | ${ukmet_PDY:2}/${ukmet_date}"
 
@@ -310,7 +310,7 @@ EOF
 
         HPCECMWF=ecmwf.${PDY}
         if [[ ! -L "${HPCECMWF}" ]]; then
-            ln -sf "${COMINecmwf}/ecmwf.${ecmwf_PDY}/gempak" "${HPCECMWF}"
+            ${NLN} "${COMINecmwf}/ecmwf.${ecmwf_PDY}/gempak" "${HPCECMWF}"
         fi
         grid2="${HPCECMWF}/ecmwf_glob_${ecmwf_date}"
 

diff --git a/gempak/ush/gfs_meta_mar_pac.sh b/gempak/ush/gfs_meta_mar_pac.sh
@@ -17,7 +17,7 @@ cp "${HOMEgfs}/gempak/fix/datatype.tbl" datatype.tbl
 #
 export COMIN="${RUN}.${PDY}${cyc}"
 if [[ ! -L ${COMIN} ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 
 mdl=gfs

diff --git a/gempak/ush/gfs_meta_mar_ql.sh b/gempak/ush/gfs_meta_mar_ql.sh
@@ -17,7 +17,7 @@ cp "${HOMEgfs}/gempak/fix/datatype.tbl" datatype.tbl
 #
 export COMIN="${RUN}.${PDY}${cyc}"
 if [[ ! -L ${COMIN} ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 
 mdl=gfs

diff --git a/gempak/ush/gfs_meta_mar_skewt.sh b/gempak/ush/gfs_meta_mar_skewt.sh
@@ -17,7 +17,7 @@ cp "${HOMEgfs}/gempak/fix/datatype.tbl" datatype.tbl
 #
 export COMIN="${RUN}.${PDY}${cyc}"
 if [[ ! -L ${COMIN} ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 
 mdl=gfs

diff --git a/gempak/ush/gfs_meta_mar_ver.sh b/gempak/ush/gfs_meta_mar_ver.sh
@@ -17,7 +17,7 @@ cp "${HOMEgfs}/gempak/fix/datatype.tbl" datatype.tbl
 #
 export COMIN="${RUN}.${PDY}${cyc}"
 if [[ ! -L ${COMIN} ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 
 mdl=gfs