PLATFORM 989/PLATFORM 1000 - Develop a label (manual) triggered pipeline based on CM team's approval of UFS-WM. New and improved, now with parallelization. (Combined PR 2057 and PR 2039)

modulefiles/ufs_gaea.intel.lua

@@ -3,7 +3,7 @@ help([[
   on the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0.
 ]])
 
-whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea C5 ]===])
+whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea ]===])
 
 load("PrgEnv-intel/8.3.3")
 load("intel-classic/2023.1.0")

modulefiles/ufs_jet.intel.lua

@@ -2,7 +2,7 @@ help([[
 loads UFS Model prerequisites for Jet/Intel
 ]])
 
-prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env-rocky8/install/modulefiles/Core")
 
 stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
 load(pathJoin("stack-intel", stack_intel_ver))

tests/ci/Jenkinsfile.combined

@@ -0,0 +1,301 @@
+matchedNode = []
+generateBaselineNode = []
+for (label in pullRequest.labels) {
+   listOfLabelNodeNames = jenkins.model.Jenkins.instance.nodes.collect {
+   node -> node.getLabelString().contains(label) ? node.name : null
+
+   if ((label.matches(node.getLabelString()+"-(.*)"))) {
+     matchedNode += node.getLabelString()
+   }
+
+   if ((label.matches(node.getLabelString()+"(.*)-BL"))) {
+     generateBaselineNode += node.getLabelString() 
+   }
+
+ }
+}
+
+modifiedLabels = matchedNode.collect{"'" + it + "'"} 
+baselineLabels = generateBaselineNode.collect{"'" + it + "'"} 
+def generateStage(nodeLabel) {
+    return {
+        stage("Running on ${nodeLabel}") {
+            node(nodeLabel) {
+               cleanWs()
+               checkout scm
+               script {
+                 try {
+                    echo "Running on ${nodeLabel}"
+                    if (baselineLabels.contains(nodeLabel)) {
+                       sh '''
+                       git submodule update --init --recursive
+                       cd tests
+		       pwd
+		       export BL_DATE=$(cat bl_date.conf | cut -d '=' -f2)
+                       export machine=${NODE_NAME}
+                       export PATH=$PATH:~/bin
+                       echo $CHANGE_ID
+                       export SSH_ORIGIN=$(curl --silent https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/$CHANGE_ID | jq -r '.head.repo.ssh_url')
+                       export FORK_BRANCH=$(curl --silent https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/$CHANGE_ID | jq -r '.head.ref')
+
+                          if [[ $machine =~ "Jet" ]] 
+                          then
+                            echo "Creating baselines on $machine"
+                            export dprefix=/lfs1/NAGAPE/$ACCNR/$USER
+                            ./rt.sh -a ${ACCNR} -c -r -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log
+                          elif [[ $machine =~ "Hercules" ]]
+                          then
+                            echo "Creating baselines on $machine"
+                            export dprefix=/work2/noaa/$ACCNR/$USER
+                            sed "s|/noaa/stmp/|/noaa/$ACCNR/stmp/|g" -i rt.sh
+                            export ACCNR=epic
+                            ./rt.sh -a ${ACCNR} -c -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log
+                            export DISKNM=/work/noaa/epic/hercules/UFS-WM_RT
+			    cd ${DISKNM}/NEMSfv3gfs/
+			    mkdir develop-${BL_DATE}
+			    cd /work2/noaa/epic/stmp/role-epic/stmp/role-epic/FV3_RT
+			    rsync -a REGRESSION_TEST/ ${DISKNM}/NEMSfv3gfs/develop-${BL_DATE}
+                            cd ${DISKNM}/NEMSfv3gfs/
+                            ./adjust_permissions.sh hercules develop-${BL_DATE}
+                            chgrp noaa-hpc develop-${BL_DATE}
+		            cd $WORKSPACE/tests
+                            ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log
+                          elif [[ $machine =~ "Orion" ]]
+                          then
+                            echo "Creating baselines on $machine"
+                            export dprefix=/work2/noaa/$ACCNR/$USER
+                            sed -i 's|/work/noaa/stmp/${USER}|/work/noaa/epic/stmp/role-epic/|g' rt.sh
+		            export ACCNR=epic
+                            ./rt.sh -a ${ACCNR} -c -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log
+                            export DISKNM=/work/noaa/epic/UFS-WM_RT
+                            cd ${DISKNM}/NEMSfv3gfs/
+                            mkdir develop-${BL_DATE}
+                            cd  /work/noaa/epic/stmp/role-epic/stmp/role-epic/FV3_RT/
+                            rsync -a REGRESSION_TEST/ ${DISKNM}/NEMSfv3gfs/develop-${BL_DATE}
+                            cd ${DISKNM}/NEMSfv3gfs/
+                            ./adjust_permissions.sh orion develop-${BL_DATE}
+                            chgrp noaa-hpc develop-${BL_DATE}
+			    cd $WORKSPACE/tests
+                            ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log
+                          elif [[ $machine =~ "Gaea" ]]
+                          then 
+                            echo "Creating baselines on $machine"
+                            ./rt.sh -a ${ACCNR} -c -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log
+                            unset LD_LIBRARY_PATH
+                            export DISKNM=/lustre/f2/pdata/ncep/role.epic/C5/RT
+                            cd ${DISKNM}/NEMSfv3gfs/
+                            mkdir develop-${BL_DATE}
+                            cd  /lustre/f2/scratch/role.epic/FV3_RT
+                            rsync -a REGRESSION_TEST/ ${DISKNM}/NEMSfv3gfs/develop-${BL_DATE}
+                            cd ${DISKNM}/NEMSfv3gfs/
+                            chgrp ncep develop-${BL_DATE}
+		            cd $WORKSPACE/tests
+                            ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log
+                          elif [[ $machine =~ "Hera" ]]
+                          then
+                            echo "Creating baselines on $machine"
+                            export ACCNR=nems
+                            ./rt.sh -a ${ACCNR} -c -r -l rt.conf
+                            export DISKNM=/scratch2/NAGAPE/epic/UFS-WM_RT
+			    cd ${DISKNM}/NEMSfv3gfs/
+			    mkdir develop-${BL_DATE}
+			    cd  /scratch1/NCEPDEV/stmp4/role.epic/FV3_RT
+			    rsync -a REGRESSION_TEST/ ${DISKNM}/NEMSfv3gfs/develop-${BL_DATE}
+			    cd $WORKSPACE/tests
+                            ./rt.sh -a ${ACCNR} -r -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log
+                           elif [[ $machine =~ "Derecho" ]]
+                           then
+                             echo "Creating baselines on $machine"
+                             export ACCNR=nral0032
+                             ./rt.sh -a ${ACCNR} -c -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log
+                             export DISKNM=/glade/derecho/scratch/epicufsrt/ufs-weather-model/RT/
+			     cd ${DISKNM}/NEMSfv3gfs/
+			     mkdir develop-${BL_DATE}
+			     cd /glade/derecho/scratch/epicufsrt/FV3_RT
+			     rsync -a REGRESSION_TEST/ ${DISKNM}/NEMSfv3gfs/develop-${BL_DATE}
+			     cd $WORKSPACE/tests
+                             ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log
+                          else
+                            echo "Creating baselines on $machine"
+                            ./rt.sh -a ${ACCNR} -c -r -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log
+                          fi
+                      git config user.email "[email protected]"
+                      git config user.name "epic-cicd-jenkins"
+                      echo "Testing concluded...removing labels for $machine from $GIT_URL"
+
+                      export machine_name_logs=$(echo $machine | awk '{ print tolower($1) }')
+                      git remote -v | grep -w sshorigin > /dev/null 2>&1 && git remote remove sshorigin > /dev/null 2>&1
+                      git remote add sshorigin $SSH_ORIGIN > /dev/null 2>&1
+                      git add logs/RegressionTests_$machine_name_logs.log
+                      git commit -m "[AutoRT] $machine Job Completed.\n\n\n on-behalf-of @ufs-community <[email protected]>"
+                      git pull sshorigin $FORK_BRANCH
+                      git push sshorigin HEAD:$FORK_BRANCH
+
+                      tar --create --gzip --verbose --dereference --file "${machine_name_logs}.tgz" ${WORKSPACE}/tests/logs/*.log
+
+                      GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4)
+                      GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1)
+
+                      curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}"  https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels/$machine-BL
+                      '''
+                      s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: true, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "**/*tgz*", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+
+                    }
+                    else {
+                       sh '''
+                       git submodule update --init --recursive
+                       pwd
+                       cd tests
+                       export machine=${NODE_NAME}
+                       export PATH=$PATH:~/bin
+                       echo $CHANGE_ID
+                       export SSH_ORIGIN=$(curl --silent https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/$CHANGE_ID | jq -r '.head.repo.ssh_url')
+                       export FORK_BRANCH=$(curl --silent https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/$CHANGE_ID | jq -r '.head.ref')
+
+                          if [[ $machine =~ "Jet" ]] 
+                          then
+                           echo "Running regression tests on $machine"
+                            export dprefix=/lfs1/NAGAPE/$ACCNR/$USER
+                            ./rt.sh -a ${ACCNR} -r -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log
+                          elif [[ $machine =~ "Hercules" ]]
+                          then
+                           echo "Running regression tests on $machine"
+                            export dprefix=/work2/noaa/$ACCNR/$USER
+                            sed "s|/noaa/stmp/|/noaa/$ACCNR/stmp/|g" -i rt.sh
+                            export ACCNR=epic
+                            ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log
+                          elif [[ $machine =~ "Orion" ]]
+                          then
+                           echo "Running regression tests on $machine"
+                            export dprefix=/work2/noaa/$ACCNR/$USER
+                            sed "s|/noaa/stmp/|/noaa/$ACCNR/stmp/|g" -i rt.sh
+                            ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log
+                          elif [[ $machine =~ "Gaea" ]]
+                          then 
+                           echo "Running regression tests on $machine"
+                            ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log
+                            unset LD_LIBRARY_PATH
+                          elif [[ $machine =~ "Hera" ]]
+                          then
+                           echo "Running regression tests on $machine"
+                           export ACCNR=nems
+                          ./rt.sh -a ${ACCNR} -r -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log
+			  elif [[ $machine =~ "Derecho" ]]
+                          then
+                           echo "Running regression tests on $machine"
+                           export ACCNR=nral0032
+                           ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log
+                          else
+                           echo "Running regression tests on $machine"
+                           ./rt.sh -a ${ACCNR} -r -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log
+                          fi
+
+                      git config user.email "[email protected]"
+                      git config user.name "epic-cicd-jenkins"
+                      export machine_name_logs=$(echo $machine | awk '{ print tolower($1) }')
+                      echo "Testing concluded...removing labels for $machine from $GIT_URL"
+                      git remote -v | grep -w sshorigin > /dev/null 2>&1 && git remote remove sshorigin > /dev/null 2>&1
+                      git remote add sshorigin $SSH_ORIGIN > /dev/null 2>&1
+                      git add logs/RegressionTests_$machine_name_logs.log
+                      git commit -m "[AutoRT] $machine Job Completed.\n\n\n on-behalf-of @ufs-community <[email protected]>"
+                      git pull sshorigin $FORK_BRANCH
+                      git push sshorigin HEAD:$FORK_BRANCH
+
+                      tar --create --gzip --verbose --dereference --file "${machine_name_logs}.tgz" ${WORKSPACE}/tests/logs/*.log
+
+                      GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4)
+                      GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1)
+
+                      curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}"  https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels/$machine-RT
+
+                       '''
+                       s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: true, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "**/*tgz*", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                    }
+                  }
+                  catch(err) {
+                    sh '''
+                      export machine=${NODE_NAME}
+                      export CHANGE_ID=${CHANGE_ID}
+                      export SSH_ORIGIN=$(curl --silent https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/$CHANGE_ID | jq -r '.head.repo.ssh_url')
+                      export FORK_BRANCH=$(curl --silent https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/$CHANGE_ID | jq -r '.head.ref')
+                      cd $WORKSPACE/tests
+                      git config user.email "[email protected]"
+                      git config user.name "epic-cicd-jenkins"
+                      export machine_name_logs=$(echo $machine | awk '{ print tolower($1) }')
+                      echo "Testing concluded...removing labels for $machine from $GIT_URL"
+                      git remote -v | grep -w sshorigin > /dev/null 2>&1 && git remote remove sshorigin > /dev/null 2>&1
+                      git remote add sshorigin $SSH_ORIGIN > /dev/null 2>&1
+
+                      if [[ -f $WORKSPACE/tests/logs/RT-run-$machine.log ]] ; then
+                         git add $WORKSPACE/tests/logs/RT-run-$machine.log
+                      fi
+                      git commit  --allow-empty -m "[AutoRT] $machine Job Failed!\n\n\n on-behalf-of @ufs-community <[email protected]>"
+
+                      git pull sshorigin $FORK_BRANCH
+                      git push sshorigin HEAD:$FORK_BRANCH
+
+                      tar --create --gzip --verbose --dereference --file "${machine_name_logs}.tgz" ${WORKSPACE}/tests/logs/*.log
+
+                      GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4)
+                      GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1)
+
+                      curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}"  https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels -d '{"labels":["$machine-.*RT|$machine-.*BL"]}'
+                    '''
+                      s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: true, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "**/*tgz*", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                    currentBuild.result = 'FAILURE'
+                  }
+                }
+              }
+           }
+       }
+}
+
+def parallelStagesMap = modifiedLabels.collectEntries {
+    ["${it}" : generateStage(it)]
+}
+
+pipeline {
+    agent none
+    environment {
+        ACCNR = 'epic'
+        GITHUB_TOKEN = credentials('GithubJenkinsNew')
+        GIT_URL = 'https://github.com/ufs-community/ufs-weather-model.git'
+    }
+    stages {
+        stage('Run SonarQube Testing') {
+            steps {
+                script {
+                    build job: '/ufs-weather-model/ufs-wm-sonarqube', parameters: [
+                        string(name: 'BRANCH_NAME', value: env.CHANGE_BRANCH ?: 'develop'),
+                        string(name: 'FORK_NAME', value: env.CHANGE_FORK ?: '')
+                    ], wait: false
+                }
+            }
+        }
+        stage('Run Regression Tests in Parallel') {
+            steps {
+                script {
+                    parallel parallelStagesMap
+                }
+            }
+        }       
+    }   
+        post {
+         success {
+             node('built-in') {
+                echo 'This will run only if successful.'
+                sh '''
+                  aws sns publish --topic-arn "arn:aws:sns:us-east-1:211527314271:Jenkins-CICD-Notifications" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":sunny: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'*  *succeeded*"}}'
+               '''
+             }
+         }
+         failure {
+             node('built-in') {
+                echo 'This will run only if the run was marked as unstable.'
+                sh '''
+                  aws sns publish --topic-arn "arn:aws:sns:us-east-1:211527314271:Jenkins-CICD-Notifications" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":warning: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'*  *failed!*"}}'
+               '''
+             }
+         }
+     } 
+}

tests/rt.sh

@@ -738,8 +738,8 @@ elif [[ $MACHINE_ID = hercules ]]; then
   ECFLOW_START=/work/noaa/epic/role-epic/spack-stack/hercules/ecflow-5.8.4/bin/ecflow_start.sh
   ECF_PORT=$(( $(id -u) + 1500 ))
 
-  QUEUE=windfall
-  COMPILE_QUEUE=windfall
+  QUEUE=batch
+  COMPILE_QUEUE=batch
   PARTITION=hercules
   dprefix=/work2/noaa/stmp/${USER}
   DISKNM=/work/noaa/epic/hercules/UFS-WM_RT
@@ -758,8 +758,12 @@ elif [[ $MACHINE_ID = jet ]]; then
   ROCOTOCOMPLETE=$(which rocotocomplete)
   ROCOTO_SCHEDULER=slurm
 
-  module load ecflow/5.5.3
-  ECFLOW_START=/apps/ecflow/5.5.3/bin/ecflow_start.sh
+  module load ecflow/5.11.4
+  ECFLOW_START=/apps/ecflow/5.11.4/bin/ecflow_start.sh
+
+  module use /mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-rocky8/install/modulefiles/Core
+  module load stack-intel/2021.5.0
+  module load stack-python/3.10.8
 
   QUEUE=batch
   COMPILE_QUEUE=batch