From 80d9fd65cc7cc816a8f66981ce342f3968652788 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 14 Jun 2024 12:58:07 -0400 Subject: [PATCH 1/9] updated Jenkinsfile with try block around scm checkout and beefed up Finilize --- ci/Jenkinsfile | 52 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index c6aa0887c7..5d0f078d30 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -3,7 +3,8 @@ def machine = 'none' def HOME = 'none' def caseList = '' // Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR. -def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES'] +def NodeName = [Hera: 'Hera-EMC', Orion: 'Orion-EMC', Hercules: 'Hercules-EMC', Gaea: 'Gaea'] +def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI'] def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git' def STATUS = 'Passed' @@ -70,13 +71,13 @@ pipeline { } stage('2. Get Common Workspace') { - agent { label "${machine}-emc" } + agent { label NodeName[machine].toLowerCase() } steps { script { Machine = machine[0].toUpperCase() + machine.substring(1) echo "Getting Common Workspace for ${Machine}" ws("${custom_workspace[machine]}/${env.CHANGE_ID}") { - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) HOME = "${WORKSPACE}" sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/*") sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" --remove-label "CI-${Machine}-Ready" """) @@ -88,7 +89,7 @@ pipeline { stage('3. Build System') { matrix { - agent { label "${machine}-emc" } + agent { label NodeName[machine].toLowerCase() } //options { // throttle(['global_matrix_build']) //} @@ -112,7 +113,16 @@ pipeline { sh(script: './link_workflow.sh') } } else { - checkout scm + try { + echo "Checking out the code for ${system} on ${Machine} using scm in ${HOMEgfs}" + checkout scm + } catch (Exception e) { + if (env.CHANGE_ID) { + sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Checkout **Failed** on ${Machine}: ${e.getMessage()}" """) + } + echo "Failed to checkout: ${e.getMessage()}" + STATUS = 'Failed' + } def gist_url = "" def error_logs = "" def error_logs_message = "" @@ -173,7 +183,7 @@ pipeline { stage('4. Run Tests') { failFast false matrix { - agent { label "${machine}-emc" } + agent { label NodeName[machine].toLowerCase() } axes { axis { name 'Case' @@ -259,19 +269,27 @@ pipeline { } } stage( '5. FINALIZE' ) { - when { - expression { - STATUS == 'Passed' - } - } - agent { label "${machine}-emc" } + agent { label NodeName[machine].toLowerCase() } steps { script { - try { - sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --remove-label "CI-${Machine}-Building" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true) - sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI ${STATUS}** ${Machine} at
Built and ran in directory \\`${HOME}\\`" """, returnStatus: true) - } catch (Exception e) { - echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}" + sh(script: """ + labels=\$(gh pr view ${env.CHANGE_ID} --repo ${repo_url} --json labels --jq '.labels[].name') + for label in \$labels; do + if [[ "\$label" == *"${Machine}"* ]]; then + gh pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "\$label" + fi + done + """, returnStatus: true) + sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true) + sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI ${STATUS}** ${Machine} at
Built and ran in directory \\`${HOME}\\`" """, returnStatus: true) + if STATUS == 'Passed' { + try { + sh(script: "rm -Rf ${HOME}/*") + } catch (Exception e) { + echo "Failed to remove custom work directory ${HOME} on ${Machine}: ${e.getMessage()}" + } + } else { + echo "Failed to build and run Global-workflow in ${home} and on ${Machine}" } } } From 690c7c83e64358a1e61a51b0649f6823b52f49df Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 14 Jun 2024 13:10:29 -0400 Subject: [PATCH 2/9] syntax error on if statment --- ci/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 5d0f078d30..6601149148 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -282,7 +282,7 @@ pipeline { """, returnStatus: true) sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true) sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI ${STATUS}** ${Machine} at
Built and ran in directory \\`${HOME}\\`" """, returnStatus: true) - if STATUS == 'Passed' { + if (STATUS == 'Passed') { try { sh(script: "rm -Rf ${HOME}/*") } catch (Exception e) { From 81638f94f3654178794adb8c2c656aff03d6eeb0 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 14 Jun 2024 13:33:59 -0400 Subject: [PATCH 3/9] had up update Machine name for map --- ci/Jenkinsfile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 6601149148..f48e87d97e 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -43,7 +43,7 @@ pipeline { if (label.matches("CI-(.*?)-Ready")) { def Machine_name = label.split('-')[1].toString() jenkins.model.Jenkins.get().computers.each { c -> - if (c.node.selfLabel.name == "${Machine_name}-EMC") { + if (c.node.selfLabel.name == NodeName[Machine_name]) { run_nodes.add(c.node.selfLabel.name) } } @@ -53,8 +53,10 @@ pipeline { if (run_nodes.size() > 1) { run_nodes.init().each { node -> def machine_name = node.split('-')[0].toLowerCase() + Machine = machine_name[0].toUpperCase() + machine_name.substring(1) echo "Spawning job on node: ${node} with machine name: ${machine_name}" build job: "/global-workflow/EMC-Global-Pipeline/PR-${env.CHANGE_ID}", parameters: [ + string(name: 'Machine', value: Machine), string(name: 'machine', value: machine_name), string(name: 'Node', value: node) ], wait: false @@ -71,10 +73,9 @@ pipeline { } stage('2. Get Common Workspace') { - agent { label NodeName[machine].toLowerCase() } + agent { label NodeName[Machine].toLowerCase() } steps { script { - Machine = machine[0].toUpperCase() + machine.substring(1) echo "Getting Common Workspace for ${Machine}" ws("${custom_workspace[machine]}/${env.CHANGE_ID}") { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) @@ -89,7 +90,7 @@ pipeline { stage('3. Build System') { matrix { - agent { label NodeName[machine].toLowerCase() } + agent { label NodeName[Machine].toLowerCase() } //options { // throttle(['global_matrix_build']) //} @@ -183,7 +184,7 @@ pipeline { stage('4. Run Tests') { failFast false matrix { - agent { label NodeName[machine].toLowerCase() } + agent { label NodeName[Machine].toLowerCase() } axes { axis { name 'Case' @@ -269,7 +270,7 @@ pipeline { } } stage( '5. FINALIZE' ) { - agent { label NodeName[machine].toLowerCase() } + agent { label NodeName[Machine].toLowerCase() } steps { script { sh(script: """ From 71845e9ffefa3954207ea24603d246bc472d7116 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 14 Jun 2024 13:44:06 -0400 Subject: [PATCH 4/9] lower case on the keys for Node Name map --- ci/Jenkinsfile | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index f48e87d97e..52961b3440 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -3,7 +3,7 @@ def machine = 'none' def HOME = 'none' def caseList = '' // Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR. -def NodeName = [Hera: 'Hera-EMC', Orion: 'Orion-EMC', Hercules: 'Hercules-EMC', Gaea: 'Gaea'] +def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea'] def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI'] def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git' def STATUS = 'Passed' @@ -41,9 +41,9 @@ pipeline { echo "This is parent job so getting list of nodes matching labels:" for (label in pullRequest.labels) { if (label.matches("CI-(.*?)-Ready")) { - def Machine_name = label.split('-')[1].toString() + def machine_name = label.split('-')[1].toString().toLowerCase() jenkins.model.Jenkins.get().computers.each { c -> - if (c.node.selfLabel.name == NodeName[Machine_name]) { + if (c.node.selfLabel.name == NodeName[machine_name]) { run_nodes.add(c.node.selfLabel.name) } } @@ -53,10 +53,8 @@ pipeline { if (run_nodes.size() > 1) { run_nodes.init().each { node -> def machine_name = node.split('-')[0].toLowerCase() - Machine = machine_name[0].toUpperCase() + machine_name.substring(1) echo "Spawning job on node: ${node} with machine name: ${machine_name}" build job: "/global-workflow/EMC-Global-Pipeline/PR-${env.CHANGE_ID}", parameters: [ - string(name: 'Machine', value: Machine), string(name: 'machine', value: machine_name), string(name: 'Node', value: node) ], wait: false @@ -73,7 +71,7 @@ pipeline { } stage('2. Get Common Workspace') { - agent { label NodeName[Machine].toLowerCase() } + agent { label NodeName[machine].toLowerCase() } steps { script { echo "Getting Common Workspace for ${Machine}" @@ -90,7 +88,7 @@ pipeline { stage('3. Build System') { matrix { - agent { label NodeName[Machine].toLowerCase() } + agent { label NodeName[machine].toLowerCase() } //options { // throttle(['global_matrix_build']) //} @@ -184,7 +182,7 @@ pipeline { stage('4. Run Tests') { failFast false matrix { - agent { label NodeName[Machine].toLowerCase() } + agent { label NodeName[machine].toLowerCase() } axes { axis { name 'Case' @@ -270,7 +268,7 @@ pipeline { } } stage( '5. FINALIZE' ) { - agent { label NodeName[Machine].toLowerCase() } + agent { label NodeName[machine].toLowerCase() } steps { script { sh(script: """ From 138f016d13ac0ddcf4bcb5c6fbc8411cd909d1d8 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 14 Jun 2024 13:50:38 -0400 Subject: [PATCH 5/9] put decleratoin of Machine name back with capatilized variable --- ci/Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 52961b3440..b8e5347d1f 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -74,6 +74,7 @@ pipeline { agent { label NodeName[machine].toLowerCase() } steps { script { + Machine = machine[0].toUpperCase() + machine.substring(1) echo "Getting Common Workspace for ${Machine}" ws("${custom_workspace[machine]}/${env.CHANGE_ID}") { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) From f04a47edffb8ff359ff3b43356913e4a5d27b4bd Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 14 Jun 2024 23:50:44 -0400 Subject: [PATCH 6/9] working on formatting file complete text to GitHub --- ci/Jenkinsfile | 5 ++++- ci/scripts/run-check_ci.sh | 4 +--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index b8e5347d1f..1693edf67f 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -281,7 +281,10 @@ pipeline { done """, returnStatus: true) sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true) - sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI ${STATUS}** ${Machine} at
Built and ran in directory \\`${HOME}\\`" """, returnStatus: true) + if (fileExists("${HOME}/RUNTESTS/ci-run_check.log")) { + sh(script: """echo "**CI ${STATUS}** ${Machine} at
Built and ran in directory \\`${HOME}\\`\n\\`\\`\\`\n" | cat - ${HOME}/RUNTESTS/ci-run_check.log > temp && mv temp ${HOME}/RUNTESTS/ci-run_check.log""", returnStatus: true) + sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body-file ${HOME}/RUNTESTS/ci-run_check.log """, returnStatus: true) + } if (STATUS == 'Passed') { try { sh(script: "rm -Rf ${HOME}/*") diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh index 5c891fc4bd..5c49a21c4b 100755 --- a/ci/scripts/run-check_ci.sh +++ b/ci/scripts/run-check_ci.sh @@ -101,9 +101,7 @@ while true; do if [[ "${ROCOTO_STATE}" == "DONE" ]]; then { - echo "Experiment ${pslot} Completed ${CYCLES_DONE} Cycles at $(date)" || true - echo "with ${SUCCEEDED} successfully completed jobs" || true - echo "Experiment ${pslot} Completed: *SUCCESS*" + echo "Experiment ${pslot} Completed ${CYCLES_DONE} Cycles: *SUCCESS* at $(date)" || true } | tee -a "${run_check_logfile}" rc=0 break From 9972320e1d314fbdf4f2300860fd998e5d27766a Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Mon, 17 Jun 2024 19:55:30 +0000 Subject: [PATCH 7/9] remove verbosing of passes cases in BASH CI --- ci/scripts/check_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh index cd907d34aa..04dd92f4a6 100755 --- a/ci/scripts/check_ci.sh +++ b/ci/scripts/check_ci.sh @@ -175,7 +175,7 @@ for pr in ${pr_list}; do DATE=$(date +'%D %r') echo "Experiment ${pslot} **SUCCESS** on ${MACHINE_ID^} at ${DATE}" >> "${output_ci_single}" echo "Experiment ${pslot} *** SUCCESS *** at ${DATE}" >> "${output_ci}" - "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}" + # "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}" fi done done From 488bdd4a695c0a42556a9fa06e90f8caa667d068 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Mon, 24 Jun 2024 10:01:14 -0400 Subject: [PATCH 8/9] disambiguated Jenkins variable HOME from shell default to CUSTOME_WORKSPACE in Jenkinsfile --- ci/Jenkinsfile | 52 +++++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 1693edf67f..2757fea0ba 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -1,6 +1,6 @@ def Machine = 'none' def machine = 'none' -def HOME = 'none' +def CUSTOME_WORKSPACE = 'none' def caseList = '' // Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR. def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea'] @@ -77,12 +77,12 @@ pipeline { Machine = machine[0].toUpperCase() + machine.substring(1) echo "Getting Common Workspace for ${Machine}" ws("${custom_workspace[machine]}/${env.CHANGE_ID}") { - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) - HOME = "${WORKSPACE}" - sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/*") + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) + CUSTOME_WORKSPACE = "${WORKSPACE}" + sh(script: "mkdir -p ${CUSTOME_WORKSPACE}/RUNTESTS;rm -Rf ${CUSTOME_WORKSPACE}/RUNTESTS/*") sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" --remove-label "CI-${Machine}-Ready" """) } - echo "Building and running on ${Machine} in directory ${HOME}" + echo "Building and running on ${Machine} in directory ${CUSTOME_WORKSPACE}" } } } @@ -103,7 +103,7 @@ pipeline { stage('build system') { steps { script { - def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to build the system on per system basis under the common workspace HOME + def HOMEgfs = "${CUSTOME_WORKSPACE}/${system}" // local HOMEgfs is used to build the system on per system basis under the custome workspace for each buile system sh(script: "mkdir -p ${HOMEgfs}") ws(HOMEgfs) { if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs @@ -199,11 +199,11 @@ pipeline { } steps { script { - sh(script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp") - def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp" + sh(script: "sed -n '/{.*}/!p' ${CUSTOME_WORKSPACE}/gfs/ci/cases/pr/${Case}.yaml > ${CUSTOME_WORKSPACEOME_WORKSPACE}/gfs/ci/cases/pr/${Case}.yaml.tmp") + def yaml_case = readYaml file: "${CUSTOME_WORKSPACE}/gfs/ci/cases/pr/${Case}.yaml.tmp" system = yaml_case.experiment.system - def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to populate the XML on per system basis - env.RUNTESTS = "${HOME}/RUNTESTS" + def HOMEgfs = "${CUSTOME_WORKSPACE}/${system}" // local HOMEgfs is used to populate the XML on per system basis + env.RUNTESTS = "${CUSTOME_WORKSPACE}/RUNTESTS" sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml") } } @@ -216,15 +216,15 @@ pipeline { failFast false steps { script { - HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments - def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim() - def error_file = "${HOME}/RUNTESTS/${pslot}_error.logs" + HOMEgfs = "${CUSTOME_WORKSPACE}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments + def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOME_WORKSPACE}/RUNTESTS ${Case}", returnStdout: true).trim() + def error_file = "${CUSTOME_WORKSPACE}/RUNTESTS/${pslot}_error.logs" sh(script: " rm -f ${error_file}") try { - sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot} ${system}") + sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOME_WORKSPACE} ${pslot} ${system}") } catch (Exception error_experment) { sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}") - ws(HOME) { + ws(CUSTOME_WORKSPACE) { def error_logs = "" def error_logs_message = "" if (fileExists(error_file)) { @@ -232,11 +232,11 @@ pipeline { def lines = fileContent.readLines() for (line in lines) { echo "archiving: ${line}" - if (fileExists("${HOME}/${line}") && readFile("${HOME}/${line}").length() > 0) { + if (fileExists("${CUSTOME_WORKSPACE}/${line}") && readFile("${CUSTOME_WORKSPACE}/${line}").length() > 0) { try { archiveArtifacts artifacts: "${line}", fingerprint: true - error_logs = error_logs + "${HOME}/${line} " - error_logs_message = error_logs_message + "${HOME}/${line}\n" + error_logs = error_logs + "${CUSTOME_WORKSPACE}/${line} " + error_logs_message = error_logs_message + "${CUSTOME_WORKSPACE}/${line}\n" } catch (Exception error_arch) { echo "Failed to archive error log ${line}: ${error_arch.getMessage()}" } @@ -250,12 +250,12 @@ pipeline { echo "Failed to comment on PR: ${error_comment.getMessage()}" } } else { - echo "No error logs found for failed cases in $HOME/RUNTESTS/${pslot}_error.logs" + echo "No error logs found for failed cases in $CUSTOME_WORKSPACE/RUNTESTS/${pslot}_error.logs" } STATUS = 'Failed' try { sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true) - sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} **FAILED** on ${Machine} in\n\\`${HOME}/RUNTESTS/${pslot}\\`" """) + sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} **FAILED** on ${Machine} in\n\\`${CUSTOME_WORKSPACE}/RUNTESTS/${pslot}\\`" """) } catch (Exception e) { echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}" } @@ -281,18 +281,18 @@ pipeline { done """, returnStatus: true) sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true) - if (fileExists("${HOME}/RUNTESTS/ci-run_check.log")) { - sh(script: """echo "**CI ${STATUS}** ${Machine} at
Built and ran in directory \\`${HOME}\\`\n\\`\\`\\`\n" | cat - ${HOME}/RUNTESTS/ci-run_check.log > temp && mv temp ${HOME}/RUNTESTS/ci-run_check.log""", returnStatus: true) - sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body-file ${HOME}/RUNTESTS/ci-run_check.log """, returnStatus: true) + if (fileExists("${CUSTOME_WORKSPACE}/RUNTESTS/ci-run_check.log")) { + sh(script: """echo "**CI ${STATUS}** ${Machine} at
Built and ran in directory \\`${CUSTOME_WORKSPACE}\\`\n\\`\\`\\`\n" | cat - ${CUSTOME_WORKSPACE}/RUNTESTS/ci-run_check.log > temp && mv temp ${CUSTOME_WORKSPACE}/RUNTESTS/ci-run_check.log""", returnStatus: true) + sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body-file ${CUSTOME_WORKSPACE}/RUNTESTS/ci-run_check.log """, returnStatus: true) } if (STATUS == 'Passed') { try { - sh(script: "rm -Rf ${HOME}/*") + sh(script: "rm -Rf ${CUSTOME_WORKSPACE}/*") } catch (Exception e) { - echo "Failed to remove custom work directory ${HOME} on ${Machine}: ${e.getMessage()}" + echo "Failed to remove custom work directory ${CUSTOME_WORKSPACE} on ${Machine}: ${e.getMessage()}" } } else { - echo "Failed to build and run Global-workflow in ${home} and on ${Machine}" + echo "Failed to build and run Global-workflow in ${CUSTOME_WORKSPACE} and on ${Machine}" } } } From 898a61adfa9c295093d0f2ce442f8e7506dc32ff Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Mon, 24 Jun 2024 11:17:04 -0400 Subject: [PATCH 9/9] malaproprated CUTOM_WORKSPCE --- ci/Jenkinsfile | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 2757fea0ba..956bd692dd 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -1,6 +1,6 @@ def Machine = 'none' def machine = 'none' -def CUSTOME_WORKSPACE = 'none' +def CUSTOM_WORKSPACE = 'none' def caseList = '' // Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR. def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea'] @@ -78,11 +78,11 @@ pipeline { echo "Getting Common Workspace for ${Machine}" ws("${custom_workspace[machine]}/${env.CHANGE_ID}") { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) - CUSTOME_WORKSPACE = "${WORKSPACE}" - sh(script: "mkdir -p ${CUSTOME_WORKSPACE}/RUNTESTS;rm -Rf ${CUSTOME_WORKSPACE}/RUNTESTS/*") + CUSTOM_WORKSPACE = "${WORKSPACE}" + sh(script: "mkdir -p ${CUSTOM_WORKSPACE}/RUNTESTS;rm -Rf ${CUSTOM_WORKSPACE}/RUNTESTS/*") sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" --remove-label "CI-${Machine}-Ready" """) } - echo "Building and running on ${Machine} in directory ${CUSTOME_WORKSPACE}" + echo "Building and running on ${Machine} in directory ${CUSTOM_WORKSPACE}" } } } @@ -103,7 +103,7 @@ pipeline { stage('build system') { steps { script { - def HOMEgfs = "${CUSTOME_WORKSPACE}/${system}" // local HOMEgfs is used to build the system on per system basis under the custome workspace for each buile system + def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to build the system on per system basis under the custome workspace for each buile system sh(script: "mkdir -p ${HOMEgfs}") ws(HOMEgfs) { if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs @@ -199,11 +199,11 @@ pipeline { } steps { script { - sh(script: "sed -n '/{.*}/!p' ${CUSTOME_WORKSPACE}/gfs/ci/cases/pr/${Case}.yaml > ${CUSTOME_WORKSPACEOME_WORKSPACE}/gfs/ci/cases/pr/${Case}.yaml.tmp") - def yaml_case = readYaml file: "${CUSTOME_WORKSPACE}/gfs/ci/cases/pr/${Case}.yaml.tmp" + sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${Case}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${Case}.yaml.tmp") + def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${Case}.yaml.tmp" system = yaml_case.experiment.system - def HOMEgfs = "${CUSTOME_WORKSPACE}/${system}" // local HOMEgfs is used to populate the XML on per system basis - env.RUNTESTS = "${CUSTOME_WORKSPACE}/RUNTESTS" + def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to populate the XML on per system basis + env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS" sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml") } } @@ -216,15 +216,15 @@ pipeline { failFast false steps { script { - HOMEgfs = "${CUSTOME_WORKSPACE}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments - def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOME_WORKSPACE}/RUNTESTS ${Case}", returnStdout: true).trim() - def error_file = "${CUSTOME_WORKSPACE}/RUNTESTS/${pslot}_error.logs" + HOMEgfs = "${CUSTOM_WORKSPACE}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments + def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${Case}", returnStdout: true).trim() + def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs" sh(script: " rm -f ${error_file}") try { - sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOME_WORKSPACE} ${pslot} ${system}") + sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${system}") } catch (Exception error_experment) { sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}") - ws(CUSTOME_WORKSPACE) { + ws(CUSTOM_WORKSPACE) { def error_logs = "" def error_logs_message = "" if (fileExists(error_file)) { @@ -232,11 +232,11 @@ pipeline { def lines = fileContent.readLines() for (line in lines) { echo "archiving: ${line}" - if (fileExists("${CUSTOME_WORKSPACE}/${line}") && readFile("${CUSTOME_WORKSPACE}/${line}").length() > 0) { + if (fileExists("${CUSTOM_WORKSPACE}/${line}") && readFile("${CUSTOM_WORKSPACE}/${line}").length() > 0) { try { archiveArtifacts artifacts: "${line}", fingerprint: true - error_logs = error_logs + "${CUSTOME_WORKSPACE}/${line} " - error_logs_message = error_logs_message + "${CUSTOME_WORKSPACE}/${line}\n" + error_logs = error_logs + "${CUSTOM_WORKSPACE}/${line} " + error_logs_message = error_logs_message + "${CUSTOM_WORKSPACE}/${line}\n" } catch (Exception error_arch) { echo "Failed to archive error log ${line}: ${error_arch.getMessage()}" } @@ -250,12 +250,12 @@ pipeline { echo "Failed to comment on PR: ${error_comment.getMessage()}" } } else { - echo "No error logs found for failed cases in $CUSTOME_WORKSPACE/RUNTESTS/${pslot}_error.logs" + echo "No error logs found for failed cases in $CUSTOM_WORKSPACE/RUNTESTS/${pslot}_error.logs" } STATUS = 'Failed' try { sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true) - sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} **FAILED** on ${Machine} in\n\\`${CUSTOME_WORKSPACE}/RUNTESTS/${pslot}\\`" """) + sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} **FAILED** on ${Machine} in\n\\`${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}\\`" """) } catch (Exception e) { echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}" } @@ -281,18 +281,18 @@ pipeline { done """, returnStatus: true) sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true) - if (fileExists("${CUSTOME_WORKSPACE}/RUNTESTS/ci-run_check.log")) { - sh(script: """echo "**CI ${STATUS}** ${Machine} at
Built and ran in directory \\`${CUSTOME_WORKSPACE}\\`\n\\`\\`\\`\n" | cat - ${CUSTOME_WORKSPACE}/RUNTESTS/ci-run_check.log > temp && mv temp ${CUSTOME_WORKSPACE}/RUNTESTS/ci-run_check.log""", returnStatus: true) - sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body-file ${CUSTOME_WORKSPACE}/RUNTESTS/ci-run_check.log """, returnStatus: true) + if (fileExists("${CUSTOM_WORKSPACE}/RUNTESTS/ci-run_check.log")) { + sh(script: """echo "**CI ${STATUS}** ${Machine} at
Built and ran in directory \\`${CUSTOM_WORKSPACE}\\`\n\\`\\`\\`\n" | cat - ${CUSTOM_WORKSPACE}/RUNTESTS/ci-run_check.log > temp && mv temp ${CUSTOM_WORKSPACE}/RUNTESTS/ci-run_check.log""", returnStatus: true) + sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body-file ${CUSTOM_WORKSPACE}/RUNTESTS/ci-run_check.log """, returnStatus: true) } if (STATUS == 'Passed') { try { - sh(script: "rm -Rf ${CUSTOME_WORKSPACE}/*") + sh(script: "rm -Rf ${CUSTOM_WORKSPACE}/*") } catch (Exception e) { - echo "Failed to remove custom work directory ${CUSTOME_WORKSPACE} on ${Machine}: ${e.getMessage()}" + echo "Failed to remove custom work directory ${CUSTOM_WORKSPACE} on ${Machine}: ${e.getMessage()}" } } else { - echo "Failed to build and run Global-workflow in ${CUSTOME_WORKSPACE} and on ${Machine}" + echo "Failed to build and run Global-workflow in ${CUSTOM_WORKSPACE} and on ${Machine}" } } }