From 1259aba25985b9500aa6c313561423ade773cba2 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Fri, 7 Jun 2024 19:29:36 +0000 Subject: [PATCH 01/10] [feature/hash_update] Update weather model hash to 485ccdf (from June 5) --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 25ec5f79b..efcaf88fb 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -12,7 +12,7 @@ protocol = git repo_url = https://github.com/ufs-community/ufs-weather-model # Specify either a branch name or a hash but not both. #branch = develop -hash = 1c6b4d4 +hash = 485ccdf local_path = sorc/ufs-weather-model required = True From e1a0cf12f89997e8e3278d0ddbc2194ec5aa74f4 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Tue, 11 Jun 2024 18:41:59 +0000 Subject: [PATCH 02/10] [feature/hash_update] update PULL_REQUEST_TEMPLATE to consolidate Tier-1 platforms and compilers for testing --- .github/PULL_REQUEST_TEMPLATE | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE index 1c363c651..29a878d4a 100644 --- a/.github/PULL_REQUEST_TEMPLATE +++ b/.github/PULL_REQUEST_TEMPLATE @@ -30,15 +30,13 @@ -- [ ] hera.intel -- [ ] orion.intel -- [ ] hercules.intel -- [ ] cheyenne.intel -- [ ] cheyenne.gnu - [ ] derecho.intel - [ ] gaea.intel -- [ ] gaeac5.intel +- [ ] hera.gnu +- [ ] hera.intel +- [ ] hercules.intel - [ ] jet.intel +- [ ] orion.intel - [ ] wcoss2.intel - [ ] NOAA Cloud (indicate which platform) - [ ] Jenkins From f6de84e5db37e9de000efbb24e12bf7b893fa598 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Thu, 27 Jun 2024 18:47:14 +0000 Subject: [PATCH 03/10] [feature/hash_update] Apply PULL_REQUEST_TEMPLATE updates to ContributorGuide --- doc/ContribGuide/contributing.rst | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/doc/ContribGuide/contributing.rst b/doc/ContribGuide/contributing.rst index ed1671363..eb995efb4 100644 --- a/doc/ContribGuide/contributing.rst +++ b/doc/ContribGuide/contributing.rst @@ -227,15 +227,13 @@ Here is the template that is provided when developers click "Create pull request - - [ ] hera.intel - - [ ] orion.intel - - [ ] hercules.intel - - [ ] cheyenne.intel - - [ ] cheyenne.gnu - [ ] derecho.intel - [ ] gaea.intel - - [ ] gaeac5.intel + - [ ] hera.gnu + - [ ] hera.intel + - [ ] hercules.intel - [ ] jet.intel + - [ ] orion.intel - [ ] wcoss2.intel - [ ] NOAA Cloud (indicate which platform) - [ ] Jenkins From 4c71aae41cdec6a3f3b0f716560b6f9aedef418e Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Tue, 30 Jul 2024 19:38:53 +0000 Subject: [PATCH 04/10] [feature/hash_update] Update verification scripts to address KeyError messages in the log files --- scripts/exregional_run_met_genensprod_or_ensemblestat.sh | 2 +- scripts/exregional_run_met_gridstat_or_pointstat_vx.sh | 2 +- scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh | 2 +- scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh | 2 +- scripts/exregional_run_met_pb2nc_obs.sh | 3 +-- scripts/exregional_run_met_pcpcombine.sh | 2 +- 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh index 05503bb96..1c09dc09c 100755 --- a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh +++ b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh @@ -10,7 +10,7 @@ . $USHdir/source_util_funcs.sh for sect in user nco platform workflow nco global verification cpl_aqm_parm \ constants fixed_files grid_params \ - task_run_post task_run_vx_ensgrid ; do + task_run_post ; do source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} done # diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh index 03c609394..abe5e3dd3 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh @@ -10,7 +10,7 @@ . $USHdir/source_util_funcs.sh for sect in user nco platform workflow nco global verification cpl_aqm_parm \ constants fixed_files grid_params \ - task_run_post task_run_vx_gridstat task_run_vx_pointstat ; do + task_run_post ; do source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} done # diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh index 12a54dc21..2c8378c12 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh @@ -10,7 +10,7 @@ . $USHdir/source_util_funcs.sh for sect in user nco platform workflow nco global verification cpl_aqm_parm \ constants fixed_files grid_params \ - task_run_post task_run_vx_ensgrid_mean task_run_vx_enspoint_mean ; do + task_run_post ; do source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} done # diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh index 8fd4a59df..eae1850ad 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh @@ -10,7 +10,7 @@ . $USHdir/source_util_funcs.sh for sect in user nco platform workflow nco global verification cpl_aqm_parm \ constants fixed_files grid_params \ - task_run_vx_ensgrid_prob task_run_vx_enspoint_prob task_run_post ; do + task_run_post ; do source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} done # diff --git a/scripts/exregional_run_met_pb2nc_obs.sh b/scripts/exregional_run_met_pb2nc_obs.sh index 5281021f0..7e79fb4ef 100755 --- a/scripts/exregional_run_met_pb2nc_obs.sh +++ b/scripts/exregional_run_met_pb2nc_obs.sh @@ -9,8 +9,7 @@ # . $USHdir/source_util_funcs.sh for sect in user nco platform workflow nco global verification cpl_aqm_parm \ - constants fixed_files grid_params \ - task_run_met_pb2nc_obs ; do + constants fixed_files grid_params ; do source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} done # diff --git a/scripts/exregional_run_met_pcpcombine.sh b/scripts/exregional_run_met_pcpcombine.sh index ce9e78ab1..026afb4eb 100755 --- a/scripts/exregional_run_met_pcpcombine.sh +++ b/scripts/exregional_run_met_pcpcombine.sh @@ -10,7 +10,7 @@ . $USHdir/source_util_funcs.sh for sect in user nco platform workflow nco global verification cpl_aqm_parm \ constants fixed_files grid_params \ - task_run_met_pcpcombine task_run_post ; do + task_run_post ; do source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} done # From 2b8b292153fd63affb66688de04810425f1867ac Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Tue, 30 Jul 2024 19:51:32 +0000 Subject: [PATCH 05/10] [feature/hash_update] Update Jenkinsfile to address Functional WorkflowTaskTests having a TASK_DEPTH set to null --- .cicd/Jenkinsfile | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 030661bd2..3d4cb95a2 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -91,6 +91,11 @@ pipeline { expression { params.SRW_COMPILER_FILTER == 'all' } expression { params.SRW_COMPILER_FILTER == env.SRW_COMPILER } } + + anyOf { + expression { params.SRW_WRAPPER_TASK_DEPTH == '9' } + expression { params.SRW_WRAPPER_TASK_DEPTH == env.TASK_DEPTH } + } } } @@ -104,6 +109,11 @@ pipeline { name 'SRW_COMPILER' values 'gnu', 'intel' } + + axis { + name 'TASK_DEPTH' + values '9', '1', '0' + } } excludes { @@ -192,10 +202,6 @@ pipeline { // Try a few Workflow Task scripts to make sure E2E tests can be launched in a follow-on 'Test' stage stage('Functional WorkflowTaskTests') { - environment { - TASK_DEPTH = "${env.SRW_WRAPPER_TASK_DEPTH}" - } - steps { dir ("${env.SRW_PLATFORM}") { echo "Running ${TASK_DEPTH} simple workflow script task tests on ${env.SRW_PLATFORM} (using ${env.WORKSPACE}/${env.SRW_PLATFORM})" From 3ed69ec31c1cc6d73d7a0ba85c00ed475d525be6 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Tue, 30 Jul 2024 19:54:31 +0000 Subject: [PATCH 06/10] [feature/hash_update] Update the Jenkinsfile to only run on Hera Intel (for testing purposes) --- .cicd/Jenkinsfile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 3d4cb95a2..294d49206 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -15,9 +15,11 @@ pipeline { // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use') // Use the line below to enable the PW AWS cluster // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use') - choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use') + // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use') + choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'hera'], description: 'Specify the platform(s) to use') // Allow job runner to filter based on compiler - choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build') + // choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build') + choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'intel'], description: 'Specify the compiler(s) to use to build') // Workflow Wrapper test depth {0..9}, 0=none, 1=simple, 9=all [default] choice(name: 'SRW_WRAPPER_TASK_DEPTH', choices: ['9', '1', '0'], description: '0=none, 1=simple, 9=all [default]') // WE2E Tests ? @@ -102,12 +104,14 @@ pipeline { axes { axis { name 'SRW_PLATFORM' - values 'derecho', 'gaea', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' + // values 'derecho', 'gaea', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' + values 'hera' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' } axis { name 'SRW_COMPILER' - values 'gnu', 'intel' + // values 'gnu', 'intel' + values 'intel' } axis { From 55fb2ff83e22aaf2aa9f78ef904f72fb9e5c7c37 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Tue, 30 Jul 2024 20:54:59 +0000 Subject: [PATCH 07/10] [feature/hash_update] Removing the TASK_DEPTH axis entry from Jenkinsfile. Replace env.SRW_WRAPPER_TASK_DEPTH with params.SRW_WRAPPER_TASK_DEPTH --- .cicd/Jenkinsfile | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 294d49206..7f1ed4296 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -93,11 +93,6 @@ pipeline { expression { params.SRW_COMPILER_FILTER == 'all' } expression { params.SRW_COMPILER_FILTER == env.SRW_COMPILER } } - - anyOf { - expression { params.SRW_WRAPPER_TASK_DEPTH == '9' } - expression { params.SRW_WRAPPER_TASK_DEPTH == env.TASK_DEPTH } - } } } @@ -113,11 +108,6 @@ pipeline { // values 'gnu', 'intel' values 'intel' } - - axis { - name 'TASK_DEPTH' - values '9', '1', '0' - } } excludes { @@ -206,6 +196,10 @@ pipeline { // Try a few Workflow Task scripts to make sure E2E tests can be launched in a follow-on 'Test' stage stage('Functional WorkflowTaskTests') { + environment { + TASK_DEPTH = "${params.SRW_WRAPPER_TASK_DEPTH}" + } + steps { dir ("${env.SRW_PLATFORM}") { echo "Running ${TASK_DEPTH} simple workflow script task tests on ${env.SRW_PLATFORM} (using ${env.WORKSPACE}/${env.SRW_PLATFORM})" From 4b7fcb843fc35fa063c43d63e7ad82c3159904f2 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Wed, 31 Jul 2024 14:44:30 +0000 Subject: [PATCH 08/10] [feature/hash_update] Add back in removed machines and compilers (that were removed for testing purposes) --- .cicd/Jenkinsfile | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 7f1ed4296..e6ed9515f 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -15,11 +15,9 @@ pipeline { // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use') // Use the line below to enable the PW AWS cluster // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use') - // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use') - choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'hera'], description: 'Specify the platform(s) to use') + choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use') // Allow job runner to filter based on compiler - // choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build') - choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'intel'], description: 'Specify the compiler(s) to use to build') + choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build') // Workflow Wrapper test depth {0..9}, 0=none, 1=simple, 9=all [default] choice(name: 'SRW_WRAPPER_TASK_DEPTH', choices: ['9', '1', '0'], description: '0=none, 1=simple, 9=all [default]') // WE2E Tests ? @@ -99,14 +97,12 @@ pipeline { axes { axis { name 'SRW_PLATFORM' - // values 'derecho', 'gaea', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' - values 'hera' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' + values 'derecho', 'gaea', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' } axis { name 'SRW_COMPILER' - // values 'gnu', 'intel' - values 'intel' + values 'gnu', 'intel' } } From e5c22ac4ab258cf2a86c164aea2f40a17f142272 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Wed, 31 Jul 2024 20:46:50 +0000 Subject: [PATCH 09/10] [feature/hash_update] Update ufs-weather-model hash to b5a1976 (July 30) --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index efcaf88fb..4545cd8ca 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -12,7 +12,7 @@ protocol = git repo_url = https://github.com/ufs-community/ufs-weather-model # Specify either a branch name or a hash but not both. #branch = develop -hash = 485ccdf +hash = b5a1976 local_path = sorc/ufs-weather-model required = True From c65bc576e0de494bacf999a8741175c63c01bc77 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Fri, 9 Aug 2024 15:51:30 +0000 Subject: [PATCH 10/10] [feature/hash_update] Update modulefiles/tasks/noaacloud/plot_allvars.local.lua so that the plotting tasks will work on NOAA Cloud platforms --- modulefiles/tasks/noaacloud/plot_allvars.local.lua | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/modulefiles/tasks/noaacloud/plot_allvars.local.lua b/modulefiles/tasks/noaacloud/plot_allvars.local.lua index 2fd9b41eb..85291013c 100644 --- a/modulefiles/tasks/noaacloud/plot_allvars.local.lua +++ b/modulefiles/tasks/noaacloud/plot_allvars.local.lua @@ -1,5 +1,2 @@ -unload("python") -append_path("MODULEPATH","/contrib/EPIC/miniconda3/modulefiles") -load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) - -setenv("SRW_GRAPHICS_ENV", "regional_workflow") +load("conda") +setenv("SRW_GRAPHICS_ENV", "srw_graphics")