From 18e5075201d10229c87234e48fa9875c1ddc9354 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Wed, 22 Mar 2023 21:56:04 -0600 Subject: [PATCH 1/4] Add Jim's changes for new GPU options based on his branch: https://github.com/jedwards4b/CMEPS/compare/ff8726f..79d6fa7 modified: cime_config/config_component.xml --- cime_config/config_component.xml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index 7f9bac96e..cadc8a433 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -784,6 +784,24 @@ If TRUE, the component libraries are always built with OpenMP capability. + + char + none, v100, a100, mi250 + none + build_def + env_build.xml + If set will compile and submit with this gpu type enabled + + + + char + none, openacc, openmp, combined + none + build_def + env_build.xml + If set will compile and submit with this gpu offload method enabled + + logical TRUE,FALSE From ebb0818566e23a99e14c3d59aff19cbaaf2e1f90 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Wed, 22 Mar 2023 22:03:18 -0600 Subject: [PATCH 2/4] Add MAX_CPUTASKS_PER_GPU_NODE XML variable Update nvhpc compiler for GPU settings Remove PGI compiler modified: cime_config/config_component.xml --- cime_config/config_component.xml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index cadc8a433..abff72296 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -1897,12 +1897,22 @@ pes or cores per node for accounting purposes + + integer + 0 + + 1 + + mach_pes_last + env_mach_pes.xml + Number of CPU cores per GPU node used for simulation + + integer 0 - 1 - 1 + 1 mach_pes env_mach_pes.xml From 72c123099cc4a8f255af4c07eb0dc26984a02340 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Fri, 24 Mar 2023 14:44:22 -0600 Subject: [PATCH 3/4] Remove default_values and valid_values for GPU_TYPE and GPU_OFFLOAD so that they could assign multiple values to the config_machines.xml file modified: cime_config/config_component.xml --- cime_config/config_component.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index abff72296..48e86f88c 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -786,8 +786,8 @@ char - none, v100, a100, mi250 - none + + build_def env_build.xml If set will compile and submit with this gpu type enabled @@ -795,8 +795,8 @@ char - none, openacc, openmp, combined - none + + build_def env_build.xml If set will compile and submit with this gpu offload method enabled From 72ee0b2fa13b125e49cfca3db1ec7ee557d30a28 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Tue, 8 Aug 2023 09:08:18 -0600 Subject: [PATCH 4/4] Add a new XML variable to apply the MPI wrapper script more generically. --- cime_config/config_component.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index 0137597af..a329be743 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -802,6 +802,16 @@ If set will compile and submit with this gpu offload method enabled + + char + + + build_def + env_build.xml + If set will attach this script to the MPI run command, mapping + different MPI ranks to different GPUs within the same compute node + + logical TRUE,FALSE