From fada2d4ef3b535275060b57a8067112be6210475 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 8 Jul 2021 11:24:56 -0600 Subject: [PATCH 01/10] code to use HPE SMARTREDIS --- cime_config/buildexe | 7 ++++++- cime_config/config_component.xml | 9 +++++++++ drivers/cime/ensemble_driver.F90 | 4 ++-- drivers/cime/esmApp.F90 | 7 ++++++- drivers/cime/esm_utils_mod.F90 | 8 ++++++-- 5 files changed, 29 insertions(+), 6 deletions(-) diff --git a/cime_config/buildexe b/cime_config/buildexe index 52640d30b..3e9b7b091 100755 --- a/cime_config/buildexe +++ b/cime_config/buildexe @@ -40,6 +40,7 @@ def _main_func(): atm_model = case.get_value("COMP_ATM") gmake_args = get_standard_makefile_args(case) esmf_aware_threading = case.get_value("ESMF_AWARE_THREADING") + use_smartsim = case.get_value("USE_SMARTSIM") # Determine valid components valid_comps = [] @@ -77,6 +78,10 @@ def _main_func(): if esmf_aware_threading: gmake_args += " USER_CPPDEFS=-DESMF_AWARE_THREADING" + if use_smartsim: + gmake_args += " USE_SMARTSIM=TRUE" + + gmake_args += " IAC_PRESENT=FALSE" expect((num_esp is None) or (int(num_esp) == 1), "ESP component restricted to one instance") @@ -109,7 +114,7 @@ def _main_func(): .format(gmake, gmake_j, exename, gmake_args, makefile) - rc, out, err = run_cmd(cmd,from_dir=bld_root) + rc, out, err = run_cmd(cmd,from_dir=bld_root, verbose=True) expect(rc==0,"Command {} failed rc={}\nout={}\nerr={}".format(cmd,rc,out,err)) logger.info(out) diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index ba73c96d6..a99397088 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -2528,6 +2528,15 @@ if true, create ESMF PET log files even if there is no error encountered + + logical + TRUE,FALSE + FALSE + run_debug + env_build.xml + if true, link with the HPE SmartRedis library + + diff --git a/drivers/cime/ensemble_driver.F90 b/drivers/cime/ensemble_driver.F90 index 8ddbb727f..29f58f2d1 100644 --- a/drivers/cime/ensemble_driver.F90 +++ b/drivers/cime/ensemble_driver.F90 @@ -134,7 +134,7 @@ subroutine SetModelServices(ensemble_driver, rc) call ReadAttributes(ensemble_driver, config, "CLOCK_attributes::", rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return - call NUOPC_CompAttributeGet(ensemble_driver, 'calendar', calendar, rc=rc) + call NUOPC_CompAttributeGet(ensemble_driver, 'calendar', calendar, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return if (calendar == 'NO_LEAP') then call ESMF_CalendarSetDefault(ESMF_CALKIND_NOLEAP, rc=rc) @@ -247,7 +247,7 @@ subroutine SetModelServices(ensemble_driver, rc) call ReadAttributes(driver, config, "DRV_modelio"//trim(inst_suffix)//"::", rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return - ! Set the driver log to the driver task 0 + ! Set the driver log to the driver task 0 if (mod(localPet, ntasks_per_member) == 0) then call NUOPC_CompAttributeGet(driver, name="diro", value=diro, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return diff --git a/drivers/cime/esmApp.F90 b/drivers/cime/esmApp.F90 index 1516ffa10..307c7b542 100644 --- a/drivers/cime/esmApp.F90 +++ b/drivers/cime/esmApp.F90 @@ -17,6 +17,9 @@ program esmApp use ensemble_driver, only : SetServices use shr_pio_mod, only : shr_pio_init1 use shr_sys_mod, only : shr_sys_abort +#ifdef SMARTREDIS + use esm_utils_mod , only : sr_client +#endif implicit none @@ -43,7 +46,9 @@ program esmApp call MPI_init(rc) #endif COMP_COMM = MPI_COMM_WORLD - +#ifdef SMARTREDIS + call sr_client%initialize(.true.) +#endif !----------------------------------------------------------------------------- ! Initialize PIO !----------------------------------------------------------------------------- diff --git a/drivers/cime/esm_utils_mod.F90 b/drivers/cime/esm_utils_mod.F90 index f6a4aeb40..73b77f668 100644 --- a/drivers/cime/esm_utils_mod.F90 +++ b/drivers/cime/esm_utils_mod.F90 @@ -1,12 +1,16 @@ module esm_utils_mod - +#ifdef SMARTREDIS + use smartredis_client, only : client_type +#endif implicit none public logical :: mastertask integer :: logunit integer :: dbug_flag = 0 - +#ifdef SMARTREDIS + type(client_type) :: sr_client +#endif character(*), parameter :: u_FILE_u = & __FILE__ From c9591d120332eb1dfe670db61acb53c5378a193b Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 16 Jul 2021 10:32:45 -0600 Subject: [PATCH 02/10] initialize a SmartRedis interface if requested --- drivers/cime/esmApp.F90 | 4 ++-- drivers/cime/esm_utils_mod.F90 | 6 ------ nuopc_cap_share/nuopc_shr_methods.F90 | 8 ++++++-- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/drivers/cime/esmApp.F90 b/drivers/cime/esmApp.F90 index 307c7b542..8994e865f 100644 --- a/drivers/cime/esmApp.F90 +++ b/drivers/cime/esmApp.F90 @@ -18,7 +18,7 @@ program esmApp use shr_pio_mod, only : shr_pio_init1 use shr_sys_mod, only : shr_sys_abort #ifdef SMARTREDIS - use esm_utils_mod , only : sr_client + use nuopc_shr_methods, only : sr_client #endif implicit none @@ -47,7 +47,7 @@ program esmApp #endif COMP_COMM = MPI_COMM_WORLD #ifdef SMARTREDIS - call sr_client%initialize(.true.) + call sr_client%initialize(.false.) #endif !----------------------------------------------------------------------------- ! Initialize PIO diff --git a/drivers/cime/esm_utils_mod.F90 b/drivers/cime/esm_utils_mod.F90 index 73b77f668..99a3f1999 100644 --- a/drivers/cime/esm_utils_mod.F90 +++ b/drivers/cime/esm_utils_mod.F90 @@ -1,16 +1,10 @@ module esm_utils_mod -#ifdef SMARTREDIS - use smartredis_client, only : client_type -#endif implicit none public logical :: mastertask integer :: logunit integer :: dbug_flag = 0 -#ifdef SMARTREDIS - type(client_type) :: sr_client -#endif character(*), parameter :: u_FILE_u = & __FILE__ diff --git a/nuopc_cap_share/nuopc_shr_methods.F90 b/nuopc_cap_share/nuopc_shr_methods.F90 index 8cbf91056..1586a9106 100644 --- a/nuopc_cap_share/nuopc_shr_methods.F90 +++ b/nuopc_cap_share/nuopc_shr_methods.F90 @@ -1,5 +1,7 @@ module nuopc_shr_methods - +#ifdef SMARTREDIS + use smartredis_client, only : client_type +#endif use ESMF , only : operator(<), operator(/=), operator(+) use ESMF , only : operator(-), operator(*) , operator(>=) use ESMF , only : operator(<=), operator(>), operator(==) @@ -68,6 +70,9 @@ module nuopc_shr_methods character(len=1024) :: msgString character(len=*), parameter :: u_FILE_u = & __FILE__ +#ifdef SMARTREDIS + type(client_type), public :: sr_client +#endif !=============================================================================== contains @@ -222,7 +227,6 @@ subroutine state_getscalar(state, scalar_id, scalar_value, flds_scalar_name, fld call ESMF_VMGet(vm, localPet=mytask, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return - call ESMF_StateGet(State, itemName=trim(flds_scalar_name), field=field, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return From a7e7811e13a6db89e4fa1a890eeec5356926a219 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 16 Jul 2021 10:34:16 -0600 Subject: [PATCH 03/10] add a testmod --- cime_config/testdefs/testmods_dirs/drv/smartsim/shell_commands | 1 + 1 file changed, 1 insertion(+) create mode 100644 cime_config/testdefs/testmods_dirs/drv/smartsim/shell_commands diff --git a/cime_config/testdefs/testmods_dirs/drv/smartsim/shell_commands b/cime_config/testdefs/testmods_dirs/drv/smartsim/shell_commands new file mode 100644 index 000000000..41e0d6658 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/drv/smartsim/shell_commands @@ -0,0 +1 @@ +./xmlchange USE_SMARTSIM=TRUE From a8f702ff39315ba779676f0c27c754023b72f23e Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 19 Jul 2021 14:25:52 -0600 Subject: [PATCH 04/10] remove ifdef and use stubs --- nuopc_cap_share/nuopc_shr_methods.F90 | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/nuopc_cap_share/nuopc_shr_methods.F90 b/nuopc_cap_share/nuopc_shr_methods.F90 index 1586a9106..48a9ff1a2 100644 --- a/nuopc_cap_share/nuopc_shr_methods.F90 +++ b/nuopc_cap_share/nuopc_shr_methods.F90 @@ -1,7 +1,5 @@ module nuopc_shr_methods -#ifdef SMARTREDIS - use smartredis_client, only : client_type -#endif + use smartredis_client, only : client_type use ESMF , only : operator(<), operator(/=), operator(+) use ESMF , only : operator(-), operator(*) , operator(>=) use ESMF , only : operator(<=), operator(>), operator(==) @@ -38,10 +36,14 @@ module nuopc_shr_methods public :: state_diagnose public :: alarmInit public :: chkerr - private :: timeInit private :: field_getfldptr +#ifdef SMARTREDIS + logical, public :: use_smartredis = .true. +#else + logical, public :: use_smartredis = .false. +#endif ! Clock and alarm options character(len=*), private, parameter :: & optNONE = "none" , & @@ -70,9 +72,9 @@ module nuopc_shr_methods character(len=1024) :: msgString character(len=*), parameter :: u_FILE_u = & __FILE__ -#ifdef SMARTREDIS + type(client_type), public :: sr_client -#endif + !=============================================================================== contains From 4b59380433b3715f13add84d6ee1e28f60532d93 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 20 Jul 2021 16:19:42 -0600 Subject: [PATCH 05/10] add CREATE_SMARTSIM_CLUSTER option --- cime_config/config_component.xml | 33 +++++++++++++-------- cime_config/namelist_definition_drv.xml | 38 ++++++++++++++++++++----- drivers/cime/esmApp.F90 | 23 ++++++++++----- 3 files changed, 68 insertions(+), 26 deletions(-) diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index a99397088..0bb0b4cd8 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -669,7 +669,7 @@ build_def env_build.xml Output root directory for each machine. - Base directory for build and run directories. + Base directory for build and run directories. @@ -1003,9 +1003,9 @@ env_run.xml Determines what ESMF log files (if any) are generated when - USE_ESMF_LIB is TRUE. + USE_ESMF_LIB is TRUE. ESMF_LOGKIND_SINGLE: Use a single log file, combining messages from - all of the PETs. Not supported on some platforms. + all of the PETs. Not supported on some platforms. ESMF_LOGKIND_MULTI: Use multiple log files -- one per PET. ESMF_LOGKIND_NONE: Do not issue messages to a log file. By default, no ESMF log files are generated. @@ -1046,8 +1046,8 @@ run_flags env_run.xml Turns on component barriers for component timing. - This variable is for testing and debugging only and should never - be set for a production run. + This variable is for testing and debugging only and should never + be set for a production run. @@ -1255,7 +1255,7 @@ run_domain env_run.xml Latitude of grid location, in single column mode interpolate datasets to this location - in single point mode assume all datasets are at this location + in single point mode assume all datasets are at this location @@ -1264,7 +1264,7 @@ run_domain env_run.xml Longitude of grid location, in single column mode interpolate datasets to this location - in single point mode assume all datasets are at this location + in single point mode assume all datasets are at this location @@ -1273,10 +1273,10 @@ run_domain env_run.xml used only if if PTS_LAT and PTS_LON are greater than or - equal to 0. If this is the case then if PTS_DOMAINFILE is not - equal to UNSET a nearest neighbor search of PTS_DOMAINFILE using - PTS_LAT and PTS_LON will be done and the component mesh will have - this nearest neighbor value. + equal to 0. If this is the case then if PTS_DOMAINFILE is not + equal to UNSET a nearest neighbor search of PTS_DOMAINFILE using + PTS_LAT and PTS_LON will be done and the component mesh will have + this nearest neighbor value. @@ -1947,7 +1947,7 @@ pio rearranger communication max pending requests (io2comp) : -2 implies that CIME internally calculates the value ( = 64), -1 implies no bound on max pending requests - 0 implies that MPI_ALLTOALL will be used + 0 implies that MPI_ALLTOALL will be used @@ -2537,6 +2537,15 @@ if true, link with the HPE SmartRedis library + + logical + TRUE,FALSE + FALSE + run_debug + env_build.xml + Set to true if SmartSim database spans multiple nodes + + diff --git a/cime_config/namelist_definition_drv.xml b/cime_config/namelist_definition_drv.xml index 71eca18ec..36393571c 100644 --- a/cime_config/namelist_definition_drv.xml +++ b/cime_config/namelist_definition_drv.xml @@ -289,6 +289,18 @@ + + logical + external_interface + nuopc_var + + Link in the CrayLabs SmartRedis code https://github.com/CrayLabs/SmartRedis for use with SmartSim https://github.com/CrayLabs/SmartSim + + + $USE_SMARTSIM + + + @@ -987,11 +999,11 @@ 'on': always do this renormalization 'off': never do this renormalization (see WARNING below) 'on_if_glc_coupled_fluxes': Determine at runtime whether to do this renormalization. - Does the renormalization if we're running a two-way-coupled glc that sends fluxes - to other components (which is the case where we need conservation). - Does NOT do the renormalization if we're running a one-way-coupled glc, or if - we're running a glc-only compset (T compsets). - (In these cases, conservation is not important.) + Does the renormalization if we're running a two-way-coupled glc that sends fluxes + to other components (which is the case where we need conservation). + Does NOT do the renormalization if we're running a one-way-coupled glc, or if + we're running a glc-only compset (T compsets). + (In these cases, conservation is not important.) Only used if running with a prognostic GLC component. @@ -3322,9 +3334,9 @@ PELAYOUT_attributes Determines what ESMF log files (if any) are generated when - USE_ESMF_LIB is TRUE. + USE_ESMF_LIB is TRUE. ESMF_LOGKIND_SINGLE: Use a single log file, combining messages from - all of the PETs. Not supported on some platforms. + all of the PETs. Not supported on some platforms. ESMF_LOGKIND_MULTI: Use multiple log files — one per PET. ESMF_LOGKIND_NONE: Do not issue messages to a log file. By default, no ESMF log files are generated. @@ -3350,6 +3362,18 @@ + + logical + smartsim + smartsim_inparm + + If the SmartSim Database spans more than one node set this true. + + + $CREATE_SMARTSIM_CLUSTER + + + diff --git a/drivers/cime/esmApp.F90 b/drivers/cime/esmApp.F90 index 8994e865f..415098b30 100644 --- a/drivers/cime/esmApp.F90 +++ b/drivers/cime/esmApp.F90 @@ -17,9 +17,7 @@ program esmApp use ensemble_driver, only : SetServices use shr_pio_mod, only : shr_pio_init1 use shr_sys_mod, only : shr_sys_abort -#ifdef SMARTREDIS use nuopc_shr_methods, only : sr_client -#endif implicit none @@ -33,9 +31,10 @@ program esmApp integer :: fileunit integer :: provided type(ESMF_VM) :: vm + logical :: create_smartsim_cluster = .false. namelist /debug_inparm / create_esmf_pet_files - + namelist /smartsim_inparm/ create_smartsim_cluster !----------------------------------------------------------------------------- ! Initiallize MPI !----------------------------------------------------------------------------- @@ -46,9 +45,7 @@ program esmApp call MPI_init(rc) #endif COMP_COMM = MPI_COMM_WORLD -#ifdef SMARTREDIS - call sr_client%initialize(.false.) -#endif + !----------------------------------------------------------------------------- ! Initialize PIO !----------------------------------------------------------------------------- @@ -71,7 +68,7 @@ program esmApp open(newunit=fileunit, status="old", file="drv_in") read(fileunit, debug_inparm, iostat=ier) if (ier > 0) then - call shr_sys_abort('esmApp: error reading in debug_inparm namelist from drv_in') + call shr_sys_abort('esmApp: error reading in debug_inparm namelist from drv_in') end if close(fileunit) end if @@ -90,6 +87,18 @@ program esmApp file=__FILE__)) & call ESMF_Finalize(endflag=ESMF_END_ABORT) + ! Initialize the CrayLabs SmartRedis client + if (iam==0) then + open(newunit=fileunit, status="old", file="drv_in") + read(fileunit, smartsim_inparm, iostat=ier) + if (ier > 0) then + call shr_sys_abort('esmApp: error reading in smartsim_inparm namelist from drv_in') + end if + close(fileunit) + end if + call mpi_bcast (create_smartsim_cluster, 1, MPI_LOGICAL, 0, COMP_COMM, ier) + call sr_client%initialize(create_smartsim_cluster) + call ESMF_VMGet(vm, mpiCommunicator=COMP_COMM, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & From 74594984017f373552bd5bdda75daef2cc076ef0 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 21 Jul 2021 09:49:36 -0600 Subject: [PATCH 06/10] add use_redis logical to drv_in --- cime_config/buildnml | 2 +- cime_config/config_component.xml | 6 ++--- cime_config/namelist_definition_drv.xml | 24 ++++++++--------- drivers/cime/esmApp.F90 | 35 +++++++++++++++---------- nuopc_cap_share/nuopc_shr_methods.F90 | 5 +--- 5 files changed, 38 insertions(+), 34 deletions(-) diff --git a/cime_config/buildnml b/cime_config/buildnml index 28e83bbd9..d199919bf 100755 --- a/cime_config/buildnml +++ b/cime_config/buildnml @@ -220,7 +220,7 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files): # (2) Write namelist file drv_in and initial input dataset list. #-------------------------------- namelist_file = os.path.join(confdir, "drv_in") - drv_namelist_groups = ["papi_inparm", "pio_default_inparm", "prof_inparm", "debug_inparm"] + drv_namelist_groups = ["smartsim_inparm", "papi_inparm", "pio_default_inparm", "prof_inparm", "debug_inparm"] nmlgen.write_output_file(namelist_file, data_list_path=data_list_path, groups=drv_namelist_groups) #-------------------------------- diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index 0bb0b4cd8..ac95cf48c 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -2532,7 +2532,7 @@ logical TRUE,FALSE FALSE - run_debug + smartsim env_build.xml if true, link with the HPE SmartRedis library @@ -2541,8 +2541,8 @@ logical TRUE,FALSE FALSE - run_debug - env_build.xml + smartsim + env_run.xml Set to true if SmartSim database spans multiple nodes diff --git a/cime_config/namelist_definition_drv.xml b/cime_config/namelist_definition_drv.xml index 36393571c..5ce00410a 100644 --- a/cime_config/namelist_definition_drv.xml +++ b/cime_config/namelist_definition_drv.xml @@ -289,18 +289,6 @@ - - logical - external_interface - nuopc_var - - Link in the CrayLabs SmartRedis code https://github.com/CrayLabs/SmartRedis for use with SmartSim https://github.com/CrayLabs/SmartSim - - - $USE_SMARTSIM - - - @@ -3362,6 +3350,18 @@ + + logical + smartsim + smartsim_inparm + + Link in the CrayLabs SmartRedis code https://github.com/CrayLabs/SmartRedis for use with SmartSim https://github.com/CrayLabs/SmartSim + + + $USE_SMARTSIM + + + logical smartsim diff --git a/drivers/cime/esmApp.F90 b/drivers/cime/esmApp.F90 index 415098b30..fc9d5747e 100644 --- a/drivers/cime/esmApp.F90 +++ b/drivers/cime/esmApp.F90 @@ -17,7 +17,7 @@ program esmApp use ensemble_driver, only : SetServices use shr_pio_mod, only : shr_pio_init1 use shr_sys_mod, only : shr_sys_abort - use nuopc_shr_methods, only : sr_client + use nuopc_shr_methods, only : sr_client, use_smartredis implicit none @@ -34,7 +34,7 @@ program esmApp logical :: create_smartsim_cluster = .false. namelist /debug_inparm / create_esmf_pet_files - namelist /smartsim_inparm/ create_smartsim_cluster + namelist /smartsim_inparm/ use_smartredis, create_smartsim_cluster !----------------------------------------------------------------------------- ! Initiallize MPI !----------------------------------------------------------------------------- @@ -87,18 +87,6 @@ program esmApp file=__FILE__)) & call ESMF_Finalize(endflag=ESMF_END_ABORT) - ! Initialize the CrayLabs SmartRedis client - if (iam==0) then - open(newunit=fileunit, status="old", file="drv_in") - read(fileunit, smartsim_inparm, iostat=ier) - if (ier > 0) then - call shr_sys_abort('esmApp: error reading in smartsim_inparm namelist from drv_in') - end if - close(fileunit) - end if - call mpi_bcast (create_smartsim_cluster, 1, MPI_LOGICAL, 0, COMP_COMM, ier) - call sr_client%initialize(create_smartsim_cluster) - call ESMF_VMGet(vm, mpiCommunicator=COMP_COMM, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & @@ -113,6 +101,25 @@ program esmApp file=__FILE__)) & call ESMF_Finalize(endflag=ESMF_END_ABORT) + !----------------------------------------------------------------------------- + ! Initialize the CrayLabs SmartRedis client + !----------------------------------------------------------------------------- + + if (iam==0) then + open(newunit=fileunit, status="old", file="drv_in") + read(fileunit, smartsim_inparm, iostat=ier) + if (ier > 0) then + call shr_sys_abort('esmApp: error reading in smartsim_inparm namelist from drv_in') + end if + close(fileunit) + end if + call mpi_bcast (use_smartredis, 1, MPI_LOGICAL, 0, COMP_COMM, ier) + call mpi_bcast (create_smartsim_cluster, 1, MPI_LOGICAL, 0, COMP_COMM, ier) + if (use_smartredis) then + call ESMF_Logwrite("Using SmartSim interface", ESMF_LOGMSG_INFO, rc=rc) + call sr_client%initialize(create_smartsim_cluster) + endif + !----------------------------------------------------------------------------- ! Operate on the NUOPC Field dictionary !----------------------------------------------------------------------------- diff --git a/nuopc_cap_share/nuopc_shr_methods.F90 b/nuopc_cap_share/nuopc_shr_methods.F90 index 48a9ff1a2..1dd9b734d 100644 --- a/nuopc_cap_share/nuopc_shr_methods.F90 +++ b/nuopc_cap_share/nuopc_shr_methods.F90 @@ -39,11 +39,8 @@ module nuopc_shr_methods private :: timeInit private :: field_getfldptr -#ifdef SMARTREDIS - logical, public :: use_smartredis = .true. -#else logical, public :: use_smartredis = .false. -#endif + ! Clock and alarm options character(len=*), private, parameter :: & optNONE = "none" , & From 3d83c68f60b9a5026356ad702dc9a06865bc29d9 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 22 Jul 2021 16:22:46 -0600 Subject: [PATCH 07/10] add documentation --- drivers/cime/esmApp.F90 | 14 +++++++++++++- nuopc_cap_share/nuopc_shr_methods.F90 | 14 +++++++++++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/drivers/cime/esmApp.F90 b/drivers/cime/esmApp.F90 index fc9d5747e..819d8b7ef 100644 --- a/drivers/cime/esmApp.F90 +++ b/drivers/cime/esmApp.F90 @@ -17,6 +17,12 @@ program esmApp use ensemble_driver, only : SetServices use shr_pio_mod, only : shr_pio_init1 use shr_sys_mod, only : shr_sys_abort + ! + ! The CrayLabs SmartSim interface is provided in directory share https://github.com/ESCOMP/CESM_share + ! Please see file cime/tools/smartsim/README.md for a complete explanation of the CESM interface to smartsim + ! create_smartsim_cluster is set to true if the database is using 3 or more nodes, false if its using 1 and + ! (in the pbs interface at least) 2 is not allowed. + ! use nuopc_shr_methods, only : sr_client, use_smartredis implicit none @@ -34,6 +40,11 @@ program esmApp logical :: create_smartsim_cluster = .false. namelist /debug_inparm / create_esmf_pet_files + ! + ! The CrayLabs SmartSim interface is provided in directory share https://github.com/ESCOMP/CESM_share + ! Please see file cime/tools/smartsim/README.md for a complete explanation of the CESM interface to smartsim + ! The use_smartredis variable is set in file drv_in and if true the variable sr_client is initialized in esmApp.F90 + ! namelist /smartsim_inparm/ use_smartredis, create_smartsim_cluster !----------------------------------------------------------------------------- ! Initiallize MPI @@ -102,7 +113,8 @@ program esmApp call ESMF_Finalize(endflag=ESMF_END_ABORT) !----------------------------------------------------------------------------- - ! Initialize the CrayLabs SmartRedis client + ! Initialize the CrayLabs SmartRedis client, a stub is provided in share if + ! smartsim is not used. This client shall be used by all component models. !----------------------------------------------------------------------------- if (iam==0) then diff --git a/nuopc_cap_share/nuopc_shr_methods.F90 b/nuopc_cap_share/nuopc_shr_methods.F90 index 1dd9b734d..68c8f8a07 100644 --- a/nuopc_cap_share/nuopc_shr_methods.F90 +++ b/nuopc_cap_share/nuopc_shr_methods.F90 @@ -1,5 +1,4 @@ module nuopc_shr_methods - use smartredis_client, only : client_type use ESMF , only : operator(<), operator(/=), operator(+) use ESMF , only : operator(-), operator(*) , operator(>=) use ESMF , only : operator(<=), operator(>), operator(==) @@ -23,6 +22,11 @@ module nuopc_shr_methods use shr_kind_mod , only : r8 => shr_kind_r8, cl=>shr_kind_cl, cs=>shr_kind_cs use shr_sys_mod , only : shr_sys_abort use shr_file_mod , only : shr_file_setlogunit, shr_file_getLogUnit + ! + ! The CrayLabs SmartSim interface is provided in directory share https://github.com/ESCOMP/CESM_share + ! Please see file cime/tools/smartsim/README.md for a complete explanation of the CESM interface to smartsim + ! + use smartredis_client, only : client_type implicit none private @@ -39,7 +43,6 @@ module nuopc_shr_methods private :: timeInit private :: field_getfldptr - logical, public :: use_smartredis = .false. ! Clock and alarm options character(len=*), private, parameter :: & @@ -69,7 +72,12 @@ module nuopc_shr_methods character(len=1024) :: msgString character(len=*), parameter :: u_FILE_u = & __FILE__ - + ! + ! The CrayLabs SmartSim interface is provided in directory share https://github.com/ESCOMP/CESM_share + ! Please see file cime/tools/smartsim/README.md for a complete explanation of the CESM interface to smartsim + ! The use_smartredis variable is set in file drv_in and if true the variable sr_client is initialized in esmApp.F90 + ! + logical, public :: use_smartredis = .false. type(client_type), public :: sr_client From fadfe82fe10b18da8221d7fdad5bca2e08ea6483 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 22 Jul 2021 16:38:29 -0600 Subject: [PATCH 08/10] add a test --- cime_config/testdefs/testlist_drv.xml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cime_config/testdefs/testlist_drv.xml b/cime_config/testdefs/testlist_drv.xml index d255baa18..9cf2ff9b4 100644 --- a/cime_config/testdefs/testlist_drv.xml +++ b/cime_config/testdefs/testlist_drv.xml @@ -110,6 +110,19 @@ + + + + + + + + + + + + + From 704026dd1e93ffaf04d785fae147ff2aceed42fc Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 22 Jul 2021 17:00:33 -0600 Subject: [PATCH 09/10] Revert "add a test" This reverts commit fadfe82fe10b18da8221d7fdad5bca2e08ea6483. --- cime_config/testdefs/testlist_drv.xml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/cime_config/testdefs/testlist_drv.xml b/cime_config/testdefs/testlist_drv.xml index 9cf2ff9b4..d255baa18 100644 --- a/cime_config/testdefs/testlist_drv.xml +++ b/cime_config/testdefs/testlist_drv.xml @@ -110,19 +110,6 @@ - - - - - - - - - - - - - From 1ab44be0bb405327d8be609709a839d418204f74 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 16 Aug 2021 14:40:21 -0600 Subject: [PATCH 10/10] add log message to driver log about smartsim --- drivers/cime/esm.F90 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/cime/esm.F90 b/drivers/cime/esm.F90 index e1a18f135..4f2a02da0 100644 --- a/drivers/cime/esm.F90 +++ b/drivers/cime/esm.F90 @@ -5,7 +5,6 @@ module ESM !----------------------------------------------------------------------------- use shr_kind_mod , only : r8=>shr_kind_r8, cl=>shr_kind_cl, cs=>shr_kind_cs - use shr_log_mod , only : shrlogunit=> shr_log_unit use shr_sys_mod , only : shr_sys_abort use shr_mpi_mod , only : shr_mpi_bcast use shr_mem_mod , only : shr_mem_init @@ -118,6 +117,7 @@ subroutine SetModelServices(driver, rc) use NUOPC , only : NUOPC_CompSetInternalEntryPoint, NUOPC_CompAttributeGet use NUOPC , only : NUOPC_CompAttributeAdd, NUOPC_CompAttributeSet use NUOPC_Driver , only : NUOPC_DriverAddComp, NUOPC_DriverGetComp + use nuopc_shr_methods, only : use_smartredis ! input/output variables type(ESMF_GridComp) :: driver @@ -217,6 +217,10 @@ subroutine SetModelServices(driver, rc) write(logunit,*) trim(meminitstr) end if + if (mastertask .and. use_smartredis) then + write(logunit,*) "Using CrayLabs SmartRedis interface" + endif + !------------------------------------------- ! Timer initialization (has to be after pelayouts are determined) !-------------------------------------------