diff --git a/cime_config/buildexe b/cime_config/buildexe index 52640d30b..3e9b7b091 100755 --- a/cime_config/buildexe +++ b/cime_config/buildexe @@ -40,6 +40,7 @@ def _main_func(): atm_model = case.get_value("COMP_ATM") gmake_args = get_standard_makefile_args(case) esmf_aware_threading = case.get_value("ESMF_AWARE_THREADING") + use_smartsim = case.get_value("USE_SMARTSIM") # Determine valid components valid_comps = [] @@ -77,6 +78,10 @@ def _main_func(): if esmf_aware_threading: gmake_args += " USER_CPPDEFS=-DESMF_AWARE_THREADING" + if use_smartsim: + gmake_args += " USE_SMARTSIM=TRUE" + + gmake_args += " IAC_PRESENT=FALSE" expect((num_esp is None) or (int(num_esp) == 1), "ESP component restricted to one instance") @@ -109,7 +114,7 @@ def _main_func(): .format(gmake, gmake_j, exename, gmake_args, makefile) - rc, out, err = run_cmd(cmd,from_dir=bld_root) + rc, out, err = run_cmd(cmd,from_dir=bld_root, verbose=True) expect(rc==0,"Command {} failed rc={}\nout={}\nerr={}".format(cmd,rc,out,err)) logger.info(out) diff --git a/cime_config/buildnml b/cime_config/buildnml index 28e83bbd9..d199919bf 100755 --- a/cime_config/buildnml +++ b/cime_config/buildnml @@ -220,7 +220,7 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files): # (2) Write namelist file drv_in and initial input dataset list. #-------------------------------- namelist_file = os.path.join(confdir, "drv_in") - drv_namelist_groups = ["papi_inparm", "pio_default_inparm", "prof_inparm", "debug_inparm"] + drv_namelist_groups = ["smartsim_inparm", "papi_inparm", "pio_default_inparm", "prof_inparm", "debug_inparm"] nmlgen.write_output_file(namelist_file, data_list_path=data_list_path, groups=drv_namelist_groups) #-------------------------------- diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index ba73c96d6..ac95cf48c 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -669,7 +669,7 @@ build_def env_build.xml Output root directory for each machine. - Base directory for build and run directories. + Base directory for build and run directories. @@ -1003,9 +1003,9 @@ env_run.xml Determines what ESMF log files (if any) are generated when - USE_ESMF_LIB is TRUE. + USE_ESMF_LIB is TRUE. ESMF_LOGKIND_SINGLE: Use a single log file, combining messages from - all of the PETs. Not supported on some platforms. + all of the PETs. Not supported on some platforms. ESMF_LOGKIND_MULTI: Use multiple log files -- one per PET. ESMF_LOGKIND_NONE: Do not issue messages to a log file. By default, no ESMF log files are generated. @@ -1046,8 +1046,8 @@ run_flags env_run.xml Turns on component barriers for component timing. - This variable is for testing and debugging only and should never - be set for a production run. + This variable is for testing and debugging only and should never + be set for a production run. @@ -1255,7 +1255,7 @@ run_domain env_run.xml Latitude of grid location, in single column mode interpolate datasets to this location - in single point mode assume all datasets are at this location + in single point mode assume all datasets are at this location @@ -1264,7 +1264,7 @@ run_domain env_run.xml Longitude of grid location, in single column mode interpolate datasets to this location - in single point mode assume all datasets are at this location + in single point mode assume all datasets are at this location @@ -1273,10 +1273,10 @@ run_domain env_run.xml used only if if PTS_LAT and PTS_LON are greater than or - equal to 0. If this is the case then if PTS_DOMAINFILE is not - equal to UNSET a nearest neighbor search of PTS_DOMAINFILE using - PTS_LAT and PTS_LON will be done and the component mesh will have - this nearest neighbor value. + equal to 0. If this is the case then if PTS_DOMAINFILE is not + equal to UNSET a nearest neighbor search of PTS_DOMAINFILE using + PTS_LAT and PTS_LON will be done and the component mesh will have + this nearest neighbor value. @@ -1947,7 +1947,7 @@ pio rearranger communication max pending requests (io2comp) : -2 implies that CIME internally calculates the value ( = 64), -1 implies no bound on max pending requests - 0 implies that MPI_ALLTOALL will be used + 0 implies that MPI_ALLTOALL will be used @@ -2528,6 +2528,24 @@ if true, create ESMF PET log files even if there is no error encountered + + logical + TRUE,FALSE + FALSE + smartsim + env_build.xml + if true, link with the HPE SmartRedis library + + + + logical + TRUE,FALSE + FALSE + smartsim + env_run.xml + Set to true if SmartSim database spans multiple nodes + + diff --git a/cime_config/namelist_definition_drv.xml b/cime_config/namelist_definition_drv.xml index 71eca18ec..5ce00410a 100644 --- a/cime_config/namelist_definition_drv.xml +++ b/cime_config/namelist_definition_drv.xml @@ -987,11 +987,11 @@ 'on': always do this renormalization 'off': never do this renormalization (see WARNING below) 'on_if_glc_coupled_fluxes': Determine at runtime whether to do this renormalization. - Does the renormalization if we're running a two-way-coupled glc that sends fluxes - to other components (which is the case where we need conservation). - Does NOT do the renormalization if we're running a one-way-coupled glc, or if - we're running a glc-only compset (T compsets). - (In these cases, conservation is not important.) + Does the renormalization if we're running a two-way-coupled glc that sends fluxes + to other components (which is the case where we need conservation). + Does NOT do the renormalization if we're running a one-way-coupled glc, or if + we're running a glc-only compset (T compsets). + (In these cases, conservation is not important.) Only used if running with a prognostic GLC component. @@ -3322,9 +3322,9 @@ PELAYOUT_attributes Determines what ESMF log files (if any) are generated when - USE_ESMF_LIB is TRUE. + USE_ESMF_LIB is TRUE. ESMF_LOGKIND_SINGLE: Use a single log file, combining messages from - all of the PETs. Not supported on some platforms. + all of the PETs. Not supported on some platforms. ESMF_LOGKIND_MULTI: Use multiple log files — one per PET. ESMF_LOGKIND_NONE: Do not issue messages to a log file. By default, no ESMF log files are generated. @@ -3350,6 +3350,30 @@ + + logical + smartsim + smartsim_inparm + + Link in the CrayLabs SmartRedis code https://github.com/CrayLabs/SmartRedis for use with SmartSim https://github.com/CrayLabs/SmartSim + + + $USE_SMARTSIM + + + + + logical + smartsim + smartsim_inparm + + If the SmartSim Database spans more than one node set this true. + + + $CREATE_SMARTSIM_CLUSTER + + + diff --git a/cime_config/testdefs/testmods_dirs/drv/smartsim/shell_commands b/cime_config/testdefs/testmods_dirs/drv/smartsim/shell_commands new file mode 100644 index 000000000..41e0d6658 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/drv/smartsim/shell_commands @@ -0,0 +1 @@ +./xmlchange USE_SMARTSIM=TRUE diff --git a/drivers/cime/ensemble_driver.F90 b/drivers/cime/ensemble_driver.F90 index 8ddbb727f..29f58f2d1 100644 --- a/drivers/cime/ensemble_driver.F90 +++ b/drivers/cime/ensemble_driver.F90 @@ -134,7 +134,7 @@ subroutine SetModelServices(ensemble_driver, rc) call ReadAttributes(ensemble_driver, config, "CLOCK_attributes::", rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return - call NUOPC_CompAttributeGet(ensemble_driver, 'calendar', calendar, rc=rc) + call NUOPC_CompAttributeGet(ensemble_driver, 'calendar', calendar, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return if (calendar == 'NO_LEAP') then call ESMF_CalendarSetDefault(ESMF_CALKIND_NOLEAP, rc=rc) @@ -247,7 +247,7 @@ subroutine SetModelServices(ensemble_driver, rc) call ReadAttributes(driver, config, "DRV_modelio"//trim(inst_suffix)//"::", rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return - ! Set the driver log to the driver task 0 + ! Set the driver log to the driver task 0 if (mod(localPet, ntasks_per_member) == 0) then call NUOPC_CompAttributeGet(driver, name="diro", value=diro, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return diff --git a/drivers/cime/esm.F90 b/drivers/cime/esm.F90 index e1a18f135..4f2a02da0 100644 --- a/drivers/cime/esm.F90 +++ b/drivers/cime/esm.F90 @@ -5,7 +5,6 @@ module ESM !----------------------------------------------------------------------------- use shr_kind_mod , only : r8=>shr_kind_r8, cl=>shr_kind_cl, cs=>shr_kind_cs - use shr_log_mod , only : shrlogunit=> shr_log_unit use shr_sys_mod , only : shr_sys_abort use shr_mpi_mod , only : shr_mpi_bcast use shr_mem_mod , only : shr_mem_init @@ -118,6 +117,7 @@ subroutine SetModelServices(driver, rc) use NUOPC , only : NUOPC_CompSetInternalEntryPoint, NUOPC_CompAttributeGet use NUOPC , only : NUOPC_CompAttributeAdd, NUOPC_CompAttributeSet use NUOPC_Driver , only : NUOPC_DriverAddComp, NUOPC_DriverGetComp + use nuopc_shr_methods, only : use_smartredis ! input/output variables type(ESMF_GridComp) :: driver @@ -217,6 +217,10 @@ subroutine SetModelServices(driver, rc) write(logunit,*) trim(meminitstr) end if + if (mastertask .and. use_smartredis) then + write(logunit,*) "Using CrayLabs SmartRedis interface" + endif + !------------------------------------------- ! Timer initialization (has to be after pelayouts are determined) !------------------------------------------- diff --git a/drivers/cime/esmApp.F90 b/drivers/cime/esmApp.F90 index 1516ffa10..819d8b7ef 100644 --- a/drivers/cime/esmApp.F90 +++ b/drivers/cime/esmApp.F90 @@ -17,6 +17,13 @@ program esmApp use ensemble_driver, only : SetServices use shr_pio_mod, only : shr_pio_init1 use shr_sys_mod, only : shr_sys_abort + ! + ! The CrayLabs SmartSim interface is provided in directory share https://github.com/ESCOMP/CESM_share + ! Please see file cime/tools/smartsim/README.md for a complete explanation of the CESM interface to smartsim + ! create_smartsim_cluster is set to true if the database is using 3 or more nodes, false if its using 1 and + ! (in the pbs interface at least) 2 is not allowed. + ! + use nuopc_shr_methods, only : sr_client, use_smartredis implicit none @@ -30,9 +37,15 @@ program esmApp integer :: fileunit integer :: provided type(ESMF_VM) :: vm + logical :: create_smartsim_cluster = .false. namelist /debug_inparm / create_esmf_pet_files - + ! + ! The CrayLabs SmartSim interface is provided in directory share https://github.com/ESCOMP/CESM_share + ! Please see file cime/tools/smartsim/README.md for a complete explanation of the CESM interface to smartsim + ! The use_smartredis variable is set in file drv_in and if true the variable sr_client is initialized in esmApp.F90 + ! + namelist /smartsim_inparm/ use_smartredis, create_smartsim_cluster !----------------------------------------------------------------------------- ! Initiallize MPI !----------------------------------------------------------------------------- @@ -66,7 +79,7 @@ program esmApp open(newunit=fileunit, status="old", file="drv_in") read(fileunit, debug_inparm, iostat=ier) if (ier > 0) then - call shr_sys_abort('esmApp: error reading in debug_inparm namelist from drv_in') + call shr_sys_abort('esmApp: error reading in debug_inparm namelist from drv_in') end if close(fileunit) end if @@ -99,6 +112,26 @@ program esmApp file=__FILE__)) & call ESMF_Finalize(endflag=ESMF_END_ABORT) + !----------------------------------------------------------------------------- + ! Initialize the CrayLabs SmartRedis client, a stub is provided in share if + ! smartsim is not used. This client shall be used by all component models. + !----------------------------------------------------------------------------- + + if (iam==0) then + open(newunit=fileunit, status="old", file="drv_in") + read(fileunit, smartsim_inparm, iostat=ier) + if (ier > 0) then + call shr_sys_abort('esmApp: error reading in smartsim_inparm namelist from drv_in') + end if + close(fileunit) + end if + call mpi_bcast (use_smartredis, 1, MPI_LOGICAL, 0, COMP_COMM, ier) + call mpi_bcast (create_smartsim_cluster, 1, MPI_LOGICAL, 0, COMP_COMM, ier) + if (use_smartredis) then + call ESMF_Logwrite("Using SmartSim interface", ESMF_LOGMSG_INFO, rc=rc) + call sr_client%initialize(create_smartsim_cluster) + endif + !----------------------------------------------------------------------------- ! Operate on the NUOPC Field dictionary !----------------------------------------------------------------------------- diff --git a/drivers/cime/esm_utils_mod.F90 b/drivers/cime/esm_utils_mod.F90 index f6a4aeb40..99a3f1999 100644 --- a/drivers/cime/esm_utils_mod.F90 +++ b/drivers/cime/esm_utils_mod.F90 @@ -1,12 +1,10 @@ module esm_utils_mod - implicit none public logical :: mastertask integer :: logunit integer :: dbug_flag = 0 - character(*), parameter :: u_FILE_u = & __FILE__ diff --git a/nuopc_cap_share/nuopc_shr_methods.F90 b/nuopc_cap_share/nuopc_shr_methods.F90 index 8cbf91056..68c8f8a07 100644 --- a/nuopc_cap_share/nuopc_shr_methods.F90 +++ b/nuopc_cap_share/nuopc_shr_methods.F90 @@ -1,5 +1,4 @@ module nuopc_shr_methods - use ESMF , only : operator(<), operator(/=), operator(+) use ESMF , only : operator(-), operator(*) , operator(>=) use ESMF , only : operator(<=), operator(>), operator(==) @@ -23,6 +22,11 @@ module nuopc_shr_methods use shr_kind_mod , only : r8 => shr_kind_r8, cl=>shr_kind_cl, cs=>shr_kind_cs use shr_sys_mod , only : shr_sys_abort use shr_file_mod , only : shr_file_setlogunit, shr_file_getLogUnit + ! + ! The CrayLabs SmartSim interface is provided in directory share https://github.com/ESCOMP/CESM_share + ! Please see file cime/tools/smartsim/README.md for a complete explanation of the CESM interface to smartsim + ! + use smartredis_client, only : client_type implicit none private @@ -36,10 +40,10 @@ module nuopc_shr_methods public :: state_diagnose public :: alarmInit public :: chkerr - private :: timeInit private :: field_getfldptr + ! Clock and alarm options character(len=*), private, parameter :: & optNONE = "none" , & @@ -68,6 +72,14 @@ module nuopc_shr_methods character(len=1024) :: msgString character(len=*), parameter :: u_FILE_u = & __FILE__ + ! + ! The CrayLabs SmartSim interface is provided in directory share https://github.com/ESCOMP/CESM_share + ! Please see file cime/tools/smartsim/README.md for a complete explanation of the CESM interface to smartsim + ! The use_smartredis variable is set in file drv_in and if true the variable sr_client is initialized in esmApp.F90 + ! + logical, public :: use_smartredis = .false. + type(client_type), public :: sr_client + !=============================================================================== contains @@ -222,7 +234,6 @@ subroutine state_getscalar(state, scalar_id, scalar_value, flds_scalar_name, fld call ESMF_VMGet(vm, localPet=mytask, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return - call ESMF_StateGet(State, itemName=trim(flds_scalar_name), field=field, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return