code to use HPE SMARTREDIS

cime_config/buildexe

@@ -40,6 +40,7 @@ def _main_func():
         atm_model = case.get_value("COMP_ATM")
         gmake_args = get_standard_makefile_args(case)
         esmf_aware_threading = case.get_value("ESMF_AWARE_THREADING")
+        use_smartsim = case.get_value("USE_SMARTSIM")
 
     # Determine valid components
     valid_comps = []
@@ -77,6 +78,10 @@ def _main_func():
     if esmf_aware_threading:
         gmake_args += " USER_CPPDEFS=-DESMF_AWARE_THREADING"
 
+    if use_smartsim:
+        gmake_args += " USE_SMARTSIM=TRUE"
+
+
     gmake_args += " IAC_PRESENT=FALSE"
     expect((num_esp is None) or (int(num_esp) == 1), "ESP component restricted to one instance")
 
@@ -109,7 +114,7 @@ def _main_func():
         .format(gmake, gmake_j, exename, gmake_args, makefile)
 
 
-    rc, out, err = run_cmd(cmd,from_dir=bld_root)
+    rc, out, err = run_cmd(cmd,from_dir=bld_root, verbose=True)
     expect(rc==0,"Command {} failed rc={}\nout={}\nerr={}".format(cmd,rc,out,err))
     logger.info(out)
 

cime_config/buildnml

@@ -220,7 +220,7 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files):
     # (2) Write namelist file drv_in and initial input dataset list.
     #--------------------------------
     namelist_file = os.path.join(confdir, "drv_in")
-    drv_namelist_groups = ["papi_inparm", "pio_default_inparm", "prof_inparm", "debug_inparm"]
+    drv_namelist_groups = ["smartsim_inparm", "papi_inparm", "pio_default_inparm", "prof_inparm", "debug_inparm"]
     nmlgen.write_output_file(namelist_file, data_list_path=data_list_path, groups=drv_namelist_groups)
 
     #--------------------------------

cime_config/config_component.xml

@@ -669,7 +669,7 @@
     <group>build_def</group>
     <file>env_build.xml</file>
     <desc>Output root directory for each machine.
-                Base directory for build and run directories.
+		Base directory for build and run directories.
     </desc>
   </entry>
 
@@ -1003,9 +1003,9 @@
     <file>env_run.xml</file>
     <desc>
       Determines what ESMF log files (if any) are generated when
-          USE_ESMF_LIB is TRUE.
+	  USE_ESMF_LIB is TRUE.
       ESMF_LOGKIND_SINGLE: Use a single log file, combining messages from
-          all of the PETs. Not supported on some platforms.
+	  all of the PETs. Not supported on some platforms.
       ESMF_LOGKIND_MULTI: Use multiple log files -- one per PET.
       ESMF_LOGKIND_NONE: Do not issue messages to a log file.
       By default, no ESMF log files are generated.
@@ -1046,8 +1046,8 @@
     <group>run_flags</group>
     <file>env_run.xml</file>
     <desc>Turns on component barriers for component timing.
-          This variable is for testing and debugging only and should never
-          be set for a production run.
+	  This variable is for testing and debugging only and should never
+	  be set for a production run.
     </desc>
   </entry>
 
@@ -1255,7 +1255,7 @@
     <group>run_domain</group>
     <file>env_run.xml</file>
     <desc>Latitude of grid location, in single column mode interpolate datasets to this location
-          in single point mode assume all datasets are at this location</desc>
+	  in single point mode assume all datasets are at this location</desc>
   </entry>
 
   <entry id="PTS_LON">
@@ -1264,7 +1264,7 @@
     <group>run_domain</group>
     <file>env_run.xml</file>
     <desc>Longitude of grid location, in single column mode interpolate datasets to this location
-          in single point mode assume all datasets are at this location</desc>
+	  in single point mode assume all datasets are at this location</desc>
   </entry>
 
   <entry id="PTS_DOMAINFILE">
@@ -1273,10 +1273,10 @@
     <group>run_domain</group>
     <file>env_run.xml</file>
     <desc>used only if if PTS_LAT and PTS_LON are greater than or
-          equal to 0.  If this is the case then if PTS_DOMAINFILE is not
-          equal to UNSET a nearest neighbor search of PTS_DOMAINFILE using
-          PTS_LAT and PTS_LON will be done and the component mesh will have
-          this nearest neighbor value. </desc>
+	  equal to 0.  If this is the case then if PTS_DOMAINFILE is not
+	  equal to UNSET a nearest neighbor search of PTS_DOMAINFILE using
+	  PTS_LAT and PTS_LON will be done and the component mesh will have
+	  this nearest neighbor value. </desc>
   </entry>
 
   <!-- ======================================================================= -->
@@ -1947,7 +1947,7 @@
     <desc>pio rearranger communication max pending requests (io2comp) :
        -2 implies that CIME internally calculates the value ( = 64),
        -1 implies no bound on max pending requests
-        0 implies that MPI_ALLTOALL will be used
+	0 implies that MPI_ALLTOALL will be used
     </desc>
   </entry>
 
@@ -2528,6 +2528,24 @@
    <desc>if true, create ESMF PET log files even if there is no error encountered </desc>
  </entry>
 
+ <entry id="USE_SMARTSIM">
+   <type>logical</type>
+   <valid_values>TRUE,FALSE</valid_values>
+   <default_value>FALSE</default_value>
+   <group>smartsim</group>
+   <file>env_build.xml</file>
+   <desc>if true, link with the HPE SmartRedis library  </desc>
+ </entry>
+
+ <entry id="CREATE_SMARTSIM_CLUSTER">
+   <type>logical</type>
+   <valid_values>TRUE,FALSE</valid_values>
+   <default_value>FALSE</default_value>
+   <group>smartsim</group>
+   <file>env_run.xml</file>
+   <desc>Set to true if SmartSim database spans multiple nodes  </desc>
+ </entry>
+
   <!-- ===================================================================== -->
   <!-- Include the AOFLUX calculation for this compset -->
   <!-- ===================================================================== -->

cime_config/namelist_definition_drv.xml

@@ -987,11 +987,11 @@
       'on': always do this renormalization
       'off': never do this renormalization (see WARNING below)
       'on_if_glc_coupled_fluxes': Determine at runtime whether to do this renormalization.
-         Does the renormalization if we're running a two-way-coupled glc that sends fluxes
-         to other components (which is the case where we need conservation).
-         Does NOT do the renormalization if we're running a one-way-coupled glc, or if
-         we're running a glc-only compset (T compsets).
-         (In these cases, conservation is not important.)
+	 Does the renormalization if we're running a two-way-coupled glc that sends fluxes
+	 to other components (which is the case where we need conservation).
+	 Does NOT do the renormalization if we're running a one-way-coupled glc, or if
+	 we're running a glc-only compset (T compsets).
+	 (In these cases, conservation is not important.)
 
       Only used if running with a prognostic GLC component.
 
@@ -3322,9 +3322,9 @@
     <group>PELAYOUT_attributes</group>
     <desc>
       Determines what ESMF log files (if any) are generated when
-          USE_ESMF_LIB is TRUE.
+	  USE_ESMF_LIB is TRUE.
       ESMF_LOGKIND_SINGLE: Use a single log file, combining messages from
-          all of the PETs. Not supported on some platforms.
+	  all of the PETs. Not supported on some platforms.
       ESMF_LOGKIND_MULTI: Use multiple log files — one per PET.
       ESMF_LOGKIND_NONE: Do not issue messages to a log file.
       By default, no ESMF log files are generated.
@@ -3350,6 +3350,30 @@
     </values>
   </entry>
 
+  <entry id="use_smartredis" modify_via_xml="USE_SMARTSIM">
+    <type>logical</type>
+    <category>smartsim</category>
+    <group>smartsim_inparm</group>
+    <desc>
+      Link in the CrayLabs SmartRedis code https://github.com/CrayLabs/SmartRedis for use with SmartSim https://github.com/CrayLabs/SmartSim
+    </desc>
+    <values>
+      <value>$USE_SMARTSIM</value>
+    </values>
+  </entry>
+
+  <entry id="create_smartsim_cluster" modify_via_xml="CREATE_SMARTSIM_CLUSTER">
+    <type>logical</type>
+    <category>smartsim</category>
+    <group>smartsim_inparm</group>
+    <desc>
+      If the SmartSim Database spans more than one node set this true.
+    </desc>
+    <values>
+      <value>$CREATE_SMARTSIM_CLUSTER</value>
+    </values>
+  </entry>
+
   <!-- =========================== -->
   <!-- group prof_inparm           -->
   <!--  in perf_mod.F90            -->

cime_config/testdefs/testmods_dirs/drv/smartsim/shell_commands

@@ -0,0 +1 @@
+./xmlchange USE_SMARTSIM=TRUE

drivers/cime/ensemble_driver.F90

@@ -134,7 +134,7 @@ subroutine SetModelServices(ensemble_driver, rc)
     call ReadAttributes(ensemble_driver, config, "CLOCK_attributes::", rc=rc)
     if (chkerr(rc,__LINE__,u_FILE_u)) return
 
-    call NUOPC_CompAttributeGet(ensemble_driver, 'calendar', calendar, rc=rc) 
+    call NUOPC_CompAttributeGet(ensemble_driver, 'calendar', calendar, rc=rc)
     if (chkerr(rc,__LINE__,u_FILE_u)) return
     if (calendar == 'NO_LEAP') then
        call ESMF_CalendarSetDefault(ESMF_CALKIND_NOLEAP, rc=rc)
@@ -247,7 +247,7 @@ subroutine SetModelServices(ensemble_driver, rc)
           call ReadAttributes(driver, config, "DRV_modelio"//trim(inst_suffix)//"::", rc=rc)
           if (chkerr(rc,__LINE__,u_FILE_u)) return
 
-          ! Set the driver log to the driver task 0 
+          ! Set the driver log to the driver task 0
           if (mod(localPet, ntasks_per_member) == 0) then
              call NUOPC_CompAttributeGet(driver, name="diro", value=diro, rc=rc)
              if (chkerr(rc,__LINE__,u_FILE_u)) return

drivers/cime/esm.F90

@@ -5,7 +5,6 @@ module ESM
   !-----------------------------------------------------------------------------
 
   use shr_kind_mod , only : r8=>shr_kind_r8, cl=>shr_kind_cl, cs=>shr_kind_cs
-  use shr_log_mod  , only : shrlogunit=> shr_log_unit
   use shr_sys_mod  , only : shr_sys_abort
   use shr_mpi_mod  , only : shr_mpi_bcast
   use shr_mem_mod  , only : shr_mem_init
@@ -118,6 +117,7 @@ subroutine SetModelServices(driver, rc)
     use NUOPC        , only : NUOPC_CompSetInternalEntryPoint, NUOPC_CompAttributeGet
     use NUOPC        , only : NUOPC_CompAttributeAdd, NUOPC_CompAttributeSet
     use NUOPC_Driver , only : NUOPC_DriverAddComp, NUOPC_DriverGetComp
+    use nuopc_shr_methods, only : use_smartredis
 
     ! input/output variables
     type(ESMF_GridComp)    :: driver
@@ -217,6 +217,10 @@ subroutine SetModelServices(driver, rc)
        write(logunit,*) trim(meminitstr)
     end if
 
+    if (mastertask .and. use_smartredis) then
+       write(logunit,*) "Using CrayLabs SmartRedis interface"
+    endif
+
     !-------------------------------------------
     ! Timer initialization (has to be after pelayouts are determined)
     !-------------------------------------------

drivers/cime/esmApp.F90

@@ -17,6 +17,13 @@ program esmApp
   use ensemble_driver, only : SetServices
   use shr_pio_mod,     only : shr_pio_init1
   use shr_sys_mod,     only : shr_sys_abort
+  !
+  ! The CrayLabs SmartSim interface is provided in directory share  https://github.com/ESCOMP/CESM_share
+  ! Please see file cime/tools/smartsim/README.md for a complete explanation of the CESM interface to smartsim
+  ! create_smartsim_cluster is set to true if the database is using 3 or more nodes, false if its using 1 and 
+  ! (in the pbs interface at least) 2 is not allowed.
+  !
+  use nuopc_shr_methods, only : sr_client, use_smartredis
 
   implicit none
 
@@ -30,9 +37,15 @@ program esmApp
   integer                 :: fileunit
   integer                 :: provided
   type(ESMF_VM)           :: vm
+  logical                 :: create_smartsim_cluster = .false.
 
   namelist /debug_inparm / create_esmf_pet_files
-
+  !
+  ! The CrayLabs SmartSim interface is provided in directory share  https://github.com/ESCOMP/CESM_share
+  ! Please see file cime/tools/smartsim/README.md for a complete explanation of the CESM interface to smartsim
+  ! The use_smartredis variable is set in file drv_in and if true the variable sr_client is initialized in esmApp.F90
+  !
+  namelist /smartsim_inparm/ use_smartredis, create_smartsim_cluster
   !-----------------------------------------------------------------------------
   ! Initiallize MPI
   !-----------------------------------------------------------------------------
@@ -66,7 +79,7 @@ program esmApp
      open(newunit=fileunit, status="old", file="drv_in")
      read(fileunit, debug_inparm, iostat=ier)
      if (ier > 0) then
-        call shr_sys_abort('esmApp: error reading in debug_inparm namelist from drv_in')
+	call shr_sys_abort('esmApp: error reading in debug_inparm namelist from drv_in')
      end if
      close(fileunit)
   end if
@@ -99,6 +112,26 @@ program esmApp
        file=__FILE__)) &
        call ESMF_Finalize(endflag=ESMF_END_ABORT)
 
+  !-----------------------------------------------------------------------------
+  ! Initialize the CrayLabs SmartRedis client, a stub is provided in share if 
+  ! smartsim is not used.  This client shall be used by all component models.
+  !-----------------------------------------------------------------------------
+
+  if (iam==0) then
+     open(newunit=fileunit, status="old", file="drv_in")
+     read(fileunit, smartsim_inparm, iostat=ier)
+     if (ier > 0) then
+	call shr_sys_abort('esmApp: error reading in smartsim_inparm namelist from drv_in')
+     end if
+     close(fileunit)
+  end if
+  call mpi_bcast (use_smartredis, 1, MPI_LOGICAL, 0, COMP_COMM, ier)
+  call mpi_bcast (create_smartsim_cluster, 1, MPI_LOGICAL, 0, COMP_COMM, ier)
+  if (use_smartredis) then
+     call ESMF_Logwrite("Using SmartSim interface", ESMF_LOGMSG_INFO, rc=rc)
+     call sr_client%initialize(create_smartsim_cluster)
+  endif
+
   !-----------------------------------------------------------------------------
   ! Operate on the NUOPC Field dictionary
   !-----------------------------------------------------------------------------

drivers/cime/esm_utils_mod.F90

@@ -1,12 +1,10 @@
 module esm_utils_mod
-
   implicit none
   public
 
   logical :: mastertask
   integer :: logunit
   integer :: dbug_flag = 0
-
   character(*), parameter :: u_FILE_u = &
        __FILE__
 

nuopc_cap_share/nuopc_shr_methods.F90

@@ -1,5 +1,4 @@
 module nuopc_shr_methods
-
   use ESMF         , only : operator(<), operator(/=), operator(+)
   use ESMF         , only : operator(-), operator(*) , operator(>=)
   use ESMF         , only : operator(<=), operator(>), operator(==)
@@ -23,6 +22,11 @@ module nuopc_shr_methods
   use shr_kind_mod , only : r8 => shr_kind_r8, cl=>shr_kind_cl, cs=>shr_kind_cs
   use shr_sys_mod  , only : shr_sys_abort
   use shr_file_mod , only : shr_file_setlogunit, shr_file_getLogUnit
+  !
+  ! The CrayLabs SmartSim interface is provided in directory share  https://github.com/ESCOMP/CESM_share
+  ! Please see file cime/tools/smartsim/README.md for a complete explanation of the CESM interface to smartsim
+  !
+  use smartredis_client, only : client_type
 
   implicit none
   private
@@ -36,10 +40,10 @@ module nuopc_shr_methods
   public  :: state_diagnose
   public  :: alarmInit
   public  :: chkerr
-
   private :: timeInit
   private :: field_getfldptr
 
+
   ! Clock and alarm options
   character(len=*), private, parameter :: &
        optNONE           = "none"      , &
@@ -68,6 +72,14 @@ module nuopc_shr_methods
   character(len=1024)                   :: msgString
   character(len=*), parameter :: u_FILE_u = &
        __FILE__
+  !
+  ! The CrayLabs SmartSim interface is provided in directory share  https://github.com/ESCOMP/CESM_share
+  ! Please see file cime/tools/smartsim/README.md for a complete explanation of the CESM interface to smartsim
+  ! The use_smartredis variable is set in file drv_in and if true the variable sr_client is initialized in esmApp.F90
+  !
+  logical, public  :: use_smartredis = .false.
+  type(client_type), public :: sr_client
+
 
 !===============================================================================
 contains
@@ -222,7 +234,6 @@ subroutine state_getscalar(state, scalar_id, scalar_value, flds_scalar_name, fld
 
     call ESMF_VMGet(vm, localPet=mytask, rc=rc)
     if (chkerr(rc,__LINE__,u_FILE_u)) return
-
     call ESMF_StateGet(State, itemName=trim(flds_scalar_name), field=field, rc=rc)
     if (chkerr(rc,__LINE__,u_FILE_u)) return