Bring CDEPS inline capability to CMEPS

.gitmodules

@@ -16,8 +16,8 @@
   branch = develop
 [submodule "CMEPS"]
   path = CMEPS-interface/CMEPS
-  url = https://github.com/NOAA-EMC/CMEPS
-  branch = emc/develop
+  url = https://github.com/uturuncoglu/CMEPS
+  branch = feature/cdeps_inline
 [submodule "HYCOM"]
   path = HYCOM-interface/HYCOM
   url = https://github.com/NOAA-EMC/HYCOM-src
@@ -32,8 +32,8 @@
   branch = emc/develop
 [submodule "CDEPS"]
   path = CDEPS-interface/CDEPS
-  url = https://github.com/NOAA-EMC/CDEPS
-  branch = develop
+  url = https://github.com/uturuncoglu/CDEPS
+  branch = feature/mask
 [submodule "GOCART"]
   path = GOCART
   url = https://github.com/GEOS-ESM/GOCART

CMEPS-interface/CMakeLists.txt

@@ -82,6 +82,11 @@ list(APPEND _mediator_files
   CMEPS/mediator/med_phases_post_rof_mod.F90
   CMEPS/mediator/med_phases_post_wav_mod.F90)
 
+if(CDEPS_INLINE)
+  list(APPEND _mediator_files CMEPS/mediator/med_phases_cdeps_mod.F90)
+  add_definitions(-DCDEPS_INLINE)
+endif()
+
 ###############################################################################
 ### Host model for CCPP
 ###############################################################################
@@ -186,6 +191,12 @@ if(OpenMP_Fortran_FOUND)
   target_link_libraries(cmeps PRIVATE OpenMP::OpenMP_Fortran)
 endif()
 
+### Set dependency for CDEPS if inline capability is requested
+if(CDEPS_INLINE)
+  add_dependencies(cmeps cdeps::cdeps)
+  target_link_libraries(cmeps PUBLIC cdeps::cdeps)
+endif()
+
 ###############################################################################
 ### Install
 ###############################################################################

CMakeLists.txt

@@ -71,6 +71,7 @@ set(PARALLEL_NETCDF OFF CACHE BOOL "Enable parallel NetCDF")
 set(JEDI_DRIVER     OFF CACHE BOOL "Enable JEDI as top level driver")
 set(CMEPS_AOFLUX    OFF CACHE BOOL "Enable atmosphere-ocean flux calculation in mediator")
 set(PDLIB           OFF CACHE BOOL "Enable Domain Decomposition in WW3 via PDLIB")
+set(CDEPS_INLINE    OFF CACHE BOOL "Enable CDEPS inline capability")
 
 set(CMAKE_Platform $ENV{CMAKE_Platform})
 if(CMAKE_Platform)
@@ -118,6 +119,7 @@ message("OPENMP ........... ${OPENMP}")
 message("PARALLEL_NETCDF .. ${PARALLEL_NETCDF}")
 message("JEDI_DRIVER ...... ${JEDI_DRIVER}")
 message("CMEPS_AOFLUX ..... ${CMEPS_AOFLUX}")
+message("CDEPS_INLINE ..... ${CDEPS_INLINE}")
 
 message("")
 

cmake/configure_apps.cmake

@@ -70,10 +70,7 @@ endif()
 
 if(APP MATCHES "^(HAFS|HAFSW|HAFS-ALL)$")
   set(CMEPS      ON  CACHE BOOL "Enable CMEPS"               FORCE)
-  if(APP MATCHES "^(HAFS-ALL)$")
-    set(CDEPS    ON  CACHE BOOL "Enable CDEPS"               FORCE)
-    message("Configuring UFS app in HAFS with CDEPS mode")
-  endif()
+  set(CDEPS      ON  CACHE BOOL "Enable CDEPS"               FORCE)
   set(FMS        ON  CACHE BOOL "Enable FMS"                 FORCE)
   set(FV3        ON  CACHE BOOL "Enable FV3"                 FORCE)
   set(STOCH_PHYS ON  CACHE BOOL "Enable Stochastic Physics"  FORCE)

tests/fv3_conf/inline_run.IN

@@ -0,0 +1,8 @@
+mkdir -p INPUT_INLINE
+
+cd INPUT_INLINE
+rsync -arv @[INPUTDATA_ROOT]/CDEPS_INLINE/datm_gfs_mesh.nc .
+rsync -arv @[INPUTDATA_ROOT]/CDEPS_INLINE/ocean_forcings_fixed.nc .
+rsync -arv @[INPUTDATA_ROOT]/CDEPS_INLINE/sst_mesh.nc .
+rsync -arv @[INPUTDATA_ROOT]/CDEPS_INLINE/sst20190829_new.nc .
+cd -

tests/parm/nems.configure.hafs_atm_ocn_wav_inline.IN

@@ -0,0 +1,168 @@
+##############################################
+#####  NEMS Run-Time Configuration File  #####
+##############################################
+
+# ESMF #
+logKindFlag:            ESMF_LOGKIND_MULTI_ON
+globalResourceControl:  true
+
+# EARTH #
+EARTH_component_list: MED ATM OCN WAV
+EARTH_attributes::
+  Verbosity = 0
+::
+
+# MED #
+MED_model:                      @[med_model]
+MED_petlist_bounds:             @[med_petlist_bounds]
+MED_omp_num_threads:            @[med_omp_num_threads]
+MED_attributes::
+  Verbosity = 0
+  Diagnostic = 0
+  ATM_model = @[atm_model]
+  OCN_model = @[ocn_model]
+  WAV_model = @[wav_model]
+  MED_model = cmeps
+  history_n = 6
+  history_option = nhours
+  history_ymd = -999
+  coupling_mode = @[CPLMODE]
+  normalization = none
+  merge_type = copy
+  pio_rearranger = @[pio_rearranger]
+::
+
+# ATM #
+ATM_model:                      @[atm_model]
+ATM_petlist_bounds:             @[atm_petlist_bounds]
+ATM_omp_num_threads:            @[atm_omp_num_threads]
+ATM_attributes::
+  Verbosity = 0
+  Diagnostic = 0
+  mesh_atm = @[MESH_ATM]
+  stop_n = @[FHMAX]
+  stop_option = nhours
+  stop_ymd = -999
+  write_restart_at_endofrun = .true.
+::
+
+# OCN #
+OCN_model:                      @[ocn_model]
+OCN_petlist_bounds:             @[ocn_petlist_bounds]
+OCN_omp_num_threads:            @[ocn_omp_num_threads]
+OCN_attributes::
+  Verbosity = 0
+  Diagnostic = 0
+  cdf_impexp_freq = 3
+  cpl_hour = 0
+  cpl_min = 0
+  cpl_sec = @[coupling_interval_sec]
+  base_dtg = @[SYEAR]@[SMONTH]@[SDAY]@[SHOUR]
+  merge_import = .true.
+  skip_first_import = .true.
+  hycom_arche_output = .false.
+  hyc_esmf_exp_output = .true.
+  hyc_esmf_imp_output = .true.
+  import_diagnostics = .false.
+  import_setting = flexible
+  hyc_impexp_file = nems.configure
+  espc_show_impexp_minmax = .true.
+  ocean_start_dtg = @[OCEAN_START_DTG]
+  start_hour = 0
+  start_min = 0
+  start_sec = 0
+  end_hour = @[FHMAX]
+  end_min = 0
+  end_sec = 0
+::
+
+# WAV #
+WAV_model:                      @[wav_model]
+WAV_petlist_bounds:             @[wav_petlist_bounds]
+WAV_omp_num_threads:            @[wav_omp_num_threads]
+WAV_attributes::
+  Verbosity = 0
+  Diagnostic = 0
+  OverwriteSlice = false
+  merge_import = .true.
+  mesh_wav = @[MESH_WAV]
+  multigrid = @[MULTIGRID]
+  user_sets_restname = true
+::
+
+# Run Sequence #
+runSeq::
+@@[coupling_interval_sec]
+  MED med_phases_cdeps_run
+  MED med_phases_prep_atm
+  MED med_phases_prep_ocn_accum
+  MED med_phases_prep_ocn_avg
+  MED med_phases_prep_wav_accum
+  MED med_phases_prep_wav_avg
+  MED -> ATM :remapMethod=redist
+  MED -> OCN :remapMethod=redist
+  MED -> WAV :remapMethod=redist
+  ATM
+  OCN
+  WAV
+  ATM -> MED :remapMethod=redist
+  OCN -> MED :remapMethod=redist
+  WAV -> MED :remapMethod=redist
+  MED med_phases_post_atm
+  MED med_phases_post_ocn
+  MED med_phases_post_wav
+  MED med_phases_restart_write
+  MED med_phases_history_write
+@
+::
+
+# Other Attributes #
+DRIVER_attributes::
+::
+
+ALLCOMP_attributes::
+  ATM_model = @[atm_model]
+  OCN_model = @[ocn_model]
+  WAV_model = @[wav_model]
+  MED_model = cmeps
+  ScalarFieldCount = 3
+  ScalarFieldIdxGridNX = 1
+  ScalarFieldIdxGridNY = 2
+  ScalarFieldIdxNextSwCday = 3
+  ScalarFieldName = cpl_scalars
+  start_type = startup
+  case_name = ufs.hafs
+  restart_n = 6
+  restart_option = nhours
+  restart_ymd = -999
+  dbug_flag = 6
+  use_coldstart = true
+  orb_eccen = 1.e36
+  orb_iyear = 2000
+  orb_iyear_align = 2000
+  orb_mode = fixed_year
+  orb_mvelp = 1.e36
+  orb_obliq = 1.e36
+  mediator_read_restart = @[USE_COLDSTART]
+  mediator_present = true
+  stop_n = @[FHMAX]
+  stop_option = nhours
+  stop_ymd = -999
+::
+
+ocn_export_fields::
+  'sst'         'sea_surface_temperature'   'K'
+  'mask'        'ocean_mask'                '1'
+  'cpl_scalars' 'cpl_scalars'               '1'
+::
+
+ocn_import_fields::
+  'taux10'      'mean_zonal_moment_flx_atm' 'N_m-2'
+  'tauy10'      'mean_merid_moment_flx_atm' 'N_m-2'
+  'prcp'        'inst_prec_rate'            'kg_m-2_s-1'
+  'swflxd'      'mean_net_sw_flx'           'W_m-2'
+  'lwflxd'      'mean_net_lw_flx'           'W_m-2'
+  'mslprs'      'inst_pres_height_surface'  'Pa'
+  'sensflx'     'mean_sensi_heat_flx'       'W_m-2'
+  'latflx'      'mean_laten_heat_flx'       'W_m-2'
+::

tests/parm/stream.config.IN

@@ -0,0 +1,32 @@
+stream_info:             stream01 stream02
+
+taxmode01:               limit
+mapalgo01:               bilinear
+tInterpAlgo01:           linear
+readMode01:              single
+dtlimit01:               1.5
+stream_offset01:         @[STREAM_OFFSET]
+yearFirst01:             @[SYEAR]
+yearLast01:              @[SYEAR]
+yearAlign01:             @[SYEAR]
+stream_vectors01:        null
+stream_mesh_file01:      @[INLINE_MESH_OCN]
+stream_lev_dimname01:    null
+stream_data_files01:     @[INLINE_STREAM_FILES_OCN]
+stream_data_variables01: "TMPSFC So_t"
+stream_dst_mask01:       1
+
+taxmode02:               limit
+mapalgo02:               bilinear
+tInterpAlgo02:           linear
+readMode02:              single
+dtlimit02:               1.5
+stream_offset02:         @[STREAM_OFFSET]
+yearFirst02:             @[SYEAR]
+yearLast02:              @[SYEAR]
+yearAlign02:             @[SYEAR]
+stream_vectors02:        null
+stream_mesh_file02:      @[INLINE_MESH_ATM]
+stream_lev_dimname02:    null
+stream_data_files02:     @[INLINE_STREAM_FILES_ATM]
+stream_data_variables02: "UFLX_surface Faxa_taux" "VFLX_surface Faxa_tauy" "PRATE_surface Faxa_rain" "NETSW_surface Faxa_swnet" "NETLW_surface Faxa_lwnet" "SHTFL_surface Faxa_sen" "LHTFL_surface Faxa_lat" "PRES_surface Sa_pslv" "UGRD_10maboveground Sa_u10m" "VGRD_10maboveground Sa_v10m"

tests/rt.conf

@@ -246,6 +246,9 @@ RUN | hafs_regional_storm_following_1nest_atm_qr        | - jet s4 cheyenne noaa
 RUN | hafs_regional_storm_following_1nest_atm_ocn       | - jet s4  noaacloud           | baseline |
 RUN | hafs_global_storm_following_1nest_atm             | - jet s4  noaacloud           | baseline |
 
+COMPILE | hafsw | intel | -DAPP=HAFSW -DMOVING_NEST=ON -DCDEPS_INLINE=ON -DCCPP_SUITES=FV3_HAFS_v1_gfdlmp_tedmf,FV3_HAFS_v1_gfdlmp_tedmf_nonsst,FV3_HAFS_v1_thompson_tedmf_gfdlsf -D32BIT=ON |  | fv3 |
+RUN | hafs_regional_atm_ocn_wav_inline                  | - noaacloud                   | baseline |
+
 COMPILE | hafsw_debug | intel | -DAPP=HAFSW -DMOVING_NEST=ON -DCCPP_SUITES=FV3_HAFS_v1_gfdlmp_tedmf_nonsst -D32BIT=ON -DDEBUG=ON | - jet noaacloud s4 | fv3 |
 RUN | hafs_regional_storm_following_1nest_atm_ocn_debug | - jet s4  noaacloud           | baseline |
 

tests/run_test.sh

@@ -275,6 +275,10 @@ if [[ $DOCN_CDEPS = 'true' ]]; then
   atparse < ${PATHRT}/parm/${DOCN_STREAM_CONFIGURE:-docn.streams.IN} > docn.streams
 fi
 
+if [[ $CDEPS_INLINE = 'true' ]]; then
+  atparse < ${PATHRT}/parm/${CDEPS_INLINE_CONFIGURE:-stream.config.IN} > stream.config
+fi
+
 TPN=$(( TPN / THRD ))
 if (( TASKS < TPN )); then
   TPN=${TASKS}