Merge tag 'cam-ew2.3.004' into add_v8mpasfiles_eworg2

Bring in dev changes from EarthWorksOrg/CAM

.gitmodules

@@ -77,10 +77,10 @@
 
 [submodule "mpas"]
 	path = src/dynamics/mpas/dycore
-	url = https://github.com/MPAS-Dev/MPAS-Model.git
+	url = https://github.com/EarthWorksOrg/MPAS-Model.git
         fxrequired = AlwaysRequired
         fxsparse = ../.mpas_sparse_checkout
-        fxtag = b8c33daa
+        fxtag = mpas-ew2.3.001
 	fxDONOTUSEurl = https://github.com/MPAS-Dev/MPAS-Model.git
 
 [submodule "cosp2"]

bld/config_files/definition.xml

@@ -127,7 +127,7 @@ Chemistry source directory generated by the chemistry preprocessor; contains F90
 <entry id="chem_src_dir" value="">
 Chemistry source directory; contains F90 files.
 </entry>
-<entry id="ocn" valid_values="docn,dom,som,socn,aquaplanet,pop,mom" value="docn">
+<entry id="ocn" valid_values="docn,dom,som,socn,aquaplanet,pop,mom,mpaso" value="docn">
 Use data ocean model (docn or dom), stub ocean (socn), or aqua planet ocean
 (aquaplanet) in cam build.  When built from the CESM scripts the value of
 ocn may be set to pop.  This doesn't impact how CAM is built, only how

bld/namelist_files/namelist_defaults_cam.xml

@@ -54,8 +54,15 @@
 <ncdata hgrid="mpasa30"  nlev="58" analytic_ic="1" >atm/cam/inic/mpas/mpasa30_L58_notopo_coords_c240814.nc</ncdata>
 <ncdata hgrid="mpasa480" nlev="93" analytic_ic="1" >atm/cam/inic/mpas/mpasa480_L93_notopo_coords_c240814.nc</ncdata>
 <ncdata hgrid="mpasa120" nlev="93" analytic_ic="1" >atm/cam/inic/mpas/mpasa120_L93_notopo_coords_c240814.nc</ncdata>
+<!-- Next entries are EarthWorks specific -->
 <ncdata hgrid="mpasa60"  nlev="93" analytic_ic="1" >atm/cam/inic/mpas/mpasa60_L93_notopo_coords_c240814.nc</ncdata>
 <ncdata hgrid="mpasa30"  nlev="93" analytic_ic="1" >atm/cam/inic/mpas/mpasa30_L93_notopo_coords_c240814.nc</ncdata>
+<ncdata hgrid="mpasa60"  nlev="32" analytic_ic="1" >atm/cam/inic/mpas/mpasa60_L32_notopo_coords_c230707.nc</ncdata>
+<ncdata hgrid="mpasa60"  nlev="32" analytic_ic="1" phys="cam_dev" >atm/cam/inic/mpas/mpasa60_L32_notopo_coords_c240507.nc</ncdata>
+<ncdata hgrid="mpasa30"  nlev="32" analytic_ic="1" >atm/cam/inic/mpas/mpasa30_L32_notopo_coords_c230707.nc</ncdata>
+<ncdata hgrid="mpasa30"  nlev="32" analytic_ic="1" phys="cam_dev" >atm/cam/inic/mpas/mpasa30_L32_notopo_coords_c240507.nc</ncdata>
+<ncdata hgrid="mpasa15"  nlev="32" analytic_ic="1" >atm/cam/inic/mpas/mpasa15_L32_notopo_coords_c240507.nc</ncdata>
+<!-- END EarthWorks specific -->
 
 <!-- Files with initial conditions -->
 <ncdata dyn="fv"  hgrid="0.23x0.31" nlev="26" ic_ymd="101"      >atm/cam/inic/fv/cami_0000-01-01_0.23x0.31_L26_c100513.nc</ncdata>
@@ -294,6 +301,16 @@
 <ncdata hgrid="mpasa120" nlev="58" >atm/cam/inic/mpas/cami_01-01-2000_00Z_mpasa120_L58_CFSR_c240814.nc</ncdata>
 <ncdata hgrid="mpasa480" nlev="93" >atm/cam/inic/mpas/cami_01-01-2000_00Z_mpasa480_L93_CFSR_c240814.nc</ncdata>
 <ncdata hgrid="mpasa120" nlev="93" >atm/cam/inic/mpas/cami_01-01-2000_00Z_mpasa120_L93_CFSR_c240814.nc</ncdata>
+<!-- Next entries are EarthWorks specific, real-data ICs -->
+<ncdata hgrid="mpasa60"  nlev="32" ic_ymd="20000101" >atm/cam/inic/mpas/cami_01-01-2000_00Z_mpasa60_L32_CFSR_c210518.nc</ncdata>
+<!-- <ncdata hgrid="mpasa60"  nlev="32" ic_ymd="20000101" phys="cam_dev" >atm/cam/inic/mpas/cami_01-01-2000_00Z_mpasa60_L32_CFSR_c240508.nc</ncdata> -->
+<ncdata hgrid="mpasa30"  nlev="32" ic_ymd="20000101" >atm/cam/inic/mpas/cami_01-01-2000_00Z_mpasa30_L32_CFSR_230302.nc</ncdata>
+<!-- <ncdata hgrid="mpasa60"  nlev="32" ic_ymd="20000101" phys="cam_dev" >atm/cam/inic/mpas/cami_01-01-2000_00Z_mpasa30_L32_CFSR_c240508.nc</ncdata> -->
+<ncdata hgrid="mpasa120" nlev="58" ic_ymd="20000101">atm/cam/inic/mpas/cami_01-01-2000_00Z_mpasa120_L58_c230901.nc</ncdata>
+<!-- <ncdata hgrid="mpasa120" nlev="58" ic_ymd="20000101" phys="cam_dev" >atm/cam/inic/mpas/cami_01-01-2000_00Z_mpasa120_L58_CFSR_c240508.nc</ncdata> -->
+<ncdata hgrid="mpasa15"  nlev="58" ic_ymd="20000101">atm/cam/inic/mpas/cami_01-01-2000_00Z_mpasa15_L58_c230316.nc</ncdata>
+<!-- <ncdata hgrid="mpasa15"  nlev="58" ic_ymd="20000101" phys="cam_dev" >atm/cam/inic/mpas/cami_01-01-2000_00Z_mpasa15_L58_CFSR_c240508.nc</ncdata> -->
+<!-- END EarthWorks specific, real-data ICs -->
 
 <!-- Topography -->
 <bnd_topo hgrid="256x512" >atm/cam/topo/topo-from-cami_0000-01-01_256x512_L26_c030918.nc</bnd_topo>
@@ -347,6 +364,17 @@
 
 <bnd_topo hgrid="mpasa480" >atm/cam/topo/mpas/mpasa480_gmted2010_modis_bedmachine_nc3000_Laplace0400_noleak_20240507.nc</bnd_topo>
 <bnd_topo hgrid="mpasa120" >atm/cam/topo/mpas/mpasa120_gmted2010_modis_bedmachine_nc3000_Laplace0100_noleak_20240507.nc</bnd_topo>
+<!-- Next entires are EarthWorks specific, for real-data cases not in ESCOMP/CAM -->
+<bnd_topo hgrid="mpasa60"  >atm/cam/topo/mpas_60_nc3000_Co030_Fi001_MulG_PF_Nsw021.nc</bnd_topo>
+<!-- <bnd_topo hgrid="mpasa60"  phys="cam_dev" >atm/cam/topo/mpas/mpasa60_gmted2010_modis_bedmachine_nc3000_Laplace0050_noleak_20240507.nc</bnd_topo> -->
+<bnd_topo hgrid="mpasa30"  >atm/cam/topo/mpas_30_nc3000_Co015_Fi001_MulG_PF_Nsw011.nc</bnd_topo>
+<!-- <bnd_topo hgrid="mpasa30"  phys="cam_dev" >atm/cam/topo/mpas/mpasa30_gmted2010_modis_bedmachine_nc3000_Laplace0025_noleak_20240507.nc</bnd_topo> -->
+<bnd_topo hgrid="mpasa15"  >atm/cam/topo/mpas_15_nc3500_c20230315.nc</bnd_topo>
+<!-- <bnd_topo hgrid="mpasa15"  phys="cam_dev" >atm/cam/topo/mpas/mpasa15_gmted2010_modis_bedmachine_nc3000_Laplace0013_noleak_20240507.nc</bnd_topo> -->
+<!-- Corresponds to cami_01-01-2000_00Z_mpasa120_L58_c230901.nc file -->
+<bnd_topo hgrid="mpasa120" nlev="58" >atm/cam/topo/mpasa120_gmted2010_modis_bedmachine_nc3000_Laplace0100_20220728.nc</bnd_topo>
+<!-- <bnd_topo hgrid="mpasa120" nlev="58" phys="cam_dev" >atm/cam/topo/mpas/mpasa120_gmted2010_modis_bedmachine_nc3000_Laplace0100_noleak_20240507.nc</bnd_topo> -->
+<!-- END EarthWorks specific, for real-data cases not in ESCOMP/CAM -->
 
 <!-- Scale Dry Air Mass: 0=> no scaling / +nnn=>scale to nnn Pressure  -->
 <scale_dry_air_mass                                                                >      0.0D0 </scale_dry_air_mass>
@@ -383,6 +411,8 @@
 <analytic_ic_type phys="kessler"    > moist_baroclinic_wave_dcmip2016 </analytic_ic_type>
 <analytic_ic_type phys="tj2016"     > moist_baroclinic_wave_dcmip2016 </analytic_ic_type>
 <analytic_ic_type phys="grayrad"    > moist_baroclinic_wave_dcmip2016 </analytic_ic_type>
+<!-- MPAS Analytic ICs for aquaplanet (experimental) -->
+<analytic_ic_type dyn="mpas" aquaplanet="1"> us_standard_atmosphere </analytic_ic_type>
 
 <!-- Frierson Gray Radiation -->
 
@@ -2032,8 +2062,11 @@
 <drydep_srf_file hgrid="C192">atm/cam/chem/trop_mam/atmsrf_C192_c200625.nc</drydep_srf_file>
 <drydep_srf_file hgrid="C384">atm/cam/chem/trop_mam/atmsrf_C384_c200625.nc</drydep_srf_file>
 
-<drydep_srf_file hgrid="mpasa120">atm/cam/chem/trop_mam/atmsrf_mpasa120_c090720.nc</drydep_srf_file>
 <drydep_srf_file hgrid="mpasa480">atm/cam/chem/trop_mam/atmsrf_mpasa480_c090720.nc</drydep_srf_file>
+<drydep_srf_file hgrid="mpasa120">atm/cam/chem/trop_mam/atmsrf_mpasa120_c090720.nc</drydep_srf_file>
+<drydep_srf_file hgrid="mpasa60">atm/cam/chem/trop_mam/atmsrf_mpasa60_c210511.nc</drydep_srf_file>
+<drydep_srf_file hgrid="mpasa30">atm/cam/chem/trop_mam/atmsrf_mpasa30_c210601.nc</drydep_srf_file>
+<drydep_srf_file hgrid="mpasa15">atm/cam/chem/trop_mam/atmsrf_mpasa15_c20210804.nc</drydep_srf_file>
 
 <!-- depvel data -->
 <depvel_lnd_file>atm/cam/chem/trop_mozart/dvel/regrid_vegetation.nc</depvel_lnd_file>
@@ -2314,6 +2347,15 @@
 <clubb_mf_ent0                     > 0.22    </clubb_mf_ent0>
 <clubb_mf_nup                      > 10      </clubb_mf_nup>
 
+<!-- EarthWorks specific -->
+<!-- Set radiation intervals based on mpas hgrids -->
+<iradlw hgrid="mpasa15"  > 10 </iradlw>
+<iradsw hgrid="mpasa15"  > 10 </iradsw>
+<iradlw hgrid="mpasa7p5" > 10 </iradlw>
+<iradsw hgrid="mpasa7p5" > 10 </iradsw>
+<iradlw hgrid="mpas3p75" > 10 </iradlw>
+<iradsw hgrid="mpas3p75" > 10 </iradsw>
+
 <!-- Microphysics scheme -->
 <micro_mg_do_hail                       > .false. </micro_mg_do_hail>
 <micro_mg_do_graupel                    > .false. </micro_mg_do_graupel>
@@ -3419,10 +3461,13 @@
 <mpas_time_integration        > SRK3 </mpas_time_integration>
 <mpas_time_integration_order  > 2 </mpas_time_integration_order>
 <mpas_dt                      > 1800.0D0 </mpas_dt>
-<mpas_dt hgrid="mpasa120"     >  900.0D0 </mpas_dt>
+<mpas_dt hgrid="mpasa120"     >  600.0D0 </mpas_dt>
 <mpas_dt hgrid="mpasa120" waccm_phys="1"> 600.D0 </mpas_dt>
-<mpas_dt hgrid="mpasa60"      >  450.0D0 </mpas_dt>
-<mpas_dt hgrid="mpasa30"      >  225.0D0 </mpas_dt>
+<mpas_dt hgrid="mpasa60"      >  300.0D0 </mpas_dt>
+<mpas_dt hgrid="mpasa30"      >  150.0D0 </mpas_dt>
+<mpas_dt hgrid="mpasa15"      >   80.0D0 </mpas_dt>
+<mpas_dt hgrid="mpasa7p5"     >   40.0D0 </mpas_dt>
+<mpas_dt hgrid="mpasa3p75"    >   20.0D0 </mpas_dt>
 
 <mpas_split_dynamics_transport>.true.</mpas_split_dynamics_transport>
 <mpas_number_of_sub_steps     > 2 </mpas_number_of_sub_steps>
@@ -3440,6 +3485,9 @@
 <mpas_len_disp hgrid="mpasa120">120000.0D0</mpas_len_disp>
 <mpas_len_disp hgrid="mpasa60">  60000.0D0</mpas_len_disp>
 <mpas_len_disp hgrid="mpasa30">  30000.0D0</mpas_len_disp>
+<mpas_len_disp hgrid="mpasa15">  15000.0D0</mpas_len_disp>
+<mpas_len_disp hgrid="mpasa7p5">  7500.0D0</mpas_len_disp>
+<mpas_len_disp hgrid="mpasa3p75"> 3750.0D0</mpas_len_disp>
 
 <mpas_visc4_2dsmag            > 0.05D0 </mpas_visc4_2dsmag>
 <mpas_del4u_div_factor        > 10.0D0 </mpas_del4u_div_factor>

cime_config/config_component.xml

@@ -206,6 +206,7 @@
       <!-- Aquaplanet -->
       <value compset="_DOCN%SOMAQP">-aquaplanet</value>
       <value compset="_DOCN%AQP"   >-aquaplanet</value>
+      <value compset="_DOCN%AQP" grid="a%mpasa[0-9]">-analytic_ic</value>
 
       <!-- PORT -->
       <value compset="%PORT">-offline_drv rad</value>

cime_config/usermods_dirs/eworg_opt_clubb/user_nl_cam

@@ -0,0 +1,3 @@
+clubb_l_diag_lscale_from_tau = .true.
+clubb_penta_solve_method = 2
+clubb_tridiag_solve_method = 2

src/dynamics/mpas/Makefile

@@ -1,4 +1,8 @@
-CPPFLAGS := -D_MPI -DMPAS_NATIVE_TIMERS -DMPAS_GIT_VERSION=unknown -DMPAS_NAMELIST_SUFFIX=atmosphere -DMPAS_CAM_DYCORE -DMPAS_PIO_SUPPORT -DMPAS_NO_ESMF_INIT
+CPPFLAGS := -D_MPI -DMPAS_NATIVE_TIMERS -DMPAS_GIT_VERSION=unknown -DMPAS_NAMELIST_SUFFIX=atmosphere -DMPAS_CAM_DYCORE -DMPAS_NO_ESMF_INIT
+$(info $$GPUFLAGS=[${GPUFLAGS}])
+ifdef GPUFLAGS
+  CPPFLAGS += -DMPAS_OPENACC
+endif
 ifdef PIODEF
   CPPFLAGS += $(PIODEF)
 endif
@@ -18,7 +22,6 @@ INTERFACE_OBJS = \
 	mpas_atm_core_interface.o \
 	mpas_atm_dimensions.o \
 	mpas_atm_threading.o \
-	mpas_atm_halos.o \
 	cam_mpas_subdriver.o
 
 DYN_OBJS = \
@@ -32,11 +35,10 @@ DIAG_OBJS = \
 
 DIAGNOSTICS = \
 	mpas_atm_diagnostic_template.o \
-	mpas_isobaric_diagnostics.o \
-	mpas_cloud_diagnostics.o \
-	mpas_convective_diagnostics.o \
-	mpas_pv_diagnostics.o \
-	mpas_soundings.o
+	isobaric_diagnostics.o \
+	convective_diagnostics.o \
+	pv_diagnostics.o \
+	soundings.o
 
 REG_OBJS = \
 	parse.o \
@@ -92,10 +94,7 @@ FRAME_OBJS = \
 	mpas_pool_routines.o \
 	xml_stream_parser.o \
 	regex_matching.o \
-	mpas_log.o \
-	mpas_halo.o \
-	mpas_string_utils.o
-
+	mpas_log.o
 
 UTIL_OBJS = \
 	ezxml.o
@@ -134,8 +133,7 @@ mpas_framework.o: mpas_dmpar.o \
                   mpas_io_units.o \
                   mpas_block_decomp.o \
                   mpas_stream_manager.o \
-                  mpas_c_interfacing.o \
-                  mpas_halo.o
+                  mpas_c_interfacing.o
 
 mpas_abort.o: mpas_kind_types.o mpas_io_units.o mpas_threading.o
 
@@ -161,7 +159,7 @@ mpas_dmpar.o: mpas_sort.o mpas_kind_types.o mpas_derived_types.o mpas_hash.o mpa
 
 mpas_sort.o: mpas_kind_types.o mpas_log.o
 
-mpas_timekeeping.o: mpas_string_utils.o mpas_kind_types.o mpas_derived_types.o mpas_dmpar.o mpas_threading.o mpas_log.o
+mpas_timekeeping.o: mpas_kind_types.o mpas_derived_types.o mpas_dmpar.o mpas_threading.o mpas_log.o
 
 mpas_timer.o: mpas_kind_types.o mpas_dmpar.o mpas_threading.o mpas_log.o
 
@@ -185,7 +183,6 @@ mpas_stream_manager.o: mpas_io_streams.o mpas_timekeeping.o mpas_derived_types.o
 
 mpas_forcing.o: mpas_derived_types.o mpas_timekeeping.o mpas_stream_manager.o mpas_log.o mpas_io_units.o
 
-mpas_halo.o: mpas_derived_types.o mpas_pool_routines.o mpas_log.o
 
 #
 # Operator dependencies
@@ -241,18 +238,17 @@ mpas_atm_time_integration.o: mpas_atm_iau.o mpas_atm_dimensions.o mpas_atm_bound
 #
 mpas_atm_diagnostics_manager.o: mpas_atm_diagnostics_utils.o $(DIAGNOSTICS)
 
-mpas_cloud_diagnostics.o: mpas_atm_diagnostics_utils.o
-mpas_convective_diagnostics.o: mpas_atm_diagnostics_utils.o
-mpas_isobaric_diagnostics.o: mpas_atm_diagnostics_utils.o
-mpas_pv_diagnostics.o: mpas_atm_diagnostics_utils.o
+convective_diagnostics.o: mpas_atm_diagnostics_utils.o
+isobaric_diagnostics.o: mpas_atm_diagnostics_utils.o
+pv_diagnostics.o: mpas_atm_diagnostics_utils.o
 
 
 #
 # Core interface objects
 #
 mpas_atm_core_interface.o: mpas_atm_core.o incs
 
-mpas_atm_core.o: mpas_atm_threading.o mpas_atm_time_integration.o mpas_atm_diagnostics_manager.o mpas_atm_halos.o
+mpas_atm_core.o: mpas_atm_threading.o mpas_atm_time_integration.o mpas_atm_diagnostics_manager.o
 
 cam_mpas_subdriver.o: mpas_atm_core_interface.o mpas_derived_types.o mpas_framework.o mpas_domain_routines.o mpas_pool_routines.o
 
@@ -268,8 +264,8 @@ clean:
 	$(CC) -c $(CFLAGS) -DUNDERSCORE $(CPPFLAGS) $(CPPINCLUDES) -I$(MPAS_SRC_ROOT)/dycore/src/external/ezxml $<
 
 %.o : %.F
-	$(FC) -c $(CPPFLAGS) $(FFLAGS) $(CPPINCLUDES) $(FCINCLUDES) $<
+	$(FC) -c $(CPPFLAGS) $(FFLAGS) $(GPUFLAGS) $(CPPINCLUDES) $(FCINCLUDES) $<
 
 %.o : %.F90
-	$(FC) -c $(CPPFLAGS) $(FFLAGS) $(CPPINCLUDES) $(FCINCLUDES) $<
+	$(FC) -c $(CPPFLAGS) $(FFLAGS) $(GPUFLAGS) $(CPPINCLUDES) $(FCINCLUDES) $<