From bb48a6dd29de8db242424ea892450ccb50158df6 Mon Sep 17 00:00:00 2001 From: Cheryl Craig Date: Tue, 14 May 2024 10:45:10 -0600 Subject: [PATCH 01/22] Update externals to match cesm2_3_alpha17f --- Externals.cfg | 25 +++++++++---------------- cime_config/testdefs/testlist_cam.xml | 2 +- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index ffe2c4b012..f967878539 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -1,5 +1,5 @@ [ccs_config] -tag = ccs_config_cesm0.0.85 +tag = ccs_config_cesm0.0.106 protocol = git repo_url = https://github.com/ESMCI/ccs_config_cesm local_path = ccs_config @@ -13,7 +13,7 @@ local_path = components/cice5 required = True [cice6] -tag = cesm_cice6_4_1_10 +tag = cesm_cice6_5_0_7 protocol = git repo_url = https://github.com/ESCOMP/CESM_CICE local_path = components/cice @@ -21,14 +21,14 @@ externals = Externals.cfg required = True [cmeps] -tag = cmeps0.14.59 +tag = cmeps0.14.60 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps required = True [cdeps] -tag = cdeps1.0.31 +tag = cdeps1.0.33 protocol = git repo_url = https://github.com/ESCOMP/CDEPS.git local_path = components/cdeps @@ -36,7 +36,7 @@ externals = Externals_CDEPS.cfg required = True [cpl7] -tag = cpl77.0.7 +tag = cpl77.0.8 protocol = git repo_url = https://github.com/ESCOMP/CESM_CPL7andDataComps local_path = components/cpl7 @@ -49,13 +49,6 @@ repo_url = https://github.com/ESCOMP/CESM_share local_path = share required = True -[mct] -tag = MCT_2.11.0 -protocol = git -repo_url = https://github.com/MCSclimate/MCT -local_path = libraries/mct -required = True - [parallelio] tag = pio2_6_2 protocol = git @@ -71,7 +64,7 @@ local_path = cime required = True [cism] -tag = cismwrap_2_1_96 +tag = cismwrap_2_1_100 protocol = git repo_url = https://github.com/ESCOMP/CISM-wrapper local_path = components/cism @@ -79,7 +72,7 @@ externals = Externals_CISM.cfg required = True [clm] -tag = ctsm5.1.dev145 +tag = ctsm5.2.005 protocol = git repo_url = https://github.com/ESCOMP/CTSM local_path = components/clm @@ -95,14 +88,14 @@ externals = Externals_FMS.cfg required = True [mosart] -tag = mosart1_0_48 +tag = mosart1_0_49 protocol = git repo_url = https://github.com/ESCOMP/MOSART local_path = components/mosart required = True [rtm] -tag = rtm1_0_78 +tag = rtm1_0_79 protocol = git repo_url = https://github.com/ESCOMP/RTM local_path = components/rtm diff --git a/cime_config/testdefs/testlist_cam.xml b/cime_config/testdefs/testlist_cam.xml index 05983cca7b..98813e6993 100644 --- a/cime_config/testdefs/testlist_cam.xml +++ b/cime_config/testdefs/testlist_cam.xml @@ -1444,7 +1444,7 @@ - + From 1f0df194d83cc5d83aa88a9016f3f2c96f9d5780 Mon Sep 17 00:00:00 2001 From: Cheryl Craig Date: Tue, 14 May 2024 11:22:48 -0600 Subject: [PATCH 02/22] put back in MCT since makefile mods would be required --- Externals.cfg | 7 +++++++ cime_config/testdefs/testlist_cam.xml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index f967878539..ad6907e697 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -49,6 +49,13 @@ repo_url = https://github.com/ESCOMP/CESM_share local_path = share required = True +[mct] +tag = MCT_2.11.0 +protocol = git +repo_url = https://github.com/MCSclimate/MCT +local_path = libraries/mct +required = True + [parallelio] tag = pio2_6_2 protocol = git diff --git a/cime_config/testdefs/testlist_cam.xml b/cime_config/testdefs/testlist_cam.xml index 98813e6993..05983cca7b 100644 --- a/cime_config/testdefs/testlist_cam.xml +++ b/cime_config/testdefs/testlist_cam.xml @@ -1444,7 +1444,7 @@ - + From 93aa30e7ed2d4c31ab94b5bf4f310ebe8f81e343 Mon Sep 17 00:00:00 2001 From: Cheryl Craig Date: Tue, 14 May 2024 15:49:34 -0600 Subject: [PATCH 03/22] Increase wall clock times --- cime_config/testdefs/testlist_cam.xml | 28 +++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/cime_config/testdefs/testlist_cam.xml b/cime_config/testdefs/testlist_cam.xml index 05983cca7b..575729a37c 100644 --- a/cime_config/testdefs/testlist_cam.xml +++ b/cime_config/testdefs/testlist_cam.xml @@ -73,7 +73,7 @@ - + @@ -83,7 +83,7 @@ - + @@ -93,7 +93,7 @@ - + @@ -1811,7 +1811,7 @@ - + @@ -1829,7 +1829,7 @@ - + @@ -2033,7 +2033,7 @@ - + @@ -2390,7 +2390,7 @@ - + @@ -2415,7 +2415,7 @@ - + @@ -2424,7 +2424,7 @@ - + @@ -2450,7 +2450,7 @@ - + @@ -2459,7 +2459,7 @@ - + @@ -2725,7 +2725,7 @@ - + @@ -2748,7 +2748,7 @@ - + @@ -2800,7 +2800,7 @@ - + From e91606adb14eb836d8218abb4f239076810d05d3 Mon Sep 17 00:00:00 2001 From: Cheryl Craig Date: Thu, 16 May 2024 13:08:22 -0600 Subject: [PATCH 04/22] revert times back for regression tests to original times --- cime_config/testdefs/testlist_cam.xml | 28 +++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/cime_config/testdefs/testlist_cam.xml b/cime_config/testdefs/testlist_cam.xml index 575729a37c..05983cca7b 100644 --- a/cime_config/testdefs/testlist_cam.xml +++ b/cime_config/testdefs/testlist_cam.xml @@ -73,7 +73,7 @@ - + @@ -83,7 +83,7 @@ - + @@ -93,7 +93,7 @@ - + @@ -1811,7 +1811,7 @@ - + @@ -1829,7 +1829,7 @@ - + @@ -2033,7 +2033,7 @@ - + @@ -2390,7 +2390,7 @@ - + @@ -2415,7 +2415,7 @@ - + @@ -2424,7 +2424,7 @@ - + @@ -2450,7 +2450,7 @@ - + @@ -2459,7 +2459,7 @@ - + @@ -2725,7 +2725,7 @@ - + @@ -2748,7 +2748,7 @@ - + @@ -2800,7 +2800,7 @@ - + From f816d1e560281734e3106a74cfa860484f5fb6a4 Mon Sep 17 00:00:00 2001 From: Cheryl Craig Date: Thu, 16 May 2024 16:34:40 -0600 Subject: [PATCH 05/22] ChangeLog for cam6_3_161 --- doc/ChangeLog | 115 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 114 insertions(+), 1 deletion(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index 9f4f0d348d..3a5c46527d 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,6 +1,120 @@ =============================================================== +Tag name: cam6_3_161 +Originator(s): cacraig +Date: May 16, 2024 +One-line Summary: Update to alpha17 externals +Github PR URL: https://github.com/ESCOMP/CAM/pull/1031 + +Purpose of changes (include the issue number and title text for each relevant GitHub issue): + - Update externals to match externals which will be used in cesm2_3_beta17: https://github.com/ESCOMP/CAM/issues/985 + - Bring in ccs_config0.0.99: https://github.com/ESCOMP/CAM/issues/1021 + - Unable to compile cam6_3_154 with nvhpc/24.3 on Derecho: https://github.com/ESCOMP/CAM/issues/1025 + +Describe any changes made to build system: N/A + +Describe any changes made to the namelist: N/A + +List any changes to the defaults for the boundary datasets: N/A + +Describe any substantial timing or memory changes: N/A + +Code reviewed by: nusbaume + +List all files eliminated: N/A + +List all files added and what they do: N/A + +List all existing files that have been modified, and describe the changes: +M Externals.cfg + - update externals to match cesm2_3_beta17 + +If there were any failures reported from running test_driver.sh on any test +platform, and checkin with these failures has been OK'd by the gatekeeper, +then copy the lines from the td.*.status files for the failed tests to the +appropriate machine below. All failed tests must be justified. + +NOTE - most tests have namelist differences due to mediator namelist changes + +derecho/intel/aux_cam: + ERP_Ln9.f09_f09_mg17.FCSD_HCO.derecho_intel.cam-outfrq9s (Overall: FAIL) details: + - pre-existing failure + + ERP_Lh12.f19_f19_mg17.FW4madSD.derecho_intel.cam-outfrq3h (Overall: FAIL) details: + - Bug during CREATE_NEWCASE in CTSM code - will go away when CTSM external is updated post git-fleximod + + SMS_D_Ln9.T42_T42.FSCAM.derecho_intel.cam-outfrq9s (Overall: FAIL) details: + - Bug in med.F90 - Will go away when CICE external is updated post git-fleximod + + ERP_Ln9.ne30_ne30_mg17.FCnudged.derecho_intel.cam-outfrq9s (Overall: FAIL) details: + FAIL ERP_Ln9.ne30_ne30_mg17.FCnudged.derecho_intel.cam-outfrq9s SHAREDLIB_BUILD failed to initialize + SMS_D_Ln9.ne16_ne16_mg17.FX2000.derecho_intel.cam-outfrq9s (Overall: PEND) details: + PEND SMS_D_Ln9.ne16_ne16_mg17.FX2000.derecho_intel.cam-outfrq9s SHAREDLIB_BUILD + SMS_D_Ln9_P1280x1.ne0ARCTICne30x4_ne0ARCTICne30x4_mt12.FHIST.derecho_intel.cam-outfrq9s (Overall: FAIL) details: + FAIL SMS_D_Ln9_P1280x1.ne0ARCTICne30x4_ne0ARCTICne30x4_mt12.FHIST.derecho_intel.cam-outfrq9s SHAREDLIB_BUILD time=2 + SMS_D_Ln9_P1280x1.ne0CONUSne30x8_ne0CONUSne30x8_mt12.FCHIST.derecho_intel.cam-outfrq9s (Overall: FAIL) details: + FAIL SMS_D_Ln9_P1280x1.ne0CONUSne30x8_ne0CONUSne30x8_mt12.FCHIST.derecho_intel.cam-outfrq9s SHAREDLIB_BUILD time=1 + - Bugs reported to CTSM and will be fixed when CTSM external is updated post git-fleximod + + ERP_D_Ln9.ne30pg3_ne30pg3_mg17.F2000dev.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_D_Ln9.ne30pg3_ne30pg3_mg17.FLTHIST.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_D_Ln9.ne30pg3_ne30pg3_mg17.FLTHIST.derecho_intel.cam-outfrq9s_rrtmgp (Overall: DIFF) details: + ERP_Ld3.f09_f09_mg17.FWHIST.derecho_intel.cam-reduced_hist1d (Overall: DIFF) details: + ERP_Ln9.C96_C96_mg17.F2000climo.derecho_intel.cam-outfrq9s_mg3 (Overall: DIFF) details: + ERP_Ln9.f09_f09_mg17.F1850.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_Ln9.f09_f09_mg17.F2000climo.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_Ln9.f09_f09_mg17.F2000dev.derecho_intel.cam-outfrq9s_mg3 (Overall: DIFF) details: + ERP_Ln9.f09_f09_mg17.F2010climo.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_Ln9.f09_f09_mg17.FHIST_BDRD.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_Ln9.f19_f19_mg17.FWsc1850.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_Ln9.ne30pg3_ne30pg3_mg17.FW2000climo.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + ERS_Ld3.f10_f10_mg37.F1850.derecho_intel.cam-outfrq1d_14dec_ghg_cam_dev (Overall: DIFF) details: + ERS_Ln9.f09_f09_mg17.FX2000.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + ERS_Ln9.f19_f19_mg17.FSPCAMS.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + ERS_Ln9.f19_f19_mg17.FXSD.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + ERS_Ln9_P288x1.mpasa120_mpasa120.F2000climo.derecho_intel.cam-outfrq9s_mpasa120 (Overall: DIFF) details: + ERS_Ln9_P36x1.mpasa480_mpasa480.F2000climo.derecho_intel.cam-outfrq9s_mpasa480 (Overall: DIFF) details: + SMS_D_Ln9.f09_f09_mg17.FCts2nudged.derecho_intel.cam-outfrq9s_leapday (Overall: DIFF) details: + SMS_D_Ln9.f09_f09_mg17.FCvbsxHIST.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + SMS_D_Ln9.f09_f09_mg17.FSD.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + SMS_D_Ln9.f19_f19_mg17.FWma2000climo.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + SMS_D_Ln9.f19_f19_mg17.FWma2000climo.derecho_intel.cam-outfrq9s_waccm_ma_mam4 (Overall: DIFF) details: + SMS_D_Ln9.f19_f19_mg17.FXHIST.derecho_intel.cam-outfrq9s_amie (Overall: DIFF) details: + SMS_D_Ln9.ne30pg3_ne30pg3_mg17.FMTHIST.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + SMS_D_Ln9_P1280x1.ne30pg3_ne30pg3_mg17.FCLTHIST.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + SMS_Ld1.f09_f09_mg17.FCHIST_GC.derecho_intel.cam-outfrq1d (Overall: DIFF) details: + SMS_Ld1.f09_f09_mg17.FW2000climo.derecho_intel.cam-outfrq1d (Overall: DIFF) details: + SMS_Ld1.f19_f19.F2000dev.derecho_intel.cam-outfrq1d (Overall: DIFF) details: + SMS_Ld1.ne30pg3_ne30pg3_mg17.FC2010climo.derecho_intel.cam-outfrq1d (Overall: DIFF) details: + SMS_Lh12.f09_f09_mg17.FCSD_HCO.derecho_intel.cam-outfrq3h (Overall: DIFF) details: + SMS_Lm13.f10_f10_mg37.F2000climo.derecho_intel.cam-outfrq1m (Overall: DIFF) details: + SMS_Ln9.f09_f09_mg17.F2010climo.derecho_intel.cam-nudging (Overall: DIFF) details: + SMS_Ln9.f09_f09_mg17.FW1850.derecho_intel.cam-reduced_hist3s (Overall: DIFF) details: + SMS_Ln9.f19_f19.F2000climo.derecho_intel.cam-silhs (Overall: DIFF) details: + SMS_Ln9.f19_f19_mg17.FHIST.derecho_intel.cam-outfrq9s_nochem (Overall: DIFF) details: + SMS_Ln9.ne30pg3_ne30pg3_mg17.FW2000climo.derecho_intel.cam-outfrq9s_rrtmgp (Overall: DIFF) details: + - Answer changes due to updated externals + +izumi/nag/aux_cam: all B4B, except: + DAE.f45_f45_mg37.FHS94.izumi_nag.cam-dae (Overall: FAIL) details: + - pre-existing failure + +izumi/gnu/aux_cam: all BFB except: + SMS_P48x1_D_Ln9.f19_f19_mg17.FW4madSD.izumi_gnu.cam-outfrq9s (Overall: FAIL) details: + - Bug during CREATE_NEWCASE in CTSM code - will go away when CTSM external is updated post git-fleximod + + SMS_D_Ln9.f10_f10_mg37.2000_CAM%DEV%GHGMAM4_CLM50%SP_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV_SESP.izumi_gnu.cam-outfrq9s (Overall: DIFF) details: + - answer changes due to updated externals + + +CAM tag used for the baseline comparison tests if different than previous +tag: cam6_3_159 as cam6_3_160 did not run regression tests + + +=============================================================== +=============================================================== + Tag name: cam6_3_160 Originator(s): cacraig, jedwards Date: April 29, 2024 @@ -461,7 +575,6 @@ derecho/intel/aux_cam: SMS_Ld1.f09_f09_mg17.FW2000climo.derecho_intel.cam-outfrq1d (Overall: DIFF) details: SMS_Ld1.f19_f19.F2000dev.derecho_intel.cam-outfrq1d (Overall: DIFF) details: SMS_Ld1.ne30pg3_ne30pg3_mg17.FC2010climo.derecho_intel.cam-outfrq1d (Overall: DIFF) details: - SMS_Lh12.f09_f09_mg17.FCSD_HCO.derecho_intel.cam-outfrq3h (Overall: DIFF) details: SMS_Lm13.f10_f10_mg37.F2000climo.derecho_intel.cam-outfrq1m (Overall: DIFF) details: SMS_Ln9.f09_f09_mg17.F2010climo.derecho_intel.cam-nudging (Overall: DIFF) details: SMS_Ln9.f09_f09_mg17.FW1850.derecho_intel.cam-reduced_hist3s (Overall: DIFF) details: From 4fc504cd98d51f10e4c3a544172c1f3dea08f98f Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Tue, 21 May 2024 11:29:39 -0600 Subject: [PATCH 06/22] remove the casper test suites --- cime_config/testdefs/testlist_cam.xml | 45 ------------------- .../cam/outfrq9s_mg2_default/shell_commands | 7 --- .../cam/outfrq9s_mg2_default/user_nl_cam | 4 -- .../cam/outfrq9s_mg2_default/user_nl_clm | 26 ----------- .../outfrq9s_mg3_nondefault/shell_commands | 8 ---- .../cam/outfrq9s_mg3_nondefault/user_nl_cam | 24 ---------- .../cam/outfrq9s_mg3_nondefault/user_nl_clm | 26 ----------- .../cam/outfrq9s_mg3_pcols1536/shell_commands | 8 ---- .../cam/outfrq9s_mg3_pcols1536/user_nl_cam | 4 -- .../cam/outfrq9s_mg3_pcols1536/user_nl_clm | 26 ----------- 10 files changed, 178 deletions(-) delete mode 100644 cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg2_default/shell_commands delete mode 100644 cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg2_default/user_nl_cam delete mode 100644 cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg2_default/user_nl_clm delete mode 100644 cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_nondefault/shell_commands delete mode 100644 cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_nondefault/user_nl_cam delete mode 100644 cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_nondefault/user_nl_clm delete mode 100644 cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols1536/shell_commands delete mode 100644 cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols1536/user_nl_cam delete mode 100644 cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols1536/user_nl_clm diff --git a/cime_config/testdefs/testlist_cam.xml b/cime_config/testdefs/testlist_cam.xml index 05983cca7b..cd4b65be41 100644 --- a/cime_config/testdefs/testlist_cam.xml +++ b/cime_config/testdefs/testlist_cam.xml @@ -1569,51 +1569,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg2_default/shell_commands b/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg2_default/shell_commands deleted file mode 100644 index 89516e5375..0000000000 --- a/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg2_default/shell_commands +++ /dev/null @@ -1,7 +0,0 @@ -./xmlchange NTASKS=36 -./xmlchange NTHRDS=1 -./xmlchange ROOTPE='0' -./xmlchange ROF_NCPL=`./xmlquery --value ATM_NCPL` -./xmlchange GLC_NCPL=`./xmlquery --value ATM_NCPL` -./xmlchange TIMER_DETAIL='6' -./xmlchange TIMER_LEVEL='999' diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg2_default/user_nl_cam b/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg2_default/user_nl_cam deleted file mode 100644 index 8482082dce..0000000000 --- a/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg2_default/user_nl_cam +++ /dev/null @@ -1,4 +0,0 @@ -mfilt=1,1,1,1,1,1 -ndens=1,1,1,1,1,1 -nhtfrq=9,9,9,9,9,9 -inithist='ENDOFRUN' diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg2_default/user_nl_clm b/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg2_default/user_nl_clm deleted file mode 100644 index 12d5a36d2b..0000000000 --- a/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg2_default/user_nl_clm +++ /dev/null @@ -1,26 +0,0 @@ -!---------------------------------------------------------------------------------- -! Users should add all user specific namelist changes below in the form of -! namelist_var = new_namelist_value -! -! Include namelist variables for drv_flds_in ONLY if -megan and/or -drydep options -! are set in the CLM_NAMELIST_OPTS env variable. -! -! EXCEPTIONS: -! Set use_cndv by the compset you use and the CLM_BLDNML_OPTS -dynamic_vegetation setting -! Set use_vichydro by the compset you use and the CLM_BLDNML_OPTS -vichydro setting -! Set use_cn by the compset you use and CLM_BLDNML_OPTS -bgc setting -! Set use_crop by the compset you use and CLM_BLDNML_OPTS -crop setting -! Set spinup_state by the CLM_BLDNML_OPTS -bgc_spinup setting -! Set irrigate by the CLM_BLDNML_OPTS -irrig setting -! Set dtime with L_NCPL option -! Set fatmlndfrc with LND_DOMAIN_PATH/LND_DOMAIN_FILE options -! Set finidat with RUN_REFCASE/RUN_REFDATE/RUN_REFTOD options for hybrid or branch cases -! (includes $inst_string for multi-ensemble cases) -! Set glc_grid with CISM_GRID option -! Set glc_smb with GLC_SMB option -! Set maxpatch_glcmec with GLC_NEC option -! Set glc_do_dynglacier with GLC_TWO_WAY_COUPLING env variable -!---------------------------------------------------------------------------------- -hist_nhtfrq = 9 -hist_mfilt = 1 -hist_ndens = 1 diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_nondefault/shell_commands b/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_nondefault/shell_commands deleted file mode 100644 index 9fdcee8bfd..0000000000 --- a/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_nondefault/shell_commands +++ /dev/null @@ -1,8 +0,0 @@ -./xmlchange NTASKS=36 -./xmlchange NTHRDS=1 -./xmlchange ROOTPE='0' -./xmlchange ROF_NCPL=`./xmlquery --value ATM_NCPL` -./xmlchange GLC_NCPL=`./xmlquery --value ATM_NCPL` -./xmlchange CAM_CONFIG_OPTS=' -microphys mg3' --append -./xmlchange TIMER_DETAIL='6' -./xmlchange TIMER_LEVEL='999' diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_nondefault/user_nl_cam b/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_nondefault/user_nl_cam deleted file mode 100644 index 8bb09f9ffc..0000000000 --- a/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_nondefault/user_nl_cam +++ /dev/null @@ -1,24 +0,0 @@ -mfilt=1,1,1,1,1,1 -ndens=1,1,1,1,1,1 -nhtfrq=9,9,9,9,9,9 -inithist='ENDOFRUN' -micro_mg_do_graupel=.false. -micro_mg_do_hail=.true. -micro_do_sb_physics=.true. -micro_do_massless_droplet_destroyer=.true. -microp_uniform=.true. -micro_mg_nccons=.true. -micro_mg_nicons=.true. -micro_mg_ngcons=.true. -micro_mg_nrcons=.true. -micro_mg_nscons=.true. -micro_mg_evap_sed_off=.true. -micro_mg_icenuc_rh_off=.true. -micro_mg_icenuc_use_meyers=.true. -micro_mg_evap_scl_ifs=.true. -micro_mg_evap_rhthrsh_ifs=.true. -micro_mg_rainfreeze_ifs=.true. -micro_mg_ifs_sed=.true. -micro_mg_precip_fall_corr=.true. -micro_mg_implicit_fall=.false. -micro_mg_accre_sees_auto=.true. diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_nondefault/user_nl_clm b/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_nondefault/user_nl_clm deleted file mode 100644 index 12d5a36d2b..0000000000 --- a/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_nondefault/user_nl_clm +++ /dev/null @@ -1,26 +0,0 @@ -!---------------------------------------------------------------------------------- -! Users should add all user specific namelist changes below in the form of -! namelist_var = new_namelist_value -! -! Include namelist variables for drv_flds_in ONLY if -megan and/or -drydep options -! are set in the CLM_NAMELIST_OPTS env variable. -! -! EXCEPTIONS: -! Set use_cndv by the compset you use and the CLM_BLDNML_OPTS -dynamic_vegetation setting -! Set use_vichydro by the compset you use and the CLM_BLDNML_OPTS -vichydro setting -! Set use_cn by the compset you use and CLM_BLDNML_OPTS -bgc setting -! Set use_crop by the compset you use and CLM_BLDNML_OPTS -crop setting -! Set spinup_state by the CLM_BLDNML_OPTS -bgc_spinup setting -! Set irrigate by the CLM_BLDNML_OPTS -irrig setting -! Set dtime with L_NCPL option -! Set fatmlndfrc with LND_DOMAIN_PATH/LND_DOMAIN_FILE options -! Set finidat with RUN_REFCASE/RUN_REFDATE/RUN_REFTOD options for hybrid or branch cases -! (includes $inst_string for multi-ensemble cases) -! Set glc_grid with CISM_GRID option -! Set glc_smb with GLC_SMB option -! Set maxpatch_glcmec with GLC_NEC option -! Set glc_do_dynglacier with GLC_TWO_WAY_COUPLING env variable -!---------------------------------------------------------------------------------- -hist_nhtfrq = 9 -hist_mfilt = 1 -hist_ndens = 1 diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols1536/shell_commands b/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols1536/shell_commands deleted file mode 100644 index d6e6750eb4..0000000000 --- a/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols1536/shell_commands +++ /dev/null @@ -1,8 +0,0 @@ -./xmlchange NTASKS=36 -./xmlchange NTHRDS=1 -./xmlchange ROOTPE='0' -./xmlchange ROF_NCPL=`./xmlquery --value ATM_NCPL` -./xmlchange GLC_NCPL=`./xmlquery --value ATM_NCPL` -./xmlchange CAM_CONFIG_OPTS=' -microphys mg3 -pcols 1536' --append -./xmlchange TIMER_DETAIL='6' -./xmlchange TIMER_LEVEL='999' diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols1536/user_nl_cam b/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols1536/user_nl_cam deleted file mode 100644 index 8482082dce..0000000000 --- a/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols1536/user_nl_cam +++ /dev/null @@ -1,4 +0,0 @@ -mfilt=1,1,1,1,1,1 -ndens=1,1,1,1,1,1 -nhtfrq=9,9,9,9,9,9 -inithist='ENDOFRUN' diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols1536/user_nl_clm b/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols1536/user_nl_clm deleted file mode 100644 index 12d5a36d2b..0000000000 --- a/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols1536/user_nl_clm +++ /dev/null @@ -1,26 +0,0 @@ -!---------------------------------------------------------------------------------- -! Users should add all user specific namelist changes below in the form of -! namelist_var = new_namelist_value -! -! Include namelist variables for drv_flds_in ONLY if -megan and/or -drydep options -! are set in the CLM_NAMELIST_OPTS env variable. -! -! EXCEPTIONS: -! Set use_cndv by the compset you use and the CLM_BLDNML_OPTS -dynamic_vegetation setting -! Set use_vichydro by the compset you use and the CLM_BLDNML_OPTS -vichydro setting -! Set use_cn by the compset you use and CLM_BLDNML_OPTS -bgc setting -! Set use_crop by the compset you use and CLM_BLDNML_OPTS -crop setting -! Set spinup_state by the CLM_BLDNML_OPTS -bgc_spinup setting -! Set irrigate by the CLM_BLDNML_OPTS -irrig setting -! Set dtime with L_NCPL option -! Set fatmlndfrc with LND_DOMAIN_PATH/LND_DOMAIN_FILE options -! Set finidat with RUN_REFCASE/RUN_REFDATE/RUN_REFTOD options for hybrid or branch cases -! (includes $inst_string for multi-ensemble cases) -! Set glc_grid with CISM_GRID option -! Set glc_smb with GLC_SMB option -! Set maxpatch_glcmec with GLC_NEC option -! Set glc_do_dynglacier with GLC_TWO_WAY_COUPLING env variable -!---------------------------------------------------------------------------------- -hist_nhtfrq = 9 -hist_mfilt = 1 -hist_ndens = 1 From e79c4a388ea7c51286a339e3c6baf9100e263842 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Tue, 21 May 2024 11:40:49 -0600 Subject: [PATCH 07/22] add GPU test suite on Derecho --- cime_config/testdefs/testlist_cam.xml | 16 ++++++++++++ .../cam/outfrq9s_mg3_default/shell_commands | 2 +- .../cam/outfrq9s_mg3_pcols760/shell_commands | 8 ++++++ .../cam/outfrq9s_mg3_pcols760/user_nl_cam | 4 +++ .../cam/outfrq9s_mg3_pcols760/user_nl_clm | 26 +++++++++++++++++++ 5 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols760/shell_commands create mode 100644 cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols760/user_nl_cam create mode 100644 cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols760/user_nl_clm diff --git a/cime_config/testdefs/testlist_cam.xml b/cime_config/testdefs/testlist_cam.xml index cd4b65be41..d464d825d1 100644 --- a/cime_config/testdefs/testlist_cam.xml +++ b/cime_config/testdefs/testlist_cam.xml @@ -1485,6 +1485,22 @@ + + + + + + + + + + + + + + + + diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_default/shell_commands b/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_default/shell_commands index 9fdcee8bfd..23dac55242 100644 --- a/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_default/shell_commands +++ b/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_default/shell_commands @@ -1,4 +1,4 @@ -./xmlchange NTASKS=36 +./xmlchange NTASKS=128 ./xmlchange NTHRDS=1 ./xmlchange ROOTPE='0' ./xmlchange ROF_NCPL=`./xmlquery --value ATM_NCPL` diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols760/shell_commands b/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols760/shell_commands new file mode 100644 index 0000000000..f9424e5025 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols760/shell_commands @@ -0,0 +1,8 @@ +./xmlchange NTASKS=64 +./xmlchange NTHRDS=1 +./xmlchange ROOTPE='0' +./xmlchange ROF_NCPL=`./xmlquery --value ATM_NCPL` +./xmlchange GLC_NCPL=`./xmlquery --value ATM_NCPL` +./xmlchange CAM_CONFIG_OPTS=' -microphys mg3 -pcols 760 ' --append +./xmlchange TIMER_DETAIL='6' +./xmlchange TIMER_LEVEL='999' diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols760/user_nl_cam b/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols760/user_nl_cam new file mode 100644 index 0000000000..8482082dce --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols760/user_nl_cam @@ -0,0 +1,4 @@ +mfilt=1,1,1,1,1,1 +ndens=1,1,1,1,1,1 +nhtfrq=9,9,9,9,9,9 +inithist='ENDOFRUN' diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols760/user_nl_clm b/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols760/user_nl_clm new file mode 100644 index 0000000000..12d5a36d2b --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols760/user_nl_clm @@ -0,0 +1,26 @@ +!---------------------------------------------------------------------------------- +! Users should add all user specific namelist changes below in the form of +! namelist_var = new_namelist_value +! +! Include namelist variables for drv_flds_in ONLY if -megan and/or -drydep options +! are set in the CLM_NAMELIST_OPTS env variable. +! +! EXCEPTIONS: +! Set use_cndv by the compset you use and the CLM_BLDNML_OPTS -dynamic_vegetation setting +! Set use_vichydro by the compset you use and the CLM_BLDNML_OPTS -vichydro setting +! Set use_cn by the compset you use and CLM_BLDNML_OPTS -bgc setting +! Set use_crop by the compset you use and CLM_BLDNML_OPTS -crop setting +! Set spinup_state by the CLM_BLDNML_OPTS -bgc_spinup setting +! Set irrigate by the CLM_BLDNML_OPTS -irrig setting +! Set dtime with L_NCPL option +! Set fatmlndfrc with LND_DOMAIN_PATH/LND_DOMAIN_FILE options +! Set finidat with RUN_REFCASE/RUN_REFDATE/RUN_REFTOD options for hybrid or branch cases +! (includes $inst_string for multi-ensemble cases) +! Set glc_grid with CISM_GRID option +! Set glc_smb with GLC_SMB option +! Set maxpatch_glcmec with GLC_NEC option +! Set glc_do_dynglacier with GLC_TWO_WAY_COUPLING env variable +!---------------------------------------------------------------------------------- +hist_nhtfrq = 9 +hist_mfilt = 1 +hist_ndens = 1 From e779d59601195b6f8ac8d0eea5980f3b055c13eb Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Fri, 24 May 2024 13:37:05 -0600 Subject: [PATCH 08/22] swithc ERP to ERS test as nvhpc seems to have a bug with openmp threading --- cime_config/testdefs/testlist_cam.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cime_config/testdefs/testlist_cam.xml b/cime_config/testdefs/testlist_cam.xml index d464d825d1..12e7648a91 100644 --- a/cime_config/testdefs/testlist_cam.xml +++ b/cime_config/testdefs/testlist_cam.xml @@ -1485,7 +1485,7 @@ - + @@ -1493,14 +1493,14 @@ - + - + From 871336fe1943562556e45def37becc21297c455b Mon Sep 17 00:00:00 2001 From: Katetc Date: Tue, 25 Jun 2024 15:29:03 -0600 Subject: [PATCH 09/22] Updates for the new external --- .gitmodules | 2 +- bld/namelist_files/namelist_defaults_cam.xml | 4 + src/physics/cam/clubb_intr.F90 | 324 +++++++++++-------- src/physics/cam/subcol_SILHS.F90 | 162 +++++----- 4 files changed, 286 insertions(+), 206 deletions(-) diff --git a/.gitmodules b/.gitmodules index 77e9c2fc56..87a783bf6f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -96,7 +96,7 @@ url = https://github.com/larson-group/clubb_release fxrequired = AlwaysRequired fxsparse = ../.clubb_sparse_checkout - fxtag = clubb_4ncar_20231115_5406350 + fxtag = clubb_4ncar_20240605_73d60f6 fxDONOTUSEurl = https://github.com/larson-group/clubb_release [submodule "cism"] diff --git a/bld/namelist_files/namelist_defaults_cam.xml b/bld/namelist_files/namelist_defaults_cam.xml index 331863e4e2..247bacf527 100644 --- a/bld/namelist_files/namelist_defaults_cam.xml +++ b/bld/namelist_files/namelist_defaults_cam.xml @@ -2236,6 +2236,7 @@ .false. .true. .true. + .true. 0.2 @@ -2261,6 +2262,7 @@ 10.0 4.0 0.0 + 5.0 .true. .false. @@ -2279,6 +2281,8 @@ .false. .false. .false. + 0.5 + 25.00 diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index c5bdcd71ce..ce8c14e096 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -31,8 +31,10 @@ module clubb_intr #ifdef CLUBB_SGS use clubb_api_module, only: pdf_parameter, implicit_coefs_terms - use clubb_api_module, only: clubb_config_flags_type, grid, stats, & - nu_vertical_res_dep, stats_metadata_type + use clubb_api_module, only: clubb_config_flags_type, grid, stats, & + nu_vertical_res_dep, stats_metadata_type, & + hm_metadata_type, sclr_idx_type + use clubb_api_module, only: nparams use clubb_mf, only: do_clubb_mf, do_clubb_mf_diag use cloud_fraction, only: dp1, dp2 @@ -51,6 +53,17 @@ module clubb_intr type (stats_metadata_type) :: & stats_metadata +!$omp threadprivate(stats_metadata) + + type (hm_metadata_type) :: & + hm_metadata + +!$omp threadprivate(hm_metadata) + + type (sclr_idx_type) :: & + sclr_idx + +!$omp threadprivate(sclr_idx) #endif @@ -84,7 +97,30 @@ module clubb_intr #ifdef CLUBB_SGS type(clubb_config_flags_type), public :: clubb_config_flags - real(r8), dimension(nparams), public :: clubb_params ! Adjustable CLUBB parameters (C1, C2 ...) + real(r8), dimension(nparams), public :: clubb_params_single_col ! Adjustable CLUBB parameters (C1, C2 ...) +#endif + + ! These are zero by default, but will be set by SILHS before + integer :: & + hydromet_dim = 0, & + pdf_dim = 0 + + + ! ------------------------ ! + ! Sometimes private data ! + ! ------------------------ ! +#ifdef SILHS + ! If SILHS is in use, it will initialize these + public :: & + hydromet_dim, & + pdf_dim, & + hm_metadata +#else + ! If SILHS is not in use, there is no need for them to be public + private :: & + hydromet_dim, & + pdf_dim, & + hm_metadata #endif ! ------------ ! @@ -93,23 +129,18 @@ module clubb_intr integer, parameter :: & grid_type = 3, & ! The 2 option specifies stretched thermodynamic levels - hydromet_dim = 0 ! The hydromet array in SAM-CLUBB is currently 0 elements + sclr_dim = 0 ! Higher-order scalars, set to zero ! Even though sclr_dim is set to 0, the dimension here is set to 1 to prevent compiler errors ! See github ticket larson-group/cam#133 for details real(r8), parameter, dimension(1) :: & sclr_tol = 1.e-8_r8 ! Total water in kg/kg - character(len=6) :: saturation_equation - real(r8), parameter :: & theta0 = 300._r8, & ! Reference temperature [K] ts_nudge = 86400._r8, & ! Time scale for u/v nudging (not used) [s] p0_clubb = 100000._r8 - integer, parameter :: & - sclr_dim = 0 ! Higher-order scalars, set to zero - real(r8), parameter :: & wp3_const = 1._r8 ! Constant to add to wp3 when moments are advected @@ -195,8 +226,8 @@ module clubb_intr clubb_ipdf_call_placement = unset_i, & ! Selected option for the placement of the call to ! CLUBB's PDF. clubb_penta_solve_method = unset_i, & ! Specifier for method to solve the penta-diagonal system - clubb_tridiag_solve_method = unset_i ! Specifier for method to solve tri-diagonal systems - + clubb_tridiag_solve_method = unset_i,& ! Specifier for method to solve tri-diagonal systems + clubb_saturation_equation = unset_i ! Specifier for which saturation formula to use logical :: & @@ -315,13 +346,15 @@ module clubb_intr clubb_l_mono_flux_lim_vm, & ! Flag to turn on monotonic flux limiter for vm clubb_l_mono_flux_lim_spikefix, & ! Flag to implement monotonic flux limiter code that ! eliminates spurious drying tendencies at model top - clubb_l_intr_sfc_flux_smooth = .false. ! Add a locally calculated roughness to upwp and vpwp sfc fluxes + clubb_l_host_applies_sfc_fluxes ! Whether the host model applies the surface fluxes + + logical :: & + clubb_l_intr_sfc_flux_smooth = .false. ! Add a locally calculated roughness to upwp and vpwp sfc fluxes ! Constant parameters logical, parameter, private :: & - l_implemented = .true., & ! Implemented in a host model (always true) - l_host_applies_sfc_fluxes = .false. ! Whether the host model applies the surface fluxes - + l_implemented = .true. ! Implemented in a host model (always true) + logical, parameter, private :: & apply_to_heat = .false. ! Apply WACCM energy fixer to heat or not (.true. = yes (duh)) @@ -487,13 +520,8 @@ subroutine clubb_register_cam( ) history_budget_out = history_budget, & history_budget_histfile_num_out = history_budget_histfile_num, & do_hb_above_clubb_out = do_hb_above_clubb) - subcol_scheme = subcol_get_scheme() - if (trim(subcol_scheme) == 'SILHS') then - saturation_equation = "flatau" - else - saturation_equation = "gfdl" ! Goff & Gratch (1946) approximation for SVP - end if + subcol_scheme = subcol_get_scheme() if (clubb_do_adv) then cnst_names =(/'THLP2 ','RTP2 ','RTPTHLP','WPTHLP ','WPRTP ','WP2 ','WP3 ','UP2 ','VP2 '/) @@ -856,6 +884,7 @@ subroutine clubb_readnl(nlfile) clubb_ipdf_call_placement, & ! Out clubb_penta_solve_method, & ! Out clubb_tridiag_solve_method, & ! Out + clubb_saturation_equation, & ! Out clubb_l_use_precip_frac, & ! Out clubb_l_predict_upwp_vpwp, & ! Out clubb_l_min_wp2_from_corr_wx, & ! Out @@ -909,7 +938,8 @@ subroutine clubb_readnl(nlfile) clubb_l_mono_flux_lim_rtm, & ! Out clubb_l_mono_flux_lim_um, & ! Out clubb_l_mono_flux_lim_vm, & ! Out - clubb_l_mono_flux_lim_spikefix ) ! Out + clubb_l_mono_flux_lim_spikefix, & ! Out + clubb_l_host_applies_sfc_fluxes ) ! Out ! Call CLUBB+MF namelist call clubb_mf_readnl(nlfile) @@ -1158,10 +1188,14 @@ subroutine clubb_readnl(nlfile) if (ierr /= 0) call endrun(sub//": FATAL: mpi_bcast: clubb_l_mono_flux_lim_vm") call mpi_bcast(clubb_l_mono_flux_lim_spikefix, 1, mpi_logical, mstrid, mpicom, ierr) if (ierr /= 0) call endrun(sub//": FATAL: mpi_bcast: clubb_l_mono_flux_lim_spikefix") + call mpi_bcast(clubb_l_host_applies_sfc_fluxes, 1, mpi_logical, mstrid, mpicom, ierr) + if (ierr /= 0) call endrun(sub//": FATAL: mpi_bcast: clubb_l_host_applies_sfc_fluxes") call mpi_bcast(clubb_penta_solve_method, 1, mpi_integer, mstrid, mpicom, ierr) if (ierr /= 0) call endrun(sub//": FATAL: mpi_bcast: clubb_penta_solve_method") call mpi_bcast(clubb_tridiag_solve_method, 1, mpi_integer, mstrid, mpicom, ierr) if (ierr /= 0) call endrun(sub//": FATAL: mpi_bcast: clubb_tridiag_solve_method") + call mpi_bcast(clubb_saturation_equation, 1, mpi_integer, mstrid, mpicom, ierr) + if (ierr /= 0) call endrun(sub//": FATAL: mpi_bcast: clubb_saturation_equation") call mpi_bcast(clubb_l_intr_sfc_flux_smooth, 1, mpi_logical, mstrid, mpicom, ierr) if (ierr /= 0) call endrun(sub//": FATAL: mpi_bcast: clubb_l_intr_sfc_flux_smooth") call mpi_bcast(clubb_l_vary_convect_depth, 1, mpi_logical, mstrid, mpicom, ierr) @@ -1267,6 +1301,7 @@ subroutine clubb_readnl(nlfile) if(clubb_detphase_lowtemp == unset_r8) call endrun(sub//": FATAL: clubb_detphase_lowtemp not set") if(clubb_penta_solve_method == unset_i) call endrun(sub//": FATAL: clubb_penta_solve_method not set") if(clubb_tridiag_solve_method == unset_i) call endrun(sub//": FATAL: clubb_tridiag_solve_method not set") + if(clubb_saturation_equation == unset_i) call endrun(sub//": FATAL: clubb_saturation_equation not set") if(clubb_detphase_lowtemp >= meltpt_temp) & call endrun(sub//": ERROR: clubb_detphase_lowtemp must be less than 268.15 K") @@ -1274,6 +1309,7 @@ subroutine clubb_readnl(nlfile) clubb_ipdf_call_placement, & ! In clubb_penta_solve_method, & ! In clubb_tridiag_solve_method, & ! In + clubb_saturation_equation, & ! In clubb_l_use_precip_frac, & ! In clubb_l_predict_upwp_vpwp, & ! In clubb_l_min_wp2_from_corr_wx, & ! In @@ -1328,6 +1364,7 @@ subroutine clubb_readnl(nlfile) clubb_l_mono_flux_lim_um, & ! In clubb_l_mono_flux_lim_vm, & ! In clubb_l_mono_flux_lim_spikefix, & ! In + clubb_l_host_applies_sfc_fluxes, & ! In clubb_config_flags ) ! Out #endif @@ -1373,7 +1410,8 @@ subroutine clubb_ini_cam(pbuf2d) use clubb_api_module, only: & print_clubb_config_flags_api, & - setup_clubb_core_api, & + setup_parameters_model_api, & + check_clubb_settings_api, & init_pdf_params_api, & time_precision, & core_rknd, & @@ -1384,19 +1422,13 @@ subroutine clubb_ini_cam(pbuf2d) read_parameters_api, & w_tol_sqd, & rt_tol, & - thl_tol - - ! These are only needed if we're using a passive scalar - use clubb_api_module, only: & - iisclr_rt, & - iisclr_thl, & - iisclr_CO2, & - iiedsclr_rt, & - iiedsclr_thl, & - iiedsclr_CO2 + thl_tol, & + saturation_bolton, & ! Constant for Bolton approximations of saturation + saturation_gfdl, & ! Constant for the GFDL approximation of saturation + saturation_flatau, & ! Constant for Flatau approximations of saturation + saturation_lookup ! Use a lookup table for mixing length use time_manager, only: is_first_step - use clubb_api_module, only: hydromet_dim use constituents, only: cnst_get_ind use phys_control, only: phys_getopts use spmd_utils, only: iam @@ -1552,13 +1584,13 @@ subroutine clubb_ini_cam(pbuf2d) npccn_idx = pbuf_get_index('NPCCN') - iisclr_rt = -1 - iisclr_thl = -1 - iisclr_CO2 = -1 + sclr_idx%iisclr_rt = -1 + sclr_idx%iisclr_thl = -1 + sclr_idx%iisclr_CO2 = -1 - iiedsclr_rt = -1 - iiedsclr_thl = -1 - iiedsclr_CO2 = -1 + sclr_idx%iiedsclr_rt = -1 + sclr_idx%iiedsclr_thl = -1 + sclr_idx%iiedsclr_CO2 = -1 ! ----------------------------------------------------------------- ! ! Define number of tracers for CLUBB to diffuse @@ -1600,7 +1632,7 @@ subroutine clubb_ini_cam(pbuf2d) Cx_min, Cx_max, Richardson_num_min, Richardson_num_max, & wpxp_Ri_exp, a3_coef_min, a_const, bv_efold, z_displace ) - call read_parameters_api( -99, "", & + call read_parameters_api( 1, -99, "", & C1, C1b, C1c, C2rt, C2thl, C2rtthl, & C4, C_uu_shr, C_uu_buoy, C6rt, C6rtb, C6rtc, & C6thl, C6thlb, C6thlc, C7, C7b, C7c, C8, C8b, C10, & @@ -1625,75 +1657,83 @@ subroutine clubb_ini_cam(pbuf2d) C_invrs_tau_wpxp_Ri, C_invrs_tau_wpxp_N2_thresh, & Cx_min, Cx_max, Richardson_num_min, Richardson_num_max, & wpxp_Ri_exp, a3_coef_min, a_const, bv_efold, z_displace, & - clubb_params ) - - clubb_params(iC2rtthl) = clubb_C2rtthl - clubb_params(iC8) = clubb_C8 - clubb_params(iC11) = clubb_c11 - clubb_params(iC11b) = clubb_c11b - clubb_params(iC14) = clubb_c14 - clubb_params(iC_wp3_pr_turb) = clubb_C_wp3_pr_turb - clubb_params(ic_K10) = clubb_c_K10 - clubb_params(imult_coef) = clubb_mult_coef - clubb_params(iSkw_denom_coef) = clubb_Skw_denom_coef - clubb_params(iC2rt) = clubb_C2rt - clubb_params(iC2thl) = clubb_C2thl - clubb_params(ibeta) = clubb_beta - clubb_params(iC6rt) = clubb_c6rt - clubb_params(iC6rtb) = clubb_c6rtb - clubb_params(iC6rtc) = clubb_c6rtc - clubb_params(iC6thl) = clubb_c6thl - clubb_params(iC6thlb) = clubb_c6thlb - clubb_params(iC6thlc) = clubb_c6thlc - clubb_params(iwpxp_L_thresh) = clubb_wpxp_L_thresh - clubb_params(iC7) = clubb_C7 - clubb_params(iC7b) = clubb_C7b - clubb_params(igamma_coef) = clubb_gamma_coef - clubb_params(ic_K10h) = clubb_c_K10h - clubb_params(ilambda0_stability_coef) = clubb_lambda0_stability_coef - clubb_params(ilmin_coef) = clubb_lmin_coef - clubb_params(iC8b) = clubb_C8b - clubb_params(iskw_max_mag) = clubb_skw_max_mag - clubb_params(iC1) = clubb_C1 - clubb_params(iC1b) = clubb_C1b - clubb_params(igamma_coefb) = clubb_gamma_coefb - clubb_params(iup2_sfc_coef) = clubb_up2_sfc_coef - clubb_params(iC4) = clubb_C4 - clubb_params(iC_uu_shr) = clubb_C_uu_shr - clubb_params(iC_uu_buoy) = clubb_C_uu_buoy - clubb_params(ic_K1) = clubb_c_K1 - clubb_params(ic_K2) = clubb_c_K2 - clubb_params(inu2) = clubb_nu2 - clubb_params(ic_K8) = clubb_c_K8 - clubb_params(ic_K9) = clubb_c_K9 - clubb_params(inu9) = clubb_nu9 - clubb_params(iC_wp2_splat) = clubb_C_wp2_splat - clubb_params(iC_invrs_tau_bkgnd) = clubb_C_invrs_tau_bkgnd - clubb_params(iC_invrs_tau_sfc) = clubb_C_invrs_tau_sfc - clubb_params(iC_invrs_tau_shear) = clubb_C_invrs_tau_shear - clubb_params(iC_invrs_tau_N2) = clubb_C_invrs_tau_N2 - clubb_params(iC_invrs_tau_N2_wp2) = clubb_C_invrs_tau_N2_wp2 - clubb_params(iC_invrs_tau_N2_xp2) = clubb_C_invrs_tau_N2_xp2 - clubb_params(iC_invrs_tau_N2_wpxp) = clubb_C_invrs_tau_N2_wpxp - clubb_params(iC_invrs_tau_N2_clear_wp3) = clubb_C_invrs_tau_N2_clear_wp3 - clubb_params(ibv_efold) = clubb_bv_efold - clubb_params(iwpxp_Ri_exp) = clubb_wpxp_Ri_exp - clubb_params(iz_displace) = clubb_z_displace + clubb_params_single_col ) + + clubb_params_single_col(iC2rtthl) = clubb_C2rtthl + clubb_params_single_col(iC8) = clubb_C8 + clubb_params_single_col(iC11) = clubb_c11 + clubb_params_single_col(iC11b) = clubb_c11b + clubb_params_single_col(iC14) = clubb_c14 + clubb_params_single_col(iC_wp3_pr_turb) = clubb_C_wp3_pr_turb + clubb_params_single_col(ic_K10) = clubb_c_K10 + clubb_params_single_col(imult_coef) = clubb_mult_coef + clubb_params_single_col(iSkw_denom_coef) = clubb_Skw_denom_coef + clubb_params_single_col(iC2rt) = clubb_C2rt + clubb_params_single_col(iC2thl) = clubb_C2thl + clubb_params_single_col(ibeta) = clubb_beta + clubb_params_single_col(iC6rt) = clubb_c6rt + clubb_params_single_col(iC6rtb) = clubb_c6rtb + clubb_params_single_col(iC6rtc) = clubb_c6rtc + clubb_params_single_col(iC6thl) = clubb_c6thl + clubb_params_single_col(iC6thlb) = clubb_c6thlb + clubb_params_single_col(iC6thlc) = clubb_c6thlc + clubb_params_single_col(iwpxp_L_thresh) = clubb_wpxp_L_thresh + clubb_params_single_col(iC7) = clubb_C7 + clubb_params_single_col(iC7b) = clubb_C7b + clubb_params_single_col(igamma_coef) = clubb_gamma_coef + clubb_params_single_col(ic_K10h) = clubb_c_K10h + clubb_params_single_col(ilambda0_stability_coef) = clubb_lambda0_stability_coef + clubb_params_single_col(ilmin_coef) = clubb_lmin_coef + clubb_params_single_col(iC8b) = clubb_C8b + clubb_params_single_col(iskw_max_mag) = clubb_skw_max_mag + clubb_params_single_col(iC1) = clubb_C1 + clubb_params_single_col(iC1b) = clubb_C1b + clubb_params_single_col(igamma_coefb) = clubb_gamma_coefb + clubb_params_single_col(iup2_sfc_coef) = clubb_up2_sfc_coef + clubb_params_single_col(iC4) = clubb_C4 + clubb_params_single_col(iC_uu_shr) = clubb_C_uu_shr + clubb_params_single_col(iC_uu_buoy) = clubb_C_uu_buoy + clubb_params_single_col(ic_K1) = clubb_c_K1 + clubb_params_single_col(ic_K2) = clubb_c_K2 + clubb_params_single_col(inu2) = clubb_nu2 + clubb_params_single_col(ic_K8) = clubb_c_K8 + clubb_params_single_col(ic_K9) = clubb_c_K9 + clubb_params_single_col(inu9) = clubb_nu9 + clubb_params_single_col(iC_wp2_splat) = clubb_C_wp2_splat + clubb_params_single_col(iC_invrs_tau_bkgnd) = clubb_C_invrs_tau_bkgnd + clubb_params_single_col(iC_invrs_tau_sfc) = clubb_C_invrs_tau_sfc + clubb_params_single_col(iC_invrs_tau_shear) = clubb_C_invrs_tau_shear + clubb_params_single_col(iC_invrs_tau_N2) = clubb_C_invrs_tau_N2 + clubb_params_single_col(iC_invrs_tau_N2_wp2) = clubb_C_invrs_tau_N2_wp2 + clubb_params_single_col(iC_invrs_tau_N2_xp2) = clubb_C_invrs_tau_N2_xp2 + clubb_params_single_col(iC_invrs_tau_N2_wpxp) = clubb_C_invrs_tau_N2_wpxp + clubb_params_single_col(iC_invrs_tau_N2_clear_wp3) = clubb_C_invrs_tau_N2_clear_wp3 + clubb_params_single_col(ibv_efold) = clubb_bv_efold + clubb_params_single_col(iwpxp_Ri_exp) = clubb_wpxp_Ri_exp + clubb_params_single_col(iz_displace) = clubb_z_displace + + + ! Override clubb default + if ( trim(subcol_scheme) == 'SILHS' ) then + clubb_config_flags%saturation_formula = saturation_flatau + else + clubb_config_flags%saturation_formula = saturation_gfdl ! Goff & Gratch (1946) approximation for SVP + end if + + + ! Define model constant parameters + call setup_parameters_model_api( theta0, ts_nudge, clubb_params_single_col(iSkw_max_mag) ) ! Set up CLUBB core. Note that some of these inputs are overwritten ! when clubb_tend_cam is called. The reason is that heights can change ! at each time step, which is why dummy arrays are read in here for heights ! as they are immediately overwrote. !$OMP PARALLEL - call setup_clubb_core_api( & - nlev+1, theta0, ts_nudge, & ! In - hydromet_dim, sclr_dim, & ! In - sclr_tol, edsclr_dim, clubb_params, & ! In - l_host_applies_sfc_fluxes, & ! In - saturation_equation, & ! In - l_input_fields, & ! In - clubb_config_flags, & ! In - err_code ) ! Out + call check_clubb_settings_api( nlev+1, clubb_params_single_col, & ! Intent(in) + l_implemented, & ! Intent(in) + l_input_fields, & ! Intent(in) + clubb_config_flags, & ! intent(in) + err_code ) ! Intent(out) if ( err_code == clubb_fatal_error ) then call endrun('clubb_ini_cam: FATAL ERROR CALLING SETUP_CLUBB_CORE') @@ -1703,7 +1743,7 @@ subroutine clubb_ini_cam(pbuf2d) ! Print the list of CLUBB parameters if ( masterproc ) then do j = 1, nparams, 1 - write(iulog,*) params_list(j), " = ", clubb_params(j) + write(iulog,*) params_list(j), " = ", clubb_params_single_col(j) enddo endif @@ -1810,18 +1850,25 @@ subroutine clubb_ini_cam(pbuf2d) call addfld ( 'edmf_qtflx' , (/ 'ilev' /), 'A', 'W/m2' , 'qt flux (EDMF)' ) end if +#ifndef SILHS + ! hm_metadata is set up by calling init_pdf_hydromet_arrays_api in subcol_init_SILHS. + ! So if we are not using silhs, we allocate the parts of hm_metadata that need allocating + ! in order to making intel debug tests happy. + allocate( hm_metadata%hydromet_list(1) ) + allocate( hm_metadata%l_mix_rat_hm(1) ) +#endif + ! Initialize statistics, below are dummy variables dum1 = 300._r8 dum2 = 1200._r8 dum3 = 300._r8 - if (stats_metadata%l_stats) then - - call stats_init_clubb( .true., dum1, dum2, & - nlev+1, nlev+1, nlev+1, dum3, & - stats_zt(:), stats_zm(:), stats_sfc(:), & - stats_rad_zt(:), stats_rad_zm(:)) + + call stats_init_clubb( .true., dum1, dum2, & + nlev+1, nlev+1, nlev+1, dum3, & + stats_zt(:), stats_zm(:), stats_sfc(:), & + stats_rad_zt(:), stats_rad_zm(:)) allocate(out_zt(pcols,pverp,stats_zt(1)%num_output_fields), stat=ierr) if( ierr /= 0 ) call endrun( 'clubb_ini_cam: Unable to allocate out_zt' ) @@ -2046,7 +2093,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & rt_tol, & thl_tol, & stats_begin_timestep_api, & - hydromet_dim, calculate_thlp2_rad_api, update_xp2_mc_api, & + calculate_thlp2_rad_api, update_xp2_mc_api, & sat_mixrat_liq_api, & fstderr, & ipdf_post_advance_fields, & @@ -2255,7 +2302,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & wp2up2_inout, & ! w'^2 u'^2 (momentum levels) wp2vp2_inout, & ! w'^2 v'^2 (momentum levels) zt_g, & ! Thermodynamic grid of CLUBB [m] - zi_g ! Momentum grid of CLUBB [m] + zi_g ! Momentum grid of CLUBB [m] ! Local CLUBB variables dimensioned as NCOL (only useful columns) to be sent into the clubb run api ! NOTE: THESE VARIABLS SHOULD NOT BE USED IN PBUF OR OUTFLD (HISTORY) SUBROUTINES @@ -2495,6 +2542,9 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & type(nu_vertical_res_dep) :: nu_vert_res_dep ! Vertical resolution dependent nu values real(r8) :: lmin + real(r8), dimension(state%ncol,nparams) :: & + clubb_params ! Adjustable CLUBB parameters (C1, C2 ...) + #endif det_s(:) = 0.0_r8 det_ice(:) = 0.0_r8 @@ -3046,6 +3096,10 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & zi_g, zt_g, & ! intent(in) gr ) ! intent(out) + do i = 1, ncol + clubb_params(i,:) = clubb_params_single_col(:) + end do + call setup_parameters_api( zi_g(:,2), clubb_params, gr, ncol, grid_type, & ! intent(in) clubb_config_flags%l_prescribed_avg_deltaz, & ! intent(in) lmin, nu_vert_res_dep, err_code ) ! intent(out) @@ -3399,7 +3453,9 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Advance CLUBB CORE one timestep in the future call advance_clubb_core_api( gr, pverp+1-top_lev, ncol, & - l_implemented, dtime, fcor, sfc_elevation, hydromet_dim, & + l_implemented, dtime, fcor, sfc_elevation, & + hydromet_dim, & + sclr_dim, sclr_tol, edsclr_dim, sclr_idx, & thlm_forcing, rtm_forcing, um_forcing, vm_forcing, & sclrm_forcing, edsclrm_forcing, wprtp_forcing, & wpthlp_forcing, rtp2_forcing, thlp2_forcing, & @@ -3410,7 +3466,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & rtm_ref, thlm_ref, um_ref, vm_ref, ug, vg, & p_in_Pa, rho_zm, rho_zt, exner, & rho_ds_zm, rho_ds_zt, invrs_rho_ds_zm, & - invrs_rho_ds_zt, thv_ds_zm, thv_ds_zt, hydromet, & + invrs_rho_ds_zt, thv_ds_zm, thv_ds_zt, & + hydromet, hm_metadata%l_mix_rat_hm, & rfrzm, radf, & wphydrometp, wp2hmp, rtphmp_zt, thlphmp_zt, & grid_dx, grid_dy, & @@ -3491,7 +3548,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & thlp2_rad_out(:,:) = 0._r8 do i=1, ncol - call calculate_thlp2_rad_api(nlev+1, rcm_out_zm(i,:), thlprcp_out(i,:), qrl_zm(i,:), clubb_params, & + call calculate_thlp2_rad_api(nlev+1, rcm_out_zm(i,:), thlprcp_out(i,:), qrl_zm(i,:), clubb_params(i,:), & thlp2_rad_out(i,:)) end do @@ -3792,22 +3849,22 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & rtm_integral_ltend(:) = 0._r8 do k=1, pver - do i=1, ncol + do i=1, ncol - ptend_loc%u(i,k) = (um(i,k) - state1%u(i,k)) / hdtime ! east-west wind - ptend_loc%v(i,k) = (vm(i,k) - state1%v(i,k)) / hdtime ! north-south wind - ptend_loc%q(i,k,ixq) = (rtm(i,k) - rcm(i,k)-state1%q(i,k,ixq)) / hdtime ! water vapor - ptend_loc%q(i,k,ixcldliq) = (rcm(i,k) - state1%q(i,k,ixcldliq)) / hdtime ! Tendency of liquid water - ptend_loc%s(i,k) = (clubb_s(i,k) - state1%s(i,k)) / hdtime ! Tendency of static energy + ptend_loc%u(i,k) = (um(i,k) - state1%u(i,k)) / hdtime ! east-west wind + ptend_loc%v(i,k) = (vm(i,k) - state1%v(i,k)) / hdtime ! north-south wind + ptend_loc%q(i,k,ixq) = (rtm(i,k) - rcm(i,k)-state1%q(i,k,ixq)) / hdtime ! water vapor + ptend_loc%q(i,k,ixcldliq) = (rcm(i,k) - state1%q(i,k,ixcldliq)) / hdtime ! Tendency of liquid water + ptend_loc%s(i,k) = (clubb_s(i,k) - state1%s(i,k)) / hdtime ! Tendency of static energy - rtm_integral_ltend(i) = rtm_integral_ltend(i) + ptend_loc%q(i,k,ixcldliq)*state1%pdel(i,k) - rtm_integral_vtend(i) = rtm_integral_vtend(i) + ptend_loc%q(i,k,ixq)*state1%pdel(i,k) + rtm_integral_ltend(i) = rtm_integral_ltend(i) + ptend_loc%q(i,k,ixcldliq)*state1%pdel(i,k) + rtm_integral_vtend(i) = rtm_integral_vtend(i) + ptend_loc%q(i,k,ixq)*state1%pdel(i,k) - end do - end do + end do + end do - rtm_integral_ltend(:) = rtm_integral_ltend(:)/gravit - rtm_integral_vtend(:) = rtm_integral_vtend(:)/gravit + rtm_integral_ltend(:) = rtm_integral_ltend(:)/gravit + rtm_integral_vtend(:) = rtm_integral_vtend(:)/gravit if (clubb_do_adv) then if (macmic_it == cld_macmic_num_steps) then @@ -4691,7 +4748,7 @@ subroutine stats_init_clubb( l_stats_in, stats_tsamp_in, stats_tout_in, & ! Set stats_variables variables with inputs from calling subroutine stats_metadata%l_stats = l_stats_in - + stats_metadata%stats_tsamp = stats_tsamp_in stats_metadata%stats_tout = stats_tout_in @@ -4795,7 +4852,9 @@ subroutine stats_init_clubb( l_stats_in, stats_tsamp_in, stats_tout_in, & if( ierr /= 0 ) call endrun("stats_init_clubb: Failed to allocate stats_zt%file%z") ! Default initialization for array indices for zt - call stats_init_zt_api( clubb_vars_zt, & + call stats_init_zt_api( hydromet_dim, sclr_dim, edsclr_dim, & + hm_metadata%hydromet_list, hm_metadata%l_mix_rat_hm, & + clubb_vars_zt, & l_error, & stats_metadata, stats_zt(j) ) @@ -4832,7 +4891,9 @@ subroutine stats_init_clubb( l_stats_in, stats_tsamp_in, stats_tout_in, & allocate( stats_zm(j)%file%grid_avg_var( stats_zm(j)%num_output_fields ) ) allocate( stats_zm(j)%file%z( stats_zm(j)%kk ) ) - call stats_init_zm_api( clubb_vars_zm, & + call stats_init_zm_api( hydromet_dim, sclr_dim, edsclr_dim, & + hm_metadata%hydromet_list, hm_metadata%l_mix_rat_hm, & + clubb_vars_zm, & l_error, & stats_metadata, stats_zm(j) ) @@ -4916,7 +4977,6 @@ subroutine stats_init_clubb( l_stats_in, stats_tsamp_in, stats_tout_in, & ! Initialize sfc (surface point) - i = 1 do while ( ichar(clubb_vars_sfc(i)(1:1)) /= 0 .and. & len_trim(clubb_vars_sfc(i)) /= 0 .and. & diff --git a/src/physics/cam/subcol_SILHS.F90 b/src/physics/cam/subcol_SILHS.F90 index e941889e50..af039a254a 100644 --- a/src/physics/cam/subcol_SILHS.F90 +++ b/src/physics/cam/subcol_SILHS.F90 @@ -19,16 +19,20 @@ module subcol_SILHS #ifdef SILHS use clubb_intr, only: & clubb_config_flags, & - clubb_params, & + clubb_params_single_col, & stats_metadata, & stats_zt, stats_zm, stats_sfc, & - pdf_params_chnk + pdf_params_chnk, & + hm_metadata, & + hydromet_dim, & + pdf_dim use clubb_api_module, only: & hmp2_ip_on_hmm2_ip_slope_type, & hmp2_ip_on_hmm2_ip_intrcpt_type, & precipitation_fractions, & - stats + stats, & + core_rknd use silhs_api_module, only: & silhs_config_flags_type @@ -59,6 +63,11 @@ module subcol_SILHS type (stats), target :: stats_lh_zt, & stats_lh_sfc !$omp threadprivate(stats_lh_zt, stats_lh_sfc) + + real( kind = core_rknd ), dimension(:,:), allocatable :: & + corr_array_n_cloud, & + corr_array_n_below + #endif !----- @@ -334,10 +343,8 @@ subroutine subcol_init_SILHS(pbuf2d) #ifdef CLUBB_SGS #ifdef SILHS use clubb_api_module, only: core_rknd, & - pdf_dim, & setup_corr_varnce_array_api, & init_pdf_hydromet_arrays_api, & - Ncnp2_on_Ncnm2, & set_clubb_debug_level_api #endif @@ -357,7 +364,6 @@ subroutine subcol_init_SILHS(pbuf2d) ! To set up CLUBB hydromet indices integer :: & - hydromet_dim, & ! Number of enabled hydrometeors iirr, & ! Hydrometeor array index for rain water mixing ratio, rr iirs, & ! Hydrometeor array index for snow mixing ratio, rs iiri, & ! Hydrometeor array index for ice mixing ratio, ri @@ -446,36 +452,38 @@ subroutine subcol_init_SILHS(pbuf2d) !------------------------------- iirr = 1 iirs = 3 - iiri = 5 + iiri = 5 iirg = -1 - iiNr = 2 + iiNr = 2 iiNs = 4 - iiNi = 6 + iiNi = 6 iiNg = -1 hydromet_dim = 6 - ! Set up pdf indices, hydromet indicies, hydromet arrays, and hydromet variance ratios - call init_pdf_hydromet_arrays_api( 1.0_core_rknd, 1.0_core_rknd, & ! intent(in) - hydromet_dim, & ! intent(in) - iirr, iiri, iirs, iirg, & ! intent(in) - iiNr, iiNi, iiNs, iiNg, & ! intent(in) - subcol_SILHS_hmp2_ip_on_hmm2_ip_slope, & ! optional(in) - subcol_SILHS_hmp2_ip_on_hmm2_ip_intrcpt ) ! optional(in) - - Ncnp2_on_Ncnm2 = subcol_SILHS_ncnp2_on_ncnm2 + call init_pdf_hydromet_arrays_api( 1.0_core_rknd, 1.0_core_rknd, hydromet_dim, & ! intent(in) + iirr, iiNr, iiri, iiNi, & ! intent(in) + iirs, iiNs, iirg, iiNg, & ! intent(in) + subcol_SILHS_ncnp2_on_ncnm2, & ! intent(in) + hm_metadata, pdf_dim, & ! intent(out) + subcol_SILHS_hmp2_ip_on_hmm2_ip_slope, & ! optional(in) + subcol_SILHS_hmp2_ip_on_hmm2_ip_intrcpt ) ! optional(in) !------------------------------- ! Set up hydrometeors and correlation arrays for SILHS !------------------------------- + allocate(corr_array_n_cloud(pdf_dim,pdf_dim)) + allocate(corr_array_n_below(pdf_dim,pdf_dim)) + corr_file_path_cloud = trim( subcol_SILHS_corr_file_path )//trim( subcol_SILHS_corr_file_name )//cloud_file_ext corr_file_path_below = trim( subcol_SILHS_corr_file_path )//trim( subcol_SILHS_corr_file_name )//below_file_ext call setup_corr_varnce_array_api( corr_file_path_cloud, corr_file_path_below, & - newunit(iunit), & - clubb_config_flags%l_fix_w_chi_eta_correlations ) + pdf_dim, hm_metadata, newunit(iunit), & + clubb_config_flags%l_fix_w_chi_eta_correlations, & ! In + corr_array_n_cloud, corr_array_n_below ) !------------------------------- ! Register output fields from SILHS @@ -600,31 +608,15 @@ subroutine subcol_gen_SILHS(state, tend, state_sc, tend_sc, pbuf) #ifdef CLUBB_SGS #ifdef SILHS - use clubb_api_module, only : hydromet_dim, & - - setup_pdf_parameters_api, & - - hydromet_pdf_parameter, & + use clubb_api_module, only : setup_pdf_parameters_api, & zm2zt_api, setup_grid_heights_api, & - iirr, iiNr, iirs, iiri, & - iirg, iiNs, & - iiNi, iiNg, & - core_rknd, & w_tol_sqd, zero_threshold, & em_min, cloud_frac_min, & ! rc_tol, & - pdf_dim, & - corr_array_n_cloud, & - corr_array_n_below, & - iiPDF_chi, iiPDF_rr, & - iiPDF_w, iiPDF_Nr, & - iiPDF_ri, iiPDF_Ni, & - iiPDF_Ncn, iiPDF_rs, iiPDF_Ns, & - genrand_intg, genrand_init_api, & nparams, ic_K, & @@ -844,6 +836,13 @@ subroutine subcol_gen_SILHS(state, tend, state_sc, tend_sc, pbuf) type(grid) :: gr type(precipitation_fractions) :: precip_fracs + + ! Used as shortcuts to avoid typing hm_metadata%iiPDF_xx + integer :: & + iiPDF_chi, iiPDF_rr, iiPDF_w, iiPDF_Nr, & + iiPDF_ri, iiPDF_Ni, iiPDF_Ncn, iiPDF_rs, iiPDF_Ns, & + iirr, iiNr, iirs, iiri, & + iirg, iiNs, iiNi, iiNg !------------------------------------------------ ! Begin Code @@ -885,6 +884,26 @@ subroutine subcol_gen_SILHS(state, tend, state_sc, tend_sc, pbuf) ! does not? ! #ERDBG: The model iteration number is not used in SILHS unless ! sequence_length > 1, but nobody runs with that option. + + ! Copy hm_metadata indices to shortcuts + iiPDF_chi = hm_metadata%iiPDF_chi + iiPDF_Ncn = hm_metadata%iiPDF_Ncn + iiPDF_rr = hm_metadata%iiPDF_rr + iiPDF_w = hm_metadata%iiPDF_w + iiPDF_Nr = hm_metadata%iiPDF_Nr + iiPDF_ri = hm_metadata%iiPDF_ri + iiPDF_Ni = hm_metadata%iiPDF_Ni + iiPDF_rs = hm_metadata%iiPDF_rs + iiPDF_Ns = hm_metadata%iiPDF_Ns + iirr = hm_metadata%iirr + iiNr = hm_metadata%iiNr + iirs = hm_metadata%iirs + iiri = hm_metadata%iiri + iirg = hm_metadata%iirg + iiNs = hm_metadata%iiNs + iiNi = hm_metadata%iiNi + iiNg = hm_metadata%iiNg + !---------------- ! Establish associations between pointers and physics buffer fields !---------------- @@ -902,7 +921,7 @@ subroutine subcol_gen_SILHS(state, tend, state_sc, tend_sc, pbuf) call pbuf_get_field(pbuf, kvh_idx, khzm_in) ! Pull c_K from clubb parameters. - c_K = clubb_params(ic_K) + c_K = clubb_params_single_col(ic_K) !---------------- ! Copy state and populate numbers and values of sub-columns @@ -1129,27 +1148,28 @@ subroutine subcol_gen_SILHS(state, tend, state_sc, tend_sc, pbuf) call init_precip_fracs_api( pverp-top_lev+1, ngrdcol, & precip_fracs ) - call setup_pdf_parameters_api( gr, pverp-top_lev+1, ngrdcol, pdf_dim, ztodt, & ! In - Nc_in_cloud, cld_frac_in, khzm, & ! In - ice_supersat_frac_in, hydromet, wphydrometp, & ! In - corr_array_n_cloud, corr_array_n_below, & ! In - pdf_params_chnk(lchnk), & ! In - clubb_params, & ! In - clubb_config_flags%iiPDF_type, & ! In - clubb_config_flags%l_use_precip_frac, & ! In - clubb_config_flags%l_predict_upwp_vpwp, & ! In - clubb_config_flags%l_diagnose_correlations, & ! In - clubb_config_flags%l_calc_w_corr, & ! In - clubb_config_flags%l_const_Nc_in_cloud, & ! In - clubb_config_flags%l_fix_w_chi_eta_correlations, & ! In - stats_metadata, & ! In - stats_zt, stats_zm, stats_sfc, & ! In - hydrometp2, & ! Inout - mu_x_1, mu_x_2, & ! Out - sigma_x_1, sigma_x_2, & ! Out - corr_array_1, corr_array_2, & ! Out - corr_cholesky_mtx_1, corr_cholesky_mtx_2, & ! Out - precip_fracs ) ! Inout + call setup_pdf_parameters_api( gr, pverp-top_lev+1, ngrdcol, pdf_dim, hydromet_dim, ztodt, & ! In + Nc_in_cloud, cld_frac_in, khzm, & ! In + ice_supersat_frac_in, hydromet, wphydrometp, & ! In + corr_array_n_cloud, corr_array_n_below, & ! In + hm_metadata, & ! In + pdf_params_chnk(lchnk), & ! In + clubb_params_single_col, & ! In + clubb_config_flags%iiPDF_type, & ! In + clubb_config_flags%l_use_precip_frac, & ! In + clubb_config_flags%l_predict_upwp_vpwp, & ! In + clubb_config_flags%l_diagnose_correlations, & ! In + clubb_config_flags%l_calc_w_corr, & ! In + clubb_config_flags%l_const_Nc_in_cloud, & ! In + clubb_config_flags%l_fix_w_chi_eta_correlations, & ! In + stats_metadata, & ! In + stats_zt, stats_zm, stats_sfc, & ! In + hydrometp2, & ! Inout + mu_x_1, mu_x_2, & ! Out + sigma_x_1, sigma_x_2, & ! Out + corr_array_1, corr_array_2, & ! Out + corr_cholesky_mtx_1, corr_cholesky_mtx_2, & ! Out + precip_fracs ) ! Inout ! In order for Lscale to be used properly, it needs to be passed out of ! advance_clubb_core, saved to the pbuf, and then pulled out of the @@ -1220,15 +1240,11 @@ subroutine subcol_gen_SILHS(state, tend, state_sc, tend_sc, pbuf) iter, pdf_dim, num_subcols, sequence_length, pverp-top_lev+1, ngrdcol, & ! In l_calc_weights_all_levs_itime, & ! In pdf_params_chnk(lchnk), delta_zm, Lscale, & ! In - lh_seed, & ! In + lh_seed, hm_metadata, & ! In rho_ds_zt, & ! In mu_x_1, mu_x_2, sigma_x_1, sigma_x_2, & ! In corr_cholesky_mtx_1, corr_cholesky_mtx_2, & ! In precip_fracs, silhs_config_flags, & ! In - clubb_params, & ! In - clubb_config_flags%l_uv_nudge, & ! In - clubb_config_flags%l_tke_aniso, & ! In - clubb_config_flags%l_standard_term_ta, & ! In vert_decorr_coef, & ! In stats_metadata, & ! In stats_lh_zt, stats_lh_sfc, & ! InOut @@ -1236,15 +1252,15 @@ subroutine subcol_gen_SILHS(state, tend, state_sc, tend_sc, pbuf) lh_sample_point_weights) ! Out ! Extract clipped variables from subcolumns - call clip_transform_silhs_output_api( gr, pverp-top_lev+1, ngrdcol, num_subcols, & ! In - pdf_dim, hydromet_dim, & ! In - X_mixt_comp_all_levs, & ! In - X_nl_all_levs, & ! In - pdf_params_chnk(lchnk), & ! In - l_use_Ncn_to_Nc, & ! In - lh_rt_clipped, lh_thl_clipped, & ! Out - lh_rc_clipped, lh_rv_clipped, & ! Out - lh_Nc_clipped ) ! Out + call clip_transform_silhs_output_api( gr, pverp-top_lev+1, ngrdcol, num_subcols, & ! In + pdf_dim, hydromet_dim, hm_metadata, & ! In + X_mixt_comp_all_levs, & ! In + X_nl_all_levs, & ! In + pdf_params_chnk(lchnk), & ! In + l_use_Ncn_to_Nc, & ! In + lh_rt_clipped, lh_thl_clipped, & ! Out + lh_rc_clipped, lh_rv_clipped, & ! Out + lh_Nc_clipped ) ! Out !$acc wait if ( l_est_kessler_microphys ) then From dc472dae1b5fa79fa7f5e8b9fbbcd7e765bf54b2 Mon Sep 17 00:00:00 2001 From: Cheryl Craig Date: Wed, 26 Jun 2024 10:36:47 -0600 Subject: [PATCH 10/22] udpates for supporting GPU testing --- test/system/archive_baseline.sh | 55 ++++++++++----------------------- test/system/test_driver.sh | 6 ++-- 2 files changed, 18 insertions(+), 43 deletions(-) diff --git a/test/system/archive_baseline.sh b/test/system/archive_baseline.sh index 8460923a1f..e457081ba4 100755 --- a/test/system/archive_baseline.sh +++ b/test/system/archive_baseline.sh @@ -12,7 +12,7 @@ cat << EOF1 NAME archive_baseline.sh - archive pretag baselines to set locations on - hobart, izumi and derecho. + izumi and derecho. SYNOPSIS @@ -24,41 +24,28 @@ SYNOPSIS ENVIROMENT VARIABLES CESM_TESTDIR - Directory that contains the CESM finished results you wish to archive. - CAM_FC - Compiler used, only used on hobart and izumi (PGI,NAG), where the compiler + CAM_FC - Compiler used, used on derecho (INTEL, NVHPC) and izumi (GNU,NAG), where the compiler name is appended to the archive directory. BASELINE ARCHIVED LOCATION - hobart, izumi: /fs/cgd/csm/models/atm/cam/pretag_bl/TAGNAME_pgi - /fs/cgd/csm/models/atm/cam/pretag_bl/TAGNAME_nag - derecho: /glade/campaign/cesm/community/amwg/cam_baselines/TAGNAME + izumi: /fs/cgd/csm/models/atm/cam/pretag_bl/TAGNAME_gnu + /fs/cgd/csm/models/atm/cam/pretag_bl/TAGNAME_nag + derecho: /glade/campaign/cesm/community/amwg/cam_baselines/TAGNAME_intel + /glade/campaign/cesm/community/amwg/cam_baselines/TAGNAME_nvhpc HOW TO USE ARCHIVE BASELINES - Set BL_TESTDIR to the archived baseline you wish to load. + on izumi: + env CESM_TESTDIR=/scratch/cluster/YourName/aux_cam_gnu_yyyymmddsssss CAM_FC=GNU ./archive_baseline.sh cam6_4_XXX + env CESM_TESTDIR=/scratch/cluster/YourName/aux_cam_nag_yyyymmddsssss CAM_FC=NAG ./archive_baseline.sh cam6_3_XXX - -WORK FLOW - - This is an example for hobart or izumi. - - Modify your sandbox with the changes you want. - setenv CAM_FC PGI - setenv CAM_TESTDIR /scratch/cluster/fischer/cam5_2_06 - Run the cam test suite. - Make your trunk tag - archive_baseline.sh cam5_2_06 - - Create a new sandbox. - setenv CAM_FC PGI - setenv CAM_TESTDIR /scratch/cluster/fischer/cam5_2_07 - setenv BL_TESTDIR /fs/cgd/csm/models/atm/cam/pretag_bl/cam5_2_06_pgi - Run the cam test suite. - Make your trunk tag - archive_baseline.sh cam5_2_07 + on derecho: + env CESM_TESTDIR=/glade/derecho/scratch/YourName/aux_cam_intel_yyyymmddsssss CAM_FC=INTEL ./archive_baseline.sh cam6_4_XXX + env CESM_TESTDIR=/glade/derecho/scratch/YourName/aux_cam_nvhpc_yyyymmddsssss CAM_FC=NVHPC ./archive_baseline.sh cam6_4_XXX WARNING @@ -73,20 +60,10 @@ fi hostname=`hostname` case $hostname in - ho*) - echo "server: hobart" - if [ -z "$CAM_FC" ]; then - CAM_FC="PGI" - fi - test_file_list="tests_pretag_hobart_${CAM_FC,,}" - cam_tag=$1_${CAM_FC,,} - baselinedir="/fs/cgd/csm/models/atm/cam/pretag_bl/$cam_tag" - ;; - iz*) echo "server: izumi" if [ -z "$CAM_FC" ]; then - CAM_FC="PGI" + echo "Must specify CAM_FC" fi test_file_list="tests_pretag_izumi_${CAM_FC,,}" cam_tag=$1_${CAM_FC,,} @@ -96,9 +73,9 @@ case $hostname in de*) echo "server: derecho" if [ -z "$CAM_FC" ]; then - CAM_FC="INTEL" + echo "Must specify CAM_FC" fi - test_file_list="tests_pretag_derecho" + test_file_list="tests_pretag_derecho_${CAM_FC,,}" cam_tag=$1 baselinedir="/glade/campaign/cesm/community/amwg/cam_baselines/$cam_tag" ;; @@ -130,7 +107,7 @@ fi case $hostname in - ch* | hobart | izumi) + de* | izumi) if [ -z "$CESM_TESTDIR" ]; then echo '***********************************************************************************' echo 'INFO: The aux_cam and test_cam tests were NOT archived' diff --git a/test/system/test_driver.sh b/test/system/test_driver.sh index 80a632b14f..e787d25386 100755 --- a/test/system/test_driver.sh +++ b/test/system/test_driver.sh @@ -466,6 +466,8 @@ if [ "${hostname:0:6}" == "casper" ] || [ "${hostname:0:5}" == "crhtc" ]; then fi if [ -n "${CAM_FC}" ]; then comp="_${CAM_FC,,}" +else + echo "ERROR: Must specify CAM_FC" fi if [ "${cesm_test_suite}" != "none" -a -n "${cesm_test_mach}" ]; then @@ -547,8 +549,6 @@ if [ "${cesm_test_suite}" != "none" -a -n "${cesm_test_mach}" ]; then if [ -n "${CAM_FC}" ]; then testargs="${testargs} --xml-compiler ${CAM_FC,,}" - else - testargs="${testargs} --xml-compiler intel" fi case $hostname in # derecho @@ -586,8 +586,6 @@ if [ "${cesm_test_suite}" != "none" -a -n "${cesm_test_mach}" ]; then cmd="query_testlists --xml-category $cesm_test --xml-machine ${cesm_test_mach}" if [ -n "${CAM_FC}" ]; then cmd="${cmd} --xml-compiler ${CAM_FC,,}" - else - cmd="${cmd} --xml-compiler intel" fi cmd="${CIME_ROOT}/scripts/"$cmd cime_testlist=`$cmd` From 9cf174c85d9a32c57d4aeafeaf81cc188c0e7da1 Mon Sep 17 00:00:00 2001 From: Katetc Date: Thu, 27 Jun 2024 11:38:41 -0600 Subject: [PATCH 11/22] Updating clubb submodule --- src/physics/clubb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/physics/clubb b/src/physics/clubb index 50cee042e5..4b3e80faef 160000 --- a/src/physics/clubb +++ b/src/physics/clubb @@ -1 +1 @@ -Subproject commit 50cee042e588fefd3fe58c2a1d638affec8c0389 +Subproject commit 4b3e80faefdbbe79cda650a8401364ec640e05e7 From 88d8c3a59fd56dbf3269d9a4c6d6221f1eead0f4 Mon Sep 17 00:00:00 2001 From: Katetc Date: Tue, 9 Jul 2024 16:41:27 -0600 Subject: [PATCH 12/22] Update atmos_phys submodule for some reason --- src/atmos_phys | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/atmos_phys b/src/atmos_phys index 4944547f04..f4c09618ea 160000 --- a/src/atmos_phys +++ b/src/atmos_phys @@ -1 +1 @@ -Subproject commit 4944547f04b1457d78bf7d3c0becddcfe0deabb9 +Subproject commit f4c09618eaaa19eaf3382f0473a531e20aa9f808 From ed9a91cc332958363c37a242e0048ff5832a382c Mon Sep 17 00:00:00 2001 From: Cheryl Craig Date: Fri, 12 Jul 2024 08:57:12 -0600 Subject: [PATCH 13/22] Add GPU tests on derecho --- cime_config/testdefs/testlist_cam.xml | 2 ++ test/system/test_driver.sh | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cime_config/testdefs/testlist_cam.xml b/cime_config/testdefs/testlist_cam.xml index 9f259ac673..f79abffd88 100644 --- a/cime_config/testdefs/testlist_cam.xml +++ b/cime_config/testdefs/testlist_cam.xml @@ -1483,6 +1483,7 @@ + @@ -1491,6 +1492,7 @@ + diff --git a/test/system/test_driver.sh b/test/system/test_driver.sh index e787d25386..ef86e43cb8 100755 --- a/test/system/test_driver.sh +++ b/test/system/test_driver.sh @@ -1,4 +1,4 @@ -#!/bin/sh +!/bin/sh # # test_driver.sh: driver for the testing of CAM with standalone scripts # @@ -468,6 +468,7 @@ if [ -n "${CAM_FC}" ]; then comp="_${CAM_FC,,}" else echo "ERROR: Must specify CAM_FC" + exit 1 fi if [ "${cesm_test_suite}" != "none" -a -n "${cesm_test_mach}" ]; then From 55db0bbf7e6f5e0a20d745f147d1dffda7434967 Mon Sep 17 00:00:00 2001 From: Katetc Date: Mon, 29 Jul 2024 12:37:57 -0600 Subject: [PATCH 14/22] Addressing first review comments --- src/physics/cam/clubb_intr.F90 | 8 +------- src/physics/cam/subcol_SILHS.F90 | 6 +++--- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index 8159d1d63d..223ff0e5b5 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -53,18 +53,12 @@ module clubb_intr type (stats_metadata_type) :: & stats_metadata -!$omp threadprivate(stats_metadata) - type (hm_metadata_type) :: & hm_metadata -!$omp threadprivate(hm_metadata) - type (sclr_idx_type) :: & sclr_idx -!$omp threadprivate(sclr_idx) - #endif private @@ -100,7 +94,7 @@ module clubb_intr real(r8), dimension(nparams), public :: clubb_params_single_col ! Adjustable CLUBB parameters (C1, C2 ...) #endif - ! These are zero by default, but will be set by SILHS before + ! These are zero by default, but will be set by SILHS before they are used by subcolumns integer :: & hydromet_dim = 0, & pdf_dim = 0 diff --git a/src/physics/cam/subcol_SILHS.F90 b/src/physics/cam/subcol_SILHS.F90 index af039a254a..c373ed6b3e 100644 --- a/src/physics/cam/subcol_SILHS.F90 +++ b/src/physics/cam/subcol_SILHS.F90 @@ -373,7 +373,7 @@ subroutine subcol_init_SILHS(pbuf2d) iiNi, & ! Hydrometeor array index for ice concentration, Ni iiNg ! Hydrometeor array index for graupel concentration, Ng - integer :: l ! Loop variable + integer :: l, ierr=0 ! Loop variable, error check ! Set CLUBB's debug level ! This is called in module clubb_intr; no need to do it here. @@ -474,8 +474,8 @@ subroutine subcol_init_SILHS(pbuf2d) !------------------------------- ! Set up hydrometeors and correlation arrays for SILHS !------------------------------- - allocate(corr_array_n_cloud(pdf_dim,pdf_dim)) - allocate(corr_array_n_below(pdf_dim,pdf_dim)) + allocate( corr_array_n_cloud(pdf_dim,pdf_dim), corr_array_n_below(pdf_dim,pdf_dim), stat=ierr) + if( ierr /= 0 ) call endrun(' subcol_init_SILHS: failed to allocate corr_array fields ') corr_file_path_cloud = trim( subcol_SILHS_corr_file_path )//trim( subcol_SILHS_corr_file_name )//cloud_file_ext corr_file_path_below = trim( subcol_SILHS_corr_file_path )//trim( subcol_SILHS_corr_file_name )//below_file_ext From afde46e8837d446e9b98b2cb23d05c8cc074f805 Mon Sep 17 00:00:00 2001 From: Katetc Date: Mon, 29 Jul 2024 13:47:53 -0600 Subject: [PATCH 15/22] A few more review comments --- src/physics/cam/clubb_intr.F90 | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index 223ff0e5b5..57b220d2a5 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -1844,13 +1844,15 @@ subroutine clubb_ini_cam(pbuf2d) call addfld ( 'edmf_qtflx' , (/ 'ilev' /), 'A', 'W/m2' , 'qt flux (EDMF)' ) end if -#ifndef SILHS - ! hm_metadata is set up by calling init_pdf_hydromet_arrays_api in subcol_init_SILHS. - ! So if we are not using silhs, we allocate the parts of hm_metadata that need allocating - ! in order to making intel debug tests happy. - allocate( hm_metadata%hydromet_list(1) ) - allocate( hm_metadata%l_mix_rat_hm(1) ) -#endif + if ( trim(subcol_scheme) .ne. 'SILHS' ) then + ! hm_metadata is set up by calling init_pdf_hydromet_arrays_api in subcol_init_SILHS. + ! So if we are not using silhs, we allocate the parts of hm_metadata that need allocating + ! in order to making intel debug tests happy. + allocate( hm_metadata%hydromet_list(1), ierr) + if( ierr /= 0 ) call endrun( 'clubb_ini_cam: Unable to allocate hm_metadata%hydromet_list' ) + allocate( hm_metadata%l_mix_rat_hm(1), ierr) + if( ierr /= 0 ) call endrun( 'clubb_ini_cam: Unable to allocate hm_metadata%l_mix_rat_hm' ) + end if ! Initialize statistics, below are dummy variables dum1 = 300._r8 From b4df86bf3963f85e89470fa89ee82e34280972e9 Mon Sep 17 00:00:00 2001 From: Katetc Date: Mon, 29 Jul 2024 16:14:50 -0600 Subject: [PATCH 16/22] Bug fixes --- src/physics/cam/clubb_intr.F90 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index 57b220d2a5..7a07d5bcbd 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -1844,13 +1844,13 @@ subroutine clubb_ini_cam(pbuf2d) call addfld ( 'edmf_qtflx' , (/ 'ilev' /), 'A', 'W/m2' , 'qt flux (EDMF)' ) end if - if ( trim(subcol_scheme) .ne. 'SILHS' ) then + if ( trim(subcol_scheme) /= 'SILHS' ) then ! hm_metadata is set up by calling init_pdf_hydromet_arrays_api in subcol_init_SILHS. ! So if we are not using silhs, we allocate the parts of hm_metadata that need allocating ! in order to making intel debug tests happy. - allocate( hm_metadata%hydromet_list(1), ierr) + allocate( hm_metadata%hydromet_list(1), stat=ierr) if( ierr /= 0 ) call endrun( 'clubb_ini_cam: Unable to allocate hm_metadata%hydromet_list' ) - allocate( hm_metadata%l_mix_rat_hm(1), ierr) + allocate( hm_metadata%l_mix_rat_hm(1), stat=ierr) if( ierr /= 0 ) call endrun( 'clubb_ini_cam: Unable to allocate hm_metadata%l_mix_rat_hm' ) end if From a251e59f94f1f9a8e3eded064a59c5a2698c7a62 Mon Sep 17 00:00:00 2001 From: Katetc Date: Tue, 30 Jul 2024 14:14:32 -0600 Subject: [PATCH 17/22] Update ChangeLog template to include nvhpc tests on Derecho --- doc/ChangeLog_template | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/ChangeLog_template b/doc/ChangeLog_template index 5919b4e11a..f646f24e78 100644 --- a/doc/ChangeLog_template +++ b/doc/ChangeLog_template @@ -31,6 +31,8 @@ appropriate machine below. All failed tests must be justified. derecho/intel/aux_cam: +derecho/nvhpc/aux_cam: + izumi/nag/aux_cam: izumi/gnu/aux_cam: From b52730b6b89c48ebbde38027704ef4b613384ca2 Mon Sep 17 00:00:00 2001 From: katetc Date: Wed, 31 Jul 2024 14:40:19 -0600 Subject: [PATCH 18/22] Bug fix for runs where CLUBB_SGS is not defined --- src/physics/cam/clubb_intr.F90 | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index 7a07d5bcbd..361f71e57f 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -41,24 +41,22 @@ module clubb_intr #endif implicit none + #ifdef CLUBB_SGS ! Variables that contains all the statistics - type (stats), target, save :: stats_zt(pcols), & ! stats_zt grid stats_zm(pcols), & ! stats_zm grid stats_rad_zt(pcols), & ! stats_rad_zt grid stats_rad_zm(pcols), & ! stats_rad_zm grid stats_sfc(pcols) ! stats_sfc - - type (stats_metadata_type) :: & - stats_metadata - type (hm_metadata_type) :: & hm_metadata + + type (stats_metadata_type) :: & + stats_metadata type (sclr_idx_type) :: & sclr_idx - #endif private @@ -103,6 +101,7 @@ module clubb_intr ! ------------------------ ! ! Sometimes private data ! ! ------------------------ ! +#ifdef CLUBB_SGS #ifdef SILHS ! If SILHS is in use, it will initialize these public :: & @@ -116,7 +115,8 @@ module clubb_intr pdf_dim, & hm_metadata #endif - +#endif + ! ------------ ! ! Private data ! ! ------------ ! From 2abc5a0164cab1476a79554b8ea59dfad74c179d Mon Sep 17 00:00:00 2001 From: katetc Date: Thu, 8 Aug 2024 20:44:53 -0600 Subject: [PATCH 19/22] Update to new new new CLUBB external (_gpufix_PosInf) and new CDEPS --- .gitmodules | 4 ++-- components/cdeps | 2 +- src/physics/clubb | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitmodules b/.gitmodules index 1b4b6d1d1c..dff6e524c6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -96,7 +96,7 @@ url = https://github.com/larson-group/clubb_release fxrequired = AlwaysRequired fxsparse = ../.clubb_sparse_checkout - fxtag = clubb_4ncar_20240605_73d60f6 + fxtag = clubb_4ncar_20240605_73d60f6_gpufixes_posinf fxDONOTUSEurl = https://github.com/larson-group/clubb_release [submodule "cism"] @@ -151,7 +151,7 @@ fxDONOTUSEurl = https://github.com/ESCOMP/CMEPS.git [submodule "cdeps"] path = components/cdeps url = https://github.com/ESCOMP/CDEPS.git -fxtag = cdeps1.0.43 +fxtag = cdeps1.0.44 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESCOMP/CDEPS.git diff --git a/components/cdeps b/components/cdeps index 453a9d175a..b7a6088204 160000 --- a/components/cdeps +++ b/components/cdeps @@ -1 +1 @@ -Subproject commit 453a9d175a5739d9cca5c4ec7b96f45b201decec +Subproject commit b7a608820480ee9a79f11e37e88bf740350f51b5 diff --git a/src/physics/clubb b/src/physics/clubb index 4b3e80faef..15e802092f 160000 --- a/src/physics/clubb +++ b/src/physics/clubb @@ -1 +1 @@ -Subproject commit 4b3e80faefdbbe79cda650a8401364ec640e05e7 +Subproject commit 15e802092f65b3a20e5d67cb32d40f8a2771ca9b From 8e05c34e5bb2c3d1b9e0eb87f9a697d16e412391 Mon Sep 17 00:00:00 2001 From: Katetc Date: Fri, 9 Aug 2024 12:33:05 -0600 Subject: [PATCH 20/22] Update to newest CDEPS tag cdeps1.0.45 --- .gitmodules | 2 +- components/cdeps | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index dff6e524c6..811bd080f3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -151,7 +151,7 @@ fxDONOTUSEurl = https://github.com/ESCOMP/CMEPS.git [submodule "cdeps"] path = components/cdeps url = https://github.com/ESCOMP/CDEPS.git -fxtag = cdeps1.0.44 +fxtag = cdeps1.0.45 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESCOMP/CDEPS.git diff --git a/components/cdeps b/components/cdeps index b7a6088204..46c10740ce 160000 --- a/components/cdeps +++ b/components/cdeps @@ -1 +1 @@ -Subproject commit b7a608820480ee9a79f11e37e88bf740350f51b5 +Subproject commit 46c10740ce83a154edfc876093f72e7f041c3659 From 86ba7de6bff6fbb966e8f50da6cec1e8d9d6a8ed Mon Sep 17 00:00:00 2001 From: katetc Date: Fri, 9 Aug 2024 17:52:41 -0600 Subject: [PATCH 21/22] Update ChangeLog with Derecho test results before machine goes down --- doc/ChangeLog | 101 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index 93142a1a4a..a5f3894e34 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,5 +1,106 @@ =============================================================== +Tag name: cam6_4_019 +Originator(s): katec, cacraig, vlarson, bstephens82, huebleruwm, zarzycki, JulioTBacmeister, jedwards4b +Date: 10 August 2024 +One-line Summary: New CLUBB external, new GPU/nvhpc test suite, new CDEPS external +Github PR URL: https://github.com/ESCOMP/CAM/pull/1086 + +Purpose of changes (include the issue number and title text for each relevant GitHub issue): + +Describe any changes made to build system: update git-fleximod + +Describe any changes made to the namelist: none + +List any changes to the defaults for the boundary datasets: none + +Describe any substantial timing or memory changes: none + +Code reviewed by: cacraigucar + +List all files eliminated: none + +List all files added and what they do: + +List all existing files that have been modified, and describe the changes: + +If there were any failures reported from running test_driver.sh on any test +platform, and checkin with these failures has been OK'd by the gatekeeper, +then copy the lines from the td.*.status files for the failed tests to the +appropriate machine below. All failed tests must be justified. + +derecho/intel/aux_cam: +ERP_Ln9.f09_f09_mg17.FCSD_HCO.derecho_intel.cam-outfrq9s (Overall: FAIL) details: +- pre-existing failure due to HEMCO not having reproducible results issues #1018 and #856 + +SMS_D_Ln9_P1280x1.ne0ARCTICne30x4_ne0ARCTICne30x4_mt12.FHIST.derecho_intel.cam-outfrq9s (Overall: FAIL) details: +SMS_D_Ln9_P1280x1.ne0CONUSne30x8_ne0CONUSne30x8_mt12.FCHIST.derecho_intel.cam-outfrq9s (Overall: PEND) details: +- pre-existing failures -- need fix in CLM external + +SMS_D_Ln9.T42_T42.FSCAM.derecho_intel.cam-outfrq9s (Overall: FAIL) details: +- pre-existing failure -- need fix in CICE external + +ERC_D_Ln9.f19_f19_mg17.QPC6.derecho_intel.cam-outfrq3s_cosp (Overall: DIFF) details: +ERC_D_Ln9_P144x1.ne16pg3_ne16pg3_mg17.QPC6HIST.derecho_intel.cam-outfrq3s_ttrac_usecase (Overall: DIFF) details: +ERP_D_Ln9.f19_f19_mg17.QPC6.derecho_intel.cam-outfrq9s (Overall: DIFF) details: +ERP_D_Ln9.ne30pg3_ne30pg3_mg17.FLTHIST.derecho_intel.cam-outfrq9s (Overall: DIFF) details: +ERP_D_Ln9.ne30pg3_ne30pg3_mg17.FLTHIST.derecho_intel.cam-outfrq9s_rrtmgp (Overall: DIFF) details: +ERP_D_Ln9_P64x2.f09_f09_mg17.QSC6.derecho_intel.cam-outfrq9s (Overall: DIFF) details: +ERP_Ld3.f09_f09_mg17.FWHIST.derecho_intel.cam-reduced_hist1d (Overall: DIFF) details: +ERP_Ln9.C96_C96_mg17.F2000climo.derecho_intel.cam-outfrq9s_mg3 (Overall: DIFF) details: +ERP_Ln9.f09_f09_mg17.F1850.derecho_intel.cam-outfrq9s (Overall: DIFF) details: +ERP_Ln9.f09_f09_mg17.F2000climo.derecho_intel.cam-outfrq9s (Overall: DIFF) details: +ERP_Ln9.f09_f09_mg17.F2010climo.derecho_intel.cam-outfrq9s (Overall: DIFF) details: +ERP_Ln9.f09_f09_mg17.FHIST_BDRD.derecho_intel.cam-outfrq9s (Overall: DIFF) details: +ERP_Ln9.f19_f19_mg17.FWsc1850.derecho_intel.cam-outfrq9s (Overall: DIFF) details: +ERP_Ln9.ne30pg3_ne30pg3_mg17.FCnudged.derecho_intel.cam-outfrq9s (Overall: DIFF) details: +ERP_Ln9.ne30pg3_ne30pg3_mg17.FW2000climo.derecho_intel.cam-outfrq9s (Overall: DIFF) details: +ERP_Ln9_P24x3.f45_f45_mg37.QPWmaC6.derecho_intel.cam-outfrq9s_mee_fluxes (Overall: DIFF) details: +ERS_Ld3.f10_f10_mg37.F1850.derecho_intel.cam-outfrq1d_14dec_ghg_cam7 (Overall: DIFF) details: +ERS_Ln9.f09_f09_mg17.FX2000.derecho_intel.cam-outfrq9s (Overall: DIFF) details: +ERS_Ln9.f19_f19_mg17.FXSD.derecho_intel.cam-outfrq9s (Overall: DIFF) details: +ERS_Ln9_P288x1.mpasa120_mpasa120.F2000climo.derecho_intel.cam-outfrq9s_mpasa120 (Overall: DIFF) details: +ERS_Ln9_P36x1.mpasa480_mpasa480.F2000climo.derecho_intel.cam-outfrq9s_mpasa480 (Overall: DIFF) details: +SMS_D_Ln9.f09_f09_mg17.FCts2nudged.derecho_intel.cam-outfrq9s_leapday (Overall: DIFF) details: +SMS_D_Ln9.f09_f09_mg17.FCvbsxHIST.derecho_intel.cam-outfrq9s (Overall: DIFF) details: +SMS_D_Ln9.f09_f09_mg17.FSD.derecho_intel.cam-outfrq9s (Overall: DIFF) details: +SMS_D_Ln9.f19_f19_mg17.FWma2000climo.derecho_intel.cam-outfrq9s (Overall: DIFF) details: +SMS_D_Ln9.f19_f19_mg17.FWma2000climo.derecho_intel.cam-outfrq9s_waccm_ma_mam4 (Overall: DIFF) details: +SMS_D_Ln9.f19_f19_mg17.FXHIST.derecho_intel.cam-outfrq9s_amie (Overall: DIFF) details: +SMS_D_Ln9.f19_f19_mg17.QPC2000climo.derecho_intel.cam-outfrq3s_usecase (Overall: DIFF) details: +SMS_D_Ln9.ne16pg3_ne16pg3_mg17.FX2000.derecho_intel.cam-outfrq9s (Overall: DIFF) details: +SMS_D_Ln9.ne30pg3_ne30pg3_mg17.FCts4MTHIST.derecho_intel.cam-outfrq9s (Overall: DIFF) details: +SMS_D_Ln9.ne30pg3_ne30pg3_mg17.FMTHIST.derecho_intel.cam-outfrq9s (Overall: DIFF) details: +SMS_D_Ln9_P1280x1.ne30pg3_ne30pg3_mg17.FCLTHIST.derecho_intel.cam-outfrq9s (Overall: DIFF) details: +SMS_Ld1.f09_f09_mg17.FCHIST_GC.derecho_intel.cam-outfrq1d (Overall: DIFF) details: +SMS_Ld1.f09_f09_mg17.FW2000climo.derecho_intel.cam-outfrq1d (Overall: DIFF) details: +SMS_Ld1.ne30pg3_ne30pg3_mg17.FC2010climo.derecho_intel.cam-outfrq1d (Overall: DIFF) details: +SMS_Lh12.f09_f09_mg17.FCSD_HCO.derecho_intel.cam-outfrq3h (Overall: DIFF) details: +SMS_Lm13.f10_f10_mg37.F2000climo.derecho_intel.cam-outfrq1m (Overall: DIFF) details: +SMS_Ln9.f09_f09_mg17.F2010climo.derecho_intel.cam-nudging (Overall: DIFF) details: +SMS_Ln9.f09_f09_mg17.FW1850.derecho_intel.cam-reduced_hist3s (Overall: DIFF) details: +SMS_Ln9.f19_f19.F2000climo.derecho_intel.cam-silhs (Overall: DIFF) details: +SMS_Ln9.f19_f19_mg17.FHIST.derecho_intel.cam-outfrq9s_nochem (Overall: DIFF) details: +SMS_Ln9.ne30pg3_ne30pg3_mg17.FW2000climo.derecho_intel.cam-outfrq9s_rrtmgp (Overall: DIFF) details: +- Expected differences due to the new CLUBB external (See PR for discussion) + +derecho/nvphc/aux_cam: + +ERS_Ln9_G4-a100-openacc.ne30pg3_ne30pg3_mg17.F2000dev.derecho_nvhpc.cam-outfrq9s_mg3_default (Overall: DIFF) + FAIL ERS_Ln9_G4-a100-openacc.ne30pg3_ne30pg3_mg17.F2000dev.derecho_nvhpc.cam-outfrq9s_mg3_default BASELINE /glade/campaign/cesm/community/amwg/cam_baselines/cam6_4_018_intel: ERROR BFAIL baseline directory '/glade/campaign/cesm/community/amwg/cam_baselines/cam6_4_018_intel/ERS_Ln9_G4-a100-openacc.ne30pg3_ne30pg3_mg17.F2000dev.derecho_nvhpc.cam-outfrq9s_mg3_default' does not exist +- Expected baseline compare fail due to no baselines stored for GPU tests that didn't exist previously + +izumi/nag/aux_cam: + +izumi/gnu/aux_cam: + +CAM tag used for the baseline comparison tests if different than previous +tag: + +Summarize any changes to answers: BFB, as expected + +=============================================================== + Tag name: cam6_4_018 Originator(s): peverwhee, jedwards4b Date: 30 July 2024 From ed684d2c73f6db60c6edf664ae2fad41ffd0578d Mon Sep 17 00:00:00 2001 From: katetc Date: Mon, 12 Aug 2024 16:50:25 -0600 Subject: [PATCH 22/22] Final updates --- doc/ChangeLog | 65 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index a5f3894e34..077a11ada7 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -2,27 +2,59 @@ Tag name: cam6_4_019 Originator(s): katec, cacraig, vlarson, bstephens82, huebleruwm, zarzycki, JulioTBacmeister, jedwards4b -Date: 10 August 2024 +Date: 12 August 2024 One-line Summary: New CLUBB external, new GPU/nvhpc test suite, new CDEPS external Github PR URL: https://github.com/ESCOMP/CAM/pull/1086 Purpose of changes (include the issue number and title text for each relevant GitHub issue): + - New CLUBB external with fixes to support GPU testing #1036 + - part of cam6_4_019: Add GPU regression test suite #1048 -Describe any changes made to build system: update git-fleximod +Describe any changes made to build system: none -Describe any changes made to the namelist: none +Describe any changes made to the namelist: + - Add default vaules for a few new CLUBB namelist parameters: clubb_bv_efold, clubb_wpxp_Ri_exp, and clubb_z_displace List any changes to the defaults for the boundary datasets: none Describe any substantial timing or memory changes: none -Code reviewed by: cacraigucar +Code reviewed by: cacraigucar, sjsprecious, adamrher, bstephens82 -List all files eliminated: none +List all files eliminated: + cime/config/testmods_dirs/cam/outfrq9s_mg3_nondefault/shell_comands + cime/config/testmods_dirs/cam/outfrq9s_mg3_nondefault/user_nl_cam + cime/config/testmods_dirs/cam/outfrq9s_mg3_nondefault/user_nl_clm + - Removed as part of GPU test updates -List all files added and what they do: +List all files added and what they do: None List all existing files that have been modified, and describe the changes: + .gitmodules + - Point to new CLUBB external (clubb_4ncar_20240605_73d60f6_gpufixes_posinf) + and new CDEPS external (cdeps1.0.45) + + cime/config/testdefs/testlist_cam.xml + - Add nvhpc gpu test on Derecho, remove Casper tests + + cime/config/testdefs/testmods_dirs/cam/outfrq9s_mg2_default/shell_commands + cime/config/testdefs/testmods_dirs/cam/outfrq9s_mg3_default/shell_commands + - Change NTASKS for Derecho gpus + + cime/config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols1536/ + - Directory renamed to cime/config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols760 + - Files updated to reflect the change + + doc/ChangeLog_template + - Added space for new derecho/nvhpc required tests + + src/physics/cam/clubb_intr.F90 + src/physics/cam/subcol_SILHS.F90 + - Updates to support the new external + + test/system/archive_baseline.sh + test/system/test_driver.sh + - Updates to require CAM_FC compiler specification on Derecho (either intel or nvhpc) If there were any failures reported from running test_driver.sh on any test platform, and checkin with these failures has been OK'd by the gatekeeper, @@ -91,13 +123,30 @@ ERS_Ln9_G4-a100-openacc.ne30pg3_ne30pg3_mg17.F2000dev.derecho_nvhpc.cam-outfrq9s - Expected baseline compare fail due to no baselines stored for GPU tests that didn't exist previously izumi/nag/aux_cam: +DAE.f45_f45_mg37.FHS94.izumi_nag.cam-dae (Overall: FAIL) details: +- pre-existing failure - issue #670 + +ERC_D_Ln9.f10_f10_mg37.QPC6.izumi_nag.cam-outfrq3s_am (Overall: DIFF) details: +ERC_D_Ln9.f10_f10_mg37.QPC6.izumi_nag.cam-outfrq3s_cospsathist (Overall: DIFF) details: +ERC_D_Ln9.f10_f10_mg37.QPC6.izumi_nag.cam-outfrq3s (Overall: DIFF) details: +ERC_D_Ln9.f10_f10_mg37.QPWmaC6.izumi_nag.cam-outfrq3s (Overall: DIFF) details: +ERI_D_Ln18.f19_f19_mg17.QPC6.izumi_nag.cam-ghgrmp_e8 (Overall: DIFF) details: +ERP_Ln9.ne5pg3_ne5pg3_mg37.QPC6.izumi_nag.cam-outfrq9s_clubbmf (Overall: DIFF) details: +SMS_D_Ln9.f10_f10_mg37.QPC6.izumi_nag.cam-outfrq3s_ba (Overall: DIFF) details: +SMS_P48x1_D_Ln3.f09_f09_mg17.QPC6HIST.izumi_nag.cam-outfrq3s_co2cycle_usecase (Overall: DIFF) details: +- Expected differences due to the new CLUBB external (See PR for discussion) izumi/gnu/aux_cam: +ERP_D_Ln9.C48_C48_mg17.QPC6.izumi_gnu.cam-outfrq9s (Overall: DIFF) details: +ERP_D_Ln9.ne3pg3_ne3pg3_mg37.QPC6.izumi_gnu.cam-outfrq9s_rrtmgp (Overall: DIFF) details: +- Expected differences due to the new CLUBB external (See PR for discussion) CAM tag used for the baseline comparison tests if different than previous -tag: +tag: cam6_4_018 -Summarize any changes to answers: BFB, as expected +Summarize any changes to answers: + All compsets that use CLUBB (cam6+) will have slight answer changes. Discussion in PR. + Nvhpc gpu tests have no stored baseline for comparison. ===============================================================