diff --git a/Externals.cfg b/Externals.cfg index 9badad437d..7dfd41a073 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -1,5 +1,5 @@ [ccs_config] -tag = ccs_config_cesm0.0.73 +tag = ccs_config_cesm0.0.78 protocol = git repo_url = https://github.com/ESMCI/ccs_config_cesm local_path = ccs_config @@ -13,7 +13,7 @@ local_path = components/cice5 required = True [cice6] -tag = cesm_cice6_4_1_8 +tag = cesm_cice6_4_1_10 protocol = git repo_url = https://github.com/ESCOMP/CESM_CICE local_path = components/cice @@ -21,14 +21,14 @@ externals = Externals.cfg required = True [cmeps] -tag = cmeps0.14.34 +tag = cmeps0.14.39 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps required = True [cdeps] -tag = cdeps1.0.14 +tag = cdeps1.0.21 protocol = git repo_url = https://github.com/ESCOMP/CDEPS.git local_path = components/cdeps @@ -36,7 +36,7 @@ externals = Externals_CDEPS.cfg required = True [cpl7] -tag = cpl77.0.5 +tag = cpl77.0.6 protocol = git repo_url = https://github.com/ESCOMP/CESM_CPL7andDataComps local_path = components/cpl7 @@ -57,21 +57,21 @@ local_path = libraries/mct required = True [parallelio] -tag = pio2_5_10 +tag = pio2_6_2 protocol = git repo_url = https://github.com/NCAR/ParallelIO local_path = libraries/parallelio required = True [cime] -tag = cime6.0.125 +tag = cime6.0.156 protocol = git repo_url = https://github.com/ESMCI/cime local_path = cime required = True [cism] -tag = cismwrap_2_1_95 +tag = cismwrap_2_1_96 protocol = git repo_url = https://github.com/ESCOMP/CISM-wrapper local_path = components/cism @@ -79,7 +79,7 @@ externals = Externals_CISM.cfg required = True [clm] -tag = ctsm5.1.dev124 +tag = ctsm5.1.dev139 protocol = git repo_url = https://github.com/ESCOMP/CTSM local_path = components/clm diff --git a/Externals_CAM.cfg b/Externals_CAM.cfg index 0770c68211..bfba41d0db 100644 --- a/Externals_CAM.cfg +++ b/Externals_CAM.cfg @@ -78,7 +78,7 @@ required = True [hemco] local_path = src/hemco -tag = hemco-cesm1_2_0_hemco3_6_2_cesm +tag = hemco-cesm1_2_1_hemco3_6_3_cesm protocol = git repo_url = https://github.com/ESCOMP/HEMCO_CESM.git required = True diff --git a/cime_config/buildlib b/cime_config/buildlib index 90322da1ac..73db5db3dd 100755 --- a/cime_config/buildlib +++ b/cime_config/buildlib @@ -3,11 +3,12 @@ """ create the cam library """ -#pylint: disable=multiple-imports, wrong-import-position, wildcard-import -#pylint: disable=unused-wildcard-import, bad-whitespace, too-many-locals -#pylint: disable=invalid-name +# pylint: disable=multiple-imports, wrong-import-position, wildcard-import +# pylint: disable=unused-wildcard-import, bad-whitespace, too-many-locals +# pylint: disable=invalid-name import sys, os, filecmp, shutil, imp + _CIMEROOT = os.environ.get("CIMEROOT") if _CIMEROOT is None: raise SystemExit("ERROR: must set CIMEROOT environment variable") @@ -17,51 +18,27 @@ sys.path.append(_LIBDIR) from standard_script_setup import * from CIME.case import Case -from CIME.utils import run_cmd, expect +from CIME.utils import run_sub_or_cmd, expect, run_cmd from CIME.buildlib import parse_input from CIME.build import get_standard_makefile_args logger = logging.getLogger(__name__) -############################################################################### -def _build_fms(caseroot, libroot, bldroot): -############################################################################### - - with Case(caseroot) as case: - - # Only need FMS for fv3 dycore - cam_dycore = case.get_value("CAM_DYCORE") - if cam_dycore == 'fv3': - # Check to see if some other component built it already - if not os.path.exists(os.path.join(libroot,"libfms.a")): - # first check for the external FMS library and build it - srcroot = case.get_value("SRCROOT") - fmsbuildlib = os.path.join(srcroot,"libraries","FMS","buildlib") - fmsbuilddir = os.path.join(case.get_value("EXEROOT"),"FMS") - expect(os.path.exists(fmsbuildlib), "FMS external not found") - stat, _, err = run_cmd("{} {} {} {}".format(fmsbuildlib, case.get_value("EXEROOT"), fmsbuilddir, caseroot), verbose=True) - expect(stat==0, "FMS build Failed {}".format(err)) - - libfms = os.path.join(bldroot,"FMS","libfms.a") - if os.path.exists(libfms): - shutil.copy(libfms, libroot) - ############################################################################### def _build_cam(caseroot, libroot, bldroot): -############################################################################### + ############################################################################### with Case(caseroot, read_only=False) as case: srcroot = case.get_value("SRCROOT") - #------------------------------------------------------- + # ------------------------------------------------------- # Call cam's buildcpp - #------------------------------------------------------- + # ------------------------------------------------------- testpath = os.path.join(srcroot, "components", "cam") if os.path.exists(testpath): srcroot = testpath - cmd = os.path.join(os.path.join(srcroot, - "cime_config", "buildcpp")) + cmd = os.path.join(os.path.join(srcroot, "cime_config", "buildcpp")) logger.info(" ...calling cam buildcpp to set build time options") try: mod = imp.load_source("buildcpp", cmd) @@ -76,16 +53,32 @@ def _build_cam(caseroot, libroot, bldroot): gmake_j = case.get_value("GMAKE_J") gmake = case.get_value("GMAKE") mach = case.get_value("MACH") - - #------------------------------------------------------- + user_incldir = None + cam_dycore = case.get_value("CAM_DYCORE") + if cam_dycore == "fv3": + slr = os.path.abspath(case.get_value("SHAREDLIBROOT")) + compiler = case.get_value("COMPILER") + mpilib = case.get_value("MPILIB") + debug = "debug" if case.get_value("DEBUG") else "nodebug" + threaded = "threads" if case.get_value("BUILD_THREADED") or case.get_value("FORCE_BUILD_SMP") else "nothreads" + comp_interface = case.get_value("COMP_INTERFACE") + fmsbuilddir = os.path.join( + slr, compiler, mpilib, debug, threaded, comp_interface) + user_incldir = '"-I{} -I{} -I{}"'.format( + os.path.join(srcroot, "libraries", "FMS", "src", "include"), + os.path.join(srcroot, "libraries", "FMS", "src", "mpp", "include"), + fmsbuilddir, + ) + + # ------------------------------------------------------- # Filepath is created in caseroot/camconf by the call # to buildcpp - this needs to be copied to bldroot - #------------------------------------------------------- + # ------------------------------------------------------- filesrc = os.path.join(caseroot, "Buildconf", "camconf", "Filepath") filedst = os.path.join(bldroot, "Filepath_tmp") shutil.copy(filesrc, filedst) - filedst = os.path.join(bldroot, "Filepath") + filedst = os.path.join(bldroot, "Filepath") filedst_tmp = os.path.join(bldroot, "Filepath_tmp") if os.path.isfile(filedst): if not filecmp.cmp(filedst_tmp, filedst): @@ -93,28 +86,42 @@ def _build_cam(caseroot, libroot, bldroot): else: shutil.move(filedst_tmp, filedst) - #------------------------------------------------------- + # ------------------------------------------------------- + # fms is needed by fv3 and should have been built by the framework + # ------------------------------------------------------- + if cam_dycore == "fv3": + libfms = os.path.join(fmsbuilddir, "libfms.a") + expect(os.path.isfile(libfms), "FMS library not found {}".format(libfms)) + shutil.copy(libfms, libroot) + + # ------------------------------------------------------- # build the library - #------------------------------------------------------- - complib = os.path.join(libroot, "libatm.a") + # ------------------------------------------------------- + complib = os.path.join(libroot, "libatm.a") makefile = os.path.join(casetools, "Makefile") - cmd = "{} complib -j {} COMP_NAME=cam COMPLIB={} -f {} {} " \ - .format(gmake, gmake_j, complib, makefile, get_standard_makefile_args(case)) + cmd = "{} complib -j {} COMP_NAME=cam COMPLIB={} -f {} {} ".format( + gmake, gmake_j, complib, makefile, get_standard_makefile_args(case) + ) if cam_cppdefs: cmd += " USER_CPPDEFS='{}'".format(cam_cppdefs) + if user_incldir: + cmd += " USER_INCLDIR={}".format(user_incldir) + rc, out, err = run_cmd(cmd) logger.info("%s: \n\n output:\n %s \n\n err:\n\n%s\n", cmd, out, err) expect(rc == 0, "Command %s failed with rc=%s" % (cmd, rc)) + ############################################################################### + def _main_func(): caseroot, libroot, bldroot = parse_input(sys.argv) - _build_fms(caseroot, libroot, bldroot) _build_cam(caseroot, libroot, bldroot) + ############################################################################### if __name__ == "__main__": diff --git a/doc/ChangeLog b/doc/ChangeLog index 2397b96619..4aa82db7fe 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,3 +1,112 @@ + +=============================================================== + +Tag name: cam6_3_129 +Originator(s): cacraig, fischer, hplin, jet, jedwards +Date: Oct 04, 2023 +One-line Summary: Update externals to match cesm2_3_alpha16d and bring in test_driver which supports derecho +Github PR URL: https://github.com/ESCOMP/CAM/pull/888 + +Purpose of changes (include the issue number and title text for each relevant GitHub issue): + - Add derecho support to test_driver (https://github.com/ESCOMP/CAM/pull/879) + - Fix long line in HEMCO external (https://github.com/ESCOMP/CAM/issues/871) + - Fix building older FMS library (no issue as was discovered during testing, but discussion is in PR#888) + +Describe any changes made to build system: N/A + +Describe any changes made to the namelist: N/A + +List any changes to the defaults for the boundary datasets: N/A + +Describe any substantial timing or memory changes: N/A + +Code reviewed by: nusbaume, jet, fvitt + +List all files eliminated: N/A + +List all files added and what they do: N/A + +List all existing files that have been modified, and describe the changes: + +M Externals.cfg + - update externals to match cesm2_3_alpha16d (ccs_config external is one tag newer) + +M Externals_CAM.cfg + - update HEMCO tag to get fix for line too long + +M cime_config/buildlib + - Updates for building older FMS library (required with changes in external). NOTE - The regression test + which tests FV3 (C96_C96_mg17) is broken. It was verified that the actual test works, but the regression test + no longer compiles properly due to changes in the SHARED_LIB build process. + +M test/system/test_driver.sh + - add derecho functionality (uses cheyenne aux_cam hooks to determine tests to run) + It is important to note that there are a number of failures when the tests are run on derecho (see + github issue https://github.com/ESCOMP/CAM/issues/892 for details) + +If there were any failures reported from running test_driver.sh on any test +platform, and checkin with these failures has been OK'd by the gatekeeper, +then copy the lines from the td.*.status files for the failed tests to the +appropriate machine below. All failed tests must be justified. + +cheyenne/intel/aux_cam: + FAIL ERP_Ln9_Vnuopc.C96_C96_mg17.F2000climo.cheyenne_intel.cam-outfrq9s_mg3 MODEL_BUILD time=2 + - Failure of FMS library compilation when more than one regression test is made due to changes in SHAREDLIB_BUILD + Was approved at AMP SE/scientist meeting to have this test fail until the upcoming FV3 PR is brought in + + FAIL ERP_Ln9_Vnuopc.f09_f09_mg17.FCSD_HCO.cheyenne_intel.cam-outfrq9s COMPARE_base_rest + FAIL SMS_Lh12_Vnuopc.f09_f09_mg17.FCSD_HCO.cheyenne_intel.cam-outfrq3h (Overall: DIFF) details: + - pre-existing failure + + ERP_D_Ln9.ne30pg3_ne30pg3_mg17.FLTHIST.cheyenne_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_D_Ln9_Vnuopc.ne30pg3_ne30pg3_mg17.F2000dev.cheyenne_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_Ld3_Vnuopc.f09_f09_mg17.FWHIST.cheyenne_intel.cam-reduced_hist1d (Overall: DIFF) details: + ERP_Lh12_Vnuopc.f19_f19_mg17.FW4madSD.cheyenne_intel.cam-outfrq3h (Overall: DIFF) details: + ERP_Ln9_Vnuopc.f09_f09_mg17.F1850.cheyenne_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_Ln9_Vnuopc.f09_f09_mg17.F2000climo.cheyenne_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_Ln9_Vnuopc.f09_f09_mg17.F2000dev.cheyenne_intel.cam-outfrq9s_mg3 (Overall: DIFF) details: + ERP_Ln9_Vnuopc.f09_f09_mg17.F2010climo.cheyenne_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_Ln9_Vnuopc.f09_f09_mg17.FHIST_BDRD.cheyenne_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_Ln9_Vnuopc.f19_f19_mg17.FWsc1850.cheyenne_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_Ln9_Vnuopc.ne30_ne30_mg17.FCnudged.cheyenne_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_Ln9_Vnuopc.ne30pg3_ne30pg3_mg17.FW2000climo.cheyenne_intel.cam-outfrq9s_wcm_ne30 (Overall: DIFF) details: + ERS_Ld3_Vnuopc.f10_f10_mg37.F1850.cheyenne_intel.cam-outfrq1d_14dec_ghg_cam_dev (Overall: DIFF) details: + ERS_Ln9_P288x1_Vnuopc.mpasa120_mpasa120.F2000climo.cheyenne_intel.cam-outfrq9s_mpasa120 (Overall: DIFF) details: + ERS_Ln9_P36x1_Vnuopc.mpasa480_mpasa480.F2000climo.cheyenne_intel.cam-outfrq9s_mpasa480 (Overall: DIFF) details: + ERS_Ln9_Vnuopc.f09_f09_mg17.FX2000.cheyenne_intel.cam-outfrq9s (Overall: DIFF) details: + ERS_Ln9_Vnuopc.f19_f19_mg17.FSPCAMS.cheyenne_intel.cam-outfrq9s (Overall: DIFF) details: + ERS_Ln9_Vnuopc.f19_f19_mg17.FXSD.cheyenne_intel.cam-outfrq9s (Overall: DIFF) details: + SMS_D_Ln9.ne30pg3_ne30pg3_mg17.FMTHIST.cheyenne_intel.cam-outfrq9s (Overall: DIFF) details: + SMS_D_Ln9_Vnuopc.f09_f09_mg17.FCts2nudged.cheyenne_intel.cam-outfrq9s_leapday (Overall: DIFF) details: + SMS_D_Ln9_Vnuopc.f09_f09_mg17.FCvbsxHIST.cheyenne_intel.cam-outfrq9s (Overall: DIFF) details: + SMS_D_Ln9_Vnuopc.f09_f09_mg17.FSD.cheyenne_intel.cam-outfrq9s (Overall: DIFF) details: + SMS_D_Ln9_Vnuopc.f19_f19_mg17.FWma2000climo.cheyenne_intel.cam-outfrq9s (Overall: DIFF) details: + SMS_D_Ln9_Vnuopc.f19_f19_mg17.FWma2000climo.cheyenne_intel.cam-outfrq9s_waccm_ma_mam4 (Overall: DIFF) details: + SMS_D_Ln9_Vnuopc.f19_f19_mg17.FXHIST.cheyenne_intel.cam-outfrq9s_amie (Overall: DIFF) details: + SMS_D_Ln9_Vnuopc.ne16_ne16_mg17.FX2000.cheyenne_intel.cam-outfrq9s (Overall: DIFF) details: + SMS_D_Ln9_Vnuopc_P720x1.ne0ARCTICne30x4_ne0ARCTICne30x4_mt12.FHIST.cheyenne_intel.cam-outfrq9s (Overall: DIFF) details: + SMS_D_Ln9_Vnuopc_P720x1.ne0CONUSne30x8_ne0CONUSne30x8_mt12.FCHIST.cheyenne_intel.cam-outfrq9s (Overall: DIFF) details: + SMS_D_Ln9_Vnuopc_P720x1.ne30pg3_ne30pg3_mg17.FCLTHIST.cheyenne_intel.cam-outfrq9s (Overall: DIFF) details: + SMS_D_Ln9_Vnuopc.T42_T42.FSCAM.cheyenne_intel.cam-outfrq9s (Overall: DIFF) details: + SMS_Ld1_Vnuopc.f09_f09_mg17.FW2000climo.cheyenne_intel.cam-outfrq1d (Overall: DIFF) details: + SMS_Ld1_Vnuopc.f19_f19.F2000dev.cheyenne_intel.cam-outfrq1d (Overall: DIFF) details: + SMS_Ld1_Vnuopc.ne30pg3_ne30pg3_mg17.FC2010climo.cheyenne_intel.cam-outfrq1d (Overall: DIFF) details: + SMS_Lm13_Vnuopc.f10_f10_mg37.F2000climo.cheyenne_intel.cam-outfrq1m (Overall: DIFF) details: + SMS_Ln9_Vnuopc.f19_f19_mg17.FHIST.cheyenne_intel.cam-outfrq9s_nochem (Overall: DIFF) details: + - answer changes due to updated externals + +izumi/nag/aux_cam: + DAE_Vnuopc.f45_f45_mg37.FHS94.izumi_nag.cam-dae (Overall: FAIL) details: + - pre-existing failure + +izumi/gnu/aux_cam: + FAIL SMS_D_Ln9.f10_f10_mg37.2000_CAM%DEV%GHGMAM4_CLM50%SP_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV_SESP.izumi_gnu.cam-outfrq9s BASELINE /fs/cgd/csm/models/atm/cam/pretag_bl/cam6_3_128_gnu: FIELDLIST field lists differ (otherwise bit-for-bit) + FAIL SMS_P48x1_D_Ln9_Vnuopc.f19_f19_mg17.FW4madSD.izumi_gnu.cam-outfrq9s BASELINE /fs/cgd/csm/models/atm/cam/pretag_bl/cam6_3_128_gnu: DIFF + - answer changes due to updated externals + +All tests had namelist changes as well + +=============================================================== =============================================================== Tag name: cam6_3_128 diff --git a/test/system/test_driver.sh b/test/system/test_driver.sh index 964f647793..a80121753d 100755 --- a/test/system/test_driver.sh +++ b/test/system/test_driver.sh @@ -264,7 +264,7 @@ cat > ${submit_script_cime} << EOF #PBS -N cime-tests #PBS -q $CAM_BATCHQ #PBS -A $CAM_ACCOUNT -#PBS -l walltime=4:00:00 +#PBS -l walltime=$wallclock_limit #PBS -l select=1:ncpus=36:mpiprocs=36 #PBS -j oe #PBS -l inception=login @@ -273,6 +273,63 @@ EOF ##^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ writing to batch script ^^^^^^^^^^^^^^^^^^^ ;; + ##derecho + derecho* | dec* ) + submit_script_cime="`pwd -P`/test_driver_derecho_cime_${cur_time}.sh" + + if [ -z "$CAM_ACCOUNT" ]; then + echo "ERROR: Must set the environment variable CAM_ACCOUNT" + exit 2 + fi + + if [ -z "$CAM_BATCHQ" ]; then + export CAM_BATCHQ="main" + fi + + # wallclock for run job + wallclock_limit="5:00:00" + + if [ $gmake_j = 0 ]; then + gmake_j=128 + fi + + # run tests on 1 node using 64 tasks/node, 2 threads/task + # These settings are ignored on cheyenne and derecho. + # PE layouts come from config_pes.xml. + CAM_TASKS=64 + CAM_THREADS=2 + + # change parallel configuration on 1 nodes using 32 tasks, 1 threads/task + # These settings are ignored on cheyenne and derecho. + # PE layouts come from config_pes.xml. + CAM_RESTART_TASKS=32 + CAM_RESTART_THREADS=1 + + mach_workspace="/glade/derecho/scratch" + + # Check for CESM baseline directory + if [ -n "${BL_TESTDIR}" ] && [ ! -d "${BL_TESTDIR}" ]; then + echo "CESM_BASELINE ${BL_TESTDIR} not found. Check BL_TESTDIR for correct tag name." + exit 3 + fi + +#------------------------------------------- + +cat > ${submit_script_cime} << EOF +#!/bin/bash +# +#PBS -N cime-tests +#PBS -q $CAM_BATCHQ +#PBS -A $CAM_ACCOUNT +#PBS -l walltime=$wallclock_limit +#PBS -l select=1:ncpus=128:mpiprocs=128 +#PBS -j oe + +EOF + +##^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ writing to batch script ^^^^^^^^^^^^^^^^^^^ + ;; + ##hobart hob* | h[[:digit:]]* ) @@ -431,7 +488,7 @@ cat > ${submit_script_cime} << EOF #PBS -N cime-tests #PBS -q $CAM_BATCHQ #PBS -A $CAM_ACCOUNT -#PBS -l walltime=2:00:00 +#PBS -l walltime=$wallclock_limit #PBS -l select=1:ncpus=36:mpiprocs=36:mem=300GB #PBS -j oe #PBS -V @@ -451,6 +508,9 @@ comp="" if [ "${hostname:0:4}" == "chey" ]; then cesm_test_mach="cheyenne" fi +if [ "${hostname:0:5}" == "derec" ] || [ "${hostname:0:3}" == "dec" ]; then + cesm_test_mach="derecho" +fi if [ "${hostname:0:6}" == "hobart" ]; then cesm_test_mach="hobart" fi @@ -465,13 +525,20 @@ if [ -n "${CAM_FC}" ]; then fi if [ "${cesm_test_suite}" != "none" -a -n "${cesm_test_mach}" ]; then - if [ "${hostname:0:5}" != "izumi" ]; then + if [ "${hostname:0:5}" != "izumi" ] && [ "${hostname:0:7}" != "derecho" ]; then module load python fi for cesm_test in ${cesm_test_suite}; do - testargs="--xml-category ${cesm_test} --xml-machine ${cesm_test_mach} --retry 2" + # Force derecho to run the cheyenne testlist. + # After the transition to derecho is completed, this if statement can be removed and + # just the else needs to remain. + if [ "${cesm_test_mach}" == "derecho" ]; then + testargs="--xml-category ${cesm_test} --xml-machine cheyenne --mach ${cesm_test_mach} --retry 2" + else + testargs="--xml-category ${cesm_test} --xml-machine ${cesm_test_mach} --retry 2" + fi if [ -n "${use_existing}" ]; then test_id="${use_existing}" @@ -551,6 +618,10 @@ if [ "${cesm_test_suite}" != "none" -a -n "${cesm_test_mach}" ]; then chey* | r* ) testargs="${testargs} --queue ${CAM_BATCHQ} --test-root ${cesm_testdir} --output-root ${cesm_testdir}" ;; + # derecho + derec* | dec* ) + testargs="${testargs} --queue ${CAM_BATCHQ} --test-root ${cesm_testdir} --output-root ${cesm_testdir}" + ;; # casper casper* | crhtc* ) testargs="${testargs} --queue ${CAM_BATCHQ} --test-root ${cesm_testdir} --output-root ${cesm_testdir}" @@ -631,6 +702,13 @@ if [ "${cesm_test_suite}" != "none" -a -n "${cesm_test_mach}" ]; then qsub ${submit_script_cime} fi + if [ "${hostname:0:2}" == "de" ]; then + echo "cd ${script_dir}" >> ${submit_script_cime} + echo './create_test' ${testargs} >> ${submit_script_cime} + chmod u+x ${submit_script_cime} + qsub ${submit_script_cime} + fi + if [ "${hostname:0:6}" == "hobart" ]; then echo "cd ${script_dir}" >> ${submit_script_cime} echo './create_test' ${testargs} >> ${submit_script_cime}