From 8322416793ae2b76c2bafa9c7b9b108c289ede9d Mon Sep 17 00:00:00 2001 From: Elizabeth Hunke Date: Fri, 18 Aug 2023 17:34:24 -0600 Subject: [PATCH] Updates to advanced snow physics implementation (#852) * Replace tr_snow flag with snwredist, snwgrain in some places (tr_snow is still used more generally). Fix intent(out) compile issue in ice_read_write.F90. Replace badger with chicoma machine files. * update icepack to 86cae16d1b7c4c4f8 --------- Co-authored-by: apcraig --- cicecore/cicedyn/general/ice_init.F90 | 2 +- cicecore/cicedyn/general/ice_step_mod.F90 | 15 ++-- .../cicedyn/infrastructure/ice_read_write.F90 | 1 + configuration/scripts/cice.batch.csh | 6 +- configuration/scripts/cice.launch.csh | 4 +- .../scripts/machines/Macros.badger_intel | 56 --------------- .../scripts/machines/Macros.chicoma_intel | 58 +++++++++++++++ .../scripts/machines/env.badger_intel | 47 ------------ .../scripts/machines/env.chicoma_intel | 71 +++++++++++++++++++ icepack | 2 +- 10 files changed, 145 insertions(+), 117 deletions(-) delete mode 100644 configuration/scripts/machines/Macros.badger_intel create mode 100644 configuration/scripts/machines/Macros.chicoma_intel delete mode 100644 configuration/scripts/machines/env.badger_intel create mode 100755 configuration/scripts/machines/env.chicoma_intel diff --git a/cicecore/cicedyn/general/ice_init.F90 b/cicecore/cicedyn/general/ice_init.F90 index 3b8d83d1f..4ed128f5e 100644 --- a/cicecore/cicedyn/general/ice_init.F90 +++ b/cicecore/cicedyn/general/ice_init.F90 @@ -1356,7 +1356,7 @@ subroutine input_data abort_list = trim(abort_list)//":8" endif - if (snwredist(1:4) /= 'none' .and. .not. tr_snow) then + if (snwredist(1:3) == 'ITD' .and. .not. tr_snow) then if (my_task == master_task) then write (nu_diag,*) 'ERROR: snwredist on but tr_snow=F' write (nu_diag,*) 'ERROR: Use tr_snow=T for snow redistribution' diff --git a/cicecore/cicedyn/general/ice_step_mod.F90 b/cicecore/cicedyn/general/ice_step_mod.F90 index 56510c247..31989c73c 100644 --- a/cicecore/cicedyn/general/ice_step_mod.F90 +++ b/cicecore/cicedyn/general/ice_step_mod.F90 @@ -275,7 +275,7 @@ subroutine step_therm1 (dt, iblk) logical (kind=log_kind) :: & tr_iage, tr_FY, tr_iso, tr_aero, tr_pond, & - tr_pond_lvl, tr_pond_topo, calc_Tsfc, highfreq, tr_snow + tr_pond_lvl, tr_pond_topo, calc_Tsfc, snwgrain real (kind=dbl_kind) :: & puny ! a very small number @@ -296,13 +296,12 @@ subroutine step_therm1 (dt, iblk) call icepack_query_parameters(puny_out=puny) call icepack_query_parameters(calc_Tsfc_out=calc_Tsfc) - call icepack_query_parameters(highfreq_out=highfreq) + call icepack_query_parameters(snwgrain_out=snwgrain) call icepack_query_tracer_sizes(ntrcr_out=ntrcr) call icepack_query_tracer_flags( & tr_iage_out=tr_iage, tr_FY_out=tr_FY, tr_iso_out=tr_iso, & tr_aero_out=tr_aero, tr_pond_out=tr_pond, & - tr_pond_lvl_out=tr_pond_lvl, tr_pond_topo_out=tr_pond_topo, & - tr_snow_out=tr_snow) + tr_pond_lvl_out=tr_pond_lvl, tr_pond_topo_out=tr_pond_topo) call icepack_query_tracer_indices( & nt_apnd_out=nt_apnd, nt_hpnd_out=nt_hpnd, nt_ipnd_out=nt_ipnd, & nt_alvl_out=nt_alvl, nt_vlvl_out=nt_vlvl, nt_Tsfc_out=nt_Tsfc, & @@ -357,7 +356,7 @@ subroutine step_therm1 (dt, iblk) do j = jlo, jhi do i = ilo, ihi - if (tr_snow) then + if (snwgrain) then do n = 1, ncat do k = 1, nslyr rsnwn (k,n) = trcrn(i,j,nt_rsnw +k-1,n,iblk) @@ -365,7 +364,7 @@ subroutine step_therm1 (dt, iblk) smliqn(k,n) = trcrn(i,j,nt_smliq+k-1,n,iblk) enddo enddo - endif ! tr_snow + endif ! snwgrain if (tr_iso) then ! trcrn(nt_iso*) has units kg/m^3 do n=1,ncat @@ -556,7 +555,7 @@ subroutine step_therm1 (dt, iblk) endif - if (tr_snow) then + if (snwgrain) then do n = 1, ncat do k = 1, nslyr trcrn(i,j,nt_rsnw +k-1,n,iblk) = rsnwn (k,n) @@ -564,7 +563,7 @@ subroutine step_therm1 (dt, iblk) trcrn(i,j,nt_smliq+k-1,n,iblk) = smliqn(k,n) enddo enddo - endif ! tr_snow + endif ! snwgrain if (tr_iso) then do n = 1, ncat diff --git a/cicecore/cicedyn/infrastructure/ice_read_write.F90 b/cicecore/cicedyn/infrastructure/ice_read_write.F90 index b9074d8f6..041f3516b 100644 --- a/cicecore/cicedyn/infrastructure/ice_read_write.F90 +++ b/cicecore/cicedyn/infrastructure/ice_read_write.F90 @@ -1272,6 +1272,7 @@ subroutine ice_read_nc_xy(fid, nrec, varname, work, diag, & if (orca_halogrid .and. .not. present(restart_ext)) deallocate(work_g2) #else + work = c0 ! to satisfy intent(out) attribute call abort_ice(subname//' ERROR: USE_NETCDF cpp not defined', & file=__FILE__, line=__LINE__) #endif diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index 5a47decf1..263b16d02 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -184,7 +184,7 @@ cat >> ${jobfile} <> ${jobfile} << EOFB #SBATCH -J ${ICE_CASENAME} #SBATCH -t ${batchtime} @@ -194,7 +194,9 @@ cat >> ${jobfile} << EOFB #SBATCH -o slurm%j.out ###SBATCH --mail-type END,FAIL ###SBATCH --mail-user=eclare@lanl.gov -#SBATCH --qos=standby +##SBATCH --qos=debug +#SBATCH --qos=standard +##SBATCH --qos=standby EOFB else if (${ICE_MACHINE} =~ fram*) then diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh index 971bc0075..fe72e5a27 100755 --- a/configuration/scripts/cice.launch.csh +++ b/configuration/scripts/cice.launch.csh @@ -119,14 +119,14 @@ EOFR endif #======= -else if (${ICE_MACHCOMP} =~ badger*) then +else if (${ICE_MACHCOMP} =~ chicoma*) then if (${ICE_COMMDIR} =~ serial*) then cat >> ${jobfile} << EOFR ./cice >&! \$ICE_RUNLOG_FILE EOFR else cat >> ${jobfile} << EOFR -mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +srun -n ${ntasks} -c ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE EOFR endif diff --git a/configuration/scripts/machines/Macros.badger_intel b/configuration/scripts/machines/Macros.badger_intel deleted file mode 100644 index ce4eccc9c..000000000 --- a/configuration/scripts/machines/Macros.badger_intel +++ /dev/null @@ -1,56 +0,0 @@ -#============================================================================== -# Macros file for LANL badger, intel compiler -#============================================================================== - -CPP := fpp -CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} -CFLAGS := -c -O2 -fp-model precise -xHost - -FIXEDFLAGS := -132 -FREEFLAGS := -FR -FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -xHost -FFLAGS_NOOPT:= -O0 - -ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -else - FFLAGS += -O2 -endif - -SCC := icc -SFC := ifort -MPICC := mpicc -MPIFC := mpif90 - -ifeq ($(ICE_COMMDIR), mpi) - FC := $(MPIFC) - CC := $(MPICC) -else - FC := $(SFC) - CC := $(SCC) -endif -LD:= $(FC) - -NETCDF_PATH := /usr/projects/hpcsoft/toss3/common/netcdf/4.4.0_intel-18.0.5 -PNETCDF_PATH := /usr/projects/hpcsoft/toss3/badger/netcdf/4.4.0_intel-18.0.5_openmpi-2.1.2 - -PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs - -ifeq ($(ICE_IOTYPE), netcdf) - INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include -I$(PNETCDF_PATH)/include - - LIB_NETCDF := $(NETCDF_PATH)/lib - LIB_PNETCDF := $(PNETCDF_PATH)/lib - LIB_MPI := $(IMPILIBDIR) - - SLIBS := -L$(LIB_NETCDF) -lnetcdf -L$(LIB_PNETCDF) -lnetcdff -else - SLIBS := -endif - -ifeq ($(ICE_THREADED), true) - LDFLAGS += -qopenmp - CFLAGS += -qopenmp - FFLAGS += -qopenmp -endif - diff --git a/configuration/scripts/machines/Macros.chicoma_intel b/configuration/scripts/machines/Macros.chicoma_intel new file mode 100644 index 000000000..7767aff8f --- /dev/null +++ b/configuration/scripts/machines/Macros.chicoma_intel @@ -0,0 +1,58 @@ +#============================================================================== +# Macros file for LANL chicoma, intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -fp-model precise -fcommon + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback +# -mcmodel medium -shared-intel +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created +# FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays +else + FFLAGS += -O2 +endif + +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +# defined by module +#NETCDF_PATH := $(NETCDF_DIR) +NETCDF_PATH := /opt/cray/pe/netcdf-hdf5parallel/4.9.0.1/intel/19.0/ +#PNETCDF_PATH := $(PNETCDF) +#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default +#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib + +#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs + +#INCLDIR := $(INCLDIR) +INCLDIR += -I$(NETCDF_PATH)/include + +LIB_NETCDF := $(NETCDF_PATH)/lib +#LIB_PNETCDF := $(PNETCDF_PATH)/lib +#LIB_MPI := $(IMPILIBDIR) +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + diff --git a/configuration/scripts/machines/env.badger_intel b/configuration/scripts/machines/env.badger_intel deleted file mode 100644 index 5532b26d6..000000000 --- a/configuration/scripts/machines/env.badger_intel +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/tcsh -f - -set inp = "undefined" -if ($#argv == 1) then - set inp = $1 -endif - -if ("$inp" != "-nomodules") then - -#source /usr/share/Modules/init/csh - -#module purge -#module load intel -#module load openmpi -module unload hdf5-serial -module unload hdf5-parallel -module unload netcdf-serial -module unload netcdf-h5parallel -module load hdf5-serial -module load netcdf-serial/4.4.0 -module load hdf5-parallel -module load netcdf-h5parallel/4.4.0 - -#setenv OMP_STACKSIZE 256M -#setenv MP_LABELIO yes -#setenv MP_INFOLEVEL 2 -#setenv MP_SHARED_MEMORY yes -#setenv MP_EUILIB us -#setenv MP_EAGER_LIMIT 0 - -endif - -setenv ICE_MACHINE_MACHNAME badger -setenv ICE_MACHINE_MACHINFO "Penguin Intel Xeon Broadwell" -setenv ICE_MACHINE_ENVNAME intel -setenv ICE_MACHINE_ENVINFO "(Note: can vary) ifort 19.0.4.243 20190416, openmpi/2.1.2, netcdf4.4.0" -setenv ICE_MACHINE_MAKE gmake -setenv ICE_MACHINE_WKDIR /net/scratch4/$user/CICE_RUNS -setenv ICE_MACHINE_INPUTDATA /usr/projects/climate/eclare/DATA/Consortium -setenv ICE_MACHINE_BASELINE /net/scratch4/$user/CICE_BASELINE -setenv ICE_MACHINE_SUBMIT "sbatch " -#setenv ICE_MACHINE_ACCT e3sm -setenv ICE_MACHINE_ACCT climatehilat -setenv ICE_MACHINE_QUEUE "default" -setenv ICE_MACHINE_TPNODE 16 -setenv ICE_MACHINE_BLDTHRDS 1 -setenv ICE_MACHINE_QSTAT "squeue --jobs=" diff --git a/configuration/scripts/machines/env.chicoma_intel b/configuration/scripts/machines/env.chicoma_intel new file mode 100755 index 000000000..a324ec0fe --- /dev/null +++ b/configuration/scripts/machines/env.chicoma_intel @@ -0,0 +1,71 @@ +#!/bin/csh -f + +# this works (current defaults with PrgEnv-intel) +#Currently Loaded Modules: +# 1) craype-x86-rome 7) python/3.10-anaconda-2023.03 13) cray-mpich/8.1.21 +# 2) libfabric/1.15.0.0 8) craype/2.7.19 14) totalview/2023.1.6 +# 3) craype-network-ofi 9) cray-dsmml/0.2.2 15) use.own +# 4) perftools-base/22.09.0 10) cray-libsci/22.11.1.2 16) idl/8.5 +# 5) xpmem/2.4.4-2.3_13.8__gff0e1d9.shasta 11) PrgEnv-intel/8.3.3 17) cray-hdf5-parallel/1.12.2.1 +# 6) git/2.40.0 12) intel/2022.2.1 18) cray-netcdf-hdf5parallel/4.9.0.1 + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +#source /opt/modules/default/init/csh + +#module unload PrgEnv-cray +#module unload PrgEnv-gnu +#module unload PrgEnv-intel +#module unload PrgEnv-pgi +#module load PrgEnv-intel/8.3.3 + +#module unload intel +#module load intel/2022.2.1 + +#module unload cray-mpich +#module unload cray-mpich2 +#module load cray-mpich/8.1.21 + +#module unload netcdf +#module unload cray-netcdf +#module unload cray-hdf5 +#module unload cray-hdf5-parallel +#module unload cray-netcdf-hdf5parallel +#module unload cray-parallel-netcdf +#module load cray-hdf5/1.12.2.1 +#module load cray-netcdf/4.9.0.1 +#module load cray-hdf5-parallel/1.12.2.1 +#module load cray-netcdf-hdf5parallel/4.9.0.1 + +#module unload cray-libsci +#module unload craype-hugepages2M +#module load craype-broadwell + +#setenv NETCDF_PATH ${NETCDF_DIR} +#setenv NETCDF_PATH /opt/cray/pe/netcdf/4.9.0.1/intel/19.0 +#setenv NETCDF_PATH /opt/cray/pe/netcdf-hdf5parallel/4.9.0.1/intel/19.0 +limit coredumpsize unlimited +limit stacksize unlimited +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_MACHNAME chicoma +setenv ICE_MACHINE_MACHINFO "HPE Cray EX, AMD EPYC 7H12 processors" +setenv ICE_MACHINE_ENVNAME intel +setenv ICE_MACHINE_ENVINFO "PrgEnv-intel/8.3.3 intel/2022.2.1 cray-mpich/8.1.21 cray-hdf4-parallel/1/12.2.1 cray-netcdf-hdf5parallel/4.9.0.1" +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR /lustre/scratch5/$user/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /usr/projects/climate/eclare/DATA/Consortium +setenv ICE_MACHINE_BASELINE /lustre/scratch5/$user/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "sbatch " +setenv ICE_MACHINE_ACCT t23_cice +setenv ICE_MACHINE_QUEUE "debug" +setenv ICE_MACHINE_TPNODE 128 # tasks per node +setenv ICE_MACHINE_BLDTHRDS 12 +setenv ICE_MACHINE_QSTAT "squeue --jobs=" diff --git a/icepack b/icepack index 4728746ea..86cae16d1 160000 --- a/icepack +++ b/icepack @@ -1 +1 @@ -Subproject commit 4728746ea2926bf10acc5de354b3eae16d418af5 +Subproject commit 86cae16d1b7c4c4f8a410fccac155374afac777f