Port to Carpenter (CICE-Consortium#939)

Port CICE to Carpenter for intel, cray, and gnu compilers with mpich. Will add weekly testing on Carpenter, Onyx was recently decommissioned. During development, - It was discovered that the serial compilers and parallel compilers produce different results. This was reported to sys admins. Serial compilation is done with the parallel compilers to ensure bit-for-bit results with serial and mpi comm versions. - It was discovered that the openmpi and intel mpi options are not working properly. This was reported to sys admins. All testing is done with mpich for now.
DeniseWorthen · Apr 13, 2024 · d11c42d · d11c42d
1 parent a5d5011
commit d11c42d
Show file tree

Hide file tree

Showing 8 changed files with 359 additions and 2 deletions.
diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh
@@ -83,7 +83,7 @@ cat >> ${jobfile} << EOFB
 #PBS -l walltime=${batchtime}
 EOFB
 
-else if (${ICE_MACHINE} =~ gaffney* || ${ICE_MACHINE} =~ koehr* || ${ICE_MACHINE} =~ mustang*) then
+else if (${ICE_MACHINE} =~ gaffney* || ${ICE_MACHINE} =~ koehr* || ${ICE_MACHINE} =~ mustang* || ${ICE_MACHINE} =~ carpenter*) then
 cat >> ${jobfile} << EOFB
 #PBS -N ${shortcase}
 #PBS -q ${queue}

diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh
@@ -89,7 +89,7 @@ mpirun -np ${ntasks} -hostfile \$PBS_NODEFILE \${EXTRA_OMPI_SETTINGS} ./cice >&!
 EOFR
 
 #=======
-else if (${ICE_MACHCOMP} =~ onyx* || ${ICE_MACHCOMP} =~ narwhal*) then
+else if (${ICE_MACHCOMP} =~ onyx* || ${ICE_MACHCOMP} =~ narwhal* || ${ICE_MACHCOMP} =~ carpenter*) then
 cat >> ${jobfile} << EOFR
 aprun -q -n ${ntasks} -N ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE
 EOFR

diff --git a/configuration/scripts/machines/Macros.carpenter_cray b/configuration/scripts/machines/Macros.carpenter_cray
@@ -0,0 +1,60 @@
+#==============================================================================
+# Macros file for NAVYDSRC narwhal, cray compiler
+#==============================================================================
+
+CPP        := ftn -e P
+CPPDEFS    := -DFORTRANUNDERSCORE -DNO_R16 ${ICE_CPPDEFS}
+CFLAGS     := -c -O2
+
+FIXEDFLAGS := -132
+FREEFLAGS  := 
+FFLAGS     := -hbyteswapio
+FFLAGS_NOOPT:= -O0
+LDFLAGS    := -hbyteswapio
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -hfp0 -g -Rbcdps -Ktrap=fp
+else
+  FFLAGS     += -O2 -hfp0     # -eo
+endif
+
+SCC   := cc 
+SFC   := ftn
+MPICC := cc
+MPIFC := ftn
+
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+LD:= $(FC)
+
+# defined by module
+#NETCDF_PATH := $(NETCDF)
+#PNETCDF_PATH := $(PNETCDF)
+#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default
+#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib
+
+#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs 
+
+INCLDIR := $(INCLDIR)
+#INCLDIR += -I$(NETCDF_PATH)/include
+
+#LIB_NETCDF := $(NETCDF_PATH)/lib
+#LIB_PNETCDF := $(PNETCDF_PATH)/lib
+#LIB_MPI := $(IMPILIBDIR)
+#SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true) 
+   LDFLAGS += -fopenmp
+   CFLAGS += -fopenmp
+   FFLAGS += -fopenmp
+else
+   LDFLAGS += -hnoomp
+#   CFLAGS += -hnoomp
+   FFLAGS += -hnoomp
+endif
+
diff --git a/configuration/scripts/machines/Macros.carpenter_gnu b/configuration/scripts/machines/Macros.carpenter_gnu
@@ -0,0 +1,69 @@
+#==============================================================================
+# Macros file for NAVYDSRC narwhal, gnu compiler
+#==============================================================================
+
+CPP        := ftn -E
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c
+
+FIXEDFLAGS := -ffixed-line-length-132
+FREEFLAGS  := -ffree-form
+FFLAGS     := -fconvert=big-endian -fbacktrace -ffree-line-length-none -fallow-argument-mismatch
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow
+  CFLAGS     += -O0
+endif
+
+ifeq ($(ICE_COVERAGE), true)
+  FFLAGS   += -O0 -g -fprofile-arcs -ftest-coverage
+  CFLAGS   += -O0 -g -coverage
+  LDFLAGS  += -g -ftest-coverage -fprofile-arcs
+endif
+
+ifneq ($(ICE_BLDDEBUG), true)
+ifneq ($(ICE_COVERAGE), true)
+  FFLAGS   += -O2
+  CFLAGS   += -O2
+endif
+endif
+
+#SCC   := gcc
+#SFC   := gfortran
+SCC   := cc
+SFC   := ftn
+MPICC := cc
+MPIFC := ftn
+
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+LD:= $(FC)
+
+# defined by module
+#NETCDF_PATH := $(NETCDF)
+#PNETCDF_PATH := $(PNETCDF)
+#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default
+#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib
+
+#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs 
+
+INCLDIR := $(INCLDIR)
+INCLDIR += -I$(NETCDF_PATH)/include
+
+LIB_NETCDF := $(NETCDF_PATH)/lib
+#LIB_PNETCDF := $(PNETCDF_PATH)/lib
+#LIB_MPI := $(IMPILIBDIR)
+SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true) 
+   LDFLAGS += -fopenmp
+   CFLAGS += -fopenmp
+   FFLAGS += -fopenmp
+endif
+
diff --git a/configuration/scripts/machines/Macros.carpenter_intel b/configuration/scripts/machines/Macros.carpenter_intel
@@ -0,0 +1,59 @@
+#==============================================================================
+# Macros file for NAVYDSRC narwhal, intel compiler
+#==============================================================================
+
+CPP        := fpp
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c -O2 -fp-model precise -fcommon
+
+FIXEDFLAGS := -132
+FREEFLAGS  := -FR
+FFLAGS     := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback
+# -mcmodel medium -shared-intel
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created
+#  FFLAGS     += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays
+else
+  FFLAGS     += -O2
+endif
+
+#SCC   := icx 
+#SFC   := ifort
+SCC   := cc
+SFC   := ftn
+MPICC := cc
+MPIFC := ftn
+
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+LD:= $(FC)
+
+# defined by module
+#NETCDF_PATH := $(NETCDF)
+#PNETCDF_PATH := $(PNETCDF)
+#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default
+#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib
+
+#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs 
+
+INCLDIR := $(INCLDIR)
+INCLDIR += -I$(NETCDF_PATH)/include
+
+LIB_NETCDF := $(NETCDF_PATH)/lib
+#LIB_PNETCDF := $(PNETCDF_PATH)/lib
+#LIB_MPI := $(IMPILIBDIR)
+SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true) 
+   LDFLAGS += -qopenmp
+   CFLAGS += -qopenmp
+   FFLAGS += -qopenmp
+endif
+
diff --git a/configuration/scripts/machines/env.carpenter_cray b/configuration/scripts/machines/env.carpenter_cray
@@ -0,0 +1,54 @@
+#!/bin/csh -f
+
+set inp = "undefined"
+if ($#argv == 1) then
+  set inp = $1
+endif
+
+if ("$inp" != "-nomodules") then
+
+source ${MODULESHOME}/init/csh
+
+module unload PrgEnv-cray
+module unload PrgEnv-gnu
+module unload PrgEnv-intel
+module unload PrgEnv-pgi
+module load PrgEnv-cray/8.4.0
+
+module unload cce
+module load cce/16.0.0
+
+module unload cray-mpich
+module load cray-mpich/8.1.26
+
+module unload cray-hdf5
+module unload cray-hdf5-parallel
+module unload cray-netcdf-hdf5parallel
+module unload cray-parallel-netcdf
+module unload netcdf
+module load cray-netcdf/4.9.0.3
+module load cray-hdf5/1.12.2.3
+
+setenv NETCDF_PATH ${NETCDF_DIR}
+limit coredumpsize unlimited
+limit stacksize unlimited
+setenv OMP_STACKSIZE 128M
+setenv OMP_WAIT_POLICY PASSIVE
+setenv FI_CXI_RX_MATCH_MODE hybrid
+
+endif
+
+setenv ICE_MACHINE_MACHNAME carpenter
+setenv ICE_MACHINE_MACHINFO "Cray EX4000 AMD 9654 Genoa 2.1GHz, Slingshot Interconnect"
+setenv ICE_MACHINE_ENVNAME cray
+setenv ICE_MACHINE_ENVINFO "Cray Fortran/Clang 16.0.0, cray-mpich/8.1.26, netcdf/4.9.0.3"
+setenv ICE_MACHINE_MAKE gmake
+setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS
+setenv ICE_MACHINE_INPUTDATA /p/app/unsupported/RASM/cice_consortium
+setenv ICE_MACHINE_BASELINE $WORKDIR/CICE_BASELINE
+setenv ICE_MACHINE_SUBMIT "qsub "
+setenv ICE_MACHINE_ACCT P00000000
+setenv ICE_MACHINE_QUEUE "debug"
+setenv ICE_MACHINE_TPNODE 192    # tasks per node
+setenv ICE_MACHINE_BLDTHRDS 12
+setenv ICE_MACHINE_QSTAT "qstat "
diff --git a/configuration/scripts/machines/env.carpenter_gnu b/configuration/scripts/machines/env.carpenter_gnu
@@ -0,0 +1,58 @@
+#!/bin/csh -f
+
+set inp = "undefined"
+if ($#argv == 1) then
+  set inp = $1
+endif
+
+if ("$inp" != "-nomodules") then
+
+source ${MODULESHOME}/init/csh
+
+module unload PrgEnv-cray
+module unload PrgEnv-gnu
+module unload PrgEnv-intel
+module unload PrgEnv-pgi
+module load PrgEnv-gnu/8.4.0
+
+module unload gcc
+module load gcc/12.2.0
+
+module unload cray-mpich
+module unload mpi
+module unload openmpi
+module load cray-mpich/8.1.26
+#module load openmpi/4.1.6
+#module load mpi/2021.11
+
+module unload cray-hdf5
+module unload cray-hdf5-parallel
+module unload cray-netcdf-hdf5parallel
+module unload cray-parallel-netcdf
+module unload netcdf
+module load cray-netcdf/4.9.0.3
+module load cray-hdf5/1.12.2.3
+
+setenv NETCDF_PATH ${NETCDF_DIR}
+limit coredumpsize unlimited
+limit stacksize unlimited
+setenv OMP_STACKSIZE 128M
+setenv OMP_WAIT_POLICY PASSIVE
+setenv FI_CXI_RX_MATCH_MODE hybrid
+
+endif
+
+setenv ICE_MACHINE_MACHNAME carpenter
+setenv ICE_MACHINE_MACHINFO "Cray EX4000 AMD 9654 Genoa 2.1GHz, Slingshot Interconnect"
+setenv ICE_MACHINE_ENVNAME gnu
+setenv ICE_MACHINE_ENVINFO "gnu gcc 12.2.0 20220819, mpich/8.1.26, netcdf/4.9.0.3"
+setenv ICE_MACHINE_MAKE gmake
+setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS
+setenv ICE_MACHINE_INPUTDATA /p/app/unsupported/RASM/cice_consortium
+setenv ICE_MACHINE_BASELINE $WORKDIR/CICE_BASELINE
+setenv ICE_MACHINE_SUBMIT "qsub "
+setenv ICE_MACHINE_ACCT P00000000
+setenv ICE_MACHINE_QUEUE "debug"
+setenv ICE_MACHINE_TPNODE 192    # tasks per node
+setenv ICE_MACHINE_BLDTHRDS 12
+setenv ICE_MACHINE_QSTAT "qstat "
diff --git a/configuration/scripts/machines/env.carpenter_intel b/configuration/scripts/machines/env.carpenter_intel
@@ -0,0 +1,57 @@
+#!/bin/csh -f
+set inp = "undefined"
+if ($#argv == 1) then
+  set inp = $1
+endif
+
+if ("$inp" != "-nomodules") then
+
+source ${MODULESHOME}/init/csh
+
+module unload PrgEnv-cray
+module unload PrgEnv-gnu
+module unload PrgEnv-intel
+module unload PrgEnv-pgi
+module load PrgEnv-intel/8.4.0
+
+module unload intel
+module load intel/2023.0.0
+
+module unload cray-mpich
+module unload mpi
+module unload openmpi
+module load cray-mpich/8.1.26
+#module load mpi/2021.11
+#module load openmpi/4.1.6
+
+module unload cray-hdf5
+module unload cray-hdf5-parallel
+module unload cray-netcdf-hdf5parallel
+module unload cray-parallel-netcdf
+module unload netcdf
+module load cray-netcdf/4.9.0.3
+module load cray-hdf5/1.12.2.3
+
+setenv NETCDF_PATH ${NETCDF_DIR}
+limit coredumpsize unlimited
+limit stacksize unlimited
+setenv OMP_STACKSIZE 128M
+setenv OMP_WAIT_POLICY PASSIVE
+setenv FI_CXI_RX_MATCH_MODE hybrid
+
+endif
+
+setenv ICE_MACHINE_MACHNAME carpenter
+setenv ICE_MACHINE_MACHINFO "Cray EX4000 AMD 9654 Genoa 2.1GHz, Slingshot Interconnect"
+setenv ICE_MACHINE_ENVNAME intel
+setenv ICE_MACHINE_ENVINFO "ifort 2021.8.0 20221119, cray-mpich/8.1.26, netcdf/4.9.0.3"
+setenv ICE_MACHINE_MAKE gmake
+setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS
+setenv ICE_MACHINE_INPUTDATA /p/app/unsupported/RASM/cice_consortium
+setenv ICE_MACHINE_BASELINE $WORKDIR/CICE_BASELINE
+setenv ICE_MACHINE_SUBMIT "qsub "
+setenv ICE_MACHINE_ACCT P00000000
+setenv ICE_MACHINE_QUEUE "debug"
+setenv ICE_MACHINE_TPNODE 192    # tasks per node
+setenv ICE_MACHINE_BLDTHRDS 12
+setenv ICE_MACHINE_QSTAT "qstat "