Port to Carpenter intel and gnu with intel mpi instead of cray-mpich (C…

…ICE-Consortium#942)

configuration/scripts/cice.launch.csh

@@ -89,11 +89,31 @@ mpirun -np ${ntasks} -hostfile \$PBS_NODEFILE \${EXTRA_OMPI_SETTINGS} ./cice >&!
 EOFR
 
 #=======
-else if (${ICE_MACHCOMP} =~ onyx* || ${ICE_MACHCOMP} =~ narwhal* || ${ICE_MACHCOMP} =~ carpenter*) then
+else if (${ICE_MACHCOMP} =~ onyx* || ${ICE_MACHCOMP} =~ narwhal*) then
 cat >> ${jobfile} << EOFR
 aprun -q -n ${ntasks} -N ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE
 EOFR
 
+#=======
+else if (${ICE_MACHCOMP} =~ carpenter*) then 
+if (${ICE_COMMDIR} =~ serial*) then
+cat >> ${jobfile} << EOFR
+./cice >&! \$ICE_RUNLOG_FILE
+EOFR
+else
+
+if (${ICE_ENVNAME} =~ intelimpi* || ${ICE_ENVNAME} =~ gnuimpi*) then
+cat >> ${jobfile} << EOFR
+mpiexec -n ${ntasks} -ppn ${taskpernodelimit} ./cice >&! \$ICE_RUNLOG_FILE
+EOFR
+else
+cat >> ${jobfile} << EOFR
+mpiexec --cpu-bind depth -n ${ntasks} -ppn ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE
+EOFR
+endif
+
+endif
+
 #=======
 else if (${ICE_MACHCOMP} =~ cori* || ${ICE_MACHCOMP} =~ perlmutter*) then
 if (${ICE_COMMDIR} =~ serial*) then

configuration/scripts/machines/Macros.carpenter_cray

@@ -1,5 +1,5 @@
 #==============================================================================
-# Macros file for NAVYDSRC narwhal, cray compiler
+# Macros file for ERDC carpenter, cray compiler
 #==============================================================================
 
 CPP        := ftn -e P

configuration/scripts/machines/Macros.carpenter_gnu

@@ -1,5 +1,5 @@
 #==============================================================================
-# Macros file for NAVYDSRC narwhal, gnu compiler
+# Macros file for ERDC carpenter, gnu compiler
 #==============================================================================
 
 CPP        := ftn -E

configuration/scripts/machines/Macros.carpenter_gnuimpi

@@ -0,0 +1,69 @@
+#==============================================================================
+# Macros file for NAVYDSRC narwhal, gnu compiler
+#==============================================================================
+
+CPP        := ftn -E
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c
+
+FIXEDFLAGS := -ffixed-line-length-132
+FREEFLAGS  := -ffree-form
+FFLAGS     := -fconvert=big-endian -fbacktrace -ffree-line-length-none -fallow-argument-mismatch
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow
+  CFLAGS     += -O0
+endif
+
+ifeq ($(ICE_COVERAGE), true)
+  FFLAGS   += -O0 -g -fprofile-arcs -ftest-coverage
+  CFLAGS   += -O0 -g -coverage
+  LDFLAGS  += -g -ftest-coverage -fprofile-arcs
+endif
+
+ifneq ($(ICE_BLDDEBUG), true)
+ifneq ($(ICE_COVERAGE), true)
+  FFLAGS   += -O2
+  CFLAGS   += -O2
+endif
+endif
+
+#SCC   := gcc
+#SFC   := gfortran
+SCC   := mpicc
+SFC   := mpif90
+MPICC := mpicc
+MPIFC := mpif90
+
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+LD:= $(FC)
+
+# defined by module
+#NETCDF_PATH := $(NETCDF)
+#PNETCDF_PATH := $(PNETCDF)
+#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default
+#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib
+
+#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs 
+
+INCLDIR := $(INCLDIR)
+INCLDIR += -I$(NETCDF_PATH)/include
+
+LIB_NETCDF := $(NETCDF_PATH)/lib
+#LIB_PNETCDF := $(PNETCDF_PATH)/lib
+#LIB_MPI := $(IMPILIBDIR)
+SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true) 
+   LDFLAGS += -fopenmp
+   CFLAGS += -fopenmp
+   FFLAGS += -fopenmp
+endif
+

configuration/scripts/machines/Macros.carpenter_intel

@@ -1,5 +1,5 @@
 #==============================================================================
-# Macros file for NAVYDSRC narwhal, intel compiler
+# Macros file for ERDC carpenter, intel compiler
 #==============================================================================
 
 CPP        := fpp

configuration/scripts/machines/Macros.carpenter_intelimpi

@@ -0,0 +1,59 @@
+#==============================================================================
+# Macros file for ERDC carpenter, intel compiler, intel mpi
+#==============================================================================
+
+CPP        := fpp
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c -O2 -fp-model precise -fcommon
+
+FIXEDFLAGS := -132
+FREEFLAGS  := -FR
+FFLAGS     := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback
+# -mcmodel medium -shared-intel
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created
+#  FFLAGS     += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays
+else
+  FFLAGS     += -O2
+endif
+
+#SCC   := icx 
+#SFC   := ifort
+SCC   := mpiicc
+SFC   := mpiifort
+MPICC := mpiicc
+MPIFC := mpiifort
+
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+LD:= $(FC)
+
+# defined by module
+#NETCDF_PATH := $(NETCDF)
+#PNETCDF_PATH := $(PNETCDF)
+#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default
+#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib
+
+#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs 
+
+INCLDIR := $(INCLDIR)
+INCLDIR += -I$(NETCDF_PATH)/include
+
+LIB_NETCDF := $(NETCDF_PATH)/lib
+#LIB_PNETCDF := $(PNETCDF_PATH)/lib
+#LIB_MPI := $(IMPILIBDIR)
+SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true) 
+   LDFLAGS += -qopenmp
+   CFLAGS += -qopenmp
+   FFLAGS += -qopenmp
+endif
+

configuration/scripts/machines/env.carpenter_gnuimpi

@@ -0,0 +1,58 @@
+#!/bin/csh -f
+
+set inp = "undefined"
+if ($#argv == 1) then
+  set inp = $1
+endif
+
+if ("$inp" != "-nomodules") then
+
+source ${MODULESHOME}/init/csh
+
+module unload PrgEnv-cray
+module unload PrgEnv-gnu
+module unload PrgEnv-intel
+module unload PrgEnv-pgi
+module load PrgEnv-gnu/8.4.0
+
+module unload gcc
+module load gcc/11.2.0
+
+module unload cray-mpich
+module unload mpi
+module unload openmpi
+#module load cray-mpich/8.1.26
+#module load openmpi/4.1.6
+module load mpi/2021.11
+
+module unload cray-hdf5
+module unload cray-hdf5-parallel
+module unload cray-netcdf-hdf5parallel
+module unload cray-parallel-netcdf
+module unload netcdf
+module load cray-netcdf/4.9.0.3
+module load cray-hdf5/1.12.2.3
+
+setenv NETCDF_PATH ${NETCDF_DIR}
+limit coredumpsize unlimited
+limit stacksize unlimited
+setenv OMP_STACKSIZE 128M
+setenv OMP_WAIT_POLICY PASSIVE
+setenv FI_CXI_RX_MATCH_MODE hybrid
+
+endif
+
+setenv ICE_MACHINE_MACHNAME carpenter
+setenv ICE_MACHINE_MACHINFO "Cray EX4000 AMD 9654 Genoa 2.1GHz, Slingshot Interconnect"
+setenv ICE_MACHINE_ENVNAME gnuimpi
+setenv ICE_MACHINE_ENVINFO "gnu gcc 11.2.0 20210728, intel mpi 2021.11, netcdf/4.9.0.3"
+setenv ICE_MACHINE_MAKE gmake
+setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS
+setenv ICE_MACHINE_INPUTDATA /p/app/unsupported/RASM/cice_consortium
+setenv ICE_MACHINE_BASELINE $WORKDIR/CICE_BASELINE
+setenv ICE_MACHINE_SUBMIT "qsub "
+setenv ICE_MACHINE_ACCT P00000000
+setenv ICE_MACHINE_QUEUE "debug"
+setenv ICE_MACHINE_TPNODE 192    # tasks per node
+setenv ICE_MACHINE_BLDTHRDS 12
+setenv ICE_MACHINE_QSTAT "qstat "

configuration/scripts/machines/env.carpenter_intelimpi

@@ -0,0 +1,57 @@
+#!/bin/csh -f
+set inp = "undefined"
+if ($#argv == 1) then
+  set inp = $1
+endif
+
+if ("$inp" != "-nomodules") then
+
+source ${MODULESHOME}/init/csh
+
+module unload PrgEnv-cray
+module unload PrgEnv-gnu
+module unload PrgEnv-intel
+module unload PrgEnv-pgi
+module load PrgEnv-intel/8.4.0
+
+module unload intel
+module load intel/2023.0.0
+
+module unload cray-mpich
+module unload mpi
+module unload openmpi
+#module load cray-mpich/8.1.26
+module load mpi/2021.11
+#module load openmpi/4.1.6
+
+module unload cray-hdf5
+module unload cray-hdf5-parallel
+module unload cray-netcdf-hdf5parallel
+module unload cray-parallel-netcdf
+module unload netcdf
+module load cray-netcdf/4.9.0.3
+module load cray-hdf5/1.12.2.3
+
+setenv NETCDF_PATH ${NETCDF_DIR}
+limit coredumpsize unlimited
+limit stacksize unlimited
+setenv OMP_STACKSIZE 128M
+setenv OMP_WAIT_POLICY PASSIVE
+setenv FI_CXI_RX_MATCH_MODE hybrid
+
+endif
+
+setenv ICE_MACHINE_MACHNAME carpenter
+setenv ICE_MACHINE_MACHINFO "Cray EX4000 AMD 9654 Genoa 2.1GHz, Slingshot Interconnect"
+setenv ICE_MACHINE_ENVNAME intelimpi
+setenv ICE_MACHINE_ENVINFO "ifort 2021.8.0 20221119, intel mpi 2021.11, netcdf/4.9.0.3"
+setenv ICE_MACHINE_MAKE gmake
+setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS
+setenv ICE_MACHINE_INPUTDATA /p/app/unsupported/RASM/cice_consortium
+setenv ICE_MACHINE_BASELINE $WORKDIR/CICE_BASELINE
+setenv ICE_MACHINE_SUBMIT "qsub "
+setenv ICE_MACHINE_ACCT P00000000
+setenv ICE_MACHINE_QUEUE "debug"
+setenv ICE_MACHINE_TPNODE 192    # tasks per node
+setenv ICE_MACHINE_BLDTHRDS 12
+setenv ICE_MACHINE_QSTAT "qstat "