Skip to content

Commit

Permalink
Port to Carpenter intel and gnu with intel mpi instead of cray-mpich
Browse files Browse the repository at this point in the history
  • Loading branch information
apcraig committed Mar 21, 2024
1 parent 740f2a6 commit db85357
Show file tree
Hide file tree
Showing 8 changed files with 267 additions and 4 deletions.
22 changes: 21 additions & 1 deletion configuration/scripts/cice.launch.csh
Original file line number Diff line number Diff line change
Expand Up @@ -89,11 +89,31 @@ mpirun -np ${ntasks} -hostfile \$PBS_NODEFILE \${EXTRA_OMPI_SETTINGS} ./cice >&!
EOFR

#=======
else if (${ICE_MACHCOMP} =~ onyx* || ${ICE_MACHCOMP} =~ narwhal* || ${ICE_MACHCOMP} =~ carpenter*) then
else if (${ICE_MACHCOMP} =~ onyx* || ${ICE_MACHCOMP} =~ narwhal*) then
cat >> ${jobfile} << EOFR
aprun -q -n ${ntasks} -N ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE
EOFR

#=======
else if (${ICE_MACHCOMP} =~ carpenter*) then
if (${ICE_COMMDIR} =~ serial*) then
cat >> ${jobfile} << EOFR
./cice >&! \$ICE_RUNLOG_FILE
EOFR
else

if (${ICE_ENVNAME} =~ intelimpi* || ${ICE_ENVNAME} =~ gnuimpi*) then
cat >> ${jobfile} << EOFR
mpiexec -n ${ntasks} -ppn ${taskpernodelimit} ./cice >&! \$ICE_RUNLOG_FILE
EOFR
else
cat >> ${jobfile} << EOFR
mpiexec --cpu-bind depth -n ${ntasks} -ppn ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE
EOFR
endif

endif

#=======
else if (${ICE_MACHCOMP} =~ cori* || ${ICE_MACHCOMP} =~ perlmutter*) then
if (${ICE_COMMDIR} =~ serial*) then
Expand Down
2 changes: 1 addition & 1 deletion configuration/scripts/machines/Macros.carpenter_cray
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#==============================================================================
# Macros file for NAVYDSRC narwhal, cray compiler
# Macros file for ERDC carpenter, cray compiler
#==============================================================================

CPP := ftn -e P
Expand Down
2 changes: 1 addition & 1 deletion configuration/scripts/machines/Macros.carpenter_gnu
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#==============================================================================
# Macros file for NAVYDSRC narwhal, gnu compiler
# Macros file for ERDC carpenter, gnu compiler
#==============================================================================

CPP := ftn -E
Expand Down
69 changes: 69 additions & 0 deletions configuration/scripts/machines/Macros.carpenter_gnuimpi
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
#==============================================================================
# Macros file for NAVYDSRC narwhal, gnu compiler
#==============================================================================

CPP := ftn -E
CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
CFLAGS := -c

FIXEDFLAGS := -ffixed-line-length-132
FREEFLAGS := -ffree-form
FFLAGS := -fconvert=big-endian -fbacktrace -ffree-line-length-none -fallow-argument-mismatch
FFLAGS_NOOPT:= -O0

ifeq ($(ICE_BLDDEBUG), true)
FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow
CFLAGS += -O0
endif

ifeq ($(ICE_COVERAGE), true)
FFLAGS += -O0 -g -fprofile-arcs -ftest-coverage
CFLAGS += -O0 -g -coverage
LDFLAGS += -g -ftest-coverage -fprofile-arcs
endif

ifneq ($(ICE_BLDDEBUG), true)
ifneq ($(ICE_COVERAGE), true)
FFLAGS += -O2
CFLAGS += -O2
endif
endif

#SCC := gcc
#SFC := gfortran
SCC := mpicc
SFC := mpif90
MPICC := mpicc
MPIFC := mpif90

ifeq ($(ICE_COMMDIR), mpi)
FC := $(MPIFC)
CC := $(MPICC)
else
FC := $(SFC)
CC := $(SCC)
endif
LD:= $(FC)

# defined by module
#NETCDF_PATH := $(NETCDF)
#PNETCDF_PATH := $(PNETCDF)
#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default
#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib

#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs

INCLDIR := $(INCLDIR)
INCLDIR += -I$(NETCDF_PATH)/include

LIB_NETCDF := $(NETCDF_PATH)/lib
#LIB_PNETCDF := $(PNETCDF_PATH)/lib
#LIB_MPI := $(IMPILIBDIR)
SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff

ifeq ($(ICE_THREADED), true)
LDFLAGS += -fopenmp
CFLAGS += -fopenmp
FFLAGS += -fopenmp
endif

2 changes: 1 addition & 1 deletion configuration/scripts/machines/Macros.carpenter_intel
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#==============================================================================
# Macros file for NAVYDSRC narwhal, intel compiler
# Macros file for ERDC carpenter, intel compiler
#==============================================================================

CPP := fpp
Expand Down
59 changes: 59 additions & 0 deletions configuration/scripts/machines/Macros.carpenter_intelimpi
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
#==============================================================================
# Macros file for ERDC carpenter, intel compiler, intel mpi
#==============================================================================

CPP := fpp
CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
CFLAGS := -c -O2 -fp-model precise -fcommon

FIXEDFLAGS := -132
FREEFLAGS := -FR
FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback
# -mcmodel medium -shared-intel
FFLAGS_NOOPT:= -O0

ifeq ($(ICE_BLDDEBUG), true)
FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created
# FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays
else
FFLAGS += -O2
endif

#SCC := icx
#SFC := ifort
SCC := mpiicc
SFC := mpiifort
MPICC := mpiicc
MPIFC := mpiifort

ifeq ($(ICE_COMMDIR), mpi)
FC := $(MPIFC)
CC := $(MPICC)
else
FC := $(SFC)
CC := $(SCC)
endif
LD:= $(FC)

# defined by module
#NETCDF_PATH := $(NETCDF)
#PNETCDF_PATH := $(PNETCDF)
#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default
#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib

#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs

INCLDIR := $(INCLDIR)
INCLDIR += -I$(NETCDF_PATH)/include

LIB_NETCDF := $(NETCDF_PATH)/lib
#LIB_PNETCDF := $(PNETCDF_PATH)/lib
#LIB_MPI := $(IMPILIBDIR)
SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff

ifeq ($(ICE_THREADED), true)
LDFLAGS += -qopenmp
CFLAGS += -qopenmp
FFLAGS += -qopenmp
endif

58 changes: 58 additions & 0 deletions configuration/scripts/machines/env.carpenter_gnuimpi
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
#!/bin/csh -f

set inp = "undefined"
if ($#argv == 1) then
set inp = $1
endif

if ("$inp" != "-nomodules") then

source ${MODULESHOME}/init/csh

module unload PrgEnv-cray
module unload PrgEnv-gnu
module unload PrgEnv-intel
module unload PrgEnv-pgi
module load PrgEnv-gnu/8.4.0

module unload gcc
module load gcc/11.2.0

module unload cray-mpich
module unload mpi
module unload openmpi
#module load cray-mpich/8.1.26
#module load openmpi/4.1.6
module load mpi/2021.11

module unload cray-hdf5
module unload cray-hdf5-parallel
module unload cray-netcdf-hdf5parallel
module unload cray-parallel-netcdf
module unload netcdf
module load cray-netcdf/4.9.0.3
module load cray-hdf5/1.12.2.3

setenv NETCDF_PATH ${NETCDF_DIR}
limit coredumpsize unlimited
limit stacksize unlimited
setenv OMP_STACKSIZE 128M
setenv OMP_WAIT_POLICY PASSIVE
setenv FI_CXI_RX_MATCH_MODE hybrid

endif

setenv ICE_MACHINE_MACHNAME carpenter
setenv ICE_MACHINE_MACHINFO "Cray EX4000 AMD 9654 Genoa 2.1GHz, Slingshot Interconnect"
setenv ICE_MACHINE_ENVNAME gnuimpi
setenv ICE_MACHINE_ENVINFO "gnu gcc 11.2.0 20210728, intel mpi 2021.11, netcdf/4.9.0.3"
setenv ICE_MACHINE_MAKE gmake
setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS
setenv ICE_MACHINE_INPUTDATA /p/app/unsupported/RASM/cice_consortium
setenv ICE_MACHINE_BASELINE $WORKDIR/CICE_BASELINE
setenv ICE_MACHINE_SUBMIT "qsub "
setenv ICE_MACHINE_ACCT P00000000
setenv ICE_MACHINE_QUEUE "debug"
setenv ICE_MACHINE_TPNODE 192 # tasks per node
setenv ICE_MACHINE_BLDTHRDS 12
setenv ICE_MACHINE_QSTAT "qstat "
57 changes: 57 additions & 0 deletions configuration/scripts/machines/env.carpenter_intelimpi
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
#!/bin/csh -f
set inp = "undefined"
if ($#argv == 1) then
set inp = $1
endif

if ("$inp" != "-nomodules") then

source ${MODULESHOME}/init/csh

module unload PrgEnv-cray
module unload PrgEnv-gnu
module unload PrgEnv-intel
module unload PrgEnv-pgi
module load PrgEnv-intel/8.4.0

module unload intel
module load intel/2023.0.0

module unload cray-mpich
module unload mpi
module unload openmpi
#module load cray-mpich/8.1.26
module load mpi/2021.11
#module load openmpi/4.1.6

module unload cray-hdf5
module unload cray-hdf5-parallel
module unload cray-netcdf-hdf5parallel
module unload cray-parallel-netcdf
module unload netcdf
module load cray-netcdf/4.9.0.3
module load cray-hdf5/1.12.2.3

setenv NETCDF_PATH ${NETCDF_DIR}
limit coredumpsize unlimited
limit stacksize unlimited
setenv OMP_STACKSIZE 128M
setenv OMP_WAIT_POLICY PASSIVE
setenv FI_CXI_RX_MATCH_MODE hybrid

endif

setenv ICE_MACHINE_MACHNAME carpenter
setenv ICE_MACHINE_MACHINFO "Cray EX4000 AMD 9654 Genoa 2.1GHz, Slingshot Interconnect"
setenv ICE_MACHINE_ENVNAME intelimpi
setenv ICE_MACHINE_ENVINFO "ifort 2021.8.0 20221119, intel mpi 2021.11, netcdf/4.9.0.3"
setenv ICE_MACHINE_MAKE gmake
setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS
setenv ICE_MACHINE_INPUTDATA /p/app/unsupported/RASM/cice_consortium
setenv ICE_MACHINE_BASELINE $WORKDIR/CICE_BASELINE
setenv ICE_MACHINE_SUBMIT "qsub "
setenv ICE_MACHINE_ACCT P00000000
setenv ICE_MACHINE_QUEUE "debug"
setenv ICE_MACHINE_TPNODE 192 # tasks per node
setenv ICE_MACHINE_BLDTHRDS 12
setenv ICE_MACHINE_QSTAT "qstat "

0 comments on commit db85357

Please sign in to comment.