From db8535758257c4f54c79e00209e8d501631577b0 Mon Sep 17 00:00:00 2001 From: apcraig Date: Wed, 20 Mar 2024 19:52:29 -0500 Subject: [PATCH] Port to Carpenter intel and gnu with intel mpi instead of cray-mpich --- configuration/scripts/cice.launch.csh | 22 +++++- .../scripts/machines/Macros.carpenter_cray | 2 +- .../scripts/machines/Macros.carpenter_gnu | 2 +- .../scripts/machines/Macros.carpenter_gnuimpi | 69 +++++++++++++++++++ .../scripts/machines/Macros.carpenter_intel | 2 +- .../machines/Macros.carpenter_intelimpi | 59 ++++++++++++++++ .../scripts/machines/env.carpenter_gnuimpi | 58 ++++++++++++++++ .../scripts/machines/env.carpenter_intelimpi | 57 +++++++++++++++ 8 files changed, 267 insertions(+), 4 deletions(-) create mode 100644 configuration/scripts/machines/Macros.carpenter_gnuimpi create mode 100644 configuration/scripts/machines/Macros.carpenter_intelimpi create mode 100644 configuration/scripts/machines/env.carpenter_gnuimpi create mode 100644 configuration/scripts/machines/env.carpenter_intelimpi diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh index 85d647d76..51c8f044f 100755 --- a/configuration/scripts/cice.launch.csh +++ b/configuration/scripts/cice.launch.csh @@ -89,11 +89,31 @@ mpirun -np ${ntasks} -hostfile \$PBS_NODEFILE \${EXTRA_OMPI_SETTINGS} ./cice >&! EOFR #======= -else if (${ICE_MACHCOMP} =~ onyx* || ${ICE_MACHCOMP} =~ narwhal* || ${ICE_MACHCOMP} =~ carpenter*) then +else if (${ICE_MACHCOMP} =~ onyx* || ${ICE_MACHCOMP} =~ narwhal*) then cat >> ${jobfile} << EOFR aprun -q -n ${ntasks} -N ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE EOFR +#======= +else if (${ICE_MACHCOMP} =~ carpenter*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else + +if (${ICE_ENVNAME} =~ intelimpi* || ${ICE_ENVNAME} =~ gnuimpi*) then +cat >> ${jobfile} << EOFR +mpiexec -n ${ntasks} -ppn ${taskpernodelimit} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +else +cat >> ${jobfile} << EOFR +mpiexec --cpu-bind depth -n ${ntasks} -ppn ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +endif + +endif + #======= else if (${ICE_MACHCOMP} =~ cori* || ${ICE_MACHCOMP} =~ perlmutter*) then if (${ICE_COMMDIR} =~ serial*) then diff --git a/configuration/scripts/machines/Macros.carpenter_cray b/configuration/scripts/machines/Macros.carpenter_cray index 8496f7a9b..1c8fb50ca 100644 --- a/configuration/scripts/machines/Macros.carpenter_cray +++ b/configuration/scripts/machines/Macros.carpenter_cray @@ -1,5 +1,5 @@ #============================================================================== -# Macros file for NAVYDSRC narwhal, cray compiler +# Macros file for ERDC carpenter, cray compiler #============================================================================== CPP := ftn -e P diff --git a/configuration/scripts/machines/Macros.carpenter_gnu b/configuration/scripts/machines/Macros.carpenter_gnu index 50dcdc08d..61efa80c2 100644 --- a/configuration/scripts/machines/Macros.carpenter_gnu +++ b/configuration/scripts/machines/Macros.carpenter_gnu @@ -1,5 +1,5 @@ #============================================================================== -# Macros file for NAVYDSRC narwhal, gnu compiler +# Macros file for ERDC carpenter, gnu compiler #============================================================================== CPP := ftn -E diff --git a/configuration/scripts/machines/Macros.carpenter_gnuimpi b/configuration/scripts/machines/Macros.carpenter_gnuimpi new file mode 100644 index 000000000..ef0c5e96a --- /dev/null +++ b/configuration/scripts/machines/Macros.carpenter_gnuimpi @@ -0,0 +1,69 @@ +#============================================================================== +# Macros file for NAVYDSRC narwhal, gnu compiler +#============================================================================== + +CPP := ftn -E +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c + +FIXEDFLAGS := -ffixed-line-length-132 +FREEFLAGS := -ffree-form +FFLAGS := -fconvert=big-endian -fbacktrace -ffree-line-length-none -fallow-argument-mismatch +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow + CFLAGS += -O0 +endif + +ifeq ($(ICE_COVERAGE), true) + FFLAGS += -O0 -g -fprofile-arcs -ftest-coverage + CFLAGS += -O0 -g -coverage + LDFLAGS += -g -ftest-coverage -fprofile-arcs +endif + +ifneq ($(ICE_BLDDEBUG), true) +ifneq ($(ICE_COVERAGE), true) + FFLAGS += -O2 + CFLAGS += -O2 +endif +endif + +#SCC := gcc +#SFC := gfortran +SCC := mpicc +SFC := mpif90 +MPICC := mpicc +MPIFC := mpif90 + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +# defined by module +#NETCDF_PATH := $(NETCDF) +#PNETCDF_PATH := $(PNETCDF) +#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default +#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib + +#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs + +INCLDIR := $(INCLDIR) +INCLDIR += -I$(NETCDF_PATH)/include + +LIB_NETCDF := $(NETCDF_PATH)/lib +#LIB_PNETCDF := $(PNETCDF_PATH)/lib +#LIB_MPI := $(IMPILIBDIR) +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif + diff --git a/configuration/scripts/machines/Macros.carpenter_intel b/configuration/scripts/machines/Macros.carpenter_intel index 53d63736d..d53f959e4 100644 --- a/configuration/scripts/machines/Macros.carpenter_intel +++ b/configuration/scripts/machines/Macros.carpenter_intel @@ -1,5 +1,5 @@ #============================================================================== -# Macros file for NAVYDSRC narwhal, intel compiler +# Macros file for ERDC carpenter, intel compiler #============================================================================== CPP := fpp diff --git a/configuration/scripts/machines/Macros.carpenter_intelimpi b/configuration/scripts/machines/Macros.carpenter_intelimpi new file mode 100644 index 000000000..0c1aa5812 --- /dev/null +++ b/configuration/scripts/machines/Macros.carpenter_intelimpi @@ -0,0 +1,59 @@ +#============================================================================== +# Macros file for ERDC carpenter, intel compiler, intel mpi +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -fp-model precise -fcommon + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback +# -mcmodel medium -shared-intel +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created +# FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays +else + FFLAGS += -O2 +endif + +#SCC := icx +#SFC := ifort +SCC := mpiicc +SFC := mpiifort +MPICC := mpiicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +# defined by module +#NETCDF_PATH := $(NETCDF) +#PNETCDF_PATH := $(PNETCDF) +#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default +#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib + +#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs + +INCLDIR := $(INCLDIR) +INCLDIR += -I$(NETCDF_PATH)/include + +LIB_NETCDF := $(NETCDF_PATH)/lib +#LIB_PNETCDF := $(PNETCDF_PATH)/lib +#LIB_MPI := $(IMPILIBDIR) +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + diff --git a/configuration/scripts/machines/env.carpenter_gnuimpi b/configuration/scripts/machines/env.carpenter_gnuimpi new file mode 100644 index 000000000..f21bf97a5 --- /dev/null +++ b/configuration/scripts/machines/env.carpenter_gnuimpi @@ -0,0 +1,58 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source ${MODULESHOME}/init/csh + +module unload PrgEnv-cray +module unload PrgEnv-gnu +module unload PrgEnv-intel +module unload PrgEnv-pgi +module load PrgEnv-gnu/8.4.0 + +module unload gcc +module load gcc/11.2.0 + +module unload cray-mpich +module unload mpi +module unload openmpi +#module load cray-mpich/8.1.26 +#module load openmpi/4.1.6 +module load mpi/2021.11 + +module unload cray-hdf5 +module unload cray-hdf5-parallel +module unload cray-netcdf-hdf5parallel +module unload cray-parallel-netcdf +module unload netcdf +module load cray-netcdf/4.9.0.3 +module load cray-hdf5/1.12.2.3 + +setenv NETCDF_PATH ${NETCDF_DIR} +limit coredumpsize unlimited +limit stacksize unlimited +setenv OMP_STACKSIZE 128M +setenv OMP_WAIT_POLICY PASSIVE +setenv FI_CXI_RX_MATCH_MODE hybrid + +endif + +setenv ICE_MACHINE_MACHNAME carpenter +setenv ICE_MACHINE_MACHINFO "Cray EX4000 AMD 9654 Genoa 2.1GHz, Slingshot Interconnect" +setenv ICE_MACHINE_ENVNAME gnuimpi +setenv ICE_MACHINE_ENVINFO "gnu gcc 11.2.0 20210728, intel mpi 2021.11, netcdf/4.9.0.3" +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /p/app/unsupported/RASM/cice_consortium +setenv ICE_MACHINE_BASELINE $WORKDIR/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "qsub " +setenv ICE_MACHINE_ACCT P00000000 +setenv ICE_MACHINE_QUEUE "debug" +setenv ICE_MACHINE_TPNODE 192 # tasks per node +setenv ICE_MACHINE_BLDTHRDS 12 +setenv ICE_MACHINE_QSTAT "qstat " diff --git a/configuration/scripts/machines/env.carpenter_intelimpi b/configuration/scripts/machines/env.carpenter_intelimpi new file mode 100644 index 000000000..25385b09a --- /dev/null +++ b/configuration/scripts/machines/env.carpenter_intelimpi @@ -0,0 +1,57 @@ +#!/bin/csh -f +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source ${MODULESHOME}/init/csh + +module unload PrgEnv-cray +module unload PrgEnv-gnu +module unload PrgEnv-intel +module unload PrgEnv-pgi +module load PrgEnv-intel/8.4.0 + +module unload intel +module load intel/2023.0.0 + +module unload cray-mpich +module unload mpi +module unload openmpi +#module load cray-mpich/8.1.26 +module load mpi/2021.11 +#module load openmpi/4.1.6 + +module unload cray-hdf5 +module unload cray-hdf5-parallel +module unload cray-netcdf-hdf5parallel +module unload cray-parallel-netcdf +module unload netcdf +module load cray-netcdf/4.9.0.3 +module load cray-hdf5/1.12.2.3 + +setenv NETCDF_PATH ${NETCDF_DIR} +limit coredumpsize unlimited +limit stacksize unlimited +setenv OMP_STACKSIZE 128M +setenv OMP_WAIT_POLICY PASSIVE +setenv FI_CXI_RX_MATCH_MODE hybrid + +endif + +setenv ICE_MACHINE_MACHNAME carpenter +setenv ICE_MACHINE_MACHINFO "Cray EX4000 AMD 9654 Genoa 2.1GHz, Slingshot Interconnect" +setenv ICE_MACHINE_ENVNAME intelimpi +setenv ICE_MACHINE_ENVINFO "ifort 2021.8.0 20221119, intel mpi 2021.11, netcdf/4.9.0.3" +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /p/app/unsupported/RASM/cice_consortium +setenv ICE_MACHINE_BASELINE $WORKDIR/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "qsub " +setenv ICE_MACHINE_ACCT P00000000 +setenv ICE_MACHINE_QUEUE "debug" +setenv ICE_MACHINE_TPNODE 192 # tasks per node +setenv ICE_MACHINE_BLDTHRDS 12 +setenv ICE_MACHINE_QSTAT "qstat "