ESCOMP · mnlevy1981 · Oct 7, 2024 · Sep 5, 2024 · Sep 6, 2024 · Sep 6, 2024
diff --git a/standalone/build/.gitignore b/standalone/build/.gitignore
@@ -2,3 +2,5 @@ intel/
 intel-*/
 gnu/
 gnu-*/
+nvhpc/
+nvhpc-*/
diff --git a/standalone/build/build_examples.sh b/standalone/build/build_examples.sh
@@ -18,6 +18,7 @@ FMS_ROOT=${CESM_ROOT}/libraries/FMS
 # Default compiler
 COMPILER="intel"
 MACHINE="ncar"
+DEBUG=0 # Set to False (REPRO Mode)!
 
 # Parse command line arguments
 while [[ "$#" -gt 0 ]]; do
@@ -28,6 +29,8 @@ while [[ "$#" -gt 0 ]]; do
         --machine) 
             MACHINE="$2"
             shift ;;
+        --debug)
+            DEBUG=1 ;;
         *) 
             echo "Unknown parameter passed: $1"
             echo "Usage: $0 [--compiler <compiler>] [--machine <machine>]"
@@ -58,38 +61,63 @@ case $MACHINE in
         JOBS=4
         ;;
     "ncar")
-        JOBS=36
+        JOBS=32
         ;;
     *)
         echo "Invalid machine type for make -j option: $MACHINE"
         exit 1
         ;;
 esac
 
-if [ -e $1 ]; then
-  BLD_ROOT=${COMPILER}
+if [ "${DEBUG}" == 1 ]; then
+  BLD_ROOT=${COMPILER}-debug
 else
-  BLD_ROOT=$1
+  BLD_ROOT=${COMPILER}
+fi
+
+
+# Load modules for NCAR
+if [ "$MACHINE" == "ncar" ]; then
+  HOST=`hostname`
+  # Load modules if on derecho
+  if [ ! "${HOST:0:5}" == "crhtc" ] && [ ! "${HOST:0:6}" == "casper" ]; then
+    module --force purge
+    . /glade/u/apps/derecho/23.09/spack/opt/spack/lmod/8.7.24/gcc/7.5.0/c645/lmod/lmod/init/sh
+    module load cesmdev/1.0 ncarenv/23.09
+    case $COMPILER in
+      "intel" )
+        module load craype intel/2023.2.1 mkl ncarcompilers/1.0.0 cmake cray-mpich/8.1.27 netcdf-mpi/4.9.2 parallel-netcdf/1.12.3 parallelio/2.6.2 esmf/8.6.0
+        ;;
+      "gnu" )
+        module load craype gcc/12.2.0 cray-libsci/23.02.1.1 ncarcompilers/1.0.0 cmake cray-mpich/8.1.27 netcdf-mpi/4.9.2 parallel-netcdf/1.12.3 parallelio/2.6.2-debug esmf/8.6.0-debug
+        ;;
+      "nvhpc" )
+        module load craype nvhpc/23.7 ncarcompilers/1.0.0 cmake cray-mpich/8.1.27 netcdf-mpi/4.9.2 parallel-netcdf/1.12.3 parallelio/2.6.2 esmf/8.6.0
+        ;;
+      *)
+        echo "Not loading any special modules for ${COMPILER}"
+        ;;
+    esac
+  fi
 fi
 
 # 1) Build FMS
 cd ${INTERFACE_ROOT}/standalone/build
 mkdir -p ${BLD_ROOT}/FMS
 cd ${BLD_ROOT}/FMS
 ${MKMF_ROOT}/list_paths ${FMS_ROOT}/src
-# We need shr_const_mod.F90 and shr_kind_mod.F90 from ${SHR_ROOT}/src
-# to build FMS
+# We need shr_const_mod.F90 and shr_kind_mod.F90 from ${SHR_ROOT}/src to build FMS
 echo "${SHR_ROOT}/src/shr_kind_mod.F90" >> path_names
 echo "${SHR_ROOT}/src/shr_const_mod.F90" >> path_names
 ${MKMF_ROOT}/mkmf -t ${TEMPLATE} -p libfms.a -c "-Duse_libMPI -Duse_netCDF -DSPMD" path_names
-make -j${JOBS} NETCDF=3 REPRO=1 libfms.a
+make -j${JOBS} DEBUG=${DEBUG} libfms.a
 
 # 2) Build MOM6
 cd ${INTERFACE_ROOT}/standalone/build
 mkdir -p ${BLD_ROOT}/MOM6
 cd ${BLD_ROOT}/MOM6
 ${MKMF_ROOT}/list_paths -l ${MOM_ROOT}/{config_src/infra/FMS2,config_src/memory/dynamic_symmetric,config_src/drivers/solo_driver,../externals/MARBL/src,config_src/external,src/{*,*/*}}/
 ${MKMF_ROOT}/mkmf -t ${TEMPLATE} -o '-I../FMS' -p MOM6 -l '-L../FMS -lfms' -c '-Duse_libMPI -Duse_netCDF -DSPMD' path_names
-make -j${JOBS}  NETCDF=3 REPRO=1 MOM6
+make -j${JOBS} DEBUG=${DEBUG} MOM6
 
-echo "Finished build at `date`"
+echo "Finished build at `date`"
diff --git a/standalone/templates/homebrew-gnu.mk b/standalone/templates/homebrew-gnu.mk
@@ -12,68 +12,36 @@ LD = mpif90 $(MAIN_PROGRAM)
 # flags #
 #########
 DEBUG =
-REPRO =
-VERBOSE =
-OPENMP =
 
 MAKEFLAGS += --jobs=2
 
 FPPFLAGS :=
 
 FFLAGS := -fcray-pointer -fdefault-double-8 -fdefault-real-8 -Waliasing -ffree-line-length-none -fno-range-check
-FFLAGS += -I$(shell nf-config --includedir)
-FFLAGS_OPT = -O3
 FFLAGS_REPRO = -O2 -fbounds-check
 FFLAGS_DEBUG = -O0 -g -W -fbounds-check -fbacktrace -ffpe-trap=invalid,zero,overflow
-FFLAGS_OPENMP = -fopenmp
-FFLAGS_VERBOSE =
 
 CFLAGS := -D__IFC
-CFLAGS += -I$(shell nc-config --includedir)
-CFLAGS_OPT = -O2
-CFLAGS_OPENMP = -fopenmp
+CFLAGS_REPRO= -O2
 CFLAGS_DEBUG = -O0 -g
 
-# Optional Testing compile flags.  Mutually exclusive from DEBUG, REPRO, and OPT
-# *_TEST will match the production if no new option(s) is(are) to be tested.
-FFLAGS_TEST = -O2
-CFLAGS_TEST = -O2
-
 LDFLAGS :=
-LDFLAGS_OPENMP := -fopenmp
-LDFLAGS_VERBOSE :=
 
-ifneq ($(REPRO),)
-CFLAGS += $(CFLAGS_REPRO)
-FFLAGS += $(FFLAGS_REPRO)
-else ifneq ($(DEBUG),)
+ifeq ($(DEBUG),1)
 CFLAGS += $(CFLAGS_DEBUG)
 FFLAGS += $(FFLAGS_DEBUG)
-else ifneq ($(TEST),)
-CFLAGS += $(CFLAGS_TEST)
-FFLAGS += $(FFLAGS_TEST)
 else
-CFLAGS += $(CFLAGS_OPT)
-FFLAGS += $(FFLAGS_OPT)
-endif
-
-ifneq ($(OPENMP),)
-CFLAGS += $(CFLAGS_OPENMP)
-FFLAGS += $(FFLAGS_OPENMP)
-LDFLAGS += $(LDFLAGS_OPENMP)
+CFLAGS += $(CFLAGS_REPRO)
+FFLAGS += $(FFLAGS_REPRO)
 endif
 
-ifneq ($(VERBOSE),)
-CFLAGS += $(CFLAGS_VERBOSE)
-FFLAGS += $(FFLAGS_VERBOSE)
-LDFLAGS += $(LDFLAGS_VERBOSE)
-endif
+# NetCDF Things 
+FFLAGS += -I$(shell nf-config --includedir)
+CFLAGS += -I$(shell nc-config --includedir)
 
-ifeq ($(NETCDF),3)
-  # add the use_LARGEFILE cppdef
-  ifneq ($(findstring -Duse_netCDF,$(CPPDEFS)),)
-    CPPDEFS += -Duse_LARGEFILE
-  endif
+# add the use_LARGEFILE cppdef
+ifneq ($(findstring -Duse_netCDF,$(CPPDEFS)),)
+  CPPDEFS += -Duse_LARGEFILE
 endif
 
 LIBS := $(shell nc-config --libs) $(shell nf-config --flibs)

diff --git a/standalone/templates/ncar-gnu.mk b/standalone/templates/ncar-gnu.mk
@@ -1,6 +1,5 @@
 # template for the GNU fortran compiler
-# typical use with mkmf
-# mkmf -t linux-gnu.mk -c"-Duse_libMPI -Duse_netCDF" path_names /usr/local/include
+
 ############
 # commands #
 ############
@@ -12,73 +11,43 @@ LD = mpif90 $(MAIN_PROGRAM)
 #########
 # flags #
 #########
-DEBUG =
-REPRO =
-VERBOSE =
-OPENMP =
 
+DEBUG =
+
 MAKEFLAGS += --jobs=$(shell grep '^processor' /proc/cpuinfo | wc -l)
 
 FPPFLAGS :=
+FC_AUTO_R8 := -fdefault-real-8 -fdefault-double-8
+FFLAGS :=  $(FC_AUTO_R8) -fconvert=big-endian -ffree-line-length-none -ffixed-line-length-none -fallow-argument-mismatch  -fallow-invalid-boz -fcray-pointer
+FFLAGS_REPRO = -O
+FFLAGS_DEBUG = -g -Wall -Og -fbacktrace -ffpe-trap=zero,overflow -fcheck=bounds
 
-FFLAGS := -fcray-pointer -fdefault-double-8 -fdefault-real-8 -Waliasing -ffree-line-length-none -fno-range-check -fallow-argument-mismatch
-FFLAGS += -I$(shell nc-config --includedir)
-FFLAGS += $(shell pkg-config --cflags-only-I mpich2-c)
-FFLAGS_OPT = -O3
-FFLAGS_REPRO = -O2 -fbounds-check
-FFLAGS_DEBUG = -O0 -g -W -fbounds-check -fbacktrace -ffpe-trap=invalid,zero,overflow
-FFLAGS_OPENMP = -fopenmp
-FFLAGS_VERBOSE =
-
-CFLAGS := -D__IFC
-CFLAGS += -I$(shell nc-config --includedir)
-CFLAGS += $(shell pkg-config --cflags-only-I mpich2-c)
-CFLAGS_OPT = -O2
-CFLAGS_OPENMP = -fopenmp
-CFLAGS_DEBUG = -O0 -g
 
-# Optional Testing compile flags.  Mutually exclusive from DEBUG, REPRO, and OPT
-# *_TEST will match the production if no new option(s) is(are) to be tested.
-FFLAGS_TEST = -O2
-CFLAGS_TEST = -O2
+CFLAGS := -std=gnu99
+CFLAGS_REPRO = -O
+CFLAGS_DEBUG = -g -Wall -Og -fbacktrace -ffpe-trap=invalid,zero,overflow -fcheck=bounds
 
 LDFLAGS :=
-LDFLAGS_OPENMP := -fopenmp
-LDFLAGS_VERBOSE :=
 
-ifneq ($(REPRO),)
-CFLAGS += $(CFLAGS_REPRO)
-FFLAGS += $(FFLAGS_REPRO)
-else ifneq ($(DEBUG),)
+
+ifeq ($(DEBUG),1)
 CFLAGS += $(CFLAGS_DEBUG)
 FFLAGS += $(FFLAGS_DEBUG)
-else ifneq ($(TEST),)
-CFLAGS += $(CFLAGS_TEST)
-FFLAGS += $(FFLAGS_TEST)
 else
-CFLAGS += $(CFLAGS_OPT)
-FFLAGS += $(FFLAGS_OPT)
+CFLAGS += $(CFLAGS_REPRO)
+FFLAGS += $(FFLAGS_REPRO)
 endif
 
-ifneq ($(OPENMP),)
-CFLAGS += $(CFLAGS_OPENMP)
-FFLAGS += $(FFLAGS_OPENMP)
-LDFLAGS += $(LDFLAGS_OPENMP)
-endif
 
-ifneq ($(VERBOSE),)
-CFLAGS += $(CFLAGS_VERBOSE)
-FFLAGS += $(FFLAGS_VERBOSE)
-LDFLAGS += $(LDFLAGS_VERBOSE)
-endif
-
-ifeq ($(NETCDF),3)
+# NetCDF Flags
+FFLAGS += -I$(shell nc-config --includedir)
+CFLAGS += -I$(shell nc-config --includedir)
   # add the use_LARGEFILE cppdef
-  ifneq ($(findstring -Duse_netCDF,$(CPPDEFS)),)
+ifneq ($(findstring -Duse_netCDF,$(CPPDEFS)),)
     CPPDEFS += -Duse_LARGEFILE
-  endif
 endif
 
+# Linking Flags
 LIBS := $(shell nf-config --flibs) $(shell pkg-config --libs mpich2-f90)
 LDFLAGS += $(LIBS)
 

diff --git a/standalone/templates/ncar-intel.mk b/standalone/templates/ncar-intel.mk
@@ -1,11 +1,8 @@
 # template for Intel compilers
-# typical use with mkmf:
-# mkmf -t nescc-intel.mk -c "-Duse_libMPI -Duse_netCDF" path_names /usr/local/include
 
 ############
 # commands #
 ############
-
 FC = mpif90
 CC = mpicc
 CXX = icpc
@@ -15,72 +12,42 @@ LD = mpif90
 #  flags   #
 ############
 
-DEBUG =
-REPRO =
-VERBOSE =
-OPENMP =
+DEBUG = 
+# Default set to REPRODUCIBLE
 
 MAKEFLAGS += --jobs=8
 
+FC_AUTO_R8 := -r8
 FPPFLAGS := -fpp -Wp,-w
+FFLAGS :=  -qno-opt-dynamic-align  -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model source  -no-fma  -qopt-report -march=core-avx2 $(FC_AUTO_R8) 
+FFLAGS_DEBUG = -O0 -g -check uninit -check bounds -check nopointer -fpe0 -check noarg_temp_created # CESM uses -check pointers, that throws an error, changed to nopointer
+FFLAGS_REPRO = -O2 -debug minimal
 
-FFLAGS := -fno-alias -auto -safe-cray-ptr -ftz -assume byterecl -i4 -r8 -nowarn -traceback
-FFLAGS += -I$(shell nf-config --includedir)
-FFLAGS_OPT = -O3 -debug minimal -fp-model source -qoverride-limits
-FFLAGS_DEBUG = -g -O0 -check -check noarg_temp_created -check nopointer -warn -warn noerrors -fpe0 -ftrapuv
-FFLAGS_REPRO = -O2 -debug minimal -fp-model source -qoverride-limits
-FFLAGS_OPENMP = -openmp
-FFLAGS_VERBOSE = -v -V -what
-
-CFLAGS := -D__IFC -sox -traceback -diag-disable=10441
+CFLAGS := -qno-opt-dynamic-align -fp-model precise -std=gnu99  -no-fma -qopt-report -march=core-avx2
 CFLAGS += -I$(NETCDF_PATH)/include
-CFLAGS_OPT = -O2 -debug minimal
-CFLAGS_OPENMP = -openmp
-CFLAGS_DEBUG = -O0 -g -ftrapuv
-
+CFLAGS_REPRO= -O2 -debug minimal
+CFLAGS_DEBUG = -O0 -g
 LDFLAGS :=
-LDFLAGS_OPENMP := -openmp
-LDFLAGS_VERBOSE := -Wl,-V,--verbose,-cref,-M
 
-# start with blank LIBS
-LIBS :=
-
-ifneq ($(REPRO),)
-  CFLAGS += $(CFLAGS_REPRO)
-  FFLAGS += $(FFLAGS_REPRO)
-else ifneq ($(DEBUG),)
+ifeq ($(DEBUG),1)
   CFLAGS += $(CFLAGS_DEBUG)
   FFLAGS += $(FFLAGS_DEBUG)
 else
-  CFLAGS += $(CFLAGS_OPT)
-  FFLAGS += $(FFLAGS_OPT)
-endif
-
-ifneq ($(OPENMP),)
-  CFLAGS += $(CFLAGS_OPENMP)
-  FFLAGS += $(FFLAGS_OPENMP)
-  LDFLAGS += $(LDFLAGS_OPENMP)
-endif
-
-ifneq ($(VERBOSE),)
-  CFLAGS += $(CFLAGS_VERBOSE)
-  FFLAGS += $(FFLAGS_VERBOSE)
-  LDFLAGS += $(LDFLAGS_VERBOSE)
+  FFLAGS += $(FFLAGS_REPRO)
+  CFLAGS += $(CFLAGS_REPRO)
 endif
 
-ifeq ($(NETCDF),3)
-  # add the use_LARGEFILE cppdef
-  ifneq ($(findstring -Duse_netCDF,$(CPPDEFS)),)
-    CPPDEFS += -Duse_LARGEFILE
-  endif
+# Add Net CDF Flags
+FFLAGS += -I$(shell nf-config --includedir)
+# add the use_LARGEFILE cppdef
+ifneq ($(findstring -Duse_netCDF,$(CPPDEFS)),)
+  CPPDEFS += -Duse_LARGEFILE
 endif
 
+CPPDEFS := $(CPPDEFS) -D__IFC
 # Add netcdf linking
 LIBS := $(shell nc-config --libs) $(shell nf-config --flibs)
 
-#LIBS += -lmpi
-#LIBS += -lmpi -lsma
-#LIBS += -lmkl_blas95_lp64 -lmkl_lapack95_lp64 -lmkl_intel_lp64 -lmkl_core -lmkl_sequential
 LDFLAGS += $(LIBS)
 
 #---------------------------------------------------------------------------