From f3c94d2f1040ff432d864ba9f9a74918a0fbf8a1 Mon Sep 17 00:00:00 2001
From: abouchat <50667934+abouchat@users.noreply.github.com>
Date: Thu, 16 May 2019 16:32:46 -0400
Subject: [PATCH] Add machine files for millikan (#313)

---
 configuration/scripts/cice.batch.csh          | 13 ++++
 configuration/scripts/cice.launch.csh         | 12 ++++
 .../scripts/machines/Macros.millikan_intel    | 72 +++++++++++++++++++
 .../scripts/machines/env.millikan_intel       | 25 +++++++
 4 files changed, 122 insertions(+)
 create mode 100644 configuration/scripts/machines/Macros.millikan_intel
 create mode 100644 configuration/scripts/machines/env.millikan_intel

diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh
index c7c410e55..07b1f06e8 100755
--- a/configuration/scripts/cice.batch.csh
+++ b/configuration/scripts/cice.batch.csh
@@ -150,6 +150,19 @@ cat >> ${jobfile} << EOFB
 #SBATCH --qos=standby
 EOFB
 
+else if (${ICE_MACHINE} =~ millikan*) then
+cat >> ${jobfile} << EOFB
+#SBATCH -J ${ICE_CASENAME}
+#SBATCH -t ${batchtime}
+#SBATCH -A ${acct}
+#SBATCH -N ${nnodes}
+#SBATCH -e slurm%j.err
+#SBATCH -o slurm%j.out
+###SBATCH --mail-type END,FAIL
+###SBATCH --mail-user=amelie.bouchat@canada.ca
+#SBATCH --qos=standby
+EOFB
+
 else if (${ICE_MACHINE} =~ testmachine*) then
 cat >> ${jobfile} << EOFB
 # nothing to do
diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh
index b5f243159..86ede7049 100755
--- a/configuration/scripts/cice.launch.csh
+++ b/configuration/scripts/cice.launch.csh
@@ -141,6 +141,18 @@ mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE
 EOFR
 endif
 
+#=======
+else if (${ICE_MACHINE} =~ millikan*) then
+if (${ICE_COMMDIR} =~ serial*) then
+cat >> ${jobfile} << EOFR
+./cice >&! \$ICE_RUNLOG_FILE
+EOFR
+else
+cat >> ${jobfile} << EOFR
+mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE
+EOFR
+endif
+
 #=======
 else if (${ICE_MACHINE} =~ testmachine*) then
 cat >> ${jobfile} << EOFR
diff --git a/configuration/scripts/machines/Macros.millikan_intel b/configuration/scripts/machines/Macros.millikan_intel
new file mode 100644
index 000000000..1d74158c9
--- /dev/null
+++ b/configuration/scripts/machines/Macros.millikan_intel
@@ -0,0 +1,72 @@
+#==============================================================================
+# Makefile macros for "millikan"
+#==============================================================================
+# For use with intel compiler
+#==============================================================================
+
+CPP        := fpp
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c -O2 -fp-model precise
+#-xHost
+
+FIXEDFLAGS := -132
+FREEFLAGS  := -FR
+FFLAGS     := -fp-model source -convert big_endian -assume byterecl -ftz -traceback  -diag-disable 5140
+#-xHost
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays
+# -heap-arrays 1024 
+else
+  FFLAGS     += -O2
+endif
+
+SCC   := icc
+SFC   := ifort
+MPICC := mpicc
+MPIFC := mpif90
+
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+LD:= $(FC)
+
+NETCDF_PATH := /fs/ssm/hpco/tmp/eccc/201402/04/intel-2016.1.150/ubuntu-14.04-amd64-64/
+
+PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs 
+
+#PNETCDF_PATH := $(PNETCDF)
+#PNETCDF_PATH := /glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib
+
+INCLDIR := $(INCLDIR)
+
+LIB_NETCDF := $(NETCDF_PATH)/lib
+LIB_PNETCDF := $(PNETCDF_PATH)/lib
+LIB_MPI := $(IMPILIBDIR)
+
+#SLIBS   := -L$(LIB_NETCDF) -lnetcdff -lnetcdf -L$(LIB_PNETCDF) -lpnetcdf -lgptl
+SLIBS   := -L$(LIB_NETCDF) -lnetcdff -lnetcdf
+
+ifeq ($(compile_threaded), true) 
+   LDFLAGS += -openmp 
+   CFLAGS += -openmp 
+   FFLAGS += -openmp 
+endif
+
+### if using parallel I/O, load all 3 libraries.  PIO must be first!
+ifeq ($(IO_TYPE), pio)
+   PIO_PATH:=/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib
+   INCLDIR += -I/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/include
+   SLIBS   := $(SLIBS) -L$(PIO_PATH) -lpiof
+
+   CPPDEFS :=  $(CPPDEFS) -Dncdf
+endif
+
+ifeq ($(IO_TYPE), netcdf)
+   CPPDEFS :=  $(CPPDEFS) -Dncdf
+endif
diff --git a/configuration/scripts/machines/env.millikan_intel b/configuration/scripts/machines/env.millikan_intel
new file mode 100644
index 000000000..fc81ac097
--- /dev/null
+++ b/configuration/scripts/machines/env.millikan_intel
@@ -0,0 +1,25 @@
+#!/bin/csh -f
+
+set ssmuse=/fs/ssm/main/env/20180430/all/bin/ssmuse-csh # package loader
+source $ssmuse -d /fs/ssm/main/opt/intelcomp/intelcomp-2016.1.156 # intel compiler
+source /fs/ssm/main/opt/intelcomp/intelcomp-2016.1.156/intelcomp_2016.1.156_multi/bin/compilervars.csh intel64 # should be sourced by above domain, but bug in csh script
+source $ssmuse -d /fs/ssm/main/opt/openmpi/openmpi-1.6.5/intelcomp-2016.1.156 # openmpi
+source $ssmuse -d /fs/ssm/hpco/tmp/eccc/201402/04/intel-2016.1.150 # netcdf (and openmpi)
+
+setenv ICE_MACHINE_ENVNAME millikan
+setenv ICE_MACHINE_COMPILER intel
+setenv ICE_MACHINE_MAKE make
+setenv ICE_MACHINE_WKDIR  /users/dor/armn/amb/data/local/runs
+setenv ICE_MACHINE_INPUTDATA /users/dor/armn/amb/data/local/forcing
+setenv ICE_MACHINE_BASELINE /users/dor/armn/amb/data/local/baseline
+setenv ICE_MACHINE_SUBMIT "qsub"
+setenv ICE_MACHINE_TPNODE 36
+setenv ICE_MACHINE_ACCT P0000000
+setenv ICE_MACHINE_QUEUE "debug"
+setenv ICE_MACHINE_BLDTHRDS 4
+setenv ICE_MACHINE_QSTAT "qstat "
+
+if (-e ~/.cice_proj) then
+   set account_name = `head -1 ~/.cice_proj`
+   setenv CICE_ACCT ${account_name}
+endif