diff --git a/README.md b/README.md index 6b8fbf589f..289e74933b 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ The `global-workflow` current supports the following tier-1 machines: * NOAA RDHPCS - Hera * MSU HPC - Orion +* MSU HPC - Hercules * NOAA's operational HPC - WCOSS2 Additionally, the following tier-2 machine is supported: diff --git a/docs/note_fixfield.txt b/docs/note_fixfield.txt index af2539e48a..a7a4001561 100644 --- a/docs/note_fixfield.txt +++ b/docs/note_fixfield.txt @@ -3,7 +3,7 @@ For EMC, the fix fields for running the model are not included in git repository They are saved locally on all platforms Hera: /scratch1/NCEPDEV/global/glopara/fix -Orion: /work/noaa/global/glopara/fix +Orion/Hercules: /work/noaa/global/glopara/fix Jet: /mnt/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix S4: /data/prod/glopara/fix diff --git a/docs/source/clone.rst b/docs/source/clone.rst index 3b2d555520..c098a34f7e 100644 --- a/docs/source/clone.rst +++ b/docs/source/clone.rst @@ -32,7 +32,7 @@ For cycled (w/ data assimilation) use the `-g` option during checkout: For coupled cycling (include new UFSDA) use the `-gu` options during checkout: -[Currently only available on Hera and Orion] +[Currently only available on Hera, Orion, and Hercules] :: @@ -110,7 +110,7 @@ Or with the ``-g`` switch to include data assimilation (GSI) for cycling: ./checkout.sh -g Or also with the ``-u`` swtich to include coupled DA (via UFSDA): -[Currently only available on Hera and Orion] +[Currently only available on Hera, Orion, and Hercules] :: diff --git a/docs/source/components.rst b/docs/source/components.rst index 59385c45d0..4d2619e44e 100644 --- a/docs/source/components.rst +++ b/docs/source/components.rst @@ -57,7 +57,7 @@ Data Observation data, also known as dump data, is prepared in production and then archived in a global dump archive (GDA) for use by users when running cycled experiments. The GDA (identified as ``$DMPDIR`` in the workflow) is available on supported platforms and the workflow system knows where to find the data. * Hera: /scratch1/NCEPDEV/global/glopara/dump -* Orion: /work/noaa/rstprod/dump +* Orion/Hercules: /work/noaa/rstprod/dump * Jet: /mnt/lfs4/HFIP/hfv3gfs/glopara/dump * WCOSS2: /lfs/h2/emc/global/noscrub/emc.global/dump * S4: /data/prod/glopara/dump diff --git a/docs/source/hpc.rst b/docs/source/hpc.rst index 7442b0bd69..3ce6a889d9 100644 --- a/docs/source/hpc.rst +++ b/docs/source/hpc.rst @@ -19,6 +19,7 @@ HPC helpdesks * WCOSS2: hpc.wcoss2-help@noaa.gov * Hera: rdhpcs.hera.help@noaa.gov * Orion: rdhpcs.orion.help@noaa.gov +* Hercules: rdhpcs.hercules.help@noaa.gov * HPSS: rdhpcs.hpss.help@noaa.gov * Gaea: oar.gfdl.help@noaa.gov * S4: david.huber@noaa.gov @@ -72,19 +73,21 @@ Version It is advised to use Git v2+ when available. At the time of writing this documentation the default Git clients on the different machines were as noted in the table below. It is recommended that you check the default modules before loading recommended ones: +---------+----------+---------------------------------------+ -| Machine | Default | Recommended | +| Machine | Default | Recommended | +---------+----------+---------------------------------------+ -| Hera | v2.18.0 | default | +| Hera | v2.18.0 | default | +---------+----------+---------------------------------------+ -| Orion | v1.8.3.1 | **module load git/2.28.0** | +| Hercules | v2.31.1 | default | +---------+----------+---------------------------------------+ -| Jet | v2.18.0 | default | +| Orion | v1.8.3.1 | **module load git/2.28.0** | +---------+----------+---------------------------------------+ -| WCOSS2 | v2.26.2 | default or **module load git/2.29.0** | +| Jet | v2.18.0 | default | +---------+----------+---------------------------------------+ -| S4 | v1.8.3.1 | **module load git/2.30.0** | +| WCOSS2 | v2.26.2 | default or **module load git/2.29.0** | +---------+----------+---------------------------------------+ -| AWS PW | v1.8.3.1 | default +| S4 | v1.8.3.1 | **module load git/2.30.0** | ++---------+----------+---------------------------------------+ +| AWS PW | v1.8.3.1 | default +---------+----------+---------------------------------------+ ^^^^^^^^^^^^^ @@ -103,7 +106,7 @@ For the manage_externals utility functioning:: Fix: git config --global ssh.variant ssh ======================================== -Stacksize on R&Ds (Hera, Orion, Jet, S4) +Stacksize on R&Ds (Hera, Orion, Hercules, Jet, S4) ======================================== Some GFS components, like the UPP, need an unlimited stacksize. Add the following setting into your appropriate .*rc file to support these components: diff --git a/docs/source/init.rst b/docs/source/init.rst index 5c9c811052..f9562a3a7d 100644 --- a/docs/source/init.rst +++ b/docs/source/init.rst @@ -49,7 +49,7 @@ Cold-start atmosphere-only cycled C96 deterministic C48 enkf (80 members) ICs ar :: Hera: /scratch1/NCEPDEV/global/glopara/data/ICSDIR/C96C48 - Orion: /work/noaa/global/glopara/data/ICSDIR/C96C48 + Orion/Hercules: /work/noaa/global/glopara/data/ICSDIR/C96C48 WCOSS2: /lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR/C96C48 Start date = 2021122018 @@ -108,7 +108,7 @@ Warm-start cycled w/ coupled (S2S) model C48 atmosphere C48 enkf (80 members) 5 :: Hera: /scratch1/NCEPDEV/global/glopara/data/ICSDIR/C48C48mx500 - Orion: /work/noaa/global/glopara/data/ICSDIR/C48C48mx500 + Orion/Hercules: /work/noaa/global/glopara/data/ICSDIR/C48C48mx500 WCOSS2: /lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR/C48C48mx500 Jet: /lfs4/HFIP/hfv3gfs/glopara/data/ICSDIR/C48C48mx500 @@ -224,7 +224,7 @@ Forecast-only P8 prototype initial conditions are made available to users on sup WCOSS2: /lfs/h2/emc/global/noscrub/emc.global/IC/COUPLED HERA: /scratch1/NCEPDEV/climate/role.ufscpara/IC - ORION: /work/noaa/global/glopara/data/ICSDIR/prototype_ICs + ORION/Hercules: /work/noaa/global/glopara/data/ICSDIR/prototype_ICs JET: /mnt/lfs4/HFIP/hfv3gfs/glopara/data/ICSDIR/prototype_ICs S4: /data/prod/glopara/coupled_ICs @@ -253,7 +253,7 @@ Not yet supported. See :ref:`Manual Generation` section below --------------------- Forecast-only coupled --------------------- -Coupled initial conditions are currently only generated offline and copied prior to the forecast run. Prototype initial conditions will automatically be used when setting up an experiment as an S2SW app, there is no need to do anything additional. Copies of initial conditions from the prototype runs are currently maintained on Hera, Orion, Jet, and WCOSS2. The locations used are determined by ``parm/config/config.coupled_ic``. If you need prototype ICs on another machine, please contact Walter (Walter.Kolczynski@noaa.gov). +Coupled initial conditions are currently only generated offline and copied prior to the forecast run. Prototype initial conditions will automatically be used when setting up an experiment as an S2SW app, there is no need to do anything additional. Copies of initial conditions from the prototype runs are currently maintained on Hera, Orion/Hercules, Jet, and WCOSS2. The locations used are determined by ``parm/config/config.coupled_ic``. If you need prototype ICs on another machine, please contact Walter (Walter.Kolczynski@noaa.gov). .. _forecastonly-atmonly: @@ -354,7 +354,7 @@ Then switch to a different tag or use the default branch (develop). where ``$MACHINE`` is ``wcoss2``, ``hera``, or ``jet``. .. note:: - UFS-UTILS builds on Orion but due to the lack of HPSS access on Orion the ``gdas_init`` utility is not supported there. + UFS-UTILS builds on Orion/Hercules but due to the lack of HPSS access on Orion/Hercules the ``gdas_init`` utility is not supported there. 3. Configure your conversion: @@ -380,7 +380,7 @@ Most users will want to adjust the following ``config`` settings for the current where ``$MACHINE`` is currently ``wcoss2``, ``hera`` or ``jet``. Additional options will be available as support for other machines expands. .. note:: - UFS-UTILS builds on Orion but due to lack of HPSS access there is no ``gdas_init`` driver for Orion nor support to pull initial conditions from HPSS for the ``gdas_init`` utility. + UFS-UTILS builds on Orion/Hercules but due to lack of HPSS access there is no ``gdas_init`` driver for Orion/Hercules nor support to pull initial conditions from HPSS for the ``gdas_init`` utility. Several small jobs will be submitted: diff --git a/docs/source/start.rst b/docs/source/start.rst index 957971e637..7dc093e8a4 100644 --- a/docs/source/start.rst +++ b/docs/source/start.rst @@ -23,7 +23,7 @@ Set up your experiment cron ^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. note:: - Orion currently only supports cron on Orion-login-1. Cron support for other login nodes is coming in the future. + Orion and Hercules currently only support cron on Orion-login-1 and Hercules-login-1, respectively. Cron support for other login nodes is coming in the future. :: diff --git a/env/HERCULES.env b/env/HERCULES.env new file mode 100755 index 0000000000..3721be2b66 --- /dev/null +++ b/env/HERCULES.env @@ -0,0 +1,69 @@ +#! /usr/bin/env bash + +if [[ $# -ne 1 ]]; then + + echo "Must specify an input argument to set runtime environment variables!" + echo "argument can be any one of the following:" + echo "fcst post" + echo "Note: Hercules is only set up to run in forecast-only mode" + exit 1 + +fi + +step=$1 + +export npe_node_max=40 +export launcher="srun -l --export=ALL" +export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" + +# Configure MPI environment +export MPI_BUFS_PER_PROC=2048 +export MPI_BUFS_PER_HOST=2048 +export MPI_GROUP_MAX=256 +export MPI_MEMMAP_OFF=1 +export MP_STDOUTMODE="ORDERED" +export KMP_AFFINITY=scatter +export OMP_STACKSIZE=2048000 +export NTHSTACK=1024000000 +#export LD_BIND_NOW=1 + +ulimit -s unlimited +ulimit -a + +if [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || \ + [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostpnt" ]] || [[ "${step}" == "wavepostbndpntbll" ]]; then + + export CFP_MP="YES" + if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi + export wavempexec=${launcher} + export wave_mpmd=${mpmd_opt} + +elif [[ "${step}" = "fcst" ]]; then + + export OMP_STACKSIZE=512M + if [[ "${CDUMP}" =~ "gfs" ]]; then + nprocs="npe_${step}_gfs" + ppn="npe_node_${step}_gfs" || ppn="npe_node_${step}" + else + nprocs="npe_${step}" + ppn="npe_node_${step}" + fi + (( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} )) + (( ntasks = nnodes*${!ppn} )) + # With ESMF threading, the model wants to use the full node + export APRUN_UFS="${launcher} -n ${ntasks}" + unset nprocs ppn nnodes ntasks + +elif [[ "${step}" = "upp" ]]; then + + nth_max=$((npe_node_max / npe_node_upp)) + + export NTHREADS_UPP=${nth_upp:-1} + [[ ${NTHREADS_UPP} -gt ${nth_max} ]] && export NTHREADS_UPP=${nth_max} + export APRUN_UPP="${launcher} -n ${npe_upp} --cpus-per-task=${NTHREADS_UPP}" + +elif [[ "${step}" = "atmos_products" ]]; then + + export USE_CFP="YES" # Use MPMD for downstream product generation + +fi diff --git a/modulefiles/module-setup.csh.inc b/modulefiles/module-setup.csh.inc index e8219424f6..7086326627 100644 --- a/modulefiles/module-setup.csh.inc +++ b/modulefiles/module-setup.csh.inc @@ -19,11 +19,18 @@ else if ( { test -d /scratch1 } ) then source /apps/lmod/lmod/init/$__ms_shell endif module purge -else if ( { test -d /work } ) then - # We are on MSU Orion - if ( ! { module help >& /dev/null } ) then - source /apps/lmod/init/$__ms_shell - endif +elif [[ -d /work ]] ; then + # We are on MSU Orion or Hercules + if [[ -d /apps/other ]] ; then + # Hercules + init_path="/apps/other/lmod/lmod/init/$__ms_shell" + else + # Orion + init_path="/apps/lmod/lmod/init/$__ms_shell" + fi + if ( ! eval module help > /dev/null 2>&1 ) ; then + source "${init_path}" + fi module purge else if ( { test -d /data/prod } ) then # We are on SSEC S4 diff --git a/modulefiles/module-setup.sh.inc b/modulefiles/module-setup.sh.inc index e5322cbb2c..db9dabffe1 100644 --- a/modulefiles/module-setup.sh.inc +++ b/modulefiles/module-setup.sh.inc @@ -35,9 +35,16 @@ elif [[ -d /scratch1 ]] ; then fi module purge elif [[ -d /work ]] ; then - # We are on MSU Orion + # We are on MSU Orion or Hercules + if [[ -d /apps/other ]] ; then + # Hercules + init_path="/apps/other/lmod/lmod/init/$__ms_shell" + else + # Orion + init_path="/apps/lmod/lmod/init/$__ms_shell" + fi if ( ! eval module help > /dev/null 2>&1 ) ; then - source /apps/lmod/lmod/init/$__ms_shell + source "${init_path}" fi module purge elif [[ -d /glade ]] ; then diff --git a/modulefiles/module_base.hercules.lua b/modulefiles/module_base.hercules.lua new file mode 100644 index 0000000000..d587b90c4f --- /dev/null +++ b/modulefiles/module_base.hercules.lua @@ -0,0 +1,46 @@ +help([[ +Load environment to run GFS on Hercules +]]) + +spack_stack_ver=(os.getenv("spack_stack_ver") or "None") +spack_env=(os.getenv("spack_env") or "None") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-" .. spack_stack_ver .. "/envs/" .. spack_env .. "/install/modulefiles/Core") + +load(pathJoin("stack-intel", os.getenv("stack_intel_ver"))) +load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver"))) +load(pathJoin("python", os.getenv("python_ver"))) + +-- TODO load NCL once the SAs remove the 'depends_on' statements within it +-- NCL is a static installation and does not depend on any libraries +-- but as is will load, among others, the system netcdf-c/4.9.0 module +--load(pathJoin("ncl", os.getenv("ncl_ver"))) +load(pathJoin("jasper", os.getenv("jasper_ver"))) +load(pathJoin("libpng", os.getenv("libpng_ver"))) +load(pathJoin("cdo", os.getenv("cdo_ver"))) + +load(pathJoin("hdf5", os.getenv("hdf5_ver"))) +load(pathJoin("netcdf-c", os.getenv("netcdf_c_ver"))) +load(pathJoin("netcdf-fortran", os.getenv("netcdf_fortran_ver"))) + +load(pathJoin("nco", os.getenv("nco_ver"))) +load(pathJoin("prod_util", os.getenv("prod_util_ver"))) +load(pathJoin("grib-util", os.getenv("grib_util_ver"))) +load(pathJoin("g2tmpl", os.getenv("g2tmpl_ver"))) +load(pathJoin("gsi-ncdiag", os.getenv("gsi_ncdiag_ver"))) +load(pathJoin("crtm", os.getenv("crtm_ver"))) +load(pathJoin("bufr", os.getenv("bufr_ver"))) +load(pathJoin("wgrib2", os.getenv("wgrib2_ver"))) +load(pathJoin("py-netcdf4", os.getenv("py_netcdf4_ver"))) +load(pathJoin("py-pyyaml", os.getenv("py_pyyaml_ver"))) +load(pathJoin("py-jinja2", os.getenv("py_jinja2_ver"))) + +setenv("WGRIB2","wgrib2") +setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) + +prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles")) +load(pathJoin("prepobs", os.getenv("prepobs_run_ver"))) + +prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) +load(pathJoin("fit2obs", os.getenv("fit2obs_ver"))) + +whatis("Description: GFS run environment") diff --git a/modulefiles/module_gwci.hercules.lua b/modulefiles/module_gwci.hercules.lua new file mode 100644 index 0000000000..9c60aed467 --- /dev/null +++ b/modulefiles/module_gwci.hercules.lua @@ -0,0 +1,15 @@ +help([[ +Load environment to run GFS workflow ci scripts on Hercules +]]) + +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") + +load(pathJoin("stack-intel", os.getenv("2021.9.0"))) +load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.9.0"))) + +load(pathJoin("netcdf-c", os.getenv("4.9.2"))) +load(pathJoin("netcdf-fortran", os.getenv("4.6.0"))) +load(pathJoin("nccmp","1.9.0.1")) +load(pathJoin("wgrib2", "3.1.1")) + +whatis("Description: GFS run ci top-level sripts environment") diff --git a/modulefiles/module_gwsetup.hercules.lua b/modulefiles/module_gwsetup.hercules.lua new file mode 100644 index 0000000000..673928605c --- /dev/null +++ b/modulefiles/module_gwsetup.hercules.lua @@ -0,0 +1,19 @@ +help([[ +Load environment to run GFS workflow ci scripts on Hercules +]]) + +load(pathJoin("contrib","0.1")) +load(pathJoin("rocoto","1.3.5")) + +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") + +local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" +local python_ver=os.getenv("python_ver") or "3.10.8" + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("python", python_ver)) +load("py-jinja2") +load("py-pyyaml") +load("py-numpy") + +whatis("Description: GFS run setup environment") diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 40860f7b3a..a50418d23a 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -43,6 +43,8 @@ elif [[ ${machine} = "S4" ]]; then fi elif [[ ${machine} = "ORION" ]]; then export npe_node_max=40 +elif [[ ${machine} = "HERCULES" ]]; then + export npe_node_max=40 fi if [[ ${step} = "prep" ]]; then diff --git a/parm/config/gefs/config.ufs b/parm/config/gefs/config.ufs index db0589ab6a..68b364529e 100644 --- a/parm/config/gefs/config.ufs +++ b/parm/config/gefs/config.ufs @@ -72,7 +72,7 @@ case "${machine}" in "WCOSS2") npe_node_max=128 ;; - "HERA" | "ORION") + "HERA" | "ORION" | "HERCULES" ) npe_node_max=40 ;; "JET") diff --git a/parm/config/gfs/config.aero b/parm/config/gfs/config.aero index 23d738c642..32993554b4 100644 --- a/parm/config/gfs/config.aero +++ b/parm/config/gfs/config.aero @@ -11,7 +11,7 @@ case ${machine} in "HERA") AERO_INPUTS_DIR="/scratch1/NCEPDEV/global/glopara/data/gocart_emissions" ;; - "ORION") + "ORION" | "HERCULES") AERO_INPUTS_DIR="/work2/noaa/global/wkolczyn/noscrub/global-workflow/gocart_emissions" ;; "S4") diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 46be4fbda4..695ad5fcc5 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -55,6 +55,8 @@ elif [[ "${machine}" = "AWSPW" ]]; then export npe_node_max=40 elif [[ ${machine} = "ORION" ]]; then export npe_node_max=40 +elif [[ ${machine} = "HERCULES" ]]; then + export npe_node_max=40 fi if [[ ${step} = "prep" ]]; then diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs index 33264ba0aa..000c8b1e99 100644 --- a/parm/config/gfs/config.ufs +++ b/parm/config/gfs/config.ufs @@ -72,7 +72,7 @@ case "${machine}" in "WCOSS2") npe_node_max=128 ;; - "HERA" | "ORION") + "HERA" | "ORION" | "HERCULES") npe_node_max=40 ;; "JET") diff --git a/scripts/exglobal_atmos_products.sh b/scripts/exglobal_atmos_products.sh index beeea99161..d2c0ed7466 100755 --- a/scripts/exglobal_atmos_products.sh +++ b/scripts/exglobal_atmos_products.sh @@ -133,7 +133,7 @@ for (( nset=1 ; nset <= downset ; nset++ )); do export err=$? else chmod 755 "${DATA}/poescript" - bash +x "${DATA}/poescript" 2>&1 mpmd.out + bash +x "${DATA}/poescript" > mpmd.out 2>&1 export err=$? fi err_chk diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 4558bed4d5..581c50e704 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -67,6 +67,7 @@ case "${machine}" in "wcoss2") FIX_DIR="/lfs/h2/emc/global/noscrub/emc.global/FIX/fix" ;; "hera") FIX_DIR="/scratch1/NCEPDEV/global/glopara/fix" ;; "orion") FIX_DIR="/work/noaa/global/glopara/fix" ;; + "hercules") FIX_DIR="/work/noaa/global/glopara/fix" ;; "jet") FIX_DIR="/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix" ;; "s4") FIX_DIR="/data/prod/glopara/fix" ;; *) diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index f7e4a9d4f3..afeb01830b 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -26,6 +26,8 @@ case $(hostname -f) in Orion-login-[1-4].HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion1-4 + hercules-login-[1-4].hpc.msstate.edu) MACHINE_ID=hercules ;; ### hercules1-4 + cheyenne[1-6].cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 cheyenne[1-6].ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 chadmin[1-6].ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 @@ -64,8 +66,13 @@ elif [[ -d /scratch1 ]] ; then # We are on NOAA Hera MACHINE_ID=hera elif [[ -d /work ]] ; then - # We are on MSU Orion - MACHINE_ID=orion + # We are on MSU Orion or Hercules + if [[ -d /apps/other ]] ; then + # We are on Hercules + MACHINE_ID=hercules + else + MACHINE_ID=orion + fi elif [[ -d /glade ]] ; then # We are on NCAR Yellowstone MACHINE_ID=cheyenne diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh index 6ceb5c63ee..b4f23fa331 100755 --- a/ush/load_fv3gfs_modules.sh +++ b/ush/load_fv3gfs_modules.sh @@ -28,8 +28,14 @@ elif [[ -d /scratch1 ]] ; then # We are on NOAA Hera module load module_base.hera elif [[ -d /work ]] ; then - # We are on MSU Orion - module load module_base.orion + # We are on MSU Orion or Hercules + if [[ -d /apps/other ]] ; then + # Hercules + module load module_base.hercules + else + # Orion + module load module_base.orion + fi elif [[ -d /glade ]] ; then # We are on NCAR Yellowstone module load module_base.cheyenne diff --git a/ush/module-setup.sh b/ush/module-setup.sh index e01e379de4..fd656966bf 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -19,10 +19,20 @@ elif [[ ${MACHINE_ID} = hera* ]] ; then module reset set -u +elif [[ ${MACHINE_ID} = hercules* ]] ; then + # We are on Hercules + if ( ! eval module help > /dev/null 2>&1 ) ; then + source /apps/other/lmod/lmod/init/bash + fi + export LMOD_SYSTEM_DEFAULT_MODULES=contrib + set +u + module reset + set -u + elif [[ ${MACHINE_ID} = orion* ]] ; then # We are on Orion if ( ! eval module help > /dev/null 2>&1 ) ; then - source /apps/lmod/init/bash + source /apps/lmod/lmod/init/bash fi export LMOD_SYSTEM_DEFAULT_MODULES=contrib set +u diff --git a/versions/build.hercules.ver b/versions/build.hercules.ver new file mode 100644 index 0000000000..5513466631 --- /dev/null +++ b/versions/build.hercules.ver @@ -0,0 +1,3 @@ +export stack_intel_ver=2021.9.0 +export stack_impi_ver=2021.9.0 +source "${HOMEgfs:-}/versions/build.spack.ver" diff --git a/versions/run.hercules.ver b/versions/run.hercules.ver new file mode 100644 index 0000000000..4bedeb1e96 --- /dev/null +++ b/versions/run.hercules.ver @@ -0,0 +1,10 @@ +export stack_intel_ver=2021.9.0 +export stack_impi_ver=2021.9.0 + +export ncl_ver=6.6.2 + +source "${HOMEgfs:-}/versions/run.spack.ver" + +# wgrib2 and cdo are different on Hercules from all the other systems +export wgrib2_ver=3.1.1 +export cdo_ver=2.2.0 diff --git a/workflow/hosts.py b/workflow/hosts.py index 0b199ba382..a17cd3f4a8 100644 --- a/workflow/hosts.py +++ b/workflow/hosts.py @@ -14,7 +14,7 @@ class Host: Gather Host specific information. """ - SUPPORTED_HOSTS = ['HERA', 'ORION', 'JET', + SUPPORTED_HOSTS = ['HERA', 'ORION', 'JET', 'HERCULES', 'WCOSS2', 'S4', 'CONTAINER', 'AWSPW'] def __init__(self, host=None): @@ -39,7 +39,10 @@ def detect(cls): if os.path.exists('/scratch1/NCEPDEV'): machine = 'HERA' elif os.path.exists('/work/noaa'): - machine = 'ORION' + if os.path.exists('/apps/other'): + machine = 'HERCULES' + else: + machine = 'ORION' elif os.path.exists('/lfs4/HFIP'): machine = 'JET' elif os.path.exists('/lfs/f1'): diff --git a/workflow/hosts/hercules.yaml b/workflow/hosts/hercules.yaml new file mode 100644 index 0000000000..e977091ba6 --- /dev/null +++ b/workflow/hosts/hercules.yaml @@ -0,0 +1,25 @@ +BASE_GIT: '/work/noaa/global/glopara/git' +DMPDIR: '/work/noaa/rstprod/dump' +BASE_CPLIC: '/work/noaa/global/glopara/data/ICSDIR/prototype_ICs' +PACKAGEROOT: '/work/noaa/global/glopara/nwpara' +COMROOT: '/work/noaa/global/glopara/com' +COMINsyn: '${COMROOT}/gfs/prod/syndat' +HOMEDIR: '/work/noaa/global/${USER}' +STMP: '/work/noaa/stmp/${USER}' +PTMP: '/work/noaa/stmp/${USER}' +NOSCRUB: $HOMEDIR +SCHEDULER: slurm +ACCOUNT: fv3-cpu +QUEUE: batch +QUEUE_SERVICE: batch +PARTITION_BATCH: hercules +PARTITION_SERVICE: service +CHGRP_RSTPROD: 'YES' +CHGRP_CMD: 'chgrp rstprod' +HPSSARCH: 'NO' +HPSS_PROJECT: emc-global +LOCALARCH: 'NO' +ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' +MAKE_NSSTBUFR: 'NO' +MAKE_ACFTBUFR: 'NO' +SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48']