diff --git a/.gitmodules b/.gitmodules index bf5a05eb3..dc0798c32 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,10 +4,8 @@ branch = main [submodule "ccpp-physics"] path = ccpp/physics - #url = https://github.com/NCAR/ccpp-physics - #branch = main - url = https://github.com/grantfirl/ccpp-physics - branch = ufs-dev-PR75 + url = https://github.com/NCAR/ccpp-physics + branch = main [submodule "CMakeModules"] path = CMakeModules url = https://github.com/noaa-emc/CMakeModules diff --git a/ccpp/physics b/ccpp/physics index 63ee841b8..867eff449 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 63ee841b85a0a71eabb9ccb18fb8963032981fe3 +Subproject commit 867eff449feaf33d1face1c649aea8fe24c88686 diff --git a/scm/doc/TechGuide/chap_quick.tex b/scm/doc/TechGuide/chap_quick.tex index 451613b59..79c539463 100644 --- a/scm/doc/TechGuide/chap_quick.tex +++ b/scm/doc/TechGuide/chap_quick.tex @@ -99,19 +99,15 @@ \subsection{Compilers} release website (\url{https://dtcenter.org/community-code/common-community-physics-package-ccpp/download}). \subsection{Using Existing Libraries on Preconfigured Platforms}\label{section: use_preconfigured_platforms} -Platform-specific scripts are provided to load modules and set the user environment for preconfigured platforms. These scripts load compiler modules (Fortran 2008-compliant), the NetCDF module, Python environment, etc. and set compiler and environment variables. From the top-level code directory (\execout{ccpp-scm} by default), source the correct script for your platform and shell. For \textit{t/csh} shells, -\begin{lstlisting}[language=csh] -source scm/etc/Hera_setup_intel.csh -source scm/etc/Cheyenne_setup_gnu.csh -source scm/etc/Cheyenne_setup_intel.csh -\end{lstlisting} -For bourne/bash shells, -\begin{lstlisting}[language=bash] -. scm/etc/Hera_setup_intel.sh -. scm/etc/Cheyenne_setup_gnu.sh -. scm/etc/Cheyenne_setup_intel.sh +Because the SCM can be built using the so-called \href{https://ufs-weather-model.readthedocs.io/en/latest/Glossary.html#term-spack-stack}{"spack-stack" libraries} maintained for the UFS Weather Model effort, there are many platforms where the SCM can be built using those existing libraries. This can be done by loading provided modules in the \execout{scm/etc/modules} directory (must be done from the top-level "ccpp-scm" directory; otherwise the \execout{module use} command should point to the corresponding absolute path): + +\begin{lstlisting}[language=sh] +module purge +module use scm/etc/modules +module load [machine]_[compiler] \end{lstlisting} +View the contents of the \execout{scm/etc/modules} directory to see if your machine/compiler combination is supported. As of this writing, modulefiles are provided for Intel and GNU compilers on the NCAR machine Derecho, the NOAA machines Hera and Jet, and the NOAA/MSU machine Orion. Loading these modules will set up all the needed compilers, libraries, and other programs needed for building, as well as the python libraries needed for both building and running the SCM. \subsection{Installing Libraries on Non-preconfigured Platforms}\label{section: setup_supported_platforms} For users on supported platforms such as generic Linux or macOS systems that have not been preconfigured, the \execout{hpc-stack} project is suggested for installing prerequisite libraries. Visit \url{https://github.com/NOAA-EMC/hpc-stack} for instructions for installing prerequisite libraries via \execout{hpc-stack} in their docs directory. UFS users who already installed libraries via the \execout{hpc-stack} package only need to set the compiler (\execout{CC, CXX, FC}), NetCDF (\execout{NetCDF\_ROOT}), and \execout{bacio}, \execout{sp} and \execout{w3emc} (\execout{bacio\_ROOT}, \execout{sp\_ROOT}, \execout{w3emc\_ROOT}) environment variables to point to their installation paths in order to compile the SCM. diff --git a/scm/doc/TechGuide/chap_repo.tex b/scm/doc/TechGuide/chap_repo.tex index 4c61361b8..f28f0588e 100644 --- a/scm/doc/TechGuide/chap_repo.tex +++ b/scm/doc/TechGuide/chap_repo.tex @@ -26,32 +26,21 @@ \section{What is included in the repository?} .3 Dockerfile\DTcomment{contains Docker instructions for building the CCPP SCM image}. .2 README.md. .2 scm/. - .3 bin/\DTcomment{build directory (initially empty; populated by cmake)}. .3 data/\DTcomment{build directory (most data directories populated by contrib/get\_all\_static\_data.sh)}. - .4 comparison\_data/\DTcomment{initially empty; contains data with which to compare SCM output}. - .4 physics\_input\_data/\DTcomment{initially empty; contains data needed by the CCPP physics}. - .4 processed\_case\_input/\DTcomment{initially empty; contains initialization and forcing data for cases}. - .4 raw\_case\_input/\DTcomment{initially empty; contains case data to be processed by scripts}. .4 vert\_coord\_data/\DTcomment{contains data to calculate vertical coordinates (from GSM-based GFS only)}. .3 doc/\DTcomment{contains this User's/Technical Guide}. .4 TechGuide/\DTcomment{contains LaTeX for this User's Guide}. .3 etc/\DTcomment{contains case configuration, machine setup scripts, and plotting scripts}. .4 case\_config/\DTcomment{contains case configuration files}. .4 CENTOS\_docker\_setup.sh\DTcomment{contains machine setup for Docker container}. - .4 Cheyenne\_setup\_gnu.csh\DTcomment{setup script for Cheyenne HPC for csh, tcsh}. - .4 Cheyenne\_setup\_gnu.sh\DTcomment{setup script for Cheyenne HPC for sh, bash}. - .4 Cheyenne\_setup\_intel.csh\DTcomment{setup script for Cheyenne HPC for csh, tcsh}. - .4 Cheyenne\_setup\_intel.sh\DTcomment{setup script for Cheyenne HPC for sh, bash}. .4 Desktop\_setup\_gfortran.csh\DTcomment{setup script for Mac Desktop for csh, tcsh}. .4 Desktop\_setup\_gfortran.sh\DTcomment{setup script for Mac Desktop for sh, bash}. - .4 Hera\_setup\_intel.csh\DTcomment{setup script for Theia HPC for csh, tcsh}. - .4 Hera\_setup\_intel.sh\DTcomment{setup script for Theia HPC for sh, bash}. + .4 modules/\DTcomment{Directory containing modulefiles for building on HPC environments (see section \ref{section: use_preconfigured_platforms})}. .4 scm\_qsub\_example.py\DTcomment{example QSUB run script}. .4 scm\_slurm\_example.py\DTcomment{example SLURM run script}. .4 scripts/\DTcomment{Python scripts for setting up cases and plotting}. .5 plot\_configs/\DTcomment{plot configuration files}. .4 tracer\_config\DTcomment{tracer configuration files}. .3 LICENSE.txt. - .3 run/\DTcomment{initially empty; populated by run\_scm.py}. .3 src/\DTcomment{source code for SCM infrastructure, Python run script, CMakeLists.txt for the SCM, example multirun setup files, suite\_info.py}. }} diff --git a/scm/etc/modules/derecho_gnu.lua b/scm/etc/modules/derecho_gnu.lua new file mode 100644 index 000000000..be07155c4 --- /dev/null +++ b/scm/etc/modules/derecho_gnu.lua @@ -0,0 +1,30 @@ +help([[ +This module loads libraries for building the CCPP Single-Column Model on +the CISL machine Derecho (Cray) using Intel-classic-2023.0.0 +]]) + +whatis([===[Loads spack-stack libraries needed for building the CCPP SCM on Derecho with GNU compilers]===]) + +setenv("LMOD_TMOD_FIND_FIRST","yes") +load("ncarenv/23.09") +load("cmake/3.26.3") + +prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/spack-stack/derecho/modulefiles") +prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core") + +load("stack-gcc/12.2.0") +load("stack-cray-mpich/8.1.25") +load("stack-python/3.10.8") +load("py-f90nml") +load("py-netcdf4/1.5.8") + +load("netcdf-c/4.9.2") +load("netcdf-fortran/4.6.0") +load("bacio/2.4.1") +load("sp/2.3.3") +load("w3emc") + +setenv("CMAKE_C_COMPILER","mpicc") +setenv("CMAKE_CXX_COMPILER","mpicxx") +setenv("CMAKE_Fortran_COMPILER","mpif90") +setenv("CMAKE_Platform","derecho.gnu") diff --git a/scm/etc/modules/derecho_intel.lua b/scm/etc/modules/derecho_intel.lua new file mode 100644 index 000000000..37f784bfe --- /dev/null +++ b/scm/etc/modules/derecho_intel.lua @@ -0,0 +1,30 @@ +help([[ +This module loads libraries for building the CCPP Single-Column Model on +the CISL machine Derecho (Cray) using Intel-classic-2023.0.0 +]]) + +whatis([===[Loads spack-stack libraries needed for building the CCPP SCM on Derecho with Intel compilers]===]) + +setenv("LMOD_TMOD_FIND_FIRST","yes") +load("ncarenv/23.09") +load("cmake/3.26.3") + +prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/spack-stack/derecho/modulefiles") +prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core") + +load("stack-intel/2021.10.0") +load("stack-cray-mpich/8.1.25") +load("stack-python/3.10.8") +load("py-f90nml") +load("py-netcdf4/1.5.8") + +load("netcdf-c/4.9.2") +load("netcdf-fortran/4.6.0") +load("bacio/2.4.1") +load("sp/2.3.3") +load("w3emc") + +setenv("CMAKE_C_COMPILER","cc") +setenv("CMAKE_CXX_COMPILER","CC") +setenv("CMAKE_Fortran_COMPILER","ftn") +setenv("CMAKE_Platform","derecho.intel") diff --git a/scm/etc/modules/hera_gnu.lua b/scm/etc/modules/hera_gnu.lua new file mode 100644 index 000000000..23867c4ac --- /dev/null +++ b/scm/etc/modules/hera_gnu.lua @@ -0,0 +1,30 @@ +help([[ +This module loads libraries for building the CCPP Single-Column Model on +the NOAA RDHPC machine Hera using GNU 9.2.0 +]]) + +whatis([===[Loads libraries needed for building the CCPP SCM on Hera with GNU compilers ]===]) + +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") + +load("cmake/3.20.1") +load("miniconda/3.9.12") + +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core") + +load("stack-gcc/9.2.0") +load("stack-openmpi/4.1.5") +load("stack-python/3.10.8") +load("py-f90nml") +load("py-netcdf4/1.5.8") + +load("netcdf-c/4.9.2") +load("netcdf-fortran/4.6.0") +load("bacio/2.4.1") +load("sp/2.3.3") +load("w3emc") + +setenv("CMAKE_C_COMPILER","mpicc") +setenv("CMAKE_CXX_COMPILER","mpicxx") +setenv("CMAKE_Fortran_COMPILER","mpif90") +setenv("CMAKE_Platform","hera.gnu") diff --git a/scm/etc/modules/hera_intel.lua b/scm/etc/modules/hera_intel.lua new file mode 100644 index 000000000..d84db665c --- /dev/null +++ b/scm/etc/modules/hera_intel.lua @@ -0,0 +1,29 @@ +help([[ +This module loads libraries for building the CCPP Single-Column Model on +the NOAA RDHPC machine Hera using Intel-2021.5.0 +]]) + +whatis([===[Loads libraries needed for building the CCPP SCM on Hera with Intel compilers ]===]) + +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") +load("cmake/3.20.1") +load("miniconda/3.9.12") + +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core") + +load("stack-intel/2021.5.0") +load("stack-intel-oneapi-mpi/2021.5.1") +load("stack-python/3.10.8") +load("py-f90nml") +load("py-netcdf4/1.5.8") + +load("netcdf-c/4.9.2") +load("netcdf-fortran/4.6.0") +load("bacio/2.4.1") +load("sp/2.3.3") +load("w3emc") + +setenv("CMAKE_C_COMPILER","mpiicc") +setenv("CMAKE_CXX_COMPILER","mpiicpc") +setenv("CMAKE_Fortran_COMPILER","mpiifort") +setenv("CMAKE_Platform","hera.intel") diff --git a/scm/etc/modules/jet_gnu.lua b/scm/etc/modules/jet_gnu.lua new file mode 100644 index 000000000..96d5a1c97 --- /dev/null +++ b/scm/etc/modules/jet_gnu.lua @@ -0,0 +1,31 @@ +help([[ +This module loads libraries for building the CCPP Single-Column Model on +the NOAA RDHPC machine Jet using GNU 9.2.0 +]]) + +whatis([===[Loads libraries needed for building the CCPP SCM on Jet with GNU compilers ]===]) + +prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/spack-stack/modulefiles") + +load("cmake/3.26.4") +load("miniconda/3.9.12") + +prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/role.epic/modulefiles") +prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core") + +load("stack-gcc/9.2.0") +load("stack-openmpi/3.1.4") +load("stack-python/3.10.8") +load("py-f90nml") +load("py-netcdf4/1.5.8") + +load("netcdf-c/4.9.2") +load("netcdf-fortran/4.6.0") +load("bacio/2.4.1") +load("sp/2.3.3") +load("w3emc") + +setenv("CMAKE_C_COMPILER","mpicc") +setenv("CMAKE_CXX_COMPILER","mpicxx") +setenv("CMAKE_Fortran_COMPILER","mpif90") +setenv("CMAKE_Platform","jet.gnu") diff --git a/scm/etc/modules/jet_intel.lua b/scm/etc/modules/jet_intel.lua new file mode 100644 index 000000000..8c232ed26 --- /dev/null +++ b/scm/etc/modules/jet_intel.lua @@ -0,0 +1,31 @@ +help([[ +This module loads libraries for building the CCPP Single-Column Model on +the NOAA RDHPC machine Jet using Intel-2021.5.0 +]]) + +whatis([===[Loads libraries needed for building the CCPP SCM on Jet with Intel compilers ]===]) + +prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/spack-stack/modulefiles") + +load("cmake/3.26.4") +load("miniconda/3.9.12") + +prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/role.epic/modulefiles") +prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core") + +load("stack-intel/2021.5.0") +load("stack-intel-oneapi-mpi/2021.5.1") +load("stack-python/3.10.8") +load("py-f90nml") +load("py-netcdf4/1.5.8") + +load("netcdf-c/4.9.2") +load("netcdf-fortran/4.6.0") +load("bacio/2.4.1") +load("sp/2.3.3") +load("w3emc") + +setenv("CMAKE_C_COMPILER","mpiicc") +setenv("CMAKE_CXX_COMPILER","mpiicpc") +setenv("CMAKE_Fortran_COMPILER","mpiifort") +setenv("CMAKE_Platform","jet.intel") diff --git a/scm/etc/modules/orion_gnu.lua b/scm/etc/modules/orion_gnu.lua new file mode 100644 index 000000000..17eb12b09 --- /dev/null +++ b/scm/etc/modules/orion_gnu.lua @@ -0,0 +1,30 @@ +help([[ +This module loads libraries for building the CCPP Single-Column Model on +the NOAA RDHPC machine orion using GNU 10.2.0 +]]) + +whatis([===[Loads libraries needed for building the CCPP SCM on Orion with GNU compilers ]===]) + +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/modulefiles") + +load("cmake/3.22.1") +load("python/3.9.2") + +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core") + +load("stack-gcc/10.2.0") +load("stack-openmpi/4.0.4") +load("stack-python/3.10.8") +load("py-f90nml") +load("py-netcdf4/1.5.8") + +load("netcdf-c/4.9.2") +load("netcdf-fortran/4.6.0") +load("bacio/2.4.1") +load("sp/2.3.3") +load("w3emc") + +setenv("CMAKE_C_COMPILER","mpicc") +setenv("CMAKE_CXX_COMPILER","mpicxx") +setenv("CMAKE_Fortran_COMPILER","mpif90") +setenv("CMAKE_Platform","orion.gnu") diff --git a/scm/etc/modules/orion_intel.lua b/scm/etc/modules/orion_intel.lua new file mode 100644 index 000000000..005119c5d --- /dev/null +++ b/scm/etc/modules/orion_intel.lua @@ -0,0 +1,30 @@ +help([[ +This module loads libraries for building the CCPP Single-Column Model on +the NOAA RDHPC machine Orion using Intel-2021.5.0 +]]) + +whatis([===[Loads libraries needed for building the CCPP SCM on Orion with Intel compilers ]===]) + +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/modulefiles") + +load("cmake/3.22.1") +load("python/3.9.2") + +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core") + +load("stack-intel/2022.0.2") +load("stack-intel-oneapi-mpi/2021.5.1") +load("stack-python/3.10.8") +load("py-f90nml") +load("py-netcdf4/1.5.8") + +load("netcdf-c/4.9.2") +load("netcdf-fortran/4.6.0") +load("bacio/2.4.1") +load("sp/2.3.3") +load("w3emc") + +setenv("CMAKE_C_COMPILER","mpiicc") +setenv("CMAKE_CXX_COMPILER","mpiicpc") +setenv("CMAKE_Fortran_COMPILER","mpiifort") +setenv("CMAKE_Platform","orion.intel") diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index b7c15b010..b952680b6 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -30,6 +30,9 @@ endif() message (STATUS "Running ccpp_prebuild.py for CCPP") # Make the directories where the ccpp_prebuild.py script wants to write caps and make snippets file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/physics) +if (NOT EXISTS "${CMAKE_SOURCE_DIR}/../../ccpp/framework/scripts/ccpp_prebuild.py") + message( FATAL_ERROR "ccpp_prebuild.py script does not exist, did you check out the code recursively?" ) +endif() if (${CMAKE_BUILD_TYPE} MATCHES "Debug") execute_process( COMMAND ccpp/framework/scripts/ccpp_prebuild.py --config=ccpp/config/ccpp_prebuild_config.py ${_ccpp_suites_arg} --builddir=${CMAKE_CURRENT_BINARY_DIR} --debug --verbose diff --git a/scm/src/run_scm.py b/scm/src/run_scm.py index 0d04edd25..20cfbce6a 100755 --- a/scm/src/run_scm.py +++ b/scm/src/run_scm.py @@ -758,13 +758,16 @@ def main(): verbose, levels, npz_type, vert_coord_file, case_data_dir, n_itt_out, \ n_itt_diag, run_dir, bin_dir, timestep) = parse_arguments() + setup_logging(verbose) + global SCM_ROOT SCM_ROOT = os.getenv('SCM_ROOT') if SCM_ROOT is None: - message = 'The SCM_ROOT environment variable is not set. Please set the SCM_ROOT environment variable to the top-level path where the model was cloned.' - logging.critical(message) - raise Exception(message) - + # Note: os.path.dirname() is the platform-agnostic way to get a parent directory + SCM_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + message = f"The SCM_ROOT environment variable is not set. Using default value:\n{SCM_ROOT}" + logging.warning(message) + global SCM_BIN if bin_dir: SCM_BIN = bin_dir @@ -782,14 +785,11 @@ def main(): global EXECUTABLE EXECUTABLE = os.path.join(SCM_RUN, EXECUTABLE_NAME) - setup_logging(verbose) - # Debugger if use_gdb: gdb = find_gdb() else: gdb = None - if (file != None): logging.info('SCM-run: Using {} to loop through defined runs'.format(file)) try: