diff --git a/.github/actions/setup-spack-stack/action.yaml b/.github/actions/setup-spack-stack/action.yaml index 23b91725d..728e9afb6 100644 --- a/.github/actions/setup-spack-stack/action.yaml +++ b/.github/actions/setup-spack-stack/action.yaml @@ -33,10 +33,13 @@ runs: - name: os-setup shell: bash run: | - if [ "$RUNNER_OS" == "Linux" ]; then + if [[ "$RUNNER_OS" == "Linux" ]]; then # Install Curl headers. Executable exists by default in spack external find. sudo apt-get install libcurl4-openssl-dev + # Install git-lfs to avoid compilation errors of "go" with Intel + sudo apt-get install git-lfs + if [[ "${{ inputs.compiler }}" == "intel"* ]]; then cd /tmp wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB @@ -47,8 +50,8 @@ runs: sudo apt-get install intel-oneapi-dev-utilities intel-oneapi-openmp intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic intel-oneapi-mpi-devel echo "source /opt/intel/oneapi/setvars.sh" >> ~/.bash_profile fi - elif [ "$RUNNER_OS" == "macOS" ]; then - echo "" + elif [[ "$RUNNER_OS" == "macOS" ]]; then + echo " " fi # Install Python poetry to avoid install errors in spack @@ -94,9 +97,10 @@ runs: if [[ "${{ inputs.mpi }}" == "openmpi"* ]]; then wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-${OPENMPI_VERSION}.tar.gz - tar -xzf openmpi-${OPENMP_VERSION}.tar.gz - cd openmpi-${OPENMP_VERSION} - ./configure --prefix=$HOME/mpi --enable-mpi-fortran --enable-mpi-cxx + tar -xzf openmpi-${OPENMPI_VERSION}.tar.gz + cd openmpi-${OPENMPI_VERSION} + # --with-hwloc=internal --with-libevent=internal : https://www.open-mpi.org/faq/?category=building#libevent-or-hwloc-errors-when-linking-fortran + ./configure --prefix=$HOME/mpi --enable-mpi-fortran --enable-mpi-cxx --with-hwloc=internal --with-libevent=internal make -j2 make install elif [[ "${{ inputs.mpi }}" == "mpich"* ]]; then @@ -122,6 +126,8 @@ runs: # So Spack can find external MPI export "PATH=$HOME/mpi/bin:${PATH}" + export MPICH_VERSION="3.4.3" + export OPENMPI_VERSION="4.1.3" # LLVM Clang not in PATH, search for it specifically # Then, Fortran compilers are null, so set to gfortran @@ -132,11 +138,13 @@ runs: sed -i".bak" "s|fc: null|fc: /usr/local/bin/gfortran-10|" ${SPACK_ENV}/spack.yaml elif [[ "${{ inputs.compiler }}" == "intel"* ]]; then spack compiler add /opt/intel/oneapi/compiler/latest/linux/bin/intel64 + # Workaround for error "libimf.so cannot be found": + sed -i 's#environment: {}#environment: {prepend_path: {LD_LIBRARY_PATH: /opt/intel/oneapi/compiler/latest/linux/compiler/lib/intel64_lin}}#g' envs/${{ inputs.app }}/spack.yaml else spack compiler find fi - # No external find for intel-oneapi-mpi + # No external find for intel-oneapi-mpi, and problems finding other MPI libraries as well # And no way to add object entry to list using "spack config add" # Add this first so "spack config add packages:" will append to this entry if [[ "${{ inputs.mpi }}" == "intel-oneapi-mpi" ]]; then @@ -148,9 +156,32 @@ runs: echo " externals:" >> ${SPACK_ENV}/spack.yaml echo " - spec: intel-oneapi-mpi@${impi_ver}" >> ${SPACK_ENV}/spack.yaml echo " prefix: /opt/intel/oneapi" >> ${SPACK_ENV}/spack.yaml + elif [[ "${{ inputs.mpi }}" == "mpich"* ]]; then + mpich_ver=${MPICH_VERSION} + echo "" >> ${SPACK_ENV}/spack.yaml + echo " packages:" >> ${SPACK_ENV}/spack.yaml + echo " mpich:" >> ${SPACK_ENV}/spack.yaml + echo " buildable: False" >> ${SPACK_ENV}/spack.yaml + echo " externals:" >> ${SPACK_ENV}/spack.yaml + echo " - spec: mpich@${mpich_ver}" >> ${SPACK_ENV}/spack.yaml + echo " prefix: $HOME/mpi" >> ${SPACK_ENV}/spack.yaml + elif [[ "${{ inputs.mpi }}" == "openmpi"* ]]; then + openmpi_ver=${OPENMPI_VERSION} + echo "" >> ${SPACK_ENV}/spack.yaml + echo " packages:" >> ${SPACK_ENV}/spack.yaml + echo " openmpi:" >> ${SPACK_ENV}/spack.yaml + echo " buildable: False" >> ${SPACK_ENV}/spack.yaml + echo " externals:" >> ${SPACK_ENV}/spack.yaml + echo " - spec: openmpi@${openmpi_ver}" >> ${SPACK_ENV}/spack.yaml + echo " prefix: $HOME/mpi" >> ${SPACK_ENV}/spack.yaml fi - spack external find + # Spack external find is by default only looking for build-tools. Either + # search for additional packages explicitly, or fall back to the old + # behavior to find all external packages. + #spack external find + #spack external find git-lfs openmpi mpich + spack external find --all - name: update-bash-profile shell: bash @@ -185,12 +216,6 @@ runs: spack config add "modules:default:tcl:whitelist:[${{ inputs.mpi }}]" spack config add "modules:default:lmod:whitelist:[${{ inputs.mpi }}]" - # Currently, this is the case for all workflows - wrap in if statement - # when this changes in the future - # Whitelist the mpi providers so that spack creates the modules for them - spack config add "modules:default:tcl:whitelist:[${{ inputs.mpi }}]" - spack config add "modules:default:lmod:whitelist:[${{ inputs.mpi }}]" - if [[ "$RUNNER_OS" == "Linux" ]]; then echo "" elif [[ "$RUNNER_OS" == "macOS" ]]; then @@ -214,13 +239,7 @@ runs: cd ${{ inputs.path }} source setup.sh spack env activate ${{ inputs.path }}/envs/${{ inputs.app }} - if [[ "${{ inputs.compiler }}" == "intel"* ]]; then - source /opt/intel/oneapi/setvars.sh - # Pass --dirty for Intel compiler environment to fix missing LD_LIBRARY_PATH - spack install --dirty --fail-fast - else - spack install --fail-fast - fi + spack install --fail-fast - name: create-meta-modules shell: bash diff --git a/.github/workflows/create-spack-mirror.yaml b/.github/workflows/create-spack-mirror.yaml index 7492420d3..1bb60e4c5 100644 --- a/.github/workflows/create-spack-mirror.yaml +++ b/.github/workflows/create-spack-mirror.yaml @@ -33,10 +33,8 @@ jobs: spack add clingo spack concretize spack mirror create -a - # upload-artifact does not allow filenames that begin with '?' - rm -f cache/source_cache/autoconf/\?id=05972f49ee632cd98057a3caf82ebfb9574846da-eaa3f69 - rm -f cache/source_cache/boost/attachment.cgi?id=356970-b6f6ce6 - rm -f cache/source_cache/py-scipy/extern_decls.patch?id=711fe05025795e44b84233e065d240859ccae5bd-5433f60 + # upload-artifact does not allow filenames that contain '?' + rm -vf `find cache/source_cache -name '*\?*'` - name: upload-mirror uses: actions/upload-artifact@v2 diff --git a/.github/workflows/macos-apple-clang.yaml b/.github/workflows/macos-apple-clang.yaml index f9bb2bc20..9735e351a 100644 --- a/.github/workflows/macos-apple-clang.yaml +++ b/.github/workflows/macos-apple-clang.yaml @@ -23,6 +23,6 @@ jobs: uses: ./.github/actions/setup-spack-stack with: app: ${{ matrix.app }} - compiler: apple-clang@13 - mpi: mpich + compiler: apple-clang + mpi: openmpi path: ${{ github.workspace }} diff --git a/.github/workflows/macos-llvm-clang.yaml b/.github/workflows/macos-llvm-clang.yaml index b5e3ed128..b8e082d5e 100644 --- a/.github/workflows/macos-llvm-clang.yaml +++ b/.github/workflows/macos-llvm-clang.yaml @@ -24,5 +24,5 @@ jobs: with: app: ${{ matrix.app }} compiler: clang - mpi: mpich + mpi: openmpi path: ${{ github.workspace }} diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index b77f8339f..a849acd86 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -59,11 +59,11 @@ esmf: version: [8.2.0] variants: ~xerces ~pnetcdf - # Attention - when updating also check macos-monterey-llvm-clang-mpich-reference site config + # Attention - when updating also check macos-monterey-llvm-clang-openmpi-reference site config fms: version: [2022.01] variants: +64bit +enable_quad_precision +gfs_phys +openmp +pic - # Attention - when updating also check macos-monterey-llvm-clang-mpich-reference site config + # Attention - when updating also check macos-monterey-llvm-clang-openmpi-reference site config fms-jcsda: version: [release-stable] variants: +64bit +enable_quad_precision +gfs_phys +openmp +pic diff --git a/configs/repos/jedi/packages/jedi-cmake/package.py b/configs/repos/jedi/packages/jedi-cmake/package.py index 297ced09d..06a3788c8 100644 --- a/configs/repos/jedi/packages/jedi-cmake/package.py +++ b/configs/repos/jedi/packages/jedi-cmake/package.py @@ -9,8 +9,8 @@ class JediCmake(CMakePackage): """CMake/ecbuild toolchains to facilitate portability on different systems.""" - homepage = "https://github.com/JCSDA-internal/jedi-cmake" - git = "https://github.com/JCSDA-internal/jedi-cmake.git" + homepage = "https://github.com/JCSDA/jedi-cmake" + git = "https://github.com/JCSDA/jedi-cmake.git" url = "https://github.com/JCSDA/jedi-cmake/archive/refs/tags/1.3.0.tar.gz" maintainers = ['climbfuji', 'rhoneyager'] diff --git a/configs/sites/macos-monterey-llvm-clang-mpich-reference/README.md b/configs/sites/macos-monterey-llvm-clang-openmpi-reference/README.md similarity index 99% rename from configs/sites/macos-monterey-llvm-clang-mpich-reference/README.md rename to configs/sites/macos-monterey-llvm-clang-openmpi-reference/README.md index 07c1b3864..1394ceeeb 100644 --- a/configs/sites/macos-monterey-llvm-clang-mpich-reference/README.md +++ b/configs/sites/macos-monterey-llvm-clang-openmpi-reference/README.md @@ -27,7 +27,7 @@ This step is only required on the new arch64 systems that are equipped with a Ap brew install coreutils@9.0 brew install gcc@11.2.0 brew install llvm13.0.0 -brew install mpich@3.4.3 +brew install openmpi@4.1.2 brew install python@3.9.10 brew install git@2.34.1 brew install git-lfs@3.0.2 diff --git a/configs/sites/macos-monterey-llvm-clang-mpich-reference/compilers.yaml b/configs/sites/macos-monterey-llvm-clang-openmpi-reference/compilers.yaml similarity index 100% rename from configs/sites/macos-monterey-llvm-clang-mpich-reference/compilers.yaml rename to configs/sites/macos-monterey-llvm-clang-openmpi-reference/compilers.yaml diff --git a/configs/sites/macos-monterey-llvm-clang-mpich-reference/config.yaml b/configs/sites/macos-monterey-llvm-clang-openmpi-reference/config.yaml similarity index 100% rename from configs/sites/macos-monterey-llvm-clang-mpich-reference/config.yaml rename to configs/sites/macos-monterey-llvm-clang-openmpi-reference/config.yaml diff --git a/configs/sites/macos-monterey-llvm-clang-mpich-reference/modules.yaml b/configs/sites/macos-monterey-llvm-clang-openmpi-reference/modules.yaml similarity index 100% rename from configs/sites/macos-monterey-llvm-clang-mpich-reference/modules.yaml rename to configs/sites/macos-monterey-llvm-clang-openmpi-reference/modules.yaml diff --git a/configs/sites/macos-monterey-llvm-clang-mpich-reference/packages.yaml b/configs/sites/macos-monterey-llvm-clang-openmpi-reference/packages.yaml similarity index 99% rename from configs/sites/macos-monterey-llvm-clang-mpich-reference/packages.yaml rename to configs/sites/macos-monterey-llvm-clang-openmpi-reference/packages.yaml index 4fa4094a8..4691404bd 100644 --- a/configs/sites/macos-monterey-llvm-clang-mpich-reference/packages.yaml +++ b/configs/sites/macos-monterey-llvm-clang-openmpi-reference/packages.yaml @@ -2,7 +2,7 @@ packages: all: compiler:: [clang@13.0.0] providers: - mpi:: [mpich@3.4.3] + mpi:: [openmpi@4.1.2] ### MPI, Python, MKL mpi: diff --git a/spack b/spack index fb489ae54..e02d348cb 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit fb489ae5430492d5047c6403a119a9f0e75f1864 +Subproject commit e02d348cbcc51be65ad811c3e4bf76158f2c6603