From 9a60e644ca6dccab553f66b6f6ae9bc7161c9586 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 4 Apr 2022 15:03:21 -0600 Subject: [PATCH 01/24] Switch to openmpi on macOS --- configs/common/packages.yaml | 4 ++-- .../README.md | 2 +- .../compilers.yaml | 0 .../config.yaml | 0 .../modules.yaml | 0 .../packages.yaml | 2 +- 6 files changed, 4 insertions(+), 4 deletions(-) rename configs/sites/{macos-monterey-llvm-clang-mpich-reference => macos-monterey-llvm-clang-openmpi-reference}/README.md (99%) rename configs/sites/{macos-monterey-llvm-clang-mpich-reference => macos-monterey-llvm-clang-openmpi-reference}/compilers.yaml (100%) rename configs/sites/{macos-monterey-llvm-clang-mpich-reference => macos-monterey-llvm-clang-openmpi-reference}/config.yaml (100%) rename configs/sites/{macos-monterey-llvm-clang-mpich-reference => macos-monterey-llvm-clang-openmpi-reference}/modules.yaml (100%) rename configs/sites/{macos-monterey-llvm-clang-mpich-reference => macos-monterey-llvm-clang-openmpi-reference}/packages.yaml (99%) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index b77f8339f..a849acd86 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -59,11 +59,11 @@ esmf: version: [8.2.0] variants: ~xerces ~pnetcdf - # Attention - when updating also check macos-monterey-llvm-clang-mpich-reference site config + # Attention - when updating also check macos-monterey-llvm-clang-openmpi-reference site config fms: version: [2022.01] variants: +64bit +enable_quad_precision +gfs_phys +openmp +pic - # Attention - when updating also check macos-monterey-llvm-clang-mpich-reference site config + # Attention - when updating also check macos-monterey-llvm-clang-openmpi-reference site config fms-jcsda: version: [release-stable] variants: +64bit +enable_quad_precision +gfs_phys +openmp +pic diff --git a/configs/sites/macos-monterey-llvm-clang-mpich-reference/README.md b/configs/sites/macos-monterey-llvm-clang-openmpi-reference/README.md similarity index 99% rename from configs/sites/macos-monterey-llvm-clang-mpich-reference/README.md rename to configs/sites/macos-monterey-llvm-clang-openmpi-reference/README.md index 07c1b3864..1394ceeeb 100644 --- a/configs/sites/macos-monterey-llvm-clang-mpich-reference/README.md +++ b/configs/sites/macos-monterey-llvm-clang-openmpi-reference/README.md @@ -27,7 +27,7 @@ This step is only required on the new arch64 systems that are equipped with a Ap brew install coreutils@9.0 brew install gcc@11.2.0 brew install llvm13.0.0 -brew install mpich@3.4.3 +brew install openmpi@4.1.2 brew install python@3.9.10 brew install git@2.34.1 brew install git-lfs@3.0.2 diff --git a/configs/sites/macos-monterey-llvm-clang-mpich-reference/compilers.yaml b/configs/sites/macos-monterey-llvm-clang-openmpi-reference/compilers.yaml similarity index 100% rename from configs/sites/macos-monterey-llvm-clang-mpich-reference/compilers.yaml rename to configs/sites/macos-monterey-llvm-clang-openmpi-reference/compilers.yaml diff --git a/configs/sites/macos-monterey-llvm-clang-mpich-reference/config.yaml b/configs/sites/macos-monterey-llvm-clang-openmpi-reference/config.yaml similarity index 100% rename from configs/sites/macos-monterey-llvm-clang-mpich-reference/config.yaml rename to configs/sites/macos-monterey-llvm-clang-openmpi-reference/config.yaml diff --git a/configs/sites/macos-monterey-llvm-clang-mpich-reference/modules.yaml b/configs/sites/macos-monterey-llvm-clang-openmpi-reference/modules.yaml similarity index 100% rename from configs/sites/macos-monterey-llvm-clang-mpich-reference/modules.yaml rename to configs/sites/macos-monterey-llvm-clang-openmpi-reference/modules.yaml diff --git a/configs/sites/macos-monterey-llvm-clang-mpich-reference/packages.yaml b/configs/sites/macos-monterey-llvm-clang-openmpi-reference/packages.yaml similarity index 99% rename from configs/sites/macos-monterey-llvm-clang-mpich-reference/packages.yaml rename to configs/sites/macos-monterey-llvm-clang-openmpi-reference/packages.yaml index 4fa4094a8..4691404bd 100644 --- a/configs/sites/macos-monterey-llvm-clang-mpich-reference/packages.yaml +++ b/configs/sites/macos-monterey-llvm-clang-openmpi-reference/packages.yaml @@ -2,7 +2,7 @@ packages: all: compiler:: [clang@13.0.0] providers: - mpi:: [mpich@3.4.3] + mpi:: [openmpi@4.1.2] ### MPI, Python, MKL mpi: From 4ced3de8c850d6fc27a28037c60dd5b0011ae8fd Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 4 Apr 2022 15:58:51 -0600 Subject: [PATCH 02/24] Update .gitmodules and submodule pointer for spack for testing --- .gitmodules | 6 ++++-- spack | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 9cadf2a04..1f1002828 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,5 +2,7 @@ path = spack #url = https://github.com/spack/spack #branch = develop - url = https://github.com/NOAA-EMC/spack.git - branch = jcsda_emc_spack_stack + #url = https://github.com/NOAA-EMC/spack.git + #branch = jcsda_emc_spack_stack + url = https://github.com/climbfuji/spack.git + branch = feature/update_jcsda_emc_spack_stack_from_authoritative_20220404 diff --git a/spack b/spack index f7edb3eeb..ebfcd17f2 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit f7edb3eebb13f80d7ba4ead2f21be4c1a7eb6b43 +Subproject commit ebfcd17f29dbaa0d82f256a7199410ce866de5d9 From 9c9b0269c917efc11bb50fba7b5ed0c558200780 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 4 Apr 2022 16:00:54 -0600 Subject: [PATCH 03/24] Use openmpi for macos-llvm-clang and macos-apple-clang github workflows --- .github/workflows/macos-apple-clang.yaml | 2 +- .github/workflows/macos-llvm-clang.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macos-apple-clang.yaml b/.github/workflows/macos-apple-clang.yaml index f9bb2bc20..245bd44d3 100644 --- a/.github/workflows/macos-apple-clang.yaml +++ b/.github/workflows/macos-apple-clang.yaml @@ -24,5 +24,5 @@ jobs: with: app: ${{ matrix.app }} compiler: apple-clang@13 - mpi: mpich + mpi: openmpi path: ${{ github.workspace }} diff --git a/.github/workflows/macos-llvm-clang.yaml b/.github/workflows/macos-llvm-clang.yaml index b5e3ed128..b8e082d5e 100644 --- a/.github/workflows/macos-llvm-clang.yaml +++ b/.github/workflows/macos-llvm-clang.yaml @@ -24,5 +24,5 @@ jobs: with: app: ${{ matrix.app }} compiler: clang - mpi: mpich + mpi: openmpi path: ${{ github.workspace }} From 7ca1c752762cdfd0af899231ff3b75f0c22f1f6e Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 6 Apr 2022 20:51:34 -0600 Subject: [PATCH 04/24] Remove version identified from clang compiler in .github/workflows/macos-apple-clang.yaml --- .github/workflows/macos-apple-clang.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos-apple-clang.yaml b/.github/workflows/macos-apple-clang.yaml index 245bd44d3..9735e351a 100644 --- a/.github/workflows/macos-apple-clang.yaml +++ b/.github/workflows/macos-apple-clang.yaml @@ -23,6 +23,6 @@ jobs: uses: ./.github/actions/setup-spack-stack with: app: ${{ matrix.app }} - compiler: apple-clang@13 + compiler: apple-clang mpi: openmpi path: ${{ github.workspace }} From 352ea6d6973896bd98d17182b7eb528f90121654 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 6 Apr 2022 20:57:08 -0600 Subject: [PATCH 05/24] Bug fix in .github/actions/setup-spack-stack/action.yaml --- .github/actions/setup-spack-stack/action.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup-spack-stack/action.yaml b/.github/actions/setup-spack-stack/action.yaml index 61e3b9efd..6707779a3 100644 --- a/.github/actions/setup-spack-stack/action.yaml +++ b/.github/actions/setup-spack-stack/action.yaml @@ -94,8 +94,8 @@ runs: if [[ "${{ inputs.mpi }}" == "openmpi"* ]]; then wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-${OPENMPI_VERSION}.tar.gz - tar -xzf openmpi-${OPENMP_VERSION}.tar.gz - cd openmpi-${OPENMP_VERSION} + tar -xzf openmpi-${OPENMPI_VERSION}.tar.gz + cd openmpi-${OPENMPI_VERSION} ./configure --prefix=$HOME/mpi --enable-mpi-fortran --enable-mpi-cxx make -j2 make install From 72a628e5a2637b0bf3fde9565b90f37663bf8ca4 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 7 Apr 2022 16:11:01 -0600 Subject: [PATCH 06/24] Update spack submodule --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index ebfcd17f2..df4315699 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit ebfcd17f29dbaa0d82f256a7199410ce866de5d9 +Subproject commit df4315699da4b0e28bc818779e91d4cdb5205630 From b9003ff75629799cceb40234f11165c654995451 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 7 Apr 2022 16:34:47 -0600 Subject: [PATCH 07/24] Update spack submodule --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index df4315699..b629a8817 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit df4315699da4b0e28bc818779e91d4cdb5205630 +Subproject commit b629a8817832bd344ce421287e2d2e06e01873e4 From 0b5c687f4d34a2db18140ba264362e616b1ab533 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 7 Apr 2022 20:24:53 -0600 Subject: [PATCH 08/24] Use internal hwloc/libevent when building openmpi --- .github/actions/setup-spack-stack/action.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup-spack-stack/action.yaml b/.github/actions/setup-spack-stack/action.yaml index 6707779a3..d2996a35b 100644 --- a/.github/actions/setup-spack-stack/action.yaml +++ b/.github/actions/setup-spack-stack/action.yaml @@ -96,7 +96,8 @@ runs: wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-${OPENMPI_VERSION}.tar.gz tar -xzf openmpi-${OPENMPI_VERSION}.tar.gz cd openmpi-${OPENMPI_VERSION} - ./configure --prefix=$HOME/mpi --enable-mpi-fortran --enable-mpi-cxx + # --with-hwloc=internal --with-libevent=internal : https://www.open-mpi.org/faq/?category=building#libevent-or-hwloc-errors-when-linking-fortran + ./configure --prefix=$HOME/mpi --enable-mpi-fortran --enable-mpi-cxx --with-hwloc=internal --with-libevent=internal make -j2 make install elif [[ "${{ inputs.mpi }}" == "mpich"* ]]; then From fe28a8d005d382be00e5e30fe262f0e29777644b Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 8 Apr 2022 06:54:20 -0600 Subject: [PATCH 09/24] For debugging --- .github/actions/setup-spack-stack/action.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/setup-spack-stack/action.yaml b/.github/actions/setup-spack-stack/action.yaml index d2996a35b..a9af41e7d 100644 --- a/.github/actions/setup-spack-stack/action.yaml +++ b/.github/actions/setup-spack-stack/action.yaml @@ -126,6 +126,8 @@ runs: # So Spack can find external MPI export "PATH=$HOME/mpi/bin:${PATH}" + which mpicc + # LLVM Clang not in PATH, search for it specifically # Then, Fortran compilers are null, so set to gfortran # Add minimal compilers just to prevent other compilers from being used by accident From 4afe505f49a6b9aee9c2a8ec0841c0936c344877 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 8 Apr 2022 11:42:21 -0600 Subject: [PATCH 10/24] Bump mpi cache version number --- .github/actions/setup-spack-stack/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup-spack-stack/action.yaml b/.github/actions/setup-spack-stack/action.yaml index a9af41e7d..8ddeaa641 100644 --- a/.github/actions/setup-spack-stack/action.yaml +++ b/.github/actions/setup-spack-stack/action.yaml @@ -56,7 +56,7 @@ runs: - name: cache-mpi id: cache-mpi - uses: actions/cache@v2 + uses: actions/cache@v2.1 with: path: ~/mpi key: mpi-${{ inputs.mpi }}-${{ inputs.compiler }}.${{ runner.os }}2 From 5aa05d3310513820876058dd2affbf0e9ff682e8 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 8 Apr 2022 11:48:17 -0600 Subject: [PATCH 11/24] Temporarily change mpi version numbers for testing --- .github/actions/setup-spack-stack/action.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup-spack-stack/action.yaml b/.github/actions/setup-spack-stack/action.yaml index d1c558f20..5c7037e9d 100644 --- a/.github/actions/setup-spack-stack/action.yaml +++ b/.github/actions/setup-spack-stack/action.yaml @@ -56,7 +56,7 @@ runs: - name: cache-mpi id: cache-mpi - uses: actions/cache@v2.1 + uses: actions/cache@v2 with: path: ~/mpi key: mpi-${{ inputs.mpi }}-${{ inputs.compiler }}.${{ runner.os }}2 @@ -89,8 +89,8 @@ runs: export FFLAGS="-fallow-argument-mismatch" fi - export MPICH_VERSION="3.4.3" - export OPENMPI_VERSION="4.1.3" + export MPICH_VERSION="3.4.2" + export OPENMPI_VERSION="4.1.2" if [[ "${{ inputs.mpi }}" == "openmpi"* ]]; then wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-${OPENMPI_VERSION}.tar.gz From 21ce595c43bbae5c9a8960dbd5c469622c87d5df Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 8 Apr 2022 15:23:12 -0600 Subject: [PATCH 12/24] Next attempt to find MPI --- .github/actions/setup-spack-stack/action.yaml | 37 +++++++++++++++---- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/.github/actions/setup-spack-stack/action.yaml b/.github/actions/setup-spack-stack/action.yaml index 5c7037e9d..e7524099e 100644 --- a/.github/actions/setup-spack-stack/action.yaml +++ b/.github/actions/setup-spack-stack/action.yaml @@ -33,10 +33,13 @@ runs: - name: os-setup shell: bash run: | - if [ "$RUNNER_OS" == "Linux" ]; then + if [[ "$RUNNER_OS" == "Linux" ]]; then # Install Curl headers. Executable exists by default in spack external find. sudo apt-get install libcurl4-openssl-dev + # Install git-lfs to avoid compilation errors of "go" with Intel + sudo apt-get install git-lfs + if [[ "${{ inputs.compiler }}" == "intel"* ]]; then cd /tmp wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB @@ -47,8 +50,8 @@ runs: sudo apt-get install intel-oneapi-dev-utilities intel-oneapi-openmp intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic intel-oneapi-mpi-devel echo "source /opt/intel/oneapi/setvars.sh" >> ~/.bash_profile fi - elif [ "$RUNNER_OS" == "macOS" ]; then - echo "" + elif [[ "$RUNNER_OS" == "macOS" ]]; then + echo " " fi # Install Python poetry to avoid install errors in spack @@ -89,8 +92,8 @@ runs: export FFLAGS="-fallow-argument-mismatch" fi - export MPICH_VERSION="3.4.2" - export OPENMPI_VERSION="4.1.2" + export MPICH_VERSION="3.4.3" + export OPENMPI_VERSION="4.1.3" if [[ "${{ inputs.mpi }}" == "openmpi"* ]]; then wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-${OPENMPI_VERSION}.tar.gz @@ -123,8 +126,8 @@ runs: # So Spack can find external MPI export "PATH=$HOME/mpi/bin:${PATH}" - - which mpicc + export MPICH_VERSION="3.4.3" + export OPENMPI_VERSION="4.1.3" # LLVM Clang not in PATH, search for it specifically # Then, Fortran compilers are null, so set to gfortran @@ -139,7 +142,7 @@ runs: spack compiler find fi - # No external find for intel-oneapi-mpi + # No external find for intel-oneapi-mpi, and problems finding other MPI libraries as well # And no way to add object entry to list using "spack config add" # Add this first so "spack config add packages:" will append to this entry if [[ "${{ inputs.mpi }}" == "intel-oneapi-mpi" ]]; then @@ -151,6 +154,24 @@ runs: echo " externals:" >> ${SPACK_ENV}/spack.yaml echo " - spec: intel-oneapi-mpi@${impi_ver}" >> ${SPACK_ENV}/spack.yaml echo " prefix: /opt/intel/oneapi" >> ${SPACK_ENV}/spack.yaml + elif [[ "${{ inputs.mpi }}" == "mpich"* ]]; then + mpich_ver=${MPICH_VERSION} + echo "" >> ${SPACK_ENV}/spack.yaml + echo " packages:" >> ${SPACK_ENV}/spack.yaml + echo " mpich:" >> ${SPACK_ENV}/spack.yaml + echo " buildable: False" >> ${SPACK_ENV}/spack.yaml + echo " externals:" >> ${SPACK_ENV}/spack.yaml + echo " - spec: mpich@${mpich_ver}" >> ${SPACK_ENV}/spack.yaml + echo " prefix: $HOME/mpi" >> ${SPACK_ENV}/spack.yaml + elif [[ "${{ inputs.mpi }}" == "openmpi"* ]]; then + openmpi_ver=${OPENMPI_VERSION} + echo "" >> ${SPACK_ENV}/spack.yaml + echo " packages:" >> ${SPACK_ENV}/spack.yaml + echo " openmpi:" >> ${SPACK_ENV}/spack.yaml + echo " buildable: False" >> ${SPACK_ENV}/spack.yaml + echo " externals:" >> ${SPACK_ENV}/spack.yaml + echo " - spec: openmpi@${openmpi_ver}" >> ${SPACK_ENV}/spack.yaml + echo " prefix: $HOME/mpi" >> ${SPACK_ENV}/spack.yaml fi spack external find From 08164733f9213cbe11551369e404a1524fc564b4 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 8 Apr 2022 17:09:02 -0600 Subject: [PATCH 13/24] Try to find git-lfs explicitly --- .github/actions/setup-spack-stack/action.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/setup-spack-stack/action.yaml b/.github/actions/setup-spack-stack/action.yaml index e7524099e..b2ec84538 100644 --- a/.github/actions/setup-spack-stack/action.yaml +++ b/.github/actions/setup-spack-stack/action.yaml @@ -174,6 +174,7 @@ runs: echo " prefix: $HOME/mpi" >> ${SPACK_ENV}/spack.yaml fi + spack external find -t git-lfs spack external find - name: update-bash-profile From 477835893e867d5e5a69b5cf7b4a9e01acf2a5cf Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Sun, 10 Apr 2022 20:26:44 -0600 Subject: [PATCH 14/24] Use git external find --all for CI tests --- .github/actions/setup-spack-stack/action.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup-spack-stack/action.yaml b/.github/actions/setup-spack-stack/action.yaml index b2ec84538..47f51b200 100644 --- a/.github/actions/setup-spack-stack/action.yaml +++ b/.github/actions/setup-spack-stack/action.yaml @@ -174,8 +174,12 @@ runs: echo " prefix: $HOME/mpi" >> ${SPACK_ENV}/spack.yaml fi - spack external find -t git-lfs - spack external find + # Spack external find is by default only looking for build-tools. Either + # search for additional packages explicitly, or fall back to the old + # behavior to find all external packages. + #spack external find + #spack external find git-lfs openmpi mpich + spack external find --all - name: update-bash-profile shell: bash From ab3eb5e159c627068c7e510e1b40737e4d250527 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Apr 2022 08:59:27 -0600 Subject: [PATCH 15/24] Change all of jedi-cmake from JCSDA-internal to JCSDA --- configs/repos/jedi/packages/jedi-cmake/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/repos/jedi/packages/jedi-cmake/package.py b/configs/repos/jedi/packages/jedi-cmake/package.py index 297ced09d..06a3788c8 100644 --- a/configs/repos/jedi/packages/jedi-cmake/package.py +++ b/configs/repos/jedi/packages/jedi-cmake/package.py @@ -9,8 +9,8 @@ class JediCmake(CMakePackage): """CMake/ecbuild toolchains to facilitate portability on different systems.""" - homepage = "https://github.com/JCSDA-internal/jedi-cmake" - git = "https://github.com/JCSDA-internal/jedi-cmake.git" + homepage = "https://github.com/JCSDA/jedi-cmake" + git = "https://github.com/JCSDA/jedi-cmake.git" url = "https://github.com/JCSDA/jedi-cmake/archive/refs/tags/1.3.0.tar.gz" maintainers = ['climbfuji', 'rhoneyager'] From 92aee3f2eef46955f4c5910ac191b54747461866 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Apr 2022 09:11:41 -0600 Subject: [PATCH 16/24] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index b629a8817..493faaed0 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit b629a8817832bd344ce421287e2d2e06e01873e4 +Subproject commit 493faaed0dc02498e04bc2ec44265ff71b78d224 From 9ffdfb5b15b307ccacb6ba0c7bf6c2fddeffff2f Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Apr 2022 10:04:24 -0600 Subject: [PATCH 17/24] Remove all download artifacts that contain a '?' --- .github/workflows/create-spack-mirror.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/create-spack-mirror.yaml b/.github/workflows/create-spack-mirror.yaml index 7492420d3..1bb60e4c5 100644 --- a/.github/workflows/create-spack-mirror.yaml +++ b/.github/workflows/create-spack-mirror.yaml @@ -33,10 +33,8 @@ jobs: spack add clingo spack concretize spack mirror create -a - # upload-artifact does not allow filenames that begin with '?' - rm -f cache/source_cache/autoconf/\?id=05972f49ee632cd98057a3caf82ebfb9574846da-eaa3f69 - rm -f cache/source_cache/boost/attachment.cgi?id=356970-b6f6ce6 - rm -f cache/source_cache/py-scipy/extern_decls.patch?id=711fe05025795e44b84233e065d240859ccae5bd-5433f60 + # upload-artifact does not allow filenames that contain '?' + rm -vf `find cache/source_cache -name '*\?*'` - name: upload-mirror uses: actions/upload-artifact@v2 From c3c0cca04545aa7ed3d9c8ba1a9bc4e7f972fb3e Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Apr 2022 11:37:56 -0600 Subject: [PATCH 18/24] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 493faaed0..53f324b87 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 493faaed0dc02498e04bc2ec44265ff71b78d224 +Subproject commit 53f324b87bc6abb6ff51501938b6e3557e99e0d4 From 62114e9477f52eca1a321f66067abb498dc0915a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Apr 2022 10:14:37 -0600 Subject: [PATCH 19/24] Temporary commit for Intel in .github/actions/setup-spack-stack/action.yaml --- .github/actions/setup-spack-stack/action.yaml | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/actions/setup-spack-stack/action.yaml b/.github/actions/setup-spack-stack/action.yaml index 47f51b200..f49544f5a 100644 --- a/.github/actions/setup-spack-stack/action.yaml +++ b/.github/actions/setup-spack-stack/action.yaml @@ -138,6 +138,11 @@ runs: sed -i".bak" "s|fc: null|fc: /usr/local/bin/gfortran-10|" ${SPACK_ENV}/spack.yaml elif [[ "${{ inputs.compiler }}" == "intel"* ]]; then spack compiler add /opt/intel/oneapi/compiler/latest/linux/bin/intel64 + # Workaround for error "libimf.so cannot be found": + cat envs/${{ inputs.app }}/spack.yaml + ls -l /opt/intel/oneapi/compiler/latest/linux/compiler/lib/intel64_lin + sed -i 's#environment: {}#environment: {prepend_path: {LD_LIBRARY_PATH: /opt/intel/oneapi/compiler/latest/linux/compiler/lib/intel64_lin}}#g' envs/${{ inputs.app }}/spack.yaml + cat envs/${{ inputs.app }}/spack.yaml else spack compiler find fi @@ -214,12 +219,6 @@ runs: spack config add "modules:default:tcl:whitelist:[${{ inputs.mpi }}]" spack config add "modules:default:lmod:whitelist:[${{ inputs.mpi }}]" - # Currently, this is the case for all workflows - wrap in if statement - # when this changes in the future - # Whitelist the mpi providers so that spack creates the modules for them - spack config add "modules:default:tcl:whitelist:[${{ inputs.mpi }}]" - spack config add "modules:default:lmod:whitelist:[${{ inputs.mpi }}]" - if [[ "$RUNNER_OS" == "Linux" ]]; then echo "" elif [[ "$RUNNER_OS" == "macOS" ]]; then @@ -243,13 +242,14 @@ runs: cd ${{ inputs.path }} source setup.sh spack env activate ${{ inputs.path }}/envs/${{ inputs.app }} - if [[ "${{ inputs.compiler }}" == "intel"* ]]; then - source /opt/intel/oneapi/setvars.sh - # Pass --dirty for Intel compiler environment to fix missing LD_LIBRARY_PATH - spack install --dirty --fail-fast - else - spack install --fail-fast - fi + #if [[ "${{ inputs.compiler }}" == "intel"* ]]; then + # source /opt/intel/oneapi/setvars.sh + # # Pass --dirty for Intel compiler environment to fix missing LD_LIBRARY_PATH + # spack install --dirty --fail-fast + #else + # spack install --fail-fast + #fi + spack install --fail-fast - name: create-meta-modules shell: bash From daaef9b85bf8afee5688817f5764ed5414e90e2f Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Apr 2022 17:22:36 -0600 Subject: [PATCH 20/24] Use py-scipy 1.7.3 on macOS with gcc --- .github/actions/setup-spack-stack/action.yaml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/actions/setup-spack-stack/action.yaml b/.github/actions/setup-spack-stack/action.yaml index f49544f5a..3be342950 100644 --- a/.github/actions/setup-spack-stack/action.yaml +++ b/.github/actions/setup-spack-stack/action.yaml @@ -139,10 +139,7 @@ runs: elif [[ "${{ inputs.compiler }}" == "intel"* ]]; then spack compiler add /opt/intel/oneapi/compiler/latest/linux/bin/intel64 # Workaround for error "libimf.so cannot be found": - cat envs/${{ inputs.app }}/spack.yaml - ls -l /opt/intel/oneapi/compiler/latest/linux/compiler/lib/intel64_lin sed -i 's#environment: {}#environment: {prepend_path: {LD_LIBRARY_PATH: /opt/intel/oneapi/compiler/latest/linux/compiler/lib/intel64_lin}}#g' envs/${{ inputs.app }}/spack.yaml - cat envs/${{ inputs.app }}/spack.yaml else spack compiler find fi @@ -222,6 +219,10 @@ runs: if [[ "$RUNNER_OS" == "Linux" ]]; then echo "" elif [[ "$RUNNER_OS" == "macOS" ]]; then + # py-scipy 1.8.0 doesn't build correctly on macOS with gcc + if [[ "${{ inputs.compiler }}" == "gcc"* ]]; then + spack config add "packages:py-scipy:version:[1.7.3]" + fi spack config add "packages:boost:version:[1.78.0]" spack config add "packages:fms:variants: +64bit +enable_quad_precision +gfs_phys ~openmp +pic" spack config add "packages:fms-jcsda:variants: +64bit +enable_quad_precision +gfs_phys ~openmp +pic" @@ -242,13 +243,6 @@ runs: cd ${{ inputs.path }} source setup.sh spack env activate ${{ inputs.path }}/envs/${{ inputs.app }} - #if [[ "${{ inputs.compiler }}" == "intel"* ]]; then - # source /opt/intel/oneapi/setvars.sh - # # Pass --dirty for Intel compiler environment to fix missing LD_LIBRARY_PATH - # spack install --dirty --fail-fast - #else - # spack install --fail-fast - #fi spack install --fail-fast - name: create-meta-modules From 835949136258258f9f0d3055d7a82e1ab1a50655 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Apr 2022 17:23:10 -0600 Subject: [PATCH 21/24] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 53f324b87..0ba55ed90 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 53f324b87bc6abb6ff51501938b6e3557e99e0d4 +Subproject commit 0ba55ed9083177ebbfc48ca1fe10981dd64a61cc From 3edb222e1fea22686680399ac24ebe0af1e1c8d9 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Apr 2022 21:37:10 -0600 Subject: [PATCH 22/24] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 0ba55ed90..ef337c6fd 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 0ba55ed9083177ebbfc48ca1fe10981dd64a61cc +Subproject commit ef337c6fd44e87c4f21342b6f4f82e77d273f2ea From 4e908dc9f7bea6f0eab84fc84eebd5ed0696464d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Apr 2022 21:44:20 -0600 Subject: [PATCH 23/24] Remove unnecessary pinning of py-scipy version 1.7.3 in .github/actions/setup-spack-stack/action.yaml --- .github/actions/setup-spack-stack/action.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/actions/setup-spack-stack/action.yaml b/.github/actions/setup-spack-stack/action.yaml index 3be342950..728e9afb6 100644 --- a/.github/actions/setup-spack-stack/action.yaml +++ b/.github/actions/setup-spack-stack/action.yaml @@ -219,10 +219,6 @@ runs: if [[ "$RUNNER_OS" == "Linux" ]]; then echo "" elif [[ "$RUNNER_OS" == "macOS" ]]; then - # py-scipy 1.8.0 doesn't build correctly on macOS with gcc - if [[ "${{ inputs.compiler }}" == "gcc"* ]]; then - spack config add "packages:py-scipy:version:[1.7.3]" - fi spack config add "packages:boost:version:[1.78.0]" spack config add "packages:fms:variants: +64bit +enable_quad_precision +gfs_phys ~openmp +pic" spack config add "packages:fms-jcsda:variants: +64bit +enable_quad_precision +gfs_phys ~openmp +pic" From 1aaaf9848f15ebaadd5e1013eeaff7f4b916202c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 12 Apr 2022 08:33:02 -0600 Subject: [PATCH 24/24] Revert .gitmodules and update submodules pointer for spack --- .gitmodules | 6 ++---- spack | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index 1f1002828..9cadf2a04 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,7 +2,5 @@ path = spack #url = https://github.com/spack/spack #branch = develop - #url = https://github.com/NOAA-EMC/spack.git - #branch = jcsda_emc_spack_stack - url = https://github.com/climbfuji/spack.git - branch = feature/update_jcsda_emc_spack_stack_from_authoritative_20220404 + url = https://github.com/NOAA-EMC/spack.git + branch = jcsda_emc_spack_stack diff --git a/spack b/spack index ef337c6fd..e02d348cb 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit ef337c6fd44e87c4f21342b6f4f82e77d273f2ea +Subproject commit e02d348cbcc51be65ad811c3e4bf76158f2c6603