From 100f9212c247f820cc8c7bc42b15ec7079daef3a Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 29 May 2024 21:07:04 +0000 Subject: [PATCH 1/6] Changes fro Grants branch --- .../workflows/ci_build_scm_ubuntu_22.04.yml | 43 ++++++------------- .github/workflows/ci_run_scm_rts.yml | 26 +++++------ scm/src/run_scm.py | 2 +- 3 files changed, 29 insertions(+), 42 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04.yml b/.github/workflows/ci_build_scm_ubuntu_22.04.yml index 6c33a30e5..a5ccb6177 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04.yml @@ -21,8 +21,9 @@ jobs: sp_ROOT: /home/runner/NCEPLIBS-sp w3emc_ROOT: /home/runner/myw3emc SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm - intel_ROOT: /home/runner/intel suites: SCM_GFS_v15p2,SCM_GFS_v16,SCM_GFS_v17_p8,SCM_HRRR,SCM_RRFS_v1beta,SCM_RAP,SCM_WoFS_v0,SCM_GFS_v15p2_ps,SCM_GFS_v16_ps,SCM_GFS_v17_p8_ps,SCM_HRRR_ps,SCM_RRFS_v1beta_ps,SCM_RAP_ps,SCM_WoFS_v0_ps + CC: mpicc + FC: mpif90 # Workflow steps steps: @@ -57,34 +58,6 @@ jobs: ####################################################################################### # Install FORTRAN dependencies ####################################################################################### - - name: Environment for ifort compiler - if: contains(matrix.fortran-compiler, 'ifort') - run: | - echo "CC=icx" >> $GITHUB_ENV - echo "FC=ifort" >> $GITHUB_ENV - echo "NFVERSION=v4.4.4" >> $GITHUB_ENV - - - name: Cache Intel compilers - id: cache-intel-compilers - if: contains(matrix.fortran-compiler, 'ifort') - uses: actions/cache@v2 - with: - path: /home/runner/intel - key: intel-${{ runner.os }}-compilers-b - - # https://software.intel.com/content/www/us/en/develop/articles/installing-intel-oneapi-toolkits-via-apt.html - # List of packages from Docker file at - # https://github.com/intel/oneapi-containers/blob/master/images/docker/hpckit-devel-ubuntu18.04/Dockerfile - - name: Install Intel compilers and libraries - if: contains(matrix.fortran-compiler, 'ifort') && steps.cache-intel-compilers.outputs.cache-hit != 'true' - run: | - wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB - sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB - sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main" - sudo apt-get update - sudo apt-get install intel-hpckit-getting-started intel-oneapi-clck intel-oneapi-common-licensing intel-oneapi-common-vars - sudo apt-get install intel-oneapi-dev-utilities intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic intel-oneapi-compiler-fortran intel-oneapi-itac - - name: Cache bacio library v2.4.1 id: cache-bacio-fortran uses: actions/cache@v3 @@ -141,6 +114,18 @@ jobs: sudo apt-get update sudo apt-get install libnetcdf-dev + - name: Install open mpi + run: | + wget https://github.com/open-mpi/ompi/archive/refs/tags/v4.1.6.tar.gz + tar -xvf v4.1.6.tar.gz + cd ompi-4.1.6 + ./autogen.pl + ./configure --prefix=/home/runner/ompi-4.1.6 + make -j4 + make install + echo "LD_LIBRARY_PATH=/home/runner/ompi-4.1.6/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=/home/runner/ompi-4.1.6/bin:$PATH" >> $GITHUB_ENV + - name: Cache NetCDF Fortran library id: cache-netcdf-fortran uses: actions/cache@v3 diff --git a/.github/workflows/ci_run_scm_rts.yml b/.github/workflows/ci_run_scm_rts.yml index f4096681d..f03399439 100644 --- a/.github/workflows/ci_run_scm_rts.yml +++ b/.github/workflows/ci_run_scm_rts.yml @@ -25,6 +25,8 @@ jobs: suites_ps: SCM_GFS_v15p2_ps,SCM_GFS_v16_ps,SCM_GFS_v17_p8_ps,SCM_HRRR_ps,SCM_RRFS_v1beta_ps,SCM_RAP_ps,SCM_WoFS_v0_ps,SCM_RRFS_v1_ps,SCM_GFS_v17_HR3_ps,SCM_GFS_v17_HR3_RRTMGP_ps dir_rt: /home/runner/work/ccpp-scm/ccpp-scm/test/artifact-${{matrix.build-type}} dir_bl: /home/runner/work/ccpp-scm/ccpp-scm/test/BL-${{matrix.build-type}} + CC: mpicc + FC: mpif90 # Workflow steps steps: @@ -59,18 +61,6 @@ jobs: ####################################################################################### # Install FORTRAN dependencies ####################################################################################### - - name: Environment for gfortran compiler - if: contains(matrix.fortran-compiler, 'gnu') - run: | - echo "FC=gfortran-11" >> $GITHUB_ENV - echo "CC=gcc-11" >> $GITHUB_ENV - - - name: Environment for ifort compiler - if: contains(matrix.fortran-compiler, 'intel') - run: | - echo "CC=icx" >> $GITHUB_ENV - echo "FC=ifort" >> $GITHUB_ENV - - name: Cache bacio library v2.4.1 id: cache-bacio-fortran uses: actions/cache@v3 @@ -125,6 +115,18 @@ jobs: - name: Install NetCDF C library run: sudo apt-get install libnetcdf-dev + - name: Install open mpi + run: | + wget https://github.com/open-mpi/ompi/archive/refs/tags/v4.1.6.tar.gz + tar -xvf v4.1.6.tar.gz + cd ompi-4.1.6 + ./autogen.pl + ./configure --prefix=/home/runner/ompi-4.1.6 + make -j4 + make install + echo "LD_LIBRARY_PATH=/home/runner/ompi-4.1.6/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=/home/runner/ompi-4.1.6/bin:$PATH" >> $GITHUB_ENV + - name: Cache NetCDF Fortran library id: cache-netcdf-fortran uses: actions/cache@v3 diff --git a/scm/src/run_scm.py b/scm/src/run_scm.py index 2f1db97c3..773b89054 100755 --- a/scm/src/run_scm.py +++ b/scm/src/run_scm.py @@ -720,7 +720,7 @@ def launch_executable(use_gdb, gdb, ignore_error = False): if use_gdb: cmd = '(cd {scm_run} && {gdb} {executable})'.format(scm_run=SCM_RUN, gdb=gdb, executable=EXECUTABLE) else: - cmd = '(cd {scm_run} && time srun -A gmtb -n 1 {executable})'.format(scm_run=SCM_RUN, executable=EXECUTABLE) + cmd = '(cd {scm_run} && time mpirun -n 1 {executable})'.format(scm_run=SCM_RUN, executable=EXECUTABLE) logging.info('Passing control to "{0}"'.format(cmd)) time.sleep(1) # This will abort in 'execute' in the event of an error if ignore_error = False From de62a3db5910320319b0f1f54a799571aff5577e Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 29 May 2024 21:13:09 +0000 Subject: [PATCH 2/6] Move openmpi build up in CI workflow --- .../workflows/ci_build_scm_ubuntu_22.04.yml | 24 +++++++++---------- .github/workflows/ci_run_scm_rts.yml | 24 +++++++++---------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04.yml b/.github/workflows/ci_build_scm_ubuntu_22.04.yml index a5ccb6177..ed2f8f884 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04.yml @@ -58,6 +58,18 @@ jobs: ####################################################################################### # Install FORTRAN dependencies ####################################################################################### + - name: Install openmpi + run: | + wget https://github.com/open-mpi/ompi/archive/refs/tags/v4.1.6.tar.gz + tar -xvf v4.1.6.tar.gz + cd ompi-4.1.6 + ./autogen.pl + ./configure --prefix=/home/runner/ompi-4.1.6 + make -j4 + make install + echo "LD_LIBRARY_PATH=/home/runner/ompi-4.1.6/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=/home/runner/ompi-4.1.6/bin:$PATH" >> $GITHUB_ENV + - name: Cache bacio library v2.4.1 id: cache-bacio-fortran uses: actions/cache@v3 @@ -114,18 +126,6 @@ jobs: sudo apt-get update sudo apt-get install libnetcdf-dev - - name: Install open mpi - run: | - wget https://github.com/open-mpi/ompi/archive/refs/tags/v4.1.6.tar.gz - tar -xvf v4.1.6.tar.gz - cd ompi-4.1.6 - ./autogen.pl - ./configure --prefix=/home/runner/ompi-4.1.6 - make -j4 - make install - echo "LD_LIBRARY_PATH=/home/runner/ompi-4.1.6/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV - echo "PATH=/home/runner/ompi-4.1.6/bin:$PATH" >> $GITHUB_ENV - - name: Cache NetCDF Fortran library id: cache-netcdf-fortran uses: actions/cache@v3 diff --git a/.github/workflows/ci_run_scm_rts.yml b/.github/workflows/ci_run_scm_rts.yml index f03399439..996bb680b 100644 --- a/.github/workflows/ci_run_scm_rts.yml +++ b/.github/workflows/ci_run_scm_rts.yml @@ -61,6 +61,18 @@ jobs: ####################################################################################### # Install FORTRAN dependencies ####################################################################################### + - name: Install openmpi + run: | + wget https://github.com/open-mpi/ompi/archive/refs/tags/v4.1.6.tar.gz + tar -xvf v4.1.6.tar.gz + cd ompi-4.1.6 + ./autogen.pl + ./configure --prefix=/home/runner/ompi-4.1.6 + make -j4 + make install + echo "LD_LIBRARY_PATH=/home/runner/ompi-4.1.6/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=/home/runner/ompi-4.1.6/bin:$PATH" >> $GITHUB_ENV + - name: Cache bacio library v2.4.1 id: cache-bacio-fortran uses: actions/cache@v3 @@ -115,18 +127,6 @@ jobs: - name: Install NetCDF C library run: sudo apt-get install libnetcdf-dev - - name: Install open mpi - run: | - wget https://github.com/open-mpi/ompi/archive/refs/tags/v4.1.6.tar.gz - tar -xvf v4.1.6.tar.gz - cd ompi-4.1.6 - ./autogen.pl - ./configure --prefix=/home/runner/ompi-4.1.6 - make -j4 - make install - echo "LD_LIBRARY_PATH=/home/runner/ompi-4.1.6/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV - echo "PATH=/home/runner/ompi-4.1.6/bin:$PATH" >> $GITHUB_ENV - - name: Cache NetCDF Fortran library id: cache-netcdf-fortran uses: actions/cache@v3 From 9ffcea06bb3fe775a593df77ba274066c13aa5f4 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 29 May 2024 21:22:12 +0000 Subject: [PATCH 3/6] CI changes --- .github/workflows/ci_build_scm_ubuntu_22.04.yml | 7 +++++-- .github/workflows/ci_run_scm_rts.yml | 8 ++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04.yml b/.github/workflows/ci_build_scm_ubuntu_22.04.yml index ed2f8f884..f2ad6421c 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04.yml @@ -22,8 +22,6 @@ jobs: w3emc_ROOT: /home/runner/myw3emc SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm suites: SCM_GFS_v15p2,SCM_GFS_v16,SCM_GFS_v17_p8,SCM_HRRR,SCM_RRFS_v1beta,SCM_RAP,SCM_WoFS_v0,SCM_GFS_v15p2_ps,SCM_GFS_v16_ps,SCM_GFS_v17_p8_ps,SCM_HRRR_ps,SCM_RRFS_v1beta_ps,SCM_RAP_ps,SCM_WoFS_v0_ps - CC: mpicc - FC: mpif90 # Workflow steps steps: @@ -143,6 +141,11 @@ jobs: sudo make install export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NFHOME}/lib + - name: Environment for openmpi compiler + run: | + echo "FC=mpif90" >> $GITHUB_ENV + echo "CC=mpicc" >> $GITHUB_ENV + ####################################################################################### # Build SCM. ####################################################################################### diff --git a/.github/workflows/ci_run_scm_rts.yml b/.github/workflows/ci_run_scm_rts.yml index 996bb680b..aab18a355 100644 --- a/.github/workflows/ci_run_scm_rts.yml +++ b/.github/workflows/ci_run_scm_rts.yml @@ -25,8 +25,6 @@ jobs: suites_ps: SCM_GFS_v15p2_ps,SCM_GFS_v16_ps,SCM_GFS_v17_p8_ps,SCM_HRRR_ps,SCM_RRFS_v1beta_ps,SCM_RAP_ps,SCM_WoFS_v0_ps,SCM_RRFS_v1_ps,SCM_GFS_v17_HR3_ps,SCM_GFS_v17_HR3_RRTMGP_ps dir_rt: /home/runner/work/ccpp-scm/ccpp-scm/test/artifact-${{matrix.build-type}} dir_bl: /home/runner/work/ccpp-scm/ccpp-scm/test/BL-${{matrix.build-type}} - CC: mpicc - FC: mpif90 # Workflow steps steps: @@ -61,6 +59,7 @@ jobs: ####################################################################################### # Install FORTRAN dependencies ####################################################################################### + - name: Install openmpi run: | wget https://github.com/open-mpi/ompi/archive/refs/tags/v4.1.6.tar.gz @@ -144,6 +143,11 @@ jobs: sudo make install export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NFHOME}/lib + - name: Environment for openmpi compiler + run: | + echo "FC=mpif90" >> $GITHUB_ENV + echo "CC=mpicc" >> $GITHUB_ENV + ####################################################################################### # Build SCM. Run regression tests. ####################################################################################### From ed1e4d81549cab92b3e764518b555a6130f4676b Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 29 May 2024 21:31:09 +0000 Subject: [PATCH 4/6] CI changes --- .github/workflows/ci_run_scm_DEPHY.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/ci_run_scm_DEPHY.yml b/.github/workflows/ci_run_scm_DEPHY.yml index eeba4c7fe..326eeb75e 100644 --- a/.github/workflows/ci_run_scm_DEPHY.yml +++ b/.github/workflows/ci_run_scm_DEPHY.yml @@ -54,6 +54,18 @@ jobs: ####################################################################################### # Install FORTRAN dependencies ####################################################################################### + - name: Install openmpi + run: | + wget https://github.com/open-mpi/ompi/archive/refs/tags/v4.1.6.tar.gz + tar -xvf v4.1.6.tar.gz + cd ompi-4.1.6 + ./autogen.pl + ./configure --prefix=/home/runner/ompi-4.1.6 + make -j4 + make install + echo "LD_LIBRARY_PATH=/home/runner/ompi-4.1.6/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=/home/runner/ompi-4.1.6/bin:$PATH" >> $GITHUB_ENV + - name: Cache bacio library v2.4.1 id: cache-bacio-fortran uses: actions/cache@v3 @@ -128,6 +140,11 @@ jobs: ####################################################################################### # Build SCM. Run DEPHYv1 case. ####################################################################################### + - name: Environment for openmpi compiler + run: | + echo "FC=mpif90" >> $GITHUB_ENV + echo "CC=mpicc" >> $GITHUB_ENV + - name: Download data for SCM run: | cd ${SCM_ROOT} From dcb2e5f1aab65634e37f8f5b2fe605d9ec2bed2c Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 29 May 2024 21:48:56 +0000 Subject: [PATCH 5/6] CI changes --- .github/workflows/ci_build_scm_ubuntu_22.04.yml | 2 +- .github/workflows/ci_run_scm_rts.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04.yml b/.github/workflows/ci_build_scm_ubuntu_22.04.yml index f2ad6421c..4a57f71a7 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04.yml @@ -67,7 +67,7 @@ jobs: make install echo "LD_LIBRARY_PATH=/home/runner/ompi-4.1.6/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV echo "PATH=/home/runner/ompi-4.1.6/bin:$PATH" >> $GITHUB_ENV - + - name: Cache bacio library v2.4.1 id: cache-bacio-fortran uses: actions/cache@v3 diff --git a/.github/workflows/ci_run_scm_rts.yml b/.github/workflows/ci_run_scm_rts.yml index aab18a355..3176274f7 100644 --- a/.github/workflows/ci_run_scm_rts.yml +++ b/.github/workflows/ci_run_scm_rts.yml @@ -147,7 +147,7 @@ jobs: run: | echo "FC=mpif90" >> $GITHUB_ENV echo "CC=mpicc" >> $GITHUB_ENV - + ####################################################################################### # Build SCM. Run regression tests. ####################################################################################### From e54a78adc1d342044e44ce7a586792f671db147c Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 29 May 2024 22:09:35 +0000 Subject: [PATCH 6/6] CI changes --- .github/workflows/ci_build_scm_ubuntu_22.04.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04.yml b/.github/workflows/ci_build_scm_ubuntu_22.04.yml index 4a57f71a7..e89dcb50c 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04.yml @@ -145,7 +145,7 @@ jobs: run: | echo "FC=mpif90" >> $GITHUB_ENV echo "CC=mpicc" >> $GITHUB_ENV - + ####################################################################################### # Build SCM. #######################################################################################