forked from NCAR/ccpp-scm
-
Notifications
You must be signed in to change notification settings - Fork 0
267 lines (231 loc) · 9.96 KB
/
ci_run_scm_rts_nvidia.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
name: Build CCPP-SCM.
on: [pull_request,workflow_dispatch]
jobs:
build_scm:
# The type of runner that the job will run on
runs-on: ubuntu-22.04
strategy:
matrix:
fortran-compiler: [nvfortran]
build-type: [Debug]#[Release, Debug]
enable-gpu-acc: [False]#[False, True]
py-version: [3.9.12]
# Environmental variables
env:
NETCDF: /home/runner/netcdf
bacio_ROOT: /home/runner/bacio
sp_ROOT: /home/runner/NCEPLIBS-sp
w3emc_ROOT: /home/runner/myw3emc
SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm
zlib_ROOT: /home/runner/zlib
HDF5_ROOT: /home/runner/hdf5
suites: SCM_GFS_v15p2,SCM_GFS_v16,SCM_GFS_v17_p8,SCM_HRRR,SCM_RRFS_v1beta,SCM_RAP,SCM_WoFS_v0
suites_ps: SCM_GFS_v15p2_ps,SCM_GFS_v16_ps,SCM_GFS_v17_p8_ps,SCM_HRRR_ps,SCM_RRFS_v1beta_ps,SCM_RAP_ps,SCM_WoFS_v0_ps
# Workflow steps
steps:
#######################################################################################
# Cleanup space
#######################################################################################
- name: Check space (pre)
run: |
df -h
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
with:
# this might remove tools that are actually needed,
# if set to "true" but frees about 6 GB
tool-cache: false
# all of these default to true, but feel free to set to
# "false" if necessary for your workflow
android: false
dotnet: false
haskell: true
large-packages: true
docker-images: false
swap-storage: false
- name: Check space (post)
run: |
df -h
#######################################################################################
# Initial
#######################################################################################
- name: Checkout SCM code (into /home/runner/work/ccpp-scm/)
uses: actions/checkout@v3
- name: Initialize submodules
run: git submodule update --init --recursive
#######################################################################################
# Python setup
#######################################################################################
- name: Set up Python
uses: actions/setup-python@v3
with:
python-version: ${{matrix.py-version}}
- name: Add conda to system path
run: |
echo $CONDA/bin >> $GITHUB_PATH
- name: Install NetCDF Python libraries
run: |
conda install --yes -c conda-forge h5py>=3.4 netCDF4 f90nml
#######################################################################################
# Install Nvidia.
#######################################################################################
- name: Nvidia setup compilers.
env:
NVCOMPILERS: /home/runner/hpc_sdk
NVARCH: Linux_x86_64
NVHPC_SILENT: true
NVHPC_INSTALL_DIR: /home/runner/hpc_sdk
NVHPC_INSTALL_TYPE: network
NVHPC_INSTALL_LOCAL_DIR: /home/runner/hpc_sdk
run: |
mkdir /home/runner/hpc_sdk && cd /home/runner/hpc_sdk
wget -q https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz
tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz
nvhpc_2024_241_Linux_x86_64_cuda_12.3/install
export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin
export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/comm_libs/mpi/bin
export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man
echo "The mpifort installed is:"
mpifort --version
echo "The path to mpifort is:"
command -v mpifort
echo "Removing tarball"
rm nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz
echo "SWALES"
ls /home/runner/hpc_sdk/Linux_x86_64/24.1/comm_libs/mpi/bin
- name: Set environment for Nvidia compiler.
run: |
echo "CC=/home/runner/hpc_sdk/Linux_x86_64/24.1/comm_libs/mpi/bin/mpicc" >> $GITHUB_ENV
echo "FC=/home/runner/hpc_sdk/Linux_x86_64/24.1/comm_libs/mpi/bin/mpifort" >> $GITHUB_ENV
echo "CMAKE_C_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/comm_libs/mpi/bin/mpicc" >> $GITHUB_ENV
echo "CMAKE_Fortran_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/comm_libs/mpi/bin/mpifort" >> $GITHUB_ENV
#######################################################################################
# Install FORTRAN dependencies
#######################################################################################
- name: Install zlib
env:
CFLAGS: -fPIC
run: |
wget https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.gz
tar -zxvf zlib-1.2.13.tar.gz
cd zlib-1.2.13
./configure --prefix=${zlib_ROOT}
make
make install
echo "LD_LIBRARY_PATH=$zlib_ROOT/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
- name: Install HDF5
env:
CPPFLAGS: -I${zlib_ROOT}/include
LDFLAGS: -L${zlib_ROOT}/lib
run: |
wget https://github.com/HDFGroup/hdf5/archive/refs/tags/hdf5-1_14_1-2.tar.gz
tar -zxvf hdf5-1_14_1-2.tar.gz
cd hdf5-hdf5-1_14_1-2
./configure --prefix=${HDF5_ROOT} --with-zlib=${zlib_ROOT}
make -j4
make install
echo "LD_LIBRARY_PATH=$HDF5_ROOT/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
echo "PATH=$HDF5_ROOT/lib:$PATH" >> $GITHUB_ENV
- name: Install Curl
run: |
sudo apt-get install curl
sudo apt-get install libssl-dev libcurl4-openssl-dev
- name: Cache NetCDF C library
id: cache-netcdf-c
uses: actions/cache@v3
with:
path: /home/runner/netcdf-c
key: cache-netcdf-c-${{matrix.fortran-compiler}}-key
- name: Install NetCDF C library
if: steps.cache-netcdf-c.outputs.cache-hit != 'true'
run: |
wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz
tar -zvxf v4.7.4.tar.gz
cd netcdf-c-4.7.4
CPPFLAGS="-I/home/runner/hdf5/include -I/home/runner/zlib/include" LDFLAGS="-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" ./configure --prefix=${NETCDF}
make
make install
echo "LD_LIBRARY_PATH=$NETCDF/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
echo "PATH=$NETCDF/lib:$PATH" >> $GITHUB_ENV
- name: Cache NetCDF Fortran library
id: cache-netcdf-fortran
uses: actions/cache@v3
with:
path: /home/runner/netcdf-fortran
key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key
- name: Install NetCDF Fortran library
if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true'
run: |
wget https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz
tar -zvxf v4.6.1.tar.gz
cd netcdf-fortran-4.6.1
FCFLAGS="-fPIC" FFLAGS="-fPIC" CPPFLAGS="-I/home/runner/hdf5/include -I/home/runner/zlib/include -I/home/runner/netcdf/include" LDFLAGS="-L/home/runner/hdf5/lib -L/home/runner/zlib/lib -L/home/runner/netcdf/lib" ./configure --prefix=${NETCDF}
make
make install
- name: Cache bacio library v2.4.1
id: cache-bacio-fortran
uses: actions/cache@v3
with:
path: /home/runner/bacio
key: cache-bacio-fortran-${{matrix.fortran-compiler}}-key
- name: Install bacio library v2.4.1
if: steps.cache-bacio-fortran.outputs.cache-hit != 'true'
run: |
git clone --branch v2.4.1 https://github.com/NOAA-EMC/NCEPLIBS-bacio.git bacio
cd bacio && mkdir build && cd build
cmake -DCMAKE_INSTALL_PREFIX=${bacio_ROOT} ../
make -j2
make install
echo "bacio_DIR=/home/runner/bacio/lib/cmake/bacio" >> $GITHUB_ENV
- name: Cache SP-library v2.3.3
id: cache-sp-fortran
uses: actions/cache@v3
with:
path: /home/runner/NCEPLIBS-sp
key: cache-sp-fortran-${{matrix.fortran-compiler}}-key
- name: Install SP-library v2.3.3
if: steps.cache-sp-fortran.outputs.cache-hit != 'true'
run: |
git clone --branch v2.3.3 https://github.com/NOAA-EMC/NCEPLIBS-sp.git NCEPLIBS-sp
cd NCEPLIBS-sp && mkdir build && cd build
cmake -DCMAKE_INSTALL_PREFIX=${sp_ROOT} ../
make -j2
make install
echo "sp_DIR=/home/runner/NCEPLIBS-sp/lib/cmake/sp" >> $GITHUB_ENV
- name: Cache w3emc library v2.9.2
id: cache-w3emc-fortran
uses: actions/cache@v3
with:
path: /home/runner/myw3emc
key: cache-w3emc-fortran-${{matrix.fortran-compiler}}-key
- name: Install w3emc library v2.9.2
if: steps.cache-w3emc-fortran.outputs.cache-hit != 'true'
run: |
git clone --branch v2.9.2 https://github.com/NOAA-EMC/NCEPLIBS-w3emc.git NCEPLIBS-w3emc
cd NCEPLIBS-w3emc && mkdir build && cd build
cmake -DCMAKE_INSTALL_PREFIX=${w3emc_ROOT} ../
make -j2
make install
echo "w3emc_DIR=/home/runner/myw3emc/lib/cmake/w3emc" >> $GITHUB_ENV
#######################################################################################
# Build and run SCM regression tests (ccpp-scm/test/rt_test_cases.py)
#######################################################################################
- name: Configure build with CMake
run: |
cd ${SCM_ROOT}/scm
mkdir bin && cd bin
cmake -DCCPP_SUITES=${suites},${suites_ps} -DCMAKE_BUILD_TYPE=${{matrix.build-type}} -DENABLE_NVIDIA_OPENACC=${{matrix.enable-gpu-acc}} ../src
- name: Build SCM.
run: |
cd ${SCM_ROOT}/scm/bin
make -j4
- name: Download data for SCM
run: |
cd ${SCM_ROOT}
./contrib/get_all_static_data.sh
./contrib/get_thompson_tables.sh
- name: Run SCM RTs (w/o GPU)
if: contains(matrix.enable-gpu-acc, 'False')
run: |
cd ${SCM_ROOT}/scm/bin
./run_scm.py --file /home/runner/work/ccpp-scm/ccpp-scm/test/rt_test_cases.py --runtime_mult 0.1 -v