-
Notifications
You must be signed in to change notification settings - Fork 11
186 lines (178 loc) · 7.18 KB
/
IDC_1100_Public_CI.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
name: openxla ci
run-name: ${{ github.actor }} is testing out GitHub Action
permissions: read-all
on:
pull_request:
branches:
- main
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
openXLA-CI:
runs-on: Intel_openxla
if: |
!(contains(github.repository, 'intel-innersource'))
steps:
- name: Check out repository code
uses: actions/checkout@v3
- name: Prepare Conda Environment
run: |
set +e
CONDA_ENV=openxla_preCI_${{ github.event.pull_request.number }}_${{ runner.name }}
source /opt/conda/bin/activate ${CONDA_ENV}
if [ $? -ne 0 ]; then
echo 'conda env does not exist'
conda create -n ${CONDA_ENV} python=3.10 -y
conda activate ${CONDA_ENV}
fi
pip install -e ./test/requirements.txt
#pip install jax==0.4.24 jaxlib==0.4.24
conda install libstdcxx-ng==12.2.0 -c conda-forge
pip install absl-py
pip list | grep numpy
pip list | grep jax
- name: Build openXLA
run: |
set +e
CONDA_ENV=openxla_preCI_${{ github.event.pull_request.number }}_${{ runner.name }}
source /opt/conda/bin/activate ${CONDA_ENV}
which bazel
if [ $? -eq 1 ]; then
wget https://github.com/bazelbuild/bazel/releases/download/5.3.0/bazel-5.3.0-installer-linux-x86_64.sh
bash bazel-5.3.0-installer-linux-x86_64.sh --user
fi
export PATH=$PATH:/home/sdp/bin
source /home/sdp/.bazel/bin/bazel-complete.bash
bazel --version
basekit_path=/home/sdp
complier_path=${basekit_path}/intel/oneapi/compiler/latest
mkl_path=${basekit_path}/intel/oneapi/mkl/latest
#get build config
if [ -d "jax_test" ]; then
echo "jax_test folder already exists. Doing nothing."
cp ./jax_test/openxla/config/openXLA_demoxla_auto_configure_mkl.exp .
else
git clone https://github.com/wendyliu235/aipc_unify_validation_infr.git jax_test
cp ./jax_test/openxla/config/openXLA_demoxla_auto_configure_mkl.exp .
fi
python --version
chmod +x ./openXLA_demoxla_auto_configure_mkl.exp
./openXLA_demoxla_auto_configure_mkl.exp $complier_path $mkl_path
bazel clean --expunge --async
bazel build //xla/tools/pip_package:build_pip_package
./bazel-bin/xla/tools/pip_package/build_pip_package ./
build_result=${PIPESTATUS[0]}
echo $build_result
if [ "$build_result" = "0" ];then
echo "Build successful"
pip install *.whl
else
echo "Build Failed"
exit 1
fi
- name: UT testing
run: |
set +e
CONDA_ENV=openxla_preCI_${{ github.event.pull_request.number }}_${{ runner.name }}
source /opt/conda/bin/activate ${CONDA_ENV}
basekit_path=/home/sdp
source ${basekit_path}/intel/oneapi/compiler/latest/env/vars.sh
source ${basekit_path}/intel/oneapi/mkl/latest/env/vars.sh
export ZE_AFFINITY_MASK=5,6
ut_branch=$(cat ./test/BRANCH_NAME)
git clone https://github.com/google/jax.git
cd jax && git checkout $ut_branch
patch -p1 -i ../test/jax_ut.patch
pip install hypothesis
pip install -e .
pip install -r build/test-requirements.txt
mkdir -p ./logs/ut
find ./tests -name "*.py" | sed 's#\./##g' |& tee ut_list.txt
file_path=./ut_list.txt
# get distribute ut list
if [ -d "jax_test" ]; then
echo "jax_test folder already exists. Doing nothing."
else
git clone https://github.com/wendyliu235/aipc_unify_validation_infr.git jax_test
fi
reference_file=jax_test/openxla/distribute_ut_list.txt
blacklist=jax_test/openxla/blacklist.txt
grep -v -f $blacklist $file_path > ut_list_filtered.txt
mv ut_list_filtered.txt $file_path
# run distribute ut first
# Check if the file exists
if [ -f "$file_path" ]; then
# Read each line using a while loop
while IFS= read -r line; do
# Process each line (you can replace this with your own logic)
echo "Processing line: $line"
if grep -qFx "$line" "$reference_file"; then
echo "'$line' is distribute case"
log_name=$(basename "$line" .py)
python $line |& tee ./logs/ut/$log_name.log
if [ ${PIPESTATUS[0]} -eq 0 ]; then
echo "$line" >> ./logs/ut/successful.log
else
echo "$line" >> ./logs/ut/err.log
fi
fi
done < "$file_path"
else
echo "File not found: $file_path"
exit
fi
# Loop through all Python files in the folder and execute them
dev_num=4 #IDC have 4 1100 pvc card
# Create a pipe and bind the file descriptor 6
tmp_fifofile="/tmp/$$.fifo"
mkfifo $tmp_fifofile
exec 6<>$tmp_fifofile
rm $tmp_fifofile
# Writing NUM_PROC blank data to the pipe
for ((i=0;i<$dev_num;i++)); do
echo "$i"
done >&6
while IFS= read -r line; do
read -u6 id
{
# Process each line (you can replace this with your own logic)
echo "Processing line: $line"
if grep -qFx "$line" "$reference_file"; then
echo "'$line' is distribute case,skip"
echo $id >&6
else
log_name=$(basename "$line" .py)
echo "running ut on device-$id"
echo $line
echo $id
echo "ZE_AFFINITY_MASK=$id python $line"
ZE_AFFINITY_MASK=$id python $line |& tee ./logs/ut/$log_name.log
if [ ${PIPESTATUS[0]} -eq 0 ]; then
echo "$line" >> ./logs/ut/successful.log
else
echo "$line" >> ./logs/ut/err.log
fi
# writing one data to the pipe for the next task
echo $id >&6
fi
} &
done < "$file_path"
wait
exec 6>&-
- name: Upload XLA CI Data
uses: actions/upload-artifact@v3
with:
name: XLA-CI-Data
path: /home/sdp/actions-runner/_work/intel-extension-for-openxla/intel-extension-for-openxla/jax/logs
- name: Test Results Check
run: |
cd /home/sdp/actions-runner/_work/intel-extension-for-openxla/intel-extension-for-openxla/jax
#UT results check
if [ -f "./logs/ut/err.log" ]; then
echo "exist fail cases"
cat ./logs/ut/err.log
exit 1
else
echo "run successful"
fi