-
Notifications
You must be signed in to change notification settings - Fork 4.2k
430 lines (413 loc) · 19.1 KB
/
test-go.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
on:
workflow_call:
inputs:
go-arch:
description: The execution architecture (arm, amd64, etc.)
required: true
type: string
enterprise:
description: A flag indicating if this workflow is executing for the enterprise repository.
required: true
type: string
total-runners:
description: Number of runners to use for executing non-binary tests.
required: true
type: string
binary-tests:
description: Whether to run the binary tests.
required: false
default: false
type: boolean
env-vars:
description: A map of environment variables as JSON.
required: false
type: string
default: '{}'
extra-flags:
description: A space-separated list of additional build flags.
required: false
type: string
default: ''
runs-on:
description: An expression indicating which kind of runners to use.
required: false
type: string
default: ubuntu-latest
go-tags:
description: A comma-separated list of additional build tags to consider satisfied during the build.
required: false
type: string
name:
description: A suffix to append to archived test results
required: false
default: ''
type: string
go-test-parallelism:
description: The parallelism parameter for Go tests
required: false
default: 20
type: number
timeout-minutes:
description: The maximum number of minutes that this workflow should run
required: false
default: 60
type: number
testonly:
description: Whether to run the tests tagged with testonly.
required: false
default: false
type: boolean
env: ${{ fromJSON(inputs.env-vars) }}
jobs:
test-matrix:
permissions:
id-token: write # Note: this permission is explicitly required for Vault auth
contents: read
runs-on: ${{ fromJSON(inputs.runs-on) }}
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
- uses: ./.github/actions/set-up-go
with:
github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
- name: Authenticate to Vault
id: vault-auth
if: github.repository == 'hashicorp/vault-enterprise'
run: vault-auth
- name: Fetch Secrets
id: secrets
if: github.repository == 'hashicorp/vault-enterprise'
uses: hashicorp/vault-action@130d1f5f4fe645bb6c83e4225c04d64cfb62de6e
with:
url: ${{ steps.vault-auth.outputs.addr }}
caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
token: ${{ steps.vault-auth.outputs.token }}
secrets: |
kv/data/github/${{ github.repository }}/datadog-ci DATADOG_API_KEY;
kv/data/github/${{ github.repository }}/github-token username-and-token | github-token;
kv/data/github/${{ github.repository }}/license license_1 | VAULT_LICENSE_CI;
kv/data/github/${{ github.repository }}/license license_2 | VAULT_LICENSE_2;
kv/data/github/${{ github.repository }}/hcp-link HCP_API_ADDRESS;
kv/data/github/${{ github.repository }}/hcp-link HCP_AUTH_URL;
kv/data/github/${{ github.repository }}/hcp-link HCP_CLIENT_ID;
kv/data/github/${{ github.repository }}/hcp-link HCP_CLIENT_SECRET;
kv/data/github/${{ github.repository }}/hcp-link HCP_RESOURCE_ID;
- id: setup-git-private
name: Setup Git configuration (private)
if: github.repository == 'hashicorp/vault-enterprise'
run: |
git config --global url."https://${{ steps.secrets.outputs.github-token }}@github.com".insteadOf https://github.com
- id: setup-git-public
name: Setup Git configuration (public)
if: github.repository != 'hashicorp/vault-enterprise'
run: |
git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN}}@github.com".insteadOf https://github.com
- uses: ./.github/actions/set-up-gotestsum
- run: mkdir -p test-results/go-test
- uses: actions/cache/restore@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
with:
path: test-results/go-test
key: go-test-reports-${{ github.run_number }}
restore-keys: go-test-reports-
- name: List cached results
id: list-cached-results
run: ls -lhR test-results/go-test
- name: Build matrix excluding binary, integration, and testonly tests
id: build-non-binary
if: ${{ !inputs.testonly }}
env:
GOPRIVATE: github.com/hashicorp/*
run: |
# testonly tests need additional build tag though let's exclude them anyway for clarity
(
go list ./... | grep -v "_binary" | grep -v "vault/integ" | grep -v "testonly" | gotestsum tool ci-matrix --debug \
--partitions "${{ inputs.total-runners }}" \
--timing-files 'test-results/go-test/*.json' > matrix.json
)
- name: Build matrix for tests tagged with testonly
if: ${{ inputs.testonly }}
env:
GOPRIVATE: github.com/hashicorp/*
run: |
set -exo pipefail
rm -f testonly_matrix.json matrix.json
# testonly tagged tests need an additional tag
# go list -tags=testonly ./... | grep "testonly" | gotestsum tool ci-matrix --debug \
# --partitions "${{ inputs.total-runners }}" \
# --timing-files 'test-results/go-test/*.json' > testonly_matrix.json
echo '{"include":[{"id":0,"estimatedRuntime":"0s","packages":"","description":""},{"id":1,"estimatedRuntime":"0s","packages":"","description":""}]}' > testonly_matrix.json
# hardcoding vault package index as there is only one test package tagged with testonly
# in the external tests. However, it would be also good to run the vault package with
# to validate the tag and also validate the vault package tests that are tagged with
# test only. There is only going to be two tests runners for testonly job.
if [ "${{ inputs.enterprise }}" == "true" ]; then
export VAULT_TEST_PACKAGE="github.com/hashicorp/vault-enterprise/vault"
else
export VAULT_TEST_PACKAGE="github.com/hashicorp/vault/vault"
fi
jq --arg VAULT_PKG "${VAULT_TEST_PACKAGE}" \
'.include[1] = {
"id": 1,
"estimatedRuntime": "N/A",
"packages": $VAULT_PKG,
"description": "partition 1 - vault test packages"
}' testonly_matrix.json > matrix.json
cat matrix.json
- name: Capture list of binary tests
if: inputs.binary-tests
id: list-binary-tests
run: |
LIST="$(go list ./... | grep "_binary" | xargs)"
echo "list=$LIST" >> "$GITHUB_OUTPUT"
- name: Build complete matrix
id: build
run: |
set -exo pipefail
matrix_file="matrix.json"
if [ "${{ inputs.binary-tests}}" == "true" ] && [ -n "${{ steps.list-binary-tests.outputs.list }}" ]; then
export BINARY_TESTS="${{ steps.list-binary-tests.outputs.list }}"
jq --arg BINARY "${BINARY_TESTS}" --arg BINARY_INDEX "${{ inputs.total-runners }}" \
'.include += [{
"id": $BINARY_INDEX,
"estimatedRuntime": "N/A",
"packages": $BINARY,
"description": "partition $BINARY_INDEX - binary test packages"
}]' matrix.json > new-matrix.json
matrix_file="new-matrix.json"
fi
# convert the json to a map keyed by id
(
echo -n "matrix="
jq -c \
'.include | map( { (.id|tostring): . } ) | add' "$matrix_file"
) >> "$GITHUB_OUTPUT"
# extract an array of ids from the json
(
echo -n "matrix_ids="
jq -c \
'[ .include[].id | tostring ]' "$matrix_file"
) >> "$GITHUB_OUTPUT"
outputs:
matrix: ${{ steps.build.outputs.matrix }}
matrix_ids: ${{ steps.build.outputs.matrix_ids }}
test-go:
needs: test-matrix
permissions:
actions: read
contents: read
id-token: write # Note: this permission is explicitly required for Vault auth
runs-on: ${{ fromJSON(inputs.runs-on) }}
strategy:
fail-fast: false
matrix:
id: ${{ fromJSON(needs.test-matrix.outputs.matrix_ids) }}
env:
GOPRIVATE: github.com/hashicorp/*
TIMEOUT_IN_MINUTES: ${{ inputs.timeout-minutes }}
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
- uses: ./.github/actions/set-up-go
with:
github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
- name: Authenticate to Vault
id: vault-auth
if: github.repository == 'hashicorp/vault-enterprise'
run: vault-auth
- name: Fetch Secrets
id: secrets
if: github.repository == 'hashicorp/vault-enterprise'
uses: hashicorp/vault-action@130d1f5f4fe645bb6c83e4225c04d64cfb62de6e
with:
url: ${{ steps.vault-auth.outputs.addr }}
caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
token: ${{ steps.vault-auth.outputs.token }}
secrets: |
kv/data/github/${{ github.repository }}/datadog-ci DATADOG_API_KEY;
kv/data/github/${{ github.repository }}/github-token username-and-token | github-token;
kv/data/github/${{ github.repository }}/license license_1 | VAULT_LICENSE_CI;
kv/data/github/${{ github.repository }}/license license_2 | VAULT_LICENSE_2;
kv/data/github/${{ github.repository }}/hcp-link HCP_API_ADDRESS;
kv/data/github/${{ github.repository }}/hcp-link HCP_AUTH_URL;
kv/data/github/${{ github.repository }}/hcp-link HCP_CLIENT_ID;
kv/data/github/${{ github.repository }}/hcp-link HCP_CLIENT_SECRET;
kv/data/github/${{ github.repository }}/hcp-link HCP_RESOURCE_ID;
- id: setup-git-private
name: Setup Git configuration (private)
if: github.repository == 'hashicorp/vault-enterprise'
run: |
git config --global url."https://${{ steps.secrets.outputs.github-token }}@github.com".insteadOf https://github.com
- id: setup-git-public
name: Setup Git configuration (public)
if: github.repository != 'hashicorp/vault-enterprise'
run: |
git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN}}@github.com".insteadOf https://github.com
- id: build
if: inputs.binary-tests && matrix.id == inputs.total-runners
env:
GOPRIVATE: github.com/hashicorp/*
run: time make ci-bootstrap dev
- uses: ./.github/actions/set-up-gotestsum
- id: run-go-tests
name: Run Go tests
timeout-minutes: ${{ fromJSON(env.TIMEOUT_IN_MINUTES) }}
env:
COMMIT_SHA: ${{ github.sha }}
run: |
set -exo pipefail
# Build the dynamically generated source files.
make prep
packages=$(echo "${{ toJSON(needs.test-matrix.outputs.matrix) }}" | jq -c -r --arg id "${{ matrix.id }}" '.[$id] | .packages')
if [ -z "$packages" ]; then
echo "no test packages to run"
exit 1
fi
# We don't want VAULT_LICENSE set when running Go tests, because that's
# not what developers have in their environments and it could break some
# tests; it would be like setting VAULT_TOKEN. However some non-Go
# CI commands, like the UI tests, shouldn't have to worry about licensing.
# So we provide the tests which want an externally supplied license with licenses
# via the VAULT_LICENSE_CI and VAULT_LICENSE_2 environment variables, and here we unset it.
# shellcheck disable=SC2034
VAULT_LICENSE=
# Assign test licenses to relevant variables if they aren't already
if [[ ${{ github.repository }} == 'hashicorp/vault' ]]; then
export VAULT_LICENSE_CI=${{ secrets.ci_license }}
export VAULT_LICENSE_2=${{ secrets.ci_license_2 }}
export HCP_API_ADDRESS=${{ secrets.HCP_API_ADDRESS }}
export HCP_AUTH_URL=${{ secrets.HCP_AUTH_URL }}
export HCP_CLIENT_ID=${{ secrets.HCP_CLIENT_ID }}
export HCP_CLIENT_SECRET=${{ secrets.HCP_CLIENT_SECRET }}
export HCP_RESOURCE_ID=${{ secrets.HCP_RESOURCE_ID }}
# Temporarily removing this variable to cause HCP Link tests
# to be skipped.
#export HCP_SCADA_ADDRESS=${{ secrets.HCP_SCADA_ADDRESS }}
fi
if [ -f bin/vault ]; then
VAULT_BINARY="$(pwd)/bin/vault"
export VAULT_BINARY
fi
# On a release branch, add a flag to rerun failed tests
# shellcheck disable=SC2193 # can get false positive for this comparision
if [[ "${{ github.base_ref }}" == release/* ]] || [[ -z "${{ github.base_ref }}" && "${{ github.ref_name }}" == release/* ]]
then
RERUN_FAILS="--rerun-fails"
fi
# shellcheck disable=SC2086 # can't quote RERUN_FAILS
GOARCH=${{ inputs.go-arch }} \
gotestsum --format=short-verbose \
--junitfile test-results/go-test/results-${{ matrix.id }}.xml \
--jsonfile test-results/go-test/results-${{ matrix.id }}.json \
--jsonfile-timing-events failure-summary-${{ matrix.id }}${{ inputs.name != '' && '-' || '' }}${{ inputs.name }}.json \
$RERUN_FAILS \
--packages "$packages" \
-- \
-tags "${{ inputs.go-tags }}" \
-timeout=${{ env.TIMEOUT_IN_MINUTES }}m \
-parallel=${{ inputs.go-test-parallelism }} \
${{ inputs.extra-flags }} \
- name: Prepare datadog-ci
if: github.repository == 'hashicorp/vault' && (success() || failure())
continue-on-error: true
run: |
curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci"
chmod +x /usr/local/bin/datadog-ci
- name: Upload test results to DataDog
continue-on-error: true
env:
DD_ENV: ci
run: |
if [[ ${{ github.repository }} == 'hashicorp/vault' ]]; then
export DATADOG_API_KEY=${{ secrets.DATADOG_API_KEY }}
fi
datadog-ci junit upload --service "$GITHUB_REPOSITORY" test-results/go-test/results-${{ matrix.id }}.xml
if: success() || failure()
- name: Archive test results
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
with:
name: test-results${{ inputs.name != '' && '-' || '' }}${{ inputs.name }}
path: test-results/go-test
if: success() || failure()
# GitHub Actions doesn't expose the job ID or the URL to the job execution,
# so we have to fetch it from the API
- name: Fetch job logs URL
uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6.4.1
if: success() || failure()
continue-on-error: true
with:
retries: 3
script: |
// We surround the whole script with a try-catch block, to avoid each of the matrix jobs
// displaying an error in the GHA workflow run annotations, which gets very noisy.
// If an error occurs, it will be logged so that we don't lose any information about the reason for failure.
try {
const fs = require("fs");
const result = await github.rest.actions.listJobsForWorkflowRun({
owner: context.repo.owner,
per_page: 100,
repo: context.repo.repo,
run_id: context.runId,
});
// Determine what job name to use for the query. These values are hardcoded, because GHA doesn't
// expose them in any of the contexts available within a workflow run.
let prefixToSearchFor;
switch ("${{ inputs.name }}") {
case "race":
prefixToSearchFor = 'Run Go tests with data race detection / test-go (${{ matrix.id }})'
break
case "fips":
prefixToSearchFor = 'Run Go tests with FIPS configuration / test-go (${{ matrix.id }})'
break
default:
prefixToSearchFor = 'Run Go tests / test-go (${{ matrix.id }})'
}
const jobData = result.data.jobs.filter(
(job) => job.name.startsWith(prefixToSearchFor)
);
const url = jobData[0].html_url;
const envVarName = "GH_JOB_URL";
const envVar = envVarName + "=" + url;
const envFile = process.env.GITHUB_ENV;
fs.appendFile(envFile, envVar, (err) => {
if (err) throw err;
console.log("Successfully set " + envVarName + " to: " + url);
});
} catch (error) {
console.log("Error: " + error);
return
}
- name: Prepare failure summary
if: success() || failure()
continue-on-error: true
run: |
# This jq query filters out successful tests, leaving only the failures.
# Then, it formats the results into rows of a Markdown table.
# An example row will resemble this:
# | github.com/hashicorp/vault/package | TestName | fips | 0 | 2 | [view results](github.com/link-to-logs) |
jq -r -n 'inputs
| select(.Action == "fail")
| "| ${{inputs.name}} | \(.Package) | \(.Test // "-") | \(.Elapsed) | ${{ matrix.id }} | [view test results :scroll:](${{ env.GH_JOB_URL }}) |"' \
failure-summary-${{ matrix.id }}${{ inputs.name != '' && '-' || '' }}${{inputs.name}}.json \
>> failure-summary-${{ matrix.id }}${{ inputs.name != '' && '-' || '' }}${{inputs.name}}.md
- name: Upload failure summary
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
if: success() || failure()
with:
name: failure-summary
path: failure-summary-${{ matrix.id }}${{ inputs.name != '' && '-' || '' }}${{inputs.name}}.md
test-collect-reports:
if: ${{ ! cancelled() }}
needs: test-go
runs-on: ${{ fromJSON(inputs.runs-on) }}
steps:
- uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
with:
path: test-results/go-test
key: go-test-reports-${{ github.run_number }}
restore-keys: go-test-reports-
- uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
name: test-results
path: test-results/go-test
- run: |
ls -lhR test-results/go-test
find test-results/go-test -mindepth 1 -mtime +3 -delete
ls -lhR test-results/go-test