test-go.yml

on:
  workflow_call:
    inputs:
      go-arch:
        description: The execution architecture (arm, amd64, etc.)
        required: true
        type: string
      enterprise:
        description: A flag indicating if this workflow is executing for the enterprise repository.
        required: true
        type: string
      total-runners:
        description: Number of runners to use for executing non-binary tests.
        required: true
        type: string
      binary-tests:
        description: Whether to run the binary tests.
        required: false
        default: false
        type: boolean
      env-vars:
        description: A map of environment variables as JSON.
        required: false
        type: string
        default: '{}'
      extra-flags:
        description: A space-separated list of additional build flags.
        required: false
        type: string
        default: ''
      runs-on:
        description: An expression indicating which kind of runners to use.
        required: false
        type: string
        default: ubuntu-latest
      go-tags:
        description: A comma-separated list of additional build tags to consider satisfied during the build.
        required: false
        type: string
      name:
        description: A suffix to append to archived test results
        required: false
        default: ''
        type: string
      go-test-parallelism:
        description: The parallelism parameter for Go tests
        required: false
        default: 20
        type: number
      timeout-minutes:
        description: The maximum number of minutes that this workflow should run
        required: false
        default: 60
        type: number
      testonly:
        description: Whether to run the tests tagged with testonly.
        required: false
        default: false
        type: boolean

env: ${{ fromJSON(inputs.env-vars) }}

jobs:
  test-matrix:
    permissions:
      id-token: write  # Note: this permission is explicitly required for Vault auth
      contents: read
    runs-on: ${{ fromJSON(inputs.runs-on) }}
    steps:
      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
      - uses: ./.github/actions/set-up-go
        with:
          github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
      - name: Authenticate to Vault
        id: vault-auth
        if: github.repository == 'hashicorp/vault-enterprise'
        run: vault-auth
      - name: Fetch Secrets
        id: secrets
        if: github.repository == 'hashicorp/vault-enterprise'
        uses: hashicorp/vault-action@130d1f5f4fe645bb6c83e4225c04d64cfb62de6e
        with:
          url: ${{ steps.vault-auth.outputs.addr }}
          caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
          token: ${{ steps.vault-auth.outputs.token }}
          secrets: |
            kv/data/github/${{ github.repository }}/datadog-ci DATADOG_API_KEY;
            kv/data/github/${{ github.repository }}/github-token username-and-token | github-token;
            kv/data/github/${{ github.repository }}/license license_1 | VAULT_LICENSE_CI;
            kv/data/github/${{ github.repository }}/license license_2 | VAULT_LICENSE_2;
            kv/data/github/${{ github.repository }}/hcp-link HCP_API_ADDRESS;
            kv/data/github/${{ github.repository }}/hcp-link HCP_AUTH_URL;
            kv/data/github/${{ github.repository }}/hcp-link HCP_CLIENT_ID;
            kv/data/github/${{ github.repository }}/hcp-link HCP_CLIENT_SECRET;
            kv/data/github/${{ github.repository }}/hcp-link HCP_RESOURCE_ID;
      - id: setup-git-private
        name: Setup Git configuration (private)
        if: github.repository == 'hashicorp/vault-enterprise'
        run: |
          git config --global url."https://${{ steps.secrets.outputs.github-token }}@github.com".insteadOf https://github.com
      - id: setup-git-public
        name: Setup Git configuration (public)
        if: github.repository != 'hashicorp/vault-enterprise'
        run: |
          git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN}}@github.com".insteadOf https://github.com
      - uses: ./.github/actions/set-up-gotestsum
      - run: mkdir -p test-results/go-test
      - uses: actions/cache/restore@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
        with:
          path: test-results/go-test
          key: go-test-reports-${{ github.run_number }}
          restore-keys: go-test-reports-
      - name: List cached results
        id: list-cached-results
        run: ls -lhR test-results/go-test
      - name: Build matrix excluding binary, integration, and testonly tests
        id: build-non-binary
        if: ${{ !inputs.testonly }}
        env:
          GOPRIVATE: github.com/hashicorp/*
        run: |
          # testonly tests need additional build tag though let's exclude them anyway for clarity
          (
            go list ./... | grep -v "_binary" | grep -v "vault/integ" | grep -v "testonly" | gotestsum tool ci-matrix --debug \
              --partitions "${{ inputs.total-runners }}" \
              --timing-files 'test-results/go-test/*.json' > matrix.json
          )
      - name: Build matrix for tests tagged with testonly
        if: ${{ inputs.testonly }}
        env:
          GOPRIVATE: github.com/hashicorp/*
        run: |
           set -exo pipefail

           # testonly tagged tests need an additional tag
           go list -tags=testonly ./... | grep "testonly" | gotestsum tool ci-matrix --debug \
             --partitions "${{ inputs.total-runners }}" \
             --timing-files 'test-results/go-test/*.json' > testonly_matrix.json

           # hardcoding vault package index as there is only one test package tagged with testonly
           # in the external tests. However, it would be also good to run the vault package with
           # to validate the tag and also validate the vault package tests that are tagged with
           # test only. There is only going to be two tests runners for testonly job.
          if [ "${{ inputs.enterprise }}" == "true" ]; then
            export VAULT_TEST_PACKAGE="github.com/hashicorp/vault-enterprise/vault"
          else
            export VAULT_TEST_PACKAGE="github.com/hashicorp/vault/vault"
          fi
          jq --arg VAULT_PKG "${VAULT_TEST_PACKAGE}" \
            '.include[1] = {
              "id": 1,
              "estimatedRuntime": "N/A",
              "packages": $VAULT_PKG,
              "description": "partition 1 - vault test packages"
          }' testonly_matrix.json > matrix.json

      - name: Capture list of binary tests
        if: inputs.binary-tests
        id: list-binary-tests
        run: |
          LIST="$(go list ./... | grep "_binary" | xargs)"
          echo "list=$LIST" >> "$GITHUB_OUTPUT"
      - name: Build complete matrix
        id: build
        run: |
            set -exo pipefail
            matrix_file="matrix.json"
            if [ "${{ inputs.binary-tests}}" == "true" ] && [ -n "${{ steps.list-binary-tests.outputs.list }}" ]; then
              export BINARY_TESTS="${{ steps.list-binary-tests.outputs.list }}"
              jq --arg BINARY "${BINARY_TESTS}" --arg BINARY_INDEX "${{ inputs.total-runners }}" \
                '.include += [{
                  "id": $BINARY_INDEX,
                  "estimatedRuntime": "N/A",
                  "packages": $BINARY,
                  "description": "partition $BINARY_INDEX - binary test packages"
              }]' matrix.json > new-matrix.json
              matrix_file="new-matrix.json"
            fi
            # convert the json to a map keyed by id
            (
              echo -n "matrix="
                jq -c \
                '.include | map( { (.id|tostring): . } ) | add' "$matrix_file"
            ) >> "$GITHUB_OUTPUT"
            # extract an array of ids from the json
            (
              echo -n "matrix_ids="
              jq -c \
                '[ .include[].id | tostring ]' "$matrix_file"
            ) >> "$GITHUB_OUTPUT"
    outputs:
      matrix: ${{ steps.build.outputs.matrix }}
      matrix_ids: ${{ steps.build.outputs.matrix_ids }}

  test-go:
    needs: test-matrix
    permissions:
      actions: read
      contents: read
      id-token: write  # Note: this permission is explicitly required for Vault auth
    runs-on: ${{ fromJSON(inputs.runs-on) }}
    strategy:
      fail-fast: false
      matrix:
        id: ${{ fromJSON(needs.test-matrix.outputs.matrix_ids) }}
    env:
      GOPRIVATE: github.com/hashicorp/*
      TIMEOUT_IN_MINUTES: ${{ inputs.timeout-minutes }}
    steps:
      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
      - uses: ./.github/actions/set-up-go
        with:
          github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
      - name: Authenticate to Vault
        id: vault-auth
        if: github.repository == 'hashicorp/vault-enterprise'
        run: vault-auth
      - name: Fetch Secrets
        id: secrets
        if: github.repository == 'hashicorp/vault-enterprise'
        uses: hashicorp/vault-action@130d1f5f4fe645bb6c83e4225c04d64cfb62de6e
        with:
          url: ${{ steps.vault-auth.outputs.addr }}
          caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
          token: ${{ steps.vault-auth.outputs.token }}
          secrets: |
            kv/data/github/${{ github.repository }}/datadog-ci DATADOG_API_KEY;
            kv/data/github/${{ github.repository }}/github-token username-and-token | github-token;
            kv/data/github/${{ github.repository }}/license license_1 | VAULT_LICENSE_CI;
            kv/data/github/${{ github.repository }}/license license_2 | VAULT_LICENSE_2;
            kv/data/github/${{ github.repository }}/hcp-link HCP_API_ADDRESS;
            kv/data/github/${{ github.repository }}/hcp-link HCP_AUTH_URL;
            kv/data/github/${{ github.repository }}/hcp-link HCP_CLIENT_ID;
            kv/data/github/${{ github.repository }}/hcp-link HCP_CLIENT_SECRET;
            kv/data/github/${{ github.repository }}/hcp-link HCP_RESOURCE_ID;
      - id: setup-git-private
        name: Setup Git configuration (private)
        if: github.repository == 'hashicorp/vault-enterprise'
        run: |
          git config --global url."https://${{ steps.secrets.outputs.github-token }}@github.com".insteadOf https://github.com
      - id: setup-git-public
        name: Setup Git configuration (public)
        if: github.repository != 'hashicorp/vault-enterprise'
        run: |
          git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN}}@github.com".insteadOf https://github.com
      - id: build
        if: inputs.binary-tests && matrix.id == inputs.total-runners
        env:
          GOPRIVATE: github.com/hashicorp/*
        run: time make ci-bootstrap dev
      - uses: ./.github/actions/set-up-gotestsum
      - id: run-go-tests
        name: Run Go tests
        timeout-minutes: ${{ fromJSON(env.TIMEOUT_IN_MINUTES) }}
        env:
          COMMIT_SHA: ${{ github.sha }}
        run: |
          set -exo pipefail

          # Build the dynamically generated source files.
          make prep

          packages=$(echo "${{ toJSON(needs.test-matrix.outputs.matrix) }}" | jq -c -r --arg id "${{ matrix.id }}" '.[$id] | .packages')

          if [ -z "$packages" ]; then
            echo "no test packages to run"
            exit 1
          fi
          # We don't want VAULT_LICENSE set when running Go tests, because that's
          # not what developers have in their environments and it could break some
          # tests; it would be like setting VAULT_TOKEN.  However some non-Go
          # CI commands, like the UI tests, shouldn't have to worry about licensing.
          # So we provide the tests which want an externally supplied license with licenses
          # via the VAULT_LICENSE_CI and VAULT_LICENSE_2 environment variables, and here we unset it.
          # shellcheck disable=SC2034
          VAULT_LICENSE=

          # Assign test licenses to relevant variables if they aren't already
          if [[ ${{ github.repository }} == 'hashicorp/vault' ]]; then
            export VAULT_LICENSE_CI=${{ secrets.ci_license }}
            export VAULT_LICENSE_2=${{ secrets.ci_license_2 }}
            export HCP_API_ADDRESS=${{ secrets.HCP_API_ADDRESS }}
            export HCP_AUTH_URL=${{ secrets.HCP_AUTH_URL }}
            export HCP_CLIENT_ID=${{ secrets.HCP_CLIENT_ID }}
            export HCP_CLIENT_SECRET=${{ secrets.HCP_CLIENT_SECRET }}
            export HCP_RESOURCE_ID=${{ secrets.HCP_RESOURCE_ID }}
            # Temporarily removing this variable to cause HCP Link tests
            # to be skipped.
            #export HCP_SCADA_ADDRESS=${{ secrets.HCP_SCADA_ADDRESS }}
          fi

          if [ -f bin/vault ]; then
            VAULT_BINARY="$(pwd)/bin/vault"
            export VAULT_BINARY
          fi

          # On a release branch, add a flag to rerun failed tests
          # shellcheck disable=SC2193 # can get false positive for this comparision
          if [[  "${{ github.base_ref }}" == release/* ]] || [[  -z "${{ github.base_ref }}" && "${{ github.ref_name }}" == release/* ]]
          then
           RERUN_FAILS="--rerun-fails"
          fi

          # shellcheck disable=SC2086 # can't quote RERUN_FAILS
          GOARCH=${{ inputs.go-arch }} \
            gotestsum --format=short-verbose \
              --junitfile test-results/go-test/results-${{ matrix.id }}.xml \
              --jsonfile test-results/go-test/results-${{ matrix.id }}.json \
              --jsonfile-timing-events failure-summary-${{ matrix.id }}${{ inputs.name != '' && '-' || '' }}${{ inputs.name }}.json \
              $RERUN_FAILS \
              --packages "$packages" \
              -- \
              -tags "${{ inputs.go-tags }}" \
              -timeout=${{ env.TIMEOUT_IN_MINUTES }}m \
              -parallel=${{ inputs.go-test-parallelism }} \
              ${{ inputs.extra-flags }} \
      - name: Prepare datadog-ci
        if: github.repository == 'hashicorp/vault' && (success() || failure())
        continue-on-error: true
        run: |
          curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci"
          chmod +x /usr/local/bin/datadog-ci
      - name: Upload test results to DataDog
        continue-on-error: true
        env:
          DD_ENV: ci
        run: |
          if [[ ${{ github.repository }} == 'hashicorp/vault' ]]; then
            export DATADOG_API_KEY=${{ secrets.DATADOG_API_KEY }}
          fi
          datadog-ci junit upload --service "$GITHUB_REPOSITORY" test-results/go-test/results-${{ matrix.id }}.xml
        if: success() || failure()
      - name: Archive test results
        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
        with:
          name: test-results${{ inputs.name != '' && '-' || '' }}${{ inputs.name }}
          path: test-results/go-test
        if: success() || failure()
      # GitHub Actions doesn't expose the job ID or the URL to the job execution,
      # so we have to fetch it from the API
      - name: Fetch job logs URL
        uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6.4.1
        if: success() || failure()
        continue-on-error: true
        with:
          retries: 3
          script: |
            // We surround the whole script with a try-catch block, to avoid each of the matrix jobs
            // displaying an error in the GHA workflow run annotations, which gets very noisy.
            // If an error occurs, it will be logged so that we don't lose any information about the reason for failure.
            try {
              const fs = require("fs");
              const result = await github.rest.actions.listJobsForWorkflowRun({
                owner: context.repo.owner,
                per_page: 100,
                repo: context.repo.repo,
                run_id: context.runId,
              });

              // Determine what job name to use for the query. These values are hardcoded, because GHA doesn't
              // expose them in any of the contexts available within a workflow run.
              let prefixToSearchFor;
              switch ("${{ inputs.name }}") {
                case "race":
                  prefixToSearchFor = 'Run Go tests with data race detection / test-go (${{ matrix.id }})'
                  break
                case "fips":
                  prefixToSearchFor = 'Run Go tests with FIPS configuration / test-go (${{ matrix.id }})'
                  break
                default:
                  prefixToSearchFor = 'Run Go tests / test-go (${{ matrix.id }})'
              }

              const jobData = result.data.jobs.filter(
                (job) => job.name.startsWith(prefixToSearchFor)
              );
              const url = jobData[0].html_url;
              const envVarName = "GH_JOB_URL";
              const envVar = envVarName + "=" + url;
              const envFile = process.env.GITHUB_ENV;

              fs.appendFile(envFile, envVar, (err) => {
                if (err) throw err;
                console.log("Successfully set " + envVarName + " to: " + url);
              });
            } catch (error) {
              console.log("Error: " + error);
              return
            }
      - name: Prepare failure summary
        if: success() || failure()
        continue-on-error: true
        run: |
          # This jq query filters out successful tests, leaving only the failures.
          # Then, it formats the results into rows of a Markdown table.
          # An example row will resemble this:
          # | github.com/hashicorp/vault/package | TestName | fips | 0 | 2 | [view results](github.com/link-to-logs) |
          jq -r -n 'inputs
          | select(.Action == "fail")
          | "| ${{inputs.name}} | \(.Package) | \(.Test // "-") | \(.Elapsed) | ${{ matrix.id }} | [view test results :scroll:](${{ env.GH_JOB_URL }}) |"' \
          failure-summary-${{ matrix.id }}${{ inputs.name != '' && '-' || '' }}${{inputs.name}}.json \
          >> failure-summary-${{ matrix.id }}${{ inputs.name != '' && '-' || '' }}${{inputs.name}}.md
      - name: Upload failure summary
        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
        if: success() || failure()
        with:
          name: failure-summary
          path: failure-summary-${{ matrix.id }}${{ inputs.name != '' && '-' || '' }}${{inputs.name}}.md

  test-collect-reports:
    if: ${{ ! cancelled() }}
    needs: test-go
    runs-on: ${{ fromJSON(inputs.runs-on) }}
    steps:
      - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
        with:
          path: test-results/go-test
          key: go-test-reports-${{ github.run_number }}
          restore-keys: go-test-reports-
      - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: test-results
          path: test-results/go-test
      - run: |
          ls -lhR test-results/go-test
          find test-results/go-test -mindepth 1 -mtime +3 -delete
          ls -lhR test-results/go-test