From 8089ff7c544f074660ceb9ddd639a01279401665 Mon Sep 17 00:00:00 2001
From: hamid ghaf <hamid@hashicorp.com>
Date: Thu, 17 Aug 2023 13:22:22 -0700
Subject: [PATCH] running tests tagged with testonly build tag

---
 .github/workflows/ci.yml      | 19 +++++++++++++++++++
 .github/workflows/test-go.yml | 35 +++++++++++++++++++++++++++++++++--
 2 files changed, 52 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index feb50b0c485b..b62e9737a2e4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -119,6 +119,25 @@ jobs:
       enterprise: ${{ needs.setup.outputs.enterprise }}
     secrets: inherit
 
+  test-go-testonly:
+    name: Run Go tests tagged with testonly
+    needs:
+      - setup
+      - verify-changes
+    # Don't run this job for docs/ui only PRs
+    if: |
+      needs.verify-changes.outputs.is_docs_change == 'false' &&
+      needs.verify-changes.outputs.is_ui_change == 'false'
+    uses: ./.github/workflows/test-go.yml
+    with:
+      testonly: true
+      total-runners: 2 # test runners cannot be less than 2
+      go-arch: amd64
+      go-tags: '${{ needs.setup.outputs.go-tags }},deadlock,testonly'
+      runs-on: ${{ needs.setup.outputs.compute-large }}
+      enterprise: ${{ needs.setup.outputs.enterprise }}
+    secrets: inherit
+
   test-go-race:
     name: Run Go tests with data race detection
     needs:
diff --git a/.github/workflows/test-go.yml b/.github/workflows/test-go.yml
index c612d9138954..3053a3ac34ee 100644
--- a/.github/workflows/test-go.yml
+++ b/.github/workflows/test-go.yml
@@ -52,6 +52,11 @@ on:
         required: false
         default: 60
         type: number
+      testonly:
+        description: Whether to run the tests tagged with testonly.
+        required: false
+        default: false
+        type: boolean
 
 env: ${{ fromJSON(inputs.env-vars) }}
 
@@ -108,16 +113,42 @@ jobs:
       - name: List cached results
         id: list-cached-results
         run: ls -lhR test-results/go-test
-      - name: Build matrix excluding binary and integration tests
+      - name: Build matrix excluding binary, integration, and testonly tests
         id: build-non-binary
+        if: ${{ !inputs.testonly }}
         env:
           GOPRIVATE: github.com/hashicorp/*
         run: |
+          # testonly tests need additional build tag though let's exclude them anyway for clarity
           (
-            go list ./... | grep -v "_binary" | grep -v "vault/integ" | gotestsum tool ci-matrix --debug \
+            go list ./... | grep -v "_binary" | grep -v "vault/integ" | grep -v "testonly" | gotestsum tool ci-matrix --debug \
               --partitions "${{ inputs.total-runners }}" \
               --timing-files 'test-results/go-test/*.json' > matrix.json
           )
+      - name: Build matrix for tests tagged with testonly
+        if: ${{ inputs.testonly }}
+        env:
+          GOPRIVATE: github.com/hashicorp/*
+          run: |
+              set -exo pipefail
+
+              # testonly tagged tests need an additional tag
+              go list -tags=testonly ./... | grep "testonly" | gotestsum tool ci-matrix --debug \
+                --partitions "${{ inputs.total-runners }}" \
+                --timing-files 'test-results/go-test/*.json' > matrix.json
+
+              # hardcoding vault package index as there is only one test package tagged with testonly
+              # in the external tests. However, it would be also good to run the vault package with
+              # to validate the tag and also validate the vault package tests that are tagged with
+              # test only. There is only going to be two tests runners for testonly job.
+              jq --arg PKG "github.com/hashicorp/vault/vault" \
+                '.include[1] = {
+                  "id": 1,
+                  "estimatedRuntime": "N/A",
+                  "packages": $PKG,
+                  "description": "partition 1 - vault test packages"
+              }' matrix.json > new-matrix.json
+              mv new-matrix.json matrix.json
       - name: Capture list of binary tests
         if: inputs.binary-tests
         id: list-binary-tests