Merge branch 'main' into ml4-patch-5

.github/actions/containerize/action.yml

@@ -90,7 +90,7 @@ runs:
         [[ ! -d "$dest_dir" ]] && mkdir -p "$dest_dir"
         [[ ! -f "$dest_path" ]] && cp ${{ inputs.vault-binary-path }} "${dest_path}"
     - if: inputs.docker == 'true'
-      uses: hashicorp/actions-docker-build@v2
+      uses: hashicorp/actions-docker-build@f22d5ac7d36868afaa4be1cc1203ec1b5865cadd
       with:
         arch: ${{ inputs.goarch }}
         do_zip_extract_step: 'false' # Don't download and extract an already present binary
@@ -99,7 +99,7 @@ runs:
         revision: ${{ steps.vars.outputs.revision }}
         version: ${{ steps.vars.outputs.container-version }}
     - if: inputs.redhat == 'true'
-      uses: hashicorp/actions-docker-build@v2
+      uses: hashicorp/actions-docker-build@f22d5ac7d36868afaa4be1cc1203ec1b5865cadd
       with:
         arch: ${{ inputs.goarch }}
         do_zip_extract_step: 'false' # Don't download and extract an already present binary

.github/workflows/build-artifacts-ce.yml

@@ -93,7 +93,7 @@ jobs:
             redhat: true
           - goos: linux
             goarch: arm64
-            redhat: false
+            redhat: true
       fail-fast: true
     runs-on: ${{ fromJSON(inputs.compute-build) }}
     name: (${{ matrix.goos }}, ${{ matrix.goarch }})

.github/workflows/ci.yml

@@ -194,6 +194,7 @@ jobs:
           secrets: |
             kv/data/github/hashicorp/vault-enterprise/github-token username-and-token | PRIVATE_REPO_GITHUB_TOKEN;
             kv/data/github/hashicorp/vault-enterprise/license license_1 | VAULT_LICENSE;
+            kv/data/github/${{ github.repository }}/datadog-ci DATADOG_API_KEY;
       - if: needs.setup.outputs.is-enterprise == 'true'
         name: Set up Git
         run: git config --global url."https://${{ steps.secrets.outputs.PRIVATE_REPO_GITHUB_TOKEN }}@github.com".insteadOf https://github.com
@@ -217,6 +218,32 @@ jobs:
         with:
           name: test-results-ui
           path: ui/test-results
+      - name: Prepare datadog-ci
+        if: (github.repository == 'hashicorp/vault' || github.repository == 'hashicorp/vault-enterprise') && (success() || failure())
+        continue-on-error: true
+        run: |
+          if type datadog-ci > /dev/null 2>&1; then
+            exit 0
+          fi
+          # Curl does not always exit 1 if things go wrong. To determine if this is successful
+          # we'll silence all non-error output and check the results to determine success.
+          if ! out="$(curl -sSL --fail https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64 --output /usr/local/bin/datadog-ci 2>&1)"; then
+            printf "failed to download datadog-ci: %s" "$out"
+          fi
+          if [[ -n "$out" ]]; then
+            printf "failed to download datadog-ci: %s" "$out"
+          fi
+          chmod +x /usr/local/bin/datadog-ci
+      - name: Upload test results to DataDog
+        if: success() || failure()
+        continue-on-error: true
+        env:
+          DD_ENV: ci
+        run: |
+          if [[ ${{ github.repository }} == 'hashicorp/vault' ]]; then
+            export DATADOG_API_KEY=${{ secrets.DATADOG_API_KEY }}
+          fi
+          datadog-ci junit upload --service "$GITHUB_REPOSITORY" 'ui/test-results/qunit/results.xml'
       - if: always()
         uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4
         with:

.github/workflows/enos-run-k8s.yml

@@ -26,7 +26,7 @@ env:
 jobs:
   enos:
     name: Integration
-    runs-on: ubuntu-latest
+    runs-on: ${{ fromJSON(contains(inputs.artifact-name, 'vault-enterprise') && (contains(inputs.artifact-name, 'arm64') && '["self-hosted","ondemand","os=ubuntu-arm","type=c6g.xlarge"]' || '["self-hosted","linux","small"]') || '"ubuntu-latest"') }}
     env:
       GITHUB_TOKEN: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
     steps:

builtin/credential/cert/path_login.go

@@ -693,6 +693,15 @@ func (b *backend) loadTrustedCerts(ctx context.Context, storage logical.Storage,
 			conf.QueryAllServers = conf.QueryAllServers || entry.OcspQueryAllServers
 			conf.OcspThisUpdateMaxAge = entry.OcspThisUpdateMaxAge
 			conf.OcspMaxRetries = entry.OcspMaxRetries
+
+			if len(entry.OcspCaCertificates) > 0 {
+				certs, err := certutil.ParseCertsPEM([]byte(entry.OcspCaCertificates))
+				if err != nil {
+					b.Logger().Error("failed to parse ocsp_ca_certificates", "name", name, "error", err)
+					continue
+				}
+				conf.ExtraCas = certs
+			}
 		}
 	}
 

builtin/credential/cert/path_login_test.go

@@ -5,6 +5,7 @@ package cert
 
 import (
 	"context"
+	"crypto"
 	"crypto/tls"
 	"crypto/x509"
 	"crypto/x509/pkix"
@@ -281,19 +282,6 @@ func TestCert_RoleResolve_RoleDoesNotExist(t *testing.T) {
 }
 
 func TestCert_RoleResolveOCSP(t *testing.T) {
-	cases := []struct {
-		name        string
-		failOpen    bool
-		certStatus  int
-		errExpected bool
-	}{
-		{"failFalseGoodCert", false, ocsp.Good, false},
-		{"failFalseRevokedCert", false, ocsp.Revoked, true},
-		{"failFalseUnknownCert", false, ocsp.Unknown, true},
-		{"failTrueGoodCert", true, ocsp.Good, false},
-		{"failTrueRevokedCert", true, ocsp.Revoked, true},
-		{"failTrueUnknownCert", true, ocsp.Unknown, false},
-	}
 	certTemplate := &x509.Certificate{
 		Subject: pkix.Name{
 			CommonName: "example.com",
@@ -332,15 +320,76 @@ func TestCert_RoleResolveOCSP(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
+	tempDir, connState2, err := generateTestCertAndConnState(t, certTemplate)
+	if err != nil {
+		t.Fatalf("error testing connection state: %v", err)
+	}
+	ca2, err := ioutil.ReadFile(filepath.Join(tempDir, "ca_cert.pem"))
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	issuer2 := parsePEM(ca2)
+	pkf2, err := ioutil.ReadFile(filepath.Join(tempDir, "ca_key.pem"))
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	pk2, err := certutil.ParsePEMBundle(string(pkf2))
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	type caData struct {
+		privateKey crypto.Signer
+		caBytes    []byte
+		caChain    []*x509.Certificate
+		connState  tls.ConnectionState
+	}
+
+	ca1Data := caData{
+		pk.PrivateKey,
+		ca,
+		issuer,
+		connState,
+	}
+	ca2Data := caData{
+		pk2.PrivateKey,
+		ca2,
+		issuer2,
+		connState2,
+	}
+
+	cases := []struct {
+		name        string
+		failOpen    bool
+		certStatus  int
+		errExpected bool
+		caData      caData
+		ocspCaCerts string
+	}{
+		{name: "failFalseGoodCert", certStatus: ocsp.Good, caData: ca1Data},
+		{name: "failFalseRevokedCert", certStatus: ocsp.Revoked, errExpected: true, caData: ca1Data},
+		{name: "failFalseUnknownCert", certStatus: ocsp.Unknown, errExpected: true, caData: ca1Data},
+		{name: "failTrueGoodCert", failOpen: true, certStatus: ocsp.Good, caData: ca1Data},
+		{name: "failTrueRevokedCert", failOpen: true, certStatus: ocsp.Revoked, errExpected: true, caData: ca1Data},
+		{name: "failTrueUnknownCert", failOpen: true, certStatus: ocsp.Unknown, caData: ca1Data},
+		{name: "failFalseGoodCertExtraCas", certStatus: ocsp.Good, caData: ca2Data, ocspCaCerts: string(pkf2)},
+		{name: "failFalseRevokedCertExtraCas", certStatus: ocsp.Revoked, errExpected: true, caData: ca2Data, ocspCaCerts: string(pkf2)},
+		{name: "failFalseUnknownCertExtraCas", certStatus: ocsp.Unknown, errExpected: true, caData: ca2Data, ocspCaCerts: string(pkf2)},
+		{name: "failTrueGoodCertExtraCas", failOpen: true, certStatus: ocsp.Good, caData: ca2Data, ocspCaCerts: string(pkf2)},
+		{name: "failTrueRevokedCertExtraCas", failOpen: true, certStatus: ocsp.Revoked, errExpected: true, caData: ca2Data, ocspCaCerts: string(pkf2)},
+		{name: "failTrueUnknownCertExtraCas", failOpen: true, certStatus: ocsp.Unknown, caData: ca2Data, ocspCaCerts: string(pkf2)},
+	}
+
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
-			resp, err := ocsp.CreateResponse(issuer[0], issuer[0], ocsp.Response{
+			resp, err := ocsp.CreateResponse(c.caData.caChain[0], c.caData.caChain[0], ocsp.Response{
 				Status:       c.certStatus,
 				SerialNumber: certTemplate.SerialNumber,
 				ProducedAt:   time.Now(),
 				ThisUpdate:   time.Now(),
 				NextUpdate:   time.Now().Add(time.Hour),
-			}, pk.PrivateKey)
+			}, c.caData.privateKey)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -351,18 +400,18 @@ func TestCert_RoleResolveOCSP(t *testing.T) {
 			var resolveStep logicaltest.TestStep
 			var loginStep logicaltest.TestStep
 			if c.errExpected {
-				loginStep = testAccStepLoginWithNameInvalid(t, connState, "web")
-				resolveStep = testAccStepResolveRoleOCSPFail(t, connState, "web")
+				loginStep = testAccStepLoginWithNameInvalid(t, c.caData.connState, "web")
+				resolveStep = testAccStepResolveRoleOCSPFail(t, c.caData.connState, "web")
 			} else {
-				loginStep = testAccStepLoginWithName(t, connState, "web")
-				resolveStep = testAccStepResolveRoleWithName(t, connState, "web")
+				loginStep = testAccStepLoginWithName(t, c.caData.connState, "web")
+				resolveStep = testAccStepResolveRoleWithName(t, c.caData.connState, "web")
 			}
 			logicaltest.Test(t, logicaltest.TestCase{
 				CredentialBackend: b,
 				Steps: []logicaltest.TestStep{
-					testAccStepCertWithExtraParams(t, "web", ca, "foo", allowed{dns: "example.com"}, false,
-						map[string]interface{}{"ocsp_enabled": true, "ocsp_fail_open": c.failOpen}),
-					testAccStepReadCertPolicy(t, "web", false, map[string]interface{}{"ocsp_enabled": true, "ocsp_fail_open": c.failOpen}),
+					testAccStepCertWithExtraParams(t, "web", c.caData.caBytes, "foo", allowed{dns: "example.com"}, false,
+						map[string]interface{}{"ocsp_enabled": true, "ocsp_fail_open": c.failOpen, "ocsp_ca_certificates": c.ocspCaCerts}),
+					testAccStepReadCertPolicy(t, "web", false, map[string]interface{}{"ocsp_enabled": true, "ocsp_fail_open": c.failOpen, "ocsp_ca_certificates": c.ocspCaCerts}),
 					loginStep,
 					resolveStep,
 				},

builtin/logical/aws/path_static_roles.go

@@ -219,6 +219,12 @@ func (b *backend) pathStaticRolesWrite(ctx context.Context, req *logical.Request
 		if err != nil {
 			return nil, fmt.Errorf("expected an item with name %q, but got an error: %w", config.Name, err)
 		}
+		// check if i is nil to prevent panic because
+		// 1. PopByKey returns nil if the key does not exist; and
+		// 2. the static cred queue is not repopulated on reload (see VAULT-30877)
+		if i == nil {
+			return nil, fmt.Errorf("expected an item with name %q, but got nil", config.Name)
+		}
 		i.Value = config
 		// update the next rotation to occur at now + the new rotation period
 		i.Priority = time.Now().Add(config.RotationPeriod).Unix()

builtin/logical/pki/backend_test.go

@@ -4114,6 +4114,7 @@ func TestBackend_RevokePlusTidy_Intermediate(t *testing.T) {
 			"tidy_revocation_queue":                 false,
 			"tidy_cross_cluster_revoked_certs":      false,
 			"tidy_cert_metadata":                    false,
+			"tidy_cmpv2_nonce_store":                false,
 			"pause_duration":                        "0s",
 			"state":                                 "Finished",
 			"error":                                 nil,
@@ -4136,6 +4137,7 @@ func TestBackend_RevokePlusTidy_Intermediate(t *testing.T) {
 			"acme_account_deleted_count":            json.Number("0"),
 			"total_acme_account_count":              json.Number("0"),
 			"cert_metadata_deleted_count":           json.Number("0"),
+			"cmpv2_nonce_deleted_count":             json.Number("0"),
 		}
 		// Let's copy the times from the response so that we can use deep.Equal()
 		timeStarted, ok := tidyStatus.Data["time_started"]

builtin/logical/pki/cmpv2_util_oss.go

@@ -0,0 +1,16 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+//go:build !enterprise
+
+package pki
+
+import (
+	"context"
+
+	"github.com/hashicorp/vault/sdk/logical"
+)
+
+func (b *backend) doTidyCMPV2NonceStore(_ context.Context, _ logical.Storage) error {
+	return nil
+}

builtin/logical/pki/fields.go

@@ -598,6 +598,11 @@ primary node.`,
 		Description: `Set to true to enable tidying up certificate metadata`,
 	}
 
+	fields["tidy_cmpv2_nonce_store"] = &framework.FieldSchema{
+		Type:        framework.TypeBool,
+		Description: `Set to true to enable tidying up the CMPv2 nonce store`,
+	}
+
 	return fields
 }