Fix/enable rate limiter for account clients (#1605)

fix: enable rate limiter for account clients
Dynatrace · Oct 15, 2024 · 0e19c4d · 0e19c4d
1 parent 7805260
commit 0e19c4d
Show file tree

Hide file tree

Showing 9 changed files with 30 additions and 29 deletions.
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
@@ -37,7 +37,7 @@ jobs:
       - name: 🛠️ Set up Go 1.x
         uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 #v5.0.2
         with:
-          go-version: '~1.22'
+          go-version: '~1.23'
 
       - name: 🏗️ Compile
         run: make compile

diff --git a/.github/workflows/dependencies-and-licenses.yml b/.github/workflows/dependencies-and-licenses.yml
@@ -17,7 +17,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 #v5.0.2
         with:
-          go-version: '~1.22'
+          go-version: '~1.23'
       - name: Install go-licence-detector
         run: |
           go install go.elastic.co/[email protected]

diff --git a/.github/workflows/end-to-end-test.yml b/.github/workflows/end-to-end-test.yml
@@ -56,7 +56,7 @@ jobs:
     - name: Set up Go 1.x
       uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 #v5.0.2
       with:
-        go-version: '~1.22'
+        go-version: '~1.23'
 
     - name: 🏁 Build release binaries
       run: make build-release
@@ -80,7 +80,7 @@ jobs:
     - name: Set up Go 1.x
       uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 #v5.0.2
       with:
-        go-version: '~1.22'
+        go-version: '~1.23'
 
     - name: 🌎 Integration test
       run: make integration-test testopts="--junitfile test-result-integration.xml"
@@ -121,7 +121,7 @@ jobs:
     - name: Set up Go 1.x
       uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 #v5.0.2
       with:
-        go-version: '~1.22'
+        go-version: '~1.23'
 
     - name: 🧓 Integration test (legacy)
       run: make integration-test-v1 testopts="--junitfile test-result-integration-legacy.xml"
@@ -162,7 +162,7 @@ jobs:
     - name: Set up Go 1.x
       uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 #v5.0.2
       with:
-        go-version: '~1.22'
+        go-version: '~1.23'
 
     - name: 📥/📤 Download/Restore test
       run: make download-restore-test testopts="--junitfile test-result-integration-download-restore.xml"
@@ -202,7 +202,7 @@ jobs:
       - name: Set up Go 1.x
         uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 #v5.0.2
         with:
-          go-version: '~1.22'
+          go-version: '~1.23'
 
       - name: 🗂️ Account Management E2E tests
         run: make account-management-test testopts="--junitfile test-result-aim.xml"
@@ -241,7 +241,7 @@ jobs:
     - name: Set up Go 1.x
       uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 #v5.0.2
       with:
-        go-version: '~1.22'
+        go-version: '~1.23'
 
     - name: 🧪 Unit test
       run: make test testopts="--junitfile test-result-windows-latest-unit.xml"
@@ -268,7 +268,7 @@ jobs:
     - name: Set up Go 1.x
       uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 #v5.0.2
       with:
-        go-version: '~1.22'
+        go-version: '~1.23'
 
     - name: 🌜 Nightly Tests
       run: make nightly-test testopts="--junitfile test-result-integration-nightly.xml"

diff --git a/.github/workflows/pr-static-code-analysis.yml b/.github/workflows/pr-static-code-analysis.yml
@@ -21,7 +21,7 @@ jobs:
       - name: 🛠️ Set up Go 1.x
         uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 #v5.0.2
         with:
-          go-version: '~1.22'
+          go-version: '~1.23'
 
       - name: ⬇️ Check out code into the Go module directory
         uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 #v4.2.1

diff --git a/cmd/monaco/dynatrace/dynatrace.go b/cmd/monaco/dynatrace/dynatrace.go
@@ -140,6 +140,8 @@ func CreateAccountClients(manifestAccounts map[string]manifest.Account) (map[acc
 			WithConcurrentRequestLimit(concurrentRequestLimit).
 			WithOAuthCredentials(oauthCreds).
 			WithUserAgent(client.DefaultMonacoUserAgent).
+			WithRateLimiter(true).
+			WithRetryOptions(&client.DefaultRetryOptions).
 			WithAccountURL(accountApiUrlOrDefault(acc.ApiUrl))
 
 		if support.SupportArchive {

diff --git a/go.mod b/go.mod
@@ -2,7 +2,7 @@ module github.com/dynatrace/dynatrace-configuration-as-code/v2
 
 require (
 	github.com/anknown/ahocorasick v0.0.0-20190904063843-d75dbd5169c0
-	github.com/dynatrace/dynatrace-configuration-as-code-core v0.6.2-0.20241004123955-8910e3e51120
+	github.com/dynatrace/dynatrace-configuration-as-code-core v0.7.1-0.20241015144435-8f403760a264
 	github.com/go-logr/logr v1.4.2
 	github.com/go-logr/zapr v1.3.0
 	github.com/google/go-cmp v0.6.0
@@ -34,7 +34,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/text v0.19.0 // indirect
-	golang.org/x/time v0.6.0 // indirect
+	golang.org/x/time v0.7.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 

diff --git a/go.sum b/go.sum
@@ -9,8 +9,8 @@ github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx2
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dynatrace/dynatrace-configuration-as-code-core v0.6.2-0.20241004123955-8910e3e51120 h1:ajBDEBpawT/Lo4owYe2bHwAZHQ5mn5TriarUwqX9WwI=
-github.com/dynatrace/dynatrace-configuration-as-code-core v0.6.2-0.20241004123955-8910e3e51120/go.mod h1:MASzWOV2hzoo8nbXuMcgMIB3XmGrcm16Fc0rakhwiTw=
+github.com/dynatrace/dynatrace-configuration-as-code-core v0.7.1-0.20241015144435-8f403760a264 h1:AvxE2prrr3Ey8rv2OCH70zb+70J5vz4CJous9ikwf+U=
+github.com/dynatrace/dynatrace-configuration-as-code-core v0.7.1-0.20241015144435-8f403760a264/go.mod h1:IMJjczhYZ3HKkjU74Cg38u6TyYZmNT1SPv8OoRyKLVQ=
 github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
 github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
@@ -59,8 +59,8 @@ golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs=
 golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
 golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
-golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
-golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ=
+golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 gonum.org/v1/gonum v0.15.1 h1:FNy7N6OUZVUaWG9pTiD+jlhdQ3lMP+/LcTpJ6+a8sQ0=
 gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

diff --git a/pkg/account/downloader/downloader.go b/pkg/account/downloader/downloader.go
@@ -23,6 +23,7 @@ import (
 	"github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/log"
 	"github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/account"
 	"github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/account/downloader/internal/http"
+	"github.com/go-logr/logr"
 )
 
 type Downloader struct {
@@ -39,7 +40,7 @@ func New(accountInfo *account.AccountInfo, client *accounts.Client) *Downloader
 
 func (a *Downloader) DownloadResources(ctx context.Context) (*account.Resources, error) {
 	log.WithCtxFields(ctx).Info("Starting download")
-
+	ctx = logr.NewContext(ctx, log.WithCtxFields(ctx).GetLogr())
 	tenants, err := a.environments(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("failed to fetch environments: %w", err)

diff --git a/pkg/account/downloader/downloader_test.go b/pkg/account/downloader/downloader_test.go
@@ -559,28 +559,26 @@ func newMockDownloader(d mockData, t *testing.T) *downloader.Downloader {
 	}
 	client := http.NewMockhttpClient(gomock.NewController(t))
 
-	ctx := gomock.AssignableToTypeOf(context.TODO())
-
-	client.EXPECT().GetEnvironmentsAndMZones(ctx, d.ai.AccountUUID).Return(d.envs, d.mzones, d.environmentsAndMZonesError).MinTimes(0).MaxTimes(1)
-	client.EXPECT().GetPolicies(ctx, d.ai.AccountUUID).Return(d.policies, d.policiesError).MinTimes(0).MaxTimes(1)
-	client.EXPECT().GetPolicyDefinition(ctx, gomock.AnyOf(toSliceOfAny(d.policies)...)).Return(d.policieDef, d.policyDefinitionError).AnyTimes()
+	client.EXPECT().GetEnvironmentsAndMZones(gomock.Any(), d.ai.AccountUUID).Return(d.envs, d.mzones, d.environmentsAndMZonesError).MinTimes(0).MaxTimes(1)
+	client.EXPECT().GetPolicies(gomock.Any(), d.ai.AccountUUID).Return(d.policies, d.policiesError).MinTimes(0).MaxTimes(1)
+	client.EXPECT().GetPolicyDefinition(gomock.Any(), gomock.AnyOf(toSliceOfAny(d.policies)...)).Return(d.policieDef, d.policyDefinitionError).AnyTimes()
 	if len(d.policyGroupBindings) == 0 {
-		client.EXPECT().GetPolicyGroupBindings(ctx, gomock.Any(), gomock.Any()).Return(&accountmanagement.LevelPolicyBindingDto{}, nil).AnyTimes()
+		client.EXPECT().GetPolicyGroupBindings(gomock.Any(), gomock.Any(), gomock.Any()).Return(&accountmanagement.LevelPolicyBindingDto{}, nil).AnyTimes()
 	} else {
 		for _, b := range d.policyGroupBindings {
-			client.EXPECT().GetPolicyGroupBindings(ctx, b.levelType, b.levelId).Return(b.bindings, b.err).MinTimes(1)
+			client.EXPECT().GetPolicyGroupBindings(gomock.Any(), b.levelType, b.levelId).Return(b.bindings, b.err).MinTimes(1)
 		}
 	}
 	if len(d.permissionsBindings) == 0 {
-		client.EXPECT().GetPermissionFor(ctx, d.ai.AccountUUID, gomock.Any()).Return(&accountmanagement.PermissionsGroupDto{}, nil).AnyTimes()
+		client.EXPECT().GetPermissionFor(gomock.Any(), d.ai.AccountUUID, gomock.Any()).Return(&accountmanagement.PermissionsGroupDto{}, nil).AnyTimes()
 	} else {
 		for _, b := range d.permissionsBindings {
-			client.EXPECT().GetPermissionFor(ctx, d.ai.AccountUUID, b.groupUUID).Return(b.bindings, b.err).AnyTimes()
+			client.EXPECT().GetPermissionFor(gomock.Any(), d.ai.AccountUUID, b.groupUUID).Return(b.bindings, b.err).AnyTimes()
 		}
 	}
-	client.EXPECT().GetGroups(ctx, d.ai.AccountUUID).Return(d.groups, d.groupsError).MinTimes(0).MaxTimes(1)
-	client.EXPECT().GetUsers(ctx, d.ai.AccountUUID).Return(d.users, d.usersError).MinTimes(0).MaxTimes(1)
-	client.EXPECT().GetGroupsForUser(ctx, userEmail(d.users), d.ai.AccountUUID).Return(d.userGroups, d.groupsForUserError).AnyTimes()
+	client.EXPECT().GetGroups(gomock.Any(), d.ai.AccountUUID).Return(d.groups, d.groupsError).MinTimes(0).MaxTimes(1)
+	client.EXPECT().GetUsers(gomock.Any(), d.ai.AccountUUID).Return(d.users, d.usersError).MinTimes(0).MaxTimes(1)
+	client.EXPECT().GetGroupsForUser(gomock.Any(), userEmail(d.users), d.ai.AccountUUID).Return(d.userGroups, d.groupsForUserError).AnyTimes()
 
 	return downloader.New4Test(d.ai, client)
 }