From 0e19c4dc3d41ac91971f865e4449b421e840cdf7 Mon Sep 17 00:00:00 2001 From: warber <72415058+warber@users.noreply.github.com> Date: Tue, 15 Oct 2024 16:54:35 +0200 Subject: [PATCH] Fix/enable rate limiter for account clients (#1605) fix: enable rate limiter for account clients --- .github/workflows/build-test.yml | 2 +- .../workflows/dependencies-and-licenses.yml | 2 +- .github/workflows/end-to-end-test.yml | 14 ++++++------ .github/workflows/pr-static-code-analysis.yml | 2 +- cmd/monaco/dynatrace/dynatrace.go | 2 ++ go.mod | 4 ++-- go.sum | 8 +++---- pkg/account/downloader/downloader.go | 3 ++- pkg/account/downloader/downloader_test.go | 22 +++++++++---------- 9 files changed, 30 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index e2be1749f..fe1d5abd4 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -37,7 +37,7 @@ jobs: - name: ๐Ÿ› ๏ธ Set up Go 1.x uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 #v5.0.2 with: - go-version: '~1.22' + go-version: '~1.23' - name: ๐Ÿ—๏ธ Compile run: make compile diff --git a/.github/workflows/dependencies-and-licenses.yml b/.github/workflows/dependencies-and-licenses.yml index 072ceda89..e2a3476f3 100644 --- a/.github/workflows/dependencies-and-licenses.yml +++ b/.github/workflows/dependencies-and-licenses.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Go uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 #v5.0.2 with: - go-version: '~1.22' + go-version: '~1.23' - name: Install go-licence-detector run: | go install go.elastic.co/go-licence-detector@v0.6.0 diff --git a/.github/workflows/end-to-end-test.yml b/.github/workflows/end-to-end-test.yml index 5ec569cbb..c4ee23179 100644 --- a/.github/workflows/end-to-end-test.yml +++ b/.github/workflows/end-to-end-test.yml @@ -56,7 +56,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 #v5.0.2 with: - go-version: '~1.22' + go-version: '~1.23' - name: ๐Ÿ Build release binaries run: make build-release @@ -80,7 +80,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 #v5.0.2 with: - go-version: '~1.22' + go-version: '~1.23' - name: ๐ŸŒŽ Integration test run: make integration-test testopts="--junitfile test-result-integration.xml" @@ -121,7 +121,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 #v5.0.2 with: - go-version: '~1.22' + go-version: '~1.23' - name: ๐Ÿง“ Integration test (legacy) run: make integration-test-v1 testopts="--junitfile test-result-integration-legacy.xml" @@ -162,7 +162,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 #v5.0.2 with: - go-version: '~1.22' + go-version: '~1.23' - name: ๐Ÿ“ฅ/๐Ÿ“ค Download/Restore test run: make download-restore-test testopts="--junitfile test-result-integration-download-restore.xml" @@ -202,7 +202,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 #v5.0.2 with: - go-version: '~1.22' + go-version: '~1.23' - name: ๐Ÿ—‚๏ธ Account Management E2E tests run: make account-management-test testopts="--junitfile test-result-aim.xml" @@ -241,7 +241,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 #v5.0.2 with: - go-version: '~1.22' + go-version: '~1.23' - name: ๐Ÿงช Unit test run: make test testopts="--junitfile test-result-windows-latest-unit.xml" @@ -268,7 +268,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 #v5.0.2 with: - go-version: '~1.22' + go-version: '~1.23' - name: ๐ŸŒœ Nightly Tests run: make nightly-test testopts="--junitfile test-result-integration-nightly.xml" diff --git a/.github/workflows/pr-static-code-analysis.yml b/.github/workflows/pr-static-code-analysis.yml index 4a6fd7c97..5dd2330c9 100644 --- a/.github/workflows/pr-static-code-analysis.yml +++ b/.github/workflows/pr-static-code-analysis.yml @@ -21,7 +21,7 @@ jobs: - name: ๐Ÿ› ๏ธ Set up Go 1.x uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 #v5.0.2 with: - go-version: '~1.22' + go-version: '~1.23' - name: โฌ‡๏ธ Check out code into the Go module directory uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 #v4.2.1 diff --git a/cmd/monaco/dynatrace/dynatrace.go b/cmd/monaco/dynatrace/dynatrace.go index b6ef741af..8911f52fd 100644 --- a/cmd/monaco/dynatrace/dynatrace.go +++ b/cmd/monaco/dynatrace/dynatrace.go @@ -140,6 +140,8 @@ func CreateAccountClients(manifestAccounts map[string]manifest.Account) (map[acc WithConcurrentRequestLimit(concurrentRequestLimit). WithOAuthCredentials(oauthCreds). WithUserAgent(client.DefaultMonacoUserAgent). + WithRateLimiter(true). + WithRetryOptions(&client.DefaultRetryOptions). WithAccountURL(accountApiUrlOrDefault(acc.ApiUrl)) if support.SupportArchive { diff --git a/go.mod b/go.mod index 1a6111637..daaa42297 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/dynatrace/dynatrace-configuration-as-code/v2 require ( github.com/anknown/ahocorasick v0.0.0-20190904063843-d75dbd5169c0 - github.com/dynatrace/dynatrace-configuration-as-code-core v0.6.2-0.20241004123955-8910e3e51120 + github.com/dynatrace/dynatrace-configuration-as-code-core v0.7.1-0.20241015144435-8f403760a264 github.com/go-logr/logr v1.4.2 github.com/go-logr/zapr v1.3.0 github.com/google/go-cmp v0.6.0 @@ -34,7 +34,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/text v0.19.0 // indirect - golang.org/x/time v0.6.0 // indirect + golang.org/x/time v0.7.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 4ed000a83..fa05ce6a7 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,8 @@ github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx2 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dynatrace/dynatrace-configuration-as-code-core v0.6.2-0.20241004123955-8910e3e51120 h1:ajBDEBpawT/Lo4owYe2bHwAZHQ5mn5TriarUwqX9WwI= -github.com/dynatrace/dynatrace-configuration-as-code-core v0.6.2-0.20241004123955-8910e3e51120/go.mod h1:MASzWOV2hzoo8nbXuMcgMIB3XmGrcm16Fc0rakhwiTw= +github.com/dynatrace/dynatrace-configuration-as-code-core v0.7.1-0.20241015144435-8f403760a264 h1:AvxE2prrr3Ey8rv2OCH70zb+70J5vz4CJous9ikwf+U= +github.com/dynatrace/dynatrace-configuration-as-code-core v0.7.1-0.20241015144435-8f403760a264/go.mod h1:IMJjczhYZ3HKkjU74Cg38u6TyYZmNT1SPv8OoRyKLVQ= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= @@ -59,8 +59,8 @@ golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= -golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= +golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= gonum.org/v1/gonum v0.15.1 h1:FNy7N6OUZVUaWG9pTiD+jlhdQ3lMP+/LcTpJ6+a8sQ0= gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= diff --git a/pkg/account/downloader/downloader.go b/pkg/account/downloader/downloader.go index d3b123892..98615fd98 100644 --- a/pkg/account/downloader/downloader.go +++ b/pkg/account/downloader/downloader.go @@ -23,6 +23,7 @@ import ( "github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/log" "github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/account" "github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/account/downloader/internal/http" + "github.com/go-logr/logr" ) type Downloader struct { @@ -39,7 +40,7 @@ func New(accountInfo *account.AccountInfo, client *accounts.Client) *Downloader func (a *Downloader) DownloadResources(ctx context.Context) (*account.Resources, error) { log.WithCtxFields(ctx).Info("Starting download") - + ctx = logr.NewContext(ctx, log.WithCtxFields(ctx).GetLogr()) tenants, err := a.environments(ctx) if err != nil { return nil, fmt.Errorf("failed to fetch environments: %w", err) diff --git a/pkg/account/downloader/downloader_test.go b/pkg/account/downloader/downloader_test.go index 08e246ae5..0d236b366 100644 --- a/pkg/account/downloader/downloader_test.go +++ b/pkg/account/downloader/downloader_test.go @@ -559,28 +559,26 @@ func newMockDownloader(d mockData, t *testing.T) *downloader.Downloader { } client := http.NewMockhttpClient(gomock.NewController(t)) - ctx := gomock.AssignableToTypeOf(context.TODO()) - - client.EXPECT().GetEnvironmentsAndMZones(ctx, d.ai.AccountUUID).Return(d.envs, d.mzones, d.environmentsAndMZonesError).MinTimes(0).MaxTimes(1) - client.EXPECT().GetPolicies(ctx, d.ai.AccountUUID).Return(d.policies, d.policiesError).MinTimes(0).MaxTimes(1) - client.EXPECT().GetPolicyDefinition(ctx, gomock.AnyOf(toSliceOfAny(d.policies)...)).Return(d.policieDef, d.policyDefinitionError).AnyTimes() + client.EXPECT().GetEnvironmentsAndMZones(gomock.Any(), d.ai.AccountUUID).Return(d.envs, d.mzones, d.environmentsAndMZonesError).MinTimes(0).MaxTimes(1) + client.EXPECT().GetPolicies(gomock.Any(), d.ai.AccountUUID).Return(d.policies, d.policiesError).MinTimes(0).MaxTimes(1) + client.EXPECT().GetPolicyDefinition(gomock.Any(), gomock.AnyOf(toSliceOfAny(d.policies)...)).Return(d.policieDef, d.policyDefinitionError).AnyTimes() if len(d.policyGroupBindings) == 0 { - client.EXPECT().GetPolicyGroupBindings(ctx, gomock.Any(), gomock.Any()).Return(&accountmanagement.LevelPolicyBindingDto{}, nil).AnyTimes() + client.EXPECT().GetPolicyGroupBindings(gomock.Any(), gomock.Any(), gomock.Any()).Return(&accountmanagement.LevelPolicyBindingDto{}, nil).AnyTimes() } else { for _, b := range d.policyGroupBindings { - client.EXPECT().GetPolicyGroupBindings(ctx, b.levelType, b.levelId).Return(b.bindings, b.err).MinTimes(1) + client.EXPECT().GetPolicyGroupBindings(gomock.Any(), b.levelType, b.levelId).Return(b.bindings, b.err).MinTimes(1) } } if len(d.permissionsBindings) == 0 { - client.EXPECT().GetPermissionFor(ctx, d.ai.AccountUUID, gomock.Any()).Return(&accountmanagement.PermissionsGroupDto{}, nil).AnyTimes() + client.EXPECT().GetPermissionFor(gomock.Any(), d.ai.AccountUUID, gomock.Any()).Return(&accountmanagement.PermissionsGroupDto{}, nil).AnyTimes() } else { for _, b := range d.permissionsBindings { - client.EXPECT().GetPermissionFor(ctx, d.ai.AccountUUID, b.groupUUID).Return(b.bindings, b.err).AnyTimes() + client.EXPECT().GetPermissionFor(gomock.Any(), d.ai.AccountUUID, b.groupUUID).Return(b.bindings, b.err).AnyTimes() } } - client.EXPECT().GetGroups(ctx, d.ai.AccountUUID).Return(d.groups, d.groupsError).MinTimes(0).MaxTimes(1) - client.EXPECT().GetUsers(ctx, d.ai.AccountUUID).Return(d.users, d.usersError).MinTimes(0).MaxTimes(1) - client.EXPECT().GetGroupsForUser(ctx, userEmail(d.users), d.ai.AccountUUID).Return(d.userGroups, d.groupsForUserError).AnyTimes() + client.EXPECT().GetGroups(gomock.Any(), d.ai.AccountUUID).Return(d.groups, d.groupsError).MinTimes(0).MaxTimes(1) + client.EXPECT().GetUsers(gomock.Any(), d.ai.AccountUUID).Return(d.users, d.usersError).MinTimes(0).MaxTimes(1) + client.EXPECT().GetGroupsForUser(gomock.Any(), userEmail(d.users), d.ai.AccountUUID).Return(d.userGroups, d.groupsForUserError).AnyTimes() return downloader.New4Test(d.ai, client) }