Update Settings based on unique key properties

For Settings schemas with unique key constraints, these key constraints are used in addition to the external ID to identify and update an existing object.

This ensures that Settings objects are not attempted to be duplicated but are correctly found and updated.

Changes in addition to the main feature:

* test(e2e): Add test which tries to deploy settings2.0 object (span-attribute) with the same unique key but different coordinates/externalID in two different projects and checks that a single object was created and updated without error.
* refactor(client): Move all settings clients test to same file
* refactor(client): Move all config client tests to same file
* refactor(client): Move all entity client tests to same file

---------

Co-authored-by: Jure Skelin <[email protected]>
Co-authored-by: UnseenWizzard <[email protected]>

cmd/monaco/integrationtest/v2/settings_integration_test.go

@@ -138,6 +138,25 @@ func TestOldExternalIDGetsUpdated(t *testing.T) {
 
 }
 
+func TestDeploySettingsWithUniqueProperties(t *testing.T) {
+	fs := testutils.CreateTestFileSystem()
+	var manifestPath = "test-resources/settings-unique-properties/manifest.yaml"
+	loadedManifest := integrationtest.LoadManifest(t, fs, manifestPath, "platform_env")
+
+	t.Cleanup(func() {
+		integrationtest.CleanupIntegrationTest(t, fs, manifestPath, loadedManifest, "")
+	})
+
+	cmd := runner.BuildCli(fs)
+	cmd.SetArgs([]string{"deploy", "-e", "platform_env", "-p", "project1", manifestPath})
+	err := cmd.Execute()
+	assert.NoError(t, err)
+
+	cmd.SetArgs([]string{"deploy", "-e", "platform_env", "-p", "project2", manifestPath})
+	err = cmd.Execute()
+	assert.NoError(t, err)
+}
+
 func createSettingsClient(t *testing.T, env manifest.EnvironmentDefinition, opts ...func(dynatraceClient *dtclient.DynatraceClient)) dtclient.SettingsClient {
 	oauthCredentials := auth.OauthCredentials{
 		ClientID:     env.Auth.OAuth.ClientID.Value,

cmd/monaco/integrationtest/v2/test-resources/settings-unique-properties/manifest.yaml

@@ -0,0 +1,26 @@
+manifestVersion: 1.0
+projects:
+- name: project1
+- name: project2
+environmentGroups:
+- name: default
+  environments:
+    - name: classic_env
+      url:
+        type: environment
+        value: URL_ENVIRONMENT_1
+      auth:
+        token:
+          name: TOKEN_ENVIRONMENT_1
+    - name: platform_env
+      url:
+        type: environment
+        value: PLATFORM_URL_ENVIRONMENT_2
+      auth:
+        token:
+          name: TOKEN_ENVIRONMENT_2
+        oAuth:
+          clientId:
+            name: OAUTH_CLIENT_ID
+          clientSecret:
+            name: OAUTH_CLIENT_SECRET

cmd/monaco/integrationtest/v2/test-resources/settings-unique-properties/project1/builtinmonitoring.slo/config.yaml

@@ -0,0 +1,11 @@
+configs:
+- id: d955d9b7-9630-3be5-aca5-3715779a7282
+  config:
+    name: d955d9b7-9630-3be5-aca5-3715779a7282
+    template: d955d9b7-9630-3be5-aca5-3715779a7282.json
+    skip: false
+  type:
+    settings:
+      schema: builtin:monitoring.slo
+      schemaVersion: 6.0.12
+      scope: environment

cmd/monaco/integrationtest/v2/test-resources/settings-unique-properties/project1/builtinmonitoring.slo/d955d9b7-9630-3be5-aca5-3715779a7282.json

@@ -0,0 +1,15 @@
+{
+  "enabled": true,
+  "name": "Just a Name",
+  "metricName": "justaname",
+  "metricExpression": "(100)*(builtin:service.errors.server.successCount:splitBy())/(builtin:service.requestCount.server:splitBy())",
+  "evaluationType": "AGGREGATE",
+  "filter": "type(\"SERVICE\")",
+  "evaluationWindow": "-1w",
+  "targetSuccess": 99.98,
+  "targetWarning": 99.99,
+  "errorBudgetBurnRate": {
+    "burnRateVisualizationEnabled": true,
+    "fastBurnThreshold": 10.0
+  }
+}

cmd/monaco/integrationtest/v2/test-resources/settings-unique-properties/project1/builtinspan-attribute/1f4b1f5c-11d2-38c8-9324-7d139c6e6452.json

@@ -0,0 +1,4 @@
+{
+    "key": "graphql.operation.name",
+    "masking": "NOT_MASKED"
+}

cmd/monaco/integrationtest/v2/test-resources/settings-unique-properties/project1/builtinspan-attribute/config.yaml

@@ -0,0 +1,11 @@
+configs:
+- id: 1f4b1f5c-11d2-38c8-9324-7d139c6e6452
+  config:
+    name: 1f4b1f5c-11d2-38c8-9324-7d139c6e6452
+    template: 1f4b1f5c-11d2-38c8-9324-7d139c6e6452.json
+    skip: false
+  type:
+    settings:
+      schema: builtin:span-attribute
+      schemaVersion: 0.0.32
+      scope: environment

cmd/monaco/integrationtest/v2/test-resources/settings-unique-properties/project2/builtinmonitoring.slo/config.yaml

@@ -0,0 +1,11 @@
+configs:
+- id: d955d9b7-9630-3be5-aca5-3715779a7282
+  config:
+    name: d955d9b7-9630-3be5-aca5-3715779a7282
+    template: d955d9b7-9630-3be5-aca5-3715779a7282.json
+    skip: false
+  type:
+    settings:
+      schema: builtin:monitoring.slo
+      schemaVersion: 6.0.12
+      scope: environment

cmd/monaco/integrationtest/v2/test-resources/settings-unique-properties/project2/builtinmonitoring.slo/d955d9b7-9630-3be5-aca5-3715779a7282.json

@@ -0,0 +1,15 @@
+{
+    "enabled": true,
+    "name": "NEW Name",
+    "metricName": "justaname",
+    "metricExpression": "(100)*(builtin:service.errors.server.successCount:splitBy())/(builtin:service.requestCount.server:splitBy())",
+    "evaluationType": "AGGREGATE",
+    "filter": "type(\"SERVICE\")",
+    "evaluationWindow": "-1w",
+    "targetSuccess": 99.98,
+    "targetWarning": 99.99,
+    "errorBudgetBurnRate": {
+        "burnRateVisualizationEnabled": true,
+        "fastBurnThreshold": 10.0
+    }
+}

cmd/monaco/integrationtest/v2/test-resources/settings-unique-properties/project2/builtinspan-attribute/1f4b1f5c-11d2-38c8-9324-7d139c6e6452.json

@@ -0,0 +1,4 @@
+{
+    "key": "graphql.operation.name",
+    "masking": "NOT_MASKED"
+}

cmd/monaco/integrationtest/v2/test-resources/settings-unique-properties/project2/builtinspan-attribute/config.yaml

@@ -0,0 +1,11 @@
+configs:
+- id: 1f4b1f5c-11d2-38c8-9324-7d139c6e6452
+  config:
+    name: 1f4b1f5c-11d2-38c8-9324-7d139c6e6452
+    template: 1f4b1f5c-11d2-38c8-9324-7d139c6e6452.json
+    skip: false
+  type:
+    settings:
+      schema: builtin:span-attribute
+      schemaVersion: 0.0.32
+      scope: environment

pkg/client/dtclient/client.go

@@ -23,7 +23,6 @@ import (
 	"fmt"
 	"github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/cache"
 	"github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/concurrency"
-	"github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/filter"
 	"github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/idutils"
 	"github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/log"
 	"github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/log/field"
@@ -560,61 +559,6 @@ func (d *DynatraceClient) getSettingById(ctx context.Context, objectId string) (
 	return &result, nil
 }
 
-func (d *DynatraceClient) ListSettings(ctx context.Context, schemaId string, opts ListSettingsOptions) (res []DownloadSettingsObject, err error) {
-	d.limiter.ExecuteBlocking(func() {
-		res, err = d.listSettings(ctx, schemaId, opts)
-	})
-	return
-}
-
-func (d *DynatraceClient) listSettings(ctx context.Context, schemaId string, opts ListSettingsOptions) ([]DownloadSettingsObject, error) {
-
-	if settings, cached := d.settingsCache.Get(schemaId); cached {
-		log.Debug("Using cached settings for schema %s", schemaId)
-		return filter.FilterSlice(settings, opts.Filter), nil
-	}
-
-	log.Debug("Downloading all settings for schema %s", schemaId)
-
-	listSettingsFields := defaultListSettingsFields
-	if opts.DiscardValue {
-		listSettingsFields = reducedListSettingsFields
-	}
-	params := url.Values{
-		"schemaIds": []string{schemaId},
-		"pageSize":  []string{defaultPageSize},
-		"fields":    []string{listSettingsFields},
-	}
-
-	result := make([]DownloadSettingsObject, 0)
-
-	addToResult := func(body []byte) (int, error) {
-		var parsed struct {
-			Items []DownloadSettingsObject `json:"items"`
-		}
-		if err := json.Unmarshal(body, &parsed); err != nil {
-			return 0, fmt.Errorf("failed to unmarshal response: %w", err)
-		}
-
-		result = append(result, parsed.Items...)
-		return len(parsed.Items), nil
-	}
-
-	u, err := buildUrl(d.environmentURL, d.settingsObjectAPIPath, params)
-	if err != nil {
-		return nil, fmt.Errorf("failed to list settings: %w", err)
-	}
-
-	_, err = rest.ListPaginated(ctx, d.platformClient, d.retrySettings, u, schemaId, addToResult)
-	if err != nil {
-		return nil, err
-	}
-
-	d.settingsCache.Set(schemaId, result)
-
-	return filter.FilterSlice(result, opts.Filter), nil
-}
-
 type EntitiesTypeListResponse struct {
 	Types []EntitiesType `json:"types"`
 }