Implement values override merging

CHANGELOG.md

@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- Implemented value file merging / overlaying. Values provided to the clusterbuilder will be merged ontop of the default ones included in this module.
+- `clusterbuilder` updated to take in a slice of values overrides that are layered ontop of the default values
+- Updated `LoadOrBuildCluster` and `standup` to work with the `clusterbuilder` refactoring.
+
 ## [1.0.0] - 2024-04-26
 
 ### Added

cmd/standup/main.go

@@ -25,6 +25,7 @@ import (
 	"github.com/giantswarm/cluster-standup-teardown/pkg/clusterbuilder/providers/capvcd"
 	"github.com/giantswarm/cluster-standup-teardown/pkg/clusterbuilder/providers/capz"
 	"github.com/giantswarm/cluster-standup-teardown/pkg/standup"
+	"github.com/giantswarm/cluster-standup-teardown/pkg/values"
 )
 
 var (
@@ -109,23 +110,26 @@ func run(cmd *cobra.Command, args []string) error {
 
 	fmt.Printf("Standing up cluster...\n\nProvider:\t\t%s\nCluster Name:\t\t%s\nOrg Name:\t\t%s\nResults Directory:\t%s\n\n", provider, clusterName, orgName, outputDirectory)
 
+	clusterValuesOverrides := []string{values.MustLoadValuesFile(clusterValues)}
+	defaultAppValuesOverrides := []string{values.MustLoadValuesFile(defaultAppValues)}
+
 	var cluster *application.Cluster
 	switch provider {
 	case application.ProviderVSphere:
 		clusterBuilder := capv.ClusterBuilder{}
-		cluster = clusterBuilder.NewClusterApp(clusterName, orgName, clusterValues, defaultAppValues).
+		cluster = clusterBuilder.NewClusterApp(clusterName, orgName, clusterValuesOverrides, defaultAppValuesOverrides).
 			WithAppVersions(clusterVersion, defaultAppVersion)
 	case application.ProviderCloudDirector:
 		clusterBuilder := capvcd.ClusterBuilder{}
-		cluster = clusterBuilder.NewClusterApp(clusterName, orgName, clusterValues, defaultAppValues).
+		cluster = clusterBuilder.NewClusterApp(clusterName, orgName, clusterValuesOverrides, defaultAppValuesOverrides).
 			WithAppVersions(clusterVersion, defaultAppVersion)
 	case application.ProviderAWS:
 		clusterBuilder := capa.ClusterBuilder{}
-		cluster = clusterBuilder.NewClusterApp(clusterName, orgName, clusterValues, defaultAppValues).
+		cluster = clusterBuilder.NewClusterApp(clusterName, orgName, clusterValuesOverrides, defaultAppValuesOverrides).
 			WithAppVersions(clusterVersion, defaultAppVersion)
 	case application.ProviderEKS:
 		clusterBuilder := capa.ManagedClusterBuilder{}
-		cluster = clusterBuilder.NewClusterApp(clusterName, orgName, clusterValues, defaultAppValues).
+		cluster = clusterBuilder.NewClusterApp(clusterName, orgName, clusterValuesOverrides, defaultAppValuesOverrides).
 			WithAppVersions(clusterVersion, defaultAppVersion)
 		// As EKS has no control plane we only check for worker nodes being ready
 		clusterReadyFns = []func(wcClient *client.Client){
@@ -139,7 +143,7 @@ func run(cmd *cobra.Command, args []string) error {
 		}
 	case application.ProviderAzure:
 		clusterBuilder := capz.ClusterBuilder{}
-		cluster = clusterBuilder.NewClusterApp(clusterName, orgName, clusterValues, defaultAppValues).
+		cluster = clusterBuilder.NewClusterApp(clusterName, orgName, clusterValuesOverrides, defaultAppValuesOverrides).
 			WithAppVersions(clusterVersion, defaultAppVersion)
 	default:
 		cluster = application.NewClusterApp(clusterName, provider).

go.mod

@@ -3,11 +3,13 @@ module github.com/giantswarm/cluster-standup-teardown
 go 1.21
 
 require (
+	dario.cat/mergo v1.0.0
 	github.com/giantswarm/apiextensions-application v0.6.1
 	github.com/giantswarm/clustertest v0.18.0
 	github.com/onsi/gomega v1.27.2
 	github.com/spf13/cobra v1.8.0
 	sigs.k8s.io/controller-runtime v0.14.5
+	sigs.k8s.io/yaml v1.4.0
 )
 
 require (
@@ -164,5 +166,4 @@ require (
 	sigs.k8s.io/kustomize/api v0.12.1 // indirect
 	sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect
-	sigs.k8s.io/yaml v1.4.0 // indirect
 )

go.sum

@@ -39,6 +39,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
+dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
+dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 h1:EKPd1INOIyr5hWOWhvpmQpY6tKjeG0hT1s3AMC/9fic=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0=

pkg/clusterbuilder/client.go

@@ -6,11 +6,13 @@ import (
 	"github.com/giantswarm/clustertest"
 	"github.com/giantswarm/clustertest/pkg/application"
 	"github.com/giantswarm/clustertest/pkg/logger"
+
+	"github.com/giantswarm/cluster-standup-teardown/pkg/values"
 )
 
 // ClusterBuilder is an interface that provides a function for building provider-specific Cluster apps
 type ClusterBuilder interface {
-	NewClusterApp(clusterName string, orgName string, clusterValuesFile string, defaultAppsValuesFile string) *application.Cluster
+	NewClusterApp(clusterName string, orgName string, clusterValuesOverrides []string, defaultAppsValuesOverrides []string) *application.Cluster
 }
 
 // LoadOrBuildCluster attempts to load a pre-built workload cluster if the appropriate env vars are set and if not will build a new Cluster
@@ -25,7 +27,11 @@ func LoadOrBuildCluster(framework *clustertest.Framework, clusterBuilder Cluster
 		return cluster
 	}
 
-	cluster = clusterBuilder.NewClusterApp("", "", "./test_data/cluster_values.yaml", "./test_data/default-apps_values.yaml")
+	cluster = clusterBuilder.NewClusterApp(
+		"", "",
+		[]string{values.MustLoadValuesFile("./test_data/cluster_values.yaml")},
+		[]string{values.MustLoadValuesFile("./test_data/default-apps_values.yaml")},
+	)
 	logger.Log("Workload cluster name: %s", cluster.Name)
 
 	return cluster

pkg/clusterbuilder/providers/capa/cluster.go

@@ -21,7 +21,7 @@ var (
 type ClusterBuilder struct{}
 
 // NewClusterApp builds a new CAPA cluster App
-func (c *ClusterBuilder) NewClusterApp(clusterName string, orgName string, clusterValuesFile string, defaultAppsValuesFile string) *application.Cluster {
+func (c *ClusterBuilder) NewClusterApp(clusterName string, orgName string, clusterValuesOverrides []string, defaultAppsValuesOverrides []string) *application.Cluster {
 	if clusterName == "" {
 		clusterName = utils.GenerateRandomName("t")
 	}
@@ -32,8 +32,8 @@ func (c *ClusterBuilder) NewClusterApp(clusterName string, orgName string, clust
 	return application.NewClusterApp(clusterName, application.ProviderAWS).
 		WithOrg(organization.New(orgName)).
 		WithAppValues(
-			values.MustOverlayValues(baseClusterValues, clusterValuesFile),
-			values.MustOverlayValues(baseDefaultAppsValues, defaultAppsValuesFile),
+			values.MustMergeValues(append([]string{baseClusterValues}, clusterValuesOverrides...)...),
+			values.MustMergeValues(append([]string{baseDefaultAppsValues}, defaultAppsValuesOverrides...)...),
 			&application.TemplateValues{
 				ClusterName:  clusterName,
 				Organization: orgName,

pkg/clusterbuilder/providers/capa/managed_controlplane_cluster.go

@@ -21,7 +21,7 @@ var (
 type ManagedClusterBuilder struct{}
 
 // NewClusterApp builds a new CAPA EKS cluster App
-func (c *ManagedClusterBuilder) NewClusterApp(clusterName string, orgName string, clusterValuesFile string, defaultAppsValuesFile string) *application.Cluster {
+func (c *ManagedClusterBuilder) NewClusterApp(clusterName string, orgName string, clusterValuesOverrides []string, defaultAppsValuesOverrides []string) *application.Cluster {
 	if clusterName == "" {
 		clusterName = utils.GenerateRandomName("t")
 	}
@@ -32,8 +32,8 @@ func (c *ManagedClusterBuilder) NewClusterApp(clusterName string, orgName string
 	return application.NewClusterApp(clusterName, application.ProviderEKS).
 		WithOrg(organization.New(orgName)).
 		WithAppValues(
-			values.MustOverlayValues(baseManagedClusterValues, clusterValuesFile),
-			values.MustOverlayValues(baseManagedDefaultAppsValues, defaultAppsValuesFile),
+			values.MustMergeValues(append([]string{baseManagedClusterValues}, clusterValuesOverrides...)...),
+			values.MustMergeValues(append([]string{baseManagedDefaultAppsValues}, defaultAppsValuesOverrides...)...),
 			&application.TemplateValues{
 				ClusterName:  clusterName,
 				Organization: orgName,

pkg/clusterbuilder/providers/capa/private_cluster.go

@@ -23,7 +23,7 @@ var (
 type PrivateClusterBuilder struct{}
 
 // NewClusterApp builds a new private CAPA cluster App
-func (c *PrivateClusterBuilder) NewClusterApp(clusterName string, orgName string, clusterValuesFile string, defaultAppsValuesFile string) *application.Cluster {
+func (c *PrivateClusterBuilder) NewClusterApp(clusterName string, orgName string, clusterValuesOverrides []string, defaultAppsValuesOverrides []string) *application.Cluster {
 	if clusterName == "" {
 		clusterName = utils.GenerateRandomName("t")
 	}
@@ -47,8 +47,9 @@ func (c *PrivateClusterBuilder) NewClusterApp(clusterName string, orgName string
 	return application.NewClusterApp(clusterName, application.ProviderAWS).
 		WithOrg(organization.New(orgName)).
 		WithAppValues(
-			values.MustOverlayValues(basePrivateClusterValues, clusterValuesFile),
-			values.MustOverlayValues(basePrivateDefaultAppsValues, defaultAppsValuesFile),
+
+			values.MustMergeValues(append([]string{basePrivateClusterValues}, clusterValuesOverrides...)...),
+			values.MustMergeValues(append([]string{basePrivateDefaultAppsValues}, defaultAppsValuesOverrides...)...),
 			templateValues,
 		)
 }

pkg/clusterbuilder/providers/capv/cluster.go

@@ -29,7 +29,7 @@ var (
 type ClusterBuilder struct{}
 
 // NewClusterApp builds a new CAPV cluster App
-func (c *ClusterBuilder) NewClusterApp(clusterName string, orgName string, clusterValuesFile string, defaultAppsValuesFile string) *application.Cluster {
+func (c *ClusterBuilder) NewClusterApp(clusterName string, orgName string, clusterValuesOverrides []string, defaultAppsValuesOverrides []string) *application.Cluster {
 	if clusterName == "" {
 		clusterName = utils.GenerateRandomName("t")
 	}
@@ -40,8 +40,8 @@ func (c *ClusterBuilder) NewClusterApp(clusterName string, orgName string, clust
 	return application.NewClusterApp(clusterName, application.ProviderVSphere).
 		WithOrg(organization.New(orgName)).
 		WithAppValues(
-			values.MustOverlayValues(baseClusterValues, clusterValuesFile),
-			values.MustOverlayValues(baseDefaultAppsValues, defaultAppsValuesFile),
+			values.MustMergeValues(append([]string{baseClusterValues}, clusterValuesOverrides...)...),
+			values.MustMergeValues(append([]string{baseDefaultAppsValues}, defaultAppsValuesOverrides...)...),
 			&application.TemplateValues{
 				ClusterName:  clusterName,
 				Organization: orgName,

pkg/clusterbuilder/providers/capvcd/cluster.go

@@ -27,7 +27,7 @@ var (
 type ClusterBuilder struct{}
 
 // NewClusterApp builds a new CAPVCD cluster App
-func (c *ClusterBuilder) NewClusterApp(clusterName string, orgName string, clusterValuesFile string, defaultAppsValuesFile string) *application.Cluster {
+func (c *ClusterBuilder) NewClusterApp(clusterName string, orgName string, clusterValuesOverrides []string, defaultAppsValuesOverrides []string) *application.Cluster {
 	if clusterName == "" {
 		clusterName = utils.GenerateRandomName("t")
 	}
@@ -38,8 +38,8 @@ func (c *ClusterBuilder) NewClusterApp(clusterName string, orgName string, clust
 	return application.NewClusterApp(clusterName, application.ProviderCloudDirector).
 		WithOrg(organization.New(orgName)).
 		WithAppValues(
-			values.MustOverlayValues(baseClusterValues, clusterValuesFile),
-			values.MustOverlayValues(baseDefaultAppsValues, defaultAppsValuesFile),
+			values.MustMergeValues(append([]string{baseClusterValues}, clusterValuesOverrides...)...),
+			values.MustMergeValues(append([]string{baseDefaultAppsValues}, defaultAppsValuesOverrides...)...),
 			&application.TemplateValues{
 				ClusterName:  clusterName,
 				Organization: orgName,

pkg/clusterbuilder/providers/capz/cluster.go

@@ -21,7 +21,7 @@ var (
 type ClusterBuilder struct{}
 
 // NewClusterApp builds a new CAPZ cluster App
-func (c *ClusterBuilder) NewClusterApp(clusterName string, orgName string, clusterValuesFile string, defaultAppsValuesFile string) *application.Cluster {
+func (c *ClusterBuilder) NewClusterApp(clusterName string, orgName string, clusterValuesOverrides []string, defaultAppsValuesOverrides []string) *application.Cluster {
 	if clusterName == "" {
 		clusterName = utils.GenerateRandomName("t")
 	}
@@ -32,8 +32,8 @@ func (c *ClusterBuilder) NewClusterApp(clusterName string, orgName string, clust
 	return application.NewClusterApp(clusterName, application.ProviderAzure).
 		WithOrg(organization.New(orgName)).
 		WithAppValues(
-			values.MustOverlayValues(baseClusterValues, clusterValuesFile),
-			values.MustOverlayValues(baseDefaultAppsValues, defaultAppsValuesFile),
+			values.MustMergeValues(append([]string{baseClusterValues}, clusterValuesOverrides...)...),
+			values.MustMergeValues(append([]string{baseDefaultAppsValues}, defaultAppsValuesOverrides...)...),
 			&application.TemplateValues{
 				ClusterName:  clusterName,
 				Organization: orgName,

pkg/values/values.go

@@ -1,25 +1,51 @@
 package values
 
-import "os"
-
-// OverlayValues takes in a default values string and an optional path to a file containing values
-// If the values file if found it will be loaded and used for the values, otherwise the default is used.
-func OverlayValues(defaultValues string, valuesFile string) (string, error) {
-	finalValues := defaultValues
-	if valuesFile != "" {
-		fileBytes, err := os.ReadFile(valuesFile)
-		if err != nil && !os.IsNotExist(err) {
-			return finalValues, err
+import (
+	"os"
+
+	"dario.cat/mergo"
+	"sigs.k8s.io/yaml"
+)
+
+// MustOverlayValues performs an OverlayValues call but ignores any errors that occur while reading the values file.
+func MustMergeValues(values ...string) string {
+	finalValues, _ := Merge(values...)
+	return finalValues
+}
+
+func Merge(layers ...string) (string, error) {
+	mergedLayers := map[string]interface{}{}
+
+	for _, layer := range layers {
+		if layer == "" {
+			continue
+		}
+
+		var rawMapData map[string]interface{}
+		err := yaml.Unmarshal([]byte(layer), &rawMapData)
+		if err != nil {
+			return "", err
+		}
+
+		err = mergo.Merge(&mergedLayers, rawMapData, mergo.WithOverride)
+		if err != nil {
+			return "", err
 		}
-		// TODO: Override / merge values together
-		finalValues = string(fileBytes)
 	}
 
-	return finalValues, nil
+	data, err := yaml.Marshal(mergedLayers)
+	if err != nil {
+		return "", err
+	}
+
+	return string(data), nil
 }
 
-// MustOverlayValues performs an OverlayValues call but ignores any errors that occur while reading the values file.
-func MustOverlayValues(defaultValues string, valuesFile string) string {
-	finalValues, _ := OverlayValues(defaultValues, valuesFile)
-	return finalValues
+// MustLoadValuesFile attempts to load a values file from the provided filePath and if fails returns an empty string
+func MustLoadValuesFile(filePath string) string {
+	fileBytes, err := os.ReadFile(filePath)
+	if err != nil {
+		return ""
+	}
+	return string(fileBytes)
 }