domain stacks (#56)

* tests passing * skip migrator on disabled tenants * resource group per domain
osstotalsoft · Jul 4, 2023 · d019f0a · d019f0a
1 parent 2ca9fcb
commit d019f0a
Show file tree

Hide file tree

Showing 9 changed files with 315 additions and 506 deletions.
diff --git a/internal/controllers/provisioning/migration/migration.go b/internal/controllers/provisioning/migration/migration.go
@@ -8,26 +8,36 @@ import (
 	v1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/klog/v2"
 	platformv1 "totalsoft.ro/platform-controllers/pkg/apis/platform/v1alpha1"
 )
 
 const (
-	JobLabelSelectorKey = "provisioning.totalsoft.ro/migration-job-template"
+	jobTemplateLabel = "provisioning.totalsoft.ro/migration-job-template"
+	domainLabel      = "platform.totalsoft.ro/domain"
 )
 
 func KubeJobsMigrationForTenant(kubeClient kubernetes.Interface,
-	nsFilter func(string, string) bool) func(platform string, tenant *platformv1.Tenant) error {
+	nsFilter func(string, string) bool) func(platform string, tenant *platformv1.Tenant, domain string) error {
 	namer := func(jName, tenant string) string {
 		return fmt.Sprintf("%s-%s-%d", jName, tenant, time.Now().Unix())
 	}
 
-	return func(platform string, tenant *platformv1.Tenant) error {
-		klog.InfoS("Creating migrations jobs", "tenant", tenant.Name)
+	return func(platform string, tenant *platformv1.Tenant, domain string) error {
+		klog.InfoS("Creating migrations jobs", "tenant", tenant.Name, "domain", domain)
+
+		labelSelector, err := labels.ValidatedSelectorFromSet(map[string]string{
+			domainLabel:      domain,
+			jobTemplateLabel: "true",
+		})
+		if err != nil {
+			return err
+		}
 
 		jobs, err := kubeClient.BatchV1().Jobs("").List(context.TODO(), metav1.ListOptions{
-			LabelSelector: JobLabelSelectorKey + "=true",
+			LabelSelector: labelSelector.String(),
 		})
 		if err != nil {
 			return err

diff --git a/internal/controllers/provisioning/migration/migration_test.go b/internal/controllers/provisioning/migration/migration_test.go
@@ -12,25 +12,28 @@ import (
 )
 
 func TestKubeJobsMigrationForTenant(t *testing.T) {
+	domain := "test-domain"
 	objects := []runtime.Object{
-		newJob("dev1", true),
-		newJob("dev2", true),
-		newJob("dev3", false),
+		newJob("dev1", domain, true),
+		newJob("dev2", domain, true),
+		newJob("dev3", domain, false),
+		newJob("dev4", "some-other-domain", true),
 	}
 	kubeClient := fake.NewSimpleClientset(objects...)
 	migrator := KubeJobsMigrationForTenant(kubeClient, func(s string, s2 string) bool {
 		return true
 	})
 	t.Run("test job selection by label", func(t *testing.T) {
-		migrator("test", newTenant("qa", "qa"))
+		migrator("test", newTenant("qa", "qa"), domain)
 		jobs, _ := kubeClient.BatchV1().Jobs(metav1.NamespaceDefault).List(context.TODO(), metav1.ListOptions{})
-		if len(jobs.Items) != 5 {
-			t.Errorf("Error running migration, expected 5 jobs but found %d", len(jobs.Items))
+		expectedNoOfJobs := 4 + 2 //4 existing + 2 new jobs
+		if len(jobs.Items) != expectedNoOfJobs {
+			t.Errorf("Error running migration, expected %d jobs but found %d", expectedNoOfJobs, len(jobs.Items))
 		}
 	})
 }
 
-func newJob(name string, template bool) *v1.Job {
+func newJob(name, domain string, template bool) *v1.Job {
 	j := &v1.Job{
 		TypeMeta: metav1.TypeMeta{APIVersion: platformv1.SchemeGroupVersion.String()},
 		ObjectMeta: metav1.ObjectMeta{
@@ -40,7 +43,10 @@ func newJob(name string, template bool) *v1.Job {
 		Spec: v1.JobSpec{},
 	}
 	if template {
-		j.SetLabels(map[string]string{JobLabelSelectorKey: "true"})
+		j.SetLabels(map[string]string{
+			jobTemplateLabel: "true",
+			domainLabel:      domain,
+		})
 	}
 	return j
 }

diff --git a/internal/controllers/provisioning/provisioners/pulumi/azure_resource_group.go b/internal/controllers/provisioning/provisioners/pulumi/azure_resource_group.go
@@ -8,9 +8,9 @@ import (
 	platformv1 "totalsoft.ro/platform-controllers/pkg/apis/platform/v1alpha1"
 )
 
-func azureRGDeployFunc(platform string, tenant *platformv1.Tenant) func(ctx *pulumi.Context) (pulumi.StringOutput, error) {
+func azureRGDeployFunc(platform string, tenant *platformv1.Tenant, domain string) func(ctx *pulumi.Context) (pulumi.StringOutput, error) {
 	return func(ctx *pulumi.Context) (pulumi.StringOutput, error) {
-		resourceGroupName := fmt.Sprintf("%s-%s", platform, tenant.Name)
+		resourceGroupName := fmt.Sprintf("%s-%s-%s", platform, tenant.Name, domain)
 		resourceGroup, err := azureResources.NewResourceGroup(ctx, resourceGroupName, &azureResources.ResourceGroupArgs{
 			ResourceGroupName: pulumi.String(resourceGroupName),
 		}, pulumi.RetainOnDelete(true))

diff --git a/internal/controllers/provisioning/provisioners/pulumi/pulumi.go b/internal/controllers/provisioning/provisioners/pulumi/pulumi.go
@@ -4,6 +4,7 @@ package pulumi
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/pulumi/pulumi/sdk/v3/go/common/apitype"
 
@@ -22,7 +23,7 @@ const (
 	PulumiRetainOnDelete = true
 )
 
-func Create(platform string, tenant *platformv1.Tenant, infra *provisioners.InfrastructureManifests) provisioners.ProvisioningResult {
+func Create(platform string, tenant *platformv1.Tenant, domain string, infra *provisioners.InfrastructureManifests) provisioners.ProvisioningResult {
 	result := provisioners.ProvisioningResult{}
 	upRes := auto.UpResult{}
 	destroyRes := auto.DestroyResult{}
@@ -37,9 +38,9 @@ func Create(platform string, tenant *platformv1.Tenant, infra *provisioners.Infr
 	anyResource := anyAzureDb || anyManagedAzureDb || anyHelmRelease || anyVirtualMachine || anyVirtualDesktop
 	needsResourceGroup := anyVirtualMachine || anyVirtualDesktop
 
-	stackName := tenant.Name
+	stackName := fmt.Sprintf("%s-%s", tenant.Name, domain)
 	if anyResource {
-		upRes, result.Error = updateStack(stackName, platform, deployFunc(platform, tenant, infra, needsResourceGroup))
+		upRes, result.Error = updateStack(stackName, platform, deployFunc(platform, tenant, domain, infra, needsResourceGroup))
 		if result.Error != nil {
 			return result
 		}
@@ -188,7 +189,7 @@ func createOrSelectStack(ctx context.Context, stackName, projectName string, dep
 	return s, nil
 }
 
-func deployFunc(platform string, tenant *platformv1.Tenant,
+func deployFunc(platform string, tenant *platformv1.Tenant, domain string,
 	infra *provisioners.InfrastructureManifests, needsResourceGroup bool) pulumi.RunFunc {
 
 	return func(ctx *pulumi.Context) error {
@@ -208,7 +209,7 @@ func deployFunc(platform string, tenant *platformv1.Tenant,
 		}
 
 		if needsResourceGroup {
-			rgName, err := azureRGDeployFunc(platform, tenant)(ctx)
+			rgName, err := azureRGDeployFunc(platform, tenant, domain)(ctx)
 			if err != nil {
 				return err
 			}

diff --git a/internal/controllers/provisioning/provisioners/types.go b/internal/controllers/provisioning/provisioners/types.go
@@ -5,8 +5,10 @@ import (
 	provisioningv1 "totalsoft.ro/platform-controllers/pkg/apis/provisioning/v1alpha1"
 )
 
-type CreateInfrastructureFunc func(platform string,
+type CreateInfrastructureFunc func(
+	platform string,
 	tenant *platformv1.Tenant,
+	domain string,
 	infra *InfrastructureManifests) ProvisioningResult
 
 type InfrastructureManifests struct {