From 7a5e879b986a2b730a9341089443c9703dced62e Mon Sep 17 00:00:00 2001 From: Jon Huhn Date: Wed, 13 Nov 2024 16:10:55 -0600 Subject: [PATCH] refactor e2e control plane initialized/ready --- test/e2e/azure_test.go | 39 +++++++++++++++++++++----------- test/e2e/capi_test.go | 36 +++++++++++++++++++---------- test/e2e/cloud-provider-azure.go | 18 --------------- test/e2e/common.go | 37 ++++++++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 43 deletions(-) diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index cc501675ce1..619f49f7a5c 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -160,7 +160,8 @@ var _ = Describe("Workload cluster creation", func() { withControlPlaneMachineCount(1), withWorkerMachineCount(1), withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }), withPostMachinesProvisioned(func() { EnsureDaemonsets(ctx, func() DaemonsetsSpecInput { @@ -210,7 +211,8 @@ var _ = Describe("Workload cluster creation", func() { withWorkerMachineCount(2), withControlPlaneInterval(specName, "wait-control-plane-ha"), withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }), withPostMachinesProvisioned(func() { EnsureDaemonsets(ctx, func() DaemonsetsSpecInput { @@ -308,7 +310,8 @@ var _ = Describe("Workload cluster creation", func() { withWorkerMachineCount(2), withControlPlaneInterval(specName, "wait-control-plane-ha"), withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }), withPostMachinesProvisioned(func() { EnsureDaemonsets(ctx, func() DaemonsetsSpecInput { @@ -367,7 +370,8 @@ var _ = Describe("Workload cluster creation", func() { withControlPlaneMachineCount(1), withWorkerMachineCount(1), withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }), withPostMachinesProvisioned(func() { EnsureDaemonsets(ctx, func() DaemonsetsSpecInput { @@ -405,7 +409,8 @@ var _ = Describe("Workload cluster creation", func() { withControlPlaneMachineCount(1), withWorkerMachineCount(1), withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }), withPostMachinesProvisioned(func() { EnsureDaemonsets(ctx, func() DaemonsetsSpecInput { @@ -442,7 +447,8 @@ var _ = Describe("Workload cluster creation", func() { withControlPlaneMachineCount(1), withWorkerMachineCount(1), withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }), withPostMachinesProvisioned(func() { EnsureDaemonsets(ctx, func() DaemonsetsSpecInput { @@ -480,7 +486,8 @@ var _ = Describe("Workload cluster creation", func() { withWorkerMachineCount(1), withControlPlaneInterval(specName, "wait-control-plane-ha"), withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }), withPostMachinesProvisioned(func() { EnsureDaemonsets(ctx, func() DaemonsetsSpecInput { @@ -540,7 +547,8 @@ var _ = Describe("Workload cluster creation", func() { withControlPlaneInterval(specName, "wait-control-plane"), withMachinePoolInterval(specName, "wait-machine-pool-nodes"), withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }), withPostMachinesProvisioned(func() { EnsureDaemonsets(ctx, func() DaemonsetsSpecInput { @@ -609,7 +617,8 @@ var _ = Describe("Workload cluster creation", func() { withWorkerMachineCount(1), withMachineDeploymentInterval(specName, "wait-gpu-nodes"), withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }), withPostMachinesProvisioned(func() { EnsureDaemonsets(ctx, func() DaemonsetsSpecInput { @@ -667,7 +676,8 @@ var _ = Describe("Workload cluster creation", func() { withWorkerMachineCount(1), withMachineDeploymentInterval(specName, ""), withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }), withMachinePoolInterval(specName, "wait-machine-pool-nodes"), withControlPlaneInterval(specName, "wait-control-plane"), @@ -1007,7 +1017,8 @@ var _ = Describe("Workload cluster creation", func() { withWorkerMachineCount(1), withControlPlaneInterval(specName, "wait-control-plane-ha"), withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }), withPostMachinesProvisioned(func() { EnsureDaemonsets(ctx, func() DaemonsetsSpecInput { @@ -1081,7 +1092,8 @@ var _ = Describe("Workload cluster creation", func() { withControlPlaneMachineCount(1), withWorkerMachineCount(1), withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }), withPostMachinesProvisioned(func() { EnsureDaemonsets(ctx, func() DaemonsetsSpecInput { @@ -1126,7 +1138,8 @@ var _ = Describe("Workload cluster creation", func() { withControlPlaneMachineCount(1), withWorkerMachineCount(1), withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }), withPostMachinesProvisioned(func() { EnsureDaemonsets(ctx, func() DaemonsetsSpecInput { diff --git a/test/e2e/capi_test.go b/test/e2e/capi_test.go index c6f1b1df680..ae38d9eb077 100644 --- a/test/e2e/capi_test.go +++ b/test/e2e/capi_test.go @@ -80,7 +80,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() { ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }, } }) @@ -95,7 +96,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() { ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }, } }) @@ -111,7 +113,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() { ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }, } }) @@ -128,7 +131,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() { ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }, } }) @@ -143,7 +147,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() { ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }, } }) @@ -158,7 +163,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() { ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }, } }) @@ -173,7 +179,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() { ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }, } }) @@ -213,7 +220,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() { PreInit: getPreInitFunc(ctx), InitWithProvidersContract: "v1beta1", ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitialized, + WaitForControlPlaneInitialized: EnsureControlPlaneInitialized, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }, InitWithKubernetesVersion: e2eConfig.GetVariable(KubernetesVersionAPIUpgradeFrom), InitWithBinary: fmt.Sprintf("https://github.com/kubernetes-sigs/cluster-api/releases/download/%s/clusterctl-{OS}-{ARCH}", e2eConfig.GetVariable(OldCAPIUpgradeVersion)), @@ -238,7 +246,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() { PreInit: getPreInitFunc(ctx), InitWithProvidersContract: "v1beta1", ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitialized, + WaitForControlPlaneInitialized: EnsureControlPlaneInitialized, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }, InitWithKubernetesVersion: e2eConfig.GetVariable(KubernetesVersionAPIUpgradeFrom), InitWithBinary: fmt.Sprintf("https://github.com/kubernetes-sigs/cluster-api/releases/download/%s/clusterctl-{OS}-{ARCH}", e2eConfig.GetVariable(LatestCAPIUpgradeVersion)), @@ -263,7 +272,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() { SkipCleanup: skipCleanup, SkipConformanceTests: true, ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitialized, + WaitForControlPlaneInitialized: EnsureControlPlaneInitialized, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }, } }) @@ -281,7 +291,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() { SkipCleanup: skipCleanup, SkipConformanceTests: true, ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitialized, + WaitForControlPlaneInitialized: EnsureControlPlaneInitialized, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }, } }) @@ -300,7 +311,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() { SkipConformanceTests: true, Flavor: ptr.To("kcp-scale-in"), ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitialized, + WaitForControlPlaneInitialized: EnsureControlPlaneInitialized, + WaitForControlPlaneMachinesReady: EnsureControlPlaneReady, }, } }) diff --git a/test/e2e/cloud-provider-azure.go b/test/e2e/cloud-provider-azure.go index 489ccea05ad..869ae19522d 100644 --- a/test/e2e/cloud-provider-azure.go +++ b/test/e2e/cloud-provider-azure.go @@ -45,7 +45,6 @@ const ( // EnsureCNIAndCloudProviderAzureHelmChart installs the official cloud-provider-azure helm chart // and a CNI and validates that expected pods exist and are Ready. func EnsureCNIAndCloudProviderAzureHelmChart(ctx context.Context, input clusterctl.ApplyCustomClusterTemplateAndWaitInput, installHelmChart bool, cidrBlocks []string, hasWindows bool) { - specName := "ensure-cloud-provider-azure" clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, input.Namespace, input.ClusterName) if installHelmChart { @@ -72,17 +71,6 @@ func EnsureCNIAndCloudProviderAzureHelmChart(ctx context.Context, input clusterc } InstallHelmChart(ctx, clusterProxy, defaultNamespace, cloudProviderAzureHelmRepoURL, cloudProviderAzureChartName, cloudProviderAzureHelmReleaseName, options, "") - } else { - By("Ensuring cloud-provider-azure is installed via CAAPH") - } - - // We do this before waiting for the pods to be ready because there is a co-dependency between CNI (nodes ready) and cloud-provider being initialized. - EnsureCNI(ctx, input, installHelmChart, cidrBlocks, hasWindows) - - By("Waiting for Ready cloud-controller-manager deployment pods") - for _, d := range []string{"cloud-controller-manager"} { - waitInput := GetWaitForDeploymentsAvailableInput(ctx, clusterProxy, d, kubesystem, specName) - WaitForDeploymentsAvailable(ctx, waitInput, e2eConfig.GetIntervals(specName, "wait-deployment")...) } } @@ -121,10 +109,4 @@ func EnsureAzureDiskCSIDriverHelmChart(ctx context.Context, input clusterctl.App g.Expect(mgmtClient.Update(ctx, cluster)).To(Succeed()) }, e2eConfig.GetIntervals(specName, "wait-deployment")...).Should(Succeed()) } - - By("Waiting for Ready csi-azuredisk-controller deployment pods") - for _, d := range []string{"csi-azuredisk-controller"} { - waitInput := GetWaitForDeploymentsAvailableInput(ctx, clusterProxy, d, kubesystem, specName) - WaitForDeploymentsAvailable(ctx, waitInput, e2eConfig.GetIntervals(specName, "wait-deployment")...) - } } diff --git a/test/e2e/common.go b/test/e2e/common.go index 466e5314228..78b65e522b6 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -308,6 +308,43 @@ func ensureControlPlaneInitialized(ctx context.Context, input clusterctl.ApplyCu result.ControlPlane = controlPlane } +// EnsureControlPlaneReady waits for the cluster KubeadmControlPlane object to be ready. +// Fulfills the clusterctl.Waiter type so that it can be used as ApplyClusterTemplateAndWaitInput data +// in the flow of a clusterctl.ApplyClusterTemplateAndWait E2E test scenario. +func EnsureControlPlaneReady(ctx context.Context, input clusterctl.ApplyCustomClusterTemplateAndWaitInput, result *clusterctl.ApplyCustomClusterTemplateAndWaitResult) { + getter := input.ClusterProxy.GetClient() + cluster := framework.GetClusterByName(ctx, framework.GetClusterByNameInput{ + Getter: getter, + Name: input.ClusterName, + Namespace: input.Namespace, + }) + + _, hasWindows := cluster.Labels["cni-windows"] + + // We do this before waiting for the pods to be ready because there is a co-dependency between CNI (nodes ready) and cloud-provider being initialized. + EnsureCNI(ctx, input, false, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks, hasWindows) + + By("Waiting for Ready cloud-controller-manager deployment pods") + clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, cluster.Namespace, cluster.Name) + specName := "ensure-cloud-provider-azure" + for _, d := range []string{"cloud-controller-manager"} { + waitInput := GetWaitForDeploymentsAvailableInput(ctx, clusterProxy, d, kubesystem, specName) + WaitForDeploymentsAvailable(ctx, waitInput, e2eConfig.GetIntervals(specName, "wait-deployment")...) + } + + By("Waiting for Ready csi-azuredisk-controller deployment pods") + for _, d := range []string{"csi-azuredisk-controller"} { + waitInput := GetWaitForDeploymentsAvailableInput(ctx, clusterProxy, d, kubesystem, specName) + WaitForDeploymentsAvailable(ctx, waitInput, e2eConfig.GetIntervals(specName, "wait-deployment")...) + } + + framework.WaitForControlPlaneAndMachinesReady(ctx, framework.WaitForControlPlaneAndMachinesReadyInput{ + GetLister: input.ClusterProxy.GetClient(), + Cluster: result.Cluster, + ControlPlane: result.ControlPlane, + }, input.WaitForControlPlaneIntervals...) +} + // CheckTestBeforeCleanup checks to see if the current running Ginkgo test failed, and prints // a status message regarding cleanup. func CheckTestBeforeCleanup() {