refactor e2e control plane initialized/ready

kubernetes-sigs · Nov 13, 2024 · 7a5e879 · 7a5e879
1 parent 9b99382
commit 7a5e879
Show file tree

Hide file tree

Showing 4 changed files with 87 additions and 43 deletions.
diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go
@@ -160,7 +160,8 @@ var _ = Describe("Workload cluster creation", func() {
 					withControlPlaneMachineCount(1),
 					withWorkerMachineCount(1),
 					withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
-						WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
+						WaitForControlPlaneInitialized:   EnsureControlPlaneInitializedNoAddons,
+						WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 					}),
 					withPostMachinesProvisioned(func() {
 						EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
@@ -210,7 +211,8 @@ var _ = Describe("Workload cluster creation", func() {
 				withWorkerMachineCount(2),
 				withControlPlaneInterval(specName, "wait-control-plane-ha"),
 				withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
-					WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneInitialized:   EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 				}),
 				withPostMachinesProvisioned(func() {
 					EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
@@ -308,7 +310,8 @@ var _ = Describe("Workload cluster creation", func() {
 				withWorkerMachineCount(2),
 				withControlPlaneInterval(specName, "wait-control-plane-ha"),
 				withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
-					WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneInitialized:   EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 				}),
 				withPostMachinesProvisioned(func() {
 					EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
@@ -367,7 +370,8 @@ var _ = Describe("Workload cluster creation", func() {
 				withControlPlaneMachineCount(1),
 				withWorkerMachineCount(1),
 				withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
-					WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneInitialized:   EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 				}),
 				withPostMachinesProvisioned(func() {
 					EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
@@ -405,7 +409,8 @@ var _ = Describe("Workload cluster creation", func() {
 				withControlPlaneMachineCount(1),
 				withWorkerMachineCount(1),
 				withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
-					WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneInitialized:   EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 				}),
 				withPostMachinesProvisioned(func() {
 					EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
@@ -442,7 +447,8 @@ var _ = Describe("Workload cluster creation", func() {
 				withControlPlaneMachineCount(1),
 				withWorkerMachineCount(1),
 				withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
-					WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneInitialized:   EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 				}),
 				withPostMachinesProvisioned(func() {
 					EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
@@ -480,7 +486,8 @@ var _ = Describe("Workload cluster creation", func() {
 				withWorkerMachineCount(1),
 				withControlPlaneInterval(specName, "wait-control-plane-ha"),
 				withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
-					WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneInitialized:   EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 				}),
 				withPostMachinesProvisioned(func() {
 					EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
@@ -540,7 +547,8 @@ var _ = Describe("Workload cluster creation", func() {
 				withControlPlaneInterval(specName, "wait-control-plane"),
 				withMachinePoolInterval(specName, "wait-machine-pool-nodes"),
 				withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
-					WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneInitialized:   EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 				}),
 				withPostMachinesProvisioned(func() {
 					EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
@@ -609,7 +617,8 @@ var _ = Describe("Workload cluster creation", func() {
 				withWorkerMachineCount(1),
 				withMachineDeploymentInterval(specName, "wait-gpu-nodes"),
 				withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
-					WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneInitialized:   EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 				}),
 				withPostMachinesProvisioned(func() {
 					EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
@@ -667,7 +676,8 @@ var _ = Describe("Workload cluster creation", func() {
 				withWorkerMachineCount(1),
 				withMachineDeploymentInterval(specName, ""),
 				withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
-					WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneInitialized:   EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 				}),
 				withMachinePoolInterval(specName, "wait-machine-pool-nodes"),
 				withControlPlaneInterval(specName, "wait-control-plane"),
@@ -1007,7 +1017,8 @@ var _ = Describe("Workload cluster creation", func() {
 				withWorkerMachineCount(1),
 				withControlPlaneInterval(specName, "wait-control-plane-ha"),
 				withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
-					WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneInitialized:   EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 				}),
 				withPostMachinesProvisioned(func() {
 					EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
@@ -1081,7 +1092,8 @@ var _ = Describe("Workload cluster creation", func() {
 				withControlPlaneMachineCount(1),
 				withWorkerMachineCount(1),
 				withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
-					WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneInitialized:   EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 				}),
 				withPostMachinesProvisioned(func() {
 					EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
@@ -1126,7 +1138,8 @@ var _ = Describe("Workload cluster creation", func() {
 				withControlPlaneMachineCount(1),
 				withWorkerMachineCount(1),
 				withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
-					WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneInitialized:   EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 				}),
 				withPostMachinesProvisioned(func() {
 					EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {

diff --git a/test/e2e/capi_test.go b/test/e2e/capi_test.go
@@ -80,7 +80,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() {
 				ArtifactFolder:        artifactFolder,
 				SkipCleanup:           skipCleanup,
 				ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
-					WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneInitialized:   EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 				},
 			}
 		})
@@ -95,7 +96,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() {
 				ArtifactFolder:        artifactFolder,
 				SkipCleanup:           skipCleanup,
 				ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
-					WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneInitialized:   EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 				},
 			}
 		})
@@ -111,7 +113,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() {
 					ArtifactFolder:        artifactFolder,
 					SkipCleanup:           skipCleanup,
 					ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
-						WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
+						WaitForControlPlaneInitialized:   EnsureControlPlaneInitializedNoAddons,
+						WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 					},
 				}
 			})
@@ -128,7 +131,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() {
 				ArtifactFolder:        artifactFolder,
 				SkipCleanup:           skipCleanup,
 				ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
-					WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneInitialized:   EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 				},
 			}
 		})
@@ -143,7 +147,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() {
 				ArtifactFolder:        artifactFolder,
 				SkipCleanup:           skipCleanup,
 				ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
-					WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneInitialized:   EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 				},
 			}
 		})
@@ -158,7 +163,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() {
 				ArtifactFolder:        artifactFolder,
 				SkipCleanup:           skipCleanup,
 				ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
-					WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneInitialized:   EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 				},
 			}
 		})
@@ -173,7 +179,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() {
 				ArtifactFolder:        artifactFolder,
 				SkipCleanup:           skipCleanup,
 				ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
-					WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneInitialized:   EnsureControlPlaneInitializedNoAddons,
+					WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 				},
 			}
 		})
@@ -213,7 +220,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() {
 						PreInit:                   getPreInitFunc(ctx),
 						InitWithProvidersContract: "v1beta1",
 						ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
-							WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
+							WaitForControlPlaneInitialized:   EnsureControlPlaneInitialized,
+							WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 						},
 						InitWithKubernetesVersion:       e2eConfig.GetVariable(KubernetesVersionAPIUpgradeFrom),
 						InitWithBinary:                  fmt.Sprintf("https://github.com/kubernetes-sigs/cluster-api/releases/download/%s/clusterctl-{OS}-{ARCH}", e2eConfig.GetVariable(OldCAPIUpgradeVersion)),
@@ -238,7 +246,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() {
 						PreInit:                   getPreInitFunc(ctx),
 						InitWithProvidersContract: "v1beta1",
 						ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
-							WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
+							WaitForControlPlaneInitialized:   EnsureControlPlaneInitialized,
+							WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 						},
 						InitWithKubernetesVersion:       e2eConfig.GetVariable(KubernetesVersionAPIUpgradeFrom),
 						InitWithBinary:                  fmt.Sprintf("https://github.com/kubernetes-sigs/cluster-api/releases/download/%s/clusterctl-{OS}-{ARCH}", e2eConfig.GetVariable(LatestCAPIUpgradeVersion)),
@@ -263,7 +272,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() {
 				SkipCleanup:           skipCleanup,
 				SkipConformanceTests:  true,
 				ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
-					WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
+					WaitForControlPlaneInitialized:   EnsureControlPlaneInitialized,
+					WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 				},
 			}
 		})
@@ -281,7 +291,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() {
 				SkipCleanup:              skipCleanup,
 				SkipConformanceTests:     true,
 				ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
-					WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
+					WaitForControlPlaneInitialized:   EnsureControlPlaneInitialized,
+					WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 				},
 			}
 		})
@@ -300,7 +311,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() {
 				SkipConformanceTests:     true,
 				Flavor:                   ptr.To("kcp-scale-in"),
 				ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
-					WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
+					WaitForControlPlaneInitialized:   EnsureControlPlaneInitialized,
+					WaitForControlPlaneMachinesReady: EnsureControlPlaneReady,
 				},
 			}
 		})

diff --git a/test/e2e/cloud-provider-azure.go b/test/e2e/cloud-provider-azure.go
@@ -45,7 +45,6 @@ const (
 // EnsureCNIAndCloudProviderAzureHelmChart installs the official cloud-provider-azure helm chart
 // and a CNI and validates that expected pods exist and are Ready.
 func EnsureCNIAndCloudProviderAzureHelmChart(ctx context.Context, input clusterctl.ApplyCustomClusterTemplateAndWaitInput, installHelmChart bool, cidrBlocks []string, hasWindows bool) {
-	specName := "ensure-cloud-provider-azure"
 	clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, input.Namespace, input.ClusterName)
 
 	if installHelmChart {
@@ -72,17 +71,6 @@ func EnsureCNIAndCloudProviderAzureHelmChart(ctx context.Context, input clusterc
 		}
 
 		InstallHelmChart(ctx, clusterProxy, defaultNamespace, cloudProviderAzureHelmRepoURL, cloudProviderAzureChartName, cloudProviderAzureHelmReleaseName, options, "")
-	} else {
-		By("Ensuring cloud-provider-azure is installed via CAAPH")
-	}
-
-	// We do this before waiting for the pods to be ready because there is a co-dependency between CNI (nodes ready) and cloud-provider being initialized.
-	EnsureCNI(ctx, input, installHelmChart, cidrBlocks, hasWindows)
-
-	By("Waiting for Ready cloud-controller-manager deployment pods")
-	for _, d := range []string{"cloud-controller-manager"} {
-		waitInput := GetWaitForDeploymentsAvailableInput(ctx, clusterProxy, d, kubesystem, specName)
-		WaitForDeploymentsAvailable(ctx, waitInput, e2eConfig.GetIntervals(specName, "wait-deployment")...)
 	}
 }
 
@@ -121,10 +109,4 @@ func EnsureAzureDiskCSIDriverHelmChart(ctx context.Context, input clusterctl.App
 			g.Expect(mgmtClient.Update(ctx, cluster)).To(Succeed())
 		}, e2eConfig.GetIntervals(specName, "wait-deployment")...).Should(Succeed())
 	}
-
-	By("Waiting for Ready csi-azuredisk-controller deployment pods")
-	for _, d := range []string{"csi-azuredisk-controller"} {
-		waitInput := GetWaitForDeploymentsAvailableInput(ctx, clusterProxy, d, kubesystem, specName)
-		WaitForDeploymentsAvailable(ctx, waitInput, e2eConfig.GetIntervals(specName, "wait-deployment")...)
-	}
 }
diff --git a/test/e2e/common.go b/test/e2e/common.go
@@ -308,6 +308,43 @@ func ensureControlPlaneInitialized(ctx context.Context, input clusterctl.ApplyCu
 	result.ControlPlane = controlPlane
 }
 
+// EnsureControlPlaneReady waits for the cluster KubeadmControlPlane object to be ready.
+// Fulfills the clusterctl.Waiter type so that it can be used as ApplyClusterTemplateAndWaitInput data
+// in the flow of a clusterctl.ApplyClusterTemplateAndWait E2E test scenario.
+func EnsureControlPlaneReady(ctx context.Context, input clusterctl.ApplyCustomClusterTemplateAndWaitInput, result *clusterctl.ApplyCustomClusterTemplateAndWaitResult) {
+	getter := input.ClusterProxy.GetClient()
+	cluster := framework.GetClusterByName(ctx, framework.GetClusterByNameInput{
+		Getter:    getter,
+		Name:      input.ClusterName,
+		Namespace: input.Namespace,
+	})
+
+	_, hasWindows := cluster.Labels["cni-windows"]
+
+	// We do this before waiting for the pods to be ready because there is a co-dependency between CNI (nodes ready) and cloud-provider being initialized.
+	EnsureCNI(ctx, input, false, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks, hasWindows)
+
+	By("Waiting for Ready cloud-controller-manager deployment pods")
+	clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, cluster.Namespace, cluster.Name)
+	specName := "ensure-cloud-provider-azure"
+	for _, d := range []string{"cloud-controller-manager"} {
+		waitInput := GetWaitForDeploymentsAvailableInput(ctx, clusterProxy, d, kubesystem, specName)
+		WaitForDeploymentsAvailable(ctx, waitInput, e2eConfig.GetIntervals(specName, "wait-deployment")...)
+	}
+
+	By("Waiting for Ready csi-azuredisk-controller deployment pods")
+	for _, d := range []string{"csi-azuredisk-controller"} {
+		waitInput := GetWaitForDeploymentsAvailableInput(ctx, clusterProxy, d, kubesystem, specName)
+		WaitForDeploymentsAvailable(ctx, waitInput, e2eConfig.GetIntervals(specName, "wait-deployment")...)
+	}
+
+	framework.WaitForControlPlaneAndMachinesReady(ctx, framework.WaitForControlPlaneAndMachinesReadyInput{
+		GetLister:    input.ClusterProxy.GetClient(),
+		Cluster:      result.Cluster,
+		ControlPlane: result.ControlPlane,
+	}, input.WaitForControlPlaneIntervals...)
+}
+
 // CheckTestBeforeCleanup checks to see if the current running Ginkgo test failed, and prints
 // a status message regarding cleanup.
 func CheckTestBeforeCleanup() {