diff --git a/.gitignore b/.gitignore index 7b63061a..b664c343 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,7 @@ bin/ out/ _releasenotes -_artifacts/ \ No newline at end of file +_artifacts/ + +# E2E test templates +test/e2e/data/infrastructure-docker/cluster-template*.yaml \ No newline at end of file diff --git a/Makefile b/Makefile index f3d146fa..0ba4e349 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ KUSTOMIZE := $(TOOLS_BIN_DIR)/$(KUSTOMIZE_BIN)-$(KUSTOMIZE_VER) # Ginkgo TEST_DIR := $(shell pwd)/test ARTIFACTS ?= $(shell pwd)/_artifacts -GINKGO_FOCUS ?= +GINKGO_FOCUS ?= GINKGO_SKIP ?= GINKGO_NODES ?= 1 # GINKGO_NODES is the number of parallel nodes to run # when running the e2e tests, 1 means no parallelism @@ -218,6 +218,8 @@ all-controlplane: manager-controlplane test-controlplane: envtest generate-controlplane generate-controlplane-conversions lint manifests-controlplane KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(TOOLS_BIN_DIR) -p path)" go test $(shell pwd)/controlplane/... -coverprofile cover.out +DOCKER_TEMPLATES := test/e2e/data/infrastructure-docker + .PHONY: docker-build-e2e docker-build-e2e: ## Run docker-build-* targets for all the images with settings to be used for the e2e tests # please ensure the generated image name matches image names used in the E2E_CONF_FILE @@ -225,8 +227,16 @@ docker-build-e2e: ## Run docker-build-* targets for all the images with settings $(MAKE) BOOTSTRAP_IMG_TAG=dev docker-build-bootstrap $(MAKE) CONTROLPLANE_IMG_TAG=dev docker-build-controlplane +.PHONY: generate-e2e-templates +generate-e2e-templates: $(KUSTOMIZE) + $(KUSTOMIZE) build $(DOCKER_TEMPLATES)/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/cluster-template.yaml + $(KUSTOMIZE) build $(DOCKER_TEMPLATES)/cluster-template-v1beta1 --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/cluster-template-v1beta1.yaml + $(KUSTOMIZE) build $(DOCKER_TEMPLATES)/cluster-template-kcp-remediation --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/cluster-template-kcp-remediation.yaml + $(KUSTOMIZE) build $(DOCKER_TEMPLATES)/cluster-template-md-remediation --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/cluster-template-md-remediation.yaml + $(KUSTOMIZE) build $(DOCKER_TEMPLATES)/cluster-template-topology --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/cluster-template-topology.yaml + .PHONY: test-e2e -test-e2e: $(GINKGO) $(KUSTOMIZE) ## Run the end-to-end tests +test-e2e: generate-e2e-templates $(GINKGO) $(KUSTOMIZE) ## Run the end-to-end tests CAPI_KUSTOMIZE_PATH="$(KUSTOMIZE)" $(GINKGO) -v --trace -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) \ -poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" \ $(_SKIP_ARGS) --nodes=$(GINKGO_NODES) --timeout=$(GINKGO_TIMEOUT) --no-color=$(GINKGO_NOCOLOR) \ diff --git a/bootstrap/config/rbac/role.yaml b/bootstrap/config/rbac/role.yaml index 0e359c91..63e35bf0 100644 --- a/bootstrap/config/rbac/role.yaml +++ b/bootstrap/config/rbac/role.yaml @@ -47,17 +47,10 @@ rules: resources: - clusters - clusters/status - - machines - - machines/status - verbs: - - get - - list - - watch -- apiGroups: - - exp.cluster.x-k8s.io - resources: - machinepools - machinepools/status + - machines + - machines/status verbs: - get - list diff --git a/bootstrap/controllers/kthreesconfig_controller.go b/bootstrap/controllers/kthreesconfig_controller.go index 09a76f69..401c1ac0 100644 --- a/bootstrap/controllers/kthreesconfig_controller.go +++ b/bootstrap/controllers/kthreesconfig_controller.go @@ -78,8 +78,7 @@ var ( // +kubebuilder:rbac:groups=bootstrap.cluster.x-k8s.io,resources=kthreesconfigs,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=bootstrap.cluster.x-k8s.io,resources=kthreesconfigs/status,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;clusters/status;machines;machines/status,verbs=get;list;watch -// +kubebuilder:rbac:groups=exp.cluster.x-k8s.io,resources=machinepools;machinepools/status,verbs=get;list;watch +// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;clusters/status;machines;machines/status;machinepools;machinepools/status,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=secrets;events;configmaps,verbs=get;list;watch;create;update;patch;delete func (r *KThreesConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ reconcile.Result, rerr error) { diff --git a/docs/tilt-setup.md b/docs/tilt-setup.md index 01973c33..52a05ac2 100644 --- a/docs/tilt-setup.md +++ b/docs/tilt-setup.md @@ -89,6 +89,7 @@ Also, visit the [Cluster API documentation on Tilt][cluster_api_tilt] for more i # add template for fast workload cluster creation, change to a different path if needed # you could also add more templates k3s-bootstrap: + # please run `make generate-e2e-templates` to generate the templates first - ../cluster-api-k3s/test/e2e/data/infrastructure-docker ``` 5. Run `tilt` in the `cluster-api` directory diff --git a/test/e2e/cluster_upgrade.go b/test/e2e/cluster_upgrade.go index d53d9fa8..48a88c07 100644 --- a/test/e2e/cluster_upgrade.go +++ b/test/e2e/cluster_upgrade.go @@ -29,11 +29,13 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" "k8s.io/utils/ptr" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/patch" ) // ClusterUpgradeSpecInput is the input for ClusterUpgradeConformanceSpec. @@ -76,8 +78,9 @@ func ClusterUpgradeSpec(ctx context.Context, inputGetter func() ClusterUpgradeSp namespace *corev1.Namespace cancelWatches context.CancelFunc - controlPlaneMachineCount int64 - workerMachineCount int64 + controlPlaneMachineCount int64 + workerMachineCount int64 + kubernetesVersionUpgradeTo string result *ApplyClusterTemplateAndWaitResult clusterName string @@ -109,6 +112,8 @@ func ClusterUpgradeSpec(ctx context.Context, inputGetter func() ClusterUpgradeSp workerMachineCount = *input.WorkerMachineCount } + kubernetesVersionUpgradeTo = input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo) + // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) @@ -154,30 +159,83 @@ func ClusterUpgradeSpec(ctx context.Context, inputGetter func() ClusterUpgradeSp WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), }, result) - By("Upgrading the Kubernetes control-plane") - UpgradeControlPlaneAndWaitForUpgrade(ctx, UpgradeControlPlaneAndWaitForUpgradeInput{ - ClusterProxy: input.BootstrapClusterProxy, - Cluster: result.Cluster, - ControlPlane: result.ControlPlane, - KubernetesUpgradeVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo), - WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"), - }) - - By("Upgrading the machine deployment") - framework.UpgradeMachineDeploymentsAndWait(ctx, framework.UpgradeMachineDeploymentsAndWaitInput{ - ClusterProxy: input.BootstrapClusterProxy, - Cluster: result.Cluster, - UpgradeVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo), - MachineDeployments: result.MachineDeployments, - WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), - }) + if result.Cluster.Spec.Topology != nil { + // Cluster is using ClusterClass, upgrade via topology. + By("Upgrading the Cluster topology") + mgmtClient := input.BootstrapClusterProxy.GetClient() + + By("Patching the new Kubernetes version to Cluster topology") + patchHelper, err := patch.NewHelper(result.Cluster, mgmtClient) + Expect(err).ToNot(HaveOccurred()) + + result.Cluster.Spec.Topology.Version = kubernetesVersionUpgradeTo + + Eventually(func() error { + return patchHelper.Patch(ctx, result.Cluster) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to patch Cluster topology %s with version %s", klog.KObj(result.Cluster), kubernetesVersionUpgradeTo) + + By("Waiting for control-plane machines to have the upgraded kubernetes version") + framework.WaitForControlPlaneMachinesToBeUpgraded(ctx, framework.WaitForControlPlaneMachinesToBeUpgradedInput{ + Lister: mgmtClient, + Cluster: result.Cluster, + MachineCount: int(*result.ControlPlane.Spec.Replicas), + KubernetesUpgradeVersion: kubernetesVersionUpgradeTo, + }, input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade")...) + + for _, deployment := range result.MachineDeployments { + if *deployment.Spec.Replicas > 0 { + Byf("Waiting for Kubernetes versions of machines in MachineDeployment %s to be upgraded to %s", + klog.KObj(deployment), kubernetesVersionUpgradeTo) + framework.WaitForMachineDeploymentMachinesToBeUpgraded(ctx, framework.WaitForMachineDeploymentMachinesToBeUpgradedInput{ + Lister: mgmtClient, + Cluster: result.Cluster, + MachineCount: int(*deployment.Spec.Replicas), + KubernetesUpgradeVersion: kubernetesVersionUpgradeTo, + MachineDeployment: *deployment, + }, input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade")...) + } + } + + for _, pool := range result.MachinePools { + if *pool.Spec.Replicas > 0 { + Byf("Waiting for Kubernetes versions of machines in MachinePool %s to be upgraded to %s", + klog.KObj(pool), kubernetesVersionUpgradeTo) + framework.WaitForMachinePoolInstancesToBeUpgraded(ctx, framework.WaitForMachinePoolInstancesToBeUpgradedInput{ + Getter: mgmtClient, + WorkloadClusterGetter: input.BootstrapClusterProxy.GetWorkloadCluster(ctx, result.Cluster.Namespace, result.Cluster.Name).GetClient(), + Cluster: result.Cluster, + MachineCount: int(*pool.Spec.Replicas), + KubernetesUpgradeVersion: kubernetesVersionUpgradeTo, + MachinePool: pool, + }, input.E2EConfig.GetIntervals(specName, "wait-machine-pool-upgrade")...) + } + } + } else { + By("Upgrading the Kubernetes control-plane") + UpgradeControlPlaneAndWaitForUpgrade(ctx, UpgradeControlPlaneAndWaitForUpgradeInput{ + ClusterProxy: input.BootstrapClusterProxy, + Cluster: result.Cluster, + ControlPlane: result.ControlPlane, + KubernetesUpgradeVersion: kubernetesVersionUpgradeTo, + WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"), + }) + + By("Upgrading the machine deployment") + framework.UpgradeMachineDeploymentsAndWait(ctx, framework.UpgradeMachineDeploymentsAndWaitInput{ + ClusterProxy: input.BootstrapClusterProxy, + Cluster: result.Cluster, + UpgradeVersion: kubernetesVersionUpgradeTo, + MachineDeployments: result.MachineDeployments, + WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + }) + } By("Waiting until nodes are ready") workloadProxy := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, result.Cluster.Name) workloadClient := workloadProxy.GetClient() framework.WaitForNodesReady(ctx, framework.WaitForNodesReadyInput{ Lister: workloadClient, - KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo), + KubernetesVersion: kubernetesVersionUpgradeTo, Count: int(result.ExpectedTotalNodes()), WaitForNodesReady: input.E2EConfig.GetIntervals(specName, "wait-nodes-ready"), }) diff --git a/test/e2e/cluster_upgrade_test.go b/test/e2e/cluster_upgrade_test.go index 4185e8d8..7ce44dd1 100644 --- a/test/e2e/cluster_upgrade_test.go +++ b/test/e2e/cluster_upgrade_test.go @@ -54,4 +54,20 @@ var _ = Describe("Workload cluster upgrade [K3s-Upgrade]", func() { } }) }) + + Context("Upgrading a cluster with ClusterClass [ClusterClass]", func() { + ClusterUpgradeSpec(ctx, func() ClusterUpgradeSpecInput { + return ClusterUpgradeSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + Flavor: ptr.To("topology"), + InfrastructureProvider: ptr.To("docker"), + ControlPlaneMachineCount: ptr.To[int64](1), + WorkerMachineCount: ptr.To[int64](2), + } + }) + }) }) diff --git a/test/e2e/config/k3s-docker.yaml b/test/e2e/config/k3s-docker.yaml index 7080539e..71bdf58d 100644 --- a/test/e2e/config/k3s-docker.yaml +++ b/test/e2e/config/k3s-docker.yaml @@ -15,8 +15,8 @@ providers: - name: cluster-api type: CoreProvider versions: - - name: v1.6.2 - value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.2/core-components.yaml + - name: v1.7.2 + value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.7.2/core-components.yaml type: url files: - sourcePath: "../data/shared/v1beta1/metadata.yaml" @@ -28,8 +28,8 @@ providers: versions: # By default, will use the latest version defined in ../data/shared/v1beta1/metadata.yaml # to init the management cluster - - name: v1.6.2 # used during e2e-test - value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.2/infrastructure-components-development.yaml + - name: v1.7.2 # used during e2e-test + value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.7.2/infrastructure-components-development.yaml type: url files: - sourcePath: "../data/shared/v1beta1/metadata.yaml" @@ -37,12 +37,12 @@ providers: - old: "imagePullPolicy: Always" new: "imagePullPolicy: IfNotPresent" - # Add v1.7.99 to support tilt (not presented in ../data/shared/v1beta1/metadata.yaml) + # Add v1.8.99 to support tilt (not presented in ../data/shared/v1beta1/metadata.yaml) # when bootstrapping with tilt, it will use # the defaultProviderVersion in https://github.com/kubernetes-sigs/cluster-api/blob/main/hack/tools/internal/tilt-prepare/main.go as # default version for docker infrastructure provider # name here should match defaultProviderVersion - - name: v1.7.99 # next; use manifest from source files + - name: v1.8.99 # next; use manifest from source files value: https://github.com/kubernetes-sigs/cluster-api/releases/latest/download/infrastructure-components-development.yaml type: url files: @@ -54,6 +54,9 @@ providers: - sourcePath: "../data/infrastructure-docker/cluster-template.yaml" - sourcePath: "../data/infrastructure-docker/cluster-template-kcp-remediation.yaml" - sourcePath: "../data/infrastructure-docker/cluster-template-md-remediation.yaml" + - sourcePath: "../data/infrastructure-docker/cluster-template-v1beta1.yaml" + - sourcePath: "../data/infrastructure-docker/cluster-template-topology.yaml" + - sourcePath: "../data/infrastructure-docker/clusterclass-k3s.yaml" - name: k3s type: BootstrapProvider versions: @@ -82,6 +85,9 @@ variables: KUBERNETES_VERSION_UPGRADE_TO: "v1.28.7+k3s1" IP_FAMILY: "IPv4" KIND_IMAGE_VERSION: "v1.28.0" + # Enabling the feature flags by setting the env variables. + CLUSTER_TOPOLOGY: "true" + EXP_MACHINE_POOL: "true" intervals: # The array is defined as [timeout, polling interval] @@ -90,10 +96,10 @@ intervals: default/wait-cluster: ["5m", "10s"] default/wait-control-plane: ["10m", "10s"] default/wait-worker-nodes: ["10m", "10s"] - default/wait-machine-pool-nodes: ["5m", "10s"] + default/wait-machine-pool-nodes: ["10m", "10s"] default/wait-delete-cluster: ["3m", "10s"] default/wait-machine-upgrade: ["30m", "10s"] - default/wait-machine-pool-upgrade: ["5m", "10s"] + default/wait-machine-pool-upgrade: ["30m", "10s"] default/wait-nodes-ready: ["10m", "10s"] default/wait-machine-remediation: ["5m", "10s"] default/wait-autoscaler: ["5m", "10s"] diff --git a/test/e2e/create_test.go b/test/e2e/create_test.go index 96646fd1..15fc51ec 100644 --- a/test/e2e/create_test.go +++ b/test/e2e/create_test.go @@ -97,4 +97,52 @@ var _ = Describe("Workload cluster creation", func() { }, result) }) }) + + Context("Creating a cluster with v1beta1 api version for testing conversion", func() { + It("Should create a workload cluster with 1 control plane and 3 worker nodes", func() { + By("Creating a workload cluster with v1beta1 controlplane") + ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: clusterctlLogFolder, + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: infrastructureProvider, + Flavor: "v1beta1", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(3), + }, + WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), + }, result) + }) + }) + + Context("Creating a cluster with clusterclass [ClusterClass]", func() { + It("Should create a workload cluster with 1 control plane, 1 machine deployment and 1 machine pool", func() { + By("Creating a workload cluster with v1beta1 controlplane") + ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: clusterctlLogFolder, + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: infrastructureProvider, + Flavor: "topology", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), + }, result) + }) + }) }) diff --git a/test/e2e/data/infrastructure-docker/bases/cluster-with-kcp-v1beta1.yaml b/test/e2e/data/infrastructure-docker/bases/cluster-with-kcp-v1beta1.yaml new file mode 100644 index 00000000..f1b3d021 --- /dev/null +++ b/test/e2e/data/infrastructure-docker/bases/cluster-with-kcp-v1beta1.yaml @@ -0,0 +1,56 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +spec: + clusterNetwork: + pods: + cidrBlocks: + - 10.45.0.0/16 + services: + cidrBlocks: + - 10.46.0.0/16 + serviceDomain: cluster.local + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KThreesControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerCluster +metadata: + name: ${CLUSTER_NAME} +spec: {} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KThreesControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} +spec: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} + kthreesConfigSpec: + serverConfig: + tlsSan: + - 0.0.0.0 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} +spec: + template: + spec: + customImage: kindest/node:${KIND_IMAGE_VERSION} \ No newline at end of file diff --git a/test/e2e/data/infrastructure-docker/bases/cluster-with-kcp.yaml b/test/e2e/data/infrastructure-docker/bases/cluster-with-kcp.yaml new file mode 100644 index 00000000..d29dc3bd --- /dev/null +++ b/test/e2e/data/infrastructure-docker/bases/cluster-with-kcp.yaml @@ -0,0 +1,57 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +spec: + clusterNetwork: + pods: + cidrBlocks: + - 10.45.0.0/16 + services: + cidrBlocks: + - 10.46.0.0/16 + serviceDomain: cluster.local + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KThreesControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerCluster +metadata: + name: ${CLUSTER_NAME} +spec: {} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: KThreesControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} +spec: + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} + kthreesConfigSpec: + serverConfig: + tlsSan: + - 0.0.0.0 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} +spec: + template: + spec: + customImage: kindest/node:${KIND_IMAGE_VERSION} \ No newline at end of file diff --git a/test/e2e/data/infrastructure-docker/bases/cluster-with-topology.yaml b/test/e2e/data/infrastructure-docker/bases/cluster-with-topology.yaml new file mode 100644 index 00000000..81edcf81 --- /dev/null +++ b/test/e2e/data/infrastructure-docker/bases/cluster-with-topology.yaml @@ -0,0 +1,41 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +spec: + clusterNetwork: + pods: + cidrBlocks: + - 10.45.0.0/16 + services: + cidrBlocks: + - 10.46.0.0/16 + serviceDomain: cluster.local + topology: + class: k3s + version: ${KUBERNETES_VERSION} + controlPlane: + nodeDeletionTimeout: "30s" + nodeVolumeDetachTimeout: "5m" + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + workers: + machineDeployments: + - class: k3s-default-worker + name: ${CLUSTER_NAME}-md-0 + nodeDeletionTimeout: "30s" + nodeVolumeDetachTimeout: "5m" + minReadySeconds: 5 + replicas: ${WORKER_MACHINE_COUNT} + + # TODO: include machinePools when the feature is ready + # machinePools: + # - class: k3s-default-worker + # name: ${CLUSTER_NAME}-mp-0 + # nodeDeletionTimeout: "30s" + # nodeVolumeDetachTimeout: "5m" + # minReadySeconds: 5 + # replicas: ${WORKER_MACHINE_COUNT} + variables: + - name: kindImageVersion + value: ${KIND_IMAGE_VERSION} \ No newline at end of file diff --git a/test/e2e/data/infrastructure-docker/bases/md-v1beta1.yaml b/test/e2e/data/infrastructure-docker/bases/md-v1beta1.yaml new file mode 100644 index 00000000..de3e602e --- /dev/null +++ b/test/e2e/data/infrastructure-docker/bases/md-v1beta1.yaml @@ -0,0 +1,52 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: ${NAMESPACE} +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + + # This label will be needed for upgrade test + # it will be used as a selector for only selecting + # machines belonging to this machine deployment + cluster.x-k8s.io/deployment-name: ${CLUSTER_NAME}-md-0 + template: + metadata: + labels: + cluster.x-k8s.io/deployment-name: ${CLUSTER_NAME}-md-0 + spec: + version: ${KUBERNETES_VERSION} + clusterName: ${CLUSTER_NAME} + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KThreesConfigTemplate + name: ${CLUSTER_NAME}-md-0 + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: ${CLUSTER_NAME}-md-0 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: ${NAMESPACE} +spec: + template: + spec: + customImage: kindest/node:${KIND_IMAGE_VERSION} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KThreesConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: ${NAMESPACE} +spec: + template: + spec: diff --git a/test/e2e/data/infrastructure-docker/bases/md.yaml b/test/e2e/data/infrastructure-docker/bases/md.yaml new file mode 100644 index 00000000..41f92a81 --- /dev/null +++ b/test/e2e/data/infrastructure-docker/bases/md.yaml @@ -0,0 +1,52 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: ${NAMESPACE} +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + + # This label will be needed for upgrade test + # it will be used as a selector for only selecting + # machines belonging to this machine deployment + cluster.x-k8s.io/deployment-name: ${CLUSTER_NAME}-md-0 + template: + metadata: + labels: + cluster.x-k8s.io/deployment-name: ${CLUSTER_NAME}-md-0 + spec: + version: ${KUBERNETES_VERSION} + clusterName: ${CLUSTER_NAME} + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KThreesConfigTemplate + name: ${CLUSTER_NAME}-md-0 + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: ${CLUSTER_NAME}-md-0 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: ${NAMESPACE} +spec: + template: + spec: + customImage: kindest/node:${KIND_IMAGE_VERSION} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: KThreesConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: ${NAMESPACE} +spec: + template: + spec: \ No newline at end of file diff --git a/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml b/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml deleted file mode 100644 index f31a91ab..00000000 --- a/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml +++ /dev/null @@ -1,157 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: ${CLUSTER_NAME} - namespace: ${NAMESPACE} -spec: - clusterNetwork: - pods: - cidrBlocks: - - 10.45.0.0/16 - services: - cidrBlocks: - - 10.46.0.0/16 - serviceDomain: cluster.local - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KThreesControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: DockerCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: DockerCluster -metadata: - name: ${CLUSTER_NAME} -spec: {} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KThreesControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: ${NAMESPACE} -spec: - infrastructureTemplate: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: DockerMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} - kthreesConfigSpec: - files: - - path: /wait-signal.sh - content: | - #!/bin/bash - - set -o errexit - set -o pipefail - - echo "Waiting for signal..." - - TOKEN=$1 - SERVER=$2 - NAMESPACE=$3 - - while true; - do - sleep 1s - - signal=$(curl -k -s --header "Authorization: Bearer $TOKEN" $SERVER/api/v1/namespaces/$NAMESPACE/configmaps/mhc-test | jq -r .data.signal?) - echo "signal $signal" - - if [ "$signal" == "pass" ]; then - curl -k -s --header "Authorization: Bearer $TOKEN" -XPATCH -H "Content-Type: application/strategic-merge-patch+json" --data '{"data": {"signal": "ack-pass"}}' $SERVER/api/v1/namespaces/$NAMESPACE/configmaps/mhc-test - exit 0 - fi - done - permissions: "0777" - preK3sCommands: - - ./wait-signal.sh "${TOKEN}" "${SERVER}" "${NAMESPACE}" - serverConfig: - tlsSan: - - localhost - - 127.0.0.1 - - 0.0.0.0 - - host.docker.internal ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: DockerMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: ${NAMESPACE} -spec: - template: - spec: - customImage: kindest/node:${KIND_IMAGE_VERSION} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: worker-md-0 - namespace: ${NAMESPACE} -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: - cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} - - # This label will be needed for upgrade test - # it will be used as a selector for only selecting - # machines belonging to this machine deployment - cluster.x-k8s.io/deployment-name: worker-md-0 - template: - metadata: - labels: - cluster.x-k8s.io/deployment-name: worker-md-0 - spec: - version: ${KUBERNETES_VERSION} - clusterName: ${CLUSTER_NAME} - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KThreesConfigTemplate - name: ${CLUSTER_NAME}-md-0 - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: DockerMachineTemplate - name: ${CLUSTER_NAME}-md-0 ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: DockerMachineTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: ${NAMESPACE} -spec: - template: - spec: - customImage: kindest/node:${KIND_IMAGE_VERSION} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KThreesConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: ${NAMESPACE} -spec: - template: - spec: ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineHealthCheck -metadata: - name: ${CLUSTER_NAME}-mhc-0 - namespace: ${NAMESPACE} -spec: - clusterName: ${CLUSTER_NAME} - maxUnhealthy: 100% - nodeStartupTimeout: 30s - selector: - matchLabels: - cluster.x-k8s.io/control-plane: "" - mhc-test: fail - unhealthyConditions: - - status: "False" - timeout: 10s - type: e2e.remediation.condition diff --git a/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation/kcp.yaml b/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation/kcp.yaml new file mode 100644 index 00000000..8bcecfa8 --- /dev/null +++ b/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation/kcp.yaml @@ -0,0 +1,42 @@ +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: KThreesControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} +spec: + kthreesConfigSpec: + files: + - path: /wait-signal.sh + content: | + #!/bin/bash + + set -o errexit + set -o pipefail + + echo "Waiting for signal..." + + TOKEN=$1 + SERVER=$2 + NAMESPACE=$3 + + while true; + do + sleep 1s + + signal=$(curl -k -s --header "Authorization: Bearer $TOKEN" $SERVER/api/v1/namespaces/$NAMESPACE/configmaps/mhc-test | jq -r .data.signal?) + echo "signal $signal" + + if [ "$signal" == "pass" ]; then + curl -k -s --header "Authorization: Bearer $TOKEN" -XPATCH -H "Content-Type: application/strategic-merge-patch+json" --data '{"data": {"signal": "ack-pass"}}' $SERVER/api/v1/namespaces/$NAMESPACE/configmaps/mhc-test + exit 0 + fi + done + permissions: "0777" + preK3sCommands: + - ./wait-signal.sh "${TOKEN}" "${SERVER}" "${NAMESPACE}" + serverConfig: + tlsSan: + - localhost + - 127.0.0.1 + - 0.0.0.0 + - host.docker.internal \ No newline at end of file diff --git a/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation/kustomization.yaml b/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation/kustomization.yaml new file mode 100644 index 00000000..40efa642 --- /dev/null +++ b/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- ../bases/cluster-with-kcp.yaml +- ../bases/md.yaml +- mhc.yaml +patches: +- path: kcp.yaml \ No newline at end of file diff --git a/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation/mhc.yaml b/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation/mhc.yaml new file mode 100644 index 00000000..28f4ca85 --- /dev/null +++ b/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation/mhc.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: ${CLUSTER_NAME}-mhc-0 + namespace: ${NAMESPACE} +spec: + clusterName: ${CLUSTER_NAME} + maxUnhealthy: 100% + nodeStartupTimeout: 30s + selector: + matchLabels: + cluster.x-k8s.io/control-plane: "" + mhc-test: fail + unhealthyConditions: + - status: "False" + timeout: 10s + type: e2e.remediation.condition diff --git a/test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml b/test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml deleted file mode 100644 index 6f353901..00000000 --- a/test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml +++ /dev/null @@ -1,129 +0,0 @@ -# TODO: copied and modified from https://github.com/k3s-io/cluster-api-k3s/pull/93/files#diff-c4a336ec56832a2ff7aed26c94d0d67ae3a0e6139d30701cc53c0f0962fe8cca -# should be the same as samples/docker/quickstart.yaml in the future -# for testing the quickstart scenario -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: ${CLUSTER_NAME} - namespace: ${NAMESPACE} -spec: - clusterNetwork: - pods: - cidrBlocks: - - 10.45.0.0/16 - services: - cidrBlocks: - - 10.46.0.0/16 - serviceDomain: cluster.local - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KThreesControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: DockerCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: DockerCluster -metadata: - name: ${CLUSTER_NAME} -spec: {} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KThreesControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: ${NAMESPACE} -spec: - infrastructureTemplate: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: DockerMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} - kthreesConfigSpec: - serverConfig: - tlsSan: - - 0.0.0.0 ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: DockerMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: ${NAMESPACE} -spec: - template: - spec: - customImage: kindest/node:${KIND_IMAGE_VERSION} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: worker-md-0 - namespace: ${NAMESPACE} -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: - cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} - - # This label will be needed for upgrade test - # it will be used as a selector for only selecting - # machines belonging to this machine deployment - cluster.x-k8s.io/deployment-name: worker-md-0 - template: - metadata: - labels: - cluster.x-k8s.io/deployment-name: worker-md-0 - e2e.remediation.label: "" - spec: - version: ${KUBERNETES_VERSION} - clusterName: ${CLUSTER_NAME} - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KThreesConfigTemplate - name: ${CLUSTER_NAME}-md-0 - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: DockerMachineTemplate - name: ${CLUSTER_NAME}-md-0 ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: DockerMachineTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: ${NAMESPACE} -spec: - template: - spec: - customImage: kindest/node:${KIND_IMAGE_VERSION} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KThreesConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: ${NAMESPACE} -spec: - template: - spec: ---- -# MachineHealthCheck object with -# - a selector that targets all the machines with label e2e.remediation.label="" -# - unhealthyConditions triggering remediation after 10s the condition is set -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineHealthCheck -metadata: - name: "${CLUSTER_NAME}-mhc-0" -spec: - clusterName: "${CLUSTER_NAME}" - maxUnhealthy: 100% - selector: - matchLabels: - e2e.remediation.label: "" - unhealthyConditions: - - type: e2e.remediation.condition - status: "False" - timeout: 10s diff --git a/test/e2e/data/infrastructure-docker/cluster-template-md-remediation/kustomization.yaml b/test/e2e/data/infrastructure-docker/cluster-template-md-remediation/kustomization.yaml new file mode 100644 index 00000000..6e28ccda --- /dev/null +++ b/test/e2e/data/infrastructure-docker/cluster-template-md-remediation/kustomization.yaml @@ -0,0 +1,7 @@ +resources: +- ../bases/cluster-with-kcp.yaml +- ../bases/md.yaml +- mhc.yaml + +patches: +- path: md.yaml diff --git a/test/e2e/data/infrastructure-docker/cluster-template-md-remediation/md.yaml b/test/e2e/data/infrastructure-docker/cluster-template-md-remediation/md.yaml new file mode 100644 index 00000000..e2b0e4c6 --- /dev/null +++ b/test/e2e/data/infrastructure-docker/cluster-template-md-remediation/md.yaml @@ -0,0 +1,10 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: ${NAMESPACE} +spec: + template: + metadata: + labels: + "e2e.remediation.label": "" diff --git a/test/e2e/data/infrastructure-docker/cluster-template-md-remediation/mhc.yaml b/test/e2e/data/infrastructure-docker/cluster-template-md-remediation/mhc.yaml new file mode 100644 index 00000000..c1072259 --- /dev/null +++ b/test/e2e/data/infrastructure-docker/cluster-template-md-remediation/mhc.yaml @@ -0,0 +1,18 @@ +--- +# MachineHealthCheck object with +# - a selector that targets all the machines with label e2e.remediation.label="" +# - unhealthyConditions triggering remediation after 10s the condition is set +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: "${CLUSTER_NAME}-mhc-0" +spec: + clusterName: "${CLUSTER_NAME}" + maxUnhealthy: 100% + selector: + matchLabels: + e2e.remediation.label: "" + unhealthyConditions: + - type: e2e.remediation.condition + status: "False" + timeout: 10s diff --git a/test/e2e/data/infrastructure-docker/cluster-template-topology/kustomization.yaml b/test/e2e/data/infrastructure-docker/cluster-template-topology/kustomization.yaml new file mode 100644 index 00000000..b1344c38 --- /dev/null +++ b/test/e2e/data/infrastructure-docker/cluster-template-topology/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- ../bases/cluster-with-topology.yaml \ No newline at end of file diff --git a/test/e2e/data/infrastructure-docker/cluster-template-v1beta1/kustomization.yaml b/test/e2e/data/infrastructure-docker/cluster-template-v1beta1/kustomization.yaml new file mode 100644 index 00000000..981c5c70 --- /dev/null +++ b/test/e2e/data/infrastructure-docker/cluster-template-v1beta1/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- ../bases/cluster-with-kcp-v1beta1.yaml +- ../bases/md-v1beta1.yaml \ No newline at end of file diff --git a/test/e2e/data/infrastructure-docker/cluster-template.yaml b/test/e2e/data/infrastructure-docker/cluster-template.yaml deleted file mode 100644 index bc554f32..00000000 --- a/test/e2e/data/infrastructure-docker/cluster-template.yaml +++ /dev/null @@ -1,110 +0,0 @@ -# TODO: copied and modified from https://github.com/k3s-io/cluster-api-k3s/pull/93/files#diff-c4a336ec56832a2ff7aed26c94d0d67ae3a0e6139d30701cc53c0f0962fe8cca -# should be the same as samples/docker/quickstart.yaml in the future -# for testing the quickstart scenario -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: ${CLUSTER_NAME} - namespace: ${NAMESPACE} -spec: - clusterNetwork: - pods: - cidrBlocks: - - 10.45.0.0/16 - services: - cidrBlocks: - - 10.46.0.0/16 - serviceDomain: cluster.local - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KThreesControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: DockerCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: DockerCluster -metadata: - name: ${CLUSTER_NAME} -spec: {} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KThreesControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: ${NAMESPACE} -spec: - infrastructureTemplate: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: DockerMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} - kthreesConfigSpec: - serverConfig: - tlsSan: - - 0.0.0.0 ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: DockerMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: ${NAMESPACE} -spec: - template: - spec: - customImage: kindest/node:${KIND_IMAGE_VERSION} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: worker-md-0 - namespace: ${NAMESPACE} -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: - cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} - - # This label will be needed for upgrade test - # it will be used as a selector for only selecting - # machines belonging to this machine deployment - cluster.x-k8s.io/deployment-name: worker-md-0 - template: - metadata: - labels: - cluster.x-k8s.io/deployment-name: worker-md-0 - spec: - version: ${KUBERNETES_VERSION} - clusterName: ${CLUSTER_NAME} - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KThreesConfigTemplate - name: ${CLUSTER_NAME}-md-0 - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: DockerMachineTemplate - name: ${CLUSTER_NAME}-md-0 ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: DockerMachineTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: ${NAMESPACE} -spec: - template: - spec: - customImage: kindest/node:${KIND_IMAGE_VERSION} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KThreesConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: ${NAMESPACE} -spec: - template: - spec: diff --git a/test/e2e/data/infrastructure-docker/cluster-template/kustomization.yaml b/test/e2e/data/infrastructure-docker/cluster-template/kustomization.yaml new file mode 100644 index 00000000..71e079ca --- /dev/null +++ b/test/e2e/data/infrastructure-docker/cluster-template/kustomization.yaml @@ -0,0 +1,4 @@ +# TODO: generated template should be the same as samples/docker/quickstart.yaml # in the future for testing the quickstart scenario +resources: +- ../bases/cluster-with-kcp.yaml +- ../bases/md.yaml \ No newline at end of file diff --git a/test/e2e/data/infrastructure-docker/clusterclass-k3s.yaml b/test/e2e/data/infrastructure-docker/clusterclass-k3s.yaml new file mode 100644 index 00000000..9616306e --- /dev/null +++ b/test/e2e/data/infrastructure-docker/clusterclass-k3s.yaml @@ -0,0 +1,157 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: k3s +spec: + controlPlane: + ref: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KThreesControlPlaneTemplate + name: k3s-control-plane + machineInfrastructure: + ref: + kind: DockerMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + name: k3s-control-plane + machineHealthCheck: + maxUnhealthy: 100% + unhealthyConditions: + - type: e2e.remediation.condition + status: "False" + timeout: 20s + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerClusterTemplate + name: k3s-cluster + workers: + machineDeployments: + - class: k3s-default-worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KThreesConfigTemplate + name: k3s-default-worker-bootstrap + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: k3s-default-worker-machinetemplate + machineHealthCheck: + maxUnhealthy: 100% + # We are intentionally not setting the 'unhealthyConditions' here to test that the field is optional. + machinePools: + - class: k3s-default-worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KThreesConfigTemplate + name: k3s-default-worker-bootstrap + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachinePoolTemplate + name: k3s-default-worker-machinepooltemplate + variables: + - name: kindImageVersion + required: true + schema: + openAPIV3Schema: + type: string + default: "v1.28.0" + patches: + - name: customImage + description: "Sets the container image that is used for running dockerMachines for the controlPlane and default-worker machineDeployments." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + matchResources: + machineDeploymentClass: + names: + - k3s-default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/customImage" + valueFrom: + template: | + kindest/node:{{ .kindImageVersion }} + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachinePoolTemplate + matchResources: + machinePoolClass: + names: + - k3s-default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/template/customImage" + valueFrom: + template: | + kindest/node:{{ .kindImageVersion }} + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/customImage" + valueFrom: + template: | + kindest/node:{{ .kindImageVersion }} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerClusterTemplate +metadata: + name: k3s-cluster +spec: + template: + spec: {} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: KThreesControlPlaneTemplate +metadata: + name: k3s-control-plane +spec: + template: + spec: + kthreesConfigSpec: + serverConfig: + tlsSan: + - 0.0.0.0 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: k3s-control-plane +spec: + template: + spec: {} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: KThreesConfigTemplate +metadata: + name: k3s-default-worker-bootstrap +spec: + template: + spec: {} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: k3s-default-worker-machinetemplate +spec: + template: + spec: {} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachinePoolTemplate +metadata: + name: k3s-default-worker-machinepooltemplate +spec: + template: + spec: + template: {} \ No newline at end of file diff --git a/test/e2e/data/shared/v1beta1/metadata.yaml b/test/e2e/data/shared/v1beta1/metadata.yaml index 92b9968a..e2d57ca9 100644 --- a/test/e2e/data/shared/v1beta1/metadata.yaml +++ b/test/e2e/data/shared/v1beta1/metadata.yaml @@ -5,5 +5,5 @@ apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 kind: Metadata releaseSeries: - major: 1 - minor: 6 + minor: 7 contract: v1beta1