Adding an E2E test that runs a full CAPL cluster test (default, k3s, & rke2) on merging to main

.github/workflows/build_test_ci.yml

@@ -74,17 +74,21 @@ jobs:
     needs:  [go-build-test, docker-build]
     runs-on: ubuntu-latest
     if: github.event.pull_request.draft == false
+    env:
+      GITHUB_TOKEN: ${{ secrets.github_token }}
+      LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }}
     steps:
     - name: Harden Runner
       uses: step-security/harden-runner@v2
       with:
         disable-sudo: true
-        egress-policy: block
+        egress-policy: audit
         allowed-endpoints: >
           api.linode.com:443
           api.github.com:443
           github.com:443
           gcr.io:443
+          ghcr.io:443
           proxy.golang.org:443
           sum.golang.org:443
           *.githubusercontent.com:443
@@ -117,11 +121,30 @@ jobs:
       with:
         key: docker-${{ runner.os }}-${{ hashFiles('go.sum') }}
 
-    - name: E2E test
+    - name: Complete E2E Test
+      if: github.ref == 'refs/heads/main' 
+      run: make e2etest
+      env:
+        INSTALL_K3S_PROVIDER: true
+        INSTALL_RKE2_PROVIDER: true
+        LINODE_REGION: us-sea
+        LINODE_CONTROL_PLANE_MACHINE_TYPE: g6-standard-2
+        LINODE_MACHINE_TYPE: g6-standard-2
+        CLUSTERCTL_CONFIG: /home/runner/work/cluster-api-provider-linode/cluster-api-provider-linode/e2e/clusterctl-config.yaml
+
+
+    - name: Quick E2E Test
       run: make e2etest
       env:
-        GITHUB_TOKEN: ${{ secrets.github_token }}
-        LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }}
+        # TODO: Switch selector to quick before the PR can be merged
+        E2E_FLAGS: '--assert-timeout 15m0s --selector rke2'
+        # TODO: ALL the var below need to be deleted before the PR can be merged
+        INSTALL_K3S_PROVIDER: true
+        INSTALL_RKE2_PROVIDER: true
+        LINODE_REGION: us-sea
+        LINODE_CONTROL_PLANE_MACHINE_TYPE: g6-standard-2
+        LINODE_MACHINE_TYPE: g6-standard-2
+        CLUSTERCTL_CONFIG: /home/runner/work/cluster-api-provider-linode/cluster-api-provider-linode/e2e/clusterctl-config.yaml
 
     - name: Copy logs
       if: ${{ always() }}

Makefile

@@ -146,8 +146,8 @@ test: generate fmt vet envtest ## Run tests.
 	rm cover.out.tmp
 
 .PHONY: e2etest
-e2etest: generate local-deploy chainsaw
-	GIT_REF=$(GIT_REF) $(CHAINSAW) test ./e2e
+e2etest: generate local-release local-deploy chainsaw
+	GIT_REF=$(GIT_REF) $(CHAINSAW) test ./e2e $(E2E_FLAGS)
 
 local-deploy: kind ctlptl tilt kustomize clusterctl
 	@echo -n "LINODE_TOKEN=$(LINODE_TOKEN)" > config/default/.env.linode

e2e/clusterctl-config.yaml

@@ -0,0 +1,16 @@
+providers:
+  - name: local-linode
+    url: /home/runner/work/cluster-api-provider-linode/cluster-api-provider-linode/infrastructure-local-linode/v0.0.0/infrastructure-components.yaml
+    type: InfrastructureProvider
+  - name: "k3s"
+    url: https://github.com/k3s-io/cluster-api-k3s/releases/latest/bootstrap-components.yaml
+    type: "BootstrapProvider"
+  - name: "k3s"
+    url: https://github.com/k3s-io/cluster-api-k3s/releases/latest/control-plane-components.yaml
+    type: "ControlPlaneProvider"
+  - name: "rke2"
+    url: "https://github.com/rancher-sandbox/cluster-api-provider-rke2/releases/latest/bootstrap-components.yaml"
+    type: "BootstrapProvider"
+  - name: "rke2"
+    url: "https://github.com/rancher-sandbox/cluster-api-provider-rke2/releases/latest/control-plane-components.yaml"
+    type: "ControlPlaneProvider"

e2e/default-CAPL-cluster/assert-capi-resources.yaml

@@ -0,0 +1,39 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: capi-controller-manager
+  namespace: capi-system
+status:
+  availableReplicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: capl-controller-manager
+  namespace: capl-system
+status:
+  availableReplicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: capi-kubeadm-bootstrap-controller-manager
+  namespace: kubeadm-bootstrap-system
+status:
+  availableReplicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: capi-kubeadm-control-plane-controller-manager
+  namespace: kubeadm-control-plane-system
+status:
+  availableReplicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: caaph-controller-manager
+  namespace: caaph-system
+status:
+  availableReplicas: 1

e2e/default-CAPL-cluster/assert-child-cluster-resources.yaml

@@ -0,0 +1,94 @@
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: LinodeMachine
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ($cluster)
+spec:
+  region: (env('LINODE_REGION'))
+  type: g6-standard-2
+status:
+  ready: true
+  instanceState: running
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Machine
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ($cluster)
+spec:
+  clusterName: ($cluster)
+status:
+  bootstrapReady: true
+  infrastructureReady: true
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ($cluster)
+spec:
+  clusterName: ($cluster)
+  replicas: 1
+status:
+  readyReplicas: 1
+  unavailableReplicas: 0
+  availableReplicas: 1
+---
+apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+kind: KubeadmControlPlane
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ($cluster)
+status:
+  readyReplicas: 1
+  unavailableReplicas: 0
+  ready: true
+---
+apiVersion: addons.cluster.x-k8s.io/v1alpha1
+kind: HelmReleaseProxy
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ($cluster)
+    helmreleaseproxy.addons.cluster.x-k8s.io/helmchartproxy-name: linode-cloud-controller-manager
+status:
+  conditions:
+  - type: Ready
+    status: "True"
+  - type: ClusterAvailable
+    status: "True"
+  - type: HelmReleaseReady
+    status: "True"
+  status: deployed
+---
+apiVersion: addons.cluster.x-k8s.io/v1alpha1
+kind: HelmReleaseProxy
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ($cluster)
+    helmreleaseproxy.addons.cluster.x-k8s.io/helmchartproxy-name: cilium
+status:
+  conditions:
+  - type: Ready
+    status: "True"
+  - type: ClusterAvailable
+    status: "True"
+  - type: HelmReleaseReady
+    status: "True"
+  status: deployed
+---
+apiVersion: addons.cluster.x-k8s.io/v1alpha1
+kind: HelmReleaseProxy
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ($cluster)
+    helmreleaseproxy.addons.cluster.x-k8s.io/helmchartproxy-name: csi-driver-linode
+status:
+  conditions:
+  - type: Ready
+    status: "True"
+  - type: ClusterAvailable
+    status: "True"
+  - type: HelmReleaseReady
+    status: "True"
+  status: deployed

e2e/default-CAPL-cluster/chainsaw-test.yaml

@@ -0,0 +1,122 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: default-capl-cluster
+  # Labels to allow the test to be triggered based on selector flag
+  labels:
+    default-cluster:
+    flavors:
+spec:
+  bindings:
+    # A short identifier for the E2E test run
+  - name: run
+    value: (join('-', ['e2e', 'default-cluster', env('GIT_REF')]))
+  - name: cluster
+    # Format the cluster name
+    value: (trim((truncate(($run), `32`)), '-'))
+  template: true
+  steps:
+  - name: Check if CAPI provider resources exist
+    try:
+    - assert:
+        file: assert-capi-resources.yaml
+  - name: Generate cluster using clusterctl
+    try:
+    - script:
+        env:
+        - name: CLUSTER
+          value: ($cluster)
+        - name: NAMESPACE
+          value: ($namespace)
+        - name: CLUSTERCTL_CONFIG
+          value: (env('CLUSTERCTL_CONFIG'))
+        content: |
+          set -e
+          clusterctl generate cluster $CLUSTER -n $NAMESPACE \
+          --kubernetes-version v1.29.1 \
+          --infrastructure local-linode:v0.0.0 \
+          --control-plane-machine-count 1 --worker-machine-count 1 \
+          --config $CLUSTERCTL_CONFIG > default-cluster.yaml
+        check:
+          ($error == null): true
+  - name: Apply generated cluster yaml
+    try:
+    - apply:
+        file: default-cluster.yaml
+    - assert:
+        file: assert-child-cluster-resources.yaml
+    catch:
+    - events:
+        namespace: ($namespace)
+    - get:
+        apiVersion: cluster.x-k8s.io/v1beta1
+        kind: MachineDeployment
+  - name: Check if the linodes are created
+    try:
+    - script:
+        env:
+        - name: TARGET_API
+          value: api.linode.com
+        - name: TARGET_API_VERSION
+          value: v4beta
+        - name: URI
+          value: linode/instances
+        - name: FILTER
+          value: (to_string({"tags":($cluster)}))
+        content: |
+          set -e
+          curl -s \
+            -H "Authorization: Bearer $LINODE_TOKEN" \
+            -H "X-Filter: $FILTER" \
+            -H "Content-Type: application/json" \
+            "https://$TARGET_API/$TARGET_API_VERSION/$URI"
+        check:
+          ($error): ~
+          (json_parse($stdout)):
+            results: 2
+  - name: Delete child cluster
+    try:
+    - delete:
+        ref:
+          apiVersion: cluster.x-k8s.io/v1beta1
+          kind: Cluster
+          name: ($cluster)
+    - delete:
+        ref:
+          apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+          kind: LinodeVPC
+          name: ($cluster)
+    - error:
+        file: check-child-cluster-and-vpc-deleted.yaml
+  - name: Check if the linodes are deleted
+    try:
+    - script:
+        env:
+        - name: TARGET_API
+          value: api.linode.com
+        - name: TARGET_API_VERSION
+          value: v4beta
+        - name: URI
+          value: linode/instances
+        - name: FILTER
+          value: (to_string({"tags":($cluster)}))
+        content: |
+          set -e
+          curl -s \
+            -H "Authorization: Bearer $LINODE_TOKEN" \
+            -H "X-Filter: $FILTER" \
+            -H "Content-Type: application/json" \
+            "https://$TARGET_API/$TARGET_API_VERSION/$URI"
+        check:
+          ($error): ~
+          (json_parse($stdout)):
+            results: 0
+  - name: Delete generated child cluster manifest yaml
+    try:
+    - script:
+        content: |
+          rm -f default-cluster.yaml
+        check:
+          ($error == null): true

e2e/default-CAPL-cluster/check-child-cluster-and-vpc-deleted.yaml

@@ -0,0 +1,10 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: LinodeMachine
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ($cluster)
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: LinodeVPC
+metadata:
+  name: ($cluster)

e2e/k3s-capl-cluster/assert-capi-resources.yaml

@@ -0,0 +1,31 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: capi-controller-manager
+  namespace: capi-system
+status:
+  availableReplicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: capl-controller-manager
+  namespace: capl-system
+status:
+  availableReplicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: capi-k3s-bootstrap-controller-manager
+  namespace: capi-k3s-bootstrap-system
+status:
+  availableReplicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: capi-k3s-control-plane-controller-manager
+  namespace: capi-k3s-control-plane-system
+status:
+  availableReplicas: 1