Adding an E2E test that runs a full CAPL cluster test (default, k3s, …

…& rke2) on merging to main (#266) * adeodhar: add try catch block --------- Co-authored-by: Khaja Omer <[email protected]> Co-authored-by: Amol Deodhar <[email protected]>
linode · May 1, 2024 · 3bc1f07 · 3bc1f07
1 parent 8a3977d
commit 3bc1f07
Show file tree

Hide file tree

Showing 53 changed files with 1,302 additions and 477 deletions.
diff --git a/.github/workflows/build_test_ci.yml b/.github/workflows/build_test_ci.yml
@@ -74,6 +74,9 @@ jobs:
     needs:  [go-build-test, docker-build]
     runs-on: ubuntu-latest
     if: github.event.pull_request.draft == false
+    env:
+      GITHUB_TOKEN: ${{ secrets.github_token }}
+      LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }}
     steps:
     - name: Harden Runner
       uses: step-security/harden-runner@v2
@@ -85,6 +88,7 @@ jobs:
           api.github.com:443
           github.com:443
           gcr.io:443
+          ghcr.io:443
           proxy.golang.org:443
           sum.golang.org:443
           *.githubusercontent.com:443
@@ -103,6 +107,8 @@ jobs:
           cloud.tilt.dev:443
           kubernetes-sigs.github.io:443
           charts.jetstack.io:443
+          helm.cilium.io:443
+          linode.github.io:443
 
     - uses: actions/checkout@v4
 
@@ -117,11 +123,24 @@ jobs:
       with:
         key: docker-${{ runner.os }}-${{ hashFiles('go.sum') }}
 
-    - name: E2E test
+    - name: Complete E2E Test
+      if: github.ref == 'refs/heads/main' 
       run: make e2etest
       env:
-        GITHUB_TOKEN: ${{ secrets.github_token }}
-        LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }}
+        E2E_FLAGS: '--assert-timeout 15m0s'
+        INSTALL_K3S_PROVIDER: true
+        INSTALL_RKE2_PROVIDER: true
+        LINODE_REGION: us-sea
+        LINODE_CONTROL_PLANE_MACHINE_TYPE: g6-standard-2
+        LINODE_MACHINE_TYPE: g6-standard-2
+        CLUSTERCTL_CONFIG: /home/runner/work/cluster-api-provider-linode/cluster-api-provider-linode/e2e/gha-clusterctl-config.yaml
+
+
+    - name: Quick E2E Test
+      if: github.ref != 'refs/heads/main'
+      run: make e2etest
+      env:
+        E2E_FLAGS: '--selector quick'
 
     - name: Copy logs
       if: ${{ always() }}

diff --git a/Makefile b/Makefile
@@ -146,8 +146,8 @@ test: generate fmt vet envtest ## Run tests.
 	rm cover.out.tmp
 
 .PHONY: e2etest
-e2etest: generate local-deploy chainsaw
-	GIT_REF=$(GIT_REF) $(CHAINSAW) test ./e2e
+e2etest: generate local-release local-deploy chainsaw
+	GIT_REF=$(GIT_REF) $(CHAINSAW) test ./e2e $(E2E_FLAGS)
 
 local-deploy: kind ctlptl tilt kustomize clusterctl
 	@echo -n "LINODE_TOKEN=$(LINODE_TOKEN)" > config/default/.env.linode

diff --git a/docs/src/developers/testing.md b/docs/src/developers/testing.md
@@ -177,14 +177,51 @@ var _ = Describe("linode creation", func() {
 For e2e tests CAPL uses the [Chainsaw project](https://kyverno.github.io/chainsaw) which leverages `kind` and `tilt` to 
 spin up a cluster with the CAPL controllers installed and then uses `chainsaw-test.yaml` files to drive e2e testing.
 
-All test live in the e2e folder with a directory structure of `e2e/${CONTROLLER_NAME}/${TEST_NAME}`
+All test live in the e2e folder with a directory structure of `e2e/${COMPONENT}/${TEST_NAME}`
 ### Running tests
-In order to run e2e tests run the following command
+In order to run e2e tests run the following commands: 
 ```bash
+# Required env vars to run e2e tests
+export INSTALL_K3S_PROVIDER=true
+export INSTALL_RKE2_PROVIDER=true
+export LINODE_REGION=us-sea
+export LINODE_CONTROL_PLANE_MACHINE_TYPE=g6-standard-2
+export LINODE_MACHINE_TYPE=g6-standard-2
+
+# IMPORTANT: Set linode, k3s, and rke2 providers in this config file.
+# Find an example at e2e/gha-clusterctl-config.yaml
+export CLUSTERCTL_CONFIG=~/.cluster-api/clusterctl.yaml
+
 make e2etest
 ```
+*Note: By default `make e2etest` runs all the e2e tests defined under `/e2e` dir*
+
+In order to run specific test, you need to pass flags to chainsaw by setting env var `E2E_FLAGS`
+
+Example: Only running e2e tests for flavors *(default, k3s, rke2)*
+```bash
+make e2etest E2E_FLAGS='--selector flavors --assert-timeout 10m0s'
+```
+*Note: We need to bump up the assert timeout to 10 mins to allow the cluster to complete building and become available*
+
+There are other selectors you can use to invoke specfic tests. Please look at the table below for all the selectors available:
+
+| Tests          | Selector     |
+| ------------- | ------------- |
+| All Controllers | `quick`     |
+| All Flavors (default, k3s, rke2) | `flavors`   |
+| K3S Cluster | `k3s` | 
+| RKE2 Cluster | `rke2` |
+| Default (kubeadm) Cluster | `default-cluster` |
+| Linode Cluster Controller | `linodecluster` |
+| Linode Machine Controller | `linodemachine` |
+| Linode Obj Controller | `linodeobj` | 
+| Linode VPC Controller | `linodevpc` | 
+
+*Note: For any flavor e2e tests, please set the required env variables*
+
 ### Adding tests
-1. Create a new directory under the controller you are testing with the naming scheme of `e2e/${CONTROLLER_NAME}/${TEST_NAME}`
+1. Create a new directory under the controller you are testing with the naming scheme of `e2e/${COMPONENT}/${TEST_NAME}`
 2. Create a minimal `chainsaw-test.yaml` file in the new test dir
     ```yaml
    # yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json

diff --git a/e2e/capl-cluster-flavors/default-CAPL-cluster/assert-capi-resources.yaml b/e2e/capl-cluster-flavors/default-CAPL-cluster/assert-capi-resources.yaml
@@ -0,0 +1,39 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: capi-controller-manager
+  namespace: capi-system
+status:
+  availableReplicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: capl-controller-manager
+  namespace: capl-system
+status:
+  availableReplicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: capi-kubeadm-bootstrap-controller-manager
+  namespace: kubeadm-bootstrap-system
+status:
+  availableReplicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: capi-kubeadm-control-plane-controller-manager
+  namespace: kubeadm-control-plane-system
+status:
+  availableReplicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: caaph-controller-manager
+  namespace: caaph-system
+status:
+  availableReplicas: 1
diff --git a/e2e/capl-cluster-flavors/default-CAPL-cluster/assert-child-cluster-resources.yaml b/e2e/capl-cluster-flavors/default-CAPL-cluster/assert-child-cluster-resources.yaml
@@ -0,0 +1,61 @@
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: LinodeMachine
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ($cluster)
+spec:
+  region: (env('LINODE_REGION'))
+  type: g6-standard-2
+status:
+  ready: true
+  instanceState: running
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Machine
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ($cluster)
+spec:
+  clusterName: ($cluster)
+status:
+  bootstrapReady: true
+  infrastructureReady: true
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ($cluster)
+spec:
+  clusterName: ($cluster)
+  replicas: 1
+status:
+  readyReplicas: 1
+  unavailableReplicas: 0
+  availableReplicas: 1
+---
+apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+kind: KubeadmControlPlane
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ($cluster)
+status:
+  readyReplicas: 1
+  unavailableReplicas: 0
+  ready: true
+---
+apiVersion: addons.cluster.x-k8s.io/v1alpha1
+kind: HelmReleaseProxy
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ($cluster)
+status:
+  conditions:
+  - type: Ready
+    status: "True"
+  - type: ClusterAvailable
+    status: "True"
+  - type: HelmReleaseReady
+    status: "True"
+  status: deployed
diff --git a/e2e/capl-cluster-flavors/default-CAPL-cluster/chainsaw-test.yaml b/e2e/capl-cluster-flavors/default-CAPL-cluster/chainsaw-test.yaml
@@ -0,0 +1,132 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: default-capl-cluster
+  # Labels to allow the test to be triggered based on selector flag
+  labels:
+    default-cluster:
+    flavors:
+spec:
+  bindings:
+    # A short identifier for the E2E test run
+    - name: run
+      value: (join('-', ['e2e', 'default-cluster', env('GIT_REF')]))
+    - name: cluster
+      # Format the cluster name
+      value: (trim((truncate(($run), `32`)), '-'))
+  template: true
+  steps:
+    - name: Check if CAPI provider resources exist
+      try:
+        - assert:
+            file: assert-capi-resources.yaml
+    - name: Generate cluster using clusterctl
+      try:
+        - script:
+            env:
+              - name: CLUSTER
+                value: ($cluster)
+              - name: NAMESPACE
+                value: ($namespace)
+              - name: CLUSTERCTL_CONFIG
+                value: (env('CLUSTERCTL_CONFIG'))
+            content: |
+              set -e
+              clusterctl generate cluster $CLUSTER -n $NAMESPACE \
+              --kubernetes-version v1.29.1 \
+              --infrastructure local-linode:v0.0.0 \
+              --control-plane-machine-count 1 --worker-machine-count 1 \
+              --config $CLUSTERCTL_CONFIG > default-cluster.yaml
+            check:
+              ($error == null): true
+    - name: Apply generated cluster yaml
+      try:
+        - apply:
+            file: default-cluster.yaml
+        - assert:
+            file: assert-child-cluster-resources.yaml
+      catch:
+        - describe:
+            apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+            kind: LinodeMachine
+        - describe:
+            apiVersion: cluster.x-k8s.io/v1beta1
+            kind: Machine
+        - describe:
+            apiVersion: cluster.x-k8s.io/v1beta1
+            kind: MachineDeployment
+        - describe:
+            apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+            kind: KubeadmControlPlane
+        - describe:
+            apiVersion: addons.cluster.x-k8s.io/v1alpha1
+            kind: HelmReleaseProxy
+    - name: Check if the linodes are created
+      try:
+        - script:
+            env:
+              - name: TARGET_API
+                value: api.linode.com
+              - name: TARGET_API_VERSION
+                value: v4beta
+              - name: URI
+                value: linode/instances
+              - name: FILTER
+                value: (to_string({"tags":($cluster)}))
+            content: |
+              set -e
+              curl -s \
+                -H "Authorization: Bearer $LINODE_TOKEN" \
+                -H "X-Filter: $FILTER" \
+                -H "Content-Type: application/json" \
+                "https://$TARGET_API/$TARGET_API_VERSION/$URI"
+            check:
+              ($error): ~
+              (json_parse($stdout)):
+                results: 2
+    - name: Delete child cluster
+      try:
+        - delete:
+            ref:
+              apiVersion: cluster.x-k8s.io/v1beta1
+              kind: Cluster
+              name: ($cluster)
+        - delete:
+            ref:
+              apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+              kind: LinodeVPC
+              name: ($cluster)
+        - error:
+            file: check-child-cluster-and-vpc-deleted.yaml
+    - name: Check if the linodes are deleted
+      try:
+        - script:
+            env:
+              - name: TARGET_API
+                value: api.linode.com
+              - name: TARGET_API_VERSION
+                value: v4beta
+              - name: URI
+                value: linode/instances
+              - name: FILTER
+                value: (to_string({"tags":($cluster)}))
+            content: |
+              set -e
+              curl -s \
+                -H "Authorization: Bearer $LINODE_TOKEN" \
+                -H "X-Filter: $FILTER" \
+                -H "Content-Type: application/json" \
+                "https://$TARGET_API/$TARGET_API_VERSION/$URI"
+            check:
+              ($error): ~
+              (json_parse($stdout)):
+                results: 0
+    - name: Delete generated child cluster manifest yaml
+      try:
+        - script:
+            content: |
+              rm -f default-cluster.yaml
+            check:
+              ($error == null): true
diff --git a/e2e/capl-cluster-flavors/default-CAPL-cluster/check-child-cluster-and-vpc-deleted.yaml b/e2e/capl-cluster-flavors/default-CAPL-cluster/check-child-cluster-and-vpc-deleted.yaml
@@ -0,0 +1,10 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: LinodeMachine
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ($cluster)
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: LinodeVPC
+metadata:
+  name: ($cluster)
diff --git a/e2e/capl-cluster-flavors/k3s-capl-cluster/assert-capi-resources.yaml b/e2e/capl-cluster-flavors/k3s-capl-cluster/assert-capi-resources.yaml
@@ -0,0 +1,31 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: capi-controller-manager
+  namespace: capi-system
+status:
+  availableReplicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: capl-controller-manager
+  namespace: capl-system
+status:
+  availableReplicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: capi-k3s-bootstrap-controller-manager
+  namespace: capi-k3s-bootstrap-system
+status:
+  availableReplicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: capi-k3s-control-plane-controller-manager
+  namespace: capi-k3s-control-plane-system
+status:
+  availableReplicas: 1