Introduce flags for speeding up test execution with "kind"

.github/workflows/ci.yaml

@@ -23,8 +23,6 @@ jobs:
     - shell: bash
       run: |
         make test
-      env:
-        HUMIO_E2E_LICENSE: ${{ secrets.HUMIO_E2E_LICENSE }}
     - name: Publish Test Report
       uses: mikepenz/action-junit-report@v4
       if: always() # always run even if the previous step fails

.github/workflows/e2e-dummy.yaml

@@ -39,7 +39,6 @@ jobs:
     - name: run e2e tests
       env:
         BIN_DIR: ${{ steps.bin_dir.outputs.BIN_DIR }}
-        HUMIO_E2E_LICENSE: ${{ secrets.HUMIO_E2E_LICENSE }}
         E2E_KIND_K8S_VERSION: ${{ matrix.kind-k8s-version }}
         E2E_LOGS_HUMIO_HOSTNAME: ${{ secrets.E2E_LOGS_HUMIO_HOSTNAME }}
         E2E_LOGS_HUMIO_INGEST_TOKEN: ${{ secrets.E2E_LOGS_HUMIO_INGEST_TOKEN }}

Makefile

@@ -49,9 +49,6 @@ vet: ## Run go vet against code.
 	go vet ./...
 
 test: manifests generate fmt vet ginkgo ## Run tests.
-ifndef HUMIO_E2E_LICENSE
-	$(error HUMIO_E2E_LICENSE not set)
-endif
 	go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
 	$(SHELL) -c "\
 		eval \$$($(GOBIN)/setup-envtest use -p env ${TEST_K8S_VERSION}); \

controllers/humiocluster_defaults.go

@@ -561,7 +561,7 @@ func (hnp *HumioNodePool) GetContainerReadinessProbe() *corev1.Probe {
 	}
 
 	if hnp.humioNodeSpec.ContainerReadinessProbe == nil {
-		return &corev1.Probe{
+		probe := &corev1.Probe{
 			ProbeHandler: corev1.ProbeHandler{
 				HTTPGet: &corev1.HTTPGetAction{
 					Path:   "/api/v1/is-node-up",
@@ -575,6 +575,10 @@ func (hnp *HumioNodePool) GetContainerReadinessProbe() *corev1.Probe {
 			SuccessThreshold:    1,
 			FailureThreshold:    10,
 		}
+		if os.Getenv("DUMMY_LOGSCALE_IMAGE") == "true" {
+			probe.InitialDelaySeconds = 0
+		}
+		return probe
 	}
 	return hnp.humioNodeSpec.ContainerReadinessProbe
 }

controllers/humiocluster_pods.go

@@ -452,22 +452,6 @@ func ConstructPod(hnp *HumioNodePool, humioNodeName string, attachments *podAtta
 				},
 			},
 		})
-		pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{
-			Name: "ca-cert",
-			VolumeSource: corev1.VolumeSource{
-				Secret: &corev1.SecretVolumeSource{
-					SecretName:  hnp.GetClusterName(),
-					DefaultMode: &mode,
-					Items: []corev1.KeyToPath{
-						{
-							Key:  "ca.crt",
-							Path: "certs/ca-bundle.crt",
-							Mode: &mode,
-						},
-					},
-				},
-			},
-		})
 	}
 
 	if attachments.bootstrapTokenSecretReference.hash != "" {

controllers/suite/clusters/humiocluster_controller_test.go

@@ -278,7 +278,7 @@ var _ = Describe("HumioCluster Controller", func() {
 				Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2"))
 			}
 
-			if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
+			if helpers.TLSEnabled(&updatedHumioCluster) {
 				suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
 				Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
 			}
@@ -462,7 +462,7 @@ var _ = Describe("HumioCluster Controller", func() {
 				Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2"))
 			}
 
-			if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
+			if helpers.TLSEnabled(&updatedHumioCluster) {
 				suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
 				Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
 			}
@@ -556,7 +556,7 @@ var _ = Describe("HumioCluster Controller", func() {
 				Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2"))
 			}
 
-			if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
+			if helpers.TLSEnabled(&updatedHumioCluster) {
 				suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
 				Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
 			}
@@ -631,7 +631,7 @@ var _ = Describe("HumioCluster Controller", func() {
 				Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2"))
 			}
 
-			if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
+			if helpers.TLSEnabled(&updatedHumioCluster) {
 				suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
 				Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
 			}
@@ -707,16 +707,16 @@ var _ = Describe("HumioCluster Controller", func() {
 				Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2"))
 			}
 
-			if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
+			if helpers.TLSEnabled(&updatedHumioCluster) {
 				suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
 				Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
 			}
 		})
 	})
 
-	Context("Humio Cluster Update EXTERNAL_URL", Label("envtest", "dummy", "real"), func() {
+	Context("Humio Cluster Update EXTERNAL_URL", Label("dummy", "real"), func() {
 		It("Update should correctly replace pods to use the new EXTERNAL_URL in a non-rolling fashion", func() {
-			if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
+			if helpers.UseCertManager() {
 				key := types.NamespacedName{
 					Name:      "humiocluster-update-ext-url",
 					Namespace: testProcessNamespace,
@@ -957,7 +957,7 @@ var _ = Describe("HumioCluster Controller", func() {
 				Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2"))
 			}
 
-			if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
+			if helpers.TLSEnabled(&updatedHumioCluster) {
 				suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
 				Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
 			}
@@ -1063,7 +1063,7 @@ var _ = Describe("HumioCluster Controller", func() {
 				Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2"))
 			}
 
-			if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
+			if helpers.TLSEnabled(&updatedHumioCluster) {
 				suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
 				Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
 			}
@@ -1177,7 +1177,7 @@ var _ = Describe("HumioCluster Controller", func() {
 				Expect(pod.Annotations[controllers.PodRevisionAnnotation]).To(Equal("3"))
 			}
 
-			if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
+			if helpers.TLSEnabled(&updatedHumioCluster) {
 				suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
 				Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
 			}
@@ -1242,7 +1242,7 @@ var _ = Describe("HumioCluster Controller", func() {
 
 			updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels())
 
-			if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
+			if helpers.TLSEnabled(&updatedHumioCluster) {
 				suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
 				Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
 			}
@@ -1314,7 +1314,7 @@ var _ = Describe("HumioCluster Controller", func() {
 
 			updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels())
 
-			if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
+			if helpers.TLSEnabled(&updatedHumioCluster) {
 				suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
 				Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
 			}
@@ -1444,7 +1444,7 @@ var _ = Describe("HumioCluster Controller", func() {
 			}, testTimeout, suite.TestInterval).Should(BeTrue())
 
 			updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, updatedHumioCluster.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetPodLabels())
-			if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
+			if helpers.TLSEnabled(&updatedHumioCluster) {
 				suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
 				Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
 			}
@@ -1666,7 +1666,7 @@ var _ = Describe("HumioCluster Controller", func() {
 			}, testTimeout, suite.TestInterval).Should(BeTrue())
 
 			updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, updatedHumioCluster.Namespace, mainNodePoolManager.GetPodLabels())
-			if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
+			if helpers.TLSEnabled(&updatedHumioCluster) {
 				suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
 				Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
 			}
@@ -1767,7 +1767,7 @@ var _ = Describe("HumioCluster Controller", func() {
 			}, testTimeout, suite.TestInterval).Should(BeTrue())
 
 			updatedClusterPods, _ = kubernetes.ListPods(ctx, k8sClient, updatedHumioCluster.Namespace, additionalNodePoolManager.GetPodLabels())
-			if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
+			if helpers.TLSEnabled(&updatedHumioCluster) {
 				suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
 				Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
 			}
@@ -3337,20 +3337,24 @@ var _ = Describe("HumioCluster Controller", func() {
 			defer suite.CleanupCluster(ctx, k8sClient, toCreate)
 
 			initialExpectedVolumesCount := 5
-			initialExpectedVolumeMountsCount := 4
+			initialExpectedHumioContainerVolumeMountsCount := 4
 
 			if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
-				// if we run on a real cluster we have TLS enabled (using 2 volumes),
-				// and k8s will automatically inject a service account token adding one more
-				initialExpectedVolumesCount += 3
-				initialExpectedVolumeMountsCount += 2
+				// k8s will automatically inject a service account token
+				initialExpectedVolumesCount += 1                    // kube-api-access-<ID>
+				initialExpectedHumioContainerVolumeMountsCount += 1 // kube-api-access-<ID>
+
+				if helpers.UseCertManager() {
+					initialExpectedVolumesCount += 1                    // tls-cert
+					initialExpectedHumioContainerVolumeMountsCount += 1 // tls-cert
+				}
 			}
 
 			clusterPods, _ := kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels())
 			for _, pod := range clusterPods {
 				Expect(pod.Spec.Volumes).To(HaveLen(initialExpectedVolumesCount))
 				humioIdx, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName)
-				Expect(pod.Spec.Containers[humioIdx].VolumeMounts).To(HaveLen(initialExpectedVolumeMountsCount))
+				Expect(pod.Spec.Containers[humioIdx].VolumeMounts).To(HaveLen(initialExpectedHumioContainerVolumeMountsCount))
 			}
 
 			suite.UsingClusterBy(key.Name, "Adding additional volumes")
@@ -3395,7 +3399,7 @@ var _ = Describe("HumioCluster Controller", func() {
 					return pod.Spec.Containers[humioIdx].VolumeMounts
 				}
 				return []corev1.VolumeMount{}
-			}, testTimeout, suite.TestInterval).Should(HaveLen(initialExpectedVolumeMountsCount + 1))
+			}, testTimeout, suite.TestInterval).Should(HaveLen(initialExpectedHumioContainerVolumeMountsCount + 1))
 			clusterPods, _ = kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels())
 			for _, pod := range clusterPods {
 				Expect(pod.Spec.Volumes).Should(ContainElement(extraVolume))
@@ -3416,7 +3420,7 @@ var _ = Describe("HumioCluster Controller", func() {
 				Type: humiov1alpha1.HumioClusterUpdateStrategyRollingUpdate,
 			}
 			protocol := "http"
-			if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
+			if helpers.TLSEnabled(toCreate) {
 				protocol = "https"
 			}
 
@@ -3815,39 +3819,40 @@ var _ = Describe("HumioCluster Controller", func() {
 		})
 	})
 
-	Context("Humio Cluster with additional hostnames for TLS", Label("envtest", "dummy", "real"), func() {
+	Context("Humio Cluster with additional hostnames for TLS", Label("dummy", "real"), func() {
 		It("Creating cluster with additional hostnames for TLS", func() {
-			if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
-				key := types.NamespacedName{
-					Name:      "humiocluster-tls-additional-hostnames",
-					Namespace: testProcessNamespace,
-				}
-				toCreate := suite.ConstructBasicSingleNodeHumioCluster(key, true)
-				toCreate.Spec.TLS = &humiov1alpha1.HumioClusterTLSSpec{
-					Enabled: helpers.BoolPtr(true),
-					ExtraHostnames: []string{
-						"something.additional",
-						"yet.another.something.additional",
-					},
-				}
+			key := types.NamespacedName{
+				Name:      "humiocluster-tls-additional-hostnames",
+				Namespace: testProcessNamespace,
+			}
+			toCreate := suite.ConstructBasicSingleNodeHumioCluster(key, true)
+			if !helpers.TLSEnabled(toCreate) {
+				return
+			}
+			toCreate.Spec.TLS = &humiov1alpha1.HumioClusterTLSSpec{
+				Enabled: helpers.BoolPtr(true),
+				ExtraHostnames: []string{
+					"something.additional",
+					"yet.another.something.additional",
+				},
+			}
 
-				suite.UsingClusterBy(key.Name, "Creating the cluster successfully")
-				ctx := context.Background()
-				suite.CreateAndBootstrapCluster(ctx, k8sClient, testHumioClient, toCreate, true, humiov1alpha1.HumioClusterStateRunning, testTimeout)
-				defer suite.CleanupCluster(ctx, k8sClient, toCreate)
+			suite.UsingClusterBy(key.Name, "Creating the cluster successfully")
+			ctx := context.Background()
+			suite.CreateAndBootstrapCluster(ctx, k8sClient, testHumioClient, toCreate, true, humiov1alpha1.HumioClusterStateRunning, testTimeout)
+			defer suite.CleanupCluster(ctx, k8sClient, toCreate)
 
-				suite.UsingClusterBy(key.Name, "Confirming certificate objects contain the additional hostnames")
+			suite.UsingClusterBy(key.Name, "Confirming certificate objects contain the additional hostnames")
 
-				Eventually(func() ([]cmapi.Certificate, error) {
-					return kubernetes.ListCertificates(ctx, k8sClient, toCreate.Namespace, kubernetes.MatchingLabelsForHumio(toCreate.Name))
-				}, testTimeout, suite.TestInterval).Should(HaveLen(2))
+			Eventually(func() ([]cmapi.Certificate, error) {
+				return kubernetes.ListCertificates(ctx, k8sClient, toCreate.Namespace, kubernetes.MatchingLabelsForHumio(toCreate.Name))
+			}, testTimeout, suite.TestInterval).Should(HaveLen(2))
 
-				var certificates []cmapi.Certificate
-				certificates, err = kubernetes.ListCertificates(ctx, k8sClient, toCreate.Namespace, kubernetes.MatchingLabelsForHumio(toCreate.Name))
-				Expect(err).To(Succeed())
-				for _, certificate := range certificates {
-					Expect(certificate.Spec.DNSNames).Should(ContainElements(toCreate.Spec.TLS.ExtraHostnames))
-				}
+			var certificates []cmapi.Certificate
+			certificates, err = kubernetes.ListCertificates(ctx, k8sClient, toCreate.Namespace, kubernetes.MatchingLabelsForHumio(toCreate.Name))
+			Expect(err).To(Succeed())
+			for _, certificate := range certificates {
+				Expect(certificate.Spec.DNSNames).Should(ContainElements(toCreate.Spec.TLS.ExtraHostnames))
 			}
 		})
 	})
@@ -4362,6 +4367,19 @@ var _ = Describe("HumioCluster Controller", func() {
 			for _, pod := range clusterPods {
 				Expect(pod.Spec.PriorityClassName).To(Equal(toCreate.Spec.PriorityClassName))
 			}
+
+			Expect(k8sClient.Delete(context.TODO(), priorityClass)).To(Succeed())
+
+			Eventually(func() bool {
+				return k8serrors.IsNotFound(k8sClient.Get(
+					context.TODO(),
+					types.NamespacedName{
+						Namespace: priorityClass.Namespace,
+						Name:      priorityClass.Name,
+					},
+					priorityClass),
+				)
+			}, testTimeout, suite.TestInterval).Should(BeTrue())
 		})
 	})
 

controllers/suite/clusters/suite_test.go

@@ -84,8 +84,6 @@ var _ = BeforeSuite(func() {
 	log = zapr.NewLogger(zapLog)
 	logf.SetLogger(log)
 
-	Expect(os.Getenv("HUMIO_E2E_LICENSE")).NotTo(BeEmpty())
-
 	By("bootstrapping test environment")
 	useExistingCluster := true
 	testProcessNamespace = fmt.Sprintf("e2e-clusters-%d", GinkgoParallelProcess())
@@ -98,6 +96,8 @@ var _ = BeforeSuite(func() {
 			testHumioClient = humio.NewMockClient()
 		} else {
 			testHumioClient = humio.NewClient(log, "")
+			By("Verifying we have a valid license, as tests will require starting up real LogScale containers")
+			Expect(os.Getenv("HUMIO_E2E_LICENSE")).NotTo(BeEmpty())
 		}
 	} else {
 		testTimeout = time.Second * 30

controllers/suite/common.go

@@ -274,7 +274,7 @@ func ConstructBasicNodeSpecForHumioCluster(key types.NamespacedName) humiov1alph
 		}
 	}
 
-	if useDockerCredentials() {
+	if UseDockerCredentials() {
 		nodeSpec.ImagePullSecrets = []corev1.LocalObjectReference{
 			{Name: DockerRegistryCredentialsSecretName},
 		}
@@ -311,12 +311,19 @@ func ConstructBasicSingleNodeHumioCluster(key types.NamespacedName, useAutoCreat
 func CreateLicenseSecret(ctx context.Context, clusterKey types.NamespacedName, k8sClient client.Client, cluster *humiov1alpha1.HumioCluster) {
 	UsingClusterBy(cluster.Name, fmt.Sprintf("Creating the license secret %s", cluster.Spec.License.SecretKeyRef.Name))
 
+	licenseString := "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzUxMiJ9.eyJpc09lbSI6ZmFsc2UsImF1ZCI6Ikh1bWlvLWxpY2Vuc2UtY2hlY2siLCJzdWIiOiJIdW1pbyBFMkUgdGVzdHMiLCJ1aWQiOiJGUXNvWlM3Yk1PUldrbEtGIiwibWF4VXNlcnMiOjEwLCJhbGxvd1NBQVMiOnRydWUsIm1heENvcmVzIjoxLCJ2YWxpZFVudGlsIjoxNzQzMTY2ODAwLCJleHAiOjE3NzQ1OTMyOTcsImlzVHJpYWwiOmZhbHNlLCJpYXQiOjE2Nzk5ODUyOTcsIm1heEluZ2VzdEdiUGVyRGF5IjoxfQ.someinvalidsignature"
+
+	// If we use a k8s that is not envtest, and we didn't specify we are using a dummy image, we require a valid license
+	if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" && os.Getenv("DUMMY_LOGSCALE_IMAGE") != "true" {
+		licenseString = os.Getenv("HUMIO_E2E_LICENSE")
+	}
+
 	licenseSecret := corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      fmt.Sprintf("%s-license", clusterKey.Name),
 			Namespace: clusterKey.Namespace,
 		},
-		StringData: map[string]string{"license": os.Getenv("HUMIO_E2E_LICENSE")},
+		StringData: map[string]string{"license": licenseString},
 		Type:       corev1.SecretTypeOpaque,
 	}
 	Expect(k8sClient.Create(ctx, &licenseSecret)).To(Succeed())
@@ -663,13 +670,13 @@ func WaitForReconcileToSync(ctx context.Context, key types.NamespacedName, k8sCl
 	}, testTimeout, TestInterval).Should(BeNumerically("==", beforeGeneration))
 }
 
-func useDockerCredentials() bool {
+func UseDockerCredentials() bool {
 	return os.Getenv(dockerUsernameEnvVar) != "" && os.Getenv(dockerPasswordEnvVar) != "" &&
 		os.Getenv(dockerUsernameEnvVar) != "none" && os.Getenv(dockerPasswordEnvVar) != "none"
 }
 
 func CreateDockerRegredSecret(ctx context.Context, namespace corev1.Namespace, k8sClient client.Client) {
-	if !useDockerCredentials() {
+	if !UseDockerCredentials() {
 		return
 	}