From 23af8e1b8debb8759a6e5b59a3a495136a52e54b Mon Sep 17 00:00:00 2001 From: Mike Rostermund Date: Tue, 24 Sep 2024 14:52:03 +0200 Subject: [PATCH 1/5] Lower initial delay seconds if we're using the dummy container --- controllers/humiocluster_defaults.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/controllers/humiocluster_defaults.go b/controllers/humiocluster_defaults.go index 9197bd7e..5b4561e8 100644 --- a/controllers/humiocluster_defaults.go +++ b/controllers/humiocluster_defaults.go @@ -561,7 +561,7 @@ func (hnp *HumioNodePool) GetContainerReadinessProbe() *corev1.Probe { } if hnp.humioNodeSpec.ContainerReadinessProbe == nil { - return &corev1.Probe{ + probe := &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/api/v1/is-node-up", @@ -575,6 +575,10 @@ func (hnp *HumioNodePool) GetContainerReadinessProbe() *corev1.Probe { SuccessThreshold: 1, FailureThreshold: 10, } + if os.Getenv("DUMMY_LOGSCALE_IMAGE") == "true" { + probe.InitialDelaySeconds = 0 + } + return probe } return hnp.humioNodeSpec.ContainerReadinessProbe } From 84bad21d11d75029017e5a5252e64ee923b4e7ca Mon Sep 17 00:00:00 2001 From: Mike Rostermund Date: Tue, 24 Sep 2024 14:52:32 +0200 Subject: [PATCH 2/5] Remove unused ca-cert volume from cluster pods --- controllers/humiocluster_pods.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/controllers/humiocluster_pods.go b/controllers/humiocluster_pods.go index 148e11dc..34e1d29c 100644 --- a/controllers/humiocluster_pods.go +++ b/controllers/humiocluster_pods.go @@ -452,22 +452,6 @@ func ConstructPod(hnp *HumioNodePool, humioNodeName string, attachments *podAtta }, }, }) - pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ - Name: "ca-cert", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: hnp.GetClusterName(), - DefaultMode: &mode, - Items: []corev1.KeyToPath{ - { - Key: "ca.crt", - Path: "certs/ca-bundle.crt", - Mode: &mode, - }, - }, - }, - }, - }) } if attachments.bootstrapTokenSecretReference.hash != "" { From 3d3fc305ac5ad07646b58d7b3f03afa249316873 Mon Sep 17 00:00:00 2001 From: Mike Rostermund Date: Tue, 24 Sep 2024 14:53:23 +0200 Subject: [PATCH 3/5] Remove unused install_go function --- hack/functions.sh | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/hack/functions.sh b/hack/functions.sh index 6528a18f..d4ab5ddd 100644 --- a/hack/functions.sh +++ b/hack/functions.sh @@ -84,23 +84,6 @@ install_helm() { $helm version } -install_go() { - if [ $(uname -o) = Darwin ]; then - # For Intel Macs - [ $(uname -m) = x86_64 ] && curl -Lo $go.tar.gz https://dl.google.com/go/go${go_version}.darwin-amd64.tar.gz && tar -zxvf $go.tar.gz -C $bin_dir && mv $bin_dir/go $bin_dir/goinstall && ln -s $bin_dir/goinstall/bin/go $go - # For M1 / ARM Macs - [ $(uname -m) = arm64 ] && curl -Lo $go.tar.gz https://dl.google.com/go/go${go_version}.darwin-arm64.tar.gz && tar -zxvf $go.tar.gz -C $bin_dir && mv $bin_dir/go $bin_dir/goinstall && ln -s $bin_dir/goinstall/bin/go $go - else - echo "Assuming Linux" - # For AMD64 / x86_64 - [ $(uname -m) = x86_64 ] && curl -Lo $go.tar.gz https://dl.google.com/go/go${go_version}.linux-amd64.tar.gz && tar -zxvf $go.tar.gz -C $bin_dir && mv $bin_dir/go $bin_dir/goinstall && ln -s $bin_dir/goinstall/bin/go $go - # For ARM64 - [ $(uname -m) = aarch64 ] && curl -Lo $go.tar.gz https://dl.google.com/go/go${go_version}.linux-arm64.tar.gz && tar -zxvf $go.tar.gz -C $bin_dir && mv $bin_dir/go $bin_dir/goinstall && ln -s $bin_dir/goinstall/bin/go $go - fi - rm $go.tar.gz - $go version -} - install_ginkgo() { go get github.com/onsi/ginkgo/v2/ginkgo go install github.com/onsi/ginkgo/v2/ginkgo From d338ef60a80aaa0eae53c8888f9217f9d4e4e4f0 Mon Sep 17 00:00:00 2001 From: Mike Rostermund Date: Tue, 24 Sep 2024 14:54:30 +0200 Subject: [PATCH 4/5] Introduce USE_CERTMANAGER and PRESERVE_KIND_CLUSTER to kind test execution and skip downloading binaries if already present USE_CERTMANAGER (default="true") can be set to "false" to disable cert-manager installation to speed up test execution. PRESERVE_KIND_CLUSTER (default="false") can be set to "true" to keep the kind cluster around after test execution so that the consecutive runs will reuse the kind cluster and existing helm installs. This also fixes a bunch of test cases where tests didn't properly clean up, causing conflicts on consecutive test runs. --- .../clusters/humiocluster_controller_test.go | 120 ++++++++++-------- controllers/suite/common.go | 6 +- .../humioresources_controller_test.go | 114 ++++++++++++++++- controllers/suite/resources/suite_test.go | 46 +++++-- hack/functions.sh | 34 ++++- hack/run-e2e-using-kind-dummy.sh | 17 ++- hack/run-e2e-using-kind.sh | 18 ++- hack/run-e2e-within-kind-test-pod-dummy.sh | 2 +- hack/run-e2e-within-kind-test-pod.sh | 2 +- 9 files changed, 277 insertions(+), 82 deletions(-) diff --git a/controllers/suite/clusters/humiocluster_controller_test.go b/controllers/suite/clusters/humiocluster_controller_test.go index e3a4ec1b..8998e023 100644 --- a/controllers/suite/clusters/humiocluster_controller_test.go +++ b/controllers/suite/clusters/humiocluster_controller_test.go @@ -278,7 +278,7 @@ var _ = Describe("HumioCluster Controller", func() { Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) } - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } @@ -462,7 +462,7 @@ var _ = Describe("HumioCluster Controller", func() { Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) } - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } @@ -556,7 +556,7 @@ var _ = Describe("HumioCluster Controller", func() { Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) } - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } @@ -631,7 +631,7 @@ var _ = Describe("HumioCluster Controller", func() { Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) } - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } @@ -707,16 +707,16 @@ var _ = Describe("HumioCluster Controller", func() { Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) } - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } }) }) - Context("Humio Cluster Update EXTERNAL_URL", Label("envtest", "dummy", "real"), func() { + Context("Humio Cluster Update EXTERNAL_URL", Label("dummy", "real"), func() { It("Update should correctly replace pods to use the new EXTERNAL_URL in a non-rolling fashion", func() { - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.UseCertManager() { key := types.NamespacedName{ Name: "humiocluster-update-ext-url", Namespace: testProcessNamespace, @@ -957,7 +957,7 @@ var _ = Describe("HumioCluster Controller", func() { Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) } - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } @@ -1063,7 +1063,7 @@ var _ = Describe("HumioCluster Controller", func() { Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) } - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } @@ -1177,7 +1177,7 @@ var _ = Describe("HumioCluster Controller", func() { Expect(pod.Annotations[controllers.PodRevisionAnnotation]).To(Equal("3")) } - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } @@ -1242,7 +1242,7 @@ var _ = Describe("HumioCluster Controller", func() { updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } @@ -1314,7 +1314,7 @@ var _ = Describe("HumioCluster Controller", func() { updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } @@ -1444,7 +1444,7 @@ var _ = Describe("HumioCluster Controller", func() { }, testTimeout, suite.TestInterval).Should(BeTrue()) updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, updatedHumioCluster.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetPodLabels()) - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } @@ -1666,7 +1666,7 @@ var _ = Describe("HumioCluster Controller", func() { }, testTimeout, suite.TestInterval).Should(BeTrue()) updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, updatedHumioCluster.Namespace, mainNodePoolManager.GetPodLabels()) - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } @@ -1767,7 +1767,7 @@ var _ = Describe("HumioCluster Controller", func() { }, testTimeout, suite.TestInterval).Should(BeTrue()) updatedClusterPods, _ = kubernetes.ListPods(ctx, k8sClient, updatedHumioCluster.Namespace, additionalNodePoolManager.GetPodLabels()) - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } @@ -3337,20 +3337,24 @@ var _ = Describe("HumioCluster Controller", func() { defer suite.CleanupCluster(ctx, k8sClient, toCreate) initialExpectedVolumesCount := 5 - initialExpectedVolumeMountsCount := 4 + initialExpectedHumioContainerVolumeMountsCount := 4 if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { - // if we run on a real cluster we have TLS enabled (using 2 volumes), - // and k8s will automatically inject a service account token adding one more - initialExpectedVolumesCount += 3 - initialExpectedVolumeMountsCount += 2 + // k8s will automatically inject a service account token + initialExpectedVolumesCount += 1 // kube-api-access- + initialExpectedHumioContainerVolumeMountsCount += 1 // kube-api-access- + + if helpers.UseCertManager() { + initialExpectedVolumesCount += 1 // tls-cert + initialExpectedHumioContainerVolumeMountsCount += 1 // tls-cert + } } clusterPods, _ := kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) for _, pod := range clusterPods { Expect(pod.Spec.Volumes).To(HaveLen(initialExpectedVolumesCount)) humioIdx, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName) - Expect(pod.Spec.Containers[humioIdx].VolumeMounts).To(HaveLen(initialExpectedVolumeMountsCount)) + Expect(pod.Spec.Containers[humioIdx].VolumeMounts).To(HaveLen(initialExpectedHumioContainerVolumeMountsCount)) } suite.UsingClusterBy(key.Name, "Adding additional volumes") @@ -3395,7 +3399,7 @@ var _ = Describe("HumioCluster Controller", func() { return pod.Spec.Containers[humioIdx].VolumeMounts } return []corev1.VolumeMount{} - }, testTimeout, suite.TestInterval).Should(HaveLen(initialExpectedVolumeMountsCount + 1)) + }, testTimeout, suite.TestInterval).Should(HaveLen(initialExpectedHumioContainerVolumeMountsCount + 1)) clusterPods, _ = kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) for _, pod := range clusterPods { Expect(pod.Spec.Volumes).Should(ContainElement(extraVolume)) @@ -3416,7 +3420,7 @@ var _ = Describe("HumioCluster Controller", func() { Type: humiov1alpha1.HumioClusterUpdateStrategyRollingUpdate, } protocol := "http" - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(toCreate) { protocol = "https" } @@ -3815,39 +3819,40 @@ var _ = Describe("HumioCluster Controller", func() { }) }) - Context("Humio Cluster with additional hostnames for TLS", Label("envtest", "dummy", "real"), func() { + Context("Humio Cluster with additional hostnames for TLS", Label("dummy", "real"), func() { It("Creating cluster with additional hostnames for TLS", func() { - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { - key := types.NamespacedName{ - Name: "humiocluster-tls-additional-hostnames", - Namespace: testProcessNamespace, - } - toCreate := suite.ConstructBasicSingleNodeHumioCluster(key, true) - toCreate.Spec.TLS = &humiov1alpha1.HumioClusterTLSSpec{ - Enabled: helpers.BoolPtr(true), - ExtraHostnames: []string{ - "something.additional", - "yet.another.something.additional", - }, - } + key := types.NamespacedName{ + Name: "humiocluster-tls-additional-hostnames", + Namespace: testProcessNamespace, + } + toCreate := suite.ConstructBasicSingleNodeHumioCluster(key, true) + if !helpers.TLSEnabled(toCreate) { + return + } + toCreate.Spec.TLS = &humiov1alpha1.HumioClusterTLSSpec{ + Enabled: helpers.BoolPtr(true), + ExtraHostnames: []string{ + "something.additional", + "yet.another.something.additional", + }, + } - suite.UsingClusterBy(key.Name, "Creating the cluster successfully") - ctx := context.Background() - suite.CreateAndBootstrapCluster(ctx, k8sClient, testHumioClient, toCreate, true, humiov1alpha1.HumioClusterStateRunning, testTimeout) - defer suite.CleanupCluster(ctx, k8sClient, toCreate) + suite.UsingClusterBy(key.Name, "Creating the cluster successfully") + ctx := context.Background() + suite.CreateAndBootstrapCluster(ctx, k8sClient, testHumioClient, toCreate, true, humiov1alpha1.HumioClusterStateRunning, testTimeout) + defer suite.CleanupCluster(ctx, k8sClient, toCreate) - suite.UsingClusterBy(key.Name, "Confirming certificate objects contain the additional hostnames") + suite.UsingClusterBy(key.Name, "Confirming certificate objects contain the additional hostnames") - Eventually(func() ([]cmapi.Certificate, error) { - return kubernetes.ListCertificates(ctx, k8sClient, toCreate.Namespace, kubernetes.MatchingLabelsForHumio(toCreate.Name)) - }, testTimeout, suite.TestInterval).Should(HaveLen(2)) + Eventually(func() ([]cmapi.Certificate, error) { + return kubernetes.ListCertificates(ctx, k8sClient, toCreate.Namespace, kubernetes.MatchingLabelsForHumio(toCreate.Name)) + }, testTimeout, suite.TestInterval).Should(HaveLen(2)) - var certificates []cmapi.Certificate - certificates, err = kubernetes.ListCertificates(ctx, k8sClient, toCreate.Namespace, kubernetes.MatchingLabelsForHumio(toCreate.Name)) - Expect(err).To(Succeed()) - for _, certificate := range certificates { - Expect(certificate.Spec.DNSNames).Should(ContainElements(toCreate.Spec.TLS.ExtraHostnames)) - } + var certificates []cmapi.Certificate + certificates, err = kubernetes.ListCertificates(ctx, k8sClient, toCreate.Namespace, kubernetes.MatchingLabelsForHumio(toCreate.Name)) + Expect(err).To(Succeed()) + for _, certificate := range certificates { + Expect(certificate.Spec.DNSNames).Should(ContainElements(toCreate.Spec.TLS.ExtraHostnames)) } }) }) @@ -4362,6 +4367,19 @@ var _ = Describe("HumioCluster Controller", func() { for _, pod := range clusterPods { Expect(pod.Spec.PriorityClassName).To(Equal(toCreate.Spec.PriorityClassName)) } + + Expect(k8sClient.Delete(context.TODO(), priorityClass)).To(Succeed()) + + Eventually(func() bool { + return k8serrors.IsNotFound(k8sClient.Get( + context.TODO(), + types.NamespacedName{ + Namespace: priorityClass.Namespace, + Name: priorityClass.Name, + }, + priorityClass), + ) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) }) diff --git a/controllers/suite/common.go b/controllers/suite/common.go index 3d820e1d..606bf2be 100644 --- a/controllers/suite/common.go +++ b/controllers/suite/common.go @@ -274,7 +274,7 @@ func ConstructBasicNodeSpecForHumioCluster(key types.NamespacedName) humiov1alph } } - if useDockerCredentials() { + if UseDockerCredentials() { nodeSpec.ImagePullSecrets = []corev1.LocalObjectReference{ {Name: DockerRegistryCredentialsSecretName}, } @@ -663,13 +663,13 @@ func WaitForReconcileToSync(ctx context.Context, key types.NamespacedName, k8sCl }, testTimeout, TestInterval).Should(BeNumerically("==", beforeGeneration)) } -func useDockerCredentials() bool { +func UseDockerCredentials() bool { return os.Getenv(dockerUsernameEnvVar) != "" && os.Getenv(dockerPasswordEnvVar) != "" && os.Getenv(dockerUsernameEnvVar) != "none" && os.Getenv(dockerPasswordEnvVar) != "none" } func CreateDockerRegredSecret(ctx context.Context, namespace corev1.Namespace, k8sClient client.Client) { - if !useDockerCredentials() { + if !UseDockerCredentials() { return } diff --git a/controllers/suite/resources/humioresources_controller_test.go b/controllers/suite/resources/humioresources_controller_test.go index 56b213cd..9b9f2fe8 100644 --- a/controllers/suite/resources/humioresources_controller_test.go +++ b/controllers/suite/resources/humioresources_controller_test.go @@ -662,7 +662,7 @@ var _ = Describe("Humio Resources Controllers", func() { Namespace: clusterKey.Namespace, } protocol := "http" - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" && helpers.UseCertManager() { protocol = "https" } @@ -1775,6 +1775,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(expectedSecretValue)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: OpsGenieProperties: Should support referencing secrets", func() { @@ -1838,6 +1845,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(expectedSecretValue)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: OpsGenieProperties: Should support direct genie key", func() { @@ -1883,6 +1897,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(expectedSecretValue)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: VictorOpsProperties: Should support referencing secrets", func() { @@ -1946,6 +1967,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(expectedSecretValue)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: VictorOpsProperties: Should support direct notify url", func() { @@ -1991,6 +2019,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(expectedSecretValue)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: SlackPostMessageProperties: Should support referencing secrets", func() { @@ -2057,6 +2092,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(expectedSecretValue)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: SlackPostMessageProperties: Should support direct api token", func() { @@ -2104,6 +2146,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(toCreateAction.Spec.SlackPostMessageProperties.ApiToken)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: SlackProperties: Should support referencing secrets", func() { @@ -2169,6 +2218,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(expectedSecretValue)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: SlackProperties: Should support direct url", func() { @@ -2216,6 +2272,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(toCreateAction.Spec.SlackProperties.Url)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: PagerDutyProperties: Should support referencing secrets", func() { @@ -2279,6 +2342,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(expectedSecretValue)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: PagerDutyProperties: Should support direct api token", func() { @@ -2324,6 +2394,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(toCreateAction.Spec.PagerDutyProperties.RoutingKey)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: WebhookProperties: Should support direct url", func() { @@ -2370,6 +2447,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(toCreateAction.Spec.WebhookProperties.Url)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: WebhookProperties: Should support referencing secret url", func() { @@ -2434,6 +2518,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(expectedSecretValue)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: WebhookProperties: Should support direct url and headers", func() { @@ -2496,6 +2587,13 @@ var _ = Describe("Humio Resources Controllers", func() { allHeaders, found := kubernetes.GetFullSetOfMergedWebhookheaders(toCreateAction) Expect(found).To(BeTrue()) Expect(allHeaders).To(HaveKeyWithValue(nonsensitiveHeaderKey, nonsensitiveHeaderValue)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: WebhookProperties: Should support direct url and mixed headers", func() { ctx := context.Background() @@ -2588,6 +2686,13 @@ var _ = Describe("Humio Resources Controllers", func() { Expect(found).To(BeTrue()) Expect(allHeaders).To(HaveKeyWithValue(headerKey1, sensitiveHeaderValue1)) Expect(allHeaders).To(HaveKeyWithValue(headerKey2, nonsensitiveHeaderValue2)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: WebhookProperties: Should support direct url and secret headers", func() { ctx := context.Background() @@ -2670,6 +2775,13 @@ var _ = Describe("Humio Resources Controllers", func() { allHeaders, found := kubernetes.GetFullSetOfMergedWebhookheaders(toCreateAction) Expect(found).To(BeTrue()) Expect(allHeaders).To(HaveKeyWithValue(headerKey, sensitiveHeaderValue)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) }) diff --git a/controllers/suite/resources/suite_test.go b/controllers/suite/resources/suite_test.go index 6b296d7b..faa4a3db 100644 --- a/controllers/suite/resources/suite_test.go +++ b/controllers/suite/resources/suite_test.go @@ -27,6 +27,7 @@ import ( "time" "github.com/humio/humio-operator/pkg/kubernetes" + k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/rest" "github.com/humio/humio-operator/controllers" @@ -103,7 +104,6 @@ var _ = BeforeSuite(func() { testEnv = &envtest.Environment{ UseExistingCluster: &useExistingCluster, } - if os.Getenv("DUMMY_LOGSCALE_IMAGE") == "true" { humioClient = humio.NewMockClient() } else { @@ -256,8 +256,7 @@ var _ = BeforeSuite(func() { Name: clusterKey.Namespace, }, } - err = k8sClient.Create(context.TODO(), &testNamespace) - Expect(err).ToNot(HaveOccurred()) + Expect(k8sClient.Create(context.TODO(), &testNamespace)).ToNot(HaveOccurred()) suite.CreateDockerRegredSecret(context.TODO(), testNamespace, k8sClient) @@ -347,24 +346,49 @@ var _ = BeforeSuite(func() { var _ = AfterSuite(func() { if k8sClient != nil { + Expect(k8sClient.Delete(context.TODO(), &corev1alpha1.HumioRepository{ + ObjectMeta: metav1.ObjectMeta{ + Name: testRepo.Name, + Namespace: testRepo.Namespace, + }, + })).To(Succeed()) + Expect(k8sClient.Delete(context.TODO(), &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: testService1.Name, + Namespace: testService1.Namespace, + }, + })).To(Succeed()) + Expect(k8sClient.Delete(context.TODO(), &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: testService2.Name, + Namespace: testService2.Namespace, + }, + })).To(Succeed()) + suite.UsingClusterBy(clusterKey.Name, "HumioCluster: Confirming resource generation wasn't updated excessively") Expect(k8sClient.Get(context.Background(), clusterKey, cluster)).Should(Succeed()) Expect(cluster.GetGeneration()).ShouldNot(BeNumerically(">", 100)) suite.CleanupCluster(context.TODO(), k8sClient, cluster) - By(fmt.Sprintf("Removing regcred secret for namespace: %s", testNamespace.Name)) - _ = k8sClient.Delete(context.TODO(), &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: suite.DockerRegistryCredentialsSecretName, - Namespace: clusterKey.Namespace, - }, - }) + if suite.UseDockerCredentials() { + By(fmt.Sprintf("Removing regcred secret for namespace: %s", testNamespace.Name)) + Expect(k8sClient.Delete(context.TODO(), &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: suite.DockerRegistryCredentialsSecretName, + Namespace: clusterKey.Namespace, + }, + })).To(Succeed()) + } - if testNamespace.ObjectMeta.Name != "" { + if testNamespace.ObjectMeta.Name != "" && os.Getenv("PRESERVE_KIND_CLUSTER") == "true" { By(fmt.Sprintf("Removing test namespace: %s", clusterKey.Namespace)) err := k8sClient.Delete(context.TODO(), &testNamespace) Expect(err).ToNot(HaveOccurred()) + + Eventually(func() bool { + return k8serrors.IsNotFound(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterKey.Namespace}, &testNamespace)) + }, testTimeout, suite.TestInterval).Should(BeTrue()) } } diff --git a/hack/functions.sh b/hack/functions.sh index d4ab5ddd..027ae544 100644 --- a/hack/functions.sh +++ b/hack/functions.sh @@ -16,6 +16,15 @@ PATH=$bin_dir/goinstall/bin:$bin_dir:/usr/local/go/bin:$PATH GOBIN=$bin_dir start_kind_cluster() { + if $kind get clusters | grep kind ; then + if ! $kubectl get daemonset -n kube-system kindnet ; then + echo "Cluster unavailable or not using a kind cluster. Only kind clusters are supported!" + exit 1 + fi + + return + fi + $kind create cluster --name kind --config hack/kind-config.yaml --image $kindest_node_image_multiplatform_amd64_arm64 --wait 300s sleep 5 @@ -29,10 +38,19 @@ start_kind_cluster() { } cleanup_kind_cluster() { - $kind delete cluster --name kind + if [[ $preserve_kind_cluster == "true" ]]; then + $kubectl delete --grace-period=1 pod test-pod + $kubectl delete -k config/crd/ + else + $kind delete cluster --name kind + fi } install_kind() { + if [ -f $kind ]; then + $kind version | grep -E "^kind v${kind_version}" && return + fi + if [ $(uname -o) = Darwin ]; then # For Intel Macs [ $(uname -m) = x86_64 ] && curl -Lo $kind https://kind.sigs.k8s.io/dl/v${kind_version}/kind-darwin-amd64 @@ -50,6 +68,10 @@ install_kind() { } install_kubectl() { + if [ -f $kubectl ]; then + $kubectl version --client | grep "GitVersion:\"v${kubectl_version}\"" && return + fi + if [ $(uname -o) = Darwin ]; then # For Intel Macs [ $(uname -m) = x86_64 ] && curl -Lo $kubectl https://dl.k8s.io/release/v${kubectl_version}/bin/darwin/amd64/kubectl @@ -67,6 +89,10 @@ install_kubectl() { } install_helm() { + if [ -f $helm ]; then + $helm version --short | grep -E "^v${helm_version}" && return + fi + if [ $(uname -o) = Darwin ]; then # For Intel Macs [ $(uname -m) = x86_64 ] && curl -Lo $helm.tar.gz https://get.helm.sh/helm-v${helm_version}-darwin-amd64.tar.gz && tar -zxvf $helm.tar.gz -C $bin_dir && mv $bin_dir/darwin-amd64/helm $helm && rm -r $bin_dir/darwin-amd64 @@ -129,6 +155,8 @@ preload_container_images() { } helm_install_shippers() { + $helm get metadata log-shipper && return + # Install components to get observability during execution of tests if [[ $humio_hostname != "none" ]] && [[ $humio_ingest_token != "none" ]]; then e2eFilterTag=$(cat < Date: Tue, 24 Sep 2024 16:05:42 +0200 Subject: [PATCH 5/5] Add fake license string to allow running tests with envtest or dummy image without a valid license --- .github/workflows/ci.yaml | 2 -- .github/workflows/e2e-dummy.yaml | 1 - Makefile | 3 --- controllers/suite/clusters/suite_test.go | 4 ++-- controllers/suite/common.go | 9 ++++++++- controllers/suite/resources/suite_test.go | 4 ++-- hack/run-e2e-using-kind-dummy.sh | 1 - 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 34b08d65..43206401 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -23,8 +23,6 @@ jobs: - shell: bash run: | make test - env: - HUMIO_E2E_LICENSE: ${{ secrets.HUMIO_E2E_LICENSE }} - name: Publish Test Report uses: mikepenz/action-junit-report@v4 if: always() # always run even if the previous step fails diff --git a/.github/workflows/e2e-dummy.yaml b/.github/workflows/e2e-dummy.yaml index 7a01a047..1c161d3a 100644 --- a/.github/workflows/e2e-dummy.yaml +++ b/.github/workflows/e2e-dummy.yaml @@ -39,7 +39,6 @@ jobs: - name: run e2e tests env: BIN_DIR: ${{ steps.bin_dir.outputs.BIN_DIR }} - HUMIO_E2E_LICENSE: ${{ secrets.HUMIO_E2E_LICENSE }} E2E_KIND_K8S_VERSION: ${{ matrix.kind-k8s-version }} E2E_LOGS_HUMIO_HOSTNAME: ${{ secrets.E2E_LOGS_HUMIO_HOSTNAME }} E2E_LOGS_HUMIO_INGEST_TOKEN: ${{ secrets.E2E_LOGS_HUMIO_INGEST_TOKEN }} diff --git a/Makefile b/Makefile index 6f7d4f97..c025c382 100644 --- a/Makefile +++ b/Makefile @@ -49,9 +49,6 @@ vet: ## Run go vet against code. go vet ./... test: manifests generate fmt vet ginkgo ## Run tests. -ifndef HUMIO_E2E_LICENSE - $(error HUMIO_E2E_LICENSE not set) -endif go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest $(SHELL) -c "\ eval \$$($(GOBIN)/setup-envtest use -p env ${TEST_K8S_VERSION}); \ diff --git a/controllers/suite/clusters/suite_test.go b/controllers/suite/clusters/suite_test.go index 727b4cf0..e75f3f7a 100644 --- a/controllers/suite/clusters/suite_test.go +++ b/controllers/suite/clusters/suite_test.go @@ -84,8 +84,6 @@ var _ = BeforeSuite(func() { log = zapr.NewLogger(zapLog) logf.SetLogger(log) - Expect(os.Getenv("HUMIO_E2E_LICENSE")).NotTo(BeEmpty()) - By("bootstrapping test environment") useExistingCluster := true testProcessNamespace = fmt.Sprintf("e2e-clusters-%d", GinkgoParallelProcess()) @@ -98,6 +96,8 @@ var _ = BeforeSuite(func() { testHumioClient = humio.NewMockClient() } else { testHumioClient = humio.NewClient(log, "") + By("Verifying we have a valid license, as tests will require starting up real LogScale containers") + Expect(os.Getenv("HUMIO_E2E_LICENSE")).NotTo(BeEmpty()) } } else { testTimeout = time.Second * 30 diff --git a/controllers/suite/common.go b/controllers/suite/common.go index 606bf2be..991f0f1b 100644 --- a/controllers/suite/common.go +++ b/controllers/suite/common.go @@ -311,12 +311,19 @@ func ConstructBasicSingleNodeHumioCluster(key types.NamespacedName, useAutoCreat func CreateLicenseSecret(ctx context.Context, clusterKey types.NamespacedName, k8sClient client.Client, cluster *humiov1alpha1.HumioCluster) { UsingClusterBy(cluster.Name, fmt.Sprintf("Creating the license secret %s", cluster.Spec.License.SecretKeyRef.Name)) + licenseString := "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzUxMiJ9.eyJpc09lbSI6ZmFsc2UsImF1ZCI6Ikh1bWlvLWxpY2Vuc2UtY2hlY2siLCJzdWIiOiJIdW1pbyBFMkUgdGVzdHMiLCJ1aWQiOiJGUXNvWlM3Yk1PUldrbEtGIiwibWF4VXNlcnMiOjEwLCJhbGxvd1NBQVMiOnRydWUsIm1heENvcmVzIjoxLCJ2YWxpZFVudGlsIjoxNzQzMTY2ODAwLCJleHAiOjE3NzQ1OTMyOTcsImlzVHJpYWwiOmZhbHNlLCJpYXQiOjE2Nzk5ODUyOTcsIm1heEluZ2VzdEdiUGVyRGF5IjoxfQ.someinvalidsignature" + + // If we use a k8s that is not envtest, and we didn't specify we are using a dummy image, we require a valid license + if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" && os.Getenv("DUMMY_LOGSCALE_IMAGE") != "true" { + licenseString = os.Getenv("HUMIO_E2E_LICENSE") + } + licenseSecret := corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-license", clusterKey.Name), Namespace: clusterKey.Namespace, }, - StringData: map[string]string{"license": os.Getenv("HUMIO_E2E_LICENSE")}, + StringData: map[string]string{"license": licenseString}, Type: corev1.SecretTypeOpaque, } Expect(k8sClient.Create(ctx, &licenseSecret)).To(Succeed()) diff --git a/controllers/suite/resources/suite_test.go b/controllers/suite/resources/suite_test.go index faa4a3db..e7db65c1 100644 --- a/controllers/suite/resources/suite_test.go +++ b/controllers/suite/resources/suite_test.go @@ -90,8 +90,6 @@ var _ = BeforeSuite(func() { log = zapr.NewLogger(zapLog) logf.SetLogger(log) - Expect(os.Getenv("HUMIO_E2E_LICENSE")).NotTo(BeEmpty()) - By("bootstrapping test environment") useExistingCluster := true clusterKey = types.NamespacedName{ @@ -108,6 +106,8 @@ var _ = BeforeSuite(func() { humioClient = humio.NewMockClient() } else { humioClient = humio.NewClient(log, "") + By("Verifying we have a valid license, as tests will require starting up real LogScale containers") + Expect(os.Getenv("HUMIO_E2E_LICENSE")).NotTo(BeEmpty()) } } else { diff --git a/hack/run-e2e-using-kind-dummy.sh b/hack/run-e2e-using-kind-dummy.sh index ea5c31b2..05949434 100755 --- a/hack/run-e2e-using-kind-dummy.sh +++ b/hack/run-e2e-using-kind-dummy.sh @@ -10,7 +10,6 @@ trap "cleanup_kind_cluster" EXIT declare -r ginkgo_nodes=${GINKGO_NODES:-6} declare -r docker=$(which docker) -declare -r humio_e2e_license=${HUMIO_E2E_LICENSE} declare -r e2e_run_ref=${GITHUB_REF:-outside-github-$(hostname)} declare -r e2e_run_id=${GITHUB_RUN_ID:-none} declare -r e2e_run_attempt=${GITHUB_RUN_ATTEMPT:-none}