From e7f66834cf54324d1375943ec01c975d2c73783c Mon Sep 17 00:00:00 2001 From: Mike Rostermund Date: Wed, 25 Sep 2024 10:00:33 +0200 Subject: [PATCH] Introduce flags for speeding up test execution with "kind" (#858) * Lower initial delay seconds if we're using the dummy container * Remove unused ca-cert volume from cluster pods * Remove unused install_go function * Introduce USE_CERTMANAGER and PRESERVE_KIND_CLUSTER to kind test execution and skip downloading binaries if already present USE_CERTMANAGER (default="true") can be set to "false" to disable cert-manager installation to speed up test execution. PRESERVE_KIND_CLUSTER (default="false") can be set to "true" to keep the kind cluster around after test execution so that the consecutive runs will reuse the kind cluster and existing helm installs. This also fixes a bunch of test cases where tests didn't properly clean up, causing conflicts on consecutive test runs. * Add fake license string to allow running tests with envtest or dummy image without a valid license --- .github/workflows/ci.yaml | 2 - .github/workflows/e2e-dummy.yaml | 1 - Makefile | 3 - controllers/humiocluster_defaults.go | 6 +- controllers/humiocluster_pods.go | 16 --- .../clusters/humiocluster_controller_test.go | 120 ++++++++++-------- controllers/suite/clusters/suite_test.go | 4 +- controllers/suite/common.go | 15 ++- .../humioresources_controller_test.go | 114 ++++++++++++++++- controllers/suite/resources/suite_test.go | 50 ++++++-- hack/functions.sh | 51 +++++--- hack/run-e2e-using-kind-dummy.sh | 18 +-- hack/run-e2e-using-kind.sh | 18 ++- hack/run-e2e-within-kind-test-pod-dummy.sh | 2 +- hack/run-e2e-within-kind-test-pod.sh | 2 +- 15 files changed, 294 insertions(+), 128 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 34b08d650..432064012 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -23,8 +23,6 @@ jobs: - shell: bash run: | make test - env: - HUMIO_E2E_LICENSE: ${{ secrets.HUMIO_E2E_LICENSE }} - name: Publish Test Report uses: mikepenz/action-junit-report@v4 if: always() # always run even if the previous step fails diff --git a/.github/workflows/e2e-dummy.yaml b/.github/workflows/e2e-dummy.yaml index 7a01a0473..1c161d3a1 100644 --- a/.github/workflows/e2e-dummy.yaml +++ b/.github/workflows/e2e-dummy.yaml @@ -39,7 +39,6 @@ jobs: - name: run e2e tests env: BIN_DIR: ${{ steps.bin_dir.outputs.BIN_DIR }} - HUMIO_E2E_LICENSE: ${{ secrets.HUMIO_E2E_LICENSE }} E2E_KIND_K8S_VERSION: ${{ matrix.kind-k8s-version }} E2E_LOGS_HUMIO_HOSTNAME: ${{ secrets.E2E_LOGS_HUMIO_HOSTNAME }} E2E_LOGS_HUMIO_INGEST_TOKEN: ${{ secrets.E2E_LOGS_HUMIO_INGEST_TOKEN }} diff --git a/Makefile b/Makefile index 6f7d4f97d..c025c3824 100644 --- a/Makefile +++ b/Makefile @@ -49,9 +49,6 @@ vet: ## Run go vet against code. go vet ./... test: manifests generate fmt vet ginkgo ## Run tests. -ifndef HUMIO_E2E_LICENSE - $(error HUMIO_E2E_LICENSE not set) -endif go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest $(SHELL) -c "\ eval \$$($(GOBIN)/setup-envtest use -p env ${TEST_K8S_VERSION}); \ diff --git a/controllers/humiocluster_defaults.go b/controllers/humiocluster_defaults.go index 9197bd7ed..5b4561e82 100644 --- a/controllers/humiocluster_defaults.go +++ b/controllers/humiocluster_defaults.go @@ -561,7 +561,7 @@ func (hnp *HumioNodePool) GetContainerReadinessProbe() *corev1.Probe { } if hnp.humioNodeSpec.ContainerReadinessProbe == nil { - return &corev1.Probe{ + probe := &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/api/v1/is-node-up", @@ -575,6 +575,10 @@ func (hnp *HumioNodePool) GetContainerReadinessProbe() *corev1.Probe { SuccessThreshold: 1, FailureThreshold: 10, } + if os.Getenv("DUMMY_LOGSCALE_IMAGE") == "true" { + probe.InitialDelaySeconds = 0 + } + return probe } return hnp.humioNodeSpec.ContainerReadinessProbe } diff --git a/controllers/humiocluster_pods.go b/controllers/humiocluster_pods.go index 148e11dc9..34e1d29cb 100644 --- a/controllers/humiocluster_pods.go +++ b/controllers/humiocluster_pods.go @@ -452,22 +452,6 @@ func ConstructPod(hnp *HumioNodePool, humioNodeName string, attachments *podAtta }, }, }) - pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ - Name: "ca-cert", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: hnp.GetClusterName(), - DefaultMode: &mode, - Items: []corev1.KeyToPath{ - { - Key: "ca.crt", - Path: "certs/ca-bundle.crt", - Mode: &mode, - }, - }, - }, - }, - }) } if attachments.bootstrapTokenSecretReference.hash != "" { diff --git a/controllers/suite/clusters/humiocluster_controller_test.go b/controllers/suite/clusters/humiocluster_controller_test.go index e3a4ec1b7..8998e023f 100644 --- a/controllers/suite/clusters/humiocluster_controller_test.go +++ b/controllers/suite/clusters/humiocluster_controller_test.go @@ -278,7 +278,7 @@ var _ = Describe("HumioCluster Controller", func() { Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) } - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } @@ -462,7 +462,7 @@ var _ = Describe("HumioCluster Controller", func() { Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) } - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } @@ -556,7 +556,7 @@ var _ = Describe("HumioCluster Controller", func() { Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) } - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } @@ -631,7 +631,7 @@ var _ = Describe("HumioCluster Controller", func() { Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) } - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } @@ -707,16 +707,16 @@ var _ = Describe("HumioCluster Controller", func() { Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) } - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } }) }) - Context("Humio Cluster Update EXTERNAL_URL", Label("envtest", "dummy", "real"), func() { + Context("Humio Cluster Update EXTERNAL_URL", Label("dummy", "real"), func() { It("Update should correctly replace pods to use the new EXTERNAL_URL in a non-rolling fashion", func() { - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.UseCertManager() { key := types.NamespacedName{ Name: "humiocluster-update-ext-url", Namespace: testProcessNamespace, @@ -957,7 +957,7 @@ var _ = Describe("HumioCluster Controller", func() { Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) } - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } @@ -1063,7 +1063,7 @@ var _ = Describe("HumioCluster Controller", func() { Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) } - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } @@ -1177,7 +1177,7 @@ var _ = Describe("HumioCluster Controller", func() { Expect(pod.Annotations[controllers.PodRevisionAnnotation]).To(Equal("3")) } - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } @@ -1242,7 +1242,7 @@ var _ = Describe("HumioCluster Controller", func() { updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } @@ -1314,7 +1314,7 @@ var _ = Describe("HumioCluster Controller", func() { updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } @@ -1444,7 +1444,7 @@ var _ = Describe("HumioCluster Controller", func() { }, testTimeout, suite.TestInterval).Should(BeTrue()) updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, updatedHumioCluster.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetPodLabels()) - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } @@ -1666,7 +1666,7 @@ var _ = Describe("HumioCluster Controller", func() { }, testTimeout, suite.TestInterval).Should(BeTrue()) updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, updatedHumioCluster.Namespace, mainNodePoolManager.GetPodLabels()) - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } @@ -1767,7 +1767,7 @@ var _ = Describe("HumioCluster Controller", func() { }, testTimeout, suite.TestInterval).Should(BeTrue()) updatedClusterPods, _ = kubernetes.ListPods(ctx, k8sClient, updatedHumioCluster.Namespace, additionalNodePoolManager.GetPodLabels()) - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(&updatedHumioCluster) { suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } @@ -3337,20 +3337,24 @@ var _ = Describe("HumioCluster Controller", func() { defer suite.CleanupCluster(ctx, k8sClient, toCreate) initialExpectedVolumesCount := 5 - initialExpectedVolumeMountsCount := 4 + initialExpectedHumioContainerVolumeMountsCount := 4 if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { - // if we run on a real cluster we have TLS enabled (using 2 volumes), - // and k8s will automatically inject a service account token adding one more - initialExpectedVolumesCount += 3 - initialExpectedVolumeMountsCount += 2 + // k8s will automatically inject a service account token + initialExpectedVolumesCount += 1 // kube-api-access- + initialExpectedHumioContainerVolumeMountsCount += 1 // kube-api-access- + + if helpers.UseCertManager() { + initialExpectedVolumesCount += 1 // tls-cert + initialExpectedHumioContainerVolumeMountsCount += 1 // tls-cert + } } clusterPods, _ := kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) for _, pod := range clusterPods { Expect(pod.Spec.Volumes).To(HaveLen(initialExpectedVolumesCount)) humioIdx, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName) - Expect(pod.Spec.Containers[humioIdx].VolumeMounts).To(HaveLen(initialExpectedVolumeMountsCount)) + Expect(pod.Spec.Containers[humioIdx].VolumeMounts).To(HaveLen(initialExpectedHumioContainerVolumeMountsCount)) } suite.UsingClusterBy(key.Name, "Adding additional volumes") @@ -3395,7 +3399,7 @@ var _ = Describe("HumioCluster Controller", func() { return pod.Spec.Containers[humioIdx].VolumeMounts } return []corev1.VolumeMount{} - }, testTimeout, suite.TestInterval).Should(HaveLen(initialExpectedVolumeMountsCount + 1)) + }, testTimeout, suite.TestInterval).Should(HaveLen(initialExpectedHumioContainerVolumeMountsCount + 1)) clusterPods, _ = kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) for _, pod := range clusterPods { Expect(pod.Spec.Volumes).Should(ContainElement(extraVolume)) @@ -3416,7 +3420,7 @@ var _ = Describe("HumioCluster Controller", func() { Type: humiov1alpha1.HumioClusterUpdateStrategyRollingUpdate, } protocol := "http" - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if helpers.TLSEnabled(toCreate) { protocol = "https" } @@ -3815,39 +3819,40 @@ var _ = Describe("HumioCluster Controller", func() { }) }) - Context("Humio Cluster with additional hostnames for TLS", Label("envtest", "dummy", "real"), func() { + Context("Humio Cluster with additional hostnames for TLS", Label("dummy", "real"), func() { It("Creating cluster with additional hostnames for TLS", func() { - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { - key := types.NamespacedName{ - Name: "humiocluster-tls-additional-hostnames", - Namespace: testProcessNamespace, - } - toCreate := suite.ConstructBasicSingleNodeHumioCluster(key, true) - toCreate.Spec.TLS = &humiov1alpha1.HumioClusterTLSSpec{ - Enabled: helpers.BoolPtr(true), - ExtraHostnames: []string{ - "something.additional", - "yet.another.something.additional", - }, - } + key := types.NamespacedName{ + Name: "humiocluster-tls-additional-hostnames", + Namespace: testProcessNamespace, + } + toCreate := suite.ConstructBasicSingleNodeHumioCluster(key, true) + if !helpers.TLSEnabled(toCreate) { + return + } + toCreate.Spec.TLS = &humiov1alpha1.HumioClusterTLSSpec{ + Enabled: helpers.BoolPtr(true), + ExtraHostnames: []string{ + "something.additional", + "yet.another.something.additional", + }, + } - suite.UsingClusterBy(key.Name, "Creating the cluster successfully") - ctx := context.Background() - suite.CreateAndBootstrapCluster(ctx, k8sClient, testHumioClient, toCreate, true, humiov1alpha1.HumioClusterStateRunning, testTimeout) - defer suite.CleanupCluster(ctx, k8sClient, toCreate) + suite.UsingClusterBy(key.Name, "Creating the cluster successfully") + ctx := context.Background() + suite.CreateAndBootstrapCluster(ctx, k8sClient, testHumioClient, toCreate, true, humiov1alpha1.HumioClusterStateRunning, testTimeout) + defer suite.CleanupCluster(ctx, k8sClient, toCreate) - suite.UsingClusterBy(key.Name, "Confirming certificate objects contain the additional hostnames") + suite.UsingClusterBy(key.Name, "Confirming certificate objects contain the additional hostnames") - Eventually(func() ([]cmapi.Certificate, error) { - return kubernetes.ListCertificates(ctx, k8sClient, toCreate.Namespace, kubernetes.MatchingLabelsForHumio(toCreate.Name)) - }, testTimeout, suite.TestInterval).Should(HaveLen(2)) + Eventually(func() ([]cmapi.Certificate, error) { + return kubernetes.ListCertificates(ctx, k8sClient, toCreate.Namespace, kubernetes.MatchingLabelsForHumio(toCreate.Name)) + }, testTimeout, suite.TestInterval).Should(HaveLen(2)) - var certificates []cmapi.Certificate - certificates, err = kubernetes.ListCertificates(ctx, k8sClient, toCreate.Namespace, kubernetes.MatchingLabelsForHumio(toCreate.Name)) - Expect(err).To(Succeed()) - for _, certificate := range certificates { - Expect(certificate.Spec.DNSNames).Should(ContainElements(toCreate.Spec.TLS.ExtraHostnames)) - } + var certificates []cmapi.Certificate + certificates, err = kubernetes.ListCertificates(ctx, k8sClient, toCreate.Namespace, kubernetes.MatchingLabelsForHumio(toCreate.Name)) + Expect(err).To(Succeed()) + for _, certificate := range certificates { + Expect(certificate.Spec.DNSNames).Should(ContainElements(toCreate.Spec.TLS.ExtraHostnames)) } }) }) @@ -4362,6 +4367,19 @@ var _ = Describe("HumioCluster Controller", func() { for _, pod := range clusterPods { Expect(pod.Spec.PriorityClassName).To(Equal(toCreate.Spec.PriorityClassName)) } + + Expect(k8sClient.Delete(context.TODO(), priorityClass)).To(Succeed()) + + Eventually(func() bool { + return k8serrors.IsNotFound(k8sClient.Get( + context.TODO(), + types.NamespacedName{ + Namespace: priorityClass.Namespace, + Name: priorityClass.Name, + }, + priorityClass), + ) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) }) diff --git a/controllers/suite/clusters/suite_test.go b/controllers/suite/clusters/suite_test.go index 727b4cf0b..e75f3f7a8 100644 --- a/controllers/suite/clusters/suite_test.go +++ b/controllers/suite/clusters/suite_test.go @@ -84,8 +84,6 @@ var _ = BeforeSuite(func() { log = zapr.NewLogger(zapLog) logf.SetLogger(log) - Expect(os.Getenv("HUMIO_E2E_LICENSE")).NotTo(BeEmpty()) - By("bootstrapping test environment") useExistingCluster := true testProcessNamespace = fmt.Sprintf("e2e-clusters-%d", GinkgoParallelProcess()) @@ -98,6 +96,8 @@ var _ = BeforeSuite(func() { testHumioClient = humio.NewMockClient() } else { testHumioClient = humio.NewClient(log, "") + By("Verifying we have a valid license, as tests will require starting up real LogScale containers") + Expect(os.Getenv("HUMIO_E2E_LICENSE")).NotTo(BeEmpty()) } } else { testTimeout = time.Second * 30 diff --git a/controllers/suite/common.go b/controllers/suite/common.go index 3d820e1d5..991f0f1b6 100644 --- a/controllers/suite/common.go +++ b/controllers/suite/common.go @@ -274,7 +274,7 @@ func ConstructBasicNodeSpecForHumioCluster(key types.NamespacedName) humiov1alph } } - if useDockerCredentials() { + if UseDockerCredentials() { nodeSpec.ImagePullSecrets = []corev1.LocalObjectReference{ {Name: DockerRegistryCredentialsSecretName}, } @@ -311,12 +311,19 @@ func ConstructBasicSingleNodeHumioCluster(key types.NamespacedName, useAutoCreat func CreateLicenseSecret(ctx context.Context, clusterKey types.NamespacedName, k8sClient client.Client, cluster *humiov1alpha1.HumioCluster) { UsingClusterBy(cluster.Name, fmt.Sprintf("Creating the license secret %s", cluster.Spec.License.SecretKeyRef.Name)) + licenseString := "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzUxMiJ9.eyJpc09lbSI6ZmFsc2UsImF1ZCI6Ikh1bWlvLWxpY2Vuc2UtY2hlY2siLCJzdWIiOiJIdW1pbyBFMkUgdGVzdHMiLCJ1aWQiOiJGUXNvWlM3Yk1PUldrbEtGIiwibWF4VXNlcnMiOjEwLCJhbGxvd1NBQVMiOnRydWUsIm1heENvcmVzIjoxLCJ2YWxpZFVudGlsIjoxNzQzMTY2ODAwLCJleHAiOjE3NzQ1OTMyOTcsImlzVHJpYWwiOmZhbHNlLCJpYXQiOjE2Nzk5ODUyOTcsIm1heEluZ2VzdEdiUGVyRGF5IjoxfQ.someinvalidsignature" + + // If we use a k8s that is not envtest, and we didn't specify we are using a dummy image, we require a valid license + if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" && os.Getenv("DUMMY_LOGSCALE_IMAGE") != "true" { + licenseString = os.Getenv("HUMIO_E2E_LICENSE") + } + licenseSecret := corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-license", clusterKey.Name), Namespace: clusterKey.Namespace, }, - StringData: map[string]string{"license": os.Getenv("HUMIO_E2E_LICENSE")}, + StringData: map[string]string{"license": licenseString}, Type: corev1.SecretTypeOpaque, } Expect(k8sClient.Create(ctx, &licenseSecret)).To(Succeed()) @@ -663,13 +670,13 @@ func WaitForReconcileToSync(ctx context.Context, key types.NamespacedName, k8sCl }, testTimeout, TestInterval).Should(BeNumerically("==", beforeGeneration)) } -func useDockerCredentials() bool { +func UseDockerCredentials() bool { return os.Getenv(dockerUsernameEnvVar) != "" && os.Getenv(dockerPasswordEnvVar) != "" && os.Getenv(dockerUsernameEnvVar) != "none" && os.Getenv(dockerPasswordEnvVar) != "none" } func CreateDockerRegredSecret(ctx context.Context, namespace corev1.Namespace, k8sClient client.Client) { - if !useDockerCredentials() { + if !UseDockerCredentials() { return } diff --git a/controllers/suite/resources/humioresources_controller_test.go b/controllers/suite/resources/humioresources_controller_test.go index 56b213cd9..9b9f2fe80 100644 --- a/controllers/suite/resources/humioresources_controller_test.go +++ b/controllers/suite/resources/humioresources_controller_test.go @@ -662,7 +662,7 @@ var _ = Describe("Humio Resources Controllers", func() { Namespace: clusterKey.Namespace, } protocol := "http" - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { + if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" && helpers.UseCertManager() { protocol = "https" } @@ -1775,6 +1775,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(expectedSecretValue)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: OpsGenieProperties: Should support referencing secrets", func() { @@ -1838,6 +1845,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(expectedSecretValue)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: OpsGenieProperties: Should support direct genie key", func() { @@ -1883,6 +1897,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(expectedSecretValue)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: VictorOpsProperties: Should support referencing secrets", func() { @@ -1946,6 +1967,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(expectedSecretValue)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: VictorOpsProperties: Should support direct notify url", func() { @@ -1991,6 +2019,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(expectedSecretValue)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: SlackPostMessageProperties: Should support referencing secrets", func() { @@ -2057,6 +2092,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(expectedSecretValue)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: SlackPostMessageProperties: Should support direct api token", func() { @@ -2104,6 +2146,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(toCreateAction.Spec.SlackPostMessageProperties.ApiToken)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: SlackProperties: Should support referencing secrets", func() { @@ -2169,6 +2218,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(expectedSecretValue)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: SlackProperties: Should support direct url", func() { @@ -2216,6 +2272,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(toCreateAction.Spec.SlackProperties.Url)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: PagerDutyProperties: Should support referencing secrets", func() { @@ -2279,6 +2342,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(expectedSecretValue)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: PagerDutyProperties: Should support direct api token", func() { @@ -2324,6 +2394,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(toCreateAction.Spec.PagerDutyProperties.RoutingKey)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: WebhookProperties: Should support direct url", func() { @@ -2370,6 +2447,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(toCreateAction.Spec.WebhookProperties.Url)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: WebhookProperties: Should support referencing secret url", func() { @@ -2434,6 +2518,13 @@ var _ = Describe("Humio Resources Controllers", func() { apiToken, found := kubernetes.GetSecretForHa(toCreateAction) Expect(found).To(BeTrue()) Expect(apiToken).To(Equal(expectedSecretValue)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: WebhookProperties: Should support direct url and headers", func() { @@ -2496,6 +2587,13 @@ var _ = Describe("Humio Resources Controllers", func() { allHeaders, found := kubernetes.GetFullSetOfMergedWebhookheaders(toCreateAction) Expect(found).To(BeTrue()) Expect(allHeaders).To(HaveKeyWithValue(nonsensitiveHeaderKey, nonsensitiveHeaderValue)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: WebhookProperties: Should support direct url and mixed headers", func() { ctx := context.Background() @@ -2588,6 +2686,13 @@ var _ = Describe("Humio Resources Controllers", func() { Expect(found).To(BeTrue()) Expect(allHeaders).To(HaveKeyWithValue(headerKey1, sensitiveHeaderValue1)) Expect(allHeaders).To(HaveKeyWithValue(headerKey2, nonsensitiveHeaderValue2)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) It("HumioAction: WebhookProperties: Should support direct url and secret headers", func() { ctx := context.Background() @@ -2670,6 +2775,13 @@ var _ = Describe("Humio Resources Controllers", func() { allHeaders, found := kubernetes.GetFullSetOfMergedWebhookheaders(toCreateAction) Expect(found).To(BeTrue()) Expect(allHeaders).To(HaveKeyWithValue(headerKey, sensitiveHeaderValue)) + + suite.UsingClusterBy(clusterKey.Name, "HumioAction: Successfully deleting it") + Expect(k8sClient.Delete(ctx, fetchedAction)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, key, fetchedAction) + return k8serrors.IsNotFound(err) + }, testTimeout, suite.TestInterval).Should(BeTrue()) }) }) diff --git a/controllers/suite/resources/suite_test.go b/controllers/suite/resources/suite_test.go index 6b296d7b6..e7db65c1f 100644 --- a/controllers/suite/resources/suite_test.go +++ b/controllers/suite/resources/suite_test.go @@ -27,6 +27,7 @@ import ( "time" "github.com/humio/humio-operator/pkg/kubernetes" + k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/rest" "github.com/humio/humio-operator/controllers" @@ -89,8 +90,6 @@ var _ = BeforeSuite(func() { log = zapr.NewLogger(zapLog) logf.SetLogger(log) - Expect(os.Getenv("HUMIO_E2E_LICENSE")).NotTo(BeEmpty()) - By("bootstrapping test environment") useExistingCluster := true clusterKey = types.NamespacedName{ @@ -103,11 +102,12 @@ var _ = BeforeSuite(func() { testEnv = &envtest.Environment{ UseExistingCluster: &useExistingCluster, } - if os.Getenv("DUMMY_LOGSCALE_IMAGE") == "true" { humioClient = humio.NewMockClient() } else { humioClient = humio.NewClient(log, "") + By("Verifying we have a valid license, as tests will require starting up real LogScale containers") + Expect(os.Getenv("HUMIO_E2E_LICENSE")).NotTo(BeEmpty()) } } else { @@ -256,8 +256,7 @@ var _ = BeforeSuite(func() { Name: clusterKey.Namespace, }, } - err = k8sClient.Create(context.TODO(), &testNamespace) - Expect(err).ToNot(HaveOccurred()) + Expect(k8sClient.Create(context.TODO(), &testNamespace)).ToNot(HaveOccurred()) suite.CreateDockerRegredSecret(context.TODO(), testNamespace, k8sClient) @@ -347,24 +346,49 @@ var _ = BeforeSuite(func() { var _ = AfterSuite(func() { if k8sClient != nil { + Expect(k8sClient.Delete(context.TODO(), &corev1alpha1.HumioRepository{ + ObjectMeta: metav1.ObjectMeta{ + Name: testRepo.Name, + Namespace: testRepo.Namespace, + }, + })).To(Succeed()) + Expect(k8sClient.Delete(context.TODO(), &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: testService1.Name, + Namespace: testService1.Namespace, + }, + })).To(Succeed()) + Expect(k8sClient.Delete(context.TODO(), &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: testService2.Name, + Namespace: testService2.Namespace, + }, + })).To(Succeed()) + suite.UsingClusterBy(clusterKey.Name, "HumioCluster: Confirming resource generation wasn't updated excessively") Expect(k8sClient.Get(context.Background(), clusterKey, cluster)).Should(Succeed()) Expect(cluster.GetGeneration()).ShouldNot(BeNumerically(">", 100)) suite.CleanupCluster(context.TODO(), k8sClient, cluster) - By(fmt.Sprintf("Removing regcred secret for namespace: %s", testNamespace.Name)) - _ = k8sClient.Delete(context.TODO(), &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: suite.DockerRegistryCredentialsSecretName, - Namespace: clusterKey.Namespace, - }, - }) + if suite.UseDockerCredentials() { + By(fmt.Sprintf("Removing regcred secret for namespace: %s", testNamespace.Name)) + Expect(k8sClient.Delete(context.TODO(), &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: suite.DockerRegistryCredentialsSecretName, + Namespace: clusterKey.Namespace, + }, + })).To(Succeed()) + } - if testNamespace.ObjectMeta.Name != "" { + if testNamespace.ObjectMeta.Name != "" && os.Getenv("PRESERVE_KIND_CLUSTER") == "true" { By(fmt.Sprintf("Removing test namespace: %s", clusterKey.Namespace)) err := k8sClient.Delete(context.TODO(), &testNamespace) Expect(err).ToNot(HaveOccurred()) + + Eventually(func() bool { + return k8serrors.IsNotFound(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterKey.Namespace}, &testNamespace)) + }, testTimeout, suite.TestInterval).Should(BeTrue()) } } diff --git a/hack/functions.sh b/hack/functions.sh index 6528a18f3..027ae544c 100644 --- a/hack/functions.sh +++ b/hack/functions.sh @@ -16,6 +16,15 @@ PATH=$bin_dir/goinstall/bin:$bin_dir:/usr/local/go/bin:$PATH GOBIN=$bin_dir start_kind_cluster() { + if $kind get clusters | grep kind ; then + if ! $kubectl get daemonset -n kube-system kindnet ; then + echo "Cluster unavailable or not using a kind cluster. Only kind clusters are supported!" + exit 1 + fi + + return + fi + $kind create cluster --name kind --config hack/kind-config.yaml --image $kindest_node_image_multiplatform_amd64_arm64 --wait 300s sleep 5 @@ -29,10 +38,19 @@ start_kind_cluster() { } cleanup_kind_cluster() { - $kind delete cluster --name kind + if [[ $preserve_kind_cluster == "true" ]]; then + $kubectl delete --grace-period=1 pod test-pod + $kubectl delete -k config/crd/ + else + $kind delete cluster --name kind + fi } install_kind() { + if [ -f $kind ]; then + $kind version | grep -E "^kind v${kind_version}" && return + fi + if [ $(uname -o) = Darwin ]; then # For Intel Macs [ $(uname -m) = x86_64 ] && curl -Lo $kind https://kind.sigs.k8s.io/dl/v${kind_version}/kind-darwin-amd64 @@ -50,6 +68,10 @@ install_kind() { } install_kubectl() { + if [ -f $kubectl ]; then + $kubectl version --client | grep "GitVersion:\"v${kubectl_version}\"" && return + fi + if [ $(uname -o) = Darwin ]; then # For Intel Macs [ $(uname -m) = x86_64 ] && curl -Lo $kubectl https://dl.k8s.io/release/v${kubectl_version}/bin/darwin/amd64/kubectl @@ -67,6 +89,10 @@ install_kubectl() { } install_helm() { + if [ -f $helm ]; then + $helm version --short | grep -E "^v${helm_version}" && return + fi + if [ $(uname -o) = Darwin ]; then # For Intel Macs [ $(uname -m) = x86_64 ] && curl -Lo $helm.tar.gz https://get.helm.sh/helm-v${helm_version}-darwin-amd64.tar.gz && tar -zxvf $helm.tar.gz -C $bin_dir && mv $bin_dir/darwin-amd64/helm $helm && rm -r $bin_dir/darwin-amd64 @@ -84,23 +110,6 @@ install_helm() { $helm version } -install_go() { - if [ $(uname -o) = Darwin ]; then - # For Intel Macs - [ $(uname -m) = x86_64 ] && curl -Lo $go.tar.gz https://dl.google.com/go/go${go_version}.darwin-amd64.tar.gz && tar -zxvf $go.tar.gz -C $bin_dir && mv $bin_dir/go $bin_dir/goinstall && ln -s $bin_dir/goinstall/bin/go $go - # For M1 / ARM Macs - [ $(uname -m) = arm64 ] && curl -Lo $go.tar.gz https://dl.google.com/go/go${go_version}.darwin-arm64.tar.gz && tar -zxvf $go.tar.gz -C $bin_dir && mv $bin_dir/go $bin_dir/goinstall && ln -s $bin_dir/goinstall/bin/go $go - else - echo "Assuming Linux" - # For AMD64 / x86_64 - [ $(uname -m) = x86_64 ] && curl -Lo $go.tar.gz https://dl.google.com/go/go${go_version}.linux-amd64.tar.gz && tar -zxvf $go.tar.gz -C $bin_dir && mv $bin_dir/go $bin_dir/goinstall && ln -s $bin_dir/goinstall/bin/go $go - # For ARM64 - [ $(uname -m) = aarch64 ] && curl -Lo $go.tar.gz https://dl.google.com/go/go${go_version}.linux-arm64.tar.gz && tar -zxvf $go.tar.gz -C $bin_dir && mv $bin_dir/go $bin_dir/goinstall && ln -s $bin_dir/goinstall/bin/go $go - fi - rm $go.tar.gz - $go version -} - install_ginkgo() { go get github.com/onsi/ginkgo/v2/ginkgo go install github.com/onsi/ginkgo/v2/ginkgo @@ -146,6 +155,8 @@ preload_container_images() { } helm_install_shippers() { + $helm get metadata log-shipper && return + # Install components to get observability during execution of tests if [[ $humio_hostname != "none" ]] && [[ $humio_ingest_token != "none" ]]; then e2eFilterTag=$(cat <