Skip to content

Commit

Permalink
Introduce USE_CERTMANAGER and PRESERVE_KIND_CLUSTER to kind test exec…
Browse files Browse the repository at this point in the history 
…ution and skip downloading binaries if already present

USE_CERTMANAGER (default="true") can be set to "false" to disable
cert-manager installation to speed up test execution.

PRESERVE_KIND_CLUSTER (default="false") can be set to "true" to keep the
kind cluster around after test execution so that the consecutive runs
will reuse the kind cluster and existing helm installs.
This also fixes a bunch of test cases where tests didn't properly clean
up, causing conflicts on consecutive test runs.
  • Loading branch information
@SaaldjorMike
SaaldjorMike committed Sep 24, 2024
1 parent 3d3fc30 commit 1369975
Show file tree
Hide file tree
Showing 9 changed files with 277 additions and 82 deletions.
120 changes: 69 additions & 51 deletions controllers/suite/clusters/humiocluster_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,7 @@ var _ = Describe("HumioCluster Controller", func() {
Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2"))
}

if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
if helpers.TLSEnabled(&updatedHumioCluster) {
suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
}
Expand Down Expand Up @@ -462,7 +462,7 @@ var _ = Describe("HumioCluster Controller", func() {
Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2"))
}

if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
if helpers.TLSEnabled(&updatedHumioCluster) {
suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
}
Expand Down Expand Up @@ -556,7 +556,7 @@ var _ = Describe("HumioCluster Controller", func() {
Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2"))
}

if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
if helpers.TLSEnabled(&updatedHumioCluster) {
suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
}
Expand Down Expand Up @@ -631,7 +631,7 @@ var _ = Describe("HumioCluster Controller", func() {
Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2"))
}

if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
if helpers.TLSEnabled(&updatedHumioCluster) {
suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
}
Expand Down Expand Up @@ -707,16 +707,16 @@ var _ = Describe("HumioCluster Controller", func() {
Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2"))
}

if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
if helpers.TLSEnabled(&updatedHumioCluster) {
suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
}
})
})

Context("Humio Cluster Update EXTERNAL_URL", Label("envtest", "dummy", "real"), func() {
Context("Humio Cluster Update EXTERNAL_URL", Label("dummy", "real"), func() {
It("Update should correctly replace pods to use the new EXTERNAL_URL in a non-rolling fashion", func() {
if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
if helpers.UseCertManager() {
key := types.NamespacedName{
Name: "humiocluster-update-ext-url",
Namespace: testProcessNamespace,
Expand Down Expand Up @@ -957,7 +957,7 @@ var _ = Describe("HumioCluster Controller", func() {
Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2"))
}

if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
if helpers.TLSEnabled(&updatedHumioCluster) {
suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
}
Expand Down Expand Up @@ -1063,7 +1063,7 @@ var _ = Describe("HumioCluster Controller", func() {
Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2"))
}

if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
if helpers.TLSEnabled(&updatedHumioCluster) {
suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
}
Expand Down Expand Up @@ -1177,7 +1177,7 @@ var _ = Describe("HumioCluster Controller", func() {
Expect(pod.Annotations[controllers.PodRevisionAnnotation]).To(Equal("3"))
}

if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
if helpers.TLSEnabled(&updatedHumioCluster) {
suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
}
Expand Down Expand Up @@ -1242,7 +1242,7 @@ var _ = Describe("HumioCluster Controller", func() {

updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels())

if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
if helpers.TLSEnabled(&updatedHumioCluster) {
suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
}
Expand Down Expand Up @@ -1314,7 +1314,7 @@ var _ = Describe("HumioCluster Controller", func() {

updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels())

if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
if helpers.TLSEnabled(&updatedHumioCluster) {
suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
}
Expand Down Expand Up @@ -1444,7 +1444,7 @@ var _ = Describe("HumioCluster Controller", func() {
}, testTimeout, suite.TestInterval).Should(BeTrue())

updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, updatedHumioCluster.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetPodLabels())
if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
if helpers.TLSEnabled(&updatedHumioCluster) {
suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
}
Expand Down Expand Up @@ -1666,7 +1666,7 @@ var _ = Describe("HumioCluster Controller", func() {
}, testTimeout, suite.TestInterval).Should(BeTrue())

updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, updatedHumioCluster.Namespace, mainNodePoolManager.GetPodLabels())
if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
if helpers.TLSEnabled(&updatedHumioCluster) {
suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
}
Expand Down Expand Up @@ -1767,7 +1767,7 @@ var _ = Describe("HumioCluster Controller", func() {
}, testTimeout, suite.TestInterval).Should(BeTrue())

updatedClusterPods, _ = kubernetes.ListPods(ctx, k8sClient, updatedHumioCluster.Namespace, additionalNodePoolManager.GetPodLabels())
if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
if helpers.TLSEnabled(&updatedHumioCluster) {
suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed")
Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods)))
}
Expand Down Expand Up @@ -3337,20 +3337,24 @@ var _ = Describe("HumioCluster Controller", func() {
defer suite.CleanupCluster(ctx, k8sClient, toCreate)

initialExpectedVolumesCount := 5
initialExpectedVolumeMountsCount := 4
initialExpectedHumioContainerVolumeMountsCount := 4

if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
// if we run on a real cluster we have TLS enabled (using 2 volumes),
// and k8s will automatically inject a service account token adding one more
initialExpectedVolumesCount += 3
initialExpectedVolumeMountsCount += 2
// k8s will automatically inject a service account token
initialExpectedVolumesCount += 1 // kube-api-access-<ID>
initialExpectedHumioContainerVolumeMountsCount += 1 // kube-api-access-<ID>

if helpers.UseCertManager() {
initialExpectedVolumesCount += 1 // tls-cert
initialExpectedHumioContainerVolumeMountsCount += 1 // tls-cert
}
}

clusterPods, _ := kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels())
for _, pod := range clusterPods {
Expect(pod.Spec.Volumes).To(HaveLen(initialExpectedVolumesCount))
humioIdx, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName)
Expect(pod.Spec.Containers[humioIdx].VolumeMounts).To(HaveLen(initialExpectedVolumeMountsCount))
Expect(pod.Spec.Containers[humioIdx].VolumeMounts).To(HaveLen(initialExpectedHumioContainerVolumeMountsCount))
}

suite.UsingClusterBy(key.Name, "Adding additional volumes")
Expand Down Expand Up @@ -3395,7 +3399,7 @@ var _ = Describe("HumioCluster Controller", func() {
return pod.Spec.Containers[humioIdx].VolumeMounts
}
return []corev1.VolumeMount{}
}, testTimeout, suite.TestInterval).Should(HaveLen(initialExpectedVolumeMountsCount + 1))
}, testTimeout, suite.TestInterval).Should(HaveLen(initialExpectedHumioContainerVolumeMountsCount + 1))
clusterPods, _ = kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels())
for _, pod := range clusterPods {
Expect(pod.Spec.Volumes).Should(ContainElement(extraVolume))
Expand All @@ -3416,7 +3420,7 @@ var _ = Describe("HumioCluster Controller", func() {
Type: humiov1alpha1.HumioClusterUpdateStrategyRollingUpdate,
}
protocol := "http"
if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
if helpers.TLSEnabled(toCreate) {
protocol = "https"
}

Expand Down Expand Up @@ -3815,39 +3819,40 @@ var _ = Describe("HumioCluster Controller", func() {
})
})

Context("Humio Cluster with additional hostnames for TLS", Label("envtest", "dummy", "real"), func() {
Context("Humio Cluster with additional hostnames for TLS", Label("dummy", "real"), func() {
It("Creating cluster with additional hostnames for TLS", func() {
if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
key := types.NamespacedName{
Name: "humiocluster-tls-additional-hostnames",
Namespace: testProcessNamespace,
}
toCreate := suite.ConstructBasicSingleNodeHumioCluster(key, true)
toCreate.Spec.TLS = &humiov1alpha1.HumioClusterTLSSpec{
Enabled: helpers.BoolPtr(true),
ExtraHostnames: []string{
"something.additional",
"yet.another.something.additional",
},
}
key := types.NamespacedName{
Name: "humiocluster-tls-additional-hostnames",
Namespace: testProcessNamespace,
}
toCreate := suite.ConstructBasicSingleNodeHumioCluster(key, true)
if !helpers.TLSEnabled(toCreate) {
return
}
toCreate.Spec.TLS = &humiov1alpha1.HumioClusterTLSSpec{
Enabled: helpers.BoolPtr(true),
ExtraHostnames: []string{
"something.additional",
"yet.another.something.additional",
},
}

suite.UsingClusterBy(key.Name, "Creating the cluster successfully")
ctx := context.Background()
suite.CreateAndBootstrapCluster(ctx, k8sClient, testHumioClient, toCreate, true, humiov1alpha1.HumioClusterStateRunning, testTimeout)
defer suite.CleanupCluster(ctx, k8sClient, toCreate)
suite.UsingClusterBy(key.Name, "Creating the cluster successfully")
ctx := context.Background()
suite.CreateAndBootstrapCluster(ctx, k8sClient, testHumioClient, toCreate, true, humiov1alpha1.HumioClusterStateRunning, testTimeout)
defer suite.CleanupCluster(ctx, k8sClient, toCreate)

suite.UsingClusterBy(key.Name, "Confirming certificate objects contain the additional hostnames")
suite.UsingClusterBy(key.Name, "Confirming certificate objects contain the additional hostnames")

Eventually(func() ([]cmapi.Certificate, error) {
return kubernetes.ListCertificates(ctx, k8sClient, toCreate.Namespace, kubernetes.MatchingLabelsForHumio(toCreate.Name))
}, testTimeout, suite.TestInterval).Should(HaveLen(2))
Eventually(func() ([]cmapi.Certificate, error) {
return kubernetes.ListCertificates(ctx, k8sClient, toCreate.Namespace, kubernetes.MatchingLabelsForHumio(toCreate.Name))
}, testTimeout, suite.TestInterval).Should(HaveLen(2))

var certificates []cmapi.Certificate
certificates, err = kubernetes.ListCertificates(ctx, k8sClient, toCreate.Namespace, kubernetes.MatchingLabelsForHumio(toCreate.Name))
Expect(err).To(Succeed())
for _, certificate := range certificates {
Expect(certificate.Spec.DNSNames).Should(ContainElements(toCreate.Spec.TLS.ExtraHostnames))
}
var certificates []cmapi.Certificate
certificates, err = kubernetes.ListCertificates(ctx, k8sClient, toCreate.Namespace, kubernetes.MatchingLabelsForHumio(toCreate.Name))
Expect(err).To(Succeed())
for _, certificate := range certificates {
Expect(certificate.Spec.DNSNames).Should(ContainElements(toCreate.Spec.TLS.ExtraHostnames))
}
})
})
Expand Down Expand Up @@ -4362,6 +4367,19 @@ var _ = Describe("HumioCluster Controller", func() {
for _, pod := range clusterPods {
Expect(pod.Spec.PriorityClassName).To(Equal(toCreate.Spec.PriorityClassName))
}

Expect(k8sClient.Delete(context.TODO(), priorityClass)).To(Succeed())

Eventually(func() bool {
return k8serrors.IsNotFound(k8sClient.Get(
context.TODO(),
types.NamespacedName{
Namespace: priorityClass.Namespace,
Name: priorityClass.Name,
},
priorityClass),
)
}, testTimeout, suite.TestInterval).Should(BeTrue())
})
})

Expand Down
6 changes: 3 additions & 3 deletions controllers/suite/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,7 @@ func ConstructBasicNodeSpecForHumioCluster(key types.NamespacedName) humiov1alph
}
}

if useDockerCredentials() {
if UseDockerCredentials() {
nodeSpec.ImagePullSecrets = []corev1.LocalObjectReference{
{Name: DockerRegistryCredentialsSecretName},
}
Expand Down Expand Up @@ -663,13 +663,13 @@ func WaitForReconcileToSync(ctx context.Context, key types.NamespacedName, k8sCl
}, testTimeout, TestInterval).Should(BeNumerically("==", beforeGeneration))
}

func useDockerCredentials() bool {
func UseDockerCredentials() bool {
return os.Getenv(dockerUsernameEnvVar) != "" && os.Getenv(dockerPasswordEnvVar) != "" &&
os.Getenv(dockerUsernameEnvVar) != "none" && os.Getenv(dockerPasswordEnvVar) != "none"
}

func CreateDockerRegredSecret(ctx context.Context, namespace corev1.Namespace, k8sClient client.Client) {
if !useDockerCredentials() {
if !UseDockerCredentials() {
return
}

Expand Down
Loading

0 comments on commit 1369975

Please sign in to comment.