diff --git a/.github/workflows/e2e_main.yaml b/.github/workflows/e2e_main.yaml index a3eee2f06..3f588b9c0 100644 --- a/.github/workflows/e2e_main.yaml +++ b/.github/workflows/e2e_main.yaml @@ -17,7 +17,6 @@ name: e2e env: CI_COMMIT_REF_NAME: ${{ github.ref_name }} GO_VERSION: "1.22.7" - GINKGO_VERSION: "2.20.0" on: workflow_dispatch: @@ -46,7 +45,8 @@ jobs: - name: Install ginkgo run: | echo "Install ginkgo" - go install "github.com/onsi/ginkgo/v2/ginkgo@v${{ env.GINKGO_VERSION }}" + GINKGO_VERSION=$(go list -f '{{.Version}}' -m github.com/onsi/ginkgo/v2) + go install "github.com/onsi/ginkgo/v2/ginkgo@v${GINKGO_VERSION}" - name: Install Deckhouse-cli run: | @@ -69,4 +69,4 @@ jobs: - name: Run E2E working-directory: ./tests/e2e/ run: | - task run -v + task run:ci -v diff --git a/tests/e2e/README.md b/tests/e2e/README.md index aa921a776..f84344c43 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -1,46 +1,82 @@ # Integration tests -## Running integration tests +## Prerequisites -Integration tests require a running deckhouse cluster with the virtualization module installed. -Once you have a running deckhouse cluster, you can use config, environment variables or flags to -point the tests to the cluster. -You can override the config file using the env var ```E2E_CONFIG```. -(default config - ```default_config.yaml```) +### Utilities -You must also have a default storage class declared. -Mark a StorageClass as default: -```yaml +Some utilities should be installed to run e2e tests: + +- task (https://taskfile.dev) +- kubectl (https://kubernetes.io/docs/tasks/tools/#kubectl) +- d8 (https://github.com/deckhouse/deckhouse-cli/releases) +- ginkgo + - Download from https://github.com/onsi/ginkgo + - Or just run `go install github.com/onsi/ginkgo/v2/ginkgo@$(go list -f '{{.Version}}' -m github.com/onsi/ginkgo/v2)` + +### Deckhouse cluster + +Integration tests require a running Deckhouse cluster with the virtualization module installed. + +### Default StorageClass + +Default storage class should be set in the cluster. Annotate a StorageClass with +storageclass.kubernetes.io/is-default-class to mark it as the default: + +```bash + +$ kubectl annotate storageclass linstor-thin-r1 storageclass.kubernetes.io/is-default-class=true + +$ kubectl get storageclass linstor-thin-r1 -o yaml | less +... metadata: annotations: storageclass.kubernetes.io/is-default-class: "true" -``` -Example: -```bash -kubectl patch storageclasses.storage.k8s.io linstor-thin-r1 --type=merge --patch='{"metadata":{"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}' +... ``` -### Local running -With "$HOME/.kube/config" by default and current cluster context: -```bash -task run_local -``` +### E2E configuration + +Temp directories, prefixes, images and ssh settings can be set in the +YAML configuration file. +Path to custom file can be set with the `E2E_CONFIG` environment variable. +Default config file is `default_config.yaml` in `tests/e2e` directory. + +You can override config field with environment variables. Use E2E_ prefix and join uppercased fields with _ (underscore). + +For example, to override curl image, set `E2E_HELPERIMAGES_CURLIMAGE` environment variable. + +### Cluster connection settings + +Connection settings priority from highest to lowest: + +- Token and endpoint in E2E_CLUSTERTRANSPORT_TOKEN and E2E_CLUSTERTRANSPORT_ENDPOINT envs. +- Token and endpoint in clusterTransport field in e2e config file. +- A path to kubeconfig file in clusterTransport.kubeConfig field in e2e config file. +- A path to kubeconfig file in KUBECONFIG env. +- A path to kubeconfig file in `~/.kube/config`. + + +## Run tests from developer machine + +Setup cluster connection in "$HOME/.kube/config" or by [switch](https://github.com/danielfoehrKn/kubeswitch)ing the `KUBECONFIG` env and run tests: -### Configuration -To override a configuration option, create an environment variable named ```E2E_variable``` where variable is the name of the configuration option and the _ (underscore) represents indention levels. -For example, you can configure the ```token``` of the ```kubectl``` and ```virtctl```: ```bash -clusterTransport: - token: "your token" +task run ``` -To override this value, set an environment variable like this: + +### Debugging options + +- Use FOCUS env to run one test. +- Use STOP_ON_FAILURE=yes env to stop tests on first failure without cleanup. + +For example, run only "Complex text" without cleanup on failure: ```bash -export E2E_CLUSTERTRANSPORT_TOKEN="your token" +FOCUS="Complex test" STOP_ON_FAILURE=yes task run ``` -### Run +## Run tests in CI ```bash -task run +task run:ci ``` ### Example @@ -86,4 +122,4 @@ export E2E_CLUSTERTRANSPORT_CERTIFICATEAUTHORITY="$PWD/ca.crt" export E2E_CLUSTERTRANSPORT_INSECURETLS="false" task run -``` \ No newline at end of file +``` diff --git a/tests/e2e/Taskfile.yaml b/tests/e2e/Taskfile.yaml index c85e0b1e5..3bd952457 100644 --- a/tests/e2e/Taskfile.yaml +++ b/tests/e2e/Taskfile.yaml @@ -3,41 +3,55 @@ version: "3" silent: true vars: - GINKGO_VERSION: "2.20.0" + GINKGO_VERSION: + # Should hide curly brackets from task templater. + sh: go list -f '{{`{{.Version}}`}}' -m github.com/onsi/ginkgo/v2 VERSION: "v1.0.0" tasks: copy: cmds: - | - rm --force --recursive /tmp/testdata + if [ "{{OS}}" == "darwin" ]; then + rm -rf /tmp/testdata + else + rm --force --recursive /tmp/testdata + fi cp -a testdata /tmp/testdata ginkgo: cmds: - | + if ! which ginkgo >/dev/null ; then + echo "Ginkgo not found or not in the PATH. Install from github or run go install github.com/onsi/ginkgo/v2/ginkgo@{{ .GINKGO_VERSION }}" + exit 1 + fi v=($(ginkgo version 2>/dev/null)) - if [ "${v[2]}" != "{{ .GINKGO_VERSION }}" ]; then - go install "github.com/onsi/ginkgo/v2/ginkgo@v{{ .GINKGO_VERSION }}" ; + if [ "v${v[2]}" != "{{ .GINKGO_VERSION }}" ]; then + echo "Ginkgo version mismatch: {{ .GINKGO_VERSION }} is required. Got ${v[2]} in PATH." + exit 2 fi - run: - desc: "Run e2e tests" - deps: - - copy + kubectl: cmds: - | - ginkgo \ - --skip-file vm_test.go \ - --skip-file vm_label_annotation_test.go \ - --skip-file ipam_test.go \ - --skip-file disks_test.go \ - -v - - run_local: - desc: "Run locally e2e tests" + if ! which kubectl >/dev/null ; then + echo "kubectl not found or not in PATH" + exit 1 + fi + d8: + cmds: + - | + if ! which d8 >/dev/null ; then + echo "d8 not found or not in PATH. Install from https://github.com/deckhouse/deckhouse-cli/releases" + exit 1 + fi + run:ci: + desc: "Separate task to run e2e tests in the CI environment" deps: - copy - ginkgo + - kubectl + - d8 cmds: - | ginkgo \ @@ -47,16 +61,21 @@ tasks: --skip-file disks_test.go \ -v - run_one: - desc: "Run one test or group" + run: + desc: "Run e2e tests" deps: - copy - ginkgo + - kubectl + - d8 cmds: - | - {{if .FOCUS }} - ginkgo --focus "{{ .FOCUS }}" -v - {{else}} - echo "Specify test to run" - echo 'Example: FOCUS="Label and Annotation" task run_one' - {{end}} + ginkgo -v \ + {{if .FOCUS -}} + --focus "{{ .FOCUS }}" + {{ else -}} + --skip-file vm_test.go \ + --skip-file vm_label_annotation_test.go \ + --skip-file ipam_test.go \ + --skip-file disks_test.go + {{end}} diff --git a/tests/e2e/complex_test.go b/tests/e2e/complex_test.go index 2764ba52b..6df73dd11 100644 --- a/tests/e2e/complex_test.go +++ b/tests/e2e/complex_test.go @@ -24,6 +24,7 @@ import ( . "github.com/onsi/gomega" virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/tests/e2e/ginkgoutil" kc "github.com/deckhouse/virtualization/tests/e2e/kubectl" ) @@ -58,7 +59,7 @@ func AssignIPToVMIP(name string) error { return nil } -var _ = Describe("Complex test", Ordered, ContinueOnFailure, func() { +var _ = Describe("Complex test", ginkgoutil.CommonE2ETestDecorators(), func() { var ( testCaseLabel = map[string]string{"testcase": "complex-test"} hasNoConsumerLabel = map[string]string{"hasNoConsumer": "complex-test"} diff --git a/tests/e2e/disks_test.go b/tests/e2e/disks_test.go index c85d77ba4..550c457a7 100644 --- a/tests/e2e/disks_test.go +++ b/tests/e2e/disks_test.go @@ -23,6 +23,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/deckhouse/virtualization/tests/e2e/ginkgoutil" kc "github.com/deckhouse/virtualization/tests/e2e/kubectl" ) @@ -42,7 +43,7 @@ func vdPath(file string) string { return path.Join(conf.Disks.VdTestDataDir, file) } -var _ = Describe("Disks", func() { +var _ = Describe("Disks", ginkgoutil.CommonE2ETestDecorators(), func() { CheckProgress := func(filepath string) { GinkgoHelper() out := "jsonpath={.status.progress}" @@ -71,7 +72,7 @@ var _ = Describe("Disks", func() { ItChekStatusPhaseFromFile(filepath, PhaseReady) } - Context("CVI", Ordered, ContinueOnFailure, func() { + Context("CVI", func() { AfterAll(func() { By("Removing resources for cvi tests") kubectl.Delete(conf.Disks.CviTestDataDir, kc.DeleteOptions{}) @@ -112,7 +113,7 @@ var _ = Describe("Disks", func() { CheckProgress(filepath) }) }) - Context("VI", Ordered, ContinueOnFailure, func() { + Context("VI", func() { AfterAll(func() { By("Removing resources for vi tests") kubectl.Delete(conf.Disks.ViTestDataDir, kc.DeleteOptions{}) @@ -153,7 +154,7 @@ var _ = Describe("Disks", func() { CheckProgress(filepath) }) }) - Context("VD", Ordered, ContinueOnFailure, func() { + Context("VD", func() { AfterAll(func() { By("Removing resources for vd tests") kubectl.Delete(conf.Disks.VdTestDataDir, kc.DeleteOptions{}) diff --git a/tests/e2e/ginkgoutil/decorators.go b/tests/e2e/ginkgoutil/decorators.go new file mode 100644 index 000000000..023e51f4f --- /dev/null +++ b/tests/e2e/ginkgoutil/decorators.go @@ -0,0 +1,86 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ginkgoutil + +import ( + "os" + + "github.com/onsi/ginkgo/v2" +) + +// Ginkgo decorators helpers: +// - Common decorators for e2e: Ordered and ContinueOnFailure. +// - ContinueOnFailure decorator is switchable and can be disabled with STOP_ON_FAILURE=yes env. +// +// A quote from Ginkgo documentation: +// Moreover, Ginkgo also supports passing in arbitrarily nested slices of decorators. +// Ginkgo will unroll these slices and process the flattened list. This makes it easier +// to pass around groups of decorators. For example, this is valid: +// markFlaky := []interface{}{Label("flaky"), FlakeAttempts(3)} +// var _ = Describe("a bunch of flaky controller tests", markFlaky, Label("controller"), func() { +// ... +// } +// The resulting tests will be decorated with FlakeAttempts(3) and the two labels flaky and controller. +// +// This helper uses this "flattening" feature, so DecoratorsFromEnv implements +// dynamic list of switchable decorators by returning an array of decorators. + +type EnvSwitchable interface { + Decorator() interface{} +} + +func DecoratorsFromEnv(decorators ...interface{}) []interface{} { + out := make([]interface{}, 0) + + for _, decorator := range decorators { + switch v := decorator.(type) { + case EnvSwitchable: + gdeco := v.Decorator() + if gdeco != nil { + out = append(out, gdeco) + } + default: + out = append(out, decorator) + } + } + + return out +} + +const StopOnFailureEnv = "STOP_ON_FAILURE" + +type FailureBehaviourEnvSwitcher struct{} + +func (f FailureBehaviourEnvSwitcher) Decorator() interface{} { + if !f.IsStopOnFailure() { + return ginkgo.ContinueOnFailure + } + return nil +} + +// IsStopOnFailure returns true if Stop on error is enabled. +func (f FailureBehaviourEnvSwitcher) IsStopOnFailure() bool { + return os.Getenv(StopOnFailureEnv) == "yes" +} + +// CommonE2ETestDecorators returns common decorators for e2e tests: Ordered and ContinueOnFailure switchable with env. +func CommonE2ETestDecorators() []interface{} { + return DecoratorsFromEnv( + ginkgo.Ordered, + FailureBehaviourEnvSwitcher{}, + ) +} diff --git a/tests/e2e/ipam_test.go b/tests/e2e/ipam_test.go index c75859348..42130ee23 100644 --- a/tests/e2e/ipam_test.go +++ b/tests/e2e/ipam_test.go @@ -22,6 +22,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/deckhouse/virtualization/tests/e2e/ginkgoutil" kc "github.com/deckhouse/virtualization/tests/e2e/kubectl" ) @@ -30,7 +31,7 @@ func ipamPath(file string) string { } var _ = Describe("Ipam", func() { - Context("VirtualMachineIPAddressClaim", Ordered, ContinueOnFailure, func() { + Context("VirtualMachineIPAddressClaim", ginkgoutil.CommonE2ETestDecorators(), func() { AfterAll(func() { By("Removing resources for vmip tests") kubectl.Delete(conf.Ipam.TestDataDir, kc.DeleteOptions{}) diff --git a/tests/e2e/sizing_policy_test.go b/tests/e2e/sizing_policy_test.go index 0ceeebcce..8a99d3936 100644 --- a/tests/e2e/sizing_policy_test.go +++ b/tests/e2e/sizing_policy_test.go @@ -26,6 +26,7 @@ import ( . "github.com/onsi/gomega" virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/tests/e2e/ginkgoutil" . "github.com/deckhouse/virtualization/tests/e2e/helper" kc "github.com/deckhouse/virtualization/tests/e2e/kubectl" ) @@ -65,7 +66,7 @@ func CompareVirtualMachineClassReadyStatus(vmName, expectedStatus string) { Expect(status).To(Equal(expectedStatus), fmt.Sprintf("VirtualMachineClassReady status should be '%s'", expectedStatus)) } -var _ = Describe("Sizing policy", Ordered, ContinueOnFailure, func() { +var _ = Describe("Sizing policy", ginkgoutil.CommonE2ETestDecorators(), func() { var ( vmNotValidSizingPolicyChanging string vmNotValidSizingPolicyCreating string diff --git a/tests/e2e/tests_suite_test.go b/tests/e2e/tests_suite_test.go index ff59eb7e2..dfda7537b 100644 --- a/tests/e2e/tests_suite_test.go +++ b/tests/e2e/tests_suite_test.go @@ -28,6 +28,7 @@ import ( "github.com/deckhouse/virtualization/tests/e2e/config" d8 "github.com/deckhouse/virtualization/tests/e2e/d8" + "github.com/deckhouse/virtualization/tests/e2e/ginkgoutil" gt "github.com/deckhouse/virtualization/tests/e2e/git" kc "github.com/deckhouse/virtualization/tests/e2e/kubectl" ) @@ -111,7 +112,9 @@ func TestTests(t *testing.T) { RegisterFailHandler(Fail) fmt.Fprintf(GinkgoWriter, "Starting test suite\n") RunSpecs(t, "Tests") - Cleanup() + if !(ginkgoutil.FailureBehaviourEnvSwitcher{}).IsStopOnFailure() { + Cleanup() + } } func Cleanup() { diff --git a/tests/e2e/vm_configuration_test.go b/tests/e2e/vm_configuration_test.go index 28975069f..5375e2897 100644 --- a/tests/e2e/vm_configuration_test.go +++ b/tests/e2e/vm_configuration_test.go @@ -25,6 +25,7 @@ import ( virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2" d8 "github.com/deckhouse/virtualization/tests/e2e/d8" + "github.com/deckhouse/virtualization/tests/e2e/ginkgoutil" kc "github.com/deckhouse/virtualization/tests/e2e/kubectl" ) @@ -79,7 +80,7 @@ func CheckCPUCoresNumberFromVirtualMachine(requiredValue string, virtualMachines } } -var _ = Describe("Virtual machine configuration", Ordered, ContinueOnFailure, func() { +var _ = Describe("Virtual machine configuration", ginkgoutil.CommonE2ETestDecorators(), func() { var ( testCaseLabel = map[string]string{"testcase": "vm-configuration"} automaticLabel = map[string]string{"vm": "automatic-conf"} diff --git a/tests/e2e/vm_connectivity_test.go b/tests/e2e/vm_connectivity_test.go index 4b839feae..d2fed6c04 100644 --- a/tests/e2e/vm_connectivity_test.go +++ b/tests/e2e/vm_connectivity_test.go @@ -29,6 +29,7 @@ import ( virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2" d8 "github.com/deckhouse/virtualization/tests/e2e/d8" "github.com/deckhouse/virtualization/tests/e2e/executor" + "github.com/deckhouse/virtualization/tests/e2e/ginkgoutil" kc "github.com/deckhouse/virtualization/tests/e2e/kubectl" ) @@ -92,7 +93,7 @@ func CheckResultSshCommand(vmName, cmd, equal string) { }).WithTimeout(Timeout).WithPolling(Interval).Should(Succeed()) } -var _ = Describe("VM connectivity", Ordered, ContinueOnFailure, func() { +var _ = Describe("VM connectivity", ginkgoutil.CommonE2ETestDecorators(), func() { var ( testCaseLabel = map[string]string{"testcase": "vm-connectivity"} aObjName = fmt.Sprintf("%s-vm-connectivity-a", namePrefix) diff --git a/tests/e2e/vm_disk_attachment_test.go b/tests/e2e/vm_disk_attachment_test.go index d87d6f9e0..31f77896f 100644 --- a/tests/e2e/vm_disk_attachment_test.go +++ b/tests/e2e/vm_disk_attachment_test.go @@ -26,6 +26,7 @@ import ( virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2" d8 "github.com/deckhouse/virtualization/tests/e2e/d8" + "github.com/deckhouse/virtualization/tests/e2e/ginkgoutil" . "github.com/deckhouse/virtualization/tests/e2e/helper" kc "github.com/deckhouse/virtualization/tests/e2e/kubectl" ) @@ -96,7 +97,7 @@ func GetDisksMetadata(vmName string, disks *Disks) { }).WithTimeout(Timeout).WithPolling(Interval).Should(Succeed()) } -var _ = Describe("Virtual disk attachment", Ordered, ContinueOnFailure, func() { +var _ = Describe("Virtual disk attachment", ginkgoutil.CommonE2ETestDecorators(), func() { var ( testCaseLabel = map[string]string{"testcase": "vm-disk-attachment"} hasNoConsumerLabel = map[string]string{"hasNoConsumer": "vm-disk-attachment"} diff --git a/tests/e2e/vm_disk_resizing_test.go b/tests/e2e/vm_disk_resizing_test.go index 73f3d6d3b..34f481bf9 100644 --- a/tests/e2e/vm_disk_resizing_test.go +++ b/tests/e2e/vm_disk_resizing_test.go @@ -28,6 +28,7 @@ import ( virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2" cfg "github.com/deckhouse/virtualization/tests/e2e/config" d8 "github.com/deckhouse/virtualization/tests/e2e/d8" + "github.com/deckhouse/virtualization/tests/e2e/ginkgoutil" kc "github.com/deckhouse/virtualization/tests/e2e/kubectl" ) @@ -149,7 +150,7 @@ func GetVirtualMachineDisks(vmName string, config *cfg.Config) (VirtualMachineDi return disks, nil } -var _ = Describe("Virtual disk resizing", Ordered, ContinueOnFailure, func() { +var _ = Describe("Virtual disk resizing", ginkgoutil.CommonE2ETestDecorators(), func() { diskResizingLabel := map[string]string{"testcase": "disk-resizing"} Context("When resources are applied:", func() { diff --git a/tests/e2e/vm_label_annotation_test.go b/tests/e2e/vm_label_annotation_test.go index 2a6bbdff8..2b4161dc4 100644 --- a/tests/e2e/vm_label_annotation_test.go +++ b/tests/e2e/vm_label_annotation_test.go @@ -27,10 +27,11 @@ import ( virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2" "github.com/deckhouse/virtualization/tests/e2e/executor" + "github.com/deckhouse/virtualization/tests/e2e/ginkgoutil" kc "github.com/deckhouse/virtualization/tests/e2e/kubectl" ) -var _ = Describe("Label and Annotation", Ordered, ContinueOnFailure, func() { +var _ = Describe("Label and Annotation", ginkgoutil.CommonE2ETestDecorators(), func() { imageManifest := vmPath("image.yaml") vmManifest := vmPath("vm_label_annotation.yaml") diff --git a/tests/e2e/vm_migration_test.go b/tests/e2e/vm_migration_test.go index 7951f30c0..aa1b6fc6e 100644 --- a/tests/e2e/vm_migration_test.go +++ b/tests/e2e/vm_migration_test.go @@ -25,6 +25,7 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" virtv1 "kubevirt.io/api/core/v1" + "github.com/deckhouse/virtualization/tests/e2e/ginkgoutil" . "github.com/deckhouse/virtualization/tests/e2e/helper" kc "github.com/deckhouse/virtualization/tests/e2e/kubectl" ) @@ -71,7 +72,7 @@ func CreateMigrationManifest(vmName, filePath string, labels map[string]string) return nil } -var _ = Describe("Virtual machine migration", Ordered, ContinueOnFailure, func() { +var _ = Describe("Virtual machine migration", ginkgoutil.CommonE2ETestDecorators(), func() { testCaseLabel := map[string]string{"testcase": "vm-migration"} Context("When resources are applied:", func() { diff --git a/tests/e2e/vm_test.go b/tests/e2e/vm_test.go index 53bb00350..3df3ffad2 100644 --- a/tests/e2e/vm_test.go +++ b/tests/e2e/vm_test.go @@ -31,6 +31,7 @@ import ( corev1 "k8s.io/api/core/v1" d8 "github.com/deckhouse/virtualization/tests/e2e/d8" + "github.com/deckhouse/virtualization/tests/e2e/ginkgoutil" kc "github.com/deckhouse/virtualization/tests/e2e/kubectl" ) @@ -48,7 +49,7 @@ func vmPath(file string) string { return path.Join(conf.VM.TestDataDir, file) } -var _ = Describe("VM", Ordered, ContinueOnFailure, func() { +var _ = Describe("VM", ginkgoutil.CommonE2ETestDecorators(), func() { imageManifest := vmPath("image.yaml") BeforeAll(func() { By("Apply image for vms")