refactor(e2e): support stopping on failure

.github/workflows/e2e_main.yaml

@@ -17,7 +17,6 @@ name: e2e
 env:
   CI_COMMIT_REF_NAME: ${{ github.ref_name }}
   GO_VERSION: "1.22.7"
-  GINKGO_VERSION: "2.20.0"
 
 on:
   workflow_dispatch:
@@ -46,7 +45,8 @@ jobs:
       - name: Install ginkgo
         run: |
           echo "Install ginkgo"
-          go install "github.com/onsi/ginkgo/v2/ginkgo@v${{ env.GINKGO_VERSION }}"
+          GINKGO_VERSION=$(go list -f '{{.Version}}' -m github.com/onsi/ginkgo/v2)
+          go install "github.com/onsi/ginkgo/v2/ginkgo@v${GINKGO_VERSION}"
 
       - name: Install Deckhouse-cli
         run: |
@@ -69,4 +69,4 @@ jobs:
       - name: Run E2E
         working-directory: ./tests/e2e/
         run: |
-          task run -v
+          task run:ci -v

tests/e2e/README.md

@@ -1,46 +1,82 @@
 # Integration tests
 
-## Running integration tests
+## Prerequisites
 
-Integration tests require a running deckhouse cluster with the virtualization module installed.  
-Once you have a running deckhouse cluster, you can use config, environment variables or flags  to
-point the tests to the cluster. 
-You can override the config file using the env var ```E2E_CONFIG```.
-(default config - ```default_config.yaml```)
+### Utilities
 
-You must also have a default storage class declared.  
-Mark a StorageClass as default:
-```yaml
+Some utilities should be installed to run e2e tests:
+
+- task (https://taskfile.dev)
+- kubectl (https://kubernetes.io/docs/tasks/tools/#kubectl)
+- d8 (https://github.com/deckhouse/deckhouse-cli/releases)
+- ginkgo
+  - Download from https://github.com/onsi/ginkgo
+  - Or just run `go install github.com/onsi/ginkgo/v2/ginkgo@$(go list -f '{{.Version}}' -m github.com/onsi/ginkgo/v2)`
+
+### Deckhouse cluster
+
+Integration tests require a running Deckhouse cluster with the virtualization module installed.
+
+### Default StorageClass
+
+Default storage class should be set in the cluster. Annotate a StorageClass with
+storageclass.kubernetes.io/is-default-class to mark it as the default:
+
+```bash
+
+$ kubectl annotate storageclass linstor-thin-r1 storageclass.kubernetes.io/is-default-class=true
+
+$ kubectl get storageclass linstor-thin-r1 -o yaml | less
+...
 metadata:
   annotations:
     storageclass.kubernetes.io/is-default-class: "true"
-```
-Example:
-```bash
-kubectl patch storageclasses.storage.k8s.io linstor-thin-r1 --type=merge --patch='{"metadata":{"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
+...
 ```
 
-### Local running
-With "$HOME/.kube/config" by default and current cluster context:
-```bash
-task run_local
-```
+### E2E configuration
+
+Temp directories, prefixes, images and ssh settings can be set in the
+YAML configuration file.
+Path to custom file can be set with the `E2E_CONFIG` environment variable.
+Default config file is `default_config.yaml` in `tests/e2e` directory.
+
+You can override config field with environment variables. Use E2E_ prefix and join uppercased fields with _ (underscore).
+
+For example, to override curl image, set `E2E_HELPERIMAGES_CURLIMAGE` environment variable.
+
+### Cluster connection settings
+
+Connection settings priority from highest to lowest:
+
+- Token and endpoint in E2E_CLUSTERTRANSPORT_TOKEN and E2E_CLUSTERTRANSPORT_ENDPOINT envs.
+- Token and endpoint in clusterTransport field in e2e config file.
+- A path to kubeconfig file in clusterTransport.kubeConfig field in e2e config file.
+- A path to kubeconfig file in KUBECONFIG env.
+- A path to kubeconfig file in `~/.kube/config`.
+
+
+## Run tests from developer machine
+
+Setup cluster connection in "$HOME/.kube/config" or by [switch](https://github.com/danielfoehrKn/kubeswitch)ing the `KUBECONFIG` env and run tests:
 
-### Configuration
-To override a configuration option, create an environment variable named ```E2E_variable``` where variable is the name of the configuration option and the _ (underscore) represents indention levels. 
-For example, you can configure the ```token``` of the ```kubectl``` and ```virtctl```:
 ```bash
-clusterTransport:
-  token: "your token"
+task run
 ```
-To override this value, set an environment variable like this:
+
+### Debugging options
+
+- Use FOCUS env to run one test.
+- Use STOP_ON_FAILURE=yes env to stop tests on first failure without cleanup.
+
+For example, run only "Complex text" without cleanup on failure:
 ```bash
-export E2E_CLUSTERTRANSPORT_TOKEN="your token"
+FOCUS="Complex test" STOP_ON_FAILURE=yes task run
 ```
 
-### Run
+## Run tests in CI
 ```bash
-task run
+task run:ci
 ```
 
 ### Example
@@ -86,4 +122,4 @@ export E2E_CLUSTERTRANSPORT_CERTIFICATEAUTHORITY="$PWD/ca.crt"
 export E2E_CLUSTERTRANSPORT_INSECURETLS="false"
 
 task run
-```
+```

tests/e2e/Taskfile.yaml

@@ -3,41 +3,55 @@ version: "3"
 silent: true
 
 vars:
-  GINKGO_VERSION: "2.20.0"
+  GINKGO_VERSION:
+    # Should hide curly brackets from task templater.
+    sh: go list -f '{{`{{.Version}}`}}' -m github.com/onsi/ginkgo/v2
   VERSION: "v1.0.0"
 
 tasks:
   copy:
     cmds:
       - |
-        rm --force --recursive /tmp/testdata
+        if [ "{{OS}}" == "darwin" ]; then
+          rm -rf /tmp/testdata
+        else
+          rm --force --recursive /tmp/testdata
+        fi
         cp -a testdata /tmp/testdata
 
   ginkgo:
     cmds:
       - |
+        if ! which ginkgo >/dev/null ; then
+          echo "Ginkgo not found or not in the PATH. Install from github or run go install github.com/onsi/ginkgo/v2/ginkgo@{{ .GINKGO_VERSION }}"
+          exit 1
+        fi
         v=($(ginkgo version 2>/dev/null))
-        if [ "${v[2]}" != "{{ .GINKGO_VERSION }}" ]; then
-          go install "github.com/onsi/ginkgo/v2/ginkgo@v{{ .GINKGO_VERSION }}" ;
+        if [ "v${v[2]}" != "{{ .GINKGO_VERSION }}" ]; then
+          echo "Ginkgo version mismatch: {{ .GINKGO_VERSION }} is required. Got ${v[2]} in PATH."
+          exit 2
         fi
-  run:
-    desc: "Run e2e tests"
-    deps:
-      - copy
+  kubectl:
     cmds:
       - |
-        ginkgo \
-          --skip-file vm_test.go \
-          --skip-file vm_label_annotation_test.go \
-          --skip-file ipam_test.go \
-          --skip-file disks_test.go \
-          -v
-
-  run_local:
-    desc: "Run locally e2e tests"
+        if ! which kubectl >/dev/null ; then
+          echo "kubectl not found or not in PATH"
+          exit 1
+        fi
+  d8:
+    cmds:
+      - |
+        if ! which d8 >/dev/null ; then
+          echo "d8 not found or not in PATH. Install from https://github.com/deckhouse/deckhouse-cli/releases"
+          exit 1
+        fi
+  run:ci:
+    desc: "Separate task to run e2e tests in the CI environment"
     deps:
       - copy
       - ginkgo
+      - kubectl
+      - d8
     cmds:
       - |
         ginkgo \
@@ -47,16 +61,21 @@ tasks:
           --skip-file disks_test.go \
           -v
 
-  run_one:
-    desc: "Run one test or group"
+  run:
+    desc: "Run e2e tests"
     deps:
       - copy
       - ginkgo
+      - kubectl
+      - d8
     cmds:
       - |
-        {{if .FOCUS }}
-          ginkgo --focus "{{ .FOCUS }}" -v
-        {{else}}
-          echo "Specify test to run"
-          echo 'Example: FOCUS="Label and Annotation" task run_one'
-        {{end}}
+        ginkgo -v \
+          {{if .FOCUS -}}
+          --focus "{{ .FOCUS }}"
+          {{ else -}}
+          --skip-file vm_test.go \
+          --skip-file vm_label_annotation_test.go \
+          --skip-file ipam_test.go \
+          --skip-file disks_test.go
+          {{end}}

tests/e2e/complex_test.go

@@ -24,6 +24,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2"
+	"github.com/deckhouse/virtualization/tests/e2e/ginkgoutil"
 	kc "github.com/deckhouse/virtualization/tests/e2e/kubectl"
 )
 
@@ -58,7 +59,7 @@ func AssignIPToVMIP(name string) error {
 	return nil
 }
 
-var _ = Describe("Complex test", Ordered, ContinueOnFailure, func() {
+var _ = Describe("Complex test", ginkgoutil.CommonE2ETestDecorators(), func() {
 	var (
 		testCaseLabel      = map[string]string{"testcase": "complex-test"}
 		hasNoConsumerLabel = map[string]string{"hasNoConsumer": "complex-test"}

tests/e2e/disks_test.go

@@ -23,6 +23,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
+	"github.com/deckhouse/virtualization/tests/e2e/ginkgoutil"
 	kc "github.com/deckhouse/virtualization/tests/e2e/kubectl"
 )
 
@@ -42,7 +43,7 @@ func vdPath(file string) string {
 	return path.Join(conf.Disks.VdTestDataDir, file)
 }
 
-var _ = Describe("Disks", func() {
+var _ = Describe("Disks", ginkgoutil.CommonE2ETestDecorators(), func() {
 	CheckProgress := func(filepath string) {
 		GinkgoHelper()
 		out := "jsonpath={.status.progress}"
@@ -71,7 +72,7 @@ var _ = Describe("Disks", func() {
 		ItChekStatusPhaseFromFile(filepath, PhaseReady)
 	}
 
-	Context("CVI", Ordered, ContinueOnFailure, func() {
+	Context("CVI", func() {
 		AfterAll(func() {
 			By("Removing resources for cvi tests")
 			kubectl.Delete(conf.Disks.CviTestDataDir, kc.DeleteOptions{})
@@ -112,7 +113,7 @@ var _ = Describe("Disks", func() {
 			CheckProgress(filepath)
 		})
 	})
-	Context("VI", Ordered, ContinueOnFailure, func() {
+	Context("VI", func() {
 		AfterAll(func() {
 			By("Removing resources for vi tests")
 			kubectl.Delete(conf.Disks.ViTestDataDir, kc.DeleteOptions{})
@@ -153,7 +154,7 @@ var _ = Describe("Disks", func() {
 			CheckProgress(filepath)
 		})
 	})
-	Context("VD", Ordered, ContinueOnFailure, func() {
+	Context("VD", func() {
 		AfterAll(func() {
 			By("Removing resources for vd tests")
 			kubectl.Delete(conf.Disks.VdTestDataDir, kc.DeleteOptions{})

tests/e2e/ginkgoutil/decorators.go

@@ -0,0 +1,86 @@
+/*
+Copyright 2024 Flant JSC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ginkgoutil
+
+import (
+	"os"
+
+	"github.com/onsi/ginkgo/v2"
+)
+
+// Ginkgo decorators helpers:
+// - Common decorators for e2e: Ordered and ContinueOnFailure.
+// - ContinueOnFailure decorator is switchable and can be disabled with STOP_ON_FAILURE=yes env.
+//
+// A quote from Ginkgo documentation:
+// Moreover, Ginkgo also supports passing in arbitrarily nested slices of decorators.
+// Ginkgo will unroll these slices and process the flattened list. This makes it easier
+// to pass around groups of decorators. For example, this is valid:
+// markFlaky := []interface{}{Label("flaky"), FlakeAttempts(3)}
+// var _ = Describe("a bunch of flaky controller tests", markFlaky, Label("controller"), func() {
+//  ...
+// }
+// The resulting tests will be decorated with FlakeAttempts(3) and the two labels flaky and controller.
+//
+// This helper uses this "flattening" feature, so DecoratorsFromEnv implements
+// dynamic list of switchable decorators by returning an array of decorators.
+
+type EnvSwitchable interface {
+	Decorator() interface{}
+}
+
+func DecoratorsFromEnv(decorators ...interface{}) []interface{} {
+	out := make([]interface{}, 0)
+
+	for _, decorator := range decorators {
+		switch v := decorator.(type) {
+		case EnvSwitchable:
+			gdeco := v.Decorator()
+			if gdeco != nil {
+				out = append(out, gdeco)
+			}
+		default:
+			out = append(out, decorator)
+		}
+	}
+
+	return out
+}
+
+const StopOnFailureEnv = "STOP_ON_FAILURE"
+
+type FailureBehaviourEnvSwitcher struct{}
+
+func (f FailureBehaviourEnvSwitcher) Decorator() interface{} {
+	if !f.IsStopOnFailure() {
+		return ginkgo.ContinueOnFailure
+	}
+	return nil
+}
+
+// IsStopOnFailure returns true if Stop on error is enabled.
+func (f FailureBehaviourEnvSwitcher) IsStopOnFailure() bool {
+	return os.Getenv(StopOnFailureEnv) == "yes"
+}
+
+// CommonE2ETestDecorators returns common decorators for e2e tests: Ordered and ContinueOnFailure switchable with env.
+func CommonE2ETestDecorators() []interface{} {
+	return DecoratorsFromEnv(
+		ginkgo.Ordered,
+		FailureBehaviourEnvSwitcher{},
+	)
+}