test(ci): add e2e github workflow

Signed-off-by: Roman Sysoev <[email protected]>

.github/workflows/e2e_main.yaml

@@ -0,0 +1,71 @@
+# Copyright 2024 Flant JSC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: e2e
+
+env:
+  CI_COMMIT_REF_NAME: ${{ github.ref_name }}
+  GO_VERSION: "1.22.7"
+  GINKGO_VERSION: "2.20.0"
+
+on:
+  schedule:
+    - cron: "0 1 * * *"
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  show_dev_manifest:
+    runs-on: ubuntu-latest
+    name: Run e2e
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go ${{ env.GO_VERSION }}
+        uses: actions/setup-go@v5
+        with:
+          go-version: "${{ env.GO_VERSION }}"
+
+      - name: Install Task
+        uses: arduino/setup-task@v2
+
+      - name: Install ginkgo
+        run: |
+          echo "Install ginkgo"
+          go install "github.com/onsi/ginkgo/v2/ginkgo@v${{ env.GINKGO_VERSION }}"
+
+      - name: Install Deckhouse-cli
+        run: |
+          echo "Install d8"
+          curl -fsSL -o d8-install.sh https://raw.githubusercontent.com/deckhouse/deckhouse-cli/main/d8-install.sh
+          bash d8-install.sh
+
+      - uses: azure/k8s-set-context@v4
+        with:
+          method: service-account
+          k8s-url: https://api.e2e.virtlab.flant.com
+          k8s-secret: ${{ secrets.E2E_VIRTUALIZATION_SA_SECRET }}
+
+      - name: Download dependencies
+        working-directory: ./tests/e2e/
+        run: |
+          echo "Download dependencies"
+          go mod download
+
+      - name: Run E2E
+        working-directory: ./tests/e2e/
+        run: |
+          task run -v

tests/e2e/Taskfile.yaml

@@ -10,29 +10,28 @@ tasks:
   copy:
     cmds:
       - |
-        rm -fr /tmp/testdata
-        mkdir /tmp/testdata
-        cp -a testdata/complex-test /tmp/testdata/complex-test
-        cp -a testdata/connectivity /tmp/testdata/connectivity
-        cp -a testdata/disk-resizing /tmp/testdata/disk-resizing
-        cp -a testdata/sizing-policy /tmp/testdata/sizing-policy
-        cp -a testdata/vm-configuration /tmp/testdata/vm-configuration
-        cp -a testdata/vm-migration /tmp/testdata/vm-migration
-        cp -a testdata/vm-disk-attachment /tmp/testdata/vm-disk-attachment
-        cp -a testdata/sshkeys /tmp/testdata/sshkeys
+        rm --force --recursive /tmp/testdata
+        cp -a testdata /tmp/testdata
+
   ginkgo:
     cmds:
       - |
         v=($(ginkgo version 2>/dev/null))
         if [ "${v[2]}" != "{{ .GINKGO_VERSION }}" ]; then
-          go install github.com/onsi/ginkgo/v2/ginkgo@v"{{ .GINKGO_VERSION }}" ;
+          go install "github.com/onsi/ginkgo/v2/ginkgo@v{{ .GINKGO_VERSION }}" ;
         fi
   run:
     desc: "Run e2e tests"
     deps:
-      - ginkgo
+      - copy
     cmds:
-      - ginkgo --focus "Virtualization resources" -v
+      - |
+        ginkgo \
+          --skip-file vm_test.go \
+          --skip-file vm_label_annotation_test.go \
+          --skip-file ipam_test.go \
+          --skip-file disks_test.go \
+          -v
 
   run_local:
     desc: "Run locally e2e tests"
@@ -55,25 +54,9 @@ tasks:
       - ginkgo
     cmds:
       - |
-        {{if .TEST }}
-          ginkgo --focus "{{ .TEST }}" -v
+        {{if .FOCUS }}
+          ginkgo --focus "{{ .FOCUS }}" -v
         {{else}}
           echo "Specify test to run"
-          echo 'Example: TEST="Label and Annotation" task run_one'
+          echo 'Example: FOCUS="Label and Annotation" task run_one'
         {{end}}
-
-  fix:ssh_key_perm:
-    desc: "Check and fix permissions for ssh keys"
-    cmds:
-      - |
-        ID_ED=testdata/vm/sshkeys/id_ed
-        case {{OS}} in
-          darwin*)      RIGHTS_SSH=$(stat -f %Lp $ID_ED) ;;
-          linux*)       RIGHTS_SSH=$(stat -c "%a" $ID_ED) ;;
-          *)            echo "unknown: {{OS}}"; exit 1       ;;
-        esac
-
-        if [ $RIGHTS_SSH -ne 600 ]; then
-          echo "Fix permissions for file $ID_ED"
-          chmod 600 $ID_ED
-        fi

tests/e2e/complex_test.go

@@ -23,9 +23,41 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
+	virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2"
 	kc "github.com/deckhouse/virtualization/tests/e2e/kubectl"
 )
 
+func AssignIPToVMIP(name string) error {
+	assignErr := fmt.Sprintf("cannot patch VMIP %q with unnassigned IP address", name)
+	unassignedIP, err := FindUnassignedIP(mc.Spec.Settings.VirtualMachineCIDRs)
+	if err != nil {
+		return fmt.Errorf("%s\n%s", assignErr, err)
+	}
+	patch := fmt.Sprintf("{\"spec\":{\"staticIP\":%q}}", unassignedIP)
+	err = MergePatchResource(kc.ResourceVMIP, name, patch)
+	if err != nil {
+		return fmt.Errorf("%s\n%s", assignErr, err)
+	}
+	vmip := virtv2.VirtualMachineIPAddress{}
+	err = GetObject(kc.ResourceVMIP, name, &vmip, kc.GetOptions{
+		Namespace: conf.Namespace,
+	})
+	if err != nil {
+		return fmt.Errorf("%s\n%s", assignErr, err)
+	}
+	jsonPath := fmt.Sprintf("'jsonpath={.status.phase}=%s'", PhaseAttached)
+	waitOpts := kc.WaitOptions{
+		Namespace: conf.Namespace,
+		For:       jsonPath,
+		Timeout:   30,
+	}
+	res := kubectl.WaitResources(kc.ResourceVMIP, waitOpts, name)
+	if res.Error() != nil {
+		return fmt.Errorf("%s\n%s", assignErr, res.StdErr())
+	}
+	return nil
+}
+
 var _ = Describe("Complex test", Ordered, ContinueOnFailure, func() {
 	var (
 		testCaseLabel      = map[string]string{"testcase": "complex-test"}
@@ -74,11 +106,10 @@ var _ = Describe("Complex test", Ordered, ContinueOnFailure, func() {
 
 	Context("When virtual machines IP addresses are applied:", func() {
 		It("patches custom VMIP with unassigned address", func() {
-			unassignedIP, err := FindUnassignedIP(mc.Spec.Settings.VirtualMachineCIDRs)
-			Expect(err).NotTo(HaveOccurred())
-			vmipMetadataName := fmt.Sprintf("%s-%s", namePrefix, "vm-custom-ip")
-			mergePatch := fmt.Sprintf("{\"spec\":{\"staticIP\":%q}}", unassignedIP)
-			MergePatchResource(kc.ResourceVMIP, vmipMetadataName, mergePatch)
+			vmipName := fmt.Sprintf("%s-%s", namePrefix, "vm-custom-ip")
+			Eventually(func() error {
+				return AssignIPToVMIP(vmipName)
+			}).Should(Succeed())
 		})
 
 		It("checks VMIPs phases", func() {

tests/e2e/d8/d8.go

@@ -68,7 +68,6 @@ func NewD8Virtualization(conf D8VirtualizationConf) (*d8VirtualizationCMD, error
 	}
 
 	connEnvs, connArgs, err := clustertransport.KubeConnectionCmdSettings(clustertransport.ClusterTransport(conf))
-
 	if err != nil {
 		return nil, fmt.Errorf("load connection config: %w", err)
 	}

tests/e2e/kubectl/kubectl.go

@@ -133,7 +133,6 @@ func NewKubectl(conf KubectlConf) (*KubectlCMD, error) {
 	}
 
 	connEnvs, connArgs, err := clustertransport.KubeConnectionCmdSettings(clustertransport.ClusterTransport(conf))
-
 	if err != nil {
 		return nil, fmt.Errorf("load connection config: %w", err)
 	}

tests/e2e/sizing_policy_test.go

@@ -161,7 +161,8 @@ var _ = Describe("Sizing policy", Ordered, ContinueOnFailure, func() {
 
 			It("changes VMClassName in VM specification with existing VMClass", func() {
 				mergePatch := fmt.Sprintf("{\"spec\":{\"virtualMachineClassName\":%q}}", vmClassDiscovery)
-				MergePatchResource(kc.ResourceVM, vmNotValidSizingPolicyChanging, mergePatch)
+				err := MergePatchResource(kc.ResourceVM, vmNotValidSizingPolicyChanging, mergePatch)
+				Expect(err).NotTo(HaveOccurred(), err)
 			})
 
 			It("checks VM phase and condition status after changing", func() {
@@ -184,7 +185,8 @@ var _ = Describe("Sizing policy", Ordered, ContinueOnFailure, func() {
 
 			It("changes VMClassName in VM specification with not existing VMClass which have correct prefix for creating", func() {
 				mergePatch := fmt.Sprintf("{\"spec\":{\"virtualMachineClassName\":%q}}", vmClassDiscoveryCopy)
-				MergePatchResource(kc.ResourceVM, vmNotValidSizingPolicyCreating, mergePatch)
+				err := MergePatchResource(kc.ResourceVM, vmNotValidSizingPolicyCreating, mergePatch)
+				Expect(err).NotTo(HaveOccurred(), err)
 			})
 
 			It("creates new `VirtualMachineClass`", func() {

tests/e2e/util_test.go

@@ -148,14 +148,16 @@ func PatchResource(resource kc.Resource, name string, patch *kc.JsonPatch) {
 		res.StdErr())
 }
 
-func MergePatchResource(resource kc.Resource, name, patch string) {
+func MergePatchResource(resource kc.Resource, name, patch string) error {
 	GinkgoHelper()
 	res := kubectl.PatchResource(resource, name, kc.PatchOptions{
 		Namespace:  conf.Namespace,
 		MergePatch: patch,
 	})
-	Expect(res.Error()).NotTo(HaveOccurred(), "patch failed %s %s/%s.\n%s", resource, conf.Namespace, name,
-		res.StdErr())
+	if res.Error() != nil {
+		return fmt.Errorf("patch failed %s %s/%s.\n%s", resource, conf.Namespace, name, res.StdErr())
+	}
+	return nil
 }
 
 func CheckField(resource kc.Resource, name, output, compareValue string) {
@@ -234,7 +236,7 @@ func WaitPhase(resource kc.Resource, phase string, opts kc.GetOptions) {
 		Timeout:   600,
 	}
 	waitResult := kubectl.WaitResources(resource, waitOpts, resources...)
-	Expect(waitResult.WasSuccess()).To(Equal(true), waitResult.StdErr())
+	Expect(waitResult.Error()).NotTo(HaveOccurred(), waitResult.StdErr())
 }
 
 func GetDefaultStorageClass() (*storagev1.StorageClass, error) {

tests/e2e/vm_disk_resizing_test.go

@@ -63,7 +63,8 @@ func ResizeDisks(addedSize *resource.Quantity, config *cfg.Config, virtualDisks
 		Expect(err).NotTo(HaveOccurred(), err)
 		newValue := resource.NewQuantity(diskObject.Spec.PersistentVolumeClaim.Size.Value()+addedSize.Value(), resource.BinarySI)
 		mergePatch := fmt.Sprintf("{\"spec\":{\"persistentVolumeClaim\":{\"size\":\"%s\"}}}", newValue.String())
-		MergePatchResource(kc.ResourceVD, vd, mergePatch)
+		err = MergePatchResource(kc.ResourceVD, vd, mergePatch)
+		Expect(err).NotTo(HaveOccurred(), err)
 	}
 }