From 0d4e80b9d408b507c6c4191af5244b5fa4059c9e Mon Sep 17 00:00:00 2001 From: Roman Sysoev Date: Tue, 3 Dec 2024 16:08:51 +0300 Subject: [PATCH] test(api): add e2e virtual disk snapshots test case Signed-off-by: Roman Sysoev --- tests/e2e/config/config.go | 1 + tests/e2e/default_config.yaml | 1 + tests/e2e/go.mod | 2 + tests/e2e/go.sum | 4 + tests/e2e/helper/helper.go | 14 + tests/e2e/kubectl/resource.go | 38 +- .../testdata/vd-snapshots/kustomization.yaml | 16 + tests/e2e/testdata/vd-snapshots/ns.yaml | 4 + .../e2e/testdata/vd-snapshots/snapshots/.keep | 0 .../testdata/vd-snapshots/transformer.yaml | 52 ++ .../vd-snapshots/vd/kustomization.yaml | 8 + .../vd-snapshots/vd/vd-alpine-http.yaml | 12 + .../vd-snapshots/vi/kustomization.yaml | 4 + .../vd-snapshots/vi/vi-alpine-http.yaml | 11 + .../vd-snapshots/vm/base/cfg/cloudinit.yaml | 51 ++ .../vd-snapshots/vm/base/kustomization.yaml | 14 + .../vd-snapshots/vm/base/transformer.yaml | 54 ++ .../vd-snapshots/vm/base/vd-root.yaml | 12 + .../e2e/testdata/vd-snapshots/vm/base/vm.yaml | 21 + .../vd-snapshots/vm/kustomization.yaml | 5 + .../automatic-with-hotplug/kustomization.yaml | 26 + .../automatic-with-hotplug/vd-attach.yaml | 7 + .../automatic-with-hotplug/vmbda.yaml | 9 + .../vm/overlays/automatic/kustomization.yaml | 24 + tests/e2e/tests_suite_test.go | 2 + tests/e2e/util_test.go | 4 +- tests/e2e/vd_snapshots_test.go | 539 ++++++++++++++++++ 27 files changed, 916 insertions(+), 19 deletions(-) create mode 100644 tests/e2e/testdata/vd-snapshots/kustomization.yaml create mode 100644 tests/e2e/testdata/vd-snapshots/ns.yaml create mode 100644 tests/e2e/testdata/vd-snapshots/snapshots/.keep create mode 100644 tests/e2e/testdata/vd-snapshots/transformer.yaml create mode 100644 tests/e2e/testdata/vd-snapshots/vd/kustomization.yaml create mode 100644 tests/e2e/testdata/vd-snapshots/vd/vd-alpine-http.yaml create mode 100644 tests/e2e/testdata/vd-snapshots/vi/kustomization.yaml create mode 100644 tests/e2e/testdata/vd-snapshots/vi/vi-alpine-http.yaml create mode 100644 tests/e2e/testdata/vd-snapshots/vm/base/cfg/cloudinit.yaml create mode 100644 tests/e2e/testdata/vd-snapshots/vm/base/kustomization.yaml create mode 100644 tests/e2e/testdata/vd-snapshots/vm/base/transformer.yaml create mode 100644 tests/e2e/testdata/vd-snapshots/vm/base/vd-root.yaml create mode 100644 tests/e2e/testdata/vd-snapshots/vm/base/vm.yaml create mode 100644 tests/e2e/testdata/vd-snapshots/vm/kustomization.yaml create mode 100644 tests/e2e/testdata/vd-snapshots/vm/overlays/automatic-with-hotplug/kustomization.yaml create mode 100644 tests/e2e/testdata/vd-snapshots/vm/overlays/automatic-with-hotplug/vd-attach.yaml create mode 100644 tests/e2e/testdata/vd-snapshots/vm/overlays/automatic-with-hotplug/vmbda.yaml create mode 100644 tests/e2e/testdata/vd-snapshots/vm/overlays/automatic/kustomization.yaml create mode 100644 tests/e2e/vd_snapshots_test.go diff --git a/tests/e2e/config/config.go b/tests/e2e/config/config.go index c22c3c709..4c8cb4cb9 100644 --- a/tests/e2e/config/config.go +++ b/tests/e2e/config/config.go @@ -137,6 +137,7 @@ type TestData struct { VmLabelAnnotation string `yaml:"vmLabelAnnotation"` VmMigration string `yaml:"vmMigration"` VmDiskAttachment string `yaml:"vmDiskAttachment"` + VdSnapshots string `yaml:"vdSnapshots"` Sshkey string `yaml:"sshKey"` SshUser string `yaml:"sshUser"` } diff --git a/tests/e2e/default_config.yaml b/tests/e2e/default_config.yaml index f46493125..b4f81e7c8 100644 --- a/tests/e2e/default_config.yaml +++ b/tests/e2e/default_config.yaml @@ -28,5 +28,6 @@ testData: vmLabelAnnotation: "/tmp/testdata/vm-label-annotation" vmMigration: "/tmp/testdata/vm-migration" vmDiskAttachment: "/tmp/testdata/vm-disk-attachment" + vdSnapshots: "/tmp/testdata/vd-snapshots" sshKey: "/tmp/testdata/sshkeys/id_ed" sshUser: "cloud" diff --git a/tests/e2e/go.mod b/tests/e2e/go.mod index ff0b138be..d5692fca8 100644 --- a/tests/e2e/go.mod +++ b/tests/e2e/go.mod @@ -3,7 +3,9 @@ module github.com/deckhouse/virtualization/tests/e2e go 1.22.7 require ( + github.com/deckhouse/sds-replicated-volume/api v0.0.0-20241109122839-a1ae840eb5db github.com/deckhouse/virtualization/api v0.0.0-20240923080356-bb5809dba578 + github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0 github.com/onsi/ginkgo/v2 v2.20.0 github.com/onsi/gomega v1.34.1 gopkg.in/yaml.v3 v3.0.1 diff --git a/tests/e2e/go.sum b/tests/e2e/go.sum index 5ca4894b9..eb086cb59 100644 --- a/tests/e2e/go.sum +++ b/tests/e2e/go.sum @@ -14,6 +14,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckhouse/sds-replicated-volume/api v0.0.0-20241109122839-a1ae840eb5db h1:ArXEN2TeIX+cX4DUGUtHpRhkJpSzzk80PNqY4PGxNi0= +github.com/deckhouse/sds-replicated-volume/api v0.0.0-20241109122839-a1ae840eb5db/go.mod h1:6yz0RtbkLVJtK2DeuvgfaqBZRl5V5ax1WsfPF5pbnvo= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -119,6 +121,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0 h1:mjQG0Vakr2h246kEDR85U8y8ZhPgT3bguTCajRa/jaw= +github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0/go.mod h1:E3vdYxHj2C2q6qo8/Da4g7P+IcwqRZyy3gJBzYybV9Y= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= diff --git a/tests/e2e/helper/helper.go b/tests/e2e/helper/helper.go index 223aed48d..a78808759 100644 --- a/tests/e2e/helper/helper.go +++ b/tests/e2e/helper/helper.go @@ -78,3 +78,17 @@ func WriteYamlObject(filePath string, obj client.Object) error { return nil } + +func UnmarshalResource(filePath string, obj client.Object) error { + file, err := os.ReadFile(filePath) + if err != nil { + return err + } + + err = yaml.Unmarshal(file, obj) + if err != nil { + return err + } + + return nil +} diff --git a/tests/e2e/kubectl/resource.go b/tests/e2e/kubectl/resource.go index 60405d7db..c85c8999d 100644 --- a/tests/e2e/kubectl/resource.go +++ b/tests/e2e/kubectl/resource.go @@ -17,21 +17,25 @@ limitations under the License. package kubectl const ( - ResourceNode Resource = "node" - ResourceNamespace Resource = "namespace" - ResourcePod Resource = "pod" - ResourceService Resource = "service" - ResourceStorageClass Resource = "storageclasses.storage.k8s.io" - ResourceKubevirtVM Resource = "internalvirtualizationvirtualmachines.internal.virtualization.deckhouse.io" - ResourceKubevirtVMI Resource = "internalvirtualizationvirtualmachineinstances.internal.virtualization.deckhouse.io" - ResourceKubevirtVMIM Resource = "internalvirtualizationvirtualmachineinstancemigrations.internal.virtualization.deckhouse.io" - ResourceModuleConfig Resource = "moduleconfigs.deckhouse.io" - ResourceVD Resource = "virtualdisks.virtualization.deckhouse.io" - ResourceVM Resource = "virtualmachine.virtualization.deckhouse.io" - ResourceVMClass Resource = "virtualmachineclasses.virtualization.deckhouse.io" - ResourceVMIP Resource = "virtualmachineipaddresses.virtualization.deckhouse.io" - ResourceVMIPLease Resource = "virtualmachineipaddressleases.virtualization.deckhouse.io" - ResourceCVI Resource = "clustervirtualimages.virtualization.deckhouse.io" - ResourceVI Resource = "virtualimages.virtualization.deckhouse.io" - ResourceVMBDA Resource = "virtualmachineblockdeviceattachments.virtualization.deckhouse.io" + ResourceNode Resource = "node" + ResourceNamespace Resource = "namespace" + ResourcePod Resource = "pod" + ResourceService Resource = "service" + ResourceStorageClass Resource = "storageclasses.storage.k8s.io" + ResourceVolumeSnapshotClass Resource = "volumesnapshotclasses.snapshot.storage.k8s.io" + ResourceReplicatedStorageClass Resource = "replicatedstorageclasses.storage.deckhouse.io" + ResourceReplicatedStoragePool Resource = "replicatedstoragepools.storage.deckhouse.io" + ResourceKubevirtVM Resource = "internalvirtualizationvirtualmachines.internal.virtualization.deckhouse.io" + ResourceKubevirtVMI Resource = "internalvirtualizationvirtualmachineinstances.internal.virtualization.deckhouse.io" + ResourceKubevirtVMIM Resource = "internalvirtualizationvirtualmachineinstancemigrations.internal.virtualization.deckhouse.io" + ResourceModuleConfig Resource = "moduleconfigs.deckhouse.io" + ResourceVD Resource = "virtualdisks.virtualization.deckhouse.io" + ResourceVDSnapshot Resource = "virtualdisksnapshots.virtualization.deckhouse.io" + ResourceVM Resource = "virtualmachine.virtualization.deckhouse.io" + ResourceVMClass Resource = "virtualmachineclasses.virtualization.deckhouse.io" + ResourceVMIP Resource = "virtualmachineipaddresses.virtualization.deckhouse.io" + ResourceVMIPLease Resource = "virtualmachineipaddressleases.virtualization.deckhouse.io" + ResourceCVI Resource = "clustervirtualimages.virtualization.deckhouse.io" + ResourceVI Resource = "virtualimages.virtualization.deckhouse.io" + ResourceVMBDA Resource = "virtualmachineblockdeviceattachments.virtualization.deckhouse.io" ) diff --git a/tests/e2e/testdata/vd-snapshots/kustomization.yaml b/tests/e2e/testdata/vd-snapshots/kustomization.yaml new file mode 100644 index 000000000..692b0657f --- /dev/null +++ b/tests/e2e/testdata/vd-snapshots/kustomization.yaml @@ -0,0 +1,16 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: testcases +namePrefix: commit- +resources: + - ns.yaml + - vd + - vi + - vm +configurations: + - transformer.yaml +labels: + - includeSelectors: true + pairs: + id: commit + testcase: vd-snapshots diff --git a/tests/e2e/testdata/vd-snapshots/ns.yaml b/tests/e2e/testdata/vd-snapshots/ns.yaml new file mode 100644 index 000000000..5efde875b --- /dev/null +++ b/tests/e2e/testdata/vd-snapshots/ns.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: default diff --git a/tests/e2e/testdata/vd-snapshots/snapshots/.keep b/tests/e2e/testdata/vd-snapshots/snapshots/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/e2e/testdata/vd-snapshots/transformer.yaml b/tests/e2e/testdata/vd-snapshots/transformer.yaml new file mode 100644 index 000000000..e827a1923 --- /dev/null +++ b/tests/e2e/testdata/vd-snapshots/transformer.yaml @@ -0,0 +1,52 @@ +namespace: + - kind: ClusterVirtualImage + path: spec/dataSource/objectRef/namespace +nameReference: + - kind: VirtualImage + version: v1alpha2 # optional + fieldSpecs: + - path: spec/dataSource/objectRef/name + kind: ClusterVirtualImage + - path: spec/dataSource/objectRef/name + kind: VirtualImage + - path: spec/dataSource/objectRef/name + kind: VirtualDisk + - path: spec/blockDeviceRefs/name + kind: VirtualMachine + - kind: ClusterVirtualImage + version: v1alpha2 # optional + fieldSpecs: + - path: spec/dataSource/objectRef/name + kind: ClusterVirtualImage + - path: spec/dataSource/objectRef/name + kind: VirtualImage + - path: spec/dataSource/objectRef/name + kind: VirtualDisk + - path: spec/blockDeviceRefs/name + kind: VirtualMachine + - kind: VirtualDisk + version: v1alpha2 # optional + fieldSpecs: + - path: spec/blockDeviceRefs/name + kind: VirtualMachine + - path: spec/blockDeviceRef/name + kind: VirtualMachineBlockDeviceAttachment + - kind: Secret + fieldSpecs: + - path: spec/provisioning/userDataRef/name + kind: VirtualMachine + - kind: VirtualMachineIPAddress + version: v1alpha2 + fieldSpecs: + - path: spec/virtualMachineIPAddressName + kind: VirtualMachine + - kind: VirtualMachine + version: v1alpha2 + fieldSpecs: + - path: spec/virtualMachineName + kind: VirtualMachineBlockDeviceAttachment + - kind: VirtualMachineClass + version: v1alpha2 + fieldSpecs: + - path: spec/virtualMachineClassName + kind: VirtualMachine diff --git a/tests/e2e/testdata/vd-snapshots/vd/kustomization.yaml b/tests/e2e/testdata/vd-snapshots/vd/kustomization.yaml new file mode 100644 index 000000000..7b38a7f14 --- /dev/null +++ b/tests/e2e/testdata/vd-snapshots/vd/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - vd-alpine-http.yaml +labels: + - includeSelectors: true + pairs: + hasNoConsumer: "vd-snapshots" diff --git a/tests/e2e/testdata/vd-snapshots/vd/vd-alpine-http.yaml b/tests/e2e/testdata/vd-snapshots/vd/vd-alpine-http.yaml new file mode 100644 index 000000000..f67f7789c --- /dev/null +++ b/tests/e2e/testdata/vd-snapshots/vd/vd-alpine-http.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: virtualization.deckhouse.io/v1alpha2 +kind: VirtualDisk +metadata: + name: vd-alpine-http +spec: + dataSource: + type: HTTP + http: + url: https://0e773854-6b4e-4e76-a65b-d9d81675451a.selstorage.ru/alpine/alpine-v3-20.qcow2 + persistentVolumeClaim: + size: 260Mi diff --git a/tests/e2e/testdata/vd-snapshots/vi/kustomization.yaml b/tests/e2e/testdata/vd-snapshots/vi/kustomization.yaml new file mode 100644 index 000000000..b807d8e2d --- /dev/null +++ b/tests/e2e/testdata/vd-snapshots/vi/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - vi-alpine-http.yaml diff --git a/tests/e2e/testdata/vd-snapshots/vi/vi-alpine-http.yaml b/tests/e2e/testdata/vd-snapshots/vi/vi-alpine-http.yaml new file mode 100644 index 000000000..2bd920826 --- /dev/null +++ b/tests/e2e/testdata/vd-snapshots/vi/vi-alpine-http.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: virtualization.deckhouse.io/v1alpha2 +kind: VirtualImage +metadata: + name: vi-alpine-http +spec: + storage: ContainerRegistry + dataSource: + type: HTTP + http: + url: https://0e773854-6b4e-4e76-a65b-d9d81675451a.selstorage.ru/alpine/alpine-v3-20.qcow2 diff --git a/tests/e2e/testdata/vd-snapshots/vm/base/cfg/cloudinit.yaml b/tests/e2e/testdata/vd-snapshots/vm/base/cfg/cloudinit.yaml new file mode 100644 index 000000000..3f922b76f --- /dev/null +++ b/tests/e2e/testdata/vd-snapshots/vm/base/cfg/cloudinit.yaml @@ -0,0 +1,51 @@ +#cloud-config +users: + - name: cloud + # passwd: cloud + passwd: $6$rounds=4096$vln/.aPHBOI7BMYR$bBMkqQvuGs5Gyd/1H5DP4m9HjQSy.kgrxpaGEHwkX7KEFV8BS.HZWPitAtZ2Vd8ZqIZRqmlykRCagTgPejt1i. + shell: /bin/bash + sudo: ALL=(ALL) NOPASSWD:ALL + chpasswd: { expire: False } + lock_passwd: false + ssh_authorized_keys: + # testcases + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFxcXHmwaGnJ8scJaEN5RzklBPZpVSic4GdaAsKjQoeA your_email@example.com +write_files: + - path: /etc/init.d/ping-service + permissions: "0755" + content: | + #!/sbin/openrc-run + + command="/usr/bin/ping-service.sh" + pidfile="/var/run/${RC_SVCNAME}.pid" + + depend() { + need localmount + after bootmisc + } + + start() { + ebegin "Starting ${RC_SVCNAME}" + start-stop-daemon --start --background --exec $command --pidfile $pidfile -- $command_args + eend $? + } + + stop() { + ebegin "Stopping ${RC_SVCNAME}" + start-stop-daemon --stop --exec $command --pidfile $pidfile + eend $? + } + - path: /usr/bin/ping-service.sh + permissions: "0755" + content: | + #!/bin/bash + + while true; do + ping -W1 -D 1.1.1.1 >> /tmp/ping.log + done +final_message: "\U0001F525\U0001F525\U0001F525 The system is finally up, after $UPTIME seconds \U0001F525\U0001F525\U0001F525" +runcmd: + - "echo \"\U0001F7E1 Starting runcmd at $(date +%H:%M:%S)\"" + - rc-update add ping-service default + - rc-service ping-service start + - "echo \"\U0001F7E1 Finished runcmd at $(date +%H:%M:%S)\"" diff --git a/tests/e2e/testdata/vd-snapshots/vm/base/kustomization.yaml b/tests/e2e/testdata/vd-snapshots/vm/base/kustomization.yaml new file mode 100644 index 000000000..d820a3f56 --- /dev/null +++ b/tests/e2e/testdata/vd-snapshots/vm/base/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./vm.yaml + - ./vd-root.yaml +configurations: + - transformer.yaml +generatorOptions: + disableNameSuffixHash: true +secretGenerator: + - files: + - userData=cfg/cloudinit.yaml + name: cloud-init + type: provisioning.virtualization.deckhouse.io/cloud-init diff --git a/tests/e2e/testdata/vd-snapshots/vm/base/transformer.yaml b/tests/e2e/testdata/vd-snapshots/vm/base/transformer.yaml new file mode 100644 index 000000000..c70c289af --- /dev/null +++ b/tests/e2e/testdata/vd-snapshots/vm/base/transformer.yaml @@ -0,0 +1,54 @@ +# https://github.com/kubernetes-sigs/kustomize/blob/master/examples/transformerconfigs/README.md#transformer-configurations + +namespace: + - kind: ClusterVirtualImage + path: spec/dataSource/objectRef/namespace +nameReference: + - kind: VirtualImage + version: v1alpha2 # optional + fieldSpecs: + - path: spec/dataSource/objectRef/name + kind: ClusterVirtualImage + - path: spec/dataSource/objectRef/name + kind: VirtualImage + - path: spec/dataSource/objectRef/name + kind: VirtualDisk + - path: spec/blockDeviceRefs/name + kind: VirtualMachine + - kind: ClusterVirtualImage + version: v1alpha2 # optional + fieldSpecs: + - path: spec/dataSource/objectRef/name + kind: ClusterVirtualImage + - path: spec/dataSource/objectRef/name + kind: VirtualImage + - path: spec/dataSource/objectRef/name + kind: VirtualDisk + - path: spec/blockDeviceRefs/name + kind: VirtualMachine + - kind: VirtualDisk + version: v1alpha2 # optional + fieldSpecs: + - path: spec/blockDeviceRefs/name + kind: VirtualMachine + - path: spec/blockDeviceRef/name + kind: VirtualMachineBlockDeviceAttachment + - kind: Secret + fieldSpecs: + - path: spec/provisioning/userDataRef/name + kind: VirtualMachine + - kind: VirtualMachineIPAddress + version: v1alpha2 + fieldSpecs: + - path: spec/virtualMachineIPAddressName + kind: VirtualMachine + - kind: VirtualMachine + version: v1alpha2 + fieldSpecs: + - path: spec/virtualMachineName + kind: VirtualMachineBlockDeviceAttachment + - kind: VirtualMachineClass + version: v1alpha2 + fieldSpecs: + - path: spec/virtualMachineClassName + kind: VirtualMachine diff --git a/tests/e2e/testdata/vd-snapshots/vm/base/vd-root.yaml b/tests/e2e/testdata/vd-snapshots/vm/base/vd-root.yaml new file mode 100644 index 000000000..79dfd103e --- /dev/null +++ b/tests/e2e/testdata/vd-snapshots/vm/base/vd-root.yaml @@ -0,0 +1,12 @@ +apiVersion: virtualization.deckhouse.io/v1alpha2 +kind: VirtualDisk +metadata: + name: vd-root +spec: + persistentVolumeClaim: + size: 512Mi + dataSource: + type: ObjectRef + objectRef: + kind: VirtualImage + name: vi-alpine-http diff --git a/tests/e2e/testdata/vd-snapshots/vm/base/vm.yaml b/tests/e2e/testdata/vd-snapshots/vm/base/vm.yaml new file mode 100644 index 000000000..a352cb713 --- /dev/null +++ b/tests/e2e/testdata/vd-snapshots/vm/base/vm.yaml @@ -0,0 +1,21 @@ +apiVersion: virtualization.deckhouse.io/v1alpha2 +kind: VirtualMachine +metadata: + name: vm +spec: + virtualMachineClassName: generic + cpu: + cores: 1 + coreFraction: 5% + memory: + size: 256Mi + disruptions: + restartApprovalMode: Manual + provisioning: + type: UserDataRef + userDataRef: + kind: Secret + name: cloud-init + blockDeviceRefs: + - kind: VirtualDisk + name: vd-root diff --git a/tests/e2e/testdata/vd-snapshots/vm/kustomization.yaml b/tests/e2e/testdata/vd-snapshots/vm/kustomization.yaml new file mode 100644 index 000000000..ada7106a4 --- /dev/null +++ b/tests/e2e/testdata/vd-snapshots/vm/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - overlays/automatic + - overlays/automatic-with-hotplug diff --git a/tests/e2e/testdata/vd-snapshots/vm/overlays/automatic-with-hotplug/kustomization.yaml b/tests/e2e/testdata/vd-snapshots/vm/overlays/automatic-with-hotplug/kustomization.yaml new file mode 100644 index 000000000..e2fb6a2e8 --- /dev/null +++ b/tests/e2e/testdata/vd-snapshots/vm/overlays/automatic-with-hotplug/kustomization.yaml @@ -0,0 +1,26 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +nameSuffix: -automatic-with-hotplug +resources: + - ../../base + - ./vd-attach.yaml + - ./vmbda.yaml +patches: + - patch: |- + - op: replace + path: /spec/runPolicy + value: AlwaysOn + target: + kind: VirtualMachine + name: vm + - patch: |- + - op: replace + path: /spec/disruptions/restartApprovalMode + value: Automatic + target: + kind: VirtualMachine + name: vm +labels: + - includeSelectors: true + pairs: + vm: automatic-with-hotplug diff --git a/tests/e2e/testdata/vd-snapshots/vm/overlays/automatic-with-hotplug/vd-attach.yaml b/tests/e2e/testdata/vd-snapshots/vm/overlays/automatic-with-hotplug/vd-attach.yaml new file mode 100644 index 000000000..a4862dd65 --- /dev/null +++ b/tests/e2e/testdata/vd-snapshots/vm/overlays/automatic-with-hotplug/vd-attach.yaml @@ -0,0 +1,7 @@ +apiVersion: virtualization.deckhouse.io/v1alpha2 +kind: VirtualDisk +metadata: + name: vd-attach +spec: + persistentVolumeClaim: + size: 100Mi diff --git a/tests/e2e/testdata/vd-snapshots/vm/overlays/automatic-with-hotplug/vmbda.yaml b/tests/e2e/testdata/vd-snapshots/vm/overlays/automatic-with-hotplug/vmbda.yaml new file mode 100644 index 000000000..30a7b6ba4 --- /dev/null +++ b/tests/e2e/testdata/vd-snapshots/vm/overlays/automatic-with-hotplug/vmbda.yaml @@ -0,0 +1,9 @@ +apiVersion: virtualization.deckhouse.io/v1alpha2 +kind: VirtualMachineBlockDeviceAttachment +metadata: + name: blank-disk-attachment +spec: + virtualMachineName: vm + blockDeviceRef: + kind: VirtualDisk + name: vd-attach diff --git a/tests/e2e/testdata/vd-snapshots/vm/overlays/automatic/kustomization.yaml b/tests/e2e/testdata/vd-snapshots/vm/overlays/automatic/kustomization.yaml new file mode 100644 index 000000000..37f7a7cf0 --- /dev/null +++ b/tests/e2e/testdata/vd-snapshots/vm/overlays/automatic/kustomization.yaml @@ -0,0 +1,24 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +nameSuffix: -automatic +resources: + - ../../base +patches: + - patch: |- + - op: replace + path: /spec/runPolicy + value: AlwaysOn + target: + kind: VirtualMachine + name: vm + - patch: |- + - op: replace + path: /spec/disruptions/restartApprovalMode + value: Automatic + target: + kind: VirtualMachine + name: vm +labels: + - includeSelectors: true + pairs: + vm: automatic diff --git a/tests/e2e/tests_suite_test.go b/tests/e2e/tests_suite_test.go index b7e3ee0c2..88ab8b608 100644 --- a/tests/e2e/tests_suite_test.go +++ b/tests/e2e/tests_suite_test.go @@ -42,6 +42,7 @@ const ( PhaseAttached = "Attached" PhaseReady = "Ready" PhaseBound = "Bound" + PhaseCompleted = "Completed" PhasePending = "Pending" PhaseReleased = "Released" PhaseSucceeded = "Succeeded" @@ -96,6 +97,7 @@ func init() { fmt.Sprintf("%s/%s", conf.TestData.Connectivity, "kustomization.yaml"), fmt.Sprintf("%s/%s", conf.TestData.DiskResizing, "kustomization.yaml"), fmt.Sprintf("%s/%s", conf.TestData.SizingPolicy, "kustomization.yaml"), + fmt.Sprintf("%s/%s", conf.TestData.VdSnapshots, "kustomization.yaml"), fmt.Sprintf("%s/%s", conf.TestData.VmConfiguration, "kustomization.yaml"), fmt.Sprintf("%s/%s", conf.TestData.VmLabelAnnotation, "kustomization.yaml"), fmt.Sprintf("%s/%s", conf.TestData.VmMigration, "kustomization.yaml"), diff --git a/tests/e2e/util_test.go b/tests/e2e/util_test.go index 4612299a0..f8313f43b 100644 --- a/tests/e2e/util_test.go +++ b/tests/e2e/util_test.go @@ -229,7 +229,7 @@ func GetObjects(resource kc.Resource, object client.ObjectList, opts kc.GetOptio } cmd := kubectl.List(resource, cmdOpts) if cmd.Error() != nil { - return fmt.Errorf(cmd.StdErr()) + return fmt.Errorf("cmd: %s\nstderr: %s", cmd.GetCmd(), cmd.StdErr()) } err := json.Unmarshal(cmd.StdOutBytes(), object) if err != nil { @@ -283,7 +283,7 @@ func WaitPhaseByLabel(resource kc.Resource, phase string, opts kc.WaitOptions) { res := kubectl.WaitResource(resource, name, waitOpts) if res.Error() != nil { mu.Lock() - waitErr = append(waitErr, res.StdErr()) + waitErr = append(waitErr, fmt.Sprintf("cmd: %s\nstderr: %s", res.GetCmd(), res.StdErr())) mu.Unlock() } }() diff --git a/tests/e2e/vd_snapshots_test.go b/tests/e2e/vd_snapshots_test.go new file mode 100644 index 000000000..6fef69136 --- /dev/null +++ b/tests/e2e/vd_snapshots_test.go @@ -0,0 +1,539 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "errors" + "fmt" + "strings" + "sync" + "time" + + sdsrepvolv1 "github.com/deckhouse/sds-replicated-volume/api/v1alpha1" + snapshotvolv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + storagev1 "k8s.io/api/storage/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/tests/e2e/ginkgoutil" + . "github.com/deckhouse/virtualization/tests/e2e/helper" + kc "github.com/deckhouse/virtualization/tests/e2e/kubectl" +) + +const ( + ReplicatedStorageClassKind = "ReplicatedStorageClass" + LinstorProviderName = "replicated.csi.storage.deckhouse.io" + LVMThinName = "LVMThin" + CephProviderName = "rbd.csi.ceph.com" + filesystemReadyTimeout = 60 * time.Second + filesystemReadyPollingInterval = 5 * time.Second + frozenReasonPollingInterval = 1 * time.Second +) + +func CreateVirtualDiskSnapshot(vdName, snapshotName, volumeSnapshotClassName string, requiredConsistency bool, labels map[string]string) error { + GinkgoHelper() + vdSnapshotName := fmt.Sprintf("%s-%s", snapshotName, volumeSnapshotClassName) + vdSnapshot := virtv2.VirtualDiskSnapshot{ + TypeMeta: v1.TypeMeta{ + APIVersion: APIVersion, + Kind: virtv2.VirtualDiskSnapshotKind, + }, + ObjectMeta: v1.ObjectMeta{ + Labels: labels, + Name: vdSnapshotName, + Namespace: conf.Namespace, + }, + Spec: virtv2.VirtualDiskSnapshotSpec{ + RequiredConsistency: requiredConsistency, + VirtualDiskName: vdName, + VolumeSnapshotClassName: volumeSnapshotClassName, + }, + } + + filePath := fmt.Sprintf("%s/snapshots/%s-%s.yaml", conf.TestData.VdSnapshots, vdSnapshotName, volumeSnapshotClassName) + err := WriteYamlObject(filePath, &vdSnapshot) + if err != nil { + return fmt.Errorf("cannot write file with virtual disk snapshot: %s\nstderr: %s", vdSnapshotName, err) + } + + res := kubectl.Apply(kc.ApplyOptions{ + Filename: []string{filePath}, + FilenameOption: kc.Filename, + }) + if res.Error() != nil { + return fmt.Errorf("cannot create virtual disk snapshot: %s\nstderr: %s", vdSnapshotName, res.StdErr()) + } + return nil +} + +func createImmediateStorageClass(provisioner string, labels map[string]string) (string, error) { + GinkgoHelper() + filePath := fmt.Sprintf("%s/immediate-storage-class.yaml", conf.TestData.VdSnapshots) + switch provisioner { + case LinstorProviderName: + replicatedStorageClassName := fmt.Sprintf("%s-linstor-immediate", namePrefix) + replicatedStoragePoolName, err := GetLVMThinReplicatedStoragePool() + if err != nil { + return "", err + } + err = createLinstorImmediateStorageClass(filePath, replicatedStorageClassName, replicatedStoragePoolName, labels) + if err != nil { + return "", err + } + return replicatedStorageClassName, nil + case CephProviderName: + storageClassName := fmt.Sprintf("%s-ceph-immediate", namePrefix) + err := createCephImmediateStorageClass(filePath, storageClassName, labels) + if err != nil { + return "", err + } + return storageClassName, nil + default: + return "", errors.New("cannot create storage class with `Immediate` volume binding mode") + } +} + +// Get first replicated storage pool with type `LVMThin` and phase `Completed` +func GetLVMThinReplicatedStoragePool() (string, error) { + GinkgoHelper() + rspObjects := sdsrepvolv1.ReplicatedStoragePoolList{} + err := GetObjects(kc.ResourceReplicatedStoragePool, &rspObjects, kc.GetOptions{}) + if err != nil { + return "", err + } + for _, rsp := range rspObjects.Items { + if rsp.Spec.Type == LVMThinName && rsp.Status.Phase == PhaseCompleted { + return rsp.Name, nil + } + } + return "", fmt.Errorf("cannot get completed replicated storage pool with type `LVMThin`") +} + +func createLinstorImmediateStorageClass(filePath, storageClassName, replicatedStoragePoolName string, labels map[string]string) error { + GinkgoHelper() + replicatedStorageClass := sdsrepvolv1.ReplicatedStorageClass{ + TypeMeta: v1.TypeMeta{ + APIVersion: sdsrepvolv1.SchemeGroupVersion.String(), + Kind: ReplicatedStorageClassKind, + }, + ObjectMeta: v1.ObjectMeta{ + Name: storageClassName, + Labels: labels, + }, + Spec: sdsrepvolv1.ReplicatedStorageClassSpec{ + ReclaimPolicy: "Delete", + Replication: "None", + StoragePool: replicatedStoragePoolName, + Topology: "Ignored", + VolumeAccess: "Any", + }, + } + + err := WriteYamlObject(filePath, &replicatedStorageClass) + if err != nil { + return nil + } + + res := kubectl.Apply(kc.ApplyOptions{ + Filename: []string{filePath}, + FilenameOption: kc.Filename, + }) + if res.Error() != nil { + return fmt.Errorf("cmd: %s\nstderr: %s", res.GetCmd(), res.StdErr()) + } + + return nil +} + +func createCephImmediateStorageClass(filePath, storageClassName string, labels map[string]string) error { + GinkgoHelper() + sc, err := GetDefaultStorageClass() + if err != nil { + return err + } + sc.ObjectMeta.Name = storageClassName + sc.ObjectMeta.Labels = labels + *sc.VolumeBindingMode = storagev1.VolumeBindingImmediate + + err = WriteYamlObject(filePath, sc) + if err != nil { + return err + } + + res := kubectl.Apply(kc.ApplyOptions{ + Filename: []string{filePath}, + FilenameOption: kc.Filename, + }) + if res.Error() != nil { + return fmt.Errorf("cmd: %s\nstderr: %s", res.GetCmd(), res.StdErr()) + } + + return nil +} + +func GetVolumeSnapshotClassName(storageClass *storagev1.StorageClass) (string, error) { + vscObjects := snapshotvolv1.VolumeSnapshotClassList{} + err := GetObjects(kc.ResourceVolumeSnapshotClass, &vscObjects, kc.GetOptions{}) + if err != nil { + return "", fmt.Errorf("cannot get `VolumeSnapshotClasses` by provisioner %q\nstderr: %s", storageClass.Provisioner, err) + } + + for _, vsc := range vscObjects.Items { + if vsc.Driver == storageClass.Provisioner { + return vsc.Name, nil + } + } + + return "", fmt.Errorf("cannot found `VolumeSnapshotClass` by provisioner %q", storageClass.Provisioner) +} + +func CheckFilesystemReadyStatus(vmName string, status v1.ConditionStatus) (string, error) { + GinkgoHelper() + vmObj := virtv2.VirtualMachine{} + filesystemReadyConditionReason := "" + err := GetObject(kc.ResourceVM, vmName, &vmObj, kc.GetOptions{Namespace: conf.Namespace}) + if err != nil { + return "", fmt.Errorf("cannot get `VirtualMachine`: %q\nstderr: %s", vmName, err) + } + + for _, condition := range vmObj.Status.Conditions { + if condition.Type == "FilesystemReady" { + if condition.Status != status { + return condition.Reason, fmt.Errorf("`FilesystemReady` status of %q is not %q: %s", vmName, status, condition.Reason) + } else { + filesystemReadyConditionReason = condition.Reason + } + } + } + + return filesystemReadyConditionReason, nil +} + +var _ = Describe("Virtual disk snapshots", ginkgoutil.CommonE2ETestDecorators(), func() { + var ( + immediateStorageClassName string // require for unattached virtual disk snapshots + defaultVolumeSnapshotClassName string + testCaseLabel = map[string]string{"testcase": "vd-snapshots"} + attachedVirtualDiskLabel = map[string]string{"attachedVirtualDisk": ""} + hasNoConsumerLabel = map[string]string{"hasNoConsumer": "vd-snapshots"} + vmAutomaticWithHotplug = map[string]string{"vm": "automatic-with-hotplug"} + ) + + Context("Environment preparing", func() { + It("prepares `Immediate` storage class and virtual disk that use it", func() { + sc, err := GetDefaultStorageClass() + Expect(err).NotTo(HaveOccurred(), "cannot get default storage class\nstderr: %s", err) + defaultVolumeSnapshotClassName, err = GetVolumeSnapshotClassName(sc) + Expect(err).NotTo(HaveOccurred(), "cannot define default `VolumeSnapshotClass`\nstderr: %s", err) + if sc.Provisioner == LinstorProviderName { + storagePoolName := sc.Parameters["replicated.csi.storage.deckhouse.io/storagePool"] + storagePoolObj := sdsrepvolv1.ReplicatedStoragePool{} + err := GetObject(kc.ResourceReplicatedStoragePool, storagePoolName, &storagePoolObj, kc.GetOptions{}) + Expect(err).NotTo(HaveOccurred(), "cannot get `storagePoolObj`: %s\nstderr: %s", storagePoolName, err) + Expect(storagePoolObj.Spec.Type).To(Equal(LVMThinName), "type of replicated storage pool should be `LVMThin`") + } + + if *sc.VolumeBindingMode != storagev1.VolumeBindingImmediate { + immediateStorageClassName, err = createImmediateStorageClass(sc.Provisioner, testCaseLabel) + Expect(err).NotTo(HaveOccurred(), "%s", err) + + virtualDiskWithoutConsumer := virtv2.VirtualDisk{} + vdWithoutConsumerFilePath := fmt.Sprintf("%s/vd/vd-alpine-http.yaml", conf.TestData.VdSnapshots) + err = UnmarshalResource(vdWithoutConsumerFilePath, &virtualDiskWithoutConsumer) + Expect(err).NotTo(HaveOccurred(), "cannot get object from file: %s\nstderr: %s", vdWithoutConsumerFilePath, err) + + virtualDiskWithoutConsumer.Spec.PersistentVolumeClaim.StorageClass = &immediateStorageClassName + err = WriteYamlObject(vdWithoutConsumerFilePath, &virtualDiskWithoutConsumer) + Expect(err).NotTo(HaveOccurred(), "cannot update virtual disk with custom storage class: %s\nstderr: %s", vdWithoutConsumerFilePath, err) + } else { + immediateStorageClassName = sc.Name + } + }) + }) + + Context("When virtualization resources are applied:", func() { + It("result should be succeeded", func() { + res := kubectl.Apply(kc.ApplyOptions{ + Filename: []string{conf.TestData.VdSnapshots}, + FilenameOption: kc.Kustomize, + }) + Expect(res.Error()).NotTo(HaveOccurred(), "cmd: %s\nstderr: %s", res.GetCmd(), res.StdErr()) + }) + }) + + Context("When virtual images are applied:", func() { + It("checks VIs phases", func() { + By(fmt.Sprintf("VIs should be in %s phases", PhaseReady)) + WaitPhaseByLabel(kc.ResourceVI, PhaseReady, kc.WaitOptions{ + Labels: testCaseLabel, + Namespace: conf.Namespace, + Timeout: MaxWaitTimeout, + }) + }) + }) + + Context("When virtual disks are applied:", func() { + It("checks VDs phases", func() { + By(fmt.Sprintf("VDs should be in %s phases", PhaseReady)) + WaitPhaseByLabel(kc.ResourceVD, PhaseReady, kc.WaitOptions{ + Labels: testCaseLabel, + Namespace: conf.Namespace, + Timeout: MaxWaitTimeout, + }) + }) + }) + + Context("When virtual machines are applied:", func() { + It("checks VMs phases", func() { + By(fmt.Sprintf("VMs should be in %s phases", PhaseRunning)) + WaitPhaseByLabel(kc.ResourceVM, PhaseRunning, kc.WaitOptions{ + Labels: testCaseLabel, + Namespace: conf.Namespace, + Timeout: MaxWaitTimeout, + }) + }) + }) + + Context("When virtual machine block device attachments are applied:", func() { + It("checks VMBDAs phases", func() { + By(fmt.Sprintf("VMBDAs should be in %s phases", PhaseAttached)) + WaitPhaseByLabel(kc.ResourceVMBDA, PhaseAttached, kc.WaitOptions{ + Labels: testCaseLabel, + Namespace: conf.Namespace, + Timeout: MaxWaitTimeout, + }) + }) + }) + + Context(fmt.Sprintf("When unattached VDs in phase %s:", PhaseReady), func() { + It("creates VDs snapshots with `requiredConsistency`", func() { + res := kubectl.List(kc.ResourceVD, kc.GetOptions{ + Labels: hasNoConsumerLabel, + Namespace: conf.Namespace, + Output: "jsonpath='{.items[*].metadata.name}'", + }) + Expect(res.Error()).NotTo(HaveOccurred(), "cmd: %s\nstderr: %s", res.GetCmd(), res.StdErr()) + + vds := strings.Split(res.StdOut(), " ") + + sc := storagev1.StorageClass{} + err := GetObject(kc.ResourceStorageClass, immediateStorageClassName, &sc, kc.GetOptions{}) + Expect(err).NotTo(HaveOccurred(), "cannot get storage class: %s\nstderr: %s", immediateStorageClassName, err) + + volumeSnapshotClassName, getErr := GetVolumeSnapshotClassName(&sc) + Expect(getErr).NotTo(HaveOccurred(), "%s", getErr) + + for _, vdName := range vds { + By(fmt.Sprintf("Create snapshot for %q with volume snapshot class %q", vdName, volumeSnapshotClassName)) + err := CreateVirtualDiskSnapshot(vdName, vdName, volumeSnapshotClassName, true, hasNoConsumerLabel) + Expect(err).NotTo(HaveOccurred(), "%s", err) + } + }) + + It("checks snapshots of unattached VDs", func() { + By(fmt.Sprintf("Snapshots should be in %s phase", PhaseReady)) + WaitPhaseByLabel(kc.ResourceVDSnapshot, PhaseReady, kc.WaitOptions{ + Labels: hasNoConsumerLabel, + Namespace: conf.Namespace, + Timeout: MaxWaitTimeout, + }) + By("Snapshots should be consistent", func() { + vdSnapshots := virtv2.VirtualDiskSnapshotList{} + err := GetObjects(kc.ResourceVDSnapshot, &vdSnapshots, kc.GetOptions{Namespace: conf.Namespace, Labels: hasNoConsumerLabel}) + Expect(err).NotTo(HaveOccurred(), "cannot get `vdSnapshots`\nstderr: %s", err) + + for _, snapshot := range vdSnapshots.Items { + Expect(*snapshot.Status.Consistent).To(BeTrue(), "consistent field should be `true`: %s", snapshot.Name) + } + }) + }) + }) + + Context(fmt.Sprintf("When virtual machines in %s phase", PhaseRunning), func() { + It("creates snapshots with `requiredConsistency` of attached VDs", func() { + vmObjects := virtv2.VirtualMachineList{} + err := GetObjects(kc.ResourceVM, &vmObjects, kc.GetOptions{Namespace: conf.Namespace}) + Expect(err).NotTo(HaveOccurred(), "cannot get virtual machines\nstderr: %s", err) + + for _, vm := range vmObjects.Items { + Eventually(func() error { + _, err := CheckFilesystemReadyStatus(vm.Name, v1.ConditionTrue) + return err + }).WithTimeout( + filesystemReadyTimeout, + ).WithPolling( + filesystemReadyPollingInterval, + ).Should(Succeed()) + + blockDevices := vm.Status.BlockDeviceRefs + for _, blockDevice := range blockDevices { + if blockDevice.Kind == virtv2.VirtualDiskKind { + By(fmt.Sprintf( + "Create snapshot for %q with volume snapshot class %q", + blockDevice.Name, + defaultVolumeSnapshotClassName, + )) + err := CreateVirtualDiskSnapshot(blockDevice.Name, blockDevice.Name, defaultVolumeSnapshotClassName, true, attachedVirtualDiskLabel) + Expect(err).NotTo(HaveOccurred(), "%s", err) + + Eventually(func() error { + reason, err := CheckFilesystemReadyStatus(vm.Name, v1.ConditionFalse) + if reason != "Frozen" { + return fmt.Errorf("`Filesystem` should be frozen when controller is snapshotting the attached virtual disk") + } + return err + }).WithTimeout( + filesystemReadyTimeout, + ).WithPolling( + frozenReasonPollingInterval, + ).Should(Succeed()) + } + } + } + }) + + It("creates `vdSnapshots` concurrently", func() { + vmObjects := virtv2.VirtualMachineList{} + err := GetObjects(kc.ResourceVM, &vmObjects, kc.GetOptions{ + Namespace: conf.Namespace, + Labels: vmAutomaticWithHotplug, + }) + Expect(err).NotTo(HaveOccurred(), "cannot get vmObject with label %q\nstderr: %s", vmAutomaticWithHotplug, err) + + for _, vm := range vmObjects.Items { + Eventually(func() error { + _, err := CheckFilesystemReadyStatus(vm.Name, v1.ConditionTrue) + return err + }).WithTimeout( + filesystemReadyTimeout, + ).WithPolling( + filesystemReadyPollingInterval, + ).Should(Succeed()) + + blockDevices := vm.Status.BlockDeviceRefs + for _, blockDevice := range blockDevices { + if blockDevice.Kind == virtv2.VirtualDiskKind { + By(fmt.Sprintf( + "Create five snapshots for %q of %q with volume snapshot class %q", + blockDevice.Name, + vm.Name, + defaultVolumeSnapshotClassName, + )) + errs := make([]error, 0, 5) + wg := sync.WaitGroup{} + for i := 0; i < 5; i++ { + wg.Add(1) + go func(index int) { + defer wg.Done() + snapshotName := fmt.Sprintf("%s-%d", blockDevice.Name, index) + err := CreateVirtualDiskSnapshot(blockDevice.Name, snapshotName, defaultVolumeSnapshotClassName, true, attachedVirtualDiskLabel) + if err != nil { + errs = append(errs, err) + } + }(i) + } + wg.Wait() + Expect(errs).To(BeEmpty(), "concurrent snapshotting error") + + Eventually(func() error { + reason, err := CheckFilesystemReadyStatus(vm.Name, v1.ConditionFalse) + if reason != "Frozen" { + return fmt.Errorf("`Filesystem` should be frozen when controller is snapshotting the attached virtual disk") + } + return err + }).WithTimeout( + filesystemReadyTimeout, + ).WithPolling( + frozenReasonPollingInterval, + ).Should(Succeed()) + } + } + } + }) + + It("checks snapshots of attached VDs", func() { + By(fmt.Sprintf("Snapshots should be in %s phase", PhaseReady)) + WaitPhaseByLabel(kc.ResourceVDSnapshot, PhaseReady, kc.WaitOptions{ + Labels: attachedVirtualDiskLabel, + Namespace: conf.Namespace, + Timeout: MaxWaitTimeout, + }) + By("Snapshots should be consistent", func() { + vdSnapshots := virtv2.VirtualDiskSnapshotList{} + err := GetObjects(kc.ResourceVDSnapshot, &vdSnapshots, kc.GetOptions{ + ExcludedLabels: []string{"hasNoConsumer"}, + Namespace: conf.Namespace, + Labels: attachedVirtualDiskLabel, + }) + Expect(err).NotTo(HaveOccurred(), "cannot get `vdSnapshots`\nstderr: %s", err) + + for _, snapshot := range vdSnapshots.Items { + Expect(*snapshot.Status.Consistent).To(BeTrue(), "consistent field should be `true`: %s", snapshot.Name) + } + }) + }) + + It("checks `FileSystemReady` status of VMs", func() { + By(fmt.Sprintf("Status should be %s", PhaseReady)) + vmObjects := virtv2.VirtualMachineList{} + err := GetObjects(kc.ResourceVM, &vmObjects, kc.GetOptions{Namespace: conf.Namespace}) + Expect(err).NotTo(HaveOccurred(), "cannot get virtual machines\nstderr: %s", err) + + for _, vm := range vmObjects.Items { + Eventually(func() error { + reason, err := CheckFilesystemReadyStatus(vm.Name, v1.ConditionTrue) + if err != nil { + return fmt.Errorf("vmName: %s\nstderr: %s\nreason: %s", vm.Name, err, reason) + } + return nil + }).WithTimeout( + filesystemReadyTimeout, + ).WithPolling( + filesystemReadyPollingInterval, + ).Should(Succeed()) + } + }) + }) + + Context("When test is completed", func() { + It("deletes test case resources", func() { + DeleteTestCaseResources(ResourcesToDelete{ + KustomizationDir: conf.TestData.VdSnapshots, + AdditionalResources: []AdditionalResource{ + { + Resource: kc.ResourceVDSnapshot, + Labels: hasNoConsumerLabel, + }, + { + Resource: kc.ResourceVDSnapshot, + Labels: attachedVirtualDiskLabel, + }, + { + Resource: kc.ResourceReplicatedStorageClass, + Labels: testCaseLabel, + }, + { + Resource: kc.ResourceStorageClass, + Labels: testCaseLabel, + }, + }, + }) + }) + }) +})