From 7b7e03aa313680bced3532cd9d93ddc7b316eacb Mon Sep 17 00:00:00 2001
From: yaroslavborbat <yaroslav.752@gmail.com>
Date: Tue, 10 Dec 2024 10:30:46 +0300
Subject: [PATCH] add

Signed-off-by: yaroslavborbat <yaroslav.752@gmail.com>
---
 Taskfile.yaml                                 |  33 +
 crds/embedded/virtualmachineinstances.yaml    |   2 +
 crds/embedded/virtualmachines.yaml            |  31 +
 .../patches/027-hotplug-container-disk.patch  | 724 ++++++++++++++++++
 images/virt-controller/debug/dlv.Dockerfile   |  31 +
 5 files changed, 821 insertions(+)
 create mode 100644 images/virt-artifact/patches/027-hotplug-container-disk.patch
 create mode 100644 images/virt-controller/debug/dlv.Dockerfile

diff --git a/Taskfile.yaml b/Taskfile.yaml
index 8af220c0a..63c0d44d5 100644
--- a/Taskfile.yaml
+++ b/Taskfile.yaml
@@ -180,3 +180,36 @@ tasks:
       }'
       kubectl -n d8-virtualization port-forward pod/<virt-handler-pod> 2345:2345
       EOF
+
+  dlv:virt-controller:build:
+    desc: "Build image virt-controller with dlv"
+    cmd: docker build -f ./images/virt-controller/debug/dlv.Dockerfile -t "{{ .DLV_IMAGE }}" .
+
+  dlv:virt-controller:build-push:
+    desc: "Build and Push image virt-controller with dlv"
+    cmds:
+      - task: dlv:virt-controller:build
+      - docker push "{{ .DLV_IMAGE }}"
+      - task: dlv:virt-controller:print
+
+  dlv:virt-controller:print:
+    desc: "Print commands for debug"
+    env:
+      IMAGE: "{{ .DLV_IMAGE }}"
+    cmd: |
+      cat <<EOF
+      kubectl -n d8-virtualization patch deploy virt-controller --type='strategic' -p '{
+        "spec": {
+          "template": {
+            "spec": {
+              "containers": [ {
+                "name": "virt-controller",
+                "image": "${IMAGE}",
+                "ports": [ { "containerPort": 2345, "name": "dlv" } ]
+              }]
+            }
+          }
+        }
+      }'
+      kubectl -n d8-virtualization port-forward deploy/virt-controller 2345:2345
+      EOF
diff --git a/crds/embedded/virtualmachineinstances.yaml b/crds/embedded/virtualmachineinstances.yaml
index c6624ebef..196797aa2 100644
--- a/crds/embedded/virtualmachineinstances.yaml
+++ b/crds/embedded/virtualmachineinstances.yaml
@@ -3076,6 +3076,8 @@ spec:
                         path:
                           description: Path defines the path to disk file in the container
                           type: string
+                        hotpluggable:
+                          type: boolean
                       required:
                       - image
                       type: object
diff --git a/crds/embedded/virtualmachines.yaml b/crds/embedded/virtualmachines.yaml
index d4774864c..b99399f64 100644
--- a/crds/embedded/virtualmachines.yaml
+++ b/crds/embedded/virtualmachines.yaml
@@ -3868,6 +3868,8 @@ spec:
                                   description: Path defines the path to disk file
                                     in the container
                                   type: string
+                                hotpluggable:
+                                  type: boolean
                               required:
                               - image
                               type: object
@@ -4491,6 +4493,35 @@ spec:
                           description: VolumeSource represents the source of the volume
                             to map to the disk.
                           properties:
+                            containerDisk:
+                              type: object
+                              description: |-
+                                ContainerDisk references a docker image, embedding a qcow or raw disk.
+                                More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html
+                              properties:
+                                image:
+                                  description: Image is the name of the image with
+                                    the embedded disk.
+                                  type: string
+                                imagePullPolicy:
+                                  description: |-
+                                    Image pull policy.
+                                    One of Always, Never, IfNotPresent.
+                                    Defaults to Always if :latest tag is specified, or IfNotPresent otherwise.
+                                    Cannot be updated.
+                                    More info: https://kubernetes.io/docs/concepts/containers/images#updating-images
+                                  type: string
+                                imagePullSecret:
+                                  description: ImagePullSecret is the name of the
+                                    Docker registry secret required to pull the image.
+                                    The secret must already exist.
+                                  type: string
+                                path:
+                                  description: Path defines the path to disk file
+                                    in the container
+                                  type: string
+                                hotpluggable:
+                                  type: boolean
                             dataVolume:
                               description: |-
                                 DataVolume represents the dynamic creation a PVC for this volume as well as
diff --git a/images/virt-artifact/patches/027-hotplug-container-disk.patch b/images/virt-artifact/patches/027-hotplug-container-disk.patch
new file mode 100644
index 000000000..399cc8cb3
--- /dev/null
+++ b/images/virt-artifact/patches/027-hotplug-container-disk.patch
@@ -0,0 +1,724 @@
+diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json
+index c4822a0448..d6bb534249 100644
+--- a/api/openapi-spec/swagger.json
++++ b/api/openapi-spec/swagger.json
+@@ -12951,6 +12951,9 @@
+      "image"
+     ],
+     "properties": {
++     "hotpluggable": {
++      "type": "boolean"
++     },
+      "image": {
+       "description": "Image is the name of the image with the embedded disk.",
+       "type": "string",
+@@ -13973,6 +13976,9 @@
+     "description": "HotplugVolumeSource Represents the source of a volume to mount which are capable of being hotplugged on a live running VMI. Only one of its members may be specified.",
+     "type": "object",
+     "properties": {
++     "containerDisk": {
++      "$ref": "#/definitions/v1.ContainerDiskSource"
++     },
+      "dataVolume": {
+       "description": "DataVolume represents the dynamic creation a PVC for this volume as well as the process of populating that PVC with a disk image.",
+       "$ref": "#/definitions/v1.DataVolumeSource"
+diff --git a/cmd/hp-container-disk/main.go b/cmd/hp-container-disk/main.go
+new file mode 100644
+index 0000000000..e4f734a516
+--- /dev/null
++++ b/cmd/hp-container-disk/main.go
+@@ -0,0 +1,16 @@
++package main
++
++import klog "kubevirt.io/client-go/log"
++
++func main() {
++	klog.InitializeLogging("virt-hp-container-disk")
++}
++
++type Config struct {
++	DstDir string  `json:"dstDir"`
++	Images []Image `json:"images"`
++}
++
++type Image struct {
++	Name string `json:"name"`
++}
+diff --git a/manifests/generated/kv-resource.yaml b/manifests/generated/kv-resource.yaml
+index 66d1b01dbf..43e36b7195 100644
+--- a/manifests/generated/kv-resource.yaml
++++ b/manifests/generated/kv-resource.yaml
+@@ -3307,9 +3307,6 @@ spec:
+     - jsonPath: .status.phase
+       name: Phase
+       type: string
+-    deprecated: true
+-    deprecationWarning: kubevirt.io/v1alpha3 is now deprecated and will be removed
+-      in a future release.
+     name: v1alpha3
+     schema:
+       openAPIV3Schema:
+diff --git a/manifests/generated/operator-csv.yaml.in b/manifests/generated/operator-csv.yaml.in
+index 400d118024..05ee099c67 100644
+--- a/manifests/generated/operator-csv.yaml.in
++++ b/manifests/generated/operator-csv.yaml.in
+@@ -605,6 +605,13 @@ spec:
+           - '*'
+           verbs:
+           - '*'
++        - apiGroups:
++          - subresources.virtualization.deckhouse.io
++          resources:
++          - virtualmachines/addvolume
++          - virtualmachines/removevolume
++          verbs:
++          - update
+         - apiGroups:
+           - subresources.kubevirt.io
+           resources:
+diff --git a/manifests/generated/rbac-operator.authorization.k8s.yaml.in b/manifests/generated/rbac-operator.authorization.k8s.yaml.in
+index 10dbb92269..1ccc9e9fa7 100644
+--- a/manifests/generated/rbac-operator.authorization.k8s.yaml.in
++++ b/manifests/generated/rbac-operator.authorization.k8s.yaml.in
+@@ -143,7 +143,7 @@ kind: RoleBinding
+ metadata:
+   labels:
+     kubevirt.io: ""
+-  name: kubevirt-operator-rolebinding
++  name: kubevirt-operator
+   namespace: {{.Namespace}}
+ roleRef:
+   apiGroup: rbac.authorization.k8s.io
+@@ -607,6 +607,13 @@ rules:
+   - '*'
+   verbs:
+   - '*'
++- apiGroups:
++  - subresources.virtualization.deckhouse.io
++  resources:
++  - virtualmachines/addvolume
++  - virtualmachines/removevolume
++  verbs:
++  - update
+ - apiGroups:
+   - subresources.kubevirt.io
+   resources:
+diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go
+index 490cc445ef..8b82437f83 100644
+--- a/pkg/controller/controller.go
++++ b/pkg/controller/controller.go
+@@ -444,6 +444,9 @@ func VMIHasHotplugVolumes(vmi *v1.VirtualMachineInstance) bool {
+ 		if volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.Hotpluggable {
+ 			return true
+ 		}
++		if volume.ContainerDisk != nil && volume.ContainerDisk.Hotpluggable {
++			return true
++		}
+ 	}
+ 	return false
+ }
+@@ -557,7 +560,7 @@ func GetHotplugVolumes(vmi *v1.VirtualMachineInstance, virtlauncherPod *k8sv1.Po
+ 		podVolumeMap[podVolume.Name] = podVolume
+ 	}
+ 	for _, vmiVolume := range vmiVolumes {
+-		if _, ok := podVolumeMap[vmiVolume.Name]; !ok && (vmiVolume.DataVolume != nil || vmiVolume.PersistentVolumeClaim != nil || vmiVolume.MemoryDump != nil) {
++		if _, ok := podVolumeMap[vmiVolume.Name]; !ok && (vmiVolume.DataVolume != nil || vmiVolume.PersistentVolumeClaim != nil || vmiVolume.MemoryDump != nil || vmiVolume.ContainerDisk != nil) {
+ 			hotplugVolumes = append(hotplugVolumes, vmiVolume.DeepCopy())
+ 		}
+ 	}
+diff --git a/pkg/virt-api/rest/subresource.go b/pkg/virt-api/rest/subresource.go
+index b5d62f5af5..fc49cbbb20 100644
+--- a/pkg/virt-api/rest/subresource.go
++++ b/pkg/virt-api/rest/subresource.go
+@@ -1125,6 +1125,8 @@ func (app *SubresourceAPIApp) addVolumeRequestHandler(request *restful.Request,
+ 		opts.VolumeSource.DataVolume.Hotpluggable = true
+ 	} else if opts.VolumeSource.PersistentVolumeClaim != nil {
+ 		opts.VolumeSource.PersistentVolumeClaim.Hotpluggable = true
++	} else if opts.VolumeSource.ContainerDisk != nil {
++		opts.VolumeSource.ContainerDisk.Hotpluggable = true
+ 	}
+ 
+ 	// inject into VMI if ephemeral, else set as a request on the VM to both make permanent and hotplug.
+diff --git a/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-update-admitter.go b/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-update-admitter.go
+index 0af25f8074..803c0ed4cd 100644
+--- a/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-update-admitter.go
++++ b/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-update-admitter.go
+@@ -200,11 +200,11 @@ func verifyHotplugVolumes(newHotplugVolumeMap, oldHotplugVolumeMap map[string]v1
+ 			}
+ 		} else {
+ 			// This is a new volume, ensure that the volume is either DV, PVC or memoryDumpVolume
+-			if v.DataVolume == nil && v.PersistentVolumeClaim == nil && v.MemoryDump == nil {
++			if v.DataVolume == nil && v.PersistentVolumeClaim == nil && v.MemoryDump == nil && v.ContainerDisk == nil {
+ 				return webhookutils.ToAdmissionResponse([]metav1.StatusCause{
+ 					{
+ 						Type:    metav1.CauseTypeFieldValueInvalid,
+-						Message: fmt.Sprintf("volume %s is not a PVC or DataVolume", k),
++						Message: fmt.Sprintf("volume %s is not a PVC,DataVolume,MemoryDumpVolume or ContainerDisk", k),
+ 					},
+ 				})
+ 			}
+diff --git a/pkg/virt-controller/services/template.go b/pkg/virt-controller/services/template.go
+index 76ed7307ec..1e9259c109 100644
+--- a/pkg/virt-controller/services/template.go
++++ b/pkg/virt-controller/services/template.go
+@@ -64,13 +64,15 @@ import (
+ )
+ 
+ const (
+-	containerDisks   = "container-disks"
+-	hotplugDisks     = "hotplug-disks"
+-	hookSidecarSocks = "hook-sidecar-sockets"
+-	varRun           = "/var/run"
+-	virtBinDir       = "virt-bin-share-dir"
+-	hotplugDisk      = "hotplug-disk"
+-	virtExporter     = "virt-exporter"
++	containerDisks        = "container-disks"
++	hotplugDisks          = "hotplug-disks"
++	hookSidecarSocks      = "hook-sidecar-sockets"
++	varRun                = "/var/run"
++	virtBinDir            = "virt-bin-share-dir"
++	hotplugDisk           = "hotplug-disk"
++	virtExporter          = "virt-exporter"
++	hotplugContainerDisks = "hotplug-container-disks"
++	HotplugContainerDisk  = "hotplug-container-disk-"
+ )
+ 
+ const KvmDevice = "devices.virtualization.deckhouse.io/kvm"
+@@ -846,6 +848,47 @@ func sidecarContainerName(i int) string {
+ 	return fmt.Sprintf("hook-sidecar-%d", i)
+ }
+ 
++func sidecarContainerHotplugContainerdDiskName(name string) string {
++	return fmt.Sprintf("%s%s", HotplugContainerDisk, name)
++}
++
++func (t *templateService) containerForHotplugContainerDisk(name string, cd *v1.ContainerDiskSource, vmi *v1.VirtualMachineInstance) k8sv1.Container {
++	runUser := int64(util.NonRootUID)
++	sharedMount := k8sv1.MountPropagationHostToContainer
++	path := fmt.Sprintf("/path/hp-%s", name)
++	command := []string{fmt.Sprintf("/init/usr/bin/container-disk --copy-path %s", path)}
++
++	return k8sv1.Container{
++		Name:      name,
++		Image:     cd.Image,
++		Command:   command,
++		Resources: hotplugContainerResourceRequirementsForVMI(vmi, t.clusterConfig),
++		SecurityContext: &k8sv1.SecurityContext{
++			AllowPrivilegeEscalation: pointer.Bool(false),
++			RunAsNonRoot:             pointer.Bool(true),
++			RunAsUser:                &runUser,
++			SeccompProfile: &k8sv1.SeccompProfile{
++				Type: k8sv1.SeccompProfileTypeRuntimeDefault,
++			},
++			Capabilities: &k8sv1.Capabilities{
++				Drop: []k8sv1.Capability{"ALL"},
++			},
++			SELinuxOptions: &k8sv1.SELinuxOptions{
++				Type:  t.clusterConfig.GetSELinuxLauncherType(),
++				Level: "s0",
++			},
++		},
++		VolumeMounts: []k8sv1.VolumeMount{
++			initContainerVolumeMount(),
++			{
++				Name:             hotplugContainerDisks,
++				MountPath:        "/path",
++				MountPropagation: &sharedMount,
++			},
++		},
++	}
++}
++
+ func (t *templateService) RenderHotplugAttachmentPodTemplate(volumes []*v1.Volume, ownerPod *k8sv1.Pod, vmi *v1.VirtualMachineInstance, claimMap map[string]*k8sv1.PersistentVolumeClaim) (*k8sv1.Pod, error) {
+ 	zero := int64(0)
+ 	runUser := int64(util.NonRootUID)
+@@ -924,6 +967,30 @@ func (t *templateService) RenderHotplugAttachmentPodTemplate(volumes []*v1.Volum
+ 			TerminationGracePeriodSeconds: &zero,
+ 		},
+ 	}
++	first := true
++	for _, vol := range vmi.Spec.Volumes {
++		if vol.ContainerDisk == nil || !vol.ContainerDisk.Hotpluggable {
++			continue
++		}
++		name := sidecarContainerHotplugContainerdDiskName(vol.Name)
++		pod.Spec.Containers = append(pod.Spec.Containers, t.containerForHotplugContainerDisk(name, vol.ContainerDisk, vmi))
++		if first {
++			first = false
++			userId := int64(util.NonRootUID)
++			initContainerCommand := []string{"/usr/bin/cp",
++				"/usr/bin/container-disk",
++				"/init/usr/bin/container-disk",
++			}
++			pod.Spec.InitContainers = append(
++				pod.Spec.InitContainers,
++				t.newInitContainerRenderer(vmi,
++					initContainerVolumeMount(),
++					initContainerResourceRequirementsForVMI(vmi, v1.ContainerDisk, t.clusterConfig),
++					userId).Render(initContainerCommand))
++			pod.Spec.Volumes = append(pod.Spec.Volumes, emptyDirVolume(hotplugContainerDisks))
++			pod.Spec.Volumes = append(pod.Spec.Volumes, emptyDirVolume(virtBinDir))
++		}
++	}
+ 
+ 	err := matchSELinuxLevelOfVMI(pod, vmi)
+ 	if err != nil {
+diff --git a/pkg/virt-controller/watch/BUILD.bazel b/pkg/virt-controller/watch/BUILD.bazel
+index 4fd325ba86..82fcaee0a3 100644
+--- a/pkg/virt-controller/watch/BUILD.bazel
++++ b/pkg/virt-controller/watch/BUILD.bazel
+@@ -101,6 +101,7 @@ go_library(
+         "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
+         "//vendor/k8s.io/client-go/util/workqueue:go_default_library",
+         "//vendor/k8s.io/utils/pointer:go_default_library",
++        "//vendor/k8s.io/utils/ptr:go_default_library",
+         "//vendor/k8s.io/utils/trace:go_default_library",
+         "//vendor/kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1:go_default_library",
+     ],
+diff --git a/pkg/virt-controller/watch/vmi.go b/pkg/virt-controller/watch/vmi.go
+index 0c4bfca389..f16d7c73b8 100644
+--- a/pkg/virt-controller/watch/vmi.go
++++ b/pkg/virt-controller/watch/vmi.go
+@@ -1816,6 +1816,10 @@ func (c *VMIController) handleHotplugVolumes(hotplugVolumes []*virtv1.Volume, ho
+ 	readyHotplugVolumes := make([]*virtv1.Volume, 0)
+ 	// Find all ready volumes
+ 	for _, volume := range hotplugVolumes {
++		if volume.ContainerDisk != nil {
++			readyHotplugVolumes = append(readyHotplugVolumes, volume)
++			continue
++		}
+ 		var err error
+ 		ready, wffc, err := storagetypes.VolumeReadyToAttachToNode(vmi.Namespace, *volume, dataVolumes, c.dataVolumeIndexer, c.pvcIndexer)
+ 		if err != nil {
+@@ -1873,10 +1877,20 @@ func (c *VMIController) podVolumesMatchesReadyVolumes(attachmentPod *k8sv1.Pod,
+ 			podVolumeMap[volume.Name] = volume
+ 		}
+ 	}
++	containerDisksNames := make(map[string]struct{})
++	for _, ctr := range attachmentPod.Spec.Containers {
++		if strings.HasPrefix(ctr.Name, services.HotplugContainerDisk) {
++			containerDisksNames[strings.TrimPrefix(ctr.Name, services.HotplugContainerDisk)] = struct{}{}
++		}
++	}
+ 	for _, volume := range volumes {
++		if volume.ContainerDisk != nil {
++			delete(containerDisksNames, volume.Name)
++			continue
++		}
+ 		delete(podVolumeMap, volume.Name)
+ 	}
+-	return len(podVolumeMap) == 0
++	return len(podVolumeMap) == 0 && len(containerDisksNames) == 0
+ }
+ 
+ func (c *VMIController) createAttachmentPod(vmi *virtv1.VirtualMachineInstance, virtLauncherPod *k8sv1.Pod, volumes []*virtv1.Volume) (*k8sv1.Pod, syncError) {
+@@ -2131,14 +2145,19 @@ func (c *VMIController) updateVolumeStatus(vmi *virtv1.VirtualMachineInstance, v
+ 					ClaimName: volume.Name,
+ 				}
+ 			}
++			if volume.ContainerDisk != nil && status.ContainerDiskVolume == nil {
++				status.ContainerDiskVolume = &virtv1.ContainerDiskInfo{}
++			}
+ 			if attachmentPod == nil {
+-				if !c.volumeReady(status.Phase) {
+-					status.HotplugVolume.AttachPodUID = ""
+-					// Volume is not hotplugged in VM and Pod is gone, or hasn't been created yet, check for the PVC associated with the volume to set phase and message
+-					phase, reason, message := c.getVolumePhaseMessageReason(&vmi.Spec.Volumes[i], vmi.Namespace)
+-					status.Phase = phase
+-					status.Message = message
+-					status.Reason = reason
++				if volume.ContainerDisk != nil {
++					if !c.volumeReady(status.Phase) {
++						status.HotplugVolume.AttachPodUID = ""
++						// Volume is not hotplugged in VM and Pod is gone, or hasn't been created yet, check for the PVC associated with the volume to set phase and message
++						phase, reason, message := c.getVolumePhaseMessageReason(&vmi.Spec.Volumes[i], vmi.Namespace)
++						status.Phase = phase
++						status.Message = message
++						status.Reason = reason
++					}
+ 				}
+ 			} else {
+ 				status.HotplugVolume.AttachPodName = attachmentPod.Name
+@@ -2156,7 +2175,6 @@ func (c *VMIController) updateVolumeStatus(vmi *virtv1.VirtualMachineInstance, v
+ 				}
+ 			}
+ 		}
+-
+ 		if volume.VolumeSource.PersistentVolumeClaim != nil || volume.VolumeSource.DataVolume != nil || volume.VolumeSource.MemoryDump != nil {
+ 
+ 			pvcName := storagetypes.PVCNameFromVirtVolume(&volume)
+diff --git a/pkg/virt-operator/resource/apply/BUILD.bazel b/pkg/virt-operator/resource/apply/BUILD.bazel
+index f6bd9bd4f1..fe6ab54f8c 100644
+--- a/pkg/virt-operator/resource/apply/BUILD.bazel
++++ b/pkg/virt-operator/resource/apply/BUILD.bazel
+@@ -4,7 +4,6 @@ go_library(
+     name = "go_default_library",
+     srcs = [
+         "admissionregistration.go",
+-        "apiservices.go",
+         "apps.go",
+         "certificates.go",
+         "core.go",
+@@ -65,7 +64,6 @@ go_library(
+         "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+         "//vendor/k8s.io/client-go/tools/record:go_default_library",
+         "//vendor/k8s.io/client-go/util/workqueue:go_default_library",
+-        "//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1:go_default_library",
+         "//vendor/k8s.io/utils/pointer:go_default_library",
+     ],
+ )
+diff --git a/pkg/virt-operator/resource/generate/components/BUILD.bazel b/pkg/virt-operator/resource/generate/components/BUILD.bazel
+index 70d2da0897..affcd3fecd 100644
+--- a/pkg/virt-operator/resource/generate/components/BUILD.bazel
++++ b/pkg/virt-operator/resource/generate/components/BUILD.bazel
+@@ -3,7 +3,6 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+ go_library(
+     name = "go_default_library",
+     srcs = [
+-        "apiservices.go",
+         "crds.go",
+         "daemonsets.go",
+         "deployments.go",
+@@ -62,7 +61,6 @@ go_library(
+         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+         "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+         "//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
+-        "//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1:go_default_library",
+         "//vendor/k8s.io/utils/pointer:go_default_library",
+     ],
+ )
+@@ -70,7 +68,6 @@ go_library(
+ go_test(
+     name = "go_default_test",
+     srcs = [
+-        "apiservices_test.go",
+         "components_suite_test.go",
+         "crds_test.go",
+         "deployments_test.go",
+@@ -85,7 +82,6 @@ go_test(
+     deps = [
+         "//pkg/certificates/bootstrap:go_default_library",
+         "//pkg/certificates/triple/cert:go_default_library",
+-        "//staging/src/kubevirt.io/api/core/v1:go_default_library",
+         "//staging/src/kubevirt.io/client-go/testutils:go_default_library",
+         "//vendor/github.com/onsi/ginkgo/v2:go_default_library",
+         "//vendor/github.com/onsi/gomega:go_default_library",
+diff --git a/pkg/virt-operator/resource/generate/components/validations_generated.go b/pkg/virt-operator/resource/generate/components/validations_generated.go
+index 4913dbead0..42225780ba 100644
+--- a/pkg/virt-operator/resource/generate/components/validations_generated.go
++++ b/pkg/virt-operator/resource/generate/components/validations_generated.go
+@@ -7723,6 +7723,8 @@ var CRDsValidation map[string]string = map[string]string{
+                           ContainerDisk references a docker image, embedding a qcow or raw disk.
+                           More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html
+                         properties:
++                          hotpluggable:
++                            type: boolean
+                           image:
+                             description: Image is the name of the image with the embedded
+                               disk.
+@@ -8355,6 +8357,35 @@ var CRDsValidation map[string]string = map[string]string{
+                     description: VolumeSource represents the source of the volume
+                       to map to the disk.
+                     properties:
++                      containerDisk:
++                        description: Represents a docker image with an embedded disk.
++                        properties:
++                          hotpluggable:
++                            type: boolean
++                          image:
++                            description: Image is the name of the image with the embedded
++                              disk.
++                            type: string
++                          imagePullPolicy:
++                            description: |-
++                              Image pull policy.
++                              One of Always, Never, IfNotPresent.
++                              Defaults to Always if :latest tag is specified, or IfNotPresent otherwise.
++                              Cannot be updated.
++                              More info: https://kubernetes.io/docs/concepts/containers/images#updating-images
++                            type: string
++                          imagePullSecret:
++                            description: ImagePullSecret is the name of the Docker
++                              registry secret required to pull the image. The secret
++                              must already exist.
++                            type: string
++                          path:
++                            description: Path defines the path to disk file in the
++                              container
++                            type: string
++                        required:
++                        - image
++                        type: object
+                       dataVolume:
+                         description: |-
+                           DataVolume represents the dynamic creation a PVC for this volume as well as
+@@ -12768,6 +12799,8 @@ var CRDsValidation map[string]string = map[string]string{
+                   ContainerDisk references a docker image, embedding a qcow or raw disk.
+                   More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html
+                 properties:
++                  hotpluggable:
++                    type: boolean
+                   image:
+                     description: Image is the name of the image with the embedded
+                       disk.
+@@ -18328,6 +18361,8 @@ var CRDsValidation map[string]string = map[string]string{
+                           ContainerDisk references a docker image, embedding a qcow or raw disk.
+                           More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html
+                         properties:
++                          hotpluggable:
++                            type: boolean
+                           image:
+                             description: Image is the name of the image with the embedded
+                               disk.
+@@ -22835,6 +22870,8 @@ var CRDsValidation map[string]string = map[string]string{
+                                   ContainerDisk references a docker image, embedding a qcow or raw disk.
+                                   More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html
+                                 properties:
++                                  hotpluggable:
++                                    type: boolean
+                                   image:
+                                     description: Image is the name of the image with
+                                       the embedded disk.
+@@ -28015,6 +28052,8 @@ var CRDsValidation map[string]string = map[string]string{
+                                       ContainerDisk references a docker image, embedding a qcow or raw disk.
+                                       More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html
+                                     properties:
++                                      hotpluggable:
++                                        type: boolean
+                                       image:
+                                         description: Image is the name of the image
+                                           with the embedded disk.
+@@ -28673,6 +28712,36 @@ var CRDsValidation map[string]string = map[string]string{
+                                 description: VolumeSource represents the source of
+                                   the volume to map to the disk.
+                                 properties:
++                                  containerDisk:
++                                    description: Represents a docker image with an
++                                      embedded disk.
++                                    properties:
++                                      hotpluggable:
++                                        type: boolean
++                                      image:
++                                        description: Image is the name of the image
++                                          with the embedded disk.
++                                        type: string
++                                      imagePullPolicy:
++                                        description: |-
++                                          Image pull policy.
++                                          One of Always, Never, IfNotPresent.
++                                          Defaults to Always if :latest tag is specified, or IfNotPresent otherwise.
++                                          Cannot be updated.
++                                          More info: https://kubernetes.io/docs/concepts/containers/images#updating-images
++                                        type: string
++                                      imagePullSecret:
++                                        description: ImagePullSecret is the name of
++                                          the Docker registry secret required to pull
++                                          the image. The secret must already exist.
++                                        type: string
++                                      path:
++                                        description: Path defines the path to disk
++                                          file in the container
++                                        type: string
++                                    required:
++                                    - image
++                                    type: object
+                                   dataVolume:
+                                     description: |-
+                                       DataVolume represents the dynamic creation a PVC for this volume as well as
+diff --git a/pkg/virt-operator/resource/generate/install/generated_mock_strategy.go b/pkg/virt-operator/resource/generate/install/generated_mock_strategy.go
+index 5f1e9a3121..1fa1416af0 100644
+--- a/pkg/virt-operator/resource/generate/install/generated_mock_strategy.go
++++ b/pkg/virt-operator/resource/generate/install/generated_mock_strategy.go
+@@ -241,16 +241,6 @@ func (_mr *_MockStrategyInterfaceRecorder) MutatingWebhookConfigurations() *gomo
+ 	return _mr.mock.ctrl.RecordCall(_mr.mock, "MutatingWebhookConfigurations")
+ }
+ 
+-func (_m *MockStrategyInterface) APIServices() []*v18.APIService {
+-	ret := _m.ctrl.Call(_m, "APIServices")
+-	ret0, _ := ret[0].([]*v18.APIService)
+-	return ret0
+-}
+-
+-func (_mr *_MockStrategyInterfaceRecorder) APIServices() *gomock.Call {
+-	return _mr.mock.ctrl.RecordCall(_mr.mock, "APIServices")
+-}
+-
+ func (_m *MockStrategyInterface) CertificateSecrets() []*v14.Secret {
+ 	ret := _m.ctrl.Call(_m, "CertificateSecrets")
+ 	ret0, _ := ret[0].([]*v14.Secret)
+diff --git a/pkg/virt-operator/resource/generate/rbac/exportproxy.go b/pkg/virt-operator/resource/generate/rbac/exportproxy.go
+index ebc9f2adbd..a0dc0586b4 100644
+--- a/pkg/virt-operator/resource/generate/rbac/exportproxy.go
++++ b/pkg/virt-operator/resource/generate/rbac/exportproxy.go
+@@ -23,6 +23,7 @@ import (
+ 	rbacv1 "k8s.io/api/rbac/v1"
+ 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ 	"k8s.io/apimachinery/pkg/runtime"
++
+ 	"kubevirt.io/kubevirt/pkg/virt-operator/resource/generate/components"
+ 
+ 	virtv1 "kubevirt.io/api/core/v1"
+diff --git a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.json b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.json
+index b651173636..3453dfb0da 100644
+--- a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.json
++++ b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.json
+@@ -754,7 +754,8 @@
+               "image": "imageValue",
+               "imagePullSecret": "imagePullSecretValue",
+               "path": "pathValue",
+-              "imagePullPolicy": "imagePullPolicyValue"
++              "imagePullPolicy": "imagePullPolicyValue",
++              "hotpluggable": true
+             },
+             "ephemeral": {
+               "persistentVolumeClaim": {
+@@ -1209,6 +1210,13 @@
+             "dataVolume": {
+               "name": "nameValue",
+               "hotpluggable": true
++            },
++            "containerDisk": {
++              "image": "imageValue",
++              "imagePullSecret": "imagePullSecretValue",
++              "path": "pathValue",
++              "imagePullPolicy": "imagePullPolicyValue",
++              "hotpluggable": true
+             }
+           },
+           "dryRun": [
+diff --git a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.yaml b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.yaml
+index 53dfdacc3b..8b23193158 100644
+--- a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.yaml
++++ b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.yaml
+@@ -719,6 +719,7 @@ spec:
+           optional: true
+           volumeLabel: volumeLabelValue
+         containerDisk:
++          hotpluggable: true
+           image: imageValue
+           imagePullPolicy: imagePullPolicyValue
+           imagePullSecret: imagePullSecretValue
+@@ -838,6 +839,12 @@ status:
+       - dryRunValue
+       name: nameValue
+       volumeSource:
++        containerDisk:
++          hotpluggable: true
++          image: imageValue
++          imagePullPolicy: imagePullPolicyValue
++          imagePullSecret: imagePullSecretValue
++          path: pathValue
+         dataVolume:
+           hotpluggable: true
+           name: nameValue
+diff --git a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.json b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.json
+index 3be904512c..f595798e89 100644
+--- a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.json
++++ b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.json
+@@ -694,7 +694,8 @@
+           "image": "imageValue",
+           "imagePullSecret": "imagePullSecretValue",
+           "path": "pathValue",
+-          "imagePullPolicy": "imagePullPolicyValue"
++          "imagePullPolicy": "imagePullPolicyValue",
++          "hotpluggable": true
+         },
+         "ephemeral": {
+           "persistentVolumeClaim": {
+diff --git a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.yaml b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.yaml
+index 6fd2ab6523..b6457ec94d 100644
+--- a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.yaml
++++ b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.yaml
+@@ -524,6 +524,7 @@ spec:
+       optional: true
+       volumeLabel: volumeLabelValue
+     containerDisk:
++      hotpluggable: true
+       image: imageValue
+       imagePullPolicy: imagePullPolicyValue
+       imagePullSecret: imagePullSecretValue
+diff --git a/staging/src/kubevirt.io/api/core/v1/BUILD.bazel b/staging/src/kubevirt.io/api/core/v1/BUILD.bazel
+index f8615293a3..0c6c166985 100644
+--- a/staging/src/kubevirt.io/api/core/v1/BUILD.bazel
++++ b/staging/src/kubevirt.io/api/core/v1/BUILD.bazel
+@@ -28,7 +28,6 @@ go_library(
+         "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+         "//vendor/k8s.io/utils/net:go_default_library",
+-        "//vendor/k8s.io/utils/pointer:go_default_library",
+         "//vendor/kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1:go_default_library",
+     ],
+ )
+diff --git a/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go b/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go
+index abd5a495d6..7372b22a9a 100644
+--- a/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go
++++ b/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go
+@@ -1948,6 +1948,11 @@ func (in *HotplugVolumeSource) DeepCopyInto(out *HotplugVolumeSource) {
+ 		*out = new(DataVolumeSource)
+ 		**out = **in
+ 	}
++	if in.ContainerDisk != nil {
++		in, out := &in.ContainerDisk, &out.ContainerDisk
++		*out = new(ContainerDiskSource)
++		**out = **in
++	}
+ 	return
+ }
+ 
+diff --git a/staging/src/kubevirt.io/api/core/v1/schema.go b/staging/src/kubevirt.io/api/core/v1/schema.go
+index 29aa3932d3..302ed9ffde 100644
+--- a/staging/src/kubevirt.io/api/core/v1/schema.go
++++ b/staging/src/kubevirt.io/api/core/v1/schema.go
+@@ -854,6 +854,8 @@ type HotplugVolumeSource struct {
+ 	// the process of populating that PVC with a disk image.
+ 	// +optional
+ 	DataVolume *DataVolumeSource `json:"dataVolume,omitempty"`
++
++	ContainerDisk *ContainerDiskSource `json:"containerDisk,omitempty"`
+ }
+ 
+ type DataVolumeSource struct {
+@@ -911,6 +913,8 @@ type ContainerDiskSource struct {
+ 	// More info: https://kubernetes.io/docs/concepts/containers/images#updating-images
+ 	// +optional
+ 	ImagePullPolicy v1.PullPolicy `json:"imagePullPolicy,omitempty"`
++
++	Hotpluggable bool `json:"hotpluggable,omitempty"`
+ }
+ 
+ // Exactly one of its members must be set.
+diff --git a/staging/src/kubevirt.io/client-go/api/openapi_generated.go b/staging/src/kubevirt.io/client-go/api/openapi_generated.go
+index cc2d743492..b982b1620c 100644
+--- a/staging/src/kubevirt.io/client-go/api/openapi_generated.go
++++ b/staging/src/kubevirt.io/client-go/api/openapi_generated.go
+@@ -17772,6 +17772,12 @@ func schema_kubevirtio_api_core_v1_ContainerDiskSource(ref common.ReferenceCallb
+ 							Enum:        []interface{}{"Always", "IfNotPresent", "Never"},
+ 						},
+ 					},
++					"hotpluggable": {
++						SchemaProps: spec.SchemaProps{
++							Type:   []string{"boolean"},
++							Format: "",
++						},
++					},
+ 				},
+ 				Required: []string{"image"},
+ 			},
+@@ -19645,11 +19651,16 @@ func schema_kubevirtio_api_core_v1_HotplugVolumeSource(ref common.ReferenceCallb
+ 							Ref:         ref("kubevirt.io/api/core/v1.DataVolumeSource"),
+ 						},
+ 					},
++					"containerDisk": {
++						SchemaProps: spec.SchemaProps{
++							Ref: ref("kubevirt.io/api/core/v1.ContainerDiskSource"),
++						},
++					},
+ 				},
+ 			},
+ 		},
+ 		Dependencies: []string{
+-			"kubevirt.io/api/core/v1.DataVolumeSource", "kubevirt.io/api/core/v1.PersistentVolumeClaimVolumeSource"},
++			"kubevirt.io/api/core/v1.ContainerDiskSource", "kubevirt.io/api/core/v1.DataVolumeSource", "kubevirt.io/api/core/v1.PersistentVolumeClaimVolumeSource"},
+ 	}
+ }
+ 
diff --git a/images/virt-controller/debug/dlv.Dockerfile b/images/virt-controller/debug/dlv.Dockerfile
new file mode 100644
index 000000000..fc947a001
--- /dev/null
+++ b/images/virt-controller/debug/dlv.Dockerfile
@@ -0,0 +1,31 @@
+FROM golang:1.22.7 AS builder
+
+ENV VERSION="1.3.1"
+ENV GOVERSION="1.22.7"
+
+RUN go install github.com/go-delve/delve/cmd/dlv@latest
+
+RUN git clone --depth 1 --branch v$VERSION https://github.com/kubevirt/kubevirt.git /kubevirt
+COPY ./images/virt-artifact/patches /patches
+WORKDIR /kubevirt
+RUN for p in /patches/*.patch ; do git apply  --ignore-space-change --ignore-whitespace ${p} && echo OK || (echo FAIL ; exit 1) ; done
+
+RUN go mod edit -go=$GOVERSION && \
+    go mod download
+
+RUN go mod vendor
+
+ENV GO111MODULE=on
+ENV GOOS=linux
+ENV CGO_ENABLED=0
+ENV GOARCH=amd64
+
+RUN go build -o /kubevirt-binaries/virt-controller ./cmd/virt-controller/
+
+FROM busybox
+
+WORKDIR /app
+COPY --from=builder /kubevirt-binaries/virt-controller /app/virt-controller
+COPY --from=builder /go/bin/dlv /app/dlv
+USER 65532:65532
+ENTRYPOINT ["./dlv", "--listen=:2345", "--headless=true", "--continue", "--log=true", "--log-output=debugger,debuglineerr,gdbwire,lldbout,rpc", "--accept-multiclient", "--api-version=2", "exec", "/app/virt-controller", "--"]