diff --git a/images/virtualization-controller/pkg/controller/vm_reconciler.go b/images/virtualization-controller/pkg/controller/vm_reconciler.go
index 7f0c287dc..7dbcd38ff 100644
--- a/images/virtualization-controller/pkg/controller/vm_reconciler.go
+++ b/images/virtualization-controller/pkg/controller/vm_reconciler.go
@@ -673,24 +673,9 @@ func (r *VMReconciler) syncMetadata(ctx context.Context, state *VMReconcilerStat
 		return nil
 	}
 
-	// Propagate user specified labels and annotations from the d8 VM to kubevirt VM.
-	metaUpdated, err := PropagateVMMetadata(state.VM.Current(), state.KVVM, state.KVVM)
-	if err != nil {
-		return err
-	}
-
-	// Ensure kubevirt VM has finalizer in case d8 VM was created manually (use case: take ownership of already existing object).
-	finalizerUpdated := controllerutil.AddFinalizer(state.KVVM, virtv2.FinalizerKVVMProtection)
-
-	if metaUpdated || finalizerUpdated {
-		if err = opts.Client.Update(ctx, state.KVVM); err != nil {
-			return fmt.Errorf("error setting finalizer on a KubeVirt VM %q: %w", state.KVVM.Name, err)
-		}
-	}
-
 	// Propagate user specified labels and annotations from the d8 VM to the kubevirt VirtualMachineInstance.
 	if state.KVVMI != nil {
-		metaUpdated, err = PropagateVMMetadata(state.VM.Current(), state.KVVM, state.KVVMI)
+		metaUpdated, err := PropagateVMMetadata(state.VM.Current(), state.KVVM, state.KVVMI)
 		if err != nil {
 			return err
 		}
@@ -709,7 +694,7 @@ func (r *VMReconciler) syncMetadata(ctx context.Context, state *VMReconcilerStat
 			if pod.Status.Phase != corev1.PodRunning {
 				continue
 			}
-			metaUpdated, err = PropagateVMMetadata(state.VM.Current(), state.KVVM, &pod)
+			metaUpdated, err := PropagateVMMetadata(state.VM.Current(), state.KVVM, &pod)
 			if err != nil {
 				return err
 			}
@@ -722,16 +707,31 @@ func (r *VMReconciler) syncMetadata(ctx context.Context, state *VMReconcilerStat
 		}
 	}
 
-	err = SetLastPropagatedLabels(state.KVVM, state.VM.Current())
+	// Propagate user specified labels and annotations from the d8 VM to kubevirt VM.
+	metaUpdated, err := PropagateVMMetadata(state.VM.Current(), state.KVVM, state.KVVM)
+	if err != nil {
+		return err
+	}
+
+	// Ensure kubevirt VM has finalizer in case d8 VM was created manually (use case: take ownership of already existing object).
+	finalizerUpdated := controllerutil.AddFinalizer(state.KVVM, virtv2.FinalizerKVVMProtection)
+
+	labelsChanged, err := SetLastPropagatedLabels(state.KVVM, state.VM.Current())
 	if err != nil {
 		return fmt.Errorf("failed to set last propagated labels: %w", err)
 	}
 
-	err = SetLastPropagatedAnnotations(state.KVVM, state.VM.Current())
+	annosChanged, err := SetLastPropagatedAnnotations(state.KVVM, state.VM.Current())
 	if err != nil {
 		return fmt.Errorf("failed to set last propagated annotations: %w", err)
 	}
 
+	if labelsChanged || annosChanged || metaUpdated || finalizerUpdated {
+		if err = opts.Client.Update(ctx, state.KVVM); err != nil {
+			return fmt.Errorf("error setting finalizer on a KubeVirt VM %q: %w", state.KVVM.Name, err)
+		}
+	}
+
 	return nil
 }
 
diff --git a/images/virtualization-controller/pkg/controller/vm_reconciler_state.go b/images/virtualization-controller/pkg/controller/vm_reconciler_state.go
index 2ca5328eb..5e92d78a1 100644
--- a/images/virtualization-controller/pkg/controller/vm_reconciler_state.go
+++ b/images/virtualization-controller/pkg/controller/vm_reconciler_state.go
@@ -536,15 +536,20 @@ func GetLastPropagatedLabels(kvvm *virtv1.VirtualMachine) (map[string]string, er
 	return lastPropagatedLabels, nil
 }
 
-func SetLastPropagatedLabels(kvvm *virtv1.VirtualMachine, vm *virtv2.VirtualMachine) error {
+func SetLastPropagatedLabels(kvvm *virtv1.VirtualMachine, vm *virtv2.VirtualMachine) (bool, error) {
 	data, err := json.Marshal(vm.GetLabels())
 	if err != nil {
-		return err
+		return false, err
 	}
 
-	common.AddLabel(kvvm, common.LastPropagatedVMLabelsAnnotation, string(data))
+	newAnnoValue := string(data)
 
-	return nil
+	if kvvm.Annotations[common.LastPropagatedVMLabelsAnnotation] == newAnnoValue {
+		return false, nil
+	}
+
+	common.AddAnnotation(kvvm, common.LastPropagatedVMLabelsAnnotation, newAnnoValue)
+	return true, nil
 }
 
 func GetLastPropagatedAnnotations(kvvm *virtv1.VirtualMachine) (map[string]string, error) {
@@ -560,13 +565,18 @@ func GetLastPropagatedAnnotations(kvvm *virtv1.VirtualMachine) (map[string]strin
 	return lastPropagatedAnno, nil
 }
 
-func SetLastPropagatedAnnotations(kvvm *virtv1.VirtualMachine, vm *virtv2.VirtualMachine) error {
+func SetLastPropagatedAnnotations(kvvm *virtv1.VirtualMachine, vm *virtv2.VirtualMachine) (bool, error) {
 	data, err := json.Marshal(RemoveNonPropagatableAnnotations(vm.GetAnnotations()))
 	if err != nil {
-		return err
+		return false, err
 	}
 
-	common.AddLabel(kvvm, common.LastPropagatedVMAnnotationsAnnotation, string(data))
+	newAnnoValue := string(data)
 
-	return nil
+	if kvvm.Annotations[common.LastPropagatedVMAnnotationsAnnotation] == newAnnoValue {
+		return false, nil
+	}
+
+	common.AddAnnotation(kvvm, common.LastPropagatedVMAnnotationsAnnotation, newAnnoValue)
+	return true, nil
 }