diff --git a/app/assets/javascripts/components/annotations/machine_annotation.ts b/app/assets/javascripts/components/annotations/machine_annotation.ts
index 93e744e0b3..2020e0bad4 100644
--- a/app/assets/javascripts/components/annotations/machine_annotation.ts
+++ b/app/assets/javascripts/components/annotations/machine_annotation.ts
@@ -31,8 +31,8 @@ export class MachineAnnotation extends ShadowlessLitElement {
     render(): TemplateResult {
         return html`
             <div class="annotation machine-annotation ${this.data.type}"
-                 @mouseenter="${() => annotationState.hoveredAnnotation = this.data}"
-                 @mouseleave="${() => annotationState.hoveredAnnotation = null}">
+                 @mouseenter="${() => annotationState.setHovered(this.data, true)}"
+                 @mouseleave="${() => annotationState.setHovered(this.data, false)}">
                 <div class="annotation-header">
                     <span class="annotation-meta">
                         ${I18n.t(`js.annotation.type.${this.data.type}`)}
diff --git a/app/assets/javascripts/components/annotations/thread.ts b/app/assets/javascripts/components/annotations/thread.ts
index 910296a278..cd8481cb5b 100644
--- a/app/assets/javascripts/components/annotations/thread.ts
+++ b/app/assets/javascripts/components/annotations/thread.ts
@@ -88,8 +88,8 @@ export class Thread extends i18nMixin(ShadowlessLitElement) {
     render(): TemplateResult {
         return this.data ? html`
             <div class="thread ${annotationState.isVisible(this.data) ? "" : "hidden"}"
-                 @mouseenter="${() => annotationState.hoveredAnnotation = this.data}"
-                 @mouseleave="${() => annotationState.hoveredAnnotation = null}"
+                 @mouseenter="${() => annotationState.setHovered(this.data, true)}"
+                 @mouseleave="${() => annotationState.setHovered(this.data, false)}"
             >
                 <d-user-annotation .data=${this.data}></d-user-annotation>
                 ${this.data.responses.map(response => html`
diff --git a/app/assets/javascripts/state/Annotations.ts b/app/assets/javascripts/state/Annotations.ts
index 98665d8206..2169dd72f5 100644
--- a/app/assets/javascripts/state/Annotations.ts
+++ b/app/assets/javascripts/state/Annotations.ts
@@ -2,6 +2,7 @@ import { MachineAnnotationData } from "state/MachineAnnotations";
 import { AnnotationType, UserAnnotationData } from "state/UserAnnotations";
 import { State } from "state/state_system/State";
 import { stateProperty } from "state/state_system/StateProperty";
+import { StateMap } from "state/state_system/StateMap";
 
 export type AnnotationVisibilityOptions = "all" | "important" | "none";
 export type AnnotationData = MachineAnnotationData | UserAnnotationData;
@@ -25,24 +26,28 @@ export function isUserAnnotation(annotation: AnnotationData): annotation is User
 class AnnotationState extends State {
     @stateProperty visibility: AnnotationVisibilityOptions = "all";
     @stateProperty isQuestionMode = false;
-    @stateProperty hoveredAnnotation: AnnotationData | null = null;
+    readonly isHoveredByKey = new StateMap<string, boolean>();
+
+    private getKeyFromAnnotation(annotation: AnnotationData): string {
+        if (annotation === undefined || annotation === null) {
+            return "";
+        }
+
+        if (isUserAnnotation(annotation)) {
+            return `${annotation.id}`;
+        }
+
+        return `${annotation.row}-${annotation.column}-${annotation.type}-${annotation.text}`;
+    }
+
+    setHovered(annotation: AnnotationData, hovered: boolean): void {
+        const key = this.getKeyFromAnnotation(annotation);
+        this.isHoveredByKey.set(key, hovered);
+    }
 
     isHovered = (annotation: AnnotationData): boolean => {
-        return this.hoveredAnnotation === annotation || (
-            this.hoveredAnnotation !== null &&
-            annotation !== null &&
-            annotation !== undefined && ((
-                isUserAnnotation(annotation) &&
-                isUserAnnotation(this.hoveredAnnotation) &&
-                annotation.id === this.hoveredAnnotation.id
-            ) || (
-                !isUserAnnotation(annotation) &&
-                !isUserAnnotation(this.hoveredAnnotation) &&
-                annotation.row === this.hoveredAnnotation.row &&
-                annotation.column === this.hoveredAnnotation.column &&
-                annotation.type === this.hoveredAnnotation.type &&
-                annotation.text === this.hoveredAnnotation.text
-            )));
+        const key = this.getKeyFromAnnotation(annotation);
+        return this.isHoveredByKey.get(key) ?? false;
     };
 
     isVisible(annotation: AnnotationData): boolean {