diff --git a/packages/text-annotator/src/SelectionHandler.ts b/packages/text-annotator/src/SelectionHandler.ts
index 801e0f1c..b61d6dc4 100644
--- a/packages/text-annotator/src/SelectionHandler.ts
+++ b/packages/text-annotator/src/SelectionHandler.ts
@@ -149,6 +149,9 @@ export const SelectionHandler = (
      */
     if (store.getAnnotation(currentTarget.annotation)) {
       store.updateTarget(currentTarget, Origin.LOCAL);
+    } else {
+      // Proper lifecycle management: clear the previous selection first...
+      selection.clear();
     }
   });
 
@@ -206,7 +209,7 @@ export const SelectionHandler = (
         if (selected.length !== 1 || selected[0].id !== hovered.id) {
           selection.userSelect(hovered.id, evt);
         }
-      } else if (!selection.isEmpty()) {
+      } else {
         selection.clear();
       }
     };