diff --git a/plugins/text-editor-resources/src/components/TextEditorToolbar.svelte b/plugins/text-editor-resources/src/components/TextEditorToolbar.svelte
index c51d60533c..97158ff4e7 100644
--- a/plugins/text-editor-resources/src/components/TextEditorToolbar.svelte
+++ b/plugins/text-editor-resources/src/components/TextEditorToolbar.svelte
@@ -97,7 +97,7 @@
 
   let selecting = false
 
-  function handleMouseDown (): void {
+  function handleMouseDown (event: MouseEvent): void {
     function handleMouseMove (): void {
       if (editor !== undefined && !editor.state.selection.empty) {
         selecting = true
@@ -112,9 +112,11 @@
       document.removeEventListener('mouseup', handleMouseUp)
     }
 
-    if (editor !== undefined) {
-      document.addEventListener('mousemove', handleMouseMove)
-      document.addEventListener('mouseup', handleMouseUp)
+    if (editor !== undefined && visible && visibleActions.length > 0) {
+      if (event.target !== null && toolbar !== null && !toolbar.contains(event.target as Node)) {
+        document.addEventListener('mousemove', handleMouseMove)
+        document.addEventListener('mouseup', handleMouseUp)
+      }
     }
   }