diff --git a/src/display/editor/tools.js b/src/display/editor/tools.js
index 541585dd7..e39006d71 100644
--- a/src/display/editor/tools.js
+++ b/src/display/editor/tools.js
@@ -637,6 +637,8 @@ class AnnotationEditorUIManager {
 
   #isEnabled = false;
 
+  #isPointerDown = false;
+
   #isWaiting = false;
 
   #keyboardManagerAC = null;
@@ -857,6 +859,20 @@ class AnnotationEditorUIManager {
       evt => this.updateParams(evt.type, evt.value),
       { signal }
     );
+    window.addEventListener(
+      "pointerdown",
+      () => {
+        this.#isPointerDown = true;
+      },
+      { capture: true, signal }
+    );
+    window.addEventListener(
+      "pointerup",
+      () => {
+        this.#isPointerDown = false;
+      },
+      { capture: true, signal }
+    );
     this.#addSelectionListener();
     this.#addDragAndDropListeners();
     this.#addKeyboardManager();
@@ -1299,22 +1315,30 @@ class AnnotationEditorUIManager {
           : null;
       activeLayer?.toggleDrawing();
 
-      const ac = new AbortController();
-      const signal = this.combinedSignal(ac);
+      if (this.#isPointerDown) {
+        const ac = new AbortController();
+        const signal = this.combinedSignal(ac);
 
-      const pointerup = e => {
-        if (e.type === "pointerup" && e.button !== 0) {
-          // Do nothing on right click.
-          return;
-        }
-        ac.abort();
+        const pointerup = e => {
+          if (e.type === "pointerup" && e.button !== 0) {
+            // Do nothing on right click.
+            return;
+          }
+          ac.abort();
+          activeLayer?.toggleDrawing(true);
+          if (e.type === "pointerup") {
+            this.#onSelectEnd("main_toolbar");
+          }
+        };
+        window.addEventListener("pointerup", pointerup, { signal });
+        window.addEventListener("blur", pointerup, { signal });
+      } else {
+        // Here neither the shift key nor the pointer is down and we've
+        // something in the selection: we can be in the case where the user is
+        // using a screen reader (see bug 1976597).
         activeLayer?.toggleDrawing(true);
-        if (e.type === "pointerup") {
-          this.#onSelectEnd("main_toolbar");
-        }
-      };
-      window.addEventListener("pointerup", pointerup, { signal });
-      window.addEventListener("blur", pointerup, { signal });
+        this.#onSelectEnd("main_toolbar");
+      }
     }
   }
 
diff --git a/test/integration/highlight_editor_spec.mjs b/test/integration/highlight_editor_spec.mjs
index 4a5977747..2c4dd0f6c 100644
--- a/test/integration/highlight_editor_spec.mjs
+++ b/test/integration/highlight_editor_spec.mjs
@@ -2801,4 +2801,61 @@ describe("Highlight Editor", () => {
       );
     });
   });
+
+  describe("Highlight the selection but without a mouse or a keyboard", () => {
+    let pages;
+
+    beforeEach(async () => {
+      pages = await loadAndWait(
+        "tracemonkey.pdf",
+        ".annotationEditorLayer",
+        null,
+        null,
+        { highlightEditorColors: "red=#AB0000" }
+      );
+    });
+
+    afterEach(async () => {
+      await closePages(pages);
+    });
+
+    it("must highlight with red color", async () => {
+      await Promise.all(
+        pages.map(async ([browserName, page]) => {
+          await switchToHighlight(page);
+
+          await page.evaluate(() => {
+            // Take the first span which contains "Trace-based Just-in-Time..."
+            const root = document.querySelector(
+              ".page[data-page-number='1'] > .textLayer"
+            );
+            const iter = document.createNodeIterator(
+              root,
+              NodeFilter.SHOW_TEXT
+            );
+            const textnode = iter.nextNode();
+            const selection = document.getSelection();
+            const range = document.createRange();
+            range.selectNodeContents(textnode);
+            selection.removeAllRanges();
+            selection.addRange(range);
+          });
+
+          await page.waitForSelector(`${getEditorSelector(0)}`);
+          await page.waitForSelector(
+            `.page[data-page-number = "1"] svg.highlightOutline.selected`
+          );
+
+          const usedColor = await page.evaluate(() => {
+            const highlight = document.querySelector(
+              `.page[data-page-number = "1"] .canvasWrapper > svg.highlight`
+            );
+            return highlight.getAttribute("fill");
+          });
+
+          expect(usedColor).withContext(`In ${browserName}`).toEqual("#AB0000");
+        })
+      );
+    });
+  });
 });