fix: Support Google Docs annotated canvas mode (#865)

- Adds support for extracting annotated text
- Unconditionally, enables annotated canvas mode in Google Docs
- Adds new background.ts tests, since all previous tests were html fallback related.

Fixes #852

.eslintrc.json

@@ -62,6 +62,11 @@
           { "allow": ["private-constructors"] }
         ],
         "tsdoc/syntax": "error"
+      },
+      "settings": {
+        "import/resolver": {
+          "typescript": {}
+        }
       }
     },
     {

extension/background.ts

@@ -39,19 +39,6 @@ chrome.runtime.onMessage.addListener(async (request, sender, response) => {
       }
       rcxMain.onTabSelect(sender.tab.id);
       break;
-    case 'forceDocsHtml?':
-      console.log('forceDocsHtml?');
-      if (rcxMain.enabled === 1) {
-        response(true);
-        chrome.tabs.sendMessage(sender.tab!.id!, {
-          type: 'showPopup',
-          text: `
-            rikaikun is forcing Google Docs to render using HTML instead of canvas.<br>
-            rikaikun can't work with canvas mode but if you need that mode, please disable rikaikun.
-          `,
-        });
-      }
-      break;
     case 'xsearch':
       console.log('xsearch');
       response(rcxMain.search(request.text, request.dictOption));

extension/docs-annotate-canvas.ts

@@ -0,0 +1,10 @@
+const injectedCode = `(function() {window['_docs_annotate_canvas_by_ext'] = '${chrome.runtime.id}';})();`;
+
+const script = document.createElement('script');
+script.textContent = injectedCode;
+(document.head || document.documentElement).appendChild(script);
+script.remove();
+
+// Empty export to satisfy `isolatedModules` compiler flag and allow including in tests.
+// Removed in production builds.
+export {};

extension/rikaicontent.ts

@@ -73,6 +73,9 @@ class RcxContent {
   private defaultDict = 2;
   private nextDict = 3;
 
+  private isGoogleDocPage =
+    document.querySelector('.kix-canvas-tile-content') !== null;
+
   // Adds the listeners and stuff.
   enableTab(config: Config) {
     if (window.rikaichan === undefined) {
@@ -709,6 +712,14 @@ class RcxContent {
     selEndList: { node: CharacterData; offset: number }[],
     maxLength: number
   ) {
+    // <rect> elements come from annotated google docs
+    if (rangeParent.nodeName === 'rect') {
+      return this.getTextFromGDoc(
+        rangeParent as SVGRectElement,
+        offset,
+        maxLength
+      );
+    }
     if (
       rangeParent.nodeName === 'TEXTAREA' ||
       rangeParent.nodeName === 'INPUT'
@@ -771,6 +782,52 @@ class RcxContent {
   // Hack because ro was coming out always 0 for some reason.
   lastRo = 0;
 
+  private getTextFromGDoc(
+    initialRect: SVGRectElement,
+    offset: number,
+    maxLength: number
+  ): string {
+    // Get text from initial rect.
+    const endIndex = Math.min(initialRect.ariaLabel.length, offset + maxLength);
+    let text = initialRect.ariaLabel.substring(offset, endIndex);
+
+    // Append text from sibling and cousin rects.
+    const rectWalker = this.createGDocTreeWalker(initialRect);
+    let rectNode: Node | null;
+    rectWalker.currentNode = initialRect;
+    while (
+      (rectNode = rectWalker.nextNode()) !== null &&
+      text.length < maxLength
+    ) {
+      const rect = rectNode as SVGRectElement;
+      const rectEndIndex = Math.min(
+        rect.ariaLabel.length,
+        maxLength - text.length
+      );
+      text += rect.ariaLabel.substring(0, rectEndIndex);
+    }
+    return text;
+  }
+
+  private createGDocTreeWalker(rect: SVGRectElement): TreeWalker {
+    return document.createTreeWalker(
+      rect.parentNode!.parentNode!, // rect is in g is in svg always
+      NodeFilter.SHOW_ELEMENT,
+      {
+        acceptNode: function (node) {
+          if (node.nodeName === 'rect') {
+            return NodeFilter.FILTER_ACCEPT;
+          }
+          if (node.nodeName === 'g') {
+            // Don't include g elements but consider their children.
+            return NodeFilter.FILTER_SKIP;
+          }
+          return NodeFilter.FILTER_REJECT;
+        },
+      }
+    );
+  }
+
   show(tdata: Rikaichan, dictOption: number) {
     const rp = tdata.prevRangeNode;
     let ro = tdata.prevRangeOfs! + tdata.uofs!;
@@ -848,8 +905,10 @@ class RcxContent {
 
     const rp = tdata.prevRangeNode;
     // don't try to highlight form elements
+    // don't try to highlight google docs
     if (
       rp &&
+      !this.isGoogleDoc(rp) &&
       ((tdata.config.highlight &&
         !this.mDown &&
         !('form' in tdata.prevTarget!)) ||
@@ -1010,10 +1069,16 @@ class RcxContent {
     //
   }
 
-  makeFake(real: HTMLTextAreaElement | HTMLInputElement) {
+  makeFake(real: HTMLTextAreaElement | HTMLInputElement | SVGRectElement) {
     const fake = document.createElement('div');
     const realRect = real.getBoundingClientRect();
-    fake.innerText = real.value;
+    let textValue = '';
+    if (real instanceof SVGRectElement) {
+      textValue = real.ariaLabel;
+    } else {
+      textValue = real.value;
+    }
+    fake.innerText = textValue;
     fake.style.cssText = document.defaultView!.getComputedStyle(
       real,
       ''
@@ -1069,6 +1134,7 @@ class RcxContent {
     }
 
     let fake;
+    let gdocRect: SVGRectElement | undefined;
     const tdata = window.rikaichan!; // per-tab data
     let range;
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -1081,16 +1147,24 @@ class RcxContent {
       HTMLSelectElement;
     // Put this in a try catch so that an exception here doesn't prevent editing due to div.
     try {
+      if (this.isGoogleDoc(eventTarget)) {
+        gdocRect = this.getRectUnderMouse(ev);
+        if (gdocRect) {
+          fake = this.makeFake(gdocRect);
+          fake.style.font = gdocRect.getAttribute('data-font-css')!;
+        }
+      }
+
       if (
         eventTarget.nodeName === 'TEXTAREA' ||
         eventTarget.nodeName === 'INPUT'
       ) {
         fake = this.makeFake(
           eventTarget as HTMLTextAreaElement | HTMLInputElement
         );
+      }
+      if (fake) {
         document.body.appendChild(fake);
-        fake.scrollTop = eventTarget.scrollTop;
-        fake.scrollLeft = eventTarget.scrollLeft;
       }
       // Calculate range and friends here after we've made our fake textarea/input divs.
       range = document.caretRangeFromPoint(ev.clientX, ev.clientY);
@@ -1171,11 +1245,19 @@ class RcxContent {
       // For text nodes do special stuff
       // we make rp the text area and keep the offset the same
       // we give the text area data so it can act normal
-      if (fake) {
+      // Google Docs case also uses a fake so exclude it here
+      if (fake && !gdocRect) {
         rp = eventTarget;
         rp.data = rp.value;
       }
 
+      // Same as text case above except we use the found gdocRect instead of
+      // eventTarget
+      if (gdocRect) {
+        rp = gdocRect;
+        rp.data = rp.ariaLabel;
+      }
+
       if (eventTarget === tdata.prevTarget && this.isVisible()) {
         // console.log("exit due to same target");
         if (tdata.title) {
@@ -1280,6 +1362,38 @@ class RcxContent {
       }
     }
   }
+
+  private getRectUnderMouse(ev: MouseEvent): SVGRectElement | undefined {
+    const gElements = document.querySelectorAll('g');
+    for (const gElement of gElements) {
+      if (this.mouseEventWasInElement(ev, gElement)) {
+        const rects = gElement.querySelectorAll('rect');
+        for (const rectChild of rects) {
+          if (this.mouseEventWasInElement(ev, rectChild)) {
+            return rectChild;
+          }
+        }
+      }
+    }
+    return undefined;
+  }
+
+  private mouseEventWasInElement(ev: MouseEvent, element: Element) {
+    const rect = element.getBoundingClientRect();
+    return (
+      ev.clientX >= rect.left &&
+      ev.clientX <= rect.right &&
+      ev.clientY >= rect.top &&
+      ev.clientY <= rect.bottom
+    );
+  }
+
+  private isGoogleDoc(eventTarget: Element | CharacterData): boolean {
+    return (
+      this.isGoogleDocPage &&
+      (eventTarget.nodeName === 'svg' || eventTarget.nodeName === 'rect')
+    );
+  }
 }
 
 const rcxContent = new RcxContent();

extension/test/background_test.ts

@@ -1,3 +1,4 @@
+import { Config } from '../configuration';
 import { RcxMain } from '../rikaichan';
 import { expect, use } from '@esm-bundle/chai';
 import chrome from 'sinon-chrome';
@@ -23,70 +24,62 @@ describe('background.ts', function () {
     chrome.tabs.sendMessage.reset();
   });
 
-  describe('when sent "forceDocsHtml?" message', function () {
-    it('should not call response callback when rikaikun disabled', async function () {
-      rcxMain.enabled = 0;
-      const responseCallback = sinon.spy();
-
-      await sendMessageToBackground({
-        type: 'forceDocsHtml?',
-        responseCallback: responseCallback,
-      });
-
-      expect(responseCallback).to.have.not.been.called;
-    });
-
-    it('should not send "showPopup" message when rikaikun disabled', async function () {
-      rcxMain.enabled = 0;
+  describe('when sent enable? message', function () {
+    it('should send "enable" message to tab', async function () {
+      rcxMain.enabled = 1;
 
-      await sendMessageToBackground({
-        type: 'forceDocsHtml?',
-      });
+      await sendMessageToBackground({ type: 'enable?' });
 
-      expect(chrome.tabs.sendMessage).to.have.not.been.called;
+      expect(chrome.tabs.sendMessage).to.have.been.calledWithMatch(
+        /* tabId= */ sinon.match.any,
+        {
+          type: 'enable',
+        }
+      );
     });
 
-    it('should pass true to response callback when rikaikun enabled', async function () {
+    it('should respond to the same tab it received a message from', async function () {
       rcxMain.enabled = 1;
-      const responseCallback = sinon.spy();
+      const tabId = 10;
 
-      await sendMessageToBackground({
-        type: 'forceDocsHtml?',
-        responseCallback: responseCallback,
-      });
+      await sendMessageToBackground({ tabId: tabId, type: 'enable?' });
 
-      expect(responseCallback).to.have.been.calledOnceWith(true);
+      expect(chrome.tabs.sendMessage).to.have.been.calledWithMatch(
+        tabId,
+        /* message= */ sinon.match.any
+      );
     });
 
-    it('should send "showPopup" message when rikaikun enabled', async function () {
+    it('should send config in message to tab', async function () {
       rcxMain.enabled = 1;
+      rcxMain.config = { copySeparator: 'testValue' } as Config;
 
-      await sendMessageToBackground({
-        type: 'forceDocsHtml?',
-      });
+      await sendMessageToBackground({ type: 'enable?' });
 
       expect(chrome.tabs.sendMessage).to.have.been.calledWithMatch(
         /* tabId= */ sinon.match.any,
-        { type: 'showPopup' }
+        { config: rcxMain.config }
       );
     });
   });
 });
 
 async function sendMessageToBackground({
+  tabId = 0,
   type,
   responseCallback = () => {
     // Do nothing by default.
   },
 }: {
+  tabId?: number;
   type: string;
   responseCallback?: (response: unknown) => void;
 }): Promise<void> {
   // In background.ts, a promise is passed to `addListener` so we can await it here.
   // eslint-disable-next-line @typescript-eslint/await-thenable
   await chrome.runtime.onMessage.addListener.yield(
     { type: type },
-    { tab: { id: 0 } },
+    { tab: { id: tabId } },
     responseCallback
   );
   return;