Skip to content

Commit

Permalink
fix: Support Google Docs annotated canvas mode (#865)
Browse files Browse the repository at this point in the history
- Adds support for extracting annotated text
- Unconditionally, enables annotated canvas mode in Google Docs
- Adds new background.ts tests, since all previous tests were html fallback related.

Fixes #852
  • Loading branch information
melink14 authored Feb 22, 2022
1 parent a155212 commit bb9b254
Show file tree
Hide file tree
Showing 14 changed files with 853 additions and 129 deletions.
5 changes: 5 additions & 0 deletions .eslintrc.json
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,11 @@
{ "allow": ["private-constructors"] }
],
"tsdoc/syntax": "error"
},
"settings": {
"import/resolver": {
"typescript": {}
}
}
},
{
Expand Down
13 changes: 0 additions & 13 deletions extension/background.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,19 +39,6 @@ chrome.runtime.onMessage.addListener(async (request, sender, response) => {
}
rcxMain.onTabSelect(sender.tab.id);
break;
case 'forceDocsHtml?':
console.log('forceDocsHtml?');
if (rcxMain.enabled === 1) {
response(true);
chrome.tabs.sendMessage(sender.tab!.id!, {
type: 'showPopup',
text: `
rikaikun is forcing Google Docs to render using HTML instead of canvas.<br>
rikaikun can't work with canvas mode but if you need that mode, please disable rikaikun.
`,
});
}
break;
case 'xsearch':
console.log('xsearch');
response(rcxMain.search(request.text, request.dictOption));
Expand Down
10 changes: 10 additions & 0 deletions extension/docs-annotate-canvas.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
const injectedCode = `(function() {window['_docs_annotate_canvas_by_ext'] = '${chrome.runtime.id}';})();`;

const script = document.createElement('script');
script.textContent = injectedCode;
(document.head || document.documentElement).appendChild(script);
script.remove();

// Empty export to satisfy `isolatedModules` compiler flag and allow including in tests.
// Removed in production builds.
export {};
23 changes: 0 additions & 23 deletions extension/docs-html-fallback.ts

This file was deleted.

124 changes: 119 additions & 5 deletions extension/rikaicontent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,9 @@ class RcxContent {
private defaultDict = 2;
private nextDict = 3;

private isGoogleDocPage =
document.querySelector('.kix-canvas-tile-content') !== null;

// Adds the listeners and stuff.
enableTab(config: Config) {
if (window.rikaichan === undefined) {
Expand Down Expand Up @@ -709,6 +712,14 @@ class RcxContent {
selEndList: { node: CharacterData; offset: number }[],
maxLength: number
) {
// <rect> elements come from annotated google docs
if (rangeParent.nodeName === 'rect') {
return this.getTextFromGDoc(
rangeParent as SVGRectElement,
offset,
maxLength
);
}
if (
rangeParent.nodeName === 'TEXTAREA' ||
rangeParent.nodeName === 'INPUT'
Expand Down Expand Up @@ -771,6 +782,52 @@ class RcxContent {
// Hack because ro was coming out always 0 for some reason.
lastRo = 0;

private getTextFromGDoc(
initialRect: SVGRectElement,
offset: number,
maxLength: number
): string {
// Get text from initial rect.
const endIndex = Math.min(initialRect.ariaLabel.length, offset + maxLength);
let text = initialRect.ariaLabel.substring(offset, endIndex);

// Append text from sibling and cousin rects.
const rectWalker = this.createGDocTreeWalker(initialRect);
let rectNode: Node | null;
rectWalker.currentNode = initialRect;
while (
(rectNode = rectWalker.nextNode()) !== null &&
text.length < maxLength
) {
const rect = rectNode as SVGRectElement;
const rectEndIndex = Math.min(
rect.ariaLabel.length,
maxLength - text.length
);
text += rect.ariaLabel.substring(0, rectEndIndex);
}
return text;
}

private createGDocTreeWalker(rect: SVGRectElement): TreeWalker {
return document.createTreeWalker(
rect.parentNode!.parentNode!, // rect is in g is in svg always
NodeFilter.SHOW_ELEMENT,
{
acceptNode: function (node) {
if (node.nodeName === 'rect') {
return NodeFilter.FILTER_ACCEPT;
}
if (node.nodeName === 'g') {
// Don't include g elements but consider their children.
return NodeFilter.FILTER_SKIP;
}
return NodeFilter.FILTER_REJECT;
},
}
);
}

show(tdata: Rikaichan, dictOption: number) {
const rp = tdata.prevRangeNode;
let ro = tdata.prevRangeOfs! + tdata.uofs!;
Expand Down Expand Up @@ -848,8 +905,10 @@ class RcxContent {

const rp = tdata.prevRangeNode;
// don't try to highlight form elements
// don't try to highlight google docs
if (
rp &&
!this.isGoogleDoc(rp) &&
((tdata.config.highlight &&
!this.mDown &&
!('form' in tdata.prevTarget!)) ||
Expand Down Expand Up @@ -1010,10 +1069,16 @@ class RcxContent {
//
}

makeFake(real: HTMLTextAreaElement | HTMLInputElement) {
makeFake(real: HTMLTextAreaElement | HTMLInputElement | SVGRectElement) {
const fake = document.createElement('div');
const realRect = real.getBoundingClientRect();
fake.innerText = real.value;
let textValue = '';
if (real instanceof SVGRectElement) {
textValue = real.ariaLabel;
} else {
textValue = real.value;
}
fake.innerText = textValue;
fake.style.cssText = document.defaultView!.getComputedStyle(
real,
''
Expand Down Expand Up @@ -1069,6 +1134,7 @@ class RcxContent {
}

let fake;
let gdocRect: SVGRectElement | undefined;
const tdata = window.rikaichan!; // per-tab data
let range;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
Expand All @@ -1081,16 +1147,24 @@ class RcxContent {
HTMLSelectElement;
// Put this in a try catch so that an exception here doesn't prevent editing due to div.
try {
if (this.isGoogleDoc(eventTarget)) {
gdocRect = this.getRectUnderMouse(ev);
if (gdocRect) {
fake = this.makeFake(gdocRect);
fake.style.font = gdocRect.getAttribute('data-font-css')!;
}
}

if (
eventTarget.nodeName === 'TEXTAREA' ||
eventTarget.nodeName === 'INPUT'
) {
fake = this.makeFake(
eventTarget as HTMLTextAreaElement | HTMLInputElement
);
}
if (fake) {
document.body.appendChild(fake);
fake.scrollTop = eventTarget.scrollTop;
fake.scrollLeft = eventTarget.scrollLeft;
}
// Calculate range and friends here after we've made our fake textarea/input divs.
range = document.caretRangeFromPoint(ev.clientX, ev.clientY);
Expand Down Expand Up @@ -1171,11 +1245,19 @@ class RcxContent {
// For text nodes do special stuff
// we make rp the text area and keep the offset the same
// we give the text area data so it can act normal
if (fake) {
// Google Docs case also uses a fake so exclude it here
if (fake && !gdocRect) {
rp = eventTarget;
rp.data = rp.value;
}

// Same as text case above except we use the found gdocRect instead of
// eventTarget
if (gdocRect) {
rp = gdocRect;
rp.data = rp.ariaLabel;
}

if (eventTarget === tdata.prevTarget && this.isVisible()) {
// console.log("exit due to same target");
if (tdata.title) {
Expand Down Expand Up @@ -1280,6 +1362,38 @@ class RcxContent {
}
}
}

private getRectUnderMouse(ev: MouseEvent): SVGRectElement | undefined {
const gElements = document.querySelectorAll('g');
for (const gElement of gElements) {
if (this.mouseEventWasInElement(ev, gElement)) {
const rects = gElement.querySelectorAll('rect');
for (const rectChild of rects) {
if (this.mouseEventWasInElement(ev, rectChild)) {
return rectChild;
}
}
}
}
return undefined;
}

private mouseEventWasInElement(ev: MouseEvent, element: Element) {
const rect = element.getBoundingClientRect();
return (
ev.clientX >= rect.left &&
ev.clientX <= rect.right &&
ev.clientY >= rect.top &&
ev.clientY <= rect.bottom
);
}

private isGoogleDoc(eventTarget: Element | CharacterData): boolean {
return (
this.isGoogleDocPage &&
(eventTarget.nodeName === 'svg' || eventTarget.nodeName === 'rect')
);
}
}

const rcxContent = new RcxContent();
Expand Down
57 changes: 25 additions & 32 deletions extension/test/background_test.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { Config } from '../configuration';
import { RcxMain } from '../rikaichan';
import { expect, use } from '@esm-bundle/chai';
import chrome from 'sinon-chrome';
Expand All @@ -23,70 +24,62 @@ describe('background.ts', function () {
chrome.tabs.sendMessage.reset();
});

describe('when sent "forceDocsHtml?" message', function () {
it('should not call response callback when rikaikun disabled', async function () {
rcxMain.enabled = 0;
const responseCallback = sinon.spy();

await sendMessageToBackground({
type: 'forceDocsHtml?',
responseCallback: responseCallback,
});

expect(responseCallback).to.have.not.been.called;
});

it('should not send "showPopup" message when rikaikun disabled', async function () {
rcxMain.enabled = 0;
describe('when sent enable? message', function () {
it('should send "enable" message to tab', async function () {
rcxMain.enabled = 1;

await sendMessageToBackground({
type: 'forceDocsHtml?',
});
await sendMessageToBackground({ type: 'enable?' });

expect(chrome.tabs.sendMessage).to.have.not.been.called;
expect(chrome.tabs.sendMessage).to.have.been.calledWithMatch(
/* tabId= */ sinon.match.any,
{
type: 'enable',
}
);
});

it('should pass true to response callback when rikaikun enabled', async function () {
it('should respond to the same tab it received a message from', async function () {
rcxMain.enabled = 1;
const responseCallback = sinon.spy();
const tabId = 10;

await sendMessageToBackground({
type: 'forceDocsHtml?',
responseCallback: responseCallback,
});
await sendMessageToBackground({ tabId: tabId, type: 'enable?' });

expect(responseCallback).to.have.been.calledOnceWith(true);
expect(chrome.tabs.sendMessage).to.have.been.calledWithMatch(
tabId,
/* message= */ sinon.match.any
);
});

it('should send "showPopup" message when rikaikun enabled', async function () {
it('should send config in message to tab', async function () {
rcxMain.enabled = 1;
rcxMain.config = { copySeparator: 'testValue' } as Config;

await sendMessageToBackground({
type: 'forceDocsHtml?',
});
await sendMessageToBackground({ type: 'enable?' });

expect(chrome.tabs.sendMessage).to.have.been.calledWithMatch(
/* tabId= */ sinon.match.any,
{ type: 'showPopup' }
{ config: rcxMain.config }
);
});
});
});

async function sendMessageToBackground({
tabId = 0,
type,
responseCallback = () => {
// Do nothing by default.
},
}: {
tabId?: number;
type: string;
responseCallback?: (response: unknown) => void;
}): Promise<void> {
// In background.ts, a promise is passed to `addListener` so we can await it here.
// eslint-disable-next-line @typescript-eslint/await-thenable
await chrome.runtime.onMessage.addListener.yield(
{ type: type },
{ tab: { id: 0 } },
{ tab: { id: tabId } },
responseCallback
);
return;
Expand Down
Loading

0 comments on commit bb9b254

Please sign in to comment.