feat: support for graphemes in key input (#2207)

Allow graphemes in key input according to step 6 of
https://w3c.github.io/webdriver/#dfn-process-a-key-action.
Use
[`Intl.Segmenter`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/Segmenter)
to detect grapheme borders.

Follow up: update WPT tests.

---------

Signed-off-by: Browser Automation Bot <browser-automation-bot@google.com>
Co-authored-by: Browser Automation Bot <browser-automation-bot@google.com>

src/bidiMapper/modules/input/ActionDispatcher.ts

@@ -23,6 +23,10 @@ import {
   type Script,
 } from '../../../protocol/protocol.js';
 import {assert} from '../../../utils/assert.js';
+import {
+  isSingleComplexGrapheme,
+  isSingleGrapheme,
+} from '../../../utils/GraphemeTools';
 import type {BrowsingContextImpl} from '../context/BrowsingContextImpl.js';
 
 import type {ActionOption} from './ActionOption.js';
@@ -562,10 +566,13 @@ export class ActionDispatcher {
     source: KeySource,
     action: Readonly<Input.KeyDownAction>
   ) {
-    if ([...action.value].length > 1) {
-      throw new InvalidArgumentException(`Invalid key value: ${action.value}`);
-    }
     const rawKey = action.value;
+    if (!isSingleGrapheme(rawKey)) {
+      // https://w3c.github.io/webdriver/#dfn-process-a-key-action
+      // WebDriver spec allows a grapheme to be used.
+      throw new InvalidArgumentException(`Invalid key value: ${rawKey}`);
+    }
+    const isGrapheme = isSingleComplexGrapheme(rawKey);
     const key = getNormalizedKey(rawKey);
     const repeat = source.pressed.has(key);
     const code = getKeyCode(rawKey);
@@ -590,7 +597,7 @@ export class ActionDispatcher {
     // --- Platform-specific code begins here ---
     // The spread is a little hack so JS gives us an array of unicode characters
     // to measure.
-    const unmodifiedText = getKeyEventUnmodifiedText(key, source);
+    const unmodifiedText = getKeyEventUnmodifiedText(key, source, isGrapheme);
     const text = getKeyEventText(code ?? '', source) ?? unmodifiedText;
     let command: string | undefined;
     // The following commands need to be declared because Chromium doesn't
@@ -649,10 +656,13 @@ export class ActionDispatcher {
   }
 
   #dispatchKeyUpAction(source: KeySource, action: Readonly<Input.KeyUpAction>) {
-    if ([...action.value].length > 1) {
-      throw new InvalidArgumentException(`Invalid key value: ${action.value}`);
-    }
     const rawKey = action.value;
+    if (!isSingleGrapheme(rawKey)) {
+      // https://w3c.github.io/webdriver/#dfn-process-a-key-action
+      // WebDriver spec allows a grapheme to be used.
+      throw new InvalidArgumentException(`Invalid key value: ${rawKey}`);
+    }
+    const isGrapheme = isSingleComplexGrapheme(rawKey);
     const key = getNormalizedKey(rawKey);
     if (!source.pressed.has(key)) {
       return;
@@ -679,7 +689,7 @@ export class ActionDispatcher {
     // --- Platform-specific code begins here ---
     // The spread is a little hack so JS gives us an array of unicode characters
     // to measure.
-    const unmodifiedText = getKeyEventUnmodifiedText(key, source);
+    const unmodifiedText = getKeyEventUnmodifiedText(key, source, isGrapheme);
     const text = getKeyEventText(code ?? '', source) ?? unmodifiedText;
     return this.#context.cdpTarget.cdpClient.sendCommand(
       'Input.dispatchKeyEvent',
@@ -700,10 +710,26 @@ export class ActionDispatcher {
   }
 }
 
-const getKeyEventUnmodifiedText = (key: string, source: KeySource) => {
+/**
+ * Translates a non-grapheme key to either an `undefined` for a special keys, or a single
+ * character modified by shift if needed.
+ */
+const getKeyEventUnmodifiedText = (
+  key: string,
+  source: KeySource,
+  isGrapheme: boolean
+) => {
+  if (isGrapheme) {
+    // Graphemes should be presented as text in the CDP command.
+    return key;
+  }
+
   if (key === 'Enter') {
     return '\r';
   }
+
+  // If key is not a single character, it is a normalized key value, and should be
+  // presented as key, not text in the CDP command.
   return [...key].length === 1
     ? source.shift
       ? key.toLocaleUpperCase('en-US')

src/utils/GraphemeTools.spec.ts

@@ -0,0 +1,116 @@
+/*
+ * Copyright 2024 Google LLC.
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {expect} from 'chai';
+
+import {isSingleComplexGrapheme, isSingleGrapheme} from './GraphemeTools';
+
+describe('GraphemeTools', () => {
+  describe('isSingleGrapheme', () => {
+    describe('should return true for a single grapheme', () => {
+      it('"a", a single char', () => {
+        expect(isSingleGrapheme('a')).to.be.true;
+      });
+
+      it('"😄", a single surrogate codepoint', () => {
+        expect(isSingleGrapheme('\ud83d\ude04')).to.be.true;
+      });
+
+      it('"நி", a grapheme containing several chars', () => {
+        expect(isSingleGrapheme('\u0BA8\u0BBF')).to.be.true;
+      });
+
+      it('"각", a grapheme containing several chars', () => {
+        expect(isSingleGrapheme('\u1100\u1161\u11A8')).to.be.true;
+      });
+
+      it('"❤️", a grapheme containing several codepoints', () => {
+        expect(isSingleGrapheme('\u2764\ufe0f')).to.be.true;
+      });
+    });
+
+    describe('should return false for multiple graphemes', () => {
+      it('2 symbols', () => {
+        expect(isSingleGrapheme('fa')).to.be.false;
+      });
+
+      it('"😄a" a codepoint with a symbol', () => {
+        expect(isSingleGrapheme('\ud83d\ude04a')).to.be.false;
+      });
+
+      it('"நிa" a grapheme with a symbol', () => {
+        expect(isSingleGrapheme('\u0BA8\u0BBFa')).to.be.false;
+      });
+
+      it('"각a" a grapheme with a symbol', () => {
+        expect(isSingleGrapheme('\u1100\u1161\u11A8a')).to.be.false;
+      });
+
+      it('"❤️a" a grapheme with a symbol', () => {
+        expect(isSingleGrapheme('\u2764\ufe0fa')).to.be.false;
+      });
+
+      it('"😄😍" 2 graphemes', () => {
+        expect(isSingleGrapheme('\ud83d\ude04\ud83d\ude0d')).to.be.false;
+      });
+
+      it('"ch" 2 graphemes', () => {
+        // https://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries
+        // Spec says it's a single grapheme in slovak locale. We support only `en` locale.
+        expect(isSingleGrapheme('\ud83d\ude04\ud83d\ude0d')).to.be.false;
+      });
+    });
+  });
+
+  describe('isSingleComplexGrapheme', () => {
+    describe('should return true', () => {
+      it('"😄", a single surrogate codepoint', () => {
+        expect(isSingleComplexGrapheme('\ud83d\ude04')).to.be.true;
+      });
+
+      it('"நி", a grapheme containing several chars', () => {
+        expect(isSingleComplexGrapheme('\u0BA8\u0BBF')).to.be.true;
+      });
+
+      it('"각", a grapheme containing several chars', () => {
+        expect(isSingleComplexGrapheme('\u1100\u1161\u11A8')).to.be.true;
+      });
+
+      it('"❤️", a grapheme containing several codepoints', () => {
+        expect(isSingleComplexGrapheme('\u2764\ufe0f')).to.be.true;
+      });
+    });
+
+    describe('should return false', () => {
+      it('"AB", 2 symbols', () => {
+        expect(isSingleComplexGrapheme('AB')).to.be.false;
+      });
+
+      it('"A", a single simple symbol', () => {
+        expect(isSingleComplexGrapheme('A')).to.be.false;
+      });
+
+      it('"1", a single simple symbol', () => {
+        expect(isSingleComplexGrapheme('1')).to.be.false;
+      });
+
+      it('empty string', () => {
+        expect(isSingleComplexGrapheme('')).to.be.false;
+      });
+    });
+  });
+});

src/utils/GraphemeTools.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2024 Google LLC.
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Check if the given string is a single complex grapheme. A complex grapheme is one that
+ * is made up of multiple characters.
+ */
+export function isSingleComplexGrapheme(value: string): boolean {
+  return isSingleGrapheme(value) && value.length > 1;
+}
+
+/**
+ * Check if the given string is a single grapheme.
+ */
+export function isSingleGrapheme(value: string): boolean {
+  // Theoretically there can be some strings considered a grapheme in some locales, like
+  // slovak "ch" digraph. Use english locale for consistency.
+  // https://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries
+  const segmenter = new Intl.Segmenter('en', {granularity: 'grapheme'});
+  return [...segmenter.segment(value)].length === 1;
+}

wpt-metadata/chromedriver/headful/webdriver/tests/bidi/input/perform_actions/pointer_pen.py.ini

@@ -1,4 +1,3 @@
 [pointer_pen.py]
-  expected: [ERROR, OK]
   [test_pen_pointer_properties]
     expected: FAIL

wpt-metadata/chromedriver/headless/webdriver/tests/bidi/input/perform_actions/pointer_pen.py.ini

@@ -1,4 +1,3 @@
 [pointer_pen.py]
-  expected: [ERROR, OK]
   [test_pen_pointer_properties]
     expected: FAIL