rework text processors (#793)

* rework text processors * rename text-preprocessors file * Fix search header left margins on small screens (#839) * Refocuses search input on backspace (#840) Fixes #775. Note that this behavior gets overridden if backspace is set as a shortcut action. * Change hotkey triggering condition to account for IME usage (#837) _isKeyCharacterInput only worked when not using an IME, as inside of an IME when a keydown event is fired, the key is reported as "Process", which does not have a key.length equal to 1. This resulted in hotkeys being triggered while typing, which this commit fixes. --------- Co-authored-by: James Maa <jmaa@berkeley.edu> Co-authored-by: Kuuuube <61125188+Kuuuube@users.noreply.github.com> Co-authored-by: Andrew Thomas Sartor <andrew@sartor.net>
yomidevs · Apr 21, 2024 · 07258ec · 07258ec
1 parent 22904d1
commit 07258ec
Show file tree

Hide file tree

Showing 21 changed files with 349 additions and 694 deletions.
diff --git a/.eslintrc.json b/.eslintrc.json
@@ -639,7 +639,6 @@
                 "ext/js/general/cache-map.js",
                 "ext/js/general/object-property-accessor.js",
                 "ext/js/general/regex-util.js",
-                "ext/js/general/text-source-map.js",
                 "ext/js/language/ar/arabic-text-preprocessors.js",
                 "ext/js/language/de/german-text-preprocessors.js",
                 "ext/js/language/de/german-transforms.js",
@@ -656,7 +655,7 @@
                 "ext/js/language/ru/russian-text-preprocessors.js",
                 "ext/js/language/sga/old-irish-transforms.js",
                 "ext/js/language/sq/albanian-transforms.js",
-                "ext/js/language/text-preprocessors.js",
+                "ext/js/language/text-processors.js",
                 "ext/js/language/translator.js",
                 "ext/js/media/audio-downloader.js",
                 "ext/js/media/media-util.js",

diff --git a/ext/js/general/regex-util.js b/ext/js/general/regex-util.js
@@ -23,13 +23,12 @@ const matchReplacementPattern = /\$(?:\$|&|`|'|(\d\d?)|<([^>]*)>)/g;
  * Applies string.replace using a regular expression and replacement string as arguments.
  * A source map of the changes is also maintained.
  * @param {string} text A string of the text to replace.
- * @param {import('./text-source-map.js').TextSourceMap} sourceMap An instance of `TextSourceMap` which corresponds to `text`.
  * @param {RegExp} pattern A regular expression to use as the replacement.
  * @param {string} replacement A replacement string that follows the format of the standard
  *   JavaScript regular expression replacement string.
  * @returns {string} A new string with the pattern replacements applied and the source map updated.
  */
-export function applyTextReplacement(text, sourceMap, pattern, replacement) {
+export function applyTextReplacement(text, pattern, replacement) {
     const isGlobal = pattern.global;
     if (isGlobal) { pattern.lastIndex = 0; }
     for (let loop = true; loop; loop = isGlobal) {
@@ -44,15 +43,6 @@ export function applyTextReplacement(text, sourceMap, pattern, replacement) {
 
         text = `${text.substring(0, index)}${actualReplacement}${text.substring(index + matchText.length)}`;
         pattern.lastIndex += delta;
-
-        if (actualReplacementLength > 0) {
-            /** @type {number[]} */
-            const zeroes = new Array(actualReplacementLength).fill(0);
-            sourceMap.insert(index, ...zeroes);
-            sourceMap.combine(index - 1 + actualReplacementLength, matchText.length);
-        } else {
-            sourceMap.combine(index, matchText.length);
-        }
     }
     return text;
 }

diff --git a/ext/js/general/text-source-map.js b/ext/js/general/text-source-map.js
diff --git a/ext/js/language/ar/arabic-text-preprocessors.js b/ext/js/language/ar/arabic-text-preprocessors.js
@@ -15,7 +15,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-import {basicTextPreprocessorOptions} from '../text-preprocessors.js';
+import {basicTextProcessorOptions} from '../text-processors.js';
 
 const optionalDiacritics = [
     '\u0618', // Small Fatha
@@ -38,11 +38,11 @@ const optionalDiacritics = [
 
 const diacriticsRegex = new RegExp(`[${optionalDiacritics.join('')}]`, 'g');
 
-/** @type {import('language').TextPreprocessor<boolean>} */
+/** @type {import('language').TextProcessor<boolean>} */
 export const removeArabicScriptDiacritics = {
     name: 'Remove diacritics',
     description: 'وَلَدَ ⬅️ ولد',
-    options: basicTextPreprocessorOptions,
+    options: basicTextProcessorOptions,
     process: (text, setting) => {
         return setting ? text.replace(diacriticsRegex, '') : text;
     }

diff --git a/ext/js/language/ja/japanese-text-preprocessors.js b/ext/js/language/ja/japanese-text-preprocessors.js
@@ -15,7 +15,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-import {basicTextPreprocessorOptions} from '../text-preprocessors.js';
+import {basicTextProcessorOptions} from '../text-processors.js';
 import {convertAlphabeticToKana} from './japanese-wanakana.js';
 import {
     collapseEmphaticSequences as collapseEmphaticSequencesFunction,
@@ -25,28 +25,28 @@ import {
     convertNumericToFullWidth
 } from './japanese.js';
 
-/** @type {import('language').TextPreprocessor<boolean>} */
+/** @type {import('language').TextProcessor<boolean>} */
 export const convertHalfWidthCharacters = {
     name: 'Convert half width characters to full width',
     description: 'ﾖﾐﾁｬﾝ → ヨミチャン',
-    options: basicTextPreprocessorOptions,
-    process: (str, setting, sourceMap) => (setting ? convertHalfWidthKanaToFullWidth(str, sourceMap) : str)
+    options: basicTextProcessorOptions,
+    process: (str, setting) => (setting ? convertHalfWidthKanaToFullWidth(str) : str)
 };
 
-/** @type {import('language').TextPreprocessor<boolean>} */
+/** @type {import('language').TextProcessor<boolean>} */
 export const convertNumericCharacters = {
     name: 'Convert numeric characters to full width',
     description: '1234 → １２３４',
-    options: basicTextPreprocessorOptions,
+    options: basicTextProcessorOptions,
     process: (str, setting) => (setting ? convertNumericToFullWidth(str) : str)
 };
 
-/** @type {import('language').TextPreprocessor<boolean>} */
+/** @type {import('language').TextProcessor<boolean>} */
 export const convertAlphabeticCharacters = {
     name: 'Convert alphabetic characters to hiragana',
     description: 'yomichan → よみちゃん',
-    options: basicTextPreprocessorOptions,
-    process: (str, setting, sourceMap) => (setting ? convertAlphabeticToKana(str, sourceMap) : str)
+    options: basicTextProcessorOptions,
+    process: (str, setting) => (setting ? convertAlphabeticToKana(str) : str)
 };
 
 /** @type {import('language').BidirectionalConversionPreprocessor} */
@@ -66,15 +66,15 @@ export const convertHiraganaToKatakana = {
     }
 };
 
-/** @type {import('language').TextPreprocessor<[collapseEmphatic: boolean, collapseEmphaticFull: boolean]>} */
+/** @type {import('language').TextProcessor<[collapseEmphatic: boolean, collapseEmphaticFull: boolean]>} */
 export const collapseEmphaticSequences = {
     name: 'Collapse emphatic character sequences',
     description: 'すっっごーーい → すっごーい / すごい',
     options: [[false, false], [true, false], [true, true]],
-    process: (str, setting, sourceMap) => {
+    process: (str, setting) => {
         const [collapseEmphatic, collapseEmphaticFull] = setting;
         if (collapseEmphatic) {
-            str = collapseEmphaticSequencesFunction(str, collapseEmphaticFull, sourceMap);
+            str = collapseEmphaticSequencesFunction(str, collapseEmphaticFull);
         }
         return str;
     }

diff --git a/ext/js/language/ja/japanese-wanakana.js b/ext/js/language/ja/japanese-wanakana.js
@@ -19,51 +19,10 @@ import * as wanakana from '../../../lib/wanakana.js';
 
 /**
  * @param {string} text
- * @param {?import('../../general/text-source-map.js').TextSourceMap} sourceMap
- * @param {number} sourceMapStart
  * @returns {string}
  */
-function convertAlphabeticPartToKana(text, sourceMap, sourceMapStart) {
-    const result = wanakana.toHiragana(text);
-
-    // Generate source mapping
-    if (sourceMap !== null) {
-        let i = 0;
-        let resultPos = 0;
-        const ii = text.length;
-        while (i < ii) {
-            // Find smallest matching substring
-            let iNext = i + 1;
-            let resultPosNext = result.length;
-            while (iNext < ii) {
-                const t = wanakana.toHiragana(text.substring(0, iNext));
-                if (t === result.substring(0, t.length)) {
-                    resultPosNext = t.length;
-                    break;
-                }
-                ++iNext;
-            }
-
-            // Merge characters
-            const removals = iNext - i - 1;
-            if (removals > 0) {
-                sourceMap.combine(sourceMapStart, removals);
-            }
-            ++sourceMapStart;
-
-            // Empty elements
-            const additions = resultPosNext - resultPos - 1;
-            for (let j = 0; j < additions; ++j) {
-                sourceMap.insert(sourceMapStart, 0);
-                ++sourceMapStart;
-            }
-
-            i = iNext;
-            resultPos = resultPosNext;
-        }
-    }
-
-    return result;
+function convertAlphabeticPartToKana(text) {
+    return wanakana.toHiragana(text);
 }
 
 /**
@@ -84,10 +43,9 @@ export function convertToRomaji(text) {
 
 /**
  * @param {string} text
- * @param {?import('../../general/text-source-map.js').TextSourceMap} sourceMap
  * @returns {string}
  */
-export function convertAlphabeticToKana(text, sourceMap = null) {
+export function convertAlphabeticToKana(text) {
     let part = '';
     let result = '';
 
@@ -106,7 +64,7 @@ export function convertAlphabeticToKana(text, sourceMap = null) {
             c = 0x2d; // '-'
         } else {
             if (part.length > 0) {
-                result += convertAlphabeticPartToKana(part, sourceMap, result.length);
+                result += convertAlphabeticPartToKana(part);
                 part = '';
             }
             result += char;
@@ -116,7 +74,7 @@ export function convertAlphabeticToKana(text, sourceMap = null) {
     }
 
     if (part.length > 0) {
-        result += convertAlphabeticPartToKana(part, sourceMap, result.length);
+        result += convertAlphabeticPartToKana(part);
     }
     return result;
 }