fix(text): handle code points > U+FFFF in levenshteinDistance

text/levenshtein_distance.ts

@@ -4,20 +4,20 @@ const { ceil } = Math;
 
 // This implements Myers' bit-vector algorithm as described here:
 // https://dl.acm.org/doi/pdf/10.1145/316542.316550
-const peq = new Uint32Array(0x10000);
+const peq = new Uint32Array(0x110000);
 
-function myers32(t: string, p: string): number {
+function myers32(t: string[], p: string[]): number {
   const n = t.length;
   const m = p.length;
   for (let i = 0; i < m; i++) {
-    peq[p.charCodeAt(i)]! |= 1 << i;
+    peq[p[i]!.codePointAt(0)!]! |= 1 << i;
   }
   const last = m - 1;
   let pv = -1;
   let mv = 0;
   let score = m;
   for (let j = 0; j < n; j++) {
-    const eq = peq[t.charCodeAt(j)]!;
+    const eq = peq[t[j]!.codePointAt(0)!]!;
     const xv = eq | mv;
     const xh = (((eq & pv) + pv) ^ pv) | eq;
     let ph = mv | ~(xh | pv);
@@ -31,12 +31,12 @@ function myers32(t: string, p: string): number {
     mv = ph & xv;
   }
   for (let i = 0; i < m; i++) {
-    peq[p.charCodeAt(i)] = 0;
+    peq[p[i]!.codePointAt(0)!] = 0;
   }
   return score;
 }
 
-function myersX(t: string, p: string): number {
+function myersX(t: string[], p: string[]): number {
   const n = t.length;
   const m = p.length;
   // Initialize the horizontal deltas to +1.
@@ -47,13 +47,13 @@ function myersX(t: string, p: string): number {
     const start = b * 32;
     const end = (b + 1) * 32;
     for (let i = start; i < end; i++) {
-      peq[p.charCodeAt(i)]! |= 1 << i;
+      peq[p[i]!.codePointAt(0)!]! |= 1 << i;
     }
     let pv = -1;
     let mv = 0;
     for (let j = 0; j < n; j++) {
       const hin = h[j]!;
-      let eq = peq[t.charCodeAt(j)]!;
+      let eq = peq[t[j]!.codePointAt(0)!]!;
       const xv = eq | mv;
       eq |= hin >>> 31;
       const xh = (((eq & pv) + pv) ^ pv) | eq;
@@ -66,20 +66,20 @@ function myersX(t: string, p: string): number {
       mv = ph & xv;
     }
     for (let i = start; i < end; i++) {
-      peq[p.charCodeAt(i)] = 0;
+      peq[p[i]!.codePointAt(0)!] = 0;
     }
   }
   const start = bmax * 32;
   for (let i = start; i < m; i++) {
-    peq[p.charCodeAt(i)]! |= 1 << i;
+    peq[p[i]!.codePointAt(0)!]! |= 1 << i;
   }
   const last = m - 1;
   let pv = -1;
   let mv = 0;
   let score = m;
   for (let j = 0; j < n; j++) {
     const hin = h[j]!;
-    let eq = peq[t.charCodeAt(j)]!;
+    let eq = peq[t[j]!.codePointAt(0)!]!;
     const xv = eq | mv;
     eq |= hin >>> 31;
     const xh = (((eq & pv) + pv) ^ pv) | eq;
@@ -92,7 +92,7 @@ function myersX(t: string, p: string): number {
     mv = ph & xv;
   }
   for (let i = start; i < m; i++) {
-    peq[p.charCodeAt(i)] = 0;
+    peq[p[i]!.codePointAt(0)!] = 0;
   }
   return score;
 }
@@ -119,13 +119,14 @@ function myersX(t: string, p: string): number {
  * @returns The Levenshtein distance between the two strings.
  */
 export function levenshteinDistance(str1: string, str2: string): number {
-  if (str1.length < str2.length) {
-    const tmp = str1;
-    str1 = str2;
-    str2 = tmp;
+  let t = [...str1];
+  let p = [...str2];
+
+  if (t.length < p.length) {
+    [p, t] = [t, p];
   }
-  if (str2.length === 0) {
-    return str1.length;
+  if (p.length === 0) {
+    return t.length;
   }
-  return str2.length <= 32 ? myers32(str1, str2) : myersX(str1, str2);
+  return p.length <= 32 ? myers32(t, p) : myersX(t, p);
 }

text/levenshtein_distance_test.ts

@@ -2,6 +2,11 @@
 import { assertEquals } from "@std/assert/equals";
 import { levenshteinDistance } from "./levenshtein_distance.ts";
 
+function assertLevenshteinBidi(a: string, b: string, distance: number) {
+  assertEquals(levenshteinDistance(a, b), distance);
+  assertEquals(levenshteinDistance(b, a), distance);
+}
+
 Deno.test("levenshteinDistance() handles basic cases", () => {
   assertEquals(levenshteinDistance("levenshtein", "levenshtein"), 0);
   assertEquals(levenshteinDistance("sitting", "kitten"), 3);
@@ -26,3 +31,36 @@ Deno.test("levenshteinDistance() handles long strings", () => {
     30,
   );
 });
+
+Deno.test("levenshteinDistance() handles code points above U+FFFF", async (t) => {
+  await t.step("one of inputs is empty fast path", () => {
+    assertLevenshteinBidi("💩", "", 1);
+    assertLevenshteinBidi("\u{10FFFF}", "", 1);
+  });
+
+  await t.step("`myers32` fast path", () => {
+    assertLevenshteinBidi("💩", "x", 1);
+    // first surrogate same
+    assertLevenshteinBidi("💩", "💫", 1);
+    // both surrogates different
+    assertLevenshteinBidi("💩", "🦄", 1);
+    // max cp
+    assertLevenshteinBidi("\u{10FFFF}x", "y", 2);
+    assertLevenshteinBidi("x\u{10FFFF}", "y", 2);
+    assertLevenshteinBidi("\u{10FFFE}", "\u{10FFFF}", 1);
+    assertLevenshteinBidi("\u{10FFFF}", "\u{10FFFF}", 0);
+  });
+
+  await t.step("`myersX` path", () => {
+    const MYERS_32_MAX = 32;
+    const n = MYERS_32_MAX + 1;
+    assertLevenshteinBidi("💩".repeat(n), "x".repeat(n), n);
+    // first surrogate same
+    assertLevenshteinBidi("💩".repeat(n), "💫".repeat(n), n);
+    // both surrogates different
+    assertLevenshteinBidi("💩".repeat(n), "🦄".repeat(n), n);
+    // max cp
+    assertLevenshteinBidi("\u{10FFFE}".repeat(n), "\u{10FFFF}".repeat(n), n);
+    assertLevenshteinBidi("\u{10FFFE}".repeat(n), "\u{10FFFF}", n);
+  });
+});