Add support for regular expressions containing octal digits greater t…

…han `\200` (#5443) * Convert octal digits greater than \200 to direct unicode char Signed-off-by: Anthony Chang <antchang@nvidia.com> * Add integration test Signed-off-by: Anthony Chang <antchang@nvidia.com>
NVIDIA · May 12, 2022 · 456dfbe · 456dfbe
1 parent fa3e435
commit 456dfbe
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 17 deletions.
diff --git a/integration_tests/src/main/python/string_test.py b/integration_tests/src/main/python/string_test.py
@@ -867,6 +867,19 @@ def test_regexp_whitespace():
             ),
         conf=_regexp_conf)
 
+def test_regexp_octal_digits():
+    gen = mk_str_gen('[abcd]\u0000\u0041\u007f\u0080\u00ff[\\\\xa0-\\\\xb0][abcd]')
+    assert_gpu_and_cpu_are_equal_collect(
+            lambda spark: unary_op_df(spark, gen).selectExpr(
+                'rlike(a, "\\\\0177")',
+                'rlike(a, "\\\\0200")',
+                'rlike(a, "\\\\0101")',
+                'regexp_extract(a, "([a-d]+)\\\\0240([a-d]+)", 1)',
+                'regexp_replace(a, "\\\\0377", "")',
+                'regexp_replace(a, "\\\\0260", "")',
+            ),
+        conf=_regexp_conf)
+
 def test_rlike():
     gen = mk_str_gen('[abcd]{1,3}')
     assert_gpu_and_cpu_are_equal_collect(

diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RegexParser.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RegexParser.scala
@@ -620,12 +620,12 @@ class CudfRegexTranspiler(mode: RegexMode) {
         } else  {
           digits
         }
-        if (Integer.parseInt(octal, 8) >= 128) {
-          // see https://github.com/NVIDIA/spark-rapids/issues/4746
-          throw new RegexUnsupportedException(
-            "cuDF does not support octal digits 0o177 < n <= 0o377")
+        val codePoint = Integer.parseInt(octal, 8)
+        if (codePoint >= 128) {
+          RegexChar(codePoint.toChar)
+        } else {
+          RegexOctalChar(octal)
         }
-        RegexOctalChar(octal)
 
       case RegexHexDigit(digits) =>
         val codePoint = Integer.parseInt(digits, 16)

diff --git a/tests/src/test/scala/com/nvidia/spark/rapids/RegularExpressionTranspilerSuite.scala b/tests/src/test/scala/com/nvidia/spark/rapids/RegularExpressionTranspilerSuite.scala
@@ -142,14 +142,6 @@ class RegularExpressionTranspilerSuite extends FunSuite with Arm {
         "cuDF does not support null characters in regular expressions"))
   }
 
-  test("cuDF does not support octal digits 0o177 < n <= 0o377") {
-    // see https://github.com/NVIDIA/spark-rapids/issues/4746
-    val patterns = Seq(raw"\0200", raw"\0377")
-    patterns.foreach(pattern =>
-      assertUnsupported(pattern, RegexFindMode,
-        "cuDF does not support octal digits 0o177 < n <= 0o377"))
-  }
-
   test("cuDF does not support octal digits in character classes") {
     // see https://github.com/NVIDIA/spark-rapids/issues/4862
     val patterns = Seq(raw"[\02]", raw"[\012]", raw"[\0177]")
@@ -170,10 +162,11 @@ class RegularExpressionTranspilerSuite extends FunSuite with Arm {
     )
   }
 
-  test("octal digits < 0o177 - find") {
-    val patterns = Seq(raw"\07", raw"\077", raw"\0177", raw"\01772")
-    assertCpuGpuMatchesRegexpFind(patterns, Seq("", "\u0007", "a\u0007b", 
-        "\u0007\u003f\u007f", "\u007f", "\u007f2"))
+  test("octal digits - find") {
+    val patterns = Seq(raw"\07", raw"\077", raw"\0177", raw"\01772", raw"\0200", 
+      raw"\0376", raw"\0377", raw"\02002")
+    assertCpuGpuMatchesRegexpFind(patterns, Seq("", "\u0007", "a\u0007b", "a\u007fb",
+        "\u0007\u003f\u007f", "\u007f", "\u0080", "a\u00fe\u00ffb", "\u007f2"))
   }
 
   test("hex digits - find") {