Qualification tool: Filter empty strings from Read Schema (#3405)

* Qualification tool: Filter empty strings from Read Schema Signed-off-by: Niranjan Artal <nartal@nvidia.com> * address review comments Signed-off-by: Niranjan Artal <nartal@nvidia.com>
NVIDIA · Sep 8, 2021 · 89299a4 · 89299a4
1 parent 8e1fd7e
commit 89299a4
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 6 deletions.
diff --git a/tools/src/main/scala/org/apache/spark/sql/rapids/tool/qualification/QualAppInfo.scala b/tools/src/main/scala/org/apache/spark/sql/rapids/tool/qualification/QualAppInfo.scala
@@ -335,7 +335,7 @@ object QualAppInfo extends Logging {
     val individualSchema: ArrayBuffer[String] = new ArrayBuffer()
     var angleBracketsCount = 0
     var parenthesesCount = 0
-    val distinctSchema = schema.distinct.mkString(",")
+    val distinctSchema = schema.distinct.filter(_.nonEmpty).mkString(",")
 
     // Get the nested types i.e everything between < >
     for (char <- distinctSchema) {
@@ -351,8 +351,7 @@ object QualAppInfo extends Logging {
       if (angleBracketsCount == 0 && parenthesesCount == 0 && char.equals(',')) {
         individualSchema += tempStringBuilder.toString
         tempStringBuilder.setLength(0)
-      }
-      else {
+      } else {
         tempStringBuilder.append(char);
       }
     }
@@ -368,13 +367,30 @@ object QualAppInfo extends Logging {
     // Check if it has types
     val incompleteTypes = incompleteSchema.map { x =>
       if (x.contains("...") && x.contains(":")) {
-        x.split(":", 2)(1).split("\\.\\.\\.")(0)
+        val schemaTypes = x.split(":", 2)
+        if (schemaTypes.size == 2) {
+          val partialSchema = schemaTypes(1).split("\\.\\.\\.")
+          if (partialSchema.size == 1) {
+            partialSchema(0)
+          } else {
+            ""
+          }
+        } else {
+          ""
+        }
       } else {
         ""
       }
     }
     // Omit columnName and get only schemas
-    val completeTypes = completeSchema.map(x => x.split(":", 2)(1))
+    val completeTypes = completeSchema.map { x =>
+      val schemaTypes = x.split(":", 2)
+      if (schemaTypes.size == 2) {
+        schemaTypes(1)
+      } else {
+        ""
+      }
+    }
     val schemaTypes = completeTypes ++ incompleteTypes
 
     // Filter only complex types.

diff --git a/tools/src/test/scala/com/nvidia/spark/rapids/tool/qualification/QualificationSuite.scala b/tools/src/test/scala/com/nvidia/spark/rapids/tool/qualification/QualificationSuite.scala
@@ -357,7 +357,7 @@ class QualificationSuite extends FunSuite with BeforeAndAfterEach with Logging {
   test("test different types in ReadSchema") {
     val testSchemas: ArrayBuffer[ArrayBuffer[String]] = ArrayBuffer(
       ArrayBuffer(""),
-      ArrayBuffer("firstName:string,lastName:string"),
+      ArrayBuffer("firstName:string,lastName:string", "", "address:string"),
       ArrayBuffer("properties:map<string,string>"),
       ArrayBuffer("name:array<string>"),
       ArrayBuffer("name:string,booksInterested:array<struct<name:string,price:decimal(8,2)," +