Qualification/Profiling tool add tests for Spark2 event logs (#2843)

* add qualification test for spark2

* Update readme

tools/README.md

@@ -13,7 +13,7 @@ GPU generated event logs.
 - Spark 3.0.1 or newer, the Qualification tool just needs the Spark jars and the Profiling tool
   runs a Spark application so needs the Spark runtime.
 - Java 8 or above
-- Complete Spark event log(s) from Spark 3.0 or above version.
+- Spark event log(s) from Spark 2.0 or above version.
   Support both rolled and compressed event logs with `.lz4`, `.lzf`, `.snappy` and `.zstd` suffixes.
   Also support Databricks specific rolled and compressed(.gz) eventlogs.
   The tool does not support nested directories, event log files or event log directories should be

tools/src/test/resources/QualificationExpectations/spark2_expectation.csv

@@ -0,0 +1,2 @@
+App Name,App ID,Score,Potential Problems,SQL Dataframe Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures
+Spark shell,local-1624892957956,21.07,"",3751,17801,58.47,false,0,""

tools/src/test/scala/com/nvidia/spark/rapids/tool/profiling/ApplicationInfoSuite.scala

@@ -132,6 +132,18 @@ class ApplicationInfoSuite extends FunSuite with Logging {
     }
   }
 
+  test("test spark2 eventlog") {
+    val eventLog = Array(s"$logDir/spark2-eventlog.zstd")
+    val apps = ToolTestUtils.processProfileApps(eventLog, sparkSession)
+    assert(apps.size == 1)
+    assert(apps.head.sparkVersion.equals("2.2.3"))
+    assert(apps.head.gpuMode.equals(false))
+    assert(apps.head.jobStart.size == 1)
+    assert(apps.head.jobStart.head.jobID.equals(0))
+    val stage0 = apps.head.stageSubmitted.filter(_.stageId == 0)
+    assert(stage0.head.numTasks.equals(6))
+  }
+
   test("malformed json eventlog") {
     val eventLog = s"$logDir/malformed_json_eventlog.zstd"
     TrampolineUtil.withTempDir { tempDir =>

tools/src/test/scala/com/nvidia/spark/rapids/tool/qualification/QualificationSuite.scala

@@ -94,6 +94,12 @@ class QualificationSuite extends FunSuite with BeforeAndAfterEach with Logging {
     runQualificationTest(logFiles, "nds_q86_test_expectation.csv")
   }
 
+  test("spark2 eventlog") {
+    val profileLogDir = ToolTestUtils.getTestResourcePath("spark-events-profiling")
+    val log = s"$profileLogDir/spark2-eventlog.zstd"
+    runQualificationTest(Array(log), "spark2_expectation.csv")
+  }
+
   test("test udf event logs") {
     val logFiles = Array(
       s"$logDir/dataset_eventlog",