diff --git a/docs/additional-functionality/qualification-profiling-tools.md b/docs/additional-functionality/qualification-profiling-tools.md index e3db672ba79..67becb1b832 100644 --- a/docs/additional-functionality/qualification-profiling-tools.md +++ b/docs/additional-functionality/qualification-profiling-tools.md @@ -60,7 +60,7 @@ Here are 2 options: ```bash git clone https://github.com/NVIDIA/spark-rapids.git cd spark-rapids -mvn -pl .,tools clean verify -DskipTests +mvn -Pdefault -pl .,tools clean verify -DskipTests ``` The jar is generated in below directory : @@ -144,14 +144,20 @@ you are running on needs to be taken into account. had to estimate it, it means the event log was missing the application finished event so we will use the last job or sql execution time we find as the end time used to calculate the duration. +`Complex Types and Unsupported Nested Complex Types` looks at the Read Schema and reports if there are any complex types(array, struct or maps) +in the schema. Nested complex types are complex types which contain other complex types (Example: array>). +Note that it can read all the schemas for DataSource V1. The Data Source V2 truncates the schema, so if you see ..., +then the full schema is not available. For such schemas we read until ... and report if there are any complex types and +nested complex types in that. + Note that SQL queries that contain failed jobs are not included. Sample output in csv: ``` -App Name,App ID,Score,Potential Problems,SQL DF Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,Read File Format Score,Unsupported Read File Formats and Types -job3,app-20210507174503-1704,4320658.0,"",9569,4320658,26171,35.34,false,0,"",20,100.0,"" -job1,app-20210507174503-2538,19864.04,"",6760,21802,83728,71.3,false,0,"",20,55.56,"Parquet[decimal]" +App Name,App ID,Score,Potential Problems,SQL DF Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,Read File Format Score,Unsupported Read File Formats and Types,Unsupported Write Data Format,Complex Types,Unsupported Nested Complex Types +job3,app-20210507174503-1704,4320658.0,"",9569,4320658,26171,35.34,false,0,"",20,100.0,"",JSON,array>;map,array> +job1,app-20210507174503-2538,19864.04,"",6760,21802,83728,71.3,false,0,"",20,55.56,"Parquet[decimal]",JSON;CSV,"","" ``` Sample output in text: diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/TypeChecks.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/TypeChecks.scala index dbffa534218..0cb177deb5c 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/TypeChecks.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/TypeChecks.scala @@ -2019,10 +2019,12 @@ object SupportedOpsForTools { val conf = new RapidsConf(Map.empty[String, String]) val types = TypeEnum.values.toSeq val header = Seq("Format", "Direction") ++ types + val writeOps: Array[String] = Array.fill(types.size)("NA") println(header.mkString(",")) GpuOverrides.fileFormats.toSeq.sortBy(_._1.toString).foreach { case (format, ioMap) => - val formatEnabled = format.toString.toLowerCase match { + val formatLowerCase = format.toString.toLowerCase + val formatEnabled = formatLowerCase match { case "csv" => conf.isCsvEnabled && conf.isCsvReadEnabled case "parquet" => conf.isParquetEnabled && conf.isParquetReadEnabled case "orc" => conf.isOrcEnabled && conf.isOrcReadEnabled @@ -2059,8 +2061,13 @@ object SupportedOpsForTools { read.support(t).text } } - // only support reads for now + // print read formats and types println(s"${(Seq(format, "read") ++ readOps).mkString(",")}") + + // print write formats and NA for types. Cannot determine types from event logs. + if (!formatLowerCase.equals("csv")) { + println(s"${(Seq(format, "write") ++ writeOps).mkString(",")}") + } } } diff --git a/tools/src/main/resources/supportedDataSource.csv b/tools/src/main/resources/supportedDataSource.csv index f5d90fb9a52..5e70a98d5d2 100644 --- a/tools/src/main/resources/supportedDataSource.csv +++ b/tools/src/main/resources/supportedDataSource.csv @@ -1,4 +1,6 @@ Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT CSV,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,S,CO,NA,NS,NA,NA,NA,NA,NA ORC,read,S,S,S,S,S,S,S,S,PS,S,CO,NA,NS,NA,PS,NS,PS,NS +ORC,write,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Parquet,read,S,S,S,S,S,S,S,S,PS,S,CO,NA,NS,NA,PS,PS,PS,NS +Parquet,write,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/tools/src/main/scala/com/nvidia/spark/rapids/tool/qualification/PluginTypeChecker.scala b/tools/src/main/scala/com/nvidia/spark/rapids/tool/qualification/PluginTypeChecker.scala index 9d1f4c8017d..c7236c0840c 100644 --- a/tools/src/main/scala/com/nvidia/spark/rapids/tool/qualification/PluginTypeChecker.scala +++ b/tools/src/main/scala/com/nvidia/spark/rapids/tool/qualification/PluginTypeChecker.scala @@ -16,7 +16,7 @@ package com.nvidia.spark.rapids.tool.qualification -import scala.collection.mutable.HashMap +import scala.collection.mutable.{ArrayBuffer,HashMap} import scala.io.{BufferedSource, Source} /** @@ -38,17 +38,22 @@ class PluginTypeChecker { private val DEFAULT_DS_FILE = "supportedDataSource.csv" // map of file format => Map[support category => Seq[Datatypes for that category]] - // contains the details of formats to which ones have datatypes not supported - // var for testing puposes - private var formatsToSupportedCategory = readSupportedTypesForPlugin + // contains the details of formats to which ones have datatypes not supported. + // Write formats contains only the formats that are supported. Types cannot be determined + // from event logs for write formats. + // var for testing purposes + private var (readFormatsAndTypes, writeFormats) = readSupportedTypesForPlugin // for testing purposes only def setPluginDataSourceFile(filePath: String): Unit = { val source = Source.fromFile(filePath) - formatsToSupportedCategory = readSupportedTypesForPlugin(source) + val (readFormatsAndTypesTest, writeFormatsTest) = readSupportedTypesForPlugin(source) + readFormatsAndTypes = readFormatsAndTypesTest + writeFormats = writeFormatsTest } - private def readSupportedTypesForPlugin: Map[String, Map[String, Seq[String]]] = { + private def readSupportedTypesForPlugin: ( + Map[String, Map[String, Seq[String]]], ArrayBuffer[String]) = { val source = Source.fromResource(DEFAULT_DS_FILE) readSupportedTypesForPlugin(source) } @@ -56,10 +61,12 @@ class PluginTypeChecker { // file format should be like this: // Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,... // CSV,read,S,S,S,S,S,S,S,S,S*,S,NS,NA,NS,NA,NA,NA,NA,NA + // Parquet,write,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA private def readSupportedTypesForPlugin( - source: BufferedSource): Map[String, Map[String, Seq[String]]] = { + source: BufferedSource): (Map[String, Map[String, Seq[String]]], ArrayBuffer[String]) = { // get the types the Rapids Plugin supports val allSupportedReadSources = HashMap.empty[String, Map[String, Seq[String]]] + val allSupportedWriteFormats = ArrayBuffer[String]() try { val fileContents = source.getLines().toSeq if (fileContents.size < 2) { @@ -85,12 +92,14 @@ class PluginTypeChecker { val allNsTypes = nsTypes.flatMap(t => getOtherTypes(t) :+ t) val allBySup = HashMap(NS -> allNsTypes) allSupportedReadSources.put(format, allBySup.toMap) + } else if (direction.equals("write")) { + allSupportedWriteFormats += format } } } finally { source.close() } - allSupportedReadSources.toMap + (allSupportedReadSources.toMap, allSupportedWriteFormats) } def getOtherTypes(typeRead: String): Seq[String] = { @@ -114,7 +123,7 @@ class PluginTypeChecker { def scoreReadDataTypes(format: String, schema: String): (Double, Set[String]) = { val schemaLower = schema.toLowerCase val formatInLower = format.toLowerCase - val typesBySup = formatsToSupportedCategory.get(formatInLower) + val typesBySup = readFormatsAndTypes.get(formatInLower) val score = typesBySup match { case Some(dtSupMap) => // check if any of the not supported types are in the schema @@ -140,4 +149,9 @@ class PluginTypeChecker { } score } + + def isWriteFormatsupported(writeFormat: ArrayBuffer[String]): ArrayBuffer[String] = { + writeFormat.map(x => x.toLowerCase.trim).filterNot( + writeFormats.map(x => x.trim).contains(_)) + } } diff --git a/tools/src/main/scala/com/nvidia/spark/rapids/tool/qualification/QualOutputWriter.scala b/tools/src/main/scala/com/nvidia/spark/rapids/tool/qualification/QualOutputWriter.scala index 9025b303ddb..230fb587fdd 100644 --- a/tools/src/main/scala/com/nvidia/spark/rapids/tool/qualification/QualOutputWriter.scala +++ b/tools/src/main/scala/com/nvidia/spark/rapids/tool/qualification/QualOutputWriter.scala @@ -44,7 +44,8 @@ class QualOutputWriter(outputDir: String, reportReadSchema: Boolean, printStdout val initHeader = s"App Name,$appIdStr,Score,Potential Problems,$sqlDurStr,$taskDurStr," + s"$appDurStr,Executor CPU Time Percent,App Duration Estimated," + "SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent," + - "Read File Format Score,Unsupported Read File Formats and Types" + "Read File Format Score,Unsupported Read File Formats and Types," + + "Unsupported Write Data Format,Unsupported Nested Types" if (reportReadSchema) { initHeader + ",Read Schema" } else { @@ -73,8 +74,11 @@ class QualOutputWriter(outputDir: String, reportReadSchema: Boolean, printStdout val failedIds = stringIfempty(appSum.failedSQLIds) // since csv, replace any commas with ; in the schema val readFileFormats = stringIfempty(appSum.readFileFormats.replace(",", ";")) + val complexTypes = stringIfempty(appSum.complexTypes.replace(",", ";")) + val nestedComplexTypes = stringIfempty(appSum.nestedComplexTypes.replace(",", ";")) val readFileScoreRounded = f"${appSum.readFileFormatScore}%1.2f" val readFormatNS = stringIfempty(appSum.readFileFormatAndTypesNotSupported) + val writeDataFormat = stringIfempty(appSum.writeDataFormat) val initRow = s"$appNameStr,$appIdStr,${appSum.score},$probStr," + @@ -82,7 +86,7 @@ class QualOutputWriter(outputDir: String, reportReadSchema: Boolean, printStdout s"${appSum.appDuration},${appSum.executorCpuTimePercent}," + s"${appSum.endDurationEstimated},${appSum.sqlDurationForProblematic},$failedIds," + s"${appSum.readScorePercent},${appSum.readFileFormatScore}," + - s"${readFormatNS}" + s"${readFormatNS},${writeDataFormat},${complexTypes},${nestedComplexTypes}" if (reportReadSchema) { initRow + s",$readFileFormats" } else { diff --git a/tools/src/main/scala/org/apache/spark/sql/rapids/tool/qualification/QualAppInfo.scala b/tools/src/main/scala/org/apache/spark/sql/rapids/tool/qualification/QualAppInfo.scala index 4ea547abcdc..97320fd7d47 100644 --- a/tools/src/main/scala/org/apache/spark/sql/rapids/tool/qualification/QualAppInfo.scala +++ b/tools/src/main/scala/org/apache/spark/sql/rapids/tool/qualification/QualAppInfo.scala @@ -41,6 +41,7 @@ class QualAppInfo( var isPluginEnabled = false var lastJobEndTime: Option[Long] = None var lastSQLEndTime: Option[Long] = None + val writeDataFormat: ArrayBuffer[String] = ArrayBuffer[String]() var appInfo: Option[QualApplicationInfo] = None val sqlStart: HashMap[Long, QualSQLExecutionInfo] = HashMap[Long, QualSQLExecutionInfo]() @@ -184,6 +185,15 @@ class QualAppInfo( }.getOrElse(1.0) } + def reportComplexTypes: (String, String) = { + if (dataSourceInfo.size != 0) { + val schema = dataSourceInfo.map { ds => ds.schema } + QualAppInfo.parseReadSchemaForNestedTypes(schema) + } else { + ("", "") + } + } + def aggregateStats(): Option[QualificationSummaryInfo] = { appInfo.map { info => val appDuration = calculateAppDuration(info.startTime).getOrElse(0L) @@ -205,12 +215,14 @@ class QualAppInfo( val typeString = types.mkString(":").replace(",", ":") s"${format}[$typeString]" }.mkString(";") + val writeFormat = writeFormatNotSupported(writeDataFormat) + val (allComplexTypes, nestedComplexTypes) = reportComplexTypes new QualificationSummaryInfo(info.appName, appId, scoreRounded, problems, sqlDataframeDur, sqlDataframeTaskDuration, appDuration, executorCpuTimePercent, endDurationEstimated, sqlDurProblem, failedIds, readScorePercent, readScoreHumanPercentRounded, notSupportFormatAndTypesString, - getAllReadFileFormats) + getAllReadFileFormats, writeFormat, allComplexTypes, nestedComplexTypes) } } @@ -228,8 +240,22 @@ class QualAppInfo( val existingIssues = sqlIDtoProblematic.getOrElse(sqlID, Set.empty[String]) sqlIDtoProblematic(sqlID) = existingIssues ++ issues } + // Get the write data format + if (node.name.contains("InsertIntoHadoopFsRelationCommand")) { + val writeFormat = node.desc.split(",")(2) + writeDataFormat += writeFormat + } } } + + def writeFormatNotSupported(writeFormat: ArrayBuffer[String]): String = { + // Filter unsupported write data format + val unSupportedWriteFormat = pluginTypeChecker.map { checker => + checker.isWriteFormatsupported(writeFormat) + }.getOrElse(ArrayBuffer[String]()) + + unSupportedWriteFormat.distinct.mkString(";").toUpperCase + } } class StageTaskQualificationSummary( @@ -273,7 +299,10 @@ case class QualificationSummaryInfo( readScorePercent: Int, readFileFormatScore: Double, readFileFormatAndTypesNotSupported: String, - readFileFormats: String) + readFileFormats: String, + writeDataFormat: String, + complexTypes: String, + nestedComplexTypes: String) object QualAppInfo extends Logging { def createApp( @@ -300,4 +329,80 @@ object QualAppInfo extends Logging { } app } + + def parseReadSchemaForNestedTypes(schema: ArrayBuffer[String]): (String, String) = { + val tempStringBuilder = new StringBuilder() + val individualSchema: ArrayBuffer[String] = new ArrayBuffer() + var angleBracketsCount = 0 + var parenthesesCount = 0 + val distinctSchema = schema.distinct.mkString(",") + + // Get the nested types i.e everything between < > + for (char <- distinctSchema) { + char match { + case '<' => angleBracketsCount += 1 + case '>' => angleBracketsCount -= 1 + // If the schema has decimals, Example decimal(6,2) then we have to make sure it has both + // opening and closing parentheses(unless the string is incomplete due to V2 reader). + case '(' => parenthesesCount += 1 + case ')' => parenthesesCount -= 1 + case _ => + } + if (angleBracketsCount == 0 && parenthesesCount == 0 && char.equals(',')) { + individualSchema += tempStringBuilder.toString + tempStringBuilder.setLength(0) + } + else { + tempStringBuilder.append(char); + } + } + if (!tempStringBuilder.isEmpty) { + individualSchema += tempStringBuilder.toString + } + + // If DataSource V2 is used, then Schema may be incomplete with ... appended at the end. + // We determine complex types and nested complex types until ... + val incompleteSchema = individualSchema.filter(x => x.contains("...")) + val completeSchema = individualSchema.filterNot(x => x.contains("...")) + + // Check if it has types + val incompleteTypes = incompleteSchema.map { x => + if (x.contains("...") && x.contains(":")) { + x.split(":", 2)(1).split("\\.\\.\\.")(0) + } else { + "" + } + } + // Omit columnName and get only schemas + val completeTypes = completeSchema.map(x => x.split(":", 2)(1)) + val schemaTypes = completeTypes ++ incompleteTypes + + // Filter only complex types. + // Example: array, array> + val complexTypes = schemaTypes.filter(x => + x.startsWith("array<") || x.startsWith("map<") || x.startsWith("struct<")) + + // Determine nested complex types from complex types + // Example: array> is nested complex type. + val nestedComplexTypes = complexTypes.filter(complexType => { + val startIndex = complexType.indexOf('<') + val closedBracket = complexType.lastIndexOf('>') + // If String is incomplete due to dsv2, then '>' may not be present. In that case traverse + // until length of the incomplete string + val lastIndex = if (closedBracket == -1) { + complexType.length - 1 + } else { + closedBracket + } + val string = complexType.substring(startIndex, lastIndex + 1) + string.contains("array<") || string.contains("struct<") || string.contains("map<") + }) + + // Since it is saved as csv, replace commas with ; + val complexTypesResult = complexTypes.filter(_.nonEmpty).mkString(";").replace(",", ";") + val nestedComplexTypesResult = nestedComplexTypes.filter( + _.nonEmpty).mkString(";").replace(",", ";") + + (complexTypesResult, nestedComplexTypesResult) + } } diff --git a/tools/src/test/resources/QualificationExpectations/complex_dec_expectation.csv b/tools/src/test/resources/QualificationExpectations/complex_dec_expectation.csv index dc35a000774..7036e8e4cc7 100644 --- a/tools/src/test/resources/QualificationExpectations/complex_dec_expectation.csv +++ b/tools/src/test/resources/QualificationExpectations/complex_dec_expectation.csv @@ -1,2 +1,2 @@ -App Name,App ID,Score,Potential Problems,SQL DF Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,Read File Format Score,Unsupported Read File Formats and Types -Spark shell,local-1626104300434,1322.1,DECIMAL,2429,1469,131104,88.35,false,160,"",20,50.0,Parquet[decimal];ORC[map:decimal] +App Name,App ID,Score,Potential Problems,SQL DF Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,Read File Format Score,Unsupported Read File Formats and Types,Unsupported Write Data Format,Complex Types,Unsupported Nested Complex Types +Spark shell,local-1626104300434,1322.1,DECIMAL,2429,1469,131104,88.35,false,160,"",20,50.0,Parquet[decimal];ORC[map:decimal],"",struct;lastname:string>;struct;previous:struct;city:string>>;array>;map;map>;map>;array>;array,struct;lastname:string>;struct;previous:struct;city:string>>;array>;map>;map>;array> diff --git a/tools/src/test/resources/QualificationExpectations/db_sim_test_expectation.csv b/tools/src/test/resources/QualificationExpectations/db_sim_test_expectation.csv index c439b9c7ab3..13e668b1d39 100644 --- a/tools/src/test/resources/QualificationExpectations/db_sim_test_expectation.csv +++ b/tools/src/test/resources/QualificationExpectations/db_sim_test_expectation.csv @@ -1,2 +1,2 @@ -App Name,App ID,Score,Potential Problems,SQL Dataframe Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,ReadFileFormat Score,Unsupported Read File Formats and Types -Spark shell,local-1623876083964,1417661.00,"",119903,1417661,133857,91.14,false,0,"",20,100.00,"" +App Name,App ID,Score,Potential Problems,SQL Dataframe Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,ReadFileFormat Score,Unsupported Read File Formats and Types,Unsupported Write Data Format,Complex Types,Unsupported Nested Complex Types +Spark shell,local-1623876083964,1417661.00,"",119903,1417661,133857,91.14,false,0,"",20,100.00,"","","","" diff --git a/tools/src/test/resources/QualificationExpectations/decimal_part_expectation.csv b/tools/src/test/resources/QualificationExpectations/decimal_part_expectation.csv index 0507cf03a4d..9ed7058426f 100644 --- a/tools/src/test/resources/QualificationExpectations/decimal_part_expectation.csv +++ b/tools/src/test/resources/QualificationExpectations/decimal_part_expectation.csv @@ -1,2 +1,2 @@ -App Name,App ID,Score,Potential Problems,SQL DF Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,Read File Format Score,Unsupported Read File Formats and Types -Spark shell,local-1626189209260,1052.3,DECIMAL,1314,1238,106033,57.21,false,1023,"",20,25.0,Parquet[decimal] +App Name,App ID,Score,Potential Problems,SQL DF Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,Read File Format Score,Unsupported Read File Formats and Types,Unsupported Write Data Format,Complex Types,Unsupported Nested Complex Types +Spark shell,local-1626189209260,1052.3,DECIMAL,1314,1238,106033,57.21,false,1023,"",20,25.0,Parquet[decimal],"",array>;map;map>;map>,array>;map>;map> diff --git a/tools/src/test/resources/QualificationExpectations/directory_test_expectation.csv b/tools/src/test/resources/QualificationExpectations/directory_test_expectation.csv index c439b9c7ab3..13e668b1d39 100644 --- a/tools/src/test/resources/QualificationExpectations/directory_test_expectation.csv +++ b/tools/src/test/resources/QualificationExpectations/directory_test_expectation.csv @@ -1,2 +1,2 @@ -App Name,App ID,Score,Potential Problems,SQL Dataframe Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,ReadFileFormat Score,Unsupported Read File Formats and Types -Spark shell,local-1623876083964,1417661.00,"",119903,1417661,133857,91.14,false,0,"",20,100.00,"" +App Name,App ID,Score,Potential Problems,SQL Dataframe Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,ReadFileFormat Score,Unsupported Read File Formats and Types,Unsupported Write Data Format,Complex Types,Unsupported Nested Complex Types +Spark shell,local-1623876083964,1417661.00,"",119903,1417661,133857,91.14,false,0,"",20,100.00,"","","","" diff --git a/tools/src/test/resources/QualificationExpectations/nds_q86_fail_test_expectation.csv b/tools/src/test/resources/QualificationExpectations/nds_q86_fail_test_expectation.csv index 34d5ee56b99..572045cdfc4 100644 --- a/tools/src/test/resources/QualificationExpectations/nds_q86_fail_test_expectation.csv +++ b/tools/src/test/resources/QualificationExpectations/nds_q86_fail_test_expectation.csv @@ -1,2 +1,2 @@ -App Name,App ID,Score,Potential Problems,SQL Dataframe Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,ReadFileFormat Score,Unsupported Read File Formats and Types -TPC-DS Like Bench q86,app-20210319163812-1778,4320658.00,"",4,4320658,26171,0.0,false,0,24,20,100.00,"" +App Name,App ID,Score,Potential Problems,SQL Dataframe Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,ReadFileFormat Score,Unsupported Read File Formats and Types,Unsupported Write Data Format,Complex Types,Unsupported Nested Complex Types +TPC-DS Like Bench q86,app-20210319163812-1778,4320658.00,"",4,4320658,26171,0.0,false,0,24,20,100.00,"","","","" diff --git a/tools/src/test/resources/QualificationExpectations/nds_q86_test_expectation.csv b/tools/src/test/resources/QualificationExpectations/nds_q86_test_expectation.csv index f0d6c657e8c..63ab5d8341f 100644 --- a/tools/src/test/resources/QualificationExpectations/nds_q86_test_expectation.csv +++ b/tools/src/test/resources/QualificationExpectations/nds_q86_test_expectation.csv @@ -1,2 +1,2 @@ -App Name,App ID,Score,Potential Problems,SQL Dataframe Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,ReadFileFormat Score,Unsupported Read File Formats and Types -TPC-DS Like Bench q86,app-20210319163812-1778,4320658.00,"",9569,4320658,26171,35.34,false,0,"",20,100.00,"" +App Name,App ID,Score,Potential Problems,SQL Dataframe Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,ReadFileFormat Score,Unsupported Read File Formats and Types,Unsupported Write Data Format,Complex Types,Unsupported Nested Complex Types +TPC-DS Like Bench q86,app-20210319163812-1778,4320658.00,"",9569,4320658,26171,35.34,false,0,"",20,100.00,"","","","" diff --git a/tools/src/test/resources/QualificationExpectations/nested_dsv2_expectation.csv b/tools/src/test/resources/QualificationExpectations/nested_dsv2_expectation.csv new file mode 100644 index 00000000000..f96af674b6d --- /dev/null +++ b/tools/src/test/resources/QualificationExpectations/nested_dsv2_expectation.csv @@ -0,0 +1,2 @@ +App Name,App ID,Score,Potential Problems,SQL DF Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,Read File Format Score,Unsupported Read File Formats and Types,Unsupported Write Data Format,Unsupported Nested Types +Spark shell,local-1630045673160,3757.0,"",1294,3757,21200,34.56,false,0,"",20,100.0,"","",array>;map>;map>;map>,array>;map> diff --git a/tools/src/test/resources/QualificationExpectations/qual_test_missing_sql_end_expectation.csv b/tools/src/test/resources/QualificationExpectations/qual_test_missing_sql_end_expectation.csv index 5c389705ffd..96c09be9200 100644 --- a/tools/src/test/resources/QualificationExpectations/qual_test_missing_sql_end_expectation.csv +++ b/tools/src/test/resources/QualificationExpectations/qual_test_missing_sql_end_expectation.csv @@ -1,2 +1,2 @@ -App Name,App ID,Score,Potential Problems,SQL Dataframe Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,ReadFileFormat Score,Unsupported Read File Formats and Types -Rapids Spark Profiling Tool Unit Tests,local-1622561780883,0.00,"",0,0,7673,0.0,false,0,"",20,100.00,"" +App Name,App ID,Score,Potential Problems,SQL Dataframe Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,ReadFileFormat Score,Unsupported Read File Formats and Types,Unsupported Write Data Format,Complex Types,Unsupported Nested Complex Types +Rapids Spark Profiling Tool Unit Tests,local-1622561780883,0.00,"",0,0,7673,0.0,false,0,"",20,100.00,"","","","" diff --git a/tools/src/test/resources/QualificationExpectations/qual_test_simple_expectation.csv b/tools/src/test/resources/QualificationExpectations/qual_test_simple_expectation.csv index d5ebc2c2e42..a80c93cd5df 100644 --- a/tools/src/test/resources/QualificationExpectations/qual_test_simple_expectation.csv +++ b/tools/src/test/resources/QualificationExpectations/qual_test_simple_expectation.csv @@ -1,5 +1,5 @@ -App Name,App ID,Score,Potential Problems,SQL Dataframe Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,ReadFileFormat Score,Unsupported Read File Formats and Types -Rapids Spark Profiling Tool Unit Tests,local-1622043423018,125035.00,"",11128,125035,16319,37.81,false,0,"",20,100.00,"" -Rapids Spark Profiling Tool Unit Tests,local-1623281204390,3732.80,UDF,2032,4666,6240,46.27,false,577,"",20,0.00,"JSON[*]" -Rapids Spark Profiling Tool Unit Tests,local-1621966649543,0.00,"",0,0,10650,0.0,false,0,"",20,100.00,"" -Rapids Spark Profiling Tool Unit Tests,local-1621955976602,0.00,"",0,0,10419,0.0,false,0,"",20,100.00,"" +App Name,App ID,Score,Potential Problems,SQL Dataframe Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,ReadFileFormat Score,Unsupported Read File Formats and Types,Unsupported Write Data Format,Complex Types,Unsupported Nested Complex Types +Rapids Spark Profiling Tool Unit Tests,local-1622043423018,125035.00,"",11128,125035,16319,37.81,false,0,"",20,100.00,"",JSON,"","" +Rapids Spark Profiling Tool Unit Tests,local-1623281204390,3732.80,UDF,2032,4666,6240,46.27,false,577,"",20,0.00,"JSON[*]",JSON,"","" +Rapids Spark Profiling Tool Unit Tests,local-1621966649543,0.00,"",0,0,10650,0.0,false,0,"",20,100.00,"",JSON,"","" +Rapids Spark Profiling Tool Unit Tests,local-1621955976602,0.00,"",0,0,10419,0.0,false,0,"",20,100.00,"",JSON,"","" diff --git a/tools/src/test/resources/QualificationExpectations/read_dsv1_expectation.csv b/tools/src/test/resources/QualificationExpectations/read_dsv1_expectation.csv index 5ee486eaa91..0a744ac5eef 100644 --- a/tools/src/test/resources/QualificationExpectations/read_dsv1_expectation.csv +++ b/tools/src/test/resources/QualificationExpectations/read_dsv1_expectation.csv @@ -1,2 +1,2 @@ -App Name,App ID,Score,Potential Problems,SQL Dataframe Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,ReadFileFormat Score,Unsupported Read File Formats and Types -Spark shell,local-1624371544219,18670.63,"",6695,20421,175293,72.15,false,0,"",20,57.14,"JSON[*];Text[*]" +App Name,App ID,Score,Potential Problems,SQL Dataframe Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,ReadFileFormat Score,Unsupported Read File Formats and Types,Unsupported Write Data Format,Complex Types,Unsupported Nested Complex Types +Spark shell,local-1624371544219,18670.63,"",6695,20421,175293,72.15,false,0,"",20,57.14,"JSON[*];Text[*]",JSON,"","" diff --git a/tools/src/test/resources/QualificationExpectations/read_dsv2_expectation.csv b/tools/src/test/resources/QualificationExpectations/read_dsv2_expectation.csv index 5f6156ce424..b785e20391f 100644 --- a/tools/src/test/resources/QualificationExpectations/read_dsv2_expectation.csv +++ b/tools/src/test/resources/QualificationExpectations/read_dsv2_expectation.csv @@ -1,2 +1,2 @@ -App Name,App ID,Score,Potential Problems,SQL Dataframe Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,ReadFileFormat Score,Unsupported Read File Formats and Types -Spark shell,local-1624371906627,19864.04,"",6760,21802,83738,71.3,false,0,"",20,55.56,"Text[*];json[*]" +App Name,App ID,Score,Potential Problems,SQL Dataframe Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,ReadFileFormat Score,Unsupported Read File Formats and Types,Unsupported Write Data Format,Complex Types,Unsupported Nested Complex Types +Spark shell,local-1624371906627,19864.04,"",6760,21802,83738,71.3,false,0,"",20,55.56,"Text[*];json[*]",JSON,"","" \ No newline at end of file diff --git a/tools/src/test/resources/QualificationExpectations/spark2_expectation.csv b/tools/src/test/resources/QualificationExpectations/spark2_expectation.csv index 4474a793267..b308e944890 100644 --- a/tools/src/test/resources/QualificationExpectations/spark2_expectation.csv +++ b/tools/src/test/resources/QualificationExpectations/spark2_expectation.csv @@ -1,2 +1,2 @@ -App Name,App ID,Score,Potential Problems,SQL Dataframe Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,ReadFileFormat Score,Unsupported Read File Formats and Types -Spark shell,local-1624892957956,37581.00,"",3751,37581,17801,58.47,false,0,"",20,100.00,"" +App Name,App ID,Score,Potential Problems,SQL Dataframe Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,ReadFileFormat Score,Unsupported Read File Formats and Types,Unsupported Write Data Format,Complex Types,Unsupported Nested Complex Types +Spark shell,local-1624892957956,37581.00,"",3751,37581,17801,58.47,false,0,"",20,100.00,"","","","" diff --git a/tools/src/test/resources/QualificationExpectations/truncated_1_end_expectation.csv b/tools/src/test/resources/QualificationExpectations/truncated_1_end_expectation.csv index e8e7b82d3d4..cab5d441751 100644 --- a/tools/src/test/resources/QualificationExpectations/truncated_1_end_expectation.csv +++ b/tools/src/test/resources/QualificationExpectations/truncated_1_end_expectation.csv @@ -1,2 +1,2 @@ -App Name,App ID,Score,Potential Problems,SQL Dataframe Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,ReadFileFormat Score,Unsupported Read File Formats and Types -Rapids Spark Profiling Tool Unit Tests,local-1622043423018,0.00,"",0,0,4872,0.0,true,0,"",20,100.00,"" +App Name,App ID,Score,Potential Problems,SQL Dataframe Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,ReadFileFormat Score,Unsupported Read File Formats and Types,Unsupported Write Data Format,Complex Types,Unsupported Nested Complex Types +Rapids Spark Profiling Tool Unit Tests,local-1622043423018,0.00,"",0,0,4872,0.0,true,0,"",20,100.00,"",JSON,"","" diff --git a/tools/src/test/resources/QualificationExpectations/write_format_expectation.csv b/tools/src/test/resources/QualificationExpectations/write_format_expectation.csv new file mode 100644 index 00000000000..1bb4ca3ac7a --- /dev/null +++ b/tools/src/test/resources/QualificationExpectations/write_format_expectation.csv @@ -0,0 +1,2 @@ +App Name,App ID,Score,Potential Problems,SQL Dataframe Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,ReadFileFormat Score,Unsupported Read File Formats and Types,Unsupported Write Data Format,Complex Types,Unsupported Nested Complex Types +Spark shell,local-1629442299891,736.0,DECIMAL,1992,920,19554,91.72,false,1992,"",20,0.0,Parquet[decimal],CSV;JSON,"","" diff --git a/tools/src/test/resources/spark-events-qualification/eventlog_complex_dec_nested b/tools/src/test/resources/spark-events-qualification/eventlog_complex_dec_nested new file mode 100644 index 00000000000..edec3b4d547 --- /dev/null +++ b/tools/src/test/resources/spark-events-qualification/eventlog_complex_dec_nested @@ -0,0 +1,101 @@ +{"Event":"SparkListenerLogStart","Spark Version":"3.1.1"} +{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"cores":{"Resource Name":"cores","Amount":1,"Discovery Script":"","Vendor":""},"memory":{"Resource Name":"memory","Amount":1024,"Discovery Script":"","Vendor":""},"offHeap":{"Resource Name":"offHeap","Amount":0,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1630551717122,"Executor ID":"driver","Executor Info":{"Host":"nartal.attlocal.net","Total Cores":8,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"nartal.attlocal.net","Port":36653},"Maximum Memory":384093388,"Timestamp":1630551717149,"Maximum Onheap Memory":384093388,"Maximum Offheap Memory":0} +{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-8-openjdk-amd64/jre","Java Version":"1.8.0_292 (Private Build)","Scala Version":"version 2.12.10"},"Spark Properties":{"spark.driver.host":"nartal.attlocal.net","spark.eventLog.enabled":"true","spark.driver.port":"44977","spark.repl.class.uri":"spark://nartal.attlocal.net:44977/classes","spark.jars":"file:///home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.repl.class.outputDir":"/tmp/spark-6437c8fb-c50a-434e-a838-d827759715c1/repl-56a3ce44-fe38-4b08-be93-b9831c0ca528","spark.app.name":"Spark shell","spark.scheduler.mode":"FIFO","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.startTime":"1630551715882","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://nartal.attlocal.net:44977/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.submit.deployMode":"client","spark.master":"local[*]","spark.home":"/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","spark.eventLog.dir":"/home/nartal/data_format_eventlog","spark.sql.catalogImplementation":"hive","spark.repl.local.jars":"file:///home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.app.id":"local-1630551716813"},"Hadoop Properties":{"hadoop.service.shutdown.timeout":"30s","yarn.resourcemanager.amlauncher.thread-count":"50","yarn.sharedcache.enabled":"false","fs.s3a.connection.maximum":"15","yarn.nodemanager.numa-awareness.numactl.cmd":"/usr/bin/numactl","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","yarn.timeline-service.timeline-client.number-of-async-entities-to-merge":"10","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.jobhistory.loadedjob.tasks.max":"-1","mapreduce.framework.name":"local","yarn.sharedcache.uploader.server.thread-count":"50","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","hadoop.security.groups.cache.background.reload.threads":"3","yarn.resourcemanager.webapp.cross-origin.enabled":"false","fs.AbstractFileSystem.ftp.impl":"org.apache.hadoop.fs.ftp.FtpFs","hadoop.registry.secure":"false","hadoop.shell.safely.delete.limit.num.files":"100","dfs.bytes-per-checksum":"512","mapreduce.job.acl-view-job":" ","fs.s3a.s3guard.ddb.background.sleep":"25ms","fs.s3a.retry.limit":"${fs.s3a.attempts.maximum}","mapreduce.jobhistory.loadedjobs.cache.size":"5","fs.s3a.s3guard.ddb.table.create":"false","yarn.nodemanager.amrmproxy.enabled":"false","yarn.timeline-service.entity-group-fs-store.with-user-dir":"false","mapreduce.input.fileinputformat.split.minsize":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","yarn.sharedcache.checksum.algo.impl":"org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl","yarn.nodemanager.amrmproxy.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor","yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size":"10485760","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","mapreduce.task.profile.maps":"0-2","yarn.scheduler.include-port-in-node-name":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","yarn.resourcemanager.node-removal-untracked.timeout-ms":"60000","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","mapreduce.jobhistory.webapp.https.address":"0.0.0.0:19890","yarn.node-labels.fs-store.impl.class":"org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore","yarn.nodemanager.collector-service.address":"${yarn.nodemanager.hostname}:8048","fs.trash.checkpoint.interval":"0","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","yarn.resourcemanager.node-ip-cache.expiry-interval-secs":"-1","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","yarn.resourcemanager.placement-constraints.handler":"disabled","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"1000","yarn.resourcemanager.system-metrics-publisher.enabled":"false","yarn.resourcemanager.placement-constraints.algorithm.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm","yarn.sharedcache.webapp.address":"0.0.0.0:8788","yarn.resourcemanager.delegation.token.renew-interval":"*********(redacted)","yarn.sharedcache.nm.uploader.replication.factor":"10","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","mapreduce.reduce.skip.proc-count.auto-incr":"true","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","ipc.client.fallback-to-simple-auth-allowed":"false","yarn.nodemanager.resource.memory.enforced":"true","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","yarn.acl.reservation-enable":"false","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","yarn.app.mapreduce.am.hard-kill-timeout-ms":"10000","fs.s3a.etag.checksum.enabled":"false","yarn.nodemanager.container-metrics.enable":"true","yarn.timeline-service.client.fd-clean-interval-secs":"60","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","hadoop.common.configuration.version":"3.0.0","fs.s3a.s3guard.ddb.table.capacity.read":"500","yarn.nodemanager.remote-app-log-dir-suffix":"logs","yarn.nodemanager.windows-container.cpu-limit.enabled":"false","yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed":"false","file.blocksize":"67108864","hadoop.registry.zk.retry.ceiling.ms":"60000","yarn.scheduler.configuration.leveldb-store.path":"${hadoop.tmp.dir}/yarn/system/confstore","yarn.sharedcache.store.in-memory.initial-delay-mins":"10","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","mapreduce.map.skip.proc-count.auto-incr":"true","fs.s3a.committer.name":"file","mapreduce.task.profile.reduces":"0-2","hadoop.zk.num-retries":"1000","yarn.webapp.xfs-filter.enabled":"true","seq.io.sort.mb":"100","yarn.scheduler.configuration.max.version":"100","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"${yarn.resourcemanager.hostname}:8030","yarn.node-labels.enabled":"false","yarn.resourcemanager.webapp.ui-actions.enabled":"true","mapreduce.task.timeout":"600000","yarn.sharedcache.client-server.thread-count":"50","hadoop.security.groups.shell.command.timeout":"0s","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.nodemanager.elastic-memory-control.oom-handler":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.DefaultOOMHandler","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"file:///","yarn.minicluster.use-rpc":"false","fs.har.impl.disable.cache":"true","yarn.webapp.ui2.enable":"false","io.compression.codec.bzip2.library":"system-native","yarn.nodemanager.distributed-scheduling.enabled":"false","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"${yarn.resourcemanager.hostname}:8090","mapreduce.jobhistory.address":"0.0.0.0:10020","yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.is.minicluster":"false","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","mapreduce.task.combine.progress.records":"10000","yarn.resourcemanager.epoch.range":"0","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","fs.AbstractFileSystem.wasbs.impl":"org.apache.hadoop.fs.azure.Wasbs","yarn.timeline-service.entity-group-fs-store.cache-store-class":"org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore","fs.ftp.transfer.mode":"BLOCK_TRANSFER_MODE","ipc.server.log.slow.rpc":"false","yarn.resourcemanager.node-labels.provider.fetch-interval-ms":"1800000","yarn.router.webapp.https.address":"0.0.0.0:8091","yarn.nodemanager.webapp.cross-origin.enabled":"false","fs.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","yarn.resourcemanager.auto-update.containers":"false","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","yarn.scheduler.configuration.zk-store.parent-path":"/confstore","yarn.nodemanager.default-container-executor.log-dirs.permissions":"710","yarn.app.attempt.diagnostics.limit.kc":"64","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"-1","fs.AbstractFileSystem.abfs.impl":"org.apache.hadoop.fs.azurebfs.Abfs","yarn.timeline-service.writer.flush-interval-seconds":"60","fs.s3a.fast.upload.active.blocks":"4","hadoop.security.credential.clear-text-fallback":"true","yarn.nodemanager.collector-service.thread-count":"5","fs.azure.secure.mode":"false","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","yarn.resourcemanager.fs.state-store.num-retries":"0","yarn.resourcemanager.nodemanager-connect-retries":"10","yarn.nodemanager.log-aggregation.num-log-files-per-app":"30","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","fs.s3a.committer.magic.enabled":"false","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","yarn.system-metrics-publisher.enabled":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","mapreduce.task.io.sort.factor":"10","yarn.nodemanager.amrmproxy.client.thread-count":"25","ha.failover-controller.new-active.rpc-timeout.ms":"60000","yarn.nodemanager.container-localizer.java.opts":"-Xmx256m","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","yarn.nodemanager.windows-container.memory-limit.enabled":"false","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.minicluster.fixed.ports":"false","hadoop.tags.system":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.cluster.max-application-priority":"0","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","hadoop.caller.context.signature.max.size":"40","yarn.client.load.resource-types.from-server":"false","ha.zookeeper.session-timeout.ms":"10000","tfile.io.chunk.size":"1048576","fs.s3a.s3guard.ddb.table.capacity.write":"100","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","yarn.scheduler.configuration.store.max-logs":"1000","yarn.nodemanager.node-attributes.provider.fetch-interval-ms":"600000","fs.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","yarn.nodemanager.local-cache.max-files-per-directory":"8192","hadoop.http.cross-origin.enabled":"false","hadoop.zk.acl":"world:anyone:rwcda","mapreduce.map.sort.spill.percent":"0.80","yarn.timeline-service.entity-group-fs-store.scan-interval-seconds":"60","yarn.node-attribute.fs-store.impl.class":"org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore","fs.s3a.retry.interval":"500ms","yarn.timeline-service.client.best-effort":"false","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","fs.AbstractFileSystem.swebhdfs.impl":"org.apache.hadoop.fs.SWebHdfs","yarn.nodemanager.elastic-memory-control.timeout-sec":"5","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","yarn.timeline-service.reader.webapp.address":"${yarn.timeline-service.webapp.address}","yarn.resourcemanager.placement-constraints.algorithm.pool-size":"1","yarn.timeline-service.hbase.coprocessor.jar.hdfs.location":"/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler","yarn.app.mapreduce.am.command-opts":"-Xmx1024m","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","io.mapfile.bloom.error.rate":"0.005","fs.client.resolve.topology.enabled":"false","yarn.nodemanager.runtime.linux.allowed-runtimes":"default","yarn.sharedcache.store.class":"org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","fs.s3a.metadatastore.impl":"org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore","io.skip.checksum.errors":"false","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","yarn.timeline-service.webapp.xfs-filter.xframe-options":"SAMEORIGIN","fs.s3a.connection.timeout":"200000","mapreduce.job.max.split.locations":"15","yarn.resourcemanager.nm-container-queuing.max-queue-length":"15","hadoop.registry.zk.session.timeout.ms":"60000","yarn.federation.cache-ttl.secs":"300","mapreduce.jvm.system-properties-to-log":"os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name","yarn.resourcemanager.opportunistic-container-allocation.nodes-used":"10","yarn.timeline-service.entity-group-fs-store.active-dir":"/tmp/entity-file-history/active","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","fs.AbstractFileSystem.abfss.impl":"org.apache.hadoop.fs.azurebfs.Abfss","yarn.sharedcache.uploader.server.address":"0.0.0.0:8046","yarn.resourcemanager.delegation-token.max-conf-size-bytes":"*********(redacted)","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","yarn.minicluster.yarn.nodemanager.resource.memory-mb":"4096","mapreduce.job.emit-timeline-data":"false","yarn.nodemanager.resource.system-reserved-memory-mb":"-1","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","yarn.resourcemanager.admin.client.thread-count":"1","yarn.dispatcher.drain-events.timeout":"300000","fs.s3a.buffer.dir":"${hadoop.tmp.dir}/s3a","hadoop.ssl.enabled.protocols":"TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2","mapreduce.jobhistory.admin.address":"0.0.0.0:10033","yarn.log-aggregation-status.time-out.ms":"600000","fs.s3a.assumed.role.sts.endpoint.region":"us-west-1","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","yarn.router.clientrm.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor","yarn.resourcemanager.zk-appid-node.split-index":"0","ftp.blocksize":"67108864","yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions":"read","yarn.router.rmadmin.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor","yarn.nodemanager.log-container-debug-info.enabled":"true","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms":"20","yarn.nodemanager.delete.debug-delay-sec":"0","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","yarn.nodemanager.container-retry-minimum-interval-ms":"1000","hadoop.security.groups.cache.secs":"300","yarn.federation.enabled":"false","fs.azure.local.sas.key.mode":"false","ipc.maximum.data.length":"67108864","mapreduce.shuffle.max.threads":"0","yarn.router.pipeline.cache-max-size":"25","yarn.resourcemanager.nm-container-queuing.load-comparator":"QUEUE_LENGTH","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","yarn.nodemanager.amrmproxy.ha.enable":"false","mapreduce.jobhistory.http.policy":"HTTP_ONLY","yarn.sharedcache.store.in-memory.check-period-mins":"720","hadoop.security.group.mapping.ldap.ssl":"false","yarn.client.application-client-protocol.poll-interval-ms":"200","yarn.scheduler.configuration.leveldb-store.compaction-interval-secs":"86400","yarn.timeline-service.writer.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl","ha.zookeeper.parent-znode":"/hadoop-ha","yarn.nodemanager.log-aggregation.policy.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","yarn.resourcemanager.placement-constraints.scheduler.pool-size":"1","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","yarn.nodemanager.recovery.supervised":"false","yarn.sharedcache.admin.thread-count":"1","yarn.resourcemanager.ha.automatic-failover.enabled":"true","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.resourcemanager.address":"${yarn.resourcemanager.hostname}:8032","ipc.client.ping":"true","mapreduce.task.local-fs.write-limit.bytes":"-1","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","yarn.nodemanager.resource-plugins.gpu.docker-plugin":"nvidia-docker-v1","hadoop.ssl.enabled":"false","fs.s3a.multipart.purge":"false","yarn.scheduler.configuration.store.class":"file","yarn.resourcemanager.nm-container-queuing.queue-limit-stdev":"1.0f","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","ipc.client.bind.wildcard.addr":"false","yarn.resourcemanager.webapp.rest-csrf.enabled":"false","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"10","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","yarn.nodemanager.runtime.linux.docker.allowed-container-networks":"host,none,bridge","yarn.nodemanager.node-labels.resync-interval-ms":"120000","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","mapreduce.job.maps":"2","mapreduce.jobhistory.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","yarn.rm.system-metrics-publisher.emit-container-events":"false","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","fs.s3a.s3guard.ddb.throttle.retry.interval":"100ms","fs.AbstractFileSystem.wasb.impl":"org.apache.hadoop.fs.azure.Wasb","mapreduce.client.submit.file.replication":"10","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"2147483647","yarn.resourcemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold":"1","yarn.nodemanager.webapp.rest-csrf.enabled":"false","fs.ftp.host.port":"21","ipc.ping.interval":"60000","yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size":"10","yarn.resourcemanager.admin.address":"${yarn.resourcemanager.hostname}:8033","file.client-write-packet-size":"65536","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"true","hadoop.security.key.default.bitlength":"128","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","yarn.nodemanager.log.deletion-threads-count":"4","yarn.webapp.filter-entity-list-by-user":"false","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"100","yarn.nodemanager.localizer.client.thread-count":"5","io.erasurecode.codec.rs.rawcoders":"rs_native,rs_java","io.erasurecode.codec.rs-legacy.rawcoders":"rs-legacy_java","yarn.sharedcache.admin.address":"0.0.0.0:8047","yarn.resourcemanager.placement-constraints.algorithm.iterator":"SERIAL","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec","mapreduce.job.cache.limit.max-resources-mb":"0","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"5000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","hadoop.workaround.non.threadsafe.getpwuid":"true","fs.df.interval":"60000","fs.s3a.multiobjectdelete.enable":"true","yarn.sharedcache.cleaner.resource-sleep-ms":"0","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"false","io.file.buffer.size":"65536","hadoop.security.group.mapping.ldap.search.attr.member":"member","hadoop.security.random.device.file.path":"/dev/urandom","hadoop.security.sensitive-config-keys":"*********(redacted)","fs.s3a.s3guard.ddb.max.retries":"9","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","yarn.intermediate-data-encryption.enable":"false","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.nodemanager.container.stderr.pattern":"{*stderr*,*STDERR*}","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/tmp/hadoop-yarn/staging","mapreduce.reduce.shuffle.read.timeout":"180000","hadoop.http.cross-origin.max-age":"1800","io.erasurecode.codec.xor.rawcoders":"xor_native,xor_java","fs.s3a.connection.establish.timeout":"5000","mapreduce.job.running.map.limit":"0","yarn.minicluster.control-resource-monitoring":"false","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","yarn.federation.state-store.class":"org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","adl.http.timeout":"-1","yarn.resourcemanager.placement-constraints.retry-attempts":"3","hadoop.caller.context.enabled":"false","yarn.nodemanager.vmem-pmem-ratio":"2.1","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","hadoop.zk.timeout-ms":"10000","fs.s3a.s3guard.cli.prune.age":"86400000","yarn.nodemanager.resource.pcores-vcores-multiplier":"1.0","yarn.nodemanager.runtime.linux.sandbox-mode":"disabled","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.s3a.committer.threads":"8","hadoop.zk.retry-interval-ms":"1000","hadoop.security.crypto.buffer.size":"8192","yarn.nodemanager.node-labels.provider.fetch-interval-ms":"600000","mapreduce.jobhistory.recovery.store.leveldb.path":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.client.failover-retries-on-socket-timeouts":"0","yarn.nodemanager.resource.memory.enabled":"false","fs.azure.authorization.caching.enable":"true","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","mapreduce.job.finish-when-all-reducers-done":"true","hadoop.registry.jaas.context":"Client","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","io.map.index.interval":"128","yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms":"100","fs.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","mapreduce.job.counters.max":"120","mapreduce.jobhistory.webapp.rest-csrf.enabled":"false","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","yarn.timeline-service.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/timeline","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","hadoop.security.auth_to_local.mechanism":"hadoop","yarn.timeline-service.app-collector.linger-period.ms":"60000","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.reservation-system.planfollower.time-step":"1000","yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed":"true","yarn.webapp.api-service.enable":"false","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","fs.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","yarn.nodemanager.container.stderr.tail.bytes":"4096","hadoop.security.group.mapping.ldap.read.timeout.ms":"60000","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","mapreduce.outputcommitter.factory.scheme.s3a":"org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory","hadoop.security.groups.cache.background.reload":"false","yarn.nodemanager.container-monitor.enabled":"true","yarn.nodemanager.elastic-memory-control.enabled":"false","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","yarn.resourcemanager.rm.container-allocation.expiry-interval-ms":"600000","mapreduce.fileoutputcommitter.algorithm.version":"1","yarn.resourcemanager.work-preserving-recovery.enabled":"true","mapreduce.map.skip.maxrecords":"0","yarn.sharedcache.root-dir":"/sharedcache","fs.s3a.retry.throttle.limit":"${fs.s3a.attempts.maximum}","hadoop.http.authentication.type":"simple","mapreduce.job.cache.limit.max-resources":"0","mapreduce.task.userlog.limit.kb":"0","yarn.resourcemanager.scheduler.monitor.enable":"false","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","yarn.nodemanager.resource-monitor.interval-ms":"3000","yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices":"auto","mapreduce.job.sharedcache.mode":"disabled","yarn.nodemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.shuffle.listen.queue.size":"128","yarn.scheduler.configuration.mutation.acl-policy.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy","mapreduce.map.cpu.vcores":"1","yarn.log-aggregation.file-formats":"TFile","yarn.timeline-service.client.fd-retain-secs":"300","hadoop.user.group.static.mapping.overrides":"dr.who=;","fs.azure.sas.expiry.period":"90d","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","yarn.router.webapp.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST","yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage":"90.0","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","yarn.nodemanager.disk-validator":"basic","yarn.app.mapreduce.client.job.max-retries":"3","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","hadoop.security.group.mapping.ldap.connection.timeout.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","yarn.nodemanager.container-diagnostics-maximum-size":"10000","hadoop.registry.zk.retry.interval.ms":"1000","yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms":"1000","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","fs.s3a.assumed.role.session.duration":"30m","hadoop.security.group.mapping.ldap.conversion.rule":"none","hadoop.ssl.server.conf":"ssl-server.xml","fs.s3a.retry.throttle.interval":"1000ms","seq.io.sort.factor":"100","yarn.sharedcache.cleaner.initial-delay-mins":"10","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.nodemanager.runtime.linux.docker.capabilities":"CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE","yarn.acl.enable":"false","yarn.timeline-service.entity-group-fs-store.done-dir":"/tmp/entity-file-history/done/","mapreduce.task.profile":"false","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","mapreduce.jobhistory.always-scan-user-dir":"false","yarn.nodemanager.opportunistic-containers-use-pause-for-preemption":"false","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","yarn.timeline-service.reader.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold":"1","yarn.resourcemanager.configuration.file-system-based-store":"/yarn/conf","mapreduce.job.cache.limit.max-single-resource-mb":"0","yarn.nodemanager.runtime.linux.docker.stop.grace-period":"10","yarn.resourcemanager.resource-profiles.source-file":"resource-profiles.json","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","fs.automatic.close":"true","yarn.nodemanager.hostname":"0.0.0.0","yarn.nodemanager.resource.memory.cgroups.swappiness":"0","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","yarn.timeline-service.app-aggregation-interval-secs":"15","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","yarn.nodemanager.container-localizer.log.level":"INFO","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60","mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","yarn.resourcemanager.delegation.token.max-lifetime":"*********(redacted)","yarn.timeline-service.client.drain-entities.timeout.ms":"2000","yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin","yarn.timeline-service.entity-group-fs-store.summary-store":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.reduce.cpu.vcores":"1","mapreduce.job.encrypted-intermediate-data.buffer.kb":"128","fs.client.resolve.remote.symlinks":"true","yarn.nodemanager.webapp.https.address":"0.0.0.0:8044","hadoop.http.cross-origin.allowed-origins":"*","mapreduce.job.encrypted-intermediate-data":"false","yarn.timeline-service.entity-group-fs-store.retain-seconds":"604800","yarn.resourcemanager.metrics.runtime.buckets":"60,300,1440","yarn.timeline-service.generic-application-history.max-applications":"10000","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","yarn.node-labels.configuration-type":"centralized","fs.s3a.path.style.access":"false","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","yarn.sharedcache.store.in-memory.staleness-period-mins":"10080","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.nodemanager.minimum.version":"NONE","mapreduce.jobhistory.webapp.xfs-filter.xframe-options":"SAMEORIGIN","yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled":"false","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","yarn.timeline-service.reader.webapp.https.address":"${yarn.timeline-service.webapp.https.address}","fs.ftp.data.connection.mode":"ACTIVE_LOCAL_DATA_CONNECTION_MODE","mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed":"true","yarn.scheduler.maximum-allocation-vcores":"4","hadoop.http.cross-origin.allowed-headers":"X-Requested-With,Content-Type,Accept,Origin","yarn.nodemanager.log-aggregation.compression-type":"none","yarn.timeline-service.version":"1.0f","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","mapreduce.reduce.maxattempts":"4","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","mapreduce.job.running.reduce.limit":"0","ipc.maximum.response.length":"134217728","yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","hadoop.caller.context.max.size":"128","yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed":"false","yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed":"false","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","fs.s3a.max.total.tasks":"5","fs.s3a.readahead.range":"64K","hadoop.http.authentication.simple.anonymous.allowed":"true","fs.s3a.attempts.maximum":"20","hadoop.registry.zk.connection.timeout.ms":"15000","yarn.resourcemanager.delegation-token-renewer.thread-count":"*********(redacted)","yarn.nodemanager.health-checker.script.timeout-ms":"1200000","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","yarn.resourcemanager.resource-profiles.enabled":"false","yarn.timeline-service.hbase-schema.prefix":"prod.","fs.azure.authorization":"false","mapreduce.map.log.level":"INFO","yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs":"20","mapreduce.output.fileoutputformat.compress.type":"RECORD","yarn.resourcemanager.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.timeline-service.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.ifile.readahead.bytes":"4194304","yarn.sharedcache.app-checker.class":"org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"true","yarn.nodemanager.resource.detect-hardware-capabilities":"false","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","hadoop.security.group.mapping.ldap.search.group.hierarchy.levels":"0","yarn.resourcemanager.fs.state-store.retry-interval-ms":"1000","file.stream-buffer-size":"4096","yarn.resourcemanager.application-timeouts.monitor.interval-ms":"3000","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.map.speculative":"true","mapreduce.job.speculative.retry-after-speculate":"15000","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","yarn.app.mapreduce.am.log.level":"INFO","mapreduce.job.reduce.slowstart.completedmaps":"0.05","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices":"0,1","yarn.timeline-service.client.internal-timers-ttl-secs":"420","hadoop.http.logs.enabled":"true","fs.s3a.block.size":"32M","yarn.sharedcache.client-server.address":"0.0.0.0:8045","yarn.nodemanager.logaggregation.threadpool-size-max":"100","yarn.resourcemanager.hostname":"0.0.0.0","yarn.resourcemanager.delegation.key.update-interval":"86400000","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"-1","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","yarn.nodemanager.disk-health-checker.enable":"true","ipc.client.tcpnodelay":"true","ipc.client.rpc-timeout.ms":"0","yarn.nodemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","ipc.client.low-latency":"false","mapreduce.input.lineinputformat.linespermap":"1","yarn.router.interceptor.user.threadpool-size":"5","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.timeline-service.entity-group-fs-store.app-cache-size":"10","fs.s3a.socket.recv.buffer":"8192","yarn.resourcemanager.resource-tracker.address":"${yarn.resourcemanager.hostname}:8031","yarn.nodemanager.node-labels.provider.fetch-timeout-ms":"1200000","mapreduce.job.heap.memory-mb.ratio":"0.8","yarn.resourcemanager.leveldb-state-store.compaction-interval-secs":"3600","yarn.resourcemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","yarn.scheduler.configuration.fs.path":"file://${hadoop.tmp.dir}/yarn/system/schedconf","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.http.lib.StaticUserWebFilter","mapreduce.reduce.memory.mb":"-1","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","yarn.nodemanager.container-metrics.period-ms":"-1","mapreduce.fileoutputcommitter.task.cleanup.enabled":"false","yarn.nodemanager.log.retain-seconds":"10800","yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds":"3600","yarn.resourcemanager.keytab":"/etc/krb5.keytab","hadoop.security.group.mapping.providers.combined":"true","mapreduce.reduce.merge.inmem.threshold":"1000","yarn.timeline-service.recovery.enabled":"false","fs.azure.saskey.usecontainersaskeyforallaccess":"true","yarn.sharedcache.nm.uploader.thread-count":"20","yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs":"3600","mapreduce.shuffle.ssl.enabled":"false","yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds":"259200000","fs.s3a.committer.staging.abort.pending.uploads":"true","yarn.nodemanager.opportunistic-containers-max-queue-length":"0","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","mapreduce.job.speculative.minimum-allowed-tasks":"10","yarn.log-aggregation.retain-seconds":"-1","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.http.cross-origin.allowed-methods":"GET,POST,HEAD","yarn.resourcemanager.opportunistic-container-allocation.enabled":"false","mapreduce.jobhistory.webapp.address":"0.0.0.0:19888","hadoop.system.tags":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.log-aggregation.file-controller.TFile.class":"org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"${yarn.resourcemanager.hostname}:8088","mapreduce.jobhistory.recovery.enable":"false","mapreduce.reduce.shuffle.parallelcopies":"5","fs.AbstractFileSystem.webhdfs.impl":"org.apache.hadoop.fs.WebHdfs","fs.trash.interval":"0","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","yarn.app.mapreduce.am.resource.mb":"1536","mapreduce.input.fileinputformat.list-status.num-threads":"1","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.resourcemanager.nm-container-queuing.min-queue-length":"5","yarn.nodemanager.resource.cpu-vcores":"-1","mapreduce.job.reduces":"1","fs.s3a.multipart.size":"100M","yarn.scheduler.minimum-allocation-vcores":"1","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","mapreduce.job.encrypted-intermediate-data-key-size-bits":"128","fs.s3a.metadatastore.authoritative":"false","yarn.nodemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","ha.health-monitor.sleep-after-disconnect.ms":"1000","yarn.app.mapreduce.shuffle.log.limit.kb":"0","hadoop.security.group.mapping":"org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback","yarn.client.application-client-protocol.poll-timeout-ms":"-1","mapreduce.jobhistory.jhist.format":"binary","yarn.resourcemanager.ha.enabled":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.jobhistory.intermediate-user-done-dir.permissions":"770","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","yarn.resourcemanager.reservation-system.enable":"false","mapreduce.map.output.compress":"false","ha.zookeeper.acl":"world:anyone:rwcda","ipc.server.max.connections":"0","yarn.nodemanager.runtime.linux.docker.default-container-network":"host","yarn.router.webapp.address":"0.0.0.0:8089","yarn.scheduler.maximum-allocation-mb":"8192","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.sharedcache.cleaner.period-mins":"1440","yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint":"http://localhost:3476/v1.0/docker/cli","yarn.app.mapreduce.am.container.log.limit.kb":"0","ipc.client.connect.retry.interval":"1000","yarn.timeline-service.http-cross-origin.enabled":"false","fs.wasbs.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure","yarn.federation.subcluster-resolver.class":"org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","mapreduce.jobhistory.cleaner.enable":"true","yarn.timeline-service.client.fd-flush-interval-secs":"10","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms":"1000","fs.s3a.committer.staging.tmp.path":"tmp/staging","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms":"10","mapreduce.job.end-notification.retry.attempts":"0","yarn.nodemanager.resource.count-logical-processors-as-cores":"false","hadoop.registry.zk.root":"/registry","adl.feature.ownerandgroup.enableupn":"false","yarn.resourcemanager.zk-max-znode-size.bytes":"1048576","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","yarn.nodemanager.localizer.cache.target-size-mb":"10240","fs.s3a.committer.staging.conflict-mode":"fail","mapreduce.client.libjars.wildcard":"true","fs.s3a.committer.staging.unique-filenames":"true","yarn.nodemanager.node-attributes.provider.fetch-timeout-ms":"1200000","fs.s3a.list.version":"2","ftp.client-write-packet-size":"65536","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","hadoop.security.key.default.cipher":"AES/CTR/NoPadding","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"86400","mapreduce.job.local-fs.single-disk-limit.check.interval-ms":"5000","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","yarn.nodemanager.amrmproxy.address":"0.0.0.0:8049","ipc.server.listen.queue.size":"128","map.sort.class":"org.apache.hadoop.util.QuickSort","fs.viewfs.rename.strategy":"SAME_MOUNTPOINT","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","fs.s3a.assumed.role.credentials.provider":"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider","yarn.nodemanager.vmem-check-enabled":"true","yarn.nodemanager.numa-awareness.enabled":"false","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","yarn.federation.registry.base-dir":"yarnfederation/","mapreduce.job.max.map":"-1","mapreduce.job.local-fs.single-disk-limit.bytes":"-1","mapreduce.job.ubertask.maxreduces":"1","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.jobhistory.jobname.limit":"50","yarn.client.nodemanager-connect.retry-interval-ms":"10000","yarn.timeline-service.state-store-class":"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ","yarn.sharedcache.nested-level":"3","yarn.timeline-service.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","fs.azure.user.agent.prefix":"unknown","yarn.resourcemanager.zk-delegation-token-node.split-index":"*********(redacted)","yarn.nodemanager.numa-awareness.read-topology":"false","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","yarn.registry.class":"org.apache.hadoop.registry.client.impl.FSRegistryOperationsService","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","yarn.scheduler.queue-placement-rules":"user-group","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","yarn.resourcemanager.recovery.enabled":"false","yarn.timeline-service.webapp.rest-csrf.enabled":"false"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.version":"1.8","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Private Build","java.vm.specification.version":"1.8","user.home":"/home/nartal","file.encoding.pkg":"sun.io","sun.nio.ch.bugLevel":"","sun.arch.data.model":"64","sun.boot.library.path":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64","user.dir":"/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","java.library.path":"/snap/bin/cmake:/usr/local/cuda-11.2/lib64::/usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib","sun.cpu.isalist":"","sun.desktop":"gnome","os.arch":"amd64","java.vm.version":"25.292-b10","jetty.git.hash":"238ec6997c7806b055319a6d11f8ae7564adc0de","java.endorsed.dirs":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/endorsed","java.runtime.version":"1.8.0_292-8u292-b10-0ubuntu1~18.04-b10","java.vm.info":"mixed mode","java.ext.dirs":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/ext:/usr/java/packages/lib/ext","java.runtime.name":"OpenJDK Runtime Environment","file.separator":"/","java.class.version":"52.0","scala.usejavacp":"true","java.specification.name":"Java Platform API Specification","sun.boot.class.path":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/resources.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/rt.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/sunrsasign.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jsse.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jce.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/charsets.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jfr.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/classes","file.encoding":"UTF-8","user.timezone":"America/Los_Angeles","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","os.version":"5.4.0-80-generic","sun.os.patch.level":"unknown","java.vm.specification.vendor":"Oracle Corporation","user.country":"US","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.url":"http://java.oracle.com/","java.awt.printerjob":"sun.print.PSPrinterJob","java.awt.graphicsenv":"sun.awt.X11GraphicsEnvironment","awt.toolkit":"sun.awt.X11.XToolkit","os.name":"Linux","java.vm.vendor":"Private Build","java.vendor.url.bug":"http://bugreport.sun.com/bugreport/","user.name":"nartal","java.vm.name":"OpenJDK 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --conf spark.eventLog.enabled=true --conf spark.eventLog.dir=/home/nartal/data_format_eventlog --class org.apache.spark.repl.Main --name Spark shell --jars /home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar spark-shell","java.home":"/usr/lib/jvm/java-8-openjdk-amd64/jre","java.version":"1.8.0_292","sun.io.unicode.encoding":"UnicodeLittle"},"Classpath Entries":{"/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/shapeless_2.12-2.3.3.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/metrics-jvm-4.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-sql_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spire-macros_2.12-0.17.0-M1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/antlr4-runtime-4.8-1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/arpack_combined_all-0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/htrace-core4-4.1.0-incubating.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/machinist_2.12-0.6.8.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jline-2.14.6.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-client-2.30.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jaxb-api-2.2.11.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-yarn-registry-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-common-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/stax2-api-3.1.4.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/osgi-resource-locator-1.0.3.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/aopalliance-1.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/janino-3.0.16.jar":"System Classpath","spark://nartal.attlocal.net:44977/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar":"Added By User","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-dataformat-yaml-2.10.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-module-scala_2.12-2.10.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/scala-collection-compat_2.12-2.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/scala-xml_2.12-1.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/leveldbjni-all-1.8.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spire-util_2.12-0.17.0-M1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/curator-recipes-2.13.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-repl_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/json4s-core_2.12-3.7.0-M5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-core-2.10.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jcl-over-slf4j-1.7.30.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/httpcore-4.4.12.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-metrics-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/generex-1.0.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jpam-1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/JLargeArrays-1.5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-core_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-yarn-server-web-proxy-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/mesos-1.4.0-shaded-protobuf.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jul-to-slf4j-1.7.30.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerby-pkix-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-apiextensions-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/json4s-ast_2.12-3.7.0-M5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/protobuf-java-2.5.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-io-2.5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jta-1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-catalyst_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/activation-1.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-common-2.3.7.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/ivy-2.4.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-container-servlet-2.30.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/py4j-0.10.9.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-codec-1.10.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-admin-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-simplekdc-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/threeten-extra-1.5.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/cats-kernel_2.12-2.0.0-M4.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hk2-locator-2.6.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-hive_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spire-platform_2.12-0.17.0-M1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-network-common_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/minlog-1.3.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/token-provider-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/aopalliance-repackaged-2.6.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/conf/":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/stax-api-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerby-config-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/javassist-3.25.0-GA.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.inject-2.6.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/opencsv-2.3.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-mapreduce-client-core-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/zjsonpatch-0.3.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/univocity-parsers-2.9.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-yarn-api-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-yarn_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-text-1.6.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jsp-api-2.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-kvstore_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-media-jaxb-2.30.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/bonecp-0.8.0.RELEASE.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-networking-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/transaction-api-1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-annotations-2.10.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/datanucleus-api-jdo-4.2.4.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-autoscaling-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/velocity-1.5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-batch-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/okio-1.14.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-streaming_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/arrow-memory-netty-2.0.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-yarn-server-common-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.servlet-api-4.0.3.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-llap-common-2.3.7.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-identity-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/okhttp-3.12.12.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/xz-1.5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/compress-lzf-1.0.3.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/arrow-format-2.0.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-jdbc-2.3.7.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-launcher_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/parquet-column-1.10.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-mapreduce-client-common-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/avro-ipc-1.8.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-mllib_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/shims-0.9.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/parquet-format-2.4.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-crypto-1.1.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/re2j-1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/automaton-1.11-8.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-policy-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/flatbuffers-java-1.9.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/parquet-jackson-1.10.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-tags_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-core-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-annotations-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-auth-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/metrics-json-4.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/parquet-encoding-1.10.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/oro-2.0.8.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/avro-1.8.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jodd-core-3.5.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-admissionregistration-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/metrics-core-4.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/parquet-common-1.10.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/istack-commons-runtime-3.0.8.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/gson-2.2.4.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-module-jaxb-annotations-2.10.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-beanutils-1.9.4.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-mesos_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/snappy-java-1.1.8.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jcip-annotations-1.0-1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/log4j-1.2.17.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerby-asn1-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/accessors-smart-1.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-module-paranamer-2.10.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/xbean-asm7-shaded-4.15.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/scala-compiler-2.12.10.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-daemon-1.0.13.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-dbcp-1.4.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/orc-core-1.5.12.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/aircompressor-0.10.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/okhttp-2.7.5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-jaxrs-json-provider-2.9.5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/metrics-graphite-4.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/stream-2.9.6.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/algebra_2.12-2.0.0-M2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-mllib-local_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-service-rpc-3.1.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-rbac-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/metrics-jmx-4.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/nimbus-jose-jwt-4.41.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-kubernetes_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/slf4j-api-1.7.30.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-common-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-compiler-3.0.16.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-metastore-2.3.7.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/core-1.1.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jaxb-runtime-2.3.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/paranamer-2.8.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/guice-4.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-pool-1.5.4.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/super-csv-2.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-exec-2.3.7-core.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-logging-1.1.3.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-settings-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-network-shuffle_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-tags_2.12-3.1.1-tests.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-configuration2-2.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-common-2.30.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-certificates-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-datatype-jsr310-2.11.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/antlr-runtime-3.5.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/curator-framework-2.13.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/parquet-hadoop-1.10.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/json-1.8.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-core-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/scala-library-2.12.10.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-cli-1.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerby-util-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/JTransforms-3.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/macro-compat_2.12-1.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-databind-2.10.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-hive-thriftserver_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hk2-api-2.6.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-shims-common-2.3.7.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/json4s-scalap_2.12-3.7.0-M5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-server-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/ST4-4.0.4.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.xml.bind-api-2.3.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-graphx_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/slf4j-log4j12-1.7.30.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jdo-api-3.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.ws.rs-api-2.1.6.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-client-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-coordination-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/orc-mapreduce-1.5.12.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-collections-3.2.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-shims-2.3.7.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-httpclient-3.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/breeze-macros_2.12-1.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-storage-api-2.7.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-crypto-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/objenesis-2.6.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/orc-shims-1.5.12.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/libthrift-0.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/snakeyaml-1.24.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-mapreduce-client-jobclient-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/zstd-jni-1.4.8-1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-serde-2.3.7.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/arrow-memory-core-2.0.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/guice-servlet-4.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-jaxrs-base-2.9.5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/javax.inject-1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/datanucleus-rdbms-4.1.19.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/derby-10.12.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kryo-shaded-4.0.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hk2-utils-2.6.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-beeline-2.3.7.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-yarn-client-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/audience-annotations-0.5.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/scala-parser-combinators_2.12-1.1.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-vector-code-gen-2.3.7.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.activation-api-1.2.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-server-2.30.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-events-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-sketch_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/logging-interceptor-3.12.12.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-client-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-hk2-2.30.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-hdfs-client-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.annotation-api-1.3.5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/netty-all-4.1.51.Final.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/pyrolite-4.30.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/httpclient-4.5.6.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/javolution-5.5.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/joda-time-2.10.5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-net-3.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/avro-mapred-1.8.2-hadoop2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-apps-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/dnsjava-2.1.7.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-storageclass-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jsr305-3.0.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/RoaringBitmap-0.9.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/zookeeper-3.4.14.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/chill_2.12-0.9.5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-math3-3.4.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/ehcache-3.3.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/chill-java-0.9.5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/guava-14.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/woodstox-core-5.0.3.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-cli-2.3.7.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-yarn-common-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/json4s-jackson_2.12-3.7.0-M5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-compress-1.20.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/javax.jdo-3.2.0-m3.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/lz4-java-1.7.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-client-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-util-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-discovery-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-scheduling-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/datanucleus-core-4.1.17.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/json-smart-2.3.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-container-servlet-core-2.30.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerby-xdr-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spire_2.12-0.17.0-M1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.validation-api-2.0.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/geronimo-jcache_1.0_spec-1.0-alpha-1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/arrow-vector-2.0.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-shims-0.23-2.3.7.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-extensions-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-lang-2.6.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/libfb303-0.9.3.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/HikariCP-2.5.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-unsafe_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/breeze_2.12-1.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-lang3-3.10.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-common-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/scala-reflect-2.12.10.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/curator-client-2.13.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-shims-scheduler-2.3.7.jar":"System Classpath"}} +{"Event":"SparkListenerApplicationStart","App Name":"Spark shell","App ID":"local-1630551716813","Timestamp":1630551715882,"User":"nartal"} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":0,"description":"save at :32","details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:36)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:38)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:40)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:42)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:44)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:46)\n$line20.$read$$iw$$iw$$iw$$iw$$iw.(:48)\n$line20.$read$$iw$$iw$$iw$$iw.(:50)\n$line20.$read$$iw$$iw$$iw.(:52)\n$line20.$read$$iw$$iw.(:54)\n$line20.$read$$iw.(:56)\n$line20.$read.(:58)\n$line20.$read$.(:62)\n$line20.$read$.()\n$line20.$eval$.$print$lzycompute(:7)\n$line20.$eval$.$print(:6)\n$line20.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)","physicalPlanDescription":"== Physical Plan ==\nExecute InsertIntoHadoopFsRelationCommand (2)\n+- * Scan ExistingRDD (1)\n\n\n(1) Scan ExistingRDD [codegen id : 1]\nOutput [2]: [name#2, booksIntersted#3]\nArguments: [name#2, booksIntersted#3], MapPartitionsRDD[1] at createDataFrame at :33, ExistingRDD, UnknownPartitioning(0)\n\n(2) Execute InsertIntoHadoopFsRelationCommand\nInput [2]: [name#2, booksIntersted#3]\nArguments: file:/home/nartal/event_logs_spark/decimalnested.parquet, false, Parquet, Map(path -> /home/nartal/event_logs_spark/decimalnested.parquet), ErrorIfExists, [name, booksIntersted]\n\n","sparkPlanInfo":{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand file:/home/nartal/event_logs_spark/decimalnested.parquet, false, Parquet, Map(path -> /home/nartal/event_logs_spark/decimalnested.parquet), ErrorIfExists, [name, booksIntersted]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan ExistingRDD","simpleString":"Scan ExistingRDD[name#2,booksIntersted#3]","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":5,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of written files","accumulatorId":0,"metricType":"sum"},{"name":"written output","accumulatorId":1,"metricType":"size"},{"name":"number of output rows","accumulatorId":2,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":3,"metricType":"sum"}]},"time":1630551740782} +{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1630551741427,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"save at :32","Number of Tasks":8,"RDD Info":[{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at :32","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"map\"}","Callsite":"createDataFrame at :33","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"parallelize at :33","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:36)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:38)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:40)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:42)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:44)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:46)\n$line20.$read$$iw$$iw$$iw$$iw$$iw.(:48)\n$line20.$read$$iw$$iw$$iw$$iw.(:50)\n$line20.$read$$iw$$iw$$iw.(:52)\n$line20.$read$$iw$$iw.(:54)\n$line20.$read$$iw.(:56)\n$line20.$read.(:58)\n$line20.$read$.(:62)\n$line20.$read$.()\n$line20.$eval$.$print$lzycompute(:7)\n$line20.$eval$.$print(:6)\n$line20.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[0],"Properties":{"spark.sql.warehouse.dir":"file:/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/spark-warehouse/","spark.driver.host":"nartal.attlocal.net","spark.eventLog.enabled":"true","spark.driver.port":"44977","spark.repl.class.uri":"spark://nartal.attlocal.net:44977/classes","spark.jars":"file:///home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.repl.class.outputDir":"/tmp/spark-6437c8fb-c50a-434e-a838-d827759715c1/repl-56a3ce44-fe38-4b08-be93-b9831c0ca528","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"2\",\"name\":\"Execute InsertIntoHadoopFsRelationCommand\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.startTime":"1630551715882","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://nartal.attlocal.net:44977/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.submit.deployMode":"client","spark.master":"local[*]","spark.home":"/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","spark.eventLog.dir":"/home/nartal/data_format_eventlog","spark.sql.execution.id":"0","spark.sql.catalogImplementation":"hive","spark.repl.local.jars":"file:///home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.app.id":"local-1630551716813"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"save at :32","Number of Tasks":8,"RDD Info":[{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at :32","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"map\"}","Callsite":"createDataFrame at :33","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"parallelize at :33","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:36)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:38)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:40)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:42)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:44)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:46)\n$line20.$read$$iw$$iw$$iw$$iw$$iw.(:48)\n$line20.$read$$iw$$iw$$iw$$iw.(:50)\n$line20.$read$$iw$$iw$$iw.(:52)\n$line20.$read$$iw$$iw.(:54)\n$line20.$read$$iw.(:56)\n$line20.$read.(:58)\n$line20.$read$.(:62)\n$line20.$read$.()\n$line20.$eval$.$print$lzycompute(:7)\n$line20.$eval$.$print(:6)\n$line20.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)","Submission Time":1630551741445,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/spark-warehouse/","spark.driver.host":"nartal.attlocal.net","spark.eventLog.enabled":"true","spark.driver.port":"44977","spark.repl.class.uri":"spark://nartal.attlocal.net:44977/classes","spark.jars":"file:///home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.repl.class.outputDir":"/tmp/spark-6437c8fb-c50a-434e-a838-d827759715c1/repl-56a3ce44-fe38-4b08-be93-b9831c0ca528","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"2\",\"name\":\"Execute InsertIntoHadoopFsRelationCommand\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.startTime":"1630551715882","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://nartal.attlocal.net:44977/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.submit.deployMode":"client","spark.master":"local[*]","spark.home":"/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","spark.eventLog.dir":"/home/nartal/data_format_eventlog","spark.sql.execution.id":"0","spark.sql.catalogImplementation":"hive","spark.repl.local.jars":"file:///home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.app.id":"local-1630551716813"}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1630551741705,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Launch Time":1630551741722,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":2,"Index":2,"Attempt":0,"Launch Time":1630551741733,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":3,"Index":3,"Attempt":0,"Launch Time":1630551741734,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":4,"Index":4,"Attempt":0,"Launch Time":1630551741735,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":5,"Index":5,"Attempt":0,"Launch Time":1630551741736,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":6,"Index":6,"Attempt":0,"Launch Time":1630551741737,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":7,"Index":7,"Attempt":0,"Launch Time":1630551741737,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":4,"Index":4,"Attempt":0,"Launch Time":1630551741735,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551742328,"Failed":false,"Killed":false,"Accumulables":[{"ID":4,"Name":"duration","Update":"27","Value":"27","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":6,"Name":"internal.metrics.executorDeserializeTime","Update":458,"Value":458,"Internal":true,"Count Failed Values":true},{"ID":7,"Name":"internal.metrics.executorDeserializeCpuTime","Update":183484552,"Value":183484552,"Internal":true,"Count Failed Values":true},{"ID":8,"Name":"internal.metrics.executorRunTime","Update":78,"Value":78,"Internal":true,"Count Failed Values":true},{"ID":9,"Name":"internal.metrics.executorCpuTime","Update":18655590,"Value":18655590,"Internal":true,"Count Failed Values":true},{"ID":10,"Name":"internal.metrics.resultSize","Update":2353,"Value":2353,"Internal":true,"Count Failed Values":true},{"ID":11,"Name":"internal.metrics.jvmGCTime","Update":12,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":12,"Name":"internal.metrics.resultSerializationTime","Update":7,"Value":7,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":458,"Executor Deserialize CPU Time":183484552,"Executor Run Time":78,"Executor CPU Time":18655590,"Peak Execution Memory":0,"Result Size":2353,"JVM GC Time":12,"Result Serialization Time":7,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":2,"Index":2,"Attempt":0,"Launch Time":1630551741733,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551742331,"Failed":false,"Killed":false,"Accumulables":[{"ID":4,"Name":"duration","Update":"32","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":6,"Name":"internal.metrics.executorDeserializeTime","Update":457,"Value":915,"Internal":true,"Count Failed Values":true},{"ID":7,"Name":"internal.metrics.executorDeserializeCpuTime","Update":191270617,"Value":374755169,"Internal":true,"Count Failed Values":true},{"ID":8,"Name":"internal.metrics.executorRunTime","Update":79,"Value":157,"Internal":true,"Count Failed Values":true},{"ID":9,"Name":"internal.metrics.executorCpuTime","Update":26214586,"Value":44870176,"Internal":true,"Count Failed Values":true},{"ID":10,"Name":"internal.metrics.resultSize","Update":2353,"Value":4706,"Internal":true,"Count Failed Values":true},{"ID":11,"Name":"internal.metrics.jvmGCTime","Update":12,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":12,"Name":"internal.metrics.resultSerializationTime","Update":8,"Value":15,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":457,"Executor Deserialize CPU Time":191270617,"Executor Run Time":79,"Executor CPU Time":26214586,"Peak Execution Memory":0,"Result Size":2353,"JVM GC Time":12,"Result Serialization Time":8,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":6,"Index":6,"Attempt":0,"Launch Time":1630551741737,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551742332,"Failed":false,"Killed":false,"Accumulables":[{"ID":4,"Name":"duration","Update":"22","Value":"81","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":6,"Name":"internal.metrics.executorDeserializeTime","Update":460,"Value":1375,"Internal":true,"Count Failed Values":true},{"ID":7,"Name":"internal.metrics.executorDeserializeCpuTime","Update":195750093,"Value":570505262,"Internal":true,"Count Failed Values":true},{"ID":8,"Name":"internal.metrics.executorRunTime","Update":76,"Value":233,"Internal":true,"Count Failed Values":true},{"ID":9,"Name":"internal.metrics.executorCpuTime","Update":22580502,"Value":67450678,"Internal":true,"Count Failed Values":true},{"ID":10,"Name":"internal.metrics.resultSize","Update":2353,"Value":7059,"Internal":true,"Count Failed Values":true},{"ID":11,"Name":"internal.metrics.jvmGCTime","Update":12,"Value":36,"Internal":true,"Count Failed Values":true},{"ID":12,"Name":"internal.metrics.resultSerializationTime","Update":8,"Value":23,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":460,"Executor Deserialize CPU Time":195750093,"Executor Run Time":76,"Executor CPU Time":22580502,"Peak Execution Memory":0,"Result Size":2353,"JVM GC Time":12,"Result Serialization Time":8,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":5,"Index":5,"Attempt":0,"Launch Time":1630551741736,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551742889,"Failed":false,"Killed":false,"Accumulables":[{"ID":4,"Name":"duration","Update":"512","Value":"593","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":5,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":6,"Name":"internal.metrics.executorDeserializeTime","Update":461,"Value":1836,"Internal":true,"Count Failed Values":true},{"ID":7,"Name":"internal.metrics.executorDeserializeCpuTime","Update":207129160,"Value":777634422,"Internal":true,"Count Failed Values":true},{"ID":8,"Name":"internal.metrics.executorRunTime","Update":672,"Value":905,"Internal":true,"Count Failed Values":true},{"ID":9,"Name":"internal.metrics.executorCpuTime","Update":87810246,"Value":155260924,"Internal":true,"Count Failed Values":true},{"ID":10,"Name":"internal.metrics.resultSize","Update":2396,"Value":9455,"Internal":true,"Count Failed Values":true},{"ID":11,"Name":"internal.metrics.jvmGCTime","Update":12,"Value":48,"Internal":true,"Count Failed Values":true},{"ID":29,"Name":"internal.metrics.output.bytesWritten","Update":1576,"Value":1576,"Internal":true,"Count Failed Values":true},{"ID":30,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":461,"Executor Deserialize CPU Time":207129160,"Executor Run Time":672,"Executor CPU Time":87810246,"Peak Execution Memory":0,"Result Size":2396,"JVM GC Time":12,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":1576,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":7,"Index":7,"Attempt":0,"Launch Time":1630551741737,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551742889,"Failed":false,"Killed":false,"Accumulables":[{"ID":4,"Name":"duration","Update":"520","Value":"1113","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":5,"Name":"number of output rows","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":6,"Name":"internal.metrics.executorDeserializeTime","Update":465,"Value":2301,"Internal":true,"Count Failed Values":true},{"ID":7,"Name":"internal.metrics.executorDeserializeCpuTime","Update":167181205,"Value":944815627,"Internal":true,"Count Failed Values":true},{"ID":8,"Name":"internal.metrics.executorRunTime","Update":669,"Value":1574,"Internal":true,"Count Failed Values":true},{"ID":9,"Name":"internal.metrics.executorCpuTime","Update":159370214,"Value":314631138,"Internal":true,"Count Failed Values":true},{"ID":10,"Name":"internal.metrics.resultSize","Update":2396,"Value":11851,"Internal":true,"Count Failed Values":true},{"ID":11,"Name":"internal.metrics.jvmGCTime","Update":12,"Value":60,"Internal":true,"Count Failed Values":true},{"ID":29,"Name":"internal.metrics.output.bytesWritten","Update":1452,"Value":3028,"Internal":true,"Count Failed Values":true},{"ID":30,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":2,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":465,"Executor Deserialize CPU Time":167181205,"Executor Run Time":669,"Executor CPU Time":159370214,"Peak Execution Memory":0,"Result Size":2396,"JVM GC Time":12,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":1452,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Launch Time":1630551741722,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551742889,"Failed":false,"Killed":false,"Accumulables":[{"ID":4,"Name":"duration","Update":"511","Value":"1624","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":5,"Name":"number of output rows","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":6,"Name":"internal.metrics.executorDeserializeTime","Update":462,"Value":2763,"Internal":true,"Count Failed Values":true},{"ID":7,"Name":"internal.metrics.executorDeserializeCpuTime","Update":185378942,"Value":1130194569,"Internal":true,"Count Failed Values":true},{"ID":8,"Name":"internal.metrics.executorRunTime","Update":672,"Value":2246,"Internal":true,"Count Failed Values":true},{"ID":9,"Name":"internal.metrics.executorCpuTime","Update":291096200,"Value":605727338,"Internal":true,"Count Failed Values":true},{"ID":10,"Name":"internal.metrics.resultSize","Update":2396,"Value":14247,"Internal":true,"Count Failed Values":true},{"ID":11,"Name":"internal.metrics.jvmGCTime","Update":12,"Value":72,"Internal":true,"Count Failed Values":true},{"ID":29,"Name":"internal.metrics.output.bytesWritten","Update":1567,"Value":4595,"Internal":true,"Count Failed Values":true},{"ID":30,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":3,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":462,"Executor Deserialize CPU Time":185378942,"Executor Run Time":672,"Executor CPU Time":291096200,"Peak Execution Memory":0,"Result Size":2396,"JVM GC Time":12,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":1567,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1630551741705,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551742889,"Failed":false,"Killed":false,"Accumulables":[{"ID":4,"Name":"duration","Update":"504","Value":"2128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":6,"Name":"internal.metrics.executorDeserializeTime","Update":459,"Value":3222,"Internal":true,"Count Failed Values":true},{"ID":7,"Name":"internal.metrics.executorDeserializeCpuTime","Update":175516068,"Value":1305710637,"Internal":true,"Count Failed Values":true},{"ID":8,"Name":"internal.metrics.executorRunTime","Update":675,"Value":2921,"Internal":true,"Count Failed Values":true},{"ID":9,"Name":"internal.metrics.executorCpuTime","Update":539168363,"Value":1144895701,"Internal":true,"Count Failed Values":true},{"ID":10,"Name":"internal.metrics.resultSize","Update":2353,"Value":16600,"Internal":true,"Count Failed Values":true},{"ID":11,"Name":"internal.metrics.jvmGCTime","Update":12,"Value":84,"Internal":true,"Count Failed Values":true},{"ID":29,"Name":"internal.metrics.output.bytesWritten","Update":825,"Value":5420,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":459,"Executor Deserialize CPU Time":175516068,"Executor Run Time":675,"Executor CPU Time":539168363,"Peak Execution Memory":0,"Result Size":2353,"JVM GC Time":12,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":825,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":3,"Index":3,"Attempt":0,"Launch Time":1630551741734,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551742890,"Failed":false,"Killed":false,"Accumulables":[{"ID":4,"Name":"duration","Update":"512","Value":"2640","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":5,"Name":"number of output rows","Update":"1","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":6,"Name":"internal.metrics.executorDeserializeTime","Update":461,"Value":3683,"Internal":true,"Count Failed Values":true},{"ID":7,"Name":"internal.metrics.executorDeserializeCpuTime","Update":195883974,"Value":1501594611,"Internal":true,"Count Failed Values":true},{"ID":8,"Name":"internal.metrics.executorRunTime","Update":672,"Value":3593,"Internal":true,"Count Failed Values":true},{"ID":9,"Name":"internal.metrics.executorCpuTime","Update":109431942,"Value":1254327643,"Internal":true,"Count Failed Values":true},{"ID":10,"Name":"internal.metrics.resultSize","Update":2396,"Value":18996,"Internal":true,"Count Failed Values":true},{"ID":11,"Name":"internal.metrics.jvmGCTime","Update":12,"Value":96,"Internal":true,"Count Failed Values":true},{"ID":29,"Name":"internal.metrics.output.bytesWritten","Update":1585,"Value":7005,"Internal":true,"Count Failed Values":true},{"ID":30,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":4,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":461,"Executor Deserialize CPU Time":195883974,"Executor Run Time":672,"Executor CPU Time":109431942,"Peak Execution Memory":0,"Result Size":2396,"JVM GC Time":12,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":1585,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"save at :32","Number of Tasks":8,"RDD Info":[{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at :32","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"map\"}","Callsite":"createDataFrame at :33","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"parallelize at :33","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:36)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:38)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:40)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:42)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:44)\n$line20.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:46)\n$line20.$read$$iw$$iw$$iw$$iw$$iw.(:48)\n$line20.$read$$iw$$iw$$iw$$iw.(:50)\n$line20.$read$$iw$$iw$$iw.(:52)\n$line20.$read$$iw$$iw.(:54)\n$line20.$read$$iw.(:56)\n$line20.$read.(:58)\n$line20.$read$.(:62)\n$line20.$read$.()\n$line20.$eval$.$print$lzycompute(:7)\n$line20.$eval$.$print(:6)\n$line20.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)","Submission Time":1630551741445,"Completion Time":1630551742894,"Accumulables":[{"ID":4,"Name":"duration","Value":"2640","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":5,"Name":"number of output rows","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":6,"Name":"internal.metrics.executorDeserializeTime","Value":3683,"Internal":true,"Count Failed Values":true},{"ID":7,"Name":"internal.metrics.executorDeserializeCpuTime","Value":1501594611,"Internal":true,"Count Failed Values":true},{"ID":8,"Name":"internal.metrics.executorRunTime","Value":3593,"Internal":true,"Count Failed Values":true},{"ID":9,"Name":"internal.metrics.executorCpuTime","Value":1254327643,"Internal":true,"Count Failed Values":true},{"ID":10,"Name":"internal.metrics.resultSize","Value":18996,"Internal":true,"Count Failed Values":true},{"ID":11,"Name":"internal.metrics.jvmGCTime","Value":96,"Internal":true,"Count Failed Values":true},{"ID":12,"Name":"internal.metrics.resultSerializationTime","Value":23,"Internal":true,"Count Failed Values":true},{"ID":29,"Name":"internal.metrics.output.bytesWritten","Value":7005,"Internal":true,"Count Failed Values":true},{"ID":30,"Name":"internal.metrics.output.recordsWritten","Value":4,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1630551742901,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[0,5],[1,7005],[2,4],[3,0]]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":0,"time":1630551742937} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":1,"description":"save at :32","details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:36)\n$line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:38)\n$line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:40)\n$line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:42)\n$line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:44)\n$line25.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:46)\n$line25.$read$$iw$$iw$$iw$$iw$$iw.(:48)\n$line25.$read$$iw$$iw$$iw$$iw.(:50)\n$line25.$read$$iw$$iw$$iw.(:52)\n$line25.$read$$iw$$iw.(:54)\n$line25.$read$$iw.(:56)\n$line25.$read.(:58)\n$line25.$read$.(:62)\n$line25.$read$.()\n$line25.$eval$.$print$lzycompute(:7)\n$line25.$eval$.$print(:6)\n$line25.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)","physicalPlanDescription":"== Physical Plan ==\nExecute InsertIntoHadoopFsRelationCommand (2)\n+- * Scan ExistingRDD (1)\n\n\n(1) Scan ExistingRDD [codegen id : 1]\nOutput [2]: [name#10, subject#11]\nArguments: [name#10, subject#11], MapPartitionsRDD[6] at createDataFrame at :33, ExistingRDD, UnknownPartitioning(0)\n\n(2) Execute InsertIntoHadoopFsRelationCommand\nInput [2]: [name#10, subject#11]\nArguments: file:/home/nartal/event_logs_spark/simpleSchema.parquet, false, Parquet, Map(path -> /home/nartal/event_logs_spark/simpleSchema.parquet), ErrorIfExists, [name, subject]\n\n","sparkPlanInfo":{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand file:/home/nartal/event_logs_spark/simpleSchema.parquet, false, Parquet, Map(path -> /home/nartal/event_logs_spark/simpleSchema.parquet), ErrorIfExists, [name, subject]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan ExistingRDD","simpleString":"Scan ExistingRDD[name#10,subject#11]","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":36,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":35,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of written files","accumulatorId":31,"metricType":"sum"},{"name":"written output","accumulatorId":32,"metricType":"size"},{"name":"number of output rows","accumulatorId":33,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":34,"metricType":"sum"}]},"time":1630551766983} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":1,"time":1630551766985} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":2,"description":"save at :32","details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:36)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:38)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:40)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:42)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:44)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:46)\n$line28.$read$$iw$$iw$$iw$$iw$$iw.(:48)\n$line28.$read$$iw$$iw$$iw$$iw.(:50)\n$line28.$read$$iw$$iw$$iw.(:52)\n$line28.$read$$iw$$iw.(:54)\n$line28.$read$$iw.(:56)\n$line28.$read.(:58)\n$line28.$read$.(:62)\n$line28.$read$.()\n$line28.$eval$.$print$lzycompute(:7)\n$line28.$eval$.$print(:6)\n$line28.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)","physicalPlanDescription":"== Physical Plan ==\nExecute InsertIntoHadoopFsRelationCommand (2)\n+- * Scan ExistingRDD (1)\n\n\n(1) Scan ExistingRDD [codegen id : 1]\nOutput [2]: [name#10, subject#11]\nArguments: [name#10, subject#11], MapPartitionsRDD[6] at createDataFrame at :33, ExistingRDD, UnknownPartitioning(0)\n\n(2) Execute InsertIntoHadoopFsRelationCommand\nInput [2]: [name#10, subject#11]\nArguments: file:/home/nartal/event_logs_spark/simpleSchema.parquet, false, Parquet, Map(path -> /home/nartal/event_logs_spark/simpleSchema.parquet), ErrorIfExists, [name, subject]\n\n","sparkPlanInfo":{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand file:/home/nartal/event_logs_spark/simpleSchema.parquet, false, Parquet, Map(path -> /home/nartal/event_logs_spark/simpleSchema.parquet), ErrorIfExists, [name, subject]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan ExistingRDD","simpleString":"Scan ExistingRDD[name#10,subject#11]","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":42,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":41,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of written files","accumulatorId":37,"metricType":"sum"},{"name":"written output","accumulatorId":38,"metricType":"size"},{"name":"number of output rows","accumulatorId":39,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":40,"metricType":"sum"}]},"time":1630551797786} +{"Event":"SparkListenerJobStart","Job ID":1,"Submission Time":1630551797867,"Stage Infos":[{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"save at :32","Number of Tasks":8,"RDD Info":[{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at :32","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"map\"}","Callsite":"createDataFrame at :33","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"8\",\"name\":\"parallelize\"}","Callsite":"parallelize at :33","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:36)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:38)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:40)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:42)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:44)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:46)\n$line28.$read$$iw$$iw$$iw$$iw$$iw.(:48)\n$line28.$read$$iw$$iw$$iw$$iw.(:50)\n$line28.$read$$iw$$iw$$iw.(:52)\n$line28.$read$$iw$$iw.(:54)\n$line28.$read$$iw.(:56)\n$line28.$read.(:58)\n$line28.$read$.(:62)\n$line28.$read$.()\n$line28.$eval$.$print$lzycompute(:7)\n$line28.$eval$.$print(:6)\n$line28.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[1],"Properties":{"spark.sql.warehouse.dir":"file:/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/spark-warehouse/","spark.driver.host":"nartal.attlocal.net","spark.eventLog.enabled":"true","spark.driver.port":"44977","spark.repl.class.uri":"spark://nartal.attlocal.net:44977/classes","spark.jars":"file:///home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.repl.class.outputDir":"/tmp/spark-6437c8fb-c50a-434e-a838-d827759715c1/repl-56a3ce44-fe38-4b08-be93-b9831c0ca528","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"11\",\"name\":\"Execute InsertIntoHadoopFsRelationCommand\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.startTime":"1630551715882","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://nartal.attlocal.net:44977/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.submit.deployMode":"client","spark.master":"local[*]","spark.home":"/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","spark.eventLog.dir":"/home/nartal/data_format_eventlog","spark.sql.execution.id":"2","spark.sql.catalogImplementation":"hive","spark.repl.local.jars":"file:///home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.app.id":"local-1630551716813"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"save at :32","Number of Tasks":8,"RDD Info":[{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at :32","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"map\"}","Callsite":"createDataFrame at :33","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"8\",\"name\":\"parallelize\"}","Callsite":"parallelize at :33","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:36)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:38)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:40)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:42)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:44)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:46)\n$line28.$read$$iw$$iw$$iw$$iw$$iw.(:48)\n$line28.$read$$iw$$iw$$iw$$iw.(:50)\n$line28.$read$$iw$$iw$$iw.(:52)\n$line28.$read$$iw$$iw.(:54)\n$line28.$read$$iw.(:56)\n$line28.$read.(:58)\n$line28.$read$.(:62)\n$line28.$read$.()\n$line28.$eval$.$print$lzycompute(:7)\n$line28.$eval$.$print(:6)\n$line28.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)","Submission Time":1630551797869,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/spark-warehouse/","spark.driver.host":"nartal.attlocal.net","spark.eventLog.enabled":"true","spark.driver.port":"44977","spark.repl.class.uri":"spark://nartal.attlocal.net:44977/classes","spark.jars":"file:///home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.repl.class.outputDir":"/tmp/spark-6437c8fb-c50a-434e-a838-d827759715c1/repl-56a3ce44-fe38-4b08-be93-b9831c0ca528","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"11\",\"name\":\"Execute InsertIntoHadoopFsRelationCommand\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.startTime":"1630551715882","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://nartal.attlocal.net:44977/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.submit.deployMode":"client","spark.master":"local[*]","spark.home":"/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","spark.eventLog.dir":"/home/nartal/data_format_eventlog","spark.sql.execution.id":"2","spark.sql.catalogImplementation":"hive","spark.repl.local.jars":"file:///home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.app.id":"local-1630551716813"}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":8,"Index":0,"Attempt":0,"Launch Time":1630551797917,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":9,"Index":1,"Attempt":0,"Launch Time":1630551797918,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":10,"Index":2,"Attempt":0,"Launch Time":1630551797918,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":11,"Index":3,"Attempt":0,"Launch Time":1630551797919,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":12,"Index":4,"Attempt":0,"Launch Time":1630551797919,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":13,"Index":5,"Attempt":0,"Launch Time":1630551797920,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":14,"Index":6,"Attempt":0,"Launch Time":1630551797920,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":15,"Index":7,"Attempt":0,"Launch Time":1630551797921,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":12,"Index":4,"Attempt":0,"Launch Time":1630551797919,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551798003,"Failed":false,"Killed":false,"Accumulables":[{"ID":41,"Name":"duration","Update":"58","Value":"58","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"internal.metrics.executorDeserializeTime","Update":40,"Value":40,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.executorDeserializeCpuTime","Update":26175462,"Value":26175462,"Internal":true,"Count Failed Values":true},{"ID":45,"Name":"internal.metrics.executorRunTime","Update":31,"Value":31,"Internal":true,"Count Failed Values":true},{"ID":46,"Name":"internal.metrics.executorCpuTime","Update":3326890,"Value":3326890,"Internal":true,"Count Failed Values":true},{"ID":47,"Name":"internal.metrics.resultSize","Update":2310,"Value":2310,"Internal":true,"Count Failed Values":true},{"ID":48,"Name":"internal.metrics.jvmGCTime","Update":20,"Value":20,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":40,"Executor Deserialize CPU Time":26175462,"Executor Run Time":31,"Executor CPU Time":3326890,"Peak Execution Memory":0,"Result Size":2310,"JVM GC Time":20,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":11,"Index":3,"Attempt":0,"Launch Time":1630551797919,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551798005,"Failed":false,"Killed":false,"Accumulables":[{"ID":41,"Name":"duration","Update":"4","Value":"62","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"internal.metrics.executorDeserializeTime","Update":70,"Value":110,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.executorDeserializeCpuTime","Update":26296759,"Value":52472221,"Internal":true,"Count Failed Values":true},{"ID":45,"Name":"internal.metrics.executorRunTime","Update":6,"Value":37,"Internal":true,"Count Failed Values":true},{"ID":46,"Name":"internal.metrics.executorCpuTime","Update":4124148,"Value":7451038,"Internal":true,"Count Failed Values":true},{"ID":47,"Name":"internal.metrics.resultSize","Update":2310,"Value":4620,"Internal":true,"Count Failed Values":true},{"ID":48,"Name":"internal.metrics.jvmGCTime","Update":20,"Value":40,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":70,"Executor Deserialize CPU Time":26296759,"Executor Run Time":6,"Executor CPU Time":4124148,"Peak Execution Memory":0,"Result Size":2310,"JVM GC Time":20,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":14,"Index":6,"Attempt":0,"Launch Time":1630551797920,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551798013,"Failed":false,"Killed":false,"Accumulables":[{"ID":41,"Name":"duration","Update":"2","Value":"64","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"internal.metrics.executorDeserializeTime","Update":85,"Value":195,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.executorDeserializeCpuTime","Update":28476689,"Value":80948910,"Internal":true,"Count Failed Values":true},{"ID":45,"Name":"internal.metrics.executorRunTime","Update":3,"Value":40,"Internal":true,"Count Failed Values":true},{"ID":46,"Name":"internal.metrics.executorCpuTime","Update":3424913,"Value":10875951,"Internal":true,"Count Failed Values":true},{"ID":47,"Name":"internal.metrics.resultSize","Update":2310,"Value":6930,"Internal":true,"Count Failed Values":true},{"ID":48,"Name":"internal.metrics.jvmGCTime","Update":20,"Value":60,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":85,"Executor Deserialize CPU Time":28476689,"Executor Run Time":3,"Executor CPU Time":3424913,"Peak Execution Memory":0,"Result Size":2310,"JVM GC Time":20,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":9,"Index":1,"Attempt":0,"Launch Time":1630551797918,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551798017,"Failed":false,"Killed":false,"Accumulables":[{"ID":41,"Name":"duration","Update":"20","Value":"84","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"internal.metrics.executorDeserializeTime","Update":79,"Value":274,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.executorDeserializeCpuTime","Update":28715207,"Value":109664117,"Internal":true,"Count Failed Values":true},{"ID":45,"Name":"internal.metrics.executorRunTime","Update":12,"Value":52,"Internal":true,"Count Failed Values":true},{"ID":46,"Name":"internal.metrics.executorCpuTime","Update":3619053,"Value":14495004,"Internal":true,"Count Failed Values":true},{"ID":47,"Name":"internal.metrics.resultSize","Update":2310,"Value":9240,"Internal":true,"Count Failed Values":true},{"ID":48,"Name":"internal.metrics.jvmGCTime","Update":20,"Value":80,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":79,"Executor Deserialize CPU Time":28715207,"Executor Run Time":12,"Executor CPU Time":3619053,"Peak Execution Memory":0,"Result Size":2310,"JVM GC Time":20,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":8,"Index":0,"Attempt":0,"Launch Time":1630551797917,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551798034,"Failed":false,"Killed":false,"Accumulables":[{"ID":41,"Name":"duration","Update":"66","Value":"150","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"internal.metrics.executorDeserializeTime","Update":34,"Value":308,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.executorDeserializeCpuTime","Update":26072946,"Value":135737063,"Internal":true,"Count Failed Values":true},{"ID":45,"Name":"internal.metrics.executorRunTime","Update":75,"Value":127,"Internal":true,"Count Failed Values":true},{"ID":46,"Name":"internal.metrics.executorCpuTime","Update":12325591,"Value":26820595,"Internal":true,"Count Failed Values":true},{"ID":47,"Name":"internal.metrics.resultSize","Update":2353,"Value":11593,"Internal":true,"Count Failed Values":true},{"ID":48,"Name":"internal.metrics.jvmGCTime","Update":20,"Value":100,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.output.bytesWritten","Update":389,"Value":389,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":34,"Executor Deserialize CPU Time":26072946,"Executor Run Time":75,"Executor CPU Time":12325591,"Peak Execution Memory":0,"Result Size":2353,"JVM GC Time":20,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":389,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":15,"Index":7,"Attempt":0,"Launch Time":1630551797921,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551798107,"Failed":false,"Killed":false,"Accumulables":[{"ID":41,"Name":"duration","Update":"138","Value":"288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"internal.metrics.executorDeserializeTime","Update":31,"Value":339,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.executorDeserializeCpuTime","Update":26783787,"Value":162520850,"Internal":true,"Count Failed Values":true},{"ID":45,"Name":"internal.metrics.executorRunTime","Update":148,"Value":275,"Internal":true,"Count Failed Values":true},{"ID":46,"Name":"internal.metrics.executorCpuTime","Update":22035952,"Value":48856547,"Internal":true,"Count Failed Values":true},{"ID":47,"Name":"internal.metrics.resultSize","Update":2396,"Value":13989,"Internal":true,"Count Failed Values":true},{"ID":48,"Name":"internal.metrics.jvmGCTime","Update":20,"Value":120,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.output.bytesWritten","Update":676,"Value":1065,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":31,"Executor Deserialize CPU Time":26783787,"Executor Run Time":148,"Executor CPU Time":22035952,"Peak Execution Memory":0,"Result Size":2396,"JVM GC Time":20,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":676,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":13,"Index":5,"Attempt":0,"Launch Time":1630551797920,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551798111,"Failed":false,"Killed":false,"Accumulables":[{"ID":41,"Name":"duration","Update":"115","Value":"403","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"number of output rows","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"internal.metrics.executorDeserializeTime","Update":66,"Value":405,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.executorDeserializeCpuTime","Update":27252870,"Value":189773720,"Internal":true,"Count Failed Values":true},{"ID":45,"Name":"internal.metrics.executorRunTime","Update":120,"Value":395,"Internal":true,"Count Failed Values":true},{"ID":46,"Name":"internal.metrics.executorCpuTime","Update":70218698,"Value":119075245,"Internal":true,"Count Failed Values":true},{"ID":47,"Name":"internal.metrics.resultSize","Update":2396,"Value":16385,"Internal":true,"Count Failed Values":true},{"ID":48,"Name":"internal.metrics.jvmGCTime","Update":20,"Value":140,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.output.bytesWritten","Update":696,"Value":1761,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":2,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":66,"Executor Deserialize CPU Time":27252870,"Executor Run Time":120,"Executor CPU Time":70218698,"Peak Execution Memory":0,"Result Size":2396,"JVM GC Time":20,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":696,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":10,"Index":2,"Attempt":0,"Launch Time":1630551797918,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551798112,"Failed":false,"Killed":false,"Accumulables":[{"ID":41,"Name":"duration","Update":"142","Value":"545","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"number of output rows","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"internal.metrics.executorDeserializeTime","Update":38,"Value":443,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.executorDeserializeCpuTime","Update":26800715,"Value":216574435,"Internal":true,"Count Failed Values":true},{"ID":45,"Name":"internal.metrics.executorRunTime","Update":148,"Value":543,"Internal":true,"Count Failed Values":true},{"ID":46,"Name":"internal.metrics.executorCpuTime","Update":24148057,"Value":143223302,"Internal":true,"Count Failed Values":true},{"ID":47,"Name":"internal.metrics.resultSize","Update":2396,"Value":18781,"Internal":true,"Count Failed Values":true},{"ID":48,"Name":"internal.metrics.jvmGCTime","Update":20,"Value":160,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.output.bytesWritten","Update":667,"Value":2428,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":3,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":38,"Executor Deserialize CPU Time":26800715,"Executor Run Time":148,"Executor CPU Time":24148057,"Peak Execution Memory":0,"Result Size":2396,"JVM GC Time":20,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":667,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"save at :32","Number of Tasks":8,"RDD Info":[{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at :32","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"map\"}","Callsite":"createDataFrame at :33","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"8\",\"name\":\"parallelize\"}","Callsite":"parallelize at :33","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:36)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:38)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:40)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:42)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:44)\n$line28.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:46)\n$line28.$read$$iw$$iw$$iw$$iw$$iw.(:48)\n$line28.$read$$iw$$iw$$iw$$iw.(:50)\n$line28.$read$$iw$$iw$$iw.(:52)\n$line28.$read$$iw$$iw.(:54)\n$line28.$read$$iw.(:56)\n$line28.$read.(:58)\n$line28.$read$.(:62)\n$line28.$read$.()\n$line28.$eval$.$print$lzycompute(:7)\n$line28.$eval$.$print(:6)\n$line28.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)","Submission Time":1630551797869,"Completion Time":1630551798113,"Accumulables":[{"ID":41,"Name":"duration","Value":"545","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"number of output rows","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"internal.metrics.executorDeserializeTime","Value":443,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.executorDeserializeCpuTime","Value":216574435,"Internal":true,"Count Failed Values":true},{"ID":45,"Name":"internal.metrics.executorRunTime","Value":543,"Internal":true,"Count Failed Values":true},{"ID":46,"Name":"internal.metrics.executorCpuTime","Value":143223302,"Internal":true,"Count Failed Values":true},{"ID":47,"Name":"internal.metrics.resultSize","Value":18781,"Internal":true,"Count Failed Values":true},{"ID":48,"Name":"internal.metrics.jvmGCTime","Value":160,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.output.bytesWritten","Value":2428,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.output.recordsWritten","Value":3,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":1,"Completion Time":1630551798113,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[37,4],[38,2428],[39,3],[40,0]]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":2,"time":1630551798129} +{"Event":"SparkListenerJobStart","Job ID":2,"Submission Time":1630551805661,"Stage Infos":[{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"load at :29","Number of Tasks":1,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"18\",\"name\":\"mapPartitions\"}","Callsite":"load at :29","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"17\",\"name\":\"parallelize\"}","Callsite":"load at :29","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:240)\n$line29.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:29)\n$line29.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:33)\n$line29.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:35)\n$line29.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:37)\n$line29.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:39)\n$line29.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:41)\n$line29.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:43)\n$line29.$read$$iw$$iw$$iw$$iw$$iw.(:45)\n$line29.$read$$iw$$iw$$iw$$iw.(:47)\n$line29.$read$$iw$$iw$$iw.(:49)\n$line29.$read$$iw$$iw.(:51)\n$line29.$read$$iw.(:53)\n$line29.$read.(:55)\n$line29.$read$.(:59)\n$line29.$read$.()\n$line29.$eval$.$print$lzycompute(:7)\n$line29.$eval$.$print(:6)\n$line29.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[2],"Properties":{"spark.rdd.scope":"{\"id\":\"19\",\"name\":\"collect\"}","spark.rdd.scope.noOverride":"true"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"load at :29","Number of Tasks":1,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"18\",\"name\":\"mapPartitions\"}","Callsite":"load at :29","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"17\",\"name\":\"parallelize\"}","Callsite":"load at :29","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:240)\n$line29.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:29)\n$line29.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:33)\n$line29.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:35)\n$line29.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:37)\n$line29.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:39)\n$line29.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:41)\n$line29.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:43)\n$line29.$read$$iw$$iw$$iw$$iw$$iw.(:45)\n$line29.$read$$iw$$iw$$iw$$iw.(:47)\n$line29.$read$$iw$$iw$$iw.(:49)\n$line29.$read$$iw$$iw.(:51)\n$line29.$read$$iw.(:53)\n$line29.$read.(:55)\n$line29.$read$.(:59)\n$line29.$read$.()\n$line29.$eval$.$print$lzycompute(:7)\n$line29.$eval$.$print(:6)\n$line29.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)","Submission Time":1630551805662,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.rdd.scope":"{\"id\":\"19\",\"name\":\"collect\"}","spark.rdd.scope.noOverride":"true"}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":16,"Index":0,"Attempt":0,"Launch Time":1630551805678,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":16,"Index":0,"Attempt":0,"Launch Time":1630551805678,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551805788,"Failed":false,"Killed":false,"Accumulables":[{"ID":68,"Name":"internal.metrics.executorDeserializeTime","Update":74,"Value":74,"Internal":true,"Count Failed Values":true},{"ID":69,"Name":"internal.metrics.executorDeserializeCpuTime","Update":73837229,"Value":73837229,"Internal":true,"Count Failed Values":true},{"ID":70,"Name":"internal.metrics.executorRunTime","Update":33,"Value":33,"Internal":true,"Count Failed Values":true},{"ID":71,"Name":"internal.metrics.executorCpuTime","Update":7938544,"Value":7938544,"Internal":true,"Count Failed Values":true},{"ID":72,"Name":"internal.metrics.resultSize","Update":2368,"Value":2368,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":74,"Executor Deserialize CPU Time":73837229,"Executor Run Time":33,"Executor CPU Time":7938544,"Peak Execution Memory":0,"Result Size":2368,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"load at :29","Number of Tasks":1,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"18\",\"name\":\"mapPartitions\"}","Callsite":"load at :29","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"17\",\"name\":\"parallelize\"}","Callsite":"load at :29","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:240)\n$line29.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:29)\n$line29.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:33)\n$line29.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:35)\n$line29.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:37)\n$line29.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:39)\n$line29.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:41)\n$line29.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:43)\n$line29.$read$$iw$$iw$$iw$$iw$$iw.(:45)\n$line29.$read$$iw$$iw$$iw$$iw.(:47)\n$line29.$read$$iw$$iw$$iw.(:49)\n$line29.$read$$iw$$iw.(:51)\n$line29.$read$$iw.(:53)\n$line29.$read.(:55)\n$line29.$read$.(:59)\n$line29.$read$.()\n$line29.$eval$.$print$lzycompute(:7)\n$line29.$eval$.$print(:6)\n$line29.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)","Submission Time":1630551805662,"Completion Time":1630551805789,"Accumulables":[{"ID":68,"Name":"internal.metrics.executorDeserializeTime","Value":74,"Internal":true,"Count Failed Values":true},{"ID":69,"Name":"internal.metrics.executorDeserializeCpuTime","Value":73837229,"Internal":true,"Count Failed Values":true},{"ID":70,"Name":"internal.metrics.executorRunTime","Value":33,"Internal":true,"Count Failed Values":true},{"ID":71,"Name":"internal.metrics.executorCpuTime","Value":7938544,"Internal":true,"Count Failed Values":true},{"ID":72,"Name":"internal.metrics.resultSize","Value":2368,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":2,"Completion Time":1630551805790,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":3,"description":"save at :32","details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:36)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:38)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:40)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:42)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:44)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:46)\n$line31.$read$$iw$$iw$$iw$$iw$$iw.(:48)\n$line31.$read$$iw$$iw$$iw$$iw.(:50)\n$line31.$read$$iw$$iw$$iw.(:52)\n$line31.$read$$iw$$iw.(:54)\n$line31.$read$$iw.(:56)\n$line31.$read.(:58)\n$line31.$read$.(:62)\n$line31.$read$.()\n$line31.$eval$.$print$lzycompute(:7)\n$line31.$eval$.$print(:6)\n$line31.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)","physicalPlanDescription":"== Physical Plan ==\nExecute InsertIntoHadoopFsRelationCommand (2)\n+- Scan parquet (1)\n\n\n(1) Scan parquet \nOutput [2]: [name#18, booksIntersted#19]\nBatched: false\nLocation: InMemoryFileIndex [file:/home/nartal/event_logs_spark/decimalnested.parquet]\nReadSchema: struct>>\n\n(2) Execute InsertIntoHadoopFsRelationCommand\nInput [2]: [name#18, booksIntersted#19]\nArguments: file:/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/complex_nested_decimal, false, Parquet, Map(path -> complex_nested_decimal), Append, [name, booksIntersted]\n\n","sparkPlanInfo":{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand file:/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/complex_nested_decimal, false, Parquet, Map(path -> complex_nested_decimal), Append, [name, booksIntersted]","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [name#18,booksIntersted#19] Batched: false, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex[file:/home/nartal/event_logs_spark/decimalnested.parquet], PartitionFilters: [], PushedFilters: [], ReadSchema: struct>>","Format":"Parquet","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of output rows","accumulatorId":97,"metricType":"sum"},{"name":"number of files read","accumulatorId":98,"metricType":"sum"},{"name":"metadata time","accumulatorId":99,"metricType":"timing"},{"name":"size of files read","accumulatorId":100,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of written files","accumulatorId":93,"metricType":"sum"},{"name":"written output","accumulatorId":94,"metricType":"size"},{"name":"number of output rows","accumulatorId":95,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":96,"metricType":"sum"}]},"time":1630551826717} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":3,"accumUpdates":[[98,5],[99,3],[100,7005]]} +{"Event":"SparkListenerJobStart","Job ID":3,"Submission Time":1630551826832,"Stage Infos":[{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"save at :32","Number of Tasks":5,"RDD Info":[{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"23\",\"name\":\"Scan parquet \"}","Callsite":"save at :32","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":5,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"FileScanRDD","Scope":"{\"id\":\"23\",\"name\":\"Scan parquet \"}","Callsite":"save at :32","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":5,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:36)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:38)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:40)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:42)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:44)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:46)\n$line31.$read$$iw$$iw$$iw$$iw$$iw.(:48)\n$line31.$read$$iw$$iw$$iw$$iw.(:50)\n$line31.$read$$iw$$iw$$iw.(:52)\n$line31.$read$$iw$$iw.(:54)\n$line31.$read$$iw.(:56)\n$line31.$read.(:58)\n$line31.$read$.(:62)\n$line31.$read$.()\n$line31.$eval$.$print$lzycompute(:7)\n$line31.$eval$.$print(:6)\n$line31.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[3],"Properties":{"spark.sql.warehouse.dir":"file:/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/spark-warehouse/","spark.driver.host":"nartal.attlocal.net","spark.eventLog.enabled":"true","spark.driver.port":"44977","spark.repl.class.uri":"spark://nartal.attlocal.net:44977/classes","spark.jars":"file:///home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.repl.class.outputDir":"/tmp/spark-6437c8fb-c50a-434e-a838-d827759715c1/repl-56a3ce44-fe38-4b08-be93-b9831c0ca528","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"22\",\"name\":\"Execute InsertIntoHadoopFsRelationCommand\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.startTime":"1630551715882","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://nartal.attlocal.net:44977/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.submit.deployMode":"client","spark.master":"local[*]","spark.home":"/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","spark.eventLog.dir":"/home/nartal/data_format_eventlog","spark.sql.execution.id":"3","spark.sql.catalogImplementation":"hive","spark.repl.local.jars":"file:///home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.app.id":"local-1630551716813"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"save at :32","Number of Tasks":5,"RDD Info":[{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"23\",\"name\":\"Scan parquet \"}","Callsite":"save at :32","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":5,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"FileScanRDD","Scope":"{\"id\":\"23\",\"name\":\"Scan parquet \"}","Callsite":"save at :32","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":5,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:36)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:38)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:40)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:42)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:44)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:46)\n$line31.$read$$iw$$iw$$iw$$iw$$iw.(:48)\n$line31.$read$$iw$$iw$$iw$$iw.(:50)\n$line31.$read$$iw$$iw$$iw.(:52)\n$line31.$read$$iw$$iw.(:54)\n$line31.$read$$iw.(:56)\n$line31.$read.(:58)\n$line31.$read$.(:62)\n$line31.$read$.()\n$line31.$eval$.$print$lzycompute(:7)\n$line31.$eval$.$print(:6)\n$line31.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)","Submission Time":1630551826836,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/spark-warehouse/","spark.driver.host":"nartal.attlocal.net","spark.eventLog.enabled":"true","spark.driver.port":"44977","spark.repl.class.uri":"spark://nartal.attlocal.net:44977/classes","spark.jars":"file:///home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.repl.class.outputDir":"/tmp/spark-6437c8fb-c50a-434e-a838-d827759715c1/repl-56a3ce44-fe38-4b08-be93-b9831c0ca528","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"22\",\"name\":\"Execute InsertIntoHadoopFsRelationCommand\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.startTime":"1630551715882","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://nartal.attlocal.net:44977/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.submit.deployMode":"client","spark.master":"local[*]","spark.home":"/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","spark.eventLog.dir":"/home/nartal/data_format_eventlog","spark.sql.execution.id":"3","spark.sql.catalogImplementation":"hive","spark.repl.local.jars":"file:///home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.app.id":"local-1630551716813"}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":17,"Index":0,"Attempt":0,"Launch Time":1630551826877,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":18,"Index":1,"Attempt":0,"Launch Time":1630551826880,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":19,"Index":2,"Attempt":0,"Launch Time":1630551826880,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":20,"Index":3,"Attempt":0,"Launch Time":1630551826880,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":21,"Index":4,"Attempt":0,"Launch Time":1630551826881,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":21,"Index":4,"Attempt":0,"Launch Time":1630551826881,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551827171,"Failed":false,"Killed":false,"Accumulables":[{"ID":101,"Name":"internal.metrics.executorDeserializeTime","Update":61,"Value":61,"Internal":true,"Count Failed Values":true},{"ID":102,"Name":"internal.metrics.executorDeserializeCpuTime","Update":51368256,"Value":51368256,"Internal":true,"Count Failed Values":true},{"ID":103,"Name":"internal.metrics.executorRunTime","Update":221,"Value":221,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorCpuTime","Update":61516954,"Value":61516954,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.resultSize","Update":2518,"Value":2518,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.jvmGCTime","Update":26,"Value":26,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.input.bytesRead","Update":2324,"Value":2324,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":61,"Executor Deserialize CPU Time":51368256,"Executor Run Time":221,"Executor CPU Time":61516954,"Peak Execution Memory":0,"Result Size":2518,"JVM GC Time":26,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":2324,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":17,"Index":0,"Attempt":0,"Launch Time":1630551826877,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551827221,"Failed":false,"Killed":false,"Accumulables":[{"ID":97,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":101,"Name":"internal.metrics.executorDeserializeTime","Update":64,"Value":125,"Internal":true,"Count Failed Values":true},{"ID":102,"Name":"internal.metrics.executorDeserializeCpuTime","Update":49715735,"Value":101083991,"Internal":true,"Count Failed Values":true},{"ID":103,"Name":"internal.metrics.executorRunTime","Update":272,"Value":493,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorCpuTime","Update":74973844,"Value":136490798,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.resultSize","Update":2647,"Value":5165,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.jvmGCTime","Update":26,"Value":52,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.input.bytesRead","Update":4949,"Value":7273,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.input.recordsRead","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.output.bytesWritten","Update":1585,"Value":1585,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":64,"Executor Deserialize CPU Time":49715735,"Executor Run Time":272,"Executor CPU Time":74973844,"Peak Execution Memory":0,"Result Size":2647,"JVM GC Time":26,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":4949,"Records Read":1},"Output Metrics":{"Bytes Written":1585,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":20,"Index":3,"Attempt":0,"Launch Time":1630551826880,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551827237,"Failed":false,"Killed":false,"Accumulables":[{"ID":97,"Name":"number of output rows","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":101,"Name":"internal.metrics.executorDeserializeTime","Update":60,"Value":185,"Internal":true,"Count Failed Values":true},{"ID":102,"Name":"internal.metrics.executorDeserializeCpuTime","Update":46769502,"Value":147853493,"Internal":true,"Count Failed Values":true},{"ID":103,"Name":"internal.metrics.executorRunTime","Update":290,"Value":783,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorCpuTime","Update":98213433,"Value":234704231,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.resultSize","Update":2647,"Value":7812,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.jvmGCTime","Update":26,"Value":78,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.input.bytesRead","Update":5288,"Value":12561,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.input.recordsRead","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.output.bytesWritten","Update":1452,"Value":3037,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":2,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":60,"Executor Deserialize CPU Time":46769502,"Executor Run Time":290,"Executor CPU Time":98213433,"Peak Execution Memory":0,"Result Size":2647,"JVM GC Time":26,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":5288,"Records Read":1},"Output Metrics":{"Bytes Written":1452,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":19,"Index":2,"Attempt":0,"Launch Time":1630551826880,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551827239,"Failed":false,"Killed":false,"Accumulables":[{"ID":97,"Name":"number of output rows","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":101,"Name":"internal.metrics.executorDeserializeTime","Update":65,"Value":250,"Internal":true,"Count Failed Values":true},{"ID":102,"Name":"internal.metrics.executorDeserializeCpuTime","Update":45325554,"Value":193179047,"Internal":true,"Count Failed Values":true},{"ID":103,"Name":"internal.metrics.executorRunTime","Update":289,"Value":1072,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorCpuTime","Update":107934080,"Value":342638311,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.resultSize","Update":2647,"Value":10459,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.jvmGCTime","Update":26,"Value":104,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.input.bytesRead","Update":4859,"Value":17420,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.input.recordsRead","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.output.bytesWritten","Update":1567,"Value":4604,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":3,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":65,"Executor Deserialize CPU Time":45325554,"Executor Run Time":289,"Executor CPU Time":107934080,"Peak Execution Memory":0,"Result Size":2647,"JVM GC Time":26,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":4859,"Records Read":1},"Output Metrics":{"Bytes Written":1567,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":18,"Index":1,"Attempt":0,"Launch Time":1630551826880,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551827239,"Failed":false,"Killed":false,"Accumulables":[{"ID":97,"Name":"number of output rows","Update":"1","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":101,"Name":"internal.metrics.executorDeserializeTime","Update":62,"Value":312,"Internal":true,"Count Failed Values":true},{"ID":102,"Name":"internal.metrics.executorDeserializeCpuTime","Update":47339918,"Value":240518965,"Internal":true,"Count Failed Values":true},{"ID":103,"Name":"internal.metrics.executorRunTime","Update":292,"Value":1364,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorCpuTime","Update":114923942,"Value":457562253,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.resultSize","Update":2647,"Value":13106,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.jvmGCTime","Update":26,"Value":130,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.input.bytesRead","Update":4904,"Value":22324,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.input.recordsRead","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.output.bytesWritten","Update":1576,"Value":6180,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":4,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":62,"Executor Deserialize CPU Time":47339918,"Executor Run Time":292,"Executor CPU Time":114923942,"Peak Execution Memory":0,"Result Size":2647,"JVM GC Time":26,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":4904,"Records Read":1},"Output Metrics":{"Bytes Written":1576,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"save at :32","Number of Tasks":5,"RDD Info":[{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"23\",\"name\":\"Scan parquet \"}","Callsite":"save at :32","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":5,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"FileScanRDD","Scope":"{\"id\":\"23\",\"name\":\"Scan parquet \"}","Callsite":"save at :32","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":5,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:36)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:38)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:40)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:42)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:44)\n$line31.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:46)\n$line31.$read$$iw$$iw$$iw$$iw$$iw.(:48)\n$line31.$read$$iw$$iw$$iw$$iw.(:50)\n$line31.$read$$iw$$iw$$iw.(:52)\n$line31.$read$$iw$$iw.(:54)\n$line31.$read$$iw.(:56)\n$line31.$read.(:58)\n$line31.$read$.(:62)\n$line31.$read$.()\n$line31.$eval$.$print$lzycompute(:7)\n$line31.$eval$.$print(:6)\n$line31.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)","Submission Time":1630551826836,"Completion Time":1630551827240,"Accumulables":[{"ID":97,"Name":"number of output rows","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":101,"Name":"internal.metrics.executorDeserializeTime","Value":312,"Internal":true,"Count Failed Values":true},{"ID":102,"Name":"internal.metrics.executorDeserializeCpuTime","Value":240518965,"Internal":true,"Count Failed Values":true},{"ID":103,"Name":"internal.metrics.executorRunTime","Value":1364,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorCpuTime","Value":457562253,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.resultSize","Value":13106,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.jvmGCTime","Value":130,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.input.bytesRead","Value":22324,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.input.recordsRead","Value":4,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.output.bytesWritten","Value":6180,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.output.recordsWritten","Value":4,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":3,"Completion Time":1630551827240,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":3,"accumUpdates":[[93,4],[94,6180],[95,4],[96,0]]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":3,"time":1630551827255} +{"Event":"SparkListenerJobStart","Job ID":4,"Submission Time":1630551835690,"Stage Infos":[{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"load at :29","Number of Tasks":1,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"mapPartitions\"}","Callsite":"load at :29","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"26\",\"name\":\"parallelize\"}","Callsite":"load at :29","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:240)\n$line32.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:29)\n$line32.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:33)\n$line32.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:35)\n$line32.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:37)\n$line32.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:39)\n$line32.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:41)\n$line32.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:43)\n$line32.$read$$iw$$iw$$iw$$iw$$iw.(:45)\n$line32.$read$$iw$$iw$$iw$$iw.(:47)\n$line32.$read$$iw$$iw$$iw.(:49)\n$line32.$read$$iw$$iw.(:51)\n$line32.$read$$iw.(:53)\n$line32.$read.(:55)\n$line32.$read$.(:59)\n$line32.$read$.()\n$line32.$eval$.$print$lzycompute(:7)\n$line32.$eval$.$print(:6)\n$line32.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[4],"Properties":{"spark.rdd.scope":"{\"id\":\"28\",\"name\":\"collect\"}","spark.rdd.scope.noOverride":"true"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"load at :29","Number of Tasks":1,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"mapPartitions\"}","Callsite":"load at :29","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"26\",\"name\":\"parallelize\"}","Callsite":"load at :29","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:240)\n$line32.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:29)\n$line32.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:33)\n$line32.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:35)\n$line32.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:37)\n$line32.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:39)\n$line32.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:41)\n$line32.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:43)\n$line32.$read$$iw$$iw$$iw$$iw$$iw.(:45)\n$line32.$read$$iw$$iw$$iw$$iw.(:47)\n$line32.$read$$iw$$iw$$iw.(:49)\n$line32.$read$$iw$$iw.(:51)\n$line32.$read$$iw.(:53)\n$line32.$read.(:55)\n$line32.$read$.(:59)\n$line32.$read$.()\n$line32.$eval$.$print$lzycompute(:7)\n$line32.$eval$.$print(:6)\n$line32.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)","Submission Time":1630551835691,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.rdd.scope":"{\"id\":\"28\",\"name\":\"collect\"}","spark.rdd.scope.noOverride":"true"}} +{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":22,"Index":0,"Attempt":0,"Launch Time":1630551835707,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":22,"Index":0,"Attempt":0,"Launch Time":1630551835707,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551835721,"Failed":false,"Killed":false,"Accumulables":[{"ID":126,"Name":"internal.metrics.executorDeserializeTime","Update":8,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":127,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6726347,"Value":6726347,"Internal":true,"Count Failed Values":true},{"ID":128,"Name":"internal.metrics.executorRunTime","Update":3,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorCpuTime","Update":1703787,"Value":1703787,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.resultSize","Update":1645,"Value":1645,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":8,"Executor Deserialize CPU Time":6726347,"Executor Run Time":3,"Executor CPU Time":1703787,"Peak Execution Memory":0,"Result Size":1645,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"load at :29","Number of Tasks":1,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"mapPartitions\"}","Callsite":"load at :29","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"26\",\"name\":\"parallelize\"}","Callsite":"load at :29","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:240)\n$line32.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:29)\n$line32.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:33)\n$line32.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:35)\n$line32.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:37)\n$line32.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:39)\n$line32.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:41)\n$line32.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:43)\n$line32.$read$$iw$$iw$$iw$$iw$$iw.(:45)\n$line32.$read$$iw$$iw$$iw$$iw.(:47)\n$line32.$read$$iw$$iw$$iw.(:49)\n$line32.$read$$iw$$iw.(:51)\n$line32.$read$$iw.(:53)\n$line32.$read.(:55)\n$line32.$read$.(:59)\n$line32.$read$.()\n$line32.$eval$.$print$lzycompute(:7)\n$line32.$eval$.$print(:6)\n$line32.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)","Submission Time":1630551835691,"Completion Time":1630551835722,"Accumulables":[{"ID":126,"Name":"internal.metrics.executorDeserializeTime","Value":8,"Internal":true,"Count Failed Values":true},{"ID":127,"Name":"internal.metrics.executorDeserializeCpuTime","Value":6726347,"Internal":true,"Count Failed Values":true},{"ID":128,"Name":"internal.metrics.executorRunTime","Value":3,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorCpuTime","Value":1703787,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.resultSize","Value":1645,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":4,"Completion Time":1630551835722,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":4,"description":"save at :32","details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:36)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:38)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:40)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:42)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:44)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:46)\n$line33.$read$$iw$$iw$$iw$$iw$$iw.(:48)\n$line33.$read$$iw$$iw$$iw$$iw.(:50)\n$line33.$read$$iw$$iw$$iw.(:52)\n$line33.$read$$iw$$iw.(:54)\n$line33.$read$$iw.(:56)\n$line33.$read.(:58)\n$line33.$read$.(:62)\n$line33.$read$.()\n$line33.$eval$.$print$lzycompute(:7)\n$line33.$eval$.$print(:6)\n$line33.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)","physicalPlanDescription":"== Physical Plan ==\nExecute InsertIntoHadoopFsRelationCommand (3)\n+- * ColumnarToRow (2)\n +- Scan parquet (1)\n\n\n(1) Scan parquet \nOutput [2]: [name#36, subject#37]\nBatched: true\nLocation: InMemoryFileIndex [file:/home/nartal/event_logs_spark/simpleSchema.parquet]\nReadSchema: struct\n\n(2) ColumnarToRow [codegen id : 1]\nInput [2]: [name#36, subject#37]\n\n(3) Execute InsertIntoHadoopFsRelationCommand\nInput [2]: [name#36, subject#37]\nArguments: file:/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/complex_nested_decimal, false, Parquet, Map(path -> complex_nested_decimal), Append, [name, subject]\n\n","sparkPlanInfo":{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand file:/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/complex_nested_decimal, false, Parquet, Map(path -> complex_nested_decimal), Append, [name, subject]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [name#36,subject#37] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex[file:/home/nartal/event_logs_spark/simpleSchema.parquet], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[file:/home/nartal/event_logs_spark/simpleSchema.parquet]","ReadSchema":"struct","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":159,"metricType":"sum"},{"name":"scan time","accumulatorId":162,"metricType":"timing"},{"name":"metadata time","accumulatorId":160,"metricType":"timing"},{"name":"size of files read","accumulatorId":161,"metricType":"size"},{"name":"number of output rows","accumulatorId":158,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":156,"metricType":"sum"},{"name":"number of input batches","accumulatorId":157,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":155,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of written files","accumulatorId":151,"metricType":"sum"},{"name":"written output","accumulatorId":152,"metricType":"size"},{"name":"number of output rows","accumulatorId":153,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":154,"metricType":"sum"}]},"time":1630551853096} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":4,"accumUpdates":[[159,4],[160,0],[161,2428]]} +{"Event":"SparkListenerJobStart","Job ID":5,"Submission Time":1630551853192,"Stage Infos":[{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"save at :32","Number of Tasks":4,"RDD Info":[{"RDD ID":20,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"32\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at :32","Parent IDs":[19],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":4,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"FileScanRDD","Scope":"{\"id\":\"35\",\"name\":\"Scan parquet \"}","Callsite":"save at :32","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":4,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"35\",\"name\":\"Scan parquet \"}","Callsite":"save at :32","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":4,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:36)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:38)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:40)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:42)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:44)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:46)\n$line33.$read$$iw$$iw$$iw$$iw$$iw.(:48)\n$line33.$read$$iw$$iw$$iw$$iw.(:50)\n$line33.$read$$iw$$iw$$iw.(:52)\n$line33.$read$$iw$$iw.(:54)\n$line33.$read$$iw.(:56)\n$line33.$read.(:58)\n$line33.$read$.(:62)\n$line33.$read$.()\n$line33.$eval$.$print$lzycompute(:7)\n$line33.$eval$.$print(:6)\n$line33.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[5],"Properties":{"spark.sql.warehouse.dir":"file:/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/spark-warehouse/","spark.driver.host":"nartal.attlocal.net","spark.eventLog.enabled":"true","spark.driver.port":"44977","spark.repl.class.uri":"spark://nartal.attlocal.net:44977/classes","spark.jars":"file:///home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.repl.class.outputDir":"/tmp/spark-6437c8fb-c50a-434e-a838-d827759715c1/repl-56a3ce44-fe38-4b08-be93-b9831c0ca528","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"31\",\"name\":\"Execute InsertIntoHadoopFsRelationCommand\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.startTime":"1630551715882","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://nartal.attlocal.net:44977/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.submit.deployMode":"client","spark.master":"local[*]","spark.home":"/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","spark.eventLog.dir":"/home/nartal/data_format_eventlog","spark.sql.execution.id":"4","spark.sql.catalogImplementation":"hive","spark.repl.local.jars":"file:///home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.app.id":"local-1630551716813"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"save at :32","Number of Tasks":4,"RDD Info":[{"RDD ID":20,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"32\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at :32","Parent IDs":[19],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":4,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"FileScanRDD","Scope":"{\"id\":\"35\",\"name\":\"Scan parquet \"}","Callsite":"save at :32","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":4,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"35\",\"name\":\"Scan parquet \"}","Callsite":"save at :32","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":4,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:36)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:38)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:40)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:42)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:44)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:46)\n$line33.$read$$iw$$iw$$iw$$iw$$iw.(:48)\n$line33.$read$$iw$$iw$$iw$$iw.(:50)\n$line33.$read$$iw$$iw$$iw.(:52)\n$line33.$read$$iw$$iw.(:54)\n$line33.$read$$iw.(:56)\n$line33.$read.(:58)\n$line33.$read$.(:62)\n$line33.$read$.()\n$line33.$eval$.$print$lzycompute(:7)\n$line33.$eval$.$print(:6)\n$line33.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)","Submission Time":1630551853194,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/spark-warehouse/","spark.driver.host":"nartal.attlocal.net","spark.eventLog.enabled":"true","spark.driver.port":"44977","spark.repl.class.uri":"spark://nartal.attlocal.net:44977/classes","spark.jars":"file:///home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.repl.class.outputDir":"/tmp/spark-6437c8fb-c50a-434e-a838-d827759715c1/repl-56a3ce44-fe38-4b08-be93-b9831c0ca528","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"31\",\"name\":\"Execute InsertIntoHadoopFsRelationCommand\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.startTime":"1630551715882","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://nartal.attlocal.net:44977/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.submit.deployMode":"client","spark.master":"local[*]","spark.home":"/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","spark.eventLog.dir":"/home/nartal/data_format_eventlog","spark.sql.execution.id":"4","spark.sql.catalogImplementation":"hive","spark.repl.local.jars":"file:///home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.app.id":"local-1630551716813"}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":23,"Index":0,"Attempt":0,"Launch Time":1630551853223,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":24,"Index":1,"Attempt":0,"Launch Time":1630551853223,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":25,"Index":2,"Attempt":0,"Launch Time":1630551853223,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":26,"Index":3,"Attempt":0,"Launch Time":1630551853224,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":26,"Index":3,"Attempt":0,"Launch Time":1630551853224,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551853275,"Failed":false,"Killed":false,"Accumulables":[{"ID":155,"Name":"duration","Update":"38","Value":"38","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":162,"Name":"scan time","Update":"17","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"internal.metrics.executorDeserializeTime","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.executorDeserializeCpuTime","Update":18906376,"Value":18906376,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.executorRunTime","Update":27,"Value":27,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.executorCpuTime","Update":13294322,"Value":13294322,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.resultSize","Update":2524,"Value":2524,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.input.bytesRead","Update":814,"Value":814,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":20,"Executor Deserialize CPU Time":18906376,"Executor Run Time":27,"Executor CPU Time":13294322,"Peak Execution Memory":0,"Result Size":2524,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":814,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":23,"Index":0,"Attempt":0,"Launch Time":1630551853223,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551853291,"Failed":false,"Killed":false,"Accumulables":[{"ID":155,"Name":"duration","Update":"35","Value":"73","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":156,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":157,"Name":"number of input batches","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":158,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":162,"Name":"scan time","Update":"5","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"internal.metrics.executorDeserializeTime","Update":17,"Value":37,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.executorDeserializeCpuTime","Update":16556383,"Value":35462759,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.executorRunTime","Update":46,"Value":73,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.executorCpuTime","Update":18906061,"Value":32200383,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.resultSize","Update":2653,"Value":5177,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.input.bytesRead","Update":2520,"Value":3334,"Internal":true,"Count Failed Values":true},{"ID":185,"Name":"internal.metrics.input.recordsRead","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":186,"Name":"internal.metrics.output.bytesWritten","Update":696,"Value":696,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":17,"Executor Deserialize CPU Time":16556383,"Executor Run Time":46,"Executor CPU Time":18906061,"Peak Execution Memory":0,"Result Size":2653,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":2520,"Records Read":1},"Output Metrics":{"Bytes Written":696,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":25,"Index":2,"Attempt":0,"Launch Time":1630551853223,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551853305,"Failed":false,"Killed":false,"Accumulables":[{"ID":155,"Name":"duration","Update":"52","Value":"125","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":156,"Name":"number of output rows","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":157,"Name":"number of input batches","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":158,"Name":"number of output rows","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":162,"Name":"scan time","Update":"29","Value":"51","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"internal.metrics.executorDeserializeTime","Update":16,"Value":53,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.executorDeserializeCpuTime","Update":15700495,"Value":51163254,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.executorRunTime","Update":62,"Value":135,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.executorCpuTime","Update":30212298,"Value":62412681,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.resultSize","Update":2653,"Value":7830,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.input.bytesRead","Update":2375,"Value":5709,"Internal":true,"Count Failed Values":true},{"ID":185,"Name":"internal.metrics.input.recordsRead","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":186,"Name":"internal.metrics.output.bytesWritten","Update":667,"Value":1363,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":2,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":16,"Executor Deserialize CPU Time":15700495,"Executor Run Time":62,"Executor CPU Time":30212298,"Peak Execution Memory":0,"Result Size":2653,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":2375,"Records Read":1},"Output Metrics":{"Bytes Written":667,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":24,"Index":1,"Attempt":0,"Launch Time":1630551853223,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630551853305,"Failed":false,"Killed":false,"Accumulables":[{"ID":155,"Name":"duration","Update":"52","Value":"177","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":156,"Name":"number of output rows","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":157,"Name":"number of input batches","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":158,"Name":"number of output rows","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":162,"Name":"scan time","Update":"29","Value":"80","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"internal.metrics.executorDeserializeTime","Update":16,"Value":69,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.executorDeserializeCpuTime","Update":14395599,"Value":65558853,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.executorRunTime","Update":62,"Value":197,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.executorCpuTime","Update":32290026,"Value":94702707,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.resultSize","Update":2653,"Value":10483,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.input.bytesRead","Update":2420,"Value":8129,"Internal":true,"Count Failed Values":true},{"ID":185,"Name":"internal.metrics.input.recordsRead","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":186,"Name":"internal.metrics.output.bytesWritten","Update":676,"Value":2039,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":3,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":16,"Executor Deserialize CPU Time":14395599,"Executor Run Time":62,"Executor CPU Time":32290026,"Peak Execution Memory":0,"Result Size":2653,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":2420,"Records Read":1},"Output Metrics":{"Bytes Written":676,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"save at :32","Number of Tasks":4,"RDD Info":[{"RDD ID":20,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"32\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at :32","Parent IDs":[19],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":4,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"FileScanRDD","Scope":"{\"id\":\"35\",\"name\":\"Scan parquet \"}","Callsite":"save at :32","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":4,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"35\",\"name\":\"Scan parquet \"}","Callsite":"save at :32","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":4,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:36)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:38)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:40)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:42)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:44)\n$line33.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:46)\n$line33.$read$$iw$$iw$$iw$$iw$$iw.(:48)\n$line33.$read$$iw$$iw$$iw$$iw.(:50)\n$line33.$read$$iw$$iw$$iw.(:52)\n$line33.$read$$iw$$iw.(:54)\n$line33.$read$$iw.(:56)\n$line33.$read.(:58)\n$line33.$read$.(:62)\n$line33.$read$.()\n$line33.$eval$.$print$lzycompute(:7)\n$line33.$eval$.$print(:6)\n$line33.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)","Submission Time":1630551853194,"Completion Time":1630551853306,"Accumulables":[{"ID":155,"Name":"duration","Value":"177","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":156,"Name":"number of output rows","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":157,"Name":"number of input batches","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":158,"Name":"number of output rows","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":162,"Name":"scan time","Value":"80","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"internal.metrics.executorDeserializeTime","Value":69,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.executorDeserializeCpuTime","Value":65558853,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.executorRunTime","Value":197,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.executorCpuTime","Value":94702707,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.resultSize","Value":10483,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.input.bytesRead","Value":8129,"Internal":true,"Count Failed Values":true},{"ID":185,"Name":"internal.metrics.input.recordsRead","Value":3,"Internal":true,"Count Failed Values":true},{"ID":186,"Name":"internal.metrics.output.bytesWritten","Value":2039,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.output.recordsWritten","Value":3,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":5,"Completion Time":1630551853307,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":4,"accumUpdates":[[151,3],[152,2039],[153,3],[154,0]]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":4,"time":1630551853321} +{"Event":"SparkListenerApplicationEnd","Timestamp":1630551854902} diff --git a/tools/src/test/resources/spark-events-qualification/eventlog_nested_dsv2 b/tools/src/test/resources/spark-events-qualification/eventlog_nested_dsv2 new file mode 100644 index 00000000000..35fa56a71a4 --- /dev/null +++ b/tools/src/test/resources/spark-events-qualification/eventlog_nested_dsv2 @@ -0,0 +1,32 @@ +{"Event":"SparkListenerLogStart","Spark Version":"3.1.1"} +{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"cores":{"Resource Name":"cores","Amount":1,"Discovery Script":"","Vendor":""},"memory":{"Resource Name":"memory","Amount":1024,"Discovery Script":"","Vendor":""},"offHeap":{"Resource Name":"offHeap","Amount":0,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1630045673517,"Executor ID":"driver","Executor Info":{"Host":"user1.attlocal.net","Total Cores":8,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"user1.attlocal.net","Port":36785},"Maximum Memory":384093388,"Timestamp":1630045673538,"Maximum Onheap Memory":384093388,"Maximum Offheap Memory":0} +{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-8-openjdk-amd64/jre","Java Version":"1.8.0_292 (Private Build)","Scala Version":"version 2.12.10"},"Spark Properties":{"spark.driver.host":"user1.attlocal.net","spark.eventLog.enabled":"true","spark.driver.port":"36303","spark.repl.class.uri":"spark://user1.attlocal.net:36303/classes","spark.sql.sources.useV1SourceList":"","spark.jars":"file:///home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.repl.class.outputDir":"/tmp/spark-e7f7c451-9bce-47e0-8e9c-a77f9923d5b4/repl-d8d58adf-1a53-4d01-9b63-9b2d150731ce","spark.app.name":"Spark shell","spark.scheduler.mode":"FIFO","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.startTime":"1630045672277","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://user1.attlocal.net:36303/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.submit.deployMode":"client","spark.master":"local[*]","spark.home":"/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","spark.eventLog.dir":"/home/user1/data_format_eventlog","spark.sql.catalogImplementation":"hive","spark.repl.local.jars":"file:///home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.app.id":"local-1630045673160"},"Hadoop Properties":{"hadoop.service.shutdown.timeout":"30s","yarn.resourcemanager.amlauncher.thread-count":"50","yarn.sharedcache.enabled":"false","fs.s3a.connection.maximum":"15","yarn.nodemanager.numa-awareness.numactl.cmd":"/usr/bin/numactl","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","yarn.timeline-service.timeline-client.number-of-async-entities-to-merge":"10","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.jobhistory.loadedjob.tasks.max":"-1","mapreduce.framework.name":"local","yarn.sharedcache.uploader.server.thread-count":"50","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","hadoop.security.groups.cache.background.reload.threads":"3","yarn.resourcemanager.webapp.cross-origin.enabled":"false","fs.AbstractFileSystem.ftp.impl":"org.apache.hadoop.fs.ftp.FtpFs","hadoop.registry.secure":"false","hadoop.shell.safely.delete.limit.num.files":"100","dfs.bytes-per-checksum":"512","mapreduce.job.acl-view-job":" ","fs.s3a.s3guard.ddb.background.sleep":"25ms","fs.s3a.retry.limit":"${fs.s3a.attempts.maximum}","mapreduce.jobhistory.loadedjobs.cache.size":"5","fs.s3a.s3guard.ddb.table.create":"false","yarn.nodemanager.amrmproxy.enabled":"false","yarn.timeline-service.entity-group-fs-store.with-user-dir":"false","mapreduce.input.fileinputformat.split.minsize":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","yarn.sharedcache.checksum.algo.impl":"org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl","yarn.nodemanager.amrmproxy.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor","yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size":"10485760","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","mapreduce.task.profile.maps":"0-2","yarn.scheduler.include-port-in-node-name":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","yarn.resourcemanager.node-removal-untracked.timeout-ms":"60000","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","mapreduce.jobhistory.webapp.https.address":"0.0.0.0:19890","yarn.node-labels.fs-store.impl.class":"org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore","yarn.nodemanager.collector-service.address":"${yarn.nodemanager.hostname}:8048","fs.trash.checkpoint.interval":"0","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","yarn.resourcemanager.node-ip-cache.expiry-interval-secs":"-1","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","yarn.resourcemanager.placement-constraints.handler":"disabled","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"1000","yarn.resourcemanager.system-metrics-publisher.enabled":"false","yarn.resourcemanager.placement-constraints.algorithm.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm","yarn.sharedcache.webapp.address":"0.0.0.0:8788","yarn.resourcemanager.delegation.token.renew-interval":"*********(redacted)","yarn.sharedcache.nm.uploader.replication.factor":"10","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","mapreduce.reduce.skip.proc-count.auto-incr":"true","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","ipc.client.fallback-to-simple-auth-allowed":"false","yarn.nodemanager.resource.memory.enforced":"true","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","yarn.acl.reservation-enable":"false","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","yarn.app.mapreduce.am.hard-kill-timeout-ms":"10000","fs.s3a.etag.checksum.enabled":"false","yarn.nodemanager.container-metrics.enable":"true","yarn.timeline-service.client.fd-clean-interval-secs":"60","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","hadoop.common.configuration.version":"3.0.0","fs.s3a.s3guard.ddb.table.capacity.read":"500","yarn.nodemanager.remote-app-log-dir-suffix":"logs","yarn.nodemanager.windows-container.cpu-limit.enabled":"false","yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed":"false","file.blocksize":"67108864","hadoop.registry.zk.retry.ceiling.ms":"60000","yarn.scheduler.configuration.leveldb-store.path":"${hadoop.tmp.dir}/yarn/system/confstore","yarn.sharedcache.store.in-memory.initial-delay-mins":"10","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","mapreduce.map.skip.proc-count.auto-incr":"true","fs.s3a.committer.name":"file","mapreduce.task.profile.reduces":"0-2","hadoop.zk.num-retries":"1000","yarn.webapp.xfs-filter.enabled":"true","seq.io.sort.mb":"100","yarn.scheduler.configuration.max.version":"100","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"${yarn.resourcemanager.hostname}:8030","yarn.node-labels.enabled":"false","yarn.resourcemanager.webapp.ui-actions.enabled":"true","mapreduce.task.timeout":"600000","yarn.sharedcache.client-server.thread-count":"50","hadoop.security.groups.shell.command.timeout":"0s","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.nodemanager.elastic-memory-control.oom-handler":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.DefaultOOMHandler","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"file:///","yarn.minicluster.use-rpc":"false","fs.har.impl.disable.cache":"true","yarn.webapp.ui2.enable":"false","io.compression.codec.bzip2.library":"system-native","yarn.nodemanager.distributed-scheduling.enabled":"false","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"${yarn.resourcemanager.hostname}:8090","mapreduce.jobhistory.address":"0.0.0.0:10020","yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.is.minicluster":"false","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","mapreduce.task.combine.progress.records":"10000","yarn.resourcemanager.epoch.range":"0","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","fs.AbstractFileSystem.wasbs.impl":"org.apache.hadoop.fs.azure.Wasbs","yarn.timeline-service.entity-group-fs-store.cache-store-class":"org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore","fs.ftp.transfer.mode":"BLOCK_TRANSFER_MODE","ipc.server.log.slow.rpc":"false","yarn.resourcemanager.node-labels.provider.fetch-interval-ms":"1800000","yarn.router.webapp.https.address":"0.0.0.0:8091","yarn.nodemanager.webapp.cross-origin.enabled":"false","fs.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","yarn.resourcemanager.auto-update.containers":"false","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","yarn.scheduler.configuration.zk-store.parent-path":"/confstore","yarn.nodemanager.default-container-executor.log-dirs.permissions":"710","yarn.app.attempt.diagnostics.limit.kc":"64","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"-1","fs.AbstractFileSystem.abfs.impl":"org.apache.hadoop.fs.azurebfs.Abfs","yarn.timeline-service.writer.flush-interval-seconds":"60","fs.s3a.fast.upload.active.blocks":"4","hadoop.security.credential.clear-text-fallback":"true","yarn.nodemanager.collector-service.thread-count":"5","fs.azure.secure.mode":"false","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","yarn.resourcemanager.fs.state-store.num-retries":"0","yarn.resourcemanager.nodemanager-connect-retries":"10","yarn.nodemanager.log-aggregation.num-log-files-per-app":"30","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","fs.s3a.committer.magic.enabled":"false","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","yarn.system-metrics-publisher.enabled":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","mapreduce.task.io.sort.factor":"10","yarn.nodemanager.amrmproxy.client.thread-count":"25","ha.failover-controller.new-active.rpc-timeout.ms":"60000","yarn.nodemanager.container-localizer.java.opts":"-Xmx256m","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","yarn.nodemanager.windows-container.memory-limit.enabled":"false","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.minicluster.fixed.ports":"false","hadoop.tags.system":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.cluster.max-application-priority":"0","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","hadoop.caller.context.signature.max.size":"40","yarn.client.load.resource-types.from-server":"false","ha.zookeeper.session-timeout.ms":"10000","tfile.io.chunk.size":"1048576","fs.s3a.s3guard.ddb.table.capacity.write":"100","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","yarn.scheduler.configuration.store.max-logs":"1000","yarn.nodemanager.node-attributes.provider.fetch-interval-ms":"600000","fs.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","yarn.nodemanager.local-cache.max-files-per-directory":"8192","hadoop.http.cross-origin.enabled":"false","hadoop.zk.acl":"world:anyone:rwcda","mapreduce.map.sort.spill.percent":"0.80","yarn.timeline-service.entity-group-fs-store.scan-interval-seconds":"60","yarn.node-attribute.fs-store.impl.class":"org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore","fs.s3a.retry.interval":"500ms","yarn.timeline-service.client.best-effort":"false","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","fs.AbstractFileSystem.swebhdfs.impl":"org.apache.hadoop.fs.SWebHdfs","yarn.nodemanager.elastic-memory-control.timeout-sec":"5","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","yarn.timeline-service.reader.webapp.address":"${yarn.timeline-service.webapp.address}","yarn.resourcemanager.placement-constraints.algorithm.pool-size":"1","yarn.timeline-service.hbase.coprocessor.jar.hdfs.location":"/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler","yarn.app.mapreduce.am.command-opts":"-Xmx1024m","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","io.mapfile.bloom.error.rate":"0.005","fs.client.resolve.topology.enabled":"false","yarn.nodemanager.runtime.linux.allowed-runtimes":"default","yarn.sharedcache.store.class":"org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","fs.s3a.metadatastore.impl":"org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore","io.skip.checksum.errors":"false","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","yarn.timeline-service.webapp.xfs-filter.xframe-options":"SAMEORIGIN","fs.s3a.connection.timeout":"200000","mapreduce.job.max.split.locations":"15","yarn.resourcemanager.nm-container-queuing.max-queue-length":"15","hadoop.registry.zk.session.timeout.ms":"60000","yarn.federation.cache-ttl.secs":"300","mapreduce.jvm.system-properties-to-log":"os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name","yarn.resourcemanager.opportunistic-container-allocation.nodes-used":"10","yarn.timeline-service.entity-group-fs-store.active-dir":"/tmp/entity-file-history/active","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","fs.AbstractFileSystem.abfss.impl":"org.apache.hadoop.fs.azurebfs.Abfss","yarn.sharedcache.uploader.server.address":"0.0.0.0:8046","yarn.resourcemanager.delegation-token.max-conf-size-bytes":"*********(redacted)","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","yarn.minicluster.yarn.nodemanager.resource.memory-mb":"4096","mapreduce.job.emit-timeline-data":"false","yarn.nodemanager.resource.system-reserved-memory-mb":"-1","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","yarn.resourcemanager.admin.client.thread-count":"1","yarn.dispatcher.drain-events.timeout":"300000","fs.s3a.buffer.dir":"${hadoop.tmp.dir}/s3a","hadoop.ssl.enabled.protocols":"TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2","mapreduce.jobhistory.admin.address":"0.0.0.0:10033","yarn.log-aggregation-status.time-out.ms":"600000","fs.s3a.assumed.role.sts.endpoint.region":"us-west-1","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","yarn.router.clientrm.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor","yarn.resourcemanager.zk-appid-node.split-index":"0","ftp.blocksize":"67108864","yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions":"read","yarn.router.rmadmin.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor","yarn.nodemanager.log-container-debug-info.enabled":"true","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms":"20","yarn.nodemanager.delete.debug-delay-sec":"0","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","yarn.nodemanager.container-retry-minimum-interval-ms":"1000","hadoop.security.groups.cache.secs":"300","yarn.federation.enabled":"false","fs.azure.local.sas.key.mode":"false","ipc.maximum.data.length":"67108864","mapreduce.shuffle.max.threads":"0","yarn.router.pipeline.cache-max-size":"25","yarn.resourcemanager.nm-container-queuing.load-comparator":"QUEUE_LENGTH","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","yarn.nodemanager.amrmproxy.ha.enable":"false","mapreduce.jobhistory.http.policy":"HTTP_ONLY","yarn.sharedcache.store.in-memory.check-period-mins":"720","hadoop.security.group.mapping.ldap.ssl":"false","yarn.client.application-client-protocol.poll-interval-ms":"200","yarn.scheduler.configuration.leveldb-store.compaction-interval-secs":"86400","yarn.timeline-service.writer.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl","ha.zookeeper.parent-znode":"/hadoop-ha","yarn.nodemanager.log-aggregation.policy.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","yarn.resourcemanager.placement-constraints.scheduler.pool-size":"1","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","yarn.nodemanager.recovery.supervised":"false","yarn.sharedcache.admin.thread-count":"1","yarn.resourcemanager.ha.automatic-failover.enabled":"true","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.resourcemanager.address":"${yarn.resourcemanager.hostname}:8032","ipc.client.ping":"true","mapreduce.task.local-fs.write-limit.bytes":"-1","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","yarn.nodemanager.resource-plugins.gpu.docker-plugin":"nvidia-docker-v1","hadoop.ssl.enabled":"false","fs.s3a.multipart.purge":"false","yarn.scheduler.configuration.store.class":"file","yarn.resourcemanager.nm-container-queuing.queue-limit-stdev":"1.0f","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","ipc.client.bind.wildcard.addr":"false","yarn.resourcemanager.webapp.rest-csrf.enabled":"false","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"10","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","yarn.nodemanager.runtime.linux.docker.allowed-container-networks":"host,none,bridge","yarn.nodemanager.node-labels.resync-interval-ms":"120000","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","mapreduce.job.maps":"2","mapreduce.jobhistory.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","yarn.rm.system-metrics-publisher.emit-container-events":"false","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","fs.s3a.s3guard.ddb.throttle.retry.interval":"100ms","fs.AbstractFileSystem.wasb.impl":"org.apache.hadoop.fs.azure.Wasb","mapreduce.client.submit.file.replication":"10","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"2147483647","yarn.resourcemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold":"1","yarn.nodemanager.webapp.rest-csrf.enabled":"false","fs.ftp.host.port":"21","ipc.ping.interval":"60000","yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size":"10","yarn.resourcemanager.admin.address":"${yarn.resourcemanager.hostname}:8033","file.client-write-packet-size":"65536","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"true","hadoop.security.key.default.bitlength":"128","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","yarn.nodemanager.log.deletion-threads-count":"4","yarn.webapp.filter-entity-list-by-user":"false","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"100","yarn.nodemanager.localizer.client.thread-count":"5","io.erasurecode.codec.rs.rawcoders":"rs_native,rs_java","io.erasurecode.codec.rs-legacy.rawcoders":"rs-legacy_java","yarn.sharedcache.admin.address":"0.0.0.0:8047","yarn.resourcemanager.placement-constraints.algorithm.iterator":"SERIAL","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec","mapreduce.job.cache.limit.max-resources-mb":"0","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"5000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","hadoop.workaround.non.threadsafe.getpwuid":"true","fs.df.interval":"60000","fs.s3a.multiobjectdelete.enable":"true","yarn.sharedcache.cleaner.resource-sleep-ms":"0","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"false","io.file.buffer.size":"65536","hadoop.security.group.mapping.ldap.search.attr.member":"member","hadoop.security.random.device.file.path":"/dev/urandom","hadoop.security.sensitive-config-keys":"*********(redacted)","fs.s3a.s3guard.ddb.max.retries":"9","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","yarn.intermediate-data-encryption.enable":"false","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.nodemanager.container.stderr.pattern":"{*stderr*,*STDERR*}","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/tmp/hadoop-yarn/staging","mapreduce.reduce.shuffle.read.timeout":"180000","hadoop.http.cross-origin.max-age":"1800","io.erasurecode.codec.xor.rawcoders":"xor_native,xor_java","fs.s3a.connection.establish.timeout":"5000","mapreduce.job.running.map.limit":"0","yarn.minicluster.control-resource-monitoring":"false","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","yarn.federation.state-store.class":"org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","adl.http.timeout":"-1","yarn.resourcemanager.placement-constraints.retry-attempts":"3","hadoop.caller.context.enabled":"false","yarn.nodemanager.vmem-pmem-ratio":"2.1","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","hadoop.zk.timeout-ms":"10000","fs.s3a.s3guard.cli.prune.age":"86400000","yarn.nodemanager.resource.pcores-vcores-multiplier":"1.0","yarn.nodemanager.runtime.linux.sandbox-mode":"disabled","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.s3a.committer.threads":"8","hadoop.zk.retry-interval-ms":"1000","hadoop.security.crypto.buffer.size":"8192","yarn.nodemanager.node-labels.provider.fetch-interval-ms":"600000","mapreduce.jobhistory.recovery.store.leveldb.path":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.client.failover-retries-on-socket-timeouts":"0","yarn.nodemanager.resource.memory.enabled":"false","fs.azure.authorization.caching.enable":"true","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","mapreduce.job.finish-when-all-reducers-done":"true","hadoop.registry.jaas.context":"Client","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","io.map.index.interval":"128","yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms":"100","fs.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","mapreduce.job.counters.max":"120","mapreduce.jobhistory.webapp.rest-csrf.enabled":"false","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","yarn.timeline-service.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/timeline","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","hadoop.security.auth_to_local.mechanism":"hadoop","yarn.timeline-service.app-collector.linger-period.ms":"60000","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.reservation-system.planfollower.time-step":"1000","yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed":"true","yarn.webapp.api-service.enable":"false","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","fs.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","yarn.nodemanager.container.stderr.tail.bytes":"4096","hadoop.security.group.mapping.ldap.read.timeout.ms":"60000","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","mapreduce.outputcommitter.factory.scheme.s3a":"org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory","hadoop.security.groups.cache.background.reload":"false","yarn.nodemanager.container-monitor.enabled":"true","yarn.nodemanager.elastic-memory-control.enabled":"false","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","yarn.resourcemanager.rm.container-allocation.expiry-interval-ms":"600000","mapreduce.fileoutputcommitter.algorithm.version":"1","yarn.resourcemanager.work-preserving-recovery.enabled":"true","mapreduce.map.skip.maxrecords":"0","yarn.sharedcache.root-dir":"/sharedcache","fs.s3a.retry.throttle.limit":"${fs.s3a.attempts.maximum}","hadoop.http.authentication.type":"simple","mapreduce.job.cache.limit.max-resources":"0","mapreduce.task.userlog.limit.kb":"0","yarn.resourcemanager.scheduler.monitor.enable":"false","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","yarn.nodemanager.resource-monitor.interval-ms":"3000","yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices":"auto","mapreduce.job.sharedcache.mode":"disabled","yarn.nodemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.shuffle.listen.queue.size":"128","yarn.scheduler.configuration.mutation.acl-policy.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy","mapreduce.map.cpu.vcores":"1","yarn.log-aggregation.file-formats":"TFile","yarn.timeline-service.client.fd-retain-secs":"300","hadoop.user.group.static.mapping.overrides":"dr.who=;","fs.azure.sas.expiry.period":"90d","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","yarn.router.webapp.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST","yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage":"90.0","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","yarn.nodemanager.disk-validator":"basic","yarn.app.mapreduce.client.job.max-retries":"3","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","hadoop.security.group.mapping.ldap.connection.timeout.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","yarn.nodemanager.container-diagnostics-maximum-size":"10000","hadoop.registry.zk.retry.interval.ms":"1000","yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms":"1000","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","fs.s3a.assumed.role.session.duration":"30m","hadoop.security.group.mapping.ldap.conversion.rule":"none","hadoop.ssl.server.conf":"ssl-server.xml","fs.s3a.retry.throttle.interval":"1000ms","seq.io.sort.factor":"100","yarn.sharedcache.cleaner.initial-delay-mins":"10","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.nodemanager.runtime.linux.docker.capabilities":"CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE","yarn.acl.enable":"false","yarn.timeline-service.entity-group-fs-store.done-dir":"/tmp/entity-file-history/done/","mapreduce.task.profile":"false","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","mapreduce.jobhistory.always-scan-user-dir":"false","yarn.nodemanager.opportunistic-containers-use-pause-for-preemption":"false","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","yarn.timeline-service.reader.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold":"1","yarn.resourcemanager.configuration.file-system-based-store":"/yarn/conf","mapreduce.job.cache.limit.max-single-resource-mb":"0","yarn.nodemanager.runtime.linux.docker.stop.grace-period":"10","yarn.resourcemanager.resource-profiles.source-file":"resource-profiles.json","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","fs.automatic.close":"true","yarn.nodemanager.hostname":"0.0.0.0","yarn.nodemanager.resource.memory.cgroups.swappiness":"0","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","yarn.timeline-service.app-aggregation-interval-secs":"15","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","yarn.nodemanager.container-localizer.log.level":"INFO","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60","mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","yarn.resourcemanager.delegation.token.max-lifetime":"*********(redacted)","yarn.timeline-service.client.drain-entities.timeout.ms":"2000","yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin","yarn.timeline-service.entity-group-fs-store.summary-store":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.reduce.cpu.vcores":"1","mapreduce.job.encrypted-intermediate-data.buffer.kb":"128","fs.client.resolve.remote.symlinks":"true","yarn.nodemanager.webapp.https.address":"0.0.0.0:8044","hadoop.http.cross-origin.allowed-origins":"*","mapreduce.job.encrypted-intermediate-data":"false","yarn.timeline-service.entity-group-fs-store.retain-seconds":"604800","yarn.resourcemanager.metrics.runtime.buckets":"60,300,1440","yarn.timeline-service.generic-application-history.max-applications":"10000","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","yarn.node-labels.configuration-type":"centralized","fs.s3a.path.style.access":"false","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","yarn.sharedcache.store.in-memory.staleness-period-mins":"10080","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.nodemanager.minimum.version":"NONE","mapreduce.jobhistory.webapp.xfs-filter.xframe-options":"SAMEORIGIN","yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled":"false","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","yarn.timeline-service.reader.webapp.https.address":"${yarn.timeline-service.webapp.https.address}","fs.ftp.data.connection.mode":"ACTIVE_LOCAL_DATA_CONNECTION_MODE","mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed":"true","yarn.scheduler.maximum-allocation-vcores":"4","hadoop.http.cross-origin.allowed-headers":"X-Requested-With,Content-Type,Accept,Origin","yarn.nodemanager.log-aggregation.compression-type":"none","yarn.timeline-service.version":"1.0f","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","mapreduce.reduce.maxattempts":"4","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","mapreduce.job.running.reduce.limit":"0","ipc.maximum.response.length":"134217728","yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","hadoop.caller.context.max.size":"128","yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed":"false","yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed":"false","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","fs.s3a.max.total.tasks":"5","fs.s3a.readahead.range":"64K","hadoop.http.authentication.simple.anonymous.allowed":"true","fs.s3a.attempts.maximum":"20","hadoop.registry.zk.connection.timeout.ms":"15000","yarn.resourcemanager.delegation-token-renewer.thread-count":"*********(redacted)","yarn.nodemanager.health-checker.script.timeout-ms":"1200000","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","yarn.resourcemanager.resource-profiles.enabled":"false","yarn.timeline-service.hbase-schema.prefix":"prod.","fs.azure.authorization":"false","mapreduce.map.log.level":"INFO","yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs":"20","mapreduce.output.fileoutputformat.compress.type":"RECORD","yarn.resourcemanager.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.timeline-service.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.ifile.readahead.bytes":"4194304","yarn.sharedcache.app-checker.class":"org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"true","yarn.nodemanager.resource.detect-hardware-capabilities":"false","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","hadoop.security.group.mapping.ldap.search.group.hierarchy.levels":"0","yarn.resourcemanager.fs.state-store.retry-interval-ms":"1000","file.stream-buffer-size":"4096","yarn.resourcemanager.application-timeouts.monitor.interval-ms":"3000","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.map.speculative":"true","mapreduce.job.speculative.retry-after-speculate":"15000","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","yarn.app.mapreduce.am.log.level":"INFO","mapreduce.job.reduce.slowstart.completedmaps":"0.05","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices":"0,1","yarn.timeline-service.client.internal-timers-ttl-secs":"420","hadoop.http.logs.enabled":"true","fs.s3a.block.size":"32M","yarn.sharedcache.client-server.address":"0.0.0.0:8045","yarn.nodemanager.logaggregation.threadpool-size-max":"100","yarn.resourcemanager.hostname":"0.0.0.0","yarn.resourcemanager.delegation.key.update-interval":"86400000","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"-1","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","yarn.nodemanager.disk-health-checker.enable":"true","ipc.client.tcpnodelay":"true","ipc.client.rpc-timeout.ms":"0","yarn.nodemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","ipc.client.low-latency":"false","mapreduce.input.lineinputformat.linespermap":"1","yarn.router.interceptor.user.threadpool-size":"5","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.timeline-service.entity-group-fs-store.app-cache-size":"10","fs.s3a.socket.recv.buffer":"8192","yarn.resourcemanager.resource-tracker.address":"${yarn.resourcemanager.hostname}:8031","yarn.nodemanager.node-labels.provider.fetch-timeout-ms":"1200000","mapreduce.job.heap.memory-mb.ratio":"0.8","yarn.resourcemanager.leveldb-state-store.compaction-interval-secs":"3600","yarn.resourcemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","yarn.scheduler.configuration.fs.path":"file://${hadoop.tmp.dir}/yarn/system/schedconf","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.http.lib.StaticUserWebFilter","mapreduce.reduce.memory.mb":"-1","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","yarn.nodemanager.container-metrics.period-ms":"-1","mapreduce.fileoutputcommitter.task.cleanup.enabled":"false","yarn.nodemanager.log.retain-seconds":"10800","yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds":"3600","yarn.resourcemanager.keytab":"/etc/krb5.keytab","hadoop.security.group.mapping.providers.combined":"true","mapreduce.reduce.merge.inmem.threshold":"1000","yarn.timeline-service.recovery.enabled":"false","fs.azure.saskey.usecontainersaskeyforallaccess":"true","yarn.sharedcache.nm.uploader.thread-count":"20","yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs":"3600","mapreduce.shuffle.ssl.enabled":"false","yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds":"259200000","fs.s3a.committer.staging.abort.pending.uploads":"true","yarn.nodemanager.opportunistic-containers-max-queue-length":"0","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","mapreduce.job.speculative.minimum-allowed-tasks":"10","yarn.log-aggregation.retain-seconds":"-1","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.http.cross-origin.allowed-methods":"GET,POST,HEAD","yarn.resourcemanager.opportunistic-container-allocation.enabled":"false","mapreduce.jobhistory.webapp.address":"0.0.0.0:19888","hadoop.system.tags":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.log-aggregation.file-controller.TFile.class":"org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"${yarn.resourcemanager.hostname}:8088","mapreduce.jobhistory.recovery.enable":"false","mapreduce.reduce.shuffle.parallelcopies":"5","fs.AbstractFileSystem.webhdfs.impl":"org.apache.hadoop.fs.WebHdfs","fs.trash.interval":"0","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","yarn.app.mapreduce.am.resource.mb":"1536","mapreduce.input.fileinputformat.list-status.num-threads":"1","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.resourcemanager.nm-container-queuing.min-queue-length":"5","yarn.nodemanager.resource.cpu-vcores":"-1","mapreduce.job.reduces":"1","fs.s3a.multipart.size":"100M","yarn.scheduler.minimum-allocation-vcores":"1","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","mapreduce.job.encrypted-intermediate-data-key-size-bits":"128","fs.s3a.metadatastore.authoritative":"false","yarn.nodemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","ha.health-monitor.sleep-after-disconnect.ms":"1000","yarn.app.mapreduce.shuffle.log.limit.kb":"0","hadoop.security.group.mapping":"org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback","yarn.client.application-client-protocol.poll-timeout-ms":"-1","mapreduce.jobhistory.jhist.format":"binary","yarn.resourcemanager.ha.enabled":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.jobhistory.intermediate-user-done-dir.permissions":"770","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","yarn.resourcemanager.reservation-system.enable":"false","mapreduce.map.output.compress":"false","ha.zookeeper.acl":"world:anyone:rwcda","ipc.server.max.connections":"0","yarn.nodemanager.runtime.linux.docker.default-container-network":"host","yarn.router.webapp.address":"0.0.0.0:8089","yarn.scheduler.maximum-allocation-mb":"8192","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.sharedcache.cleaner.period-mins":"1440","yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint":"http://localhost:3476/v1.0/docker/cli","yarn.app.mapreduce.am.container.log.limit.kb":"0","ipc.client.connect.retry.interval":"1000","yarn.timeline-service.http-cross-origin.enabled":"false","fs.wasbs.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure","yarn.federation.subcluster-resolver.class":"org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","mapreduce.jobhistory.cleaner.enable":"true","yarn.timeline-service.client.fd-flush-interval-secs":"10","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms":"1000","fs.s3a.committer.staging.tmp.path":"tmp/staging","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms":"10","mapreduce.job.end-notification.retry.attempts":"0","yarn.nodemanager.resource.count-logical-processors-as-cores":"false","hadoop.registry.zk.root":"/registry","adl.feature.ownerandgroup.enableupn":"false","yarn.resourcemanager.zk-max-znode-size.bytes":"1048576","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","yarn.nodemanager.localizer.cache.target-size-mb":"10240","fs.s3a.committer.staging.conflict-mode":"fail","mapreduce.client.libjars.wildcard":"true","fs.s3a.committer.staging.unique-filenames":"true","yarn.nodemanager.node-attributes.provider.fetch-timeout-ms":"1200000","fs.s3a.list.version":"2","ftp.client-write-packet-size":"65536","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","hadoop.security.key.default.cipher":"AES/CTR/NoPadding","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"86400","mapreduce.job.local-fs.single-disk-limit.check.interval-ms":"5000","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","yarn.nodemanager.amrmproxy.address":"0.0.0.0:8049","ipc.server.listen.queue.size":"128","map.sort.class":"org.apache.hadoop.util.QuickSort","fs.viewfs.rename.strategy":"SAME_MOUNTPOINT","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","fs.s3a.assumed.role.credentials.provider":"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider","yarn.nodemanager.vmem-check-enabled":"true","yarn.nodemanager.numa-awareness.enabled":"false","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","yarn.federation.registry.base-dir":"yarnfederation/","mapreduce.job.max.map":"-1","mapreduce.job.local-fs.single-disk-limit.bytes":"-1","mapreduce.job.ubertask.maxreduces":"1","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.jobhistory.jobname.limit":"50","yarn.client.nodemanager-connect.retry-interval-ms":"10000","yarn.timeline-service.state-store-class":"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ","yarn.sharedcache.nested-level":"3","yarn.timeline-service.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","fs.azure.user.agent.prefix":"unknown","yarn.resourcemanager.zk-delegation-token-node.split-index":"*********(redacted)","yarn.nodemanager.numa-awareness.read-topology":"false","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","yarn.registry.class":"org.apache.hadoop.registry.client.impl.FSRegistryOperationsService","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","yarn.scheduler.queue-placement-rules":"user-group","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","yarn.resourcemanager.recovery.enabled":"false","yarn.timeline-service.webapp.rest-csrf.enabled":"false"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.version":"1.8","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Private Build","java.vm.specification.version":"1.8","user.home":"/home/user1","file.encoding.pkg":"sun.io","sun.nio.ch.bugLevel":"","sun.arch.data.model":"64","sun.boot.library.path":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64","user.dir":"/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","java.library.path":"/snap/bin/cmake:/usr/local/cuda-11.2/lib64::/usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib","sun.cpu.isalist":"","sun.desktop":"gnome","os.arch":"amd64","java.vm.version":"25.292-b10","jetty.git.hash":"238ec6997c7806b055319a6d11f8ae7564adc0de","java.endorsed.dirs":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/endorsed","java.runtime.version":"1.8.0_292-8u292-b10-0ubuntu1~18.04-b10","java.vm.info":"mixed mode","java.ext.dirs":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/ext:/usr/java/packages/lib/ext","java.runtime.name":"OpenJDK Runtime Environment","file.separator":"/","java.class.version":"52.0","scala.usejavacp":"true","java.specification.name":"Java Platform API Specification","sun.boot.class.path":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/resources.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/rt.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/sunrsasign.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jsse.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jce.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/charsets.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jfr.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/classes","file.encoding":"UTF-8","user.timezone":"America/Los_Angeles","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","os.version":"5.4.0-80-generic","sun.os.patch.level":"unknown","java.vm.specification.vendor":"Oracle Corporation","user.country":"US","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.url":"http://java.oracle.com/","java.awt.printerjob":"sun.print.PSPrinterJob","java.awt.graphicsenv":"sun.awt.X11GraphicsEnvironment","awt.toolkit":"sun.awt.X11.XToolkit","os.name":"Linux","java.vm.vendor":"Private Build","java.vendor.url.bug":"http://bugreport.sun.com/bugreport/","user.name":"user1","java.vm.name":"OpenJDK 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --conf spark.sql.sources.useV1SourceList= --conf spark.eventLog.enabled=true --conf spark.eventLog.dir=/home/user1/data_format_eventlog --class org.apache.spark.repl.Main --name Spark shell --jars /home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar spark-shell","java.home":"/usr/lib/jvm/java-8-openjdk-amd64/jre","java.version":"1.8.0_292","sun.io.unicode.encoding":"UnicodeLittle"},"Classpath Entries":{"/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/shapeless_2.12-2.3.3.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/metrics-jvm-4.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-sql_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spire-macros_2.12-0.17.0-M1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/antlr4-runtime-4.8-1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/arpack_combined_all-0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/htrace-core4-4.1.0-incubating.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/machinist_2.12-0.6.8.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jline-2.14.6.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-client-2.30.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jaxb-api-2.2.11.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-yarn-registry-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-common-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/stax2-api-3.1.4.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/osgi-resource-locator-1.0.3.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/aopalliance-1.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/janino-3.0.16.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-dataformat-yaml-2.10.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-module-scala_2.12-2.10.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/scala-collection-compat_2.12-2.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/scala-xml_2.12-1.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/leveldbjni-all-1.8.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spire-util_2.12-0.17.0-M1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/curator-recipes-2.13.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-repl_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/json4s-core_2.12-3.7.0-M5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-core-2.10.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jcl-over-slf4j-1.7.30.jar":"System Classpath","spark://user1.attlocal.net:36303/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar":"Added By User","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/httpcore-4.4.12.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-metrics-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/generex-1.0.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jpam-1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/JLargeArrays-1.5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-core_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-yarn-server-web-proxy-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/mesos-1.4.0-shaded-protobuf.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jul-to-slf4j-1.7.30.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerby-pkix-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-apiextensions-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/json4s-ast_2.12-3.7.0-M5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/protobuf-java-2.5.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-io-2.5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jta-1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-catalyst_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/activation-1.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-common-2.3.7.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/ivy-2.4.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-container-servlet-2.30.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/py4j-0.10.9.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-codec-1.10.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-admin-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-simplekdc-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/threeten-extra-1.5.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/cats-kernel_2.12-2.0.0-M4.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hk2-locator-2.6.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-hive_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spire-platform_2.12-0.17.0-M1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-network-common_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/minlog-1.3.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/token-provider-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/aopalliance-repackaged-2.6.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/conf/":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/stax-api-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerby-config-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/javassist-3.25.0-GA.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.inject-2.6.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/opencsv-2.3.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-mapreduce-client-core-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/zjsonpatch-0.3.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/univocity-parsers-2.9.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-yarn-api-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-yarn_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-text-1.6.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jsp-api-2.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-kvstore_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-media-jaxb-2.30.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/bonecp-0.8.0.RELEASE.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-networking-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/transaction-api-1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-annotations-2.10.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/datanucleus-api-jdo-4.2.4.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-autoscaling-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/velocity-1.5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-batch-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/okio-1.14.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-streaming_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/arrow-memory-netty-2.0.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-yarn-server-common-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.servlet-api-4.0.3.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-llap-common-2.3.7.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-identity-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/okhttp-3.12.12.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/xz-1.5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/compress-lzf-1.0.3.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/arrow-format-2.0.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-jdbc-2.3.7.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-launcher_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/parquet-column-1.10.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-mapreduce-client-common-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/avro-ipc-1.8.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-mllib_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/shims-0.9.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/parquet-format-2.4.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-crypto-1.1.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/re2j-1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/automaton-1.11-8.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-policy-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/flatbuffers-java-1.9.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/parquet-jackson-1.10.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-tags_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-core-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-annotations-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-auth-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/metrics-json-4.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/parquet-encoding-1.10.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/oro-2.0.8.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/avro-1.8.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jodd-core-3.5.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-admissionregistration-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/metrics-core-4.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/parquet-common-1.10.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/istack-commons-runtime-3.0.8.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/gson-2.2.4.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-module-jaxb-annotations-2.10.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-beanutils-1.9.4.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-mesos_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/snappy-java-1.1.8.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jcip-annotations-1.0-1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/log4j-1.2.17.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerby-asn1-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/accessors-smart-1.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-module-paranamer-2.10.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/xbean-asm7-shaded-4.15.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/scala-compiler-2.12.10.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-daemon-1.0.13.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-dbcp-1.4.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/orc-core-1.5.12.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/aircompressor-0.10.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/okhttp-2.7.5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-jaxrs-json-provider-2.9.5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/metrics-graphite-4.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/stream-2.9.6.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/algebra_2.12-2.0.0-M2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-mllib-local_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-service-rpc-3.1.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-rbac-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/metrics-jmx-4.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/nimbus-jose-jwt-4.41.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-kubernetes_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/slf4j-api-1.7.30.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-common-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-compiler-3.0.16.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-metastore-2.3.7.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/core-1.1.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jaxb-runtime-2.3.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/paranamer-2.8.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/guice-4.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-pool-1.5.4.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/super-csv-2.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-exec-2.3.7-core.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-logging-1.1.3.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-settings-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-network-shuffle_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-tags_2.12-3.1.1-tests.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-configuration2-2.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-common-2.30.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-certificates-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-datatype-jsr310-2.11.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/antlr-runtime-3.5.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/curator-framework-2.13.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/parquet-hadoop-1.10.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/json-1.8.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-core-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/scala-library-2.12.10.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-cli-1.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerby-util-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/JTransforms-3.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/macro-compat_2.12-1.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-databind-2.10.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-hive-thriftserver_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hk2-api-2.6.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-shims-common-2.3.7.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/json4s-scalap_2.12-3.7.0-M5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-server-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/ST4-4.0.4.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.xml.bind-api-2.3.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-graphx_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/slf4j-log4j12-1.7.30.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jdo-api-3.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.ws.rs-api-2.1.6.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-client-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-coordination-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/orc-mapreduce-1.5.12.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-collections-3.2.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-shims-2.3.7.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-httpclient-3.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/breeze-macros_2.12-1.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-storage-api-2.7.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-crypto-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/objenesis-2.6.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/orc-shims-1.5.12.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/libthrift-0.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/snakeyaml-1.24.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-mapreduce-client-jobclient-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/zstd-jni-1.4.8-1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-serde-2.3.7.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/arrow-memory-core-2.0.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/guice-servlet-4.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-jaxrs-base-2.9.5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/javax.inject-1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/datanucleus-rdbms-4.1.19.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/derby-10.12.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kryo-shaded-4.0.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hk2-utils-2.6.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-beeline-2.3.7.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-yarn-client-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/audience-annotations-0.5.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/scala-parser-combinators_2.12-1.1.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-vector-code-gen-2.3.7.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.activation-api-1.2.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-server-2.30.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-events-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-sketch_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/logging-interceptor-3.12.12.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-client-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-hk2-2.30.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-hdfs-client-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.annotation-api-1.3.5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/netty-all-4.1.51.Final.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/pyrolite-4.30.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/httpclient-4.5.6.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/javolution-5.5.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/joda-time-2.10.5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-net-3.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/avro-mapred-1.8.2-hadoop2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-apps-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/dnsjava-2.1.7.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-storageclass-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jsr305-3.0.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/RoaringBitmap-0.9.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/zookeeper-3.4.14.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/chill_2.12-0.9.5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-math3-3.4.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/ehcache-3.3.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/chill-java-0.9.5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/guava-14.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/woodstox-core-5.0.3.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-cli-2.3.7.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-yarn-common-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/json4s-jackson_2.12-3.7.0-M5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-compress-1.20.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/javax.jdo-3.2.0-m3.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/lz4-java-1.7.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-client-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-util-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-discovery-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-scheduling-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/datanucleus-core-4.1.17.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/json-smart-2.3.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-container-servlet-core-2.30.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerby-xdr-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spire_2.12-0.17.0-M1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.validation-api-2.0.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/geronimo-jcache_1.0_spec-1.0-alpha-1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/arrow-vector-2.0.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-shims-0.23-2.3.7.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-extensions-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-lang-2.6.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/libfb303-0.9.3.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/HikariCP-2.5.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-unsafe_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/breeze_2.12-1.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-lang3-3.10.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-common-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/scala-reflect-2.12.10.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/curator-client-2.13.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-shims-scheduler-2.3.7.jar":"System Classpath"}} +{"Event":"SparkListenerApplicationStart","App Name":"Spark shell","App ID":"local-1630045673160","Timestamp":1630045672277,"User":"user1"} +{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1630045686521,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"load at :23","Number of Tasks":1,"RDD Info":[{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"load at :23","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"load at :23","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:240)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:23)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:27)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:29)\n$line14.$read$$iw$$iw$$iw$$iw$$iw.(:31)\n$line14.$read$$iw$$iw$$iw$$iw.(:33)\n$line14.$read$$iw$$iw$$iw.(:35)\n$line14.$read$$iw$$iw.(:37)\n$line14.$read$$iw.(:39)\n$line14.$read.(:41)\n$line14.$read$.(:45)\n$line14.$read$.()\n$line14.$eval$.$print$lzycompute(:7)\n$line14.$eval$.$print(:6)\n$line14.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[0],"Properties":{"spark.rdd.scope.noOverride":"true","spark.rdd.scope":"{\"id\":\"2\",\"name\":\"collect\"}"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"load at :23","Number of Tasks":1,"RDD Info":[{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"load at :23","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"load at :23","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:240)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:23)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:27)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:29)\n$line14.$read$$iw$$iw$$iw$$iw$$iw.(:31)\n$line14.$read$$iw$$iw$$iw$$iw.(:33)\n$line14.$read$$iw$$iw$$iw.(:35)\n$line14.$read$$iw$$iw.(:37)\n$line14.$read$$iw.(:39)\n$line14.$read.(:41)\n$line14.$read$.(:45)\n$line14.$read$.()\n$line14.$eval$.$print$lzycompute(:7)\n$line14.$eval$.$print(:6)\n$line14.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Submission Time":1630045686548,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.rdd.scope.noOverride":"true","spark.rdd.scope":"{\"id\":\"2\",\"name\":\"collect\"}"}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1630045686745,"Executor ID":"driver","Host":"user1.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1630045686745,"Executor ID":"driver","Host":"user1.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630045687377,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"internal.metrics.executorDeserializeTime","Update":307,"Value":307,"Internal":true,"Count Failed Values":true},{"ID":1,"Name":"internal.metrics.executorDeserializeCpuTime","Update":303463911,"Value":303463911,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"internal.metrics.executorRunTime","Update":238,"Value":238,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"internal.metrics.executorCpuTime","Update":52005348,"Value":52005348,"Internal":true,"Count Failed Values":true},{"ID":4,"Name":"internal.metrics.resultSize","Update":2148,"Value":2148,"Internal":true,"Count Failed Values":true},{"ID":6,"Name":"internal.metrics.resultSerializationTime","Update":4,"Value":4,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":185115640,"JVMOffHeapMemory":150901280,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":117738,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":117738,"OffHeapUnifiedMemory":0,"DirectPoolMemory":8193,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":10,"MinorGCTime":142,"MajorGCCount":4,"MajorGCTime":820},"Task Metrics":{"Executor Deserialize Time":307,"Executor Deserialize CPU Time":303463911,"Executor Run Time":238,"Executor CPU Time":52005348,"Peak Execution Memory":0,"Result Size":2148,"JVM GC Time":0,"Result Serialization Time":4,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"load at :23","Number of Tasks":1,"RDD Info":[{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"load at :23","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"load at :23","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:240)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:23)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:27)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:29)\n$line14.$read$$iw$$iw$$iw$$iw$$iw.(:31)\n$line14.$read$$iw$$iw$$iw$$iw.(:33)\n$line14.$read$$iw$$iw$$iw.(:35)\n$line14.$read$$iw$$iw.(:37)\n$line14.$read$$iw.(:39)\n$line14.$read.(:41)\n$line14.$read$.(:45)\n$line14.$read$.()\n$line14.$eval$.$print$lzycompute(:7)\n$line14.$eval$.$print(:6)\n$line14.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Submission Time":1630045686548,"Completion Time":1630045687387,"Accumulables":[{"ID":0,"Name":"internal.metrics.executorDeserializeTime","Value":307,"Internal":true,"Count Failed Values":true},{"ID":1,"Name":"internal.metrics.executorDeserializeCpuTime","Value":303463911,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"internal.metrics.executorRunTime","Value":238,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"internal.metrics.executorCpuTime","Value":52005348,"Internal":true,"Count Failed Values":true},{"ID":4,"Name":"internal.metrics.resultSize","Value":2148,"Internal":true,"Count Failed Values":true},{"ID":6,"Name":"internal.metrics.resultSerializationTime","Value":4,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1630045687393,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":0,"description":"save at :26","details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:30)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line15.$read$$iw$$iw$$iw$$iw$$iw.(:34)\n$line15.$read$$iw$$iw$$iw$$iw.(:36)\n$line15.$read$$iw$$iw$$iw.(:38)\n$line15.$read$$iw$$iw.(:40)\n$line15.$read$$iw.(:42)\n$line15.$read.(:44)\n$line15.$read$.(:48)\n$line15.$read$.()\n$line15.$eval$.$print$lzycompute(:7)\n$line15.$eval$.$print(:6)\n$line15.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","physicalPlanDescription":"== Physical Plan ==\nExecute InsertIntoHadoopFsRelationCommand (3)\n+- * Project (2)\n +- BatchScan (1)\n\n\n(1) BatchScan\nOutput [3]: [name#0, addresses#1, testmapmap#4]\nFormat: parquet\nLocation: InMemoryFileIndex[file:/home/user1/event_logs_spark/nested_type/mapcomplex.parquet]\nReadSchema: struct>,testmapmap:map>>\n\n(2) Project [codegen id : 1]\nOutput [3]: [name#0, addresses#1, testmapmap#4]\nInput [3]: [name#0, addresses#1, testmapmap#4]\n\n(3) Execute InsertIntoHadoopFsRelationCommand\nInput [3]: [name#0, addresses#1, testmapmap#4]\nArguments: file:/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/nestedType.parquet, false, Parquet, Map(path -> nestedType.parquet), ErrorIfExists, [name, addresses, testmapmap]\n\n","sparkPlanInfo":{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand file:/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/nestedType.parquet, false, Parquet, Map(path -> nestedType.parquet), ErrorIfExists, [name, addresses, testmapmap]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [name#0, addresses#1, testmapmap#4]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BatchScan","simpleString":"BatchScan[name#0, addresses#1, testmapmap#4] ParquetScan DataFilters: [], Format: parquet, Location: InMemoryFileIndex[file:/home/user1/event_logs_spark/nested_type/mapcomplex.parquet], PartitionFilters: [], PushedFilers: [], ReadSchema: struct>,testmapmap:map:26","Number of Tasks":6,"RDD Info":[{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"6\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at :26","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"BatchScan\"}","Callsite":"save at :26","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"DataSourceRDD","Scope":"{\"id\":\"9\",\"name\":\"BatchScan\"}","Callsite":"save at :26","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:30)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line15.$read$$iw$$iw$$iw$$iw$$iw.(:34)\n$line15.$read$$iw$$iw$$iw$$iw.(:36)\n$line15.$read$$iw$$iw$$iw.(:38)\n$line15.$read$$iw$$iw.(:40)\n$line15.$read$$iw.(:42)\n$line15.$read.(:44)\n$line15.$read$.(:48)\n$line15.$read$.()\n$line15.$eval$.$print$lzycompute(:7)\n$line15.$eval$.$print(:6)\n$line15.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[1],"Properties":{"spark.sql.warehouse.dir":"file:/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/spark-warehouse/","spark.driver.host":"user1.attlocal.net","spark.eventLog.enabled":"true","spark.driver.port":"36303","spark.repl.class.uri":"spark://user1.attlocal.net:36303/classes","spark.sql.sources.useV1SourceList":"","spark.jars":"file:///home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.repl.class.outputDir":"/tmp/spark-e7f7c451-9bce-47e0-8e9c-a77f9923d5b4/repl-d8d58adf-1a53-4d01-9b63-9b2d150731ce","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"5\",\"name\":\"Execute InsertIntoHadoopFsRelationCommand\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.startTime":"1630045672277","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://user1.attlocal.net:36303/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.submit.deployMode":"client","spark.master":"local[*]","spark.home":"/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","spark.eventLog.dir":"/home/user1/data_format_eventlog","spark.sql.execution.id":"0","spark.sql.catalogImplementation":"hive","spark.repl.local.jars":"file:///home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.app.id":"local-1630045673160"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"save at :26","Number of Tasks":6,"RDD Info":[{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"6\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at :26","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"BatchScan\"}","Callsite":"save at :26","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"DataSourceRDD","Scope":"{\"id\":\"9\",\"name\":\"BatchScan\"}","Callsite":"save at :26","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:30)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line15.$read$$iw$$iw$$iw$$iw$$iw.(:34)\n$line15.$read$$iw$$iw$$iw$$iw.(:36)\n$line15.$read$$iw$$iw$$iw.(:38)\n$line15.$read$$iw$$iw.(:40)\n$line15.$read$$iw.(:42)\n$line15.$read.(:44)\n$line15.$read$.(:48)\n$line15.$read$.()\n$line15.$eval$.$print$lzycompute(:7)\n$line15.$eval$.$print(:6)\n$line15.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Submission Time":1630045690501,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/spark-warehouse/","spark.driver.host":"user1.attlocal.net","spark.eventLog.enabled":"true","spark.driver.port":"36303","spark.repl.class.uri":"spark://user1.attlocal.net:36303/classes","spark.sql.sources.useV1SourceList":"","spark.jars":"file:///home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.repl.class.outputDir":"/tmp/spark-e7f7c451-9bce-47e0-8e9c-a77f9923d5b4/repl-d8d58adf-1a53-4d01-9b63-9b2d150731ce","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"5\",\"name\":\"Execute InsertIntoHadoopFsRelationCommand\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.startTime":"1630045672277","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://user1.attlocal.net:36303/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.submit.deployMode":"client","spark.master":"local[*]","spark.home":"/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","spark.eventLog.dir":"/home/user1/data_format_eventlog","spark.sql.execution.id":"0","spark.sql.catalogImplementation":"hive","spark.repl.local.jars":"file:///home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.app.id":"local-1630045673160"}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Launch Time":1630045690541,"Executor ID":"driver","Host":"user1.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":2,"Index":1,"Attempt":0,"Launch Time":1630045690546,"Executor ID":"driver","Host":"user1.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":3,"Index":2,"Attempt":0,"Launch Time":1630045690547,"Executor ID":"driver","Host":"user1.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":4,"Index":3,"Attempt":0,"Launch Time":1630045690548,"Executor ID":"driver","Host":"user1.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":5,"Index":4,"Attempt":0,"Launch Time":1630045690549,"Executor ID":"driver","Host":"user1.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":6,"Index":5,"Attempt":0,"Launch Time":1630045690549,"Executor ID":"driver","Host":"user1.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":6,"Index":5,"Attempt":0,"Launch Time":1630045690549,"Executor ID":"driver","Host":"user1.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630045690971,"Failed":false,"Killed":false,"Accumulables":[{"ID":29,"Name":"duration","Update":"541","Value":"541","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"internal.metrics.executorDeserializeTime","Update":99,"Value":99,"Internal":true,"Count Failed Values":true},{"ID":32,"Name":"internal.metrics.executorDeserializeCpuTime","Update":51564630,"Value":51564630,"Internal":true,"Count Failed Values":true},{"ID":33,"Name":"internal.metrics.executorRunTime","Update":301,"Value":301,"Internal":true,"Count Failed Values":true},{"ID":34,"Name":"internal.metrics.executorCpuTime","Update":175645587,"Value":175645587,"Internal":true,"Count Failed Values":true},{"ID":35,"Name":"internal.metrics.resultSize","Update":2439,"Value":2439,"Internal":true,"Count Failed Values":true},{"ID":36,"Name":"internal.metrics.jvmGCTime","Update":23,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":37,"Name":"internal.metrics.resultSerializationTime","Update":6,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":52,"Name":"internal.metrics.input.bytesRead","Update":3844,"Value":3844,"Internal":true,"Count Failed Values":true},{"ID":53,"Name":"internal.metrics.input.recordsRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":99,"Executor Deserialize CPU Time":51564630,"Executor Run Time":301,"Executor CPU Time":175645587,"Peak Execution Memory":0,"Result Size":2439,"JVM GC Time":23,"Result Serialization Time":6,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":3844,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":3,"Index":2,"Attempt":0,"Launch Time":1630045690547,"Executor ID":"driver","Host":"user1.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630045691210,"Failed":false,"Killed":false,"Accumulables":[{"ID":29,"Name":"duration","Update":"507","Value":"1048","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"internal.metrics.executorDeserializeTime","Update":106,"Value":205,"Internal":true,"Count Failed Values":true},{"ID":32,"Name":"internal.metrics.executorDeserializeCpuTime","Update":50578665,"Value":102143295,"Internal":true,"Count Failed Values":true},{"ID":33,"Name":"internal.metrics.executorRunTime","Update":548,"Value":849,"Internal":true,"Count Failed Values":true},{"ID":34,"Name":"internal.metrics.executorCpuTime","Update":99380078,"Value":275025665,"Internal":true,"Count Failed Values":true},{"ID":35,"Name":"internal.metrics.resultSize","Update":2482,"Value":4921,"Internal":true,"Count Failed Values":true},{"ID":36,"Name":"internal.metrics.jvmGCTime","Update":23,"Value":46,"Internal":true,"Count Failed Values":true},{"ID":52,"Name":"internal.metrics.input.bytesRead","Update":9289,"Value":13133,"Internal":true,"Count Failed Values":true},{"ID":53,"Name":"internal.metrics.input.recordsRead","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":54,"Name":"internal.metrics.output.bytesWritten","Update":1968,"Value":1968,"Internal":true,"Count Failed Values":true},{"ID":55,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":106,"Executor Deserialize CPU Time":50578665,"Executor Run Time":548,"Executor CPU Time":99380078,"Peak Execution Memory":0,"Result Size":2482,"JVM GC Time":23,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":9289,"Records Read":1},"Output Metrics":{"Bytes Written":1968,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Launch Time":1630045690541,"Executor ID":"driver","Host":"user1.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630045691211,"Failed":false,"Killed":false,"Accumulables":[{"ID":29,"Name":"duration","Update":"478","Value":"1526","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"number of output rows","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"internal.metrics.executorDeserializeTime","Update":106,"Value":311,"Internal":true,"Count Failed Values":true},{"ID":32,"Name":"internal.metrics.executorDeserializeCpuTime","Update":55749123,"Value":157892418,"Internal":true,"Count Failed Values":true},{"ID":33,"Name":"internal.metrics.executorRunTime","Update":549,"Value":1398,"Internal":true,"Count Failed Values":true},{"ID":34,"Name":"internal.metrics.executorCpuTime","Update":407598450,"Value":682624115,"Internal":true,"Count Failed Values":true},{"ID":35,"Name":"internal.metrics.resultSize","Update":2482,"Value":7403,"Internal":true,"Count Failed Values":true},{"ID":36,"Name":"internal.metrics.jvmGCTime","Update":23,"Value":69,"Internal":true,"Count Failed Values":true},{"ID":52,"Name":"internal.metrics.input.bytesRead","Update":9329,"Value":22462,"Internal":true,"Count Failed Values":true},{"ID":53,"Name":"internal.metrics.input.recordsRead","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":54,"Name":"internal.metrics.output.bytesWritten","Update":1910,"Value":3878,"Internal":true,"Count Failed Values":true},{"ID":55,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":2,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":106,"Executor Deserialize CPU Time":55749123,"Executor Run Time":549,"Executor CPU Time":407598450,"Peak Execution Memory":0,"Result Size":2482,"JVM GC Time":23,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":9329,"Records Read":1},"Output Metrics":{"Bytes Written":1910,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":2,"Index":1,"Attempt":0,"Launch Time":1630045690546,"Executor ID":"driver","Host":"user1.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630045691214,"Failed":false,"Killed":false,"Accumulables":[{"ID":29,"Name":"duration","Update":"506","Value":"2032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"number of output rows","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"internal.metrics.executorDeserializeTime","Update":102,"Value":413,"Internal":true,"Count Failed Values":true},{"ID":32,"Name":"internal.metrics.executorDeserializeCpuTime","Update":71151632,"Value":229044050,"Internal":true,"Count Failed Values":true},{"ID":33,"Name":"internal.metrics.executorRunTime","Update":558,"Value":1956,"Internal":true,"Count Failed Values":true},{"ID":34,"Name":"internal.metrics.executorCpuTime","Update":102929213,"Value":785553328,"Internal":true,"Count Failed Values":true},{"ID":35,"Name":"internal.metrics.resultSize","Update":2482,"Value":9885,"Internal":true,"Count Failed Values":true},{"ID":36,"Name":"internal.metrics.jvmGCTime","Update":23,"Value":92,"Internal":true,"Count Failed Values":true},{"ID":52,"Name":"internal.metrics.input.bytesRead","Update":9309,"Value":31771,"Internal":true,"Count Failed Values":true},{"ID":53,"Name":"internal.metrics.input.recordsRead","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":54,"Name":"internal.metrics.output.bytesWritten","Update":1964,"Value":5842,"Internal":true,"Count Failed Values":true},{"ID":55,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":3,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":102,"Executor Deserialize CPU Time":71151632,"Executor Run Time":558,"Executor CPU Time":102929213,"Peak Execution Memory":0,"Result Size":2482,"JVM GC Time":23,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":9309,"Records Read":1},"Output Metrics":{"Bytes Written":1964,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":4,"Index":3,"Attempt":0,"Launch Time":1630045690548,"Executor ID":"driver","Host":"user1.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630045691215,"Failed":false,"Killed":false,"Accumulables":[{"ID":29,"Name":"duration","Update":"506","Value":"2538","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"number of output rows","Update":"1","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"internal.metrics.executorDeserializeTime","Update":103,"Value":516,"Internal":true,"Count Failed Values":true},{"ID":32,"Name":"internal.metrics.executorDeserializeCpuTime","Update":54187190,"Value":283231240,"Internal":true,"Count Failed Values":true},{"ID":33,"Name":"internal.metrics.executorRunTime","Update":551,"Value":2507,"Internal":true,"Count Failed Values":true},{"ID":34,"Name":"internal.metrics.executorCpuTime","Update":129069858,"Value":914623186,"Internal":true,"Count Failed Values":true},{"ID":35,"Name":"internal.metrics.resultSize","Update":2482,"Value":12367,"Internal":true,"Count Failed Values":true},{"ID":36,"Name":"internal.metrics.jvmGCTime","Update":23,"Value":115,"Internal":true,"Count Failed Values":true},{"ID":52,"Name":"internal.metrics.input.bytesRead","Update":9279,"Value":41050,"Internal":true,"Count Failed Values":true},{"ID":53,"Name":"internal.metrics.input.recordsRead","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":54,"Name":"internal.metrics.output.bytesWritten","Update":1947,"Value":7789,"Internal":true,"Count Failed Values":true},{"ID":55,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":4,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":103,"Executor Deserialize CPU Time":54187190,"Executor Run Time":551,"Executor CPU Time":129069858,"Peak Execution Memory":0,"Result Size":2482,"JVM GC Time":23,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":9279,"Records Read":1},"Output Metrics":{"Bytes Written":1947,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":5,"Index":4,"Attempt":0,"Launch Time":1630045690549,"Executor ID":"driver","Host":"user1.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1630045691216,"Failed":false,"Killed":false,"Accumulables":[{"ID":29,"Name":"duration","Update":"521","Value":"3059","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"number of output rows","Update":"1","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"internal.metrics.executorDeserializeTime","Update":92,"Value":608,"Internal":true,"Count Failed Values":true},{"ID":32,"Name":"internal.metrics.executorDeserializeCpuTime","Update":57508252,"Value":340739492,"Internal":true,"Count Failed Values":true},{"ID":33,"Name":"internal.metrics.executorRunTime","Update":557,"Value":3064,"Internal":true,"Count Failed Values":true},{"ID":34,"Name":"internal.metrics.executorCpuTime","Update":147896964,"Value":1062520150,"Internal":true,"Count Failed Values":true},{"ID":35,"Name":"internal.metrics.resultSize","Update":2482,"Value":14849,"Internal":true,"Count Failed Values":true},{"ID":36,"Name":"internal.metrics.jvmGCTime","Update":23,"Value":138,"Internal":true,"Count Failed Values":true},{"ID":52,"Name":"internal.metrics.input.bytesRead","Update":8624,"Value":49674,"Internal":true,"Count Failed Values":true},{"ID":53,"Name":"internal.metrics.input.recordsRead","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":54,"Name":"internal.metrics.output.bytesWritten","Update":1824,"Value":9613,"Internal":true,"Count Failed Values":true},{"ID":55,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":5,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":92,"Executor Deserialize CPU Time":57508252,"Executor Run Time":557,"Executor CPU Time":147896964,"Peak Execution Memory":0,"Result Size":2482,"JVM GC Time":23,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":8624,"Records Read":1},"Output Metrics":{"Bytes Written":1824,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"save at :26","Number of Tasks":6,"RDD Info":[{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"6\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at :26","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"BatchScan\"}","Callsite":"save at :26","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"DataSourceRDD","Scope":"{\"id\":\"9\",\"name\":\"BatchScan\"}","Callsite":"save at :26","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:30)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line15.$read$$iw$$iw$$iw$$iw$$iw.(:34)\n$line15.$read$$iw$$iw$$iw$$iw.(:36)\n$line15.$read$$iw$$iw$$iw.(:38)\n$line15.$read$$iw$$iw.(:40)\n$line15.$read$$iw.(:42)\n$line15.$read.(:44)\n$line15.$read$.(:48)\n$line15.$read$.()\n$line15.$eval$.$print$lzycompute(:7)\n$line15.$eval$.$print(:6)\n$line15.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Submission Time":1630045690501,"Completion Time":1630045691217,"Accumulables":[{"ID":29,"Name":"duration","Value":"3059","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"number of output rows","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"internal.metrics.executorDeserializeTime","Value":608,"Internal":true,"Count Failed Values":true},{"ID":32,"Name":"internal.metrics.executorDeserializeCpuTime","Value":340739492,"Internal":true,"Count Failed Values":true},{"ID":33,"Name":"internal.metrics.executorRunTime","Value":3064,"Internal":true,"Count Failed Values":true},{"ID":34,"Name":"internal.metrics.executorCpuTime","Value":1062520150,"Internal":true,"Count Failed Values":true},{"ID":35,"Name":"internal.metrics.resultSize","Value":14849,"Internal":true,"Count Failed Values":true},{"ID":36,"Name":"internal.metrics.jvmGCTime","Value":138,"Internal":true,"Count Failed Values":true},{"ID":37,"Name":"internal.metrics.resultSerializationTime","Value":6,"Internal":true,"Count Failed Values":true},{"ID":52,"Name":"internal.metrics.input.bytesRead","Value":49674,"Internal":true,"Count Failed Values":true},{"ID":53,"Name":"internal.metrics.input.recordsRead","Value":5,"Internal":true,"Count Failed Values":true},{"ID":54,"Name":"internal.metrics.output.bytesWritten","Value":9613,"Internal":true,"Count Failed Values":true},{"ID":55,"Name":"internal.metrics.output.recordsWritten","Value":5,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":1,"Completion Time":1630045691217,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[25,5],[26,9613],[27,5],[28,0]]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":0,"time":1630045691247} +{"Event":"SparkListenerApplicationEnd","Timestamp":1630045693477} diff --git a/tools/src/test/resources/spark-events-qualification/nested_type_eventlog b/tools/src/test/resources/spark-events-qualification/nested_type_eventlog new file mode 100644 index 00000000000..bcbb0739959 --- /dev/null +++ b/tools/src/test/resources/spark-events-qualification/nested_type_eventlog @@ -0,0 +1,33 @@ +{"Event":"SparkListenerLogStart","Spark Version":"3.1.1"} +{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"cores":{"Resource Name":"cores","Amount":1,"Discovery Script":"","Vendor":""},"memory":{"Resource Name":"memory","Amount":1024,"Discovery Script":"","Vendor":""},"offHeap":{"Resource Name":"offHeap","Amount":0,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1629446107028,"Executor ID":"driver","Executor Info":{"Host":"nartal.attlocal.net","Total Cores":8,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"nartal.attlocal.net","Port":34251},"Maximum Memory":384093388,"Timestamp":1629446107049,"Maximum Onheap Memory":384093388,"Maximum Offheap Memory":0} +{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-8-openjdk-amd64/jre","Java Version":"1.8.0_292 (Private Build)","Scala Version":"version 2.12.10"},"Spark Properties":{"spark.driver.host":"nartal.attlocal.net","spark.eventLog.enabled":"true","spark.driver.port":"40637","spark.repl.class.uri":"spark://nartal.attlocal.net:40637/classes","spark.jars":"file:///home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.repl.class.outputDir":"/tmp/spark-90f7e1c4-0b66-4c35-95ba-b0df2a943d36/repl-cf3d7009-52a2-41f6-8ee9-3bef87fa92d7","spark.app.name":"Spark shell","spark.scheduler.mode":"FIFO","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.startTime":"1629446105772","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://nartal.attlocal.net:40637/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.submit.deployMode":"client","spark.master":"local[*]","spark.home":"/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","spark.eventLog.dir":"/home/nartal/data_format_eventlog","spark.sql.catalogImplementation":"hive","spark.repl.local.jars":"file:///home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.app.id":"local-1629446106683"},"Hadoop Properties":{"hadoop.service.shutdown.timeout":"30s","yarn.resourcemanager.amlauncher.thread-count":"50","yarn.sharedcache.enabled":"false","fs.s3a.connection.maximum":"15","yarn.nodemanager.numa-awareness.numactl.cmd":"/usr/bin/numactl","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","yarn.timeline-service.timeline-client.number-of-async-entities-to-merge":"10","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.jobhistory.loadedjob.tasks.max":"-1","mapreduce.framework.name":"local","yarn.sharedcache.uploader.server.thread-count":"50","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","hadoop.security.groups.cache.background.reload.threads":"3","yarn.resourcemanager.webapp.cross-origin.enabled":"false","fs.AbstractFileSystem.ftp.impl":"org.apache.hadoop.fs.ftp.FtpFs","hadoop.registry.secure":"false","hadoop.shell.safely.delete.limit.num.files":"100","dfs.bytes-per-checksum":"512","mapreduce.job.acl-view-job":" ","fs.s3a.s3guard.ddb.background.sleep":"25ms","fs.s3a.retry.limit":"${fs.s3a.attempts.maximum}","mapreduce.jobhistory.loadedjobs.cache.size":"5","fs.s3a.s3guard.ddb.table.create":"false","yarn.nodemanager.amrmproxy.enabled":"false","yarn.timeline-service.entity-group-fs-store.with-user-dir":"false","mapreduce.input.fileinputformat.split.minsize":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","yarn.sharedcache.checksum.algo.impl":"org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl","yarn.nodemanager.amrmproxy.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor","yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size":"10485760","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","mapreduce.task.profile.maps":"0-2","yarn.scheduler.include-port-in-node-name":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","yarn.resourcemanager.node-removal-untracked.timeout-ms":"60000","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","mapreduce.jobhistory.webapp.https.address":"0.0.0.0:19890","yarn.node-labels.fs-store.impl.class":"org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore","yarn.nodemanager.collector-service.address":"${yarn.nodemanager.hostname}:8048","fs.trash.checkpoint.interval":"0","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","yarn.resourcemanager.node-ip-cache.expiry-interval-secs":"-1","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","yarn.resourcemanager.placement-constraints.handler":"disabled","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"1000","yarn.resourcemanager.system-metrics-publisher.enabled":"false","yarn.resourcemanager.placement-constraints.algorithm.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm","yarn.sharedcache.webapp.address":"0.0.0.0:8788","yarn.resourcemanager.delegation.token.renew-interval":"*********(redacted)","yarn.sharedcache.nm.uploader.replication.factor":"10","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","mapreduce.reduce.skip.proc-count.auto-incr":"true","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","ipc.client.fallback-to-simple-auth-allowed":"false","yarn.nodemanager.resource.memory.enforced":"true","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","yarn.acl.reservation-enable":"false","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","yarn.app.mapreduce.am.hard-kill-timeout-ms":"10000","fs.s3a.etag.checksum.enabled":"false","yarn.nodemanager.container-metrics.enable":"true","yarn.timeline-service.client.fd-clean-interval-secs":"60","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","hadoop.common.configuration.version":"3.0.0","fs.s3a.s3guard.ddb.table.capacity.read":"500","yarn.nodemanager.remote-app-log-dir-suffix":"logs","yarn.nodemanager.windows-container.cpu-limit.enabled":"false","yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed":"false","file.blocksize":"67108864","hadoop.registry.zk.retry.ceiling.ms":"60000","yarn.scheduler.configuration.leveldb-store.path":"${hadoop.tmp.dir}/yarn/system/confstore","yarn.sharedcache.store.in-memory.initial-delay-mins":"10","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","mapreduce.map.skip.proc-count.auto-incr":"true","fs.s3a.committer.name":"file","mapreduce.task.profile.reduces":"0-2","hadoop.zk.num-retries":"1000","yarn.webapp.xfs-filter.enabled":"true","seq.io.sort.mb":"100","yarn.scheduler.configuration.max.version":"100","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"${yarn.resourcemanager.hostname}:8030","yarn.node-labels.enabled":"false","yarn.resourcemanager.webapp.ui-actions.enabled":"true","mapreduce.task.timeout":"600000","yarn.sharedcache.client-server.thread-count":"50","hadoop.security.groups.shell.command.timeout":"0s","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.nodemanager.elastic-memory-control.oom-handler":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.DefaultOOMHandler","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"file:///","yarn.minicluster.use-rpc":"false","fs.har.impl.disable.cache":"true","yarn.webapp.ui2.enable":"false","io.compression.codec.bzip2.library":"system-native","yarn.nodemanager.distributed-scheduling.enabled":"false","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"${yarn.resourcemanager.hostname}:8090","mapreduce.jobhistory.address":"0.0.0.0:10020","yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.is.minicluster":"false","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","mapreduce.task.combine.progress.records":"10000","yarn.resourcemanager.epoch.range":"0","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","fs.AbstractFileSystem.wasbs.impl":"org.apache.hadoop.fs.azure.Wasbs","yarn.timeline-service.entity-group-fs-store.cache-store-class":"org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore","fs.ftp.transfer.mode":"BLOCK_TRANSFER_MODE","ipc.server.log.slow.rpc":"false","yarn.resourcemanager.node-labels.provider.fetch-interval-ms":"1800000","yarn.router.webapp.https.address":"0.0.0.0:8091","yarn.nodemanager.webapp.cross-origin.enabled":"false","fs.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","yarn.resourcemanager.auto-update.containers":"false","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","yarn.scheduler.configuration.zk-store.parent-path":"/confstore","yarn.nodemanager.default-container-executor.log-dirs.permissions":"710","yarn.app.attempt.diagnostics.limit.kc":"64","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"-1","fs.AbstractFileSystem.abfs.impl":"org.apache.hadoop.fs.azurebfs.Abfs","yarn.timeline-service.writer.flush-interval-seconds":"60","fs.s3a.fast.upload.active.blocks":"4","hadoop.security.credential.clear-text-fallback":"true","yarn.nodemanager.collector-service.thread-count":"5","fs.azure.secure.mode":"false","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","yarn.resourcemanager.fs.state-store.num-retries":"0","yarn.resourcemanager.nodemanager-connect-retries":"10","yarn.nodemanager.log-aggregation.num-log-files-per-app":"30","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","fs.s3a.committer.magic.enabled":"false","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","yarn.system-metrics-publisher.enabled":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","mapreduce.task.io.sort.factor":"10","yarn.nodemanager.amrmproxy.client.thread-count":"25","ha.failover-controller.new-active.rpc-timeout.ms":"60000","yarn.nodemanager.container-localizer.java.opts":"-Xmx256m","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","yarn.nodemanager.windows-container.memory-limit.enabled":"false","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.minicluster.fixed.ports":"false","hadoop.tags.system":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.cluster.max-application-priority":"0","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","hadoop.caller.context.signature.max.size":"40","yarn.client.load.resource-types.from-server":"false","ha.zookeeper.session-timeout.ms":"10000","tfile.io.chunk.size":"1048576","fs.s3a.s3guard.ddb.table.capacity.write":"100","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","yarn.scheduler.configuration.store.max-logs":"1000","yarn.nodemanager.node-attributes.provider.fetch-interval-ms":"600000","fs.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","yarn.nodemanager.local-cache.max-files-per-directory":"8192","hadoop.http.cross-origin.enabled":"false","hadoop.zk.acl":"world:anyone:rwcda","mapreduce.map.sort.spill.percent":"0.80","yarn.timeline-service.entity-group-fs-store.scan-interval-seconds":"60","yarn.node-attribute.fs-store.impl.class":"org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore","fs.s3a.retry.interval":"500ms","yarn.timeline-service.client.best-effort":"false","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","fs.AbstractFileSystem.swebhdfs.impl":"org.apache.hadoop.fs.SWebHdfs","yarn.nodemanager.elastic-memory-control.timeout-sec":"5","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","yarn.timeline-service.reader.webapp.address":"${yarn.timeline-service.webapp.address}","yarn.resourcemanager.placement-constraints.algorithm.pool-size":"1","yarn.timeline-service.hbase.coprocessor.jar.hdfs.location":"/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler","yarn.app.mapreduce.am.command-opts":"-Xmx1024m","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","io.mapfile.bloom.error.rate":"0.005","fs.client.resolve.topology.enabled":"false","yarn.nodemanager.runtime.linux.allowed-runtimes":"default","yarn.sharedcache.store.class":"org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","fs.s3a.metadatastore.impl":"org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore","io.skip.checksum.errors":"false","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","yarn.timeline-service.webapp.xfs-filter.xframe-options":"SAMEORIGIN","fs.s3a.connection.timeout":"200000","mapreduce.job.max.split.locations":"15","yarn.resourcemanager.nm-container-queuing.max-queue-length":"15","hadoop.registry.zk.session.timeout.ms":"60000","yarn.federation.cache-ttl.secs":"300","mapreduce.jvm.system-properties-to-log":"os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name","yarn.resourcemanager.opportunistic-container-allocation.nodes-used":"10","yarn.timeline-service.entity-group-fs-store.active-dir":"/tmp/entity-file-history/active","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","fs.AbstractFileSystem.abfss.impl":"org.apache.hadoop.fs.azurebfs.Abfss","yarn.sharedcache.uploader.server.address":"0.0.0.0:8046","yarn.resourcemanager.delegation-token.max-conf-size-bytes":"*********(redacted)","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","yarn.minicluster.yarn.nodemanager.resource.memory-mb":"4096","mapreduce.job.emit-timeline-data":"false","yarn.nodemanager.resource.system-reserved-memory-mb":"-1","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","yarn.resourcemanager.admin.client.thread-count":"1","yarn.dispatcher.drain-events.timeout":"300000","fs.s3a.buffer.dir":"${hadoop.tmp.dir}/s3a","hadoop.ssl.enabled.protocols":"TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2","mapreduce.jobhistory.admin.address":"0.0.0.0:10033","yarn.log-aggregation-status.time-out.ms":"600000","fs.s3a.assumed.role.sts.endpoint.region":"us-west-1","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","yarn.router.clientrm.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor","yarn.resourcemanager.zk-appid-node.split-index":"0","ftp.blocksize":"67108864","yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions":"read","yarn.router.rmadmin.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor","yarn.nodemanager.log-container-debug-info.enabled":"true","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms":"20","yarn.nodemanager.delete.debug-delay-sec":"0","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","yarn.nodemanager.container-retry-minimum-interval-ms":"1000","hadoop.security.groups.cache.secs":"300","yarn.federation.enabled":"false","fs.azure.local.sas.key.mode":"false","ipc.maximum.data.length":"67108864","mapreduce.shuffle.max.threads":"0","yarn.router.pipeline.cache-max-size":"25","yarn.resourcemanager.nm-container-queuing.load-comparator":"QUEUE_LENGTH","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","yarn.nodemanager.amrmproxy.ha.enable":"false","mapreduce.jobhistory.http.policy":"HTTP_ONLY","yarn.sharedcache.store.in-memory.check-period-mins":"720","hadoop.security.group.mapping.ldap.ssl":"false","yarn.client.application-client-protocol.poll-interval-ms":"200","yarn.scheduler.configuration.leveldb-store.compaction-interval-secs":"86400","yarn.timeline-service.writer.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl","ha.zookeeper.parent-znode":"/hadoop-ha","yarn.nodemanager.log-aggregation.policy.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","yarn.resourcemanager.placement-constraints.scheduler.pool-size":"1","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","yarn.nodemanager.recovery.supervised":"false","yarn.sharedcache.admin.thread-count":"1","yarn.resourcemanager.ha.automatic-failover.enabled":"true","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.resourcemanager.address":"${yarn.resourcemanager.hostname}:8032","ipc.client.ping":"true","mapreduce.task.local-fs.write-limit.bytes":"-1","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","yarn.nodemanager.resource-plugins.gpu.docker-plugin":"nvidia-docker-v1","hadoop.ssl.enabled":"false","fs.s3a.multipart.purge":"false","yarn.scheduler.configuration.store.class":"file","yarn.resourcemanager.nm-container-queuing.queue-limit-stdev":"1.0f","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","ipc.client.bind.wildcard.addr":"false","yarn.resourcemanager.webapp.rest-csrf.enabled":"false","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"10","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","yarn.nodemanager.runtime.linux.docker.allowed-container-networks":"host,none,bridge","yarn.nodemanager.node-labels.resync-interval-ms":"120000","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","mapreduce.job.maps":"2","mapreduce.jobhistory.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","yarn.rm.system-metrics-publisher.emit-container-events":"false","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","fs.s3a.s3guard.ddb.throttle.retry.interval":"100ms","fs.AbstractFileSystem.wasb.impl":"org.apache.hadoop.fs.azure.Wasb","mapreduce.client.submit.file.replication":"10","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"2147483647","yarn.resourcemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold":"1","yarn.nodemanager.webapp.rest-csrf.enabled":"false","fs.ftp.host.port":"21","ipc.ping.interval":"60000","yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size":"10","yarn.resourcemanager.admin.address":"${yarn.resourcemanager.hostname}:8033","file.client-write-packet-size":"65536","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"true","hadoop.security.key.default.bitlength":"128","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","yarn.nodemanager.log.deletion-threads-count":"4","yarn.webapp.filter-entity-list-by-user":"false","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"100","yarn.nodemanager.localizer.client.thread-count":"5","io.erasurecode.codec.rs.rawcoders":"rs_native,rs_java","io.erasurecode.codec.rs-legacy.rawcoders":"rs-legacy_java","yarn.sharedcache.admin.address":"0.0.0.0:8047","yarn.resourcemanager.placement-constraints.algorithm.iterator":"SERIAL","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec","mapreduce.job.cache.limit.max-resources-mb":"0","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"5000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","hadoop.workaround.non.threadsafe.getpwuid":"true","fs.df.interval":"60000","fs.s3a.multiobjectdelete.enable":"true","yarn.sharedcache.cleaner.resource-sleep-ms":"0","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"false","io.file.buffer.size":"65536","hadoop.security.group.mapping.ldap.search.attr.member":"member","hadoop.security.random.device.file.path":"/dev/urandom","hadoop.security.sensitive-config-keys":"*********(redacted)","fs.s3a.s3guard.ddb.max.retries":"9","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","yarn.intermediate-data-encryption.enable":"false","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.nodemanager.container.stderr.pattern":"{*stderr*,*STDERR*}","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/tmp/hadoop-yarn/staging","mapreduce.reduce.shuffle.read.timeout":"180000","hadoop.http.cross-origin.max-age":"1800","io.erasurecode.codec.xor.rawcoders":"xor_native,xor_java","fs.s3a.connection.establish.timeout":"5000","mapreduce.job.running.map.limit":"0","yarn.minicluster.control-resource-monitoring":"false","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","yarn.federation.state-store.class":"org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","adl.http.timeout":"-1","yarn.resourcemanager.placement-constraints.retry-attempts":"3","hadoop.caller.context.enabled":"false","yarn.nodemanager.vmem-pmem-ratio":"2.1","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","hadoop.zk.timeout-ms":"10000","fs.s3a.s3guard.cli.prune.age":"86400000","yarn.nodemanager.resource.pcores-vcores-multiplier":"1.0","yarn.nodemanager.runtime.linux.sandbox-mode":"disabled","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.s3a.committer.threads":"8","hadoop.zk.retry-interval-ms":"1000","hadoop.security.crypto.buffer.size":"8192","yarn.nodemanager.node-labels.provider.fetch-interval-ms":"600000","mapreduce.jobhistory.recovery.store.leveldb.path":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.client.failover-retries-on-socket-timeouts":"0","yarn.nodemanager.resource.memory.enabled":"false","fs.azure.authorization.caching.enable":"true","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","mapreduce.job.finish-when-all-reducers-done":"true","hadoop.registry.jaas.context":"Client","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","io.map.index.interval":"128","yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms":"100","fs.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","mapreduce.job.counters.max":"120","mapreduce.jobhistory.webapp.rest-csrf.enabled":"false","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","yarn.timeline-service.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/timeline","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","hadoop.security.auth_to_local.mechanism":"hadoop","yarn.timeline-service.app-collector.linger-period.ms":"60000","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.reservation-system.planfollower.time-step":"1000","yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed":"true","yarn.webapp.api-service.enable":"false","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","fs.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","yarn.nodemanager.container.stderr.tail.bytes":"4096","hadoop.security.group.mapping.ldap.read.timeout.ms":"60000","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","mapreduce.outputcommitter.factory.scheme.s3a":"org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory","hadoop.security.groups.cache.background.reload":"false","yarn.nodemanager.container-monitor.enabled":"true","yarn.nodemanager.elastic-memory-control.enabled":"false","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","yarn.resourcemanager.rm.container-allocation.expiry-interval-ms":"600000","mapreduce.fileoutputcommitter.algorithm.version":"1","yarn.resourcemanager.work-preserving-recovery.enabled":"true","mapreduce.map.skip.maxrecords":"0","yarn.sharedcache.root-dir":"/sharedcache","fs.s3a.retry.throttle.limit":"${fs.s3a.attempts.maximum}","hadoop.http.authentication.type":"simple","mapreduce.job.cache.limit.max-resources":"0","mapreduce.task.userlog.limit.kb":"0","yarn.resourcemanager.scheduler.monitor.enable":"false","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","yarn.nodemanager.resource-monitor.interval-ms":"3000","yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices":"auto","mapreduce.job.sharedcache.mode":"disabled","yarn.nodemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.shuffle.listen.queue.size":"128","yarn.scheduler.configuration.mutation.acl-policy.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy","mapreduce.map.cpu.vcores":"1","yarn.log-aggregation.file-formats":"TFile","yarn.timeline-service.client.fd-retain-secs":"300","hadoop.user.group.static.mapping.overrides":"dr.who=;","fs.azure.sas.expiry.period":"90d","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","yarn.router.webapp.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST","yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage":"90.0","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","yarn.nodemanager.disk-validator":"basic","yarn.app.mapreduce.client.job.max-retries":"3","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","hadoop.security.group.mapping.ldap.connection.timeout.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","yarn.nodemanager.container-diagnostics-maximum-size":"10000","hadoop.registry.zk.retry.interval.ms":"1000","yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms":"1000","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","fs.s3a.assumed.role.session.duration":"30m","hadoop.security.group.mapping.ldap.conversion.rule":"none","hadoop.ssl.server.conf":"ssl-server.xml","fs.s3a.retry.throttle.interval":"1000ms","seq.io.sort.factor":"100","yarn.sharedcache.cleaner.initial-delay-mins":"10","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.nodemanager.runtime.linux.docker.capabilities":"CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE","yarn.acl.enable":"false","yarn.timeline-service.entity-group-fs-store.done-dir":"/tmp/entity-file-history/done/","mapreduce.task.profile":"false","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","mapreduce.jobhistory.always-scan-user-dir":"false","yarn.nodemanager.opportunistic-containers-use-pause-for-preemption":"false","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","yarn.timeline-service.reader.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold":"1","yarn.resourcemanager.configuration.file-system-based-store":"/yarn/conf","mapreduce.job.cache.limit.max-single-resource-mb":"0","yarn.nodemanager.runtime.linux.docker.stop.grace-period":"10","yarn.resourcemanager.resource-profiles.source-file":"resource-profiles.json","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","fs.automatic.close":"true","yarn.nodemanager.hostname":"0.0.0.0","yarn.nodemanager.resource.memory.cgroups.swappiness":"0","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","yarn.timeline-service.app-aggregation-interval-secs":"15","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","yarn.nodemanager.container-localizer.log.level":"INFO","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60","mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","yarn.resourcemanager.delegation.token.max-lifetime":"*********(redacted)","yarn.timeline-service.client.drain-entities.timeout.ms":"2000","yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin","yarn.timeline-service.entity-group-fs-store.summary-store":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.reduce.cpu.vcores":"1","mapreduce.job.encrypted-intermediate-data.buffer.kb":"128","fs.client.resolve.remote.symlinks":"true","yarn.nodemanager.webapp.https.address":"0.0.0.0:8044","hadoop.http.cross-origin.allowed-origins":"*","mapreduce.job.encrypted-intermediate-data":"false","yarn.timeline-service.entity-group-fs-store.retain-seconds":"604800","yarn.resourcemanager.metrics.runtime.buckets":"60,300,1440","yarn.timeline-service.generic-application-history.max-applications":"10000","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","yarn.node-labels.configuration-type":"centralized","fs.s3a.path.style.access":"false","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","yarn.sharedcache.store.in-memory.staleness-period-mins":"10080","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.nodemanager.minimum.version":"NONE","mapreduce.jobhistory.webapp.xfs-filter.xframe-options":"SAMEORIGIN","yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled":"false","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","yarn.timeline-service.reader.webapp.https.address":"${yarn.timeline-service.webapp.https.address}","fs.ftp.data.connection.mode":"ACTIVE_LOCAL_DATA_CONNECTION_MODE","mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed":"true","yarn.scheduler.maximum-allocation-vcores":"4","hadoop.http.cross-origin.allowed-headers":"X-Requested-With,Content-Type,Accept,Origin","yarn.nodemanager.log-aggregation.compression-type":"none","yarn.timeline-service.version":"1.0f","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","mapreduce.reduce.maxattempts":"4","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","mapreduce.job.running.reduce.limit":"0","ipc.maximum.response.length":"134217728","yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","hadoop.caller.context.max.size":"128","yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed":"false","yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed":"false","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","fs.s3a.max.total.tasks":"5","fs.s3a.readahead.range":"64K","hadoop.http.authentication.simple.anonymous.allowed":"true","fs.s3a.attempts.maximum":"20","hadoop.registry.zk.connection.timeout.ms":"15000","yarn.resourcemanager.delegation-token-renewer.thread-count":"*********(redacted)","yarn.nodemanager.health-checker.script.timeout-ms":"1200000","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","yarn.resourcemanager.resource-profiles.enabled":"false","yarn.timeline-service.hbase-schema.prefix":"prod.","fs.azure.authorization":"false","mapreduce.map.log.level":"INFO","yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs":"20","mapreduce.output.fileoutputformat.compress.type":"RECORD","yarn.resourcemanager.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.timeline-service.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.ifile.readahead.bytes":"4194304","yarn.sharedcache.app-checker.class":"org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"true","yarn.nodemanager.resource.detect-hardware-capabilities":"false","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","hadoop.security.group.mapping.ldap.search.group.hierarchy.levels":"0","yarn.resourcemanager.fs.state-store.retry-interval-ms":"1000","file.stream-buffer-size":"4096","yarn.resourcemanager.application-timeouts.monitor.interval-ms":"3000","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.map.speculative":"true","mapreduce.job.speculative.retry-after-speculate":"15000","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","yarn.app.mapreduce.am.log.level":"INFO","mapreduce.job.reduce.slowstart.completedmaps":"0.05","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices":"0,1","yarn.timeline-service.client.internal-timers-ttl-secs":"420","hadoop.http.logs.enabled":"true","fs.s3a.block.size":"32M","yarn.sharedcache.client-server.address":"0.0.0.0:8045","yarn.nodemanager.logaggregation.threadpool-size-max":"100","yarn.resourcemanager.hostname":"0.0.0.0","yarn.resourcemanager.delegation.key.update-interval":"86400000","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"-1","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","yarn.nodemanager.disk-health-checker.enable":"true","ipc.client.tcpnodelay":"true","ipc.client.rpc-timeout.ms":"0","yarn.nodemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","ipc.client.low-latency":"false","mapreduce.input.lineinputformat.linespermap":"1","yarn.router.interceptor.user.threadpool-size":"5","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.timeline-service.entity-group-fs-store.app-cache-size":"10","fs.s3a.socket.recv.buffer":"8192","yarn.resourcemanager.resource-tracker.address":"${yarn.resourcemanager.hostname}:8031","yarn.nodemanager.node-labels.provider.fetch-timeout-ms":"1200000","mapreduce.job.heap.memory-mb.ratio":"0.8","yarn.resourcemanager.leveldb-state-store.compaction-interval-secs":"3600","yarn.resourcemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","yarn.scheduler.configuration.fs.path":"file://${hadoop.tmp.dir}/yarn/system/schedconf","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.http.lib.StaticUserWebFilter","mapreduce.reduce.memory.mb":"-1","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","yarn.nodemanager.container-metrics.period-ms":"-1","mapreduce.fileoutputcommitter.task.cleanup.enabled":"false","yarn.nodemanager.log.retain-seconds":"10800","yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds":"3600","yarn.resourcemanager.keytab":"/etc/krb5.keytab","hadoop.security.group.mapping.providers.combined":"true","mapreduce.reduce.merge.inmem.threshold":"1000","yarn.timeline-service.recovery.enabled":"false","fs.azure.saskey.usecontainersaskeyforallaccess":"true","yarn.sharedcache.nm.uploader.thread-count":"20","yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs":"3600","mapreduce.shuffle.ssl.enabled":"false","yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds":"259200000","fs.s3a.committer.staging.abort.pending.uploads":"true","yarn.nodemanager.opportunistic-containers-max-queue-length":"0","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","mapreduce.job.speculative.minimum-allowed-tasks":"10","yarn.log-aggregation.retain-seconds":"-1","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.http.cross-origin.allowed-methods":"GET,POST,HEAD","yarn.resourcemanager.opportunistic-container-allocation.enabled":"false","mapreduce.jobhistory.webapp.address":"0.0.0.0:19888","hadoop.system.tags":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.log-aggregation.file-controller.TFile.class":"org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"${yarn.resourcemanager.hostname}:8088","mapreduce.jobhistory.recovery.enable":"false","mapreduce.reduce.shuffle.parallelcopies":"5","fs.AbstractFileSystem.webhdfs.impl":"org.apache.hadoop.fs.WebHdfs","fs.trash.interval":"0","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","yarn.app.mapreduce.am.resource.mb":"1536","mapreduce.input.fileinputformat.list-status.num-threads":"1","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.resourcemanager.nm-container-queuing.min-queue-length":"5","yarn.nodemanager.resource.cpu-vcores":"-1","mapreduce.job.reduces":"1","fs.s3a.multipart.size":"100M","yarn.scheduler.minimum-allocation-vcores":"1","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","mapreduce.job.encrypted-intermediate-data-key-size-bits":"128","fs.s3a.metadatastore.authoritative":"false","yarn.nodemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","ha.health-monitor.sleep-after-disconnect.ms":"1000","yarn.app.mapreduce.shuffle.log.limit.kb":"0","hadoop.security.group.mapping":"org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback","yarn.client.application-client-protocol.poll-timeout-ms":"-1","mapreduce.jobhistory.jhist.format":"binary","yarn.resourcemanager.ha.enabled":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.jobhistory.intermediate-user-done-dir.permissions":"770","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","yarn.resourcemanager.reservation-system.enable":"false","mapreduce.map.output.compress":"false","ha.zookeeper.acl":"world:anyone:rwcda","ipc.server.max.connections":"0","yarn.nodemanager.runtime.linux.docker.default-container-network":"host","yarn.router.webapp.address":"0.0.0.0:8089","yarn.scheduler.maximum-allocation-mb":"8192","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.sharedcache.cleaner.period-mins":"1440","yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint":"http://localhost:3476/v1.0/docker/cli","yarn.app.mapreduce.am.container.log.limit.kb":"0","ipc.client.connect.retry.interval":"1000","yarn.timeline-service.http-cross-origin.enabled":"false","fs.wasbs.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure","yarn.federation.subcluster-resolver.class":"org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","mapreduce.jobhistory.cleaner.enable":"true","yarn.timeline-service.client.fd-flush-interval-secs":"10","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms":"1000","fs.s3a.committer.staging.tmp.path":"tmp/staging","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms":"10","mapreduce.job.end-notification.retry.attempts":"0","yarn.nodemanager.resource.count-logical-processors-as-cores":"false","hadoop.registry.zk.root":"/registry","adl.feature.ownerandgroup.enableupn":"false","yarn.resourcemanager.zk-max-znode-size.bytes":"1048576","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","yarn.nodemanager.localizer.cache.target-size-mb":"10240","fs.s3a.committer.staging.conflict-mode":"fail","mapreduce.client.libjars.wildcard":"true","fs.s3a.committer.staging.unique-filenames":"true","yarn.nodemanager.node-attributes.provider.fetch-timeout-ms":"1200000","fs.s3a.list.version":"2","ftp.client-write-packet-size":"65536","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","hadoop.security.key.default.cipher":"AES/CTR/NoPadding","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"86400","mapreduce.job.local-fs.single-disk-limit.check.interval-ms":"5000","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","yarn.nodemanager.amrmproxy.address":"0.0.0.0:8049","ipc.server.listen.queue.size":"128","map.sort.class":"org.apache.hadoop.util.QuickSort","fs.viewfs.rename.strategy":"SAME_MOUNTPOINT","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","fs.s3a.assumed.role.credentials.provider":"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider","yarn.nodemanager.vmem-check-enabled":"true","yarn.nodemanager.numa-awareness.enabled":"false","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","yarn.federation.registry.base-dir":"yarnfederation/","mapreduce.job.max.map":"-1","mapreduce.job.local-fs.single-disk-limit.bytes":"-1","mapreduce.job.ubertask.maxreduces":"1","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.jobhistory.jobname.limit":"50","yarn.client.nodemanager-connect.retry-interval-ms":"10000","yarn.timeline-service.state-store-class":"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ","yarn.sharedcache.nested-level":"3","yarn.timeline-service.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","fs.azure.user.agent.prefix":"unknown","yarn.resourcemanager.zk-delegation-token-node.split-index":"*********(redacted)","yarn.nodemanager.numa-awareness.read-topology":"false","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","yarn.registry.class":"org.apache.hadoop.registry.client.impl.FSRegistryOperationsService","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","yarn.scheduler.queue-placement-rules":"user-group","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","yarn.resourcemanager.recovery.enabled":"false","yarn.timeline-service.webapp.rest-csrf.enabled":"false"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.version":"1.8","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Private Build","java.vm.specification.version":"1.8","user.home":"/home/nartal","file.encoding.pkg":"sun.io","sun.nio.ch.bugLevel":"","sun.arch.data.model":"64","sun.boot.library.path":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64","user.dir":"/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","java.library.path":"/snap/bin/cmake:/usr/local/cuda-11.2/lib64::/usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib","sun.cpu.isalist":"","sun.desktop":"gnome","os.arch":"amd64","java.vm.version":"25.292-b10","jetty.git.hash":"238ec6997c7806b055319a6d11f8ae7564adc0de","java.endorsed.dirs":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/endorsed","java.runtime.version":"1.8.0_292-8u292-b10-0ubuntu1~18.04-b10","java.vm.info":"mixed mode","java.ext.dirs":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/ext:/usr/java/packages/lib/ext","java.runtime.name":"OpenJDK Runtime Environment","file.separator":"/","java.class.version":"52.0","scala.usejavacp":"true","java.specification.name":"Java Platform API Specification","sun.boot.class.path":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/resources.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/rt.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/sunrsasign.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jsse.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jce.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/charsets.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jfr.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/classes","file.encoding":"UTF-8","user.timezone":"America/Los_Angeles","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","os.version":"5.4.0-80-generic","sun.os.patch.level":"unknown","java.vm.specification.vendor":"Oracle Corporation","user.country":"US","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.url":"http://java.oracle.com/","java.awt.printerjob":"sun.print.PSPrinterJob","java.awt.graphicsenv":"sun.awt.X11GraphicsEnvironment","awt.toolkit":"sun.awt.X11.XToolkit","os.name":"Linux","java.vm.vendor":"Private Build","java.vendor.url.bug":"http://bugreport.sun.com/bugreport/","user.name":"nartal","java.vm.name":"OpenJDK 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --conf spark.eventLog.enabled=true --conf spark.eventLog.dir=/home/nartal/data_format_eventlog --class org.apache.spark.repl.Main --name Spark shell --jars /home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar spark-shell","java.home":"/usr/lib/jvm/java-8-openjdk-amd64/jre","java.version":"1.8.0_292","sun.io.unicode.encoding":"UnicodeLittle"},"Classpath Entries":{"/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/shapeless_2.12-2.3.3.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/metrics-jvm-4.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-sql_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spire-macros_2.12-0.17.0-M1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/antlr4-runtime-4.8-1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/arpack_combined_all-0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/htrace-core4-4.1.0-incubating.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/machinist_2.12-0.6.8.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jline-2.14.6.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-client-2.30.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jaxb-api-2.2.11.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-yarn-registry-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-common-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/stax2-api-3.1.4.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/osgi-resource-locator-1.0.3.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/aopalliance-1.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/janino-3.0.16.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-dataformat-yaml-2.10.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-module-scala_2.12-2.10.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/scala-collection-compat_2.12-2.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/scala-xml_2.12-1.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/leveldbjni-all-1.8.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spire-util_2.12-0.17.0-M1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/curator-recipes-2.13.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-repl_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/json4s-core_2.12-3.7.0-M5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-core-2.10.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jcl-over-slf4j-1.7.30.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/httpcore-4.4.12.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-metrics-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/generex-1.0.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jpam-1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/JLargeArrays-1.5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-core_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-yarn-server-web-proxy-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/mesos-1.4.0-shaded-protobuf.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jul-to-slf4j-1.7.30.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerby-pkix-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-apiextensions-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/json4s-ast_2.12-3.7.0-M5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/protobuf-java-2.5.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-io-2.5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jta-1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-catalyst_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/activation-1.1.1.jar":"System Classpath","spark://nartal.attlocal.net:40637/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar":"Added By User","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-common-2.3.7.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/ivy-2.4.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-container-servlet-2.30.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/py4j-0.10.9.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-codec-1.10.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-admin-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-simplekdc-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/threeten-extra-1.5.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/cats-kernel_2.12-2.0.0-M4.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hk2-locator-2.6.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-hive_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spire-platform_2.12-0.17.0-M1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-network-common_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/minlog-1.3.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/token-provider-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/aopalliance-repackaged-2.6.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/conf/":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/stax-api-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerby-config-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/javassist-3.25.0-GA.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.inject-2.6.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/opencsv-2.3.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-mapreduce-client-core-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/zjsonpatch-0.3.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/univocity-parsers-2.9.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-yarn-api-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-yarn_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-text-1.6.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jsp-api-2.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-kvstore_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-media-jaxb-2.30.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/bonecp-0.8.0.RELEASE.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-networking-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/transaction-api-1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-annotations-2.10.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/datanucleus-api-jdo-4.2.4.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-autoscaling-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/velocity-1.5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-batch-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/okio-1.14.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-streaming_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/arrow-memory-netty-2.0.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-yarn-server-common-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.servlet-api-4.0.3.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-llap-common-2.3.7.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-identity-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/okhttp-3.12.12.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/xz-1.5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/compress-lzf-1.0.3.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/arrow-format-2.0.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-jdbc-2.3.7.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-launcher_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/parquet-column-1.10.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-mapreduce-client-common-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/avro-ipc-1.8.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-mllib_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/shims-0.9.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/parquet-format-2.4.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-crypto-1.1.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/re2j-1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/automaton-1.11-8.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-policy-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/flatbuffers-java-1.9.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/parquet-jackson-1.10.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-tags_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-core-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-annotations-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-auth-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/metrics-json-4.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/parquet-encoding-1.10.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/oro-2.0.8.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/avro-1.8.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jodd-core-3.5.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-admissionregistration-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/metrics-core-4.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/parquet-common-1.10.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/istack-commons-runtime-3.0.8.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/gson-2.2.4.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-module-jaxb-annotations-2.10.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-beanutils-1.9.4.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-mesos_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/snappy-java-1.1.8.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jcip-annotations-1.0-1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/log4j-1.2.17.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerby-asn1-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/accessors-smart-1.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-module-paranamer-2.10.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/xbean-asm7-shaded-4.15.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/scala-compiler-2.12.10.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-daemon-1.0.13.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-dbcp-1.4.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/orc-core-1.5.12.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/aircompressor-0.10.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/okhttp-2.7.5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-jaxrs-json-provider-2.9.5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/metrics-graphite-4.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/stream-2.9.6.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/algebra_2.12-2.0.0-M2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-mllib-local_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-service-rpc-3.1.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-rbac-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/metrics-jmx-4.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/nimbus-jose-jwt-4.41.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-kubernetes_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/slf4j-api-1.7.30.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-common-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-compiler-3.0.16.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-metastore-2.3.7.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/core-1.1.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jaxb-runtime-2.3.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/paranamer-2.8.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/guice-4.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-pool-1.5.4.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/super-csv-2.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-exec-2.3.7-core.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-logging-1.1.3.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-settings-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-network-shuffle_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-tags_2.12-3.1.1-tests.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-configuration2-2.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-common-2.30.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-certificates-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-datatype-jsr310-2.11.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/antlr-runtime-3.5.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/curator-framework-2.13.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/parquet-hadoop-1.10.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/json-1.8.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-core-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/scala-library-2.12.10.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-cli-1.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerby-util-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/JTransforms-3.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/macro-compat_2.12-1.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-databind-2.10.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-hive-thriftserver_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hk2-api-2.6.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-shims-common-2.3.7.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/json4s-scalap_2.12-3.7.0-M5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-server-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/ST4-4.0.4.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.xml.bind-api-2.3.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-graphx_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/slf4j-log4j12-1.7.30.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jdo-api-3.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.ws.rs-api-2.1.6.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-client-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-coordination-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/orc-mapreduce-1.5.12.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-collections-3.2.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-shims-2.3.7.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-httpclient-3.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/breeze-macros_2.12-1.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-storage-api-2.7.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-crypto-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/objenesis-2.6.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/orc-shims-1.5.12.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/libthrift-0.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/snakeyaml-1.24.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-mapreduce-client-jobclient-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/zstd-jni-1.4.8-1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-serde-2.3.7.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/arrow-memory-core-2.0.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/guice-servlet-4.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-jaxrs-base-2.9.5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/javax.inject-1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/datanucleus-rdbms-4.1.19.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/derby-10.12.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kryo-shaded-4.0.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hk2-utils-2.6.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-beeline-2.3.7.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-yarn-client-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/audience-annotations-0.5.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/scala-parser-combinators_2.12-1.1.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-vector-code-gen-2.3.7.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.activation-api-1.2.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-server-2.30.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-events-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-sketch_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/logging-interceptor-3.12.12.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-client-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-hk2-2.30.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-hdfs-client-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.annotation-api-1.3.5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/netty-all-4.1.51.Final.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/pyrolite-4.30.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/httpclient-4.5.6.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/javolution-5.5.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/joda-time-2.10.5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-net-3.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/avro-mapred-1.8.2-hadoop2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-apps-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/dnsjava-2.1.7.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-storageclass-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jsr305-3.0.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/RoaringBitmap-0.9.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/zookeeper-3.4.14.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/chill_2.12-0.9.5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-math3-3.4.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/ehcache-3.3.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/chill-java-0.9.5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/guava-14.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/woodstox-core-5.0.3.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-cli-2.3.7.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-yarn-common-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/json4s-jackson_2.12-3.7.0-M5.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-compress-1.20.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/javax.jdo-3.2.0-m3.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/lz4-java-1.7.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-client-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-util-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-discovery-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-scheduling-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/datanucleus-core-4.1.17.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/json-smart-2.3.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-container-servlet-core-2.30.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerby-xdr-1.0.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spire_2.12-0.17.0-M1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.validation-api-2.0.2.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/geronimo-jcache_1.0_spec-1.0-alpha-1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/arrow-vector-2.0.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-shims-0.23-2.3.7.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-extensions-4.12.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-lang-2.6.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/libfb303-0.9.3.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/HikariCP-2.5.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-unsafe_2.12-3.1.1.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/breeze_2.12-1.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-lang3-3.10.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-common-3.2.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/scala-reflect-2.12.10.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/curator-client-2.13.0.jar":"System Classpath","/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-shims-scheduler-2.3.7.jar":"System Classpath"}} +{"Event":"SparkListenerApplicationStart","App Name":"Spark shell","App ID":"local-1629446106683","Timestamp":1629446105772,"User":"nartal"} +{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1629446114033,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"load at :23","Number of Tasks":1,"RDD Info":[{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"load at :23","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"load at :23","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:240)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:23)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:27)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:29)\n$line14.$read$$iw$$iw$$iw$$iw$$iw.(:31)\n$line14.$read$$iw$$iw$$iw$$iw.(:33)\n$line14.$read$$iw$$iw$$iw.(:35)\n$line14.$read$$iw$$iw.(:37)\n$line14.$read$$iw.(:39)\n$line14.$read.(:41)\n$line14.$read$.(:45)\n$line14.$read$.()\n$line14.$eval$.$print$lzycompute(:7)\n$line14.$eval$.$print(:6)\n$line14.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[0],"Properties":{"spark.rdd.scope.noOverride":"true","spark.rdd.scope":"{\"id\":\"2\",\"name\":\"collect\"}"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"load at :23","Number of Tasks":1,"RDD Info":[{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"load at :23","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"load at :23","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:240)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:23)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:27)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:29)\n$line14.$read$$iw$$iw$$iw$$iw$$iw.(:31)\n$line14.$read$$iw$$iw$$iw$$iw.(:33)\n$line14.$read$$iw$$iw$$iw.(:35)\n$line14.$read$$iw$$iw.(:37)\n$line14.$read$$iw.(:39)\n$line14.$read.(:41)\n$line14.$read$.(:45)\n$line14.$read$.()\n$line14.$eval$.$print$lzycompute(:7)\n$line14.$eval$.$print(:6)\n$line14.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Submission Time":1629446114059,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.rdd.scope.noOverride":"true","spark.rdd.scope":"{\"id\":\"2\",\"name\":\"collect\"}"}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1629446114215,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1629446114215,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1629446114930,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"internal.metrics.executorDeserializeTime","Update":343,"Value":343,"Internal":true,"Count Failed Values":true},{"ID":1,"Name":"internal.metrics.executorDeserializeCpuTime","Update":336799450,"Value":336799450,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"internal.metrics.executorRunTime","Update":287,"Value":287,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"internal.metrics.executorCpuTime","Update":52326880,"Value":52326880,"Internal":true,"Count Failed Values":true},{"ID":4,"Name":"internal.metrics.resultSize","Update":2148,"Value":2148,"Internal":true,"Count Failed Values":true},{"ID":6,"Name":"internal.metrics.resultSerializationTime","Update":4,"Value":4,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":343,"Executor Deserialize CPU Time":336799450,"Executor Run Time":287,"Executor CPU Time":52326880,"Peak Execution Memory":0,"Result Size":2148,"JVM GC Time":0,"Result Serialization Time":4,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"load at :23","Number of Tasks":1,"RDD Info":[{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"load at :23","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"load at :23","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:240)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:23)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:27)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:29)\n$line14.$read$$iw$$iw$$iw$$iw$$iw.(:31)\n$line14.$read$$iw$$iw$$iw$$iw.(:33)\n$line14.$read$$iw$$iw$$iw.(:35)\n$line14.$read$$iw$$iw.(:37)\n$line14.$read$$iw.(:39)\n$line14.$read.(:41)\n$line14.$read$.(:45)\n$line14.$read$.()\n$line14.$eval$.$print$lzycompute(:7)\n$line14.$eval$.$print(:6)\n$line14.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Submission Time":1629446114059,"Completion Time":1629446114941,"Accumulables":[{"ID":0,"Name":"internal.metrics.executorDeserializeTime","Value":343,"Internal":true,"Count Failed Values":true},{"ID":1,"Name":"internal.metrics.executorDeserializeCpuTime","Value":336799450,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"internal.metrics.executorRunTime","Value":287,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"internal.metrics.executorCpuTime","Value":52326880,"Internal":true,"Count Failed Values":true},{"ID":4,"Name":"internal.metrics.resultSize","Value":2148,"Internal":true,"Count Failed Values":true},{"ID":6,"Name":"internal.metrics.resultSerializationTime","Value":4,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1629446114947,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":0,"description":"save at :26","details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:30)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line15.$read$$iw$$iw$$iw$$iw$$iw.(:34)\n$line15.$read$$iw$$iw$$iw$$iw.(:36)\n$line15.$read$$iw$$iw$$iw.(:38)\n$line15.$read$$iw$$iw.(:40)\n$line15.$read$$iw.(:42)\n$line15.$read.(:44)\n$line15.$read$.(:48)\n$line15.$read$.()\n$line15.$eval$.$print$lzycompute(:7)\n$line15.$eval$.$print(:6)\n$line15.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","physicalPlanDescription":"== Physical Plan ==\nExecute InsertIntoHadoopFsRelationCommand (2)\n+- Scan parquet (1)\n\n\n(1) Scan parquet \nOutput [3]: [name#0, addresses#1, testmapmap#4]\nBatched: false\nLocation: InMemoryFileIndex [file:/home/nartal/event_logs_spark/nested_type/mapcomplex.parquet]\nReadSchema: struct>,testmapmap:map>>\n\n(2) Execute InsertIntoHadoopFsRelationCommand\nInput [3]: [name#0, addresses#1, testmapmap#4]\nArguments: file:/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/nestedType.parquet, false, Parquet, Map(path -> nestedType.parquet), ErrorIfExists, [name, addresses, testmapmap]\n\n","sparkPlanInfo":{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand file:/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/nestedType.parquet, false, Parquet, Map(path -> nestedType.parquet), ErrorIfExists, [name, addresses, testmapmap]","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [name#0,addresses#1,testmapmap#4] Batched: false, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex[file:/home/nartal/event_logs_spark/nested_type/mapcomplex.parquet], PartitionFilters: [], PushedFilters: [], ReadSchema: struct>,testmapmap:map>,testmapmap:map>>","Format":"Parquet","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of output rows","accumulatorId":29,"metricType":"sum"},{"name":"number of files read","accumulatorId":30,"metricType":"sum"},{"name":"metadata time","accumulatorId":31,"metricType":"timing"},{"name":"size of files read","accumulatorId":32,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of written files","accumulatorId":25,"metricType":"sum"},{"name":"written output","accumulatorId":26,"metricType":"size"},{"name":"number of output rows","accumulatorId":27,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":28,"metricType":"sum"}]},"time":1629446117657} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[30,6],[31,3],[32,16639]]} +{"Event":"SparkListenerJobStart","Job ID":1,"Submission Time":1629446117875,"Stage Infos":[{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"save at :26","Number of Tasks":6,"RDD Info":[{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"6\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"FileScanRDD","Scope":"{\"id\":\"6\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:30)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line15.$read$$iw$$iw$$iw$$iw$$iw.(:34)\n$line15.$read$$iw$$iw$$iw$$iw.(:36)\n$line15.$read$$iw$$iw$$iw.(:38)\n$line15.$read$$iw$$iw.(:40)\n$line15.$read$$iw.(:42)\n$line15.$read.(:44)\n$line15.$read$.(:48)\n$line15.$read$.()\n$line15.$eval$.$print$lzycompute(:7)\n$line15.$eval$.$print(:6)\n$line15.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[1],"Properties":{"spark.sql.warehouse.dir":"file:/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/spark-warehouse/","spark.driver.host":"nartal.attlocal.net","spark.eventLog.enabled":"true","spark.driver.port":"40637","spark.repl.class.uri":"spark://nartal.attlocal.net:40637/classes","spark.jars":"file:///home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.repl.class.outputDir":"/tmp/spark-90f7e1c4-0b66-4c35-95ba-b0df2a943d36/repl-cf3d7009-52a2-41f6-8ee9-3bef87fa92d7","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"5\",\"name\":\"Execute InsertIntoHadoopFsRelationCommand\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.startTime":"1629446105772","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://nartal.attlocal.net:40637/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.submit.deployMode":"client","spark.master":"local[*]","spark.home":"/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","spark.eventLog.dir":"/home/nartal/data_format_eventlog","spark.sql.execution.id":"0","spark.sql.catalogImplementation":"hive","spark.repl.local.jars":"file:///home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.app.id":"local-1629446106683"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"save at :26","Number of Tasks":6,"RDD Info":[{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"6\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"FileScanRDD","Scope":"{\"id\":\"6\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:30)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line15.$read$$iw$$iw$$iw$$iw$$iw.(:34)\n$line15.$read$$iw$$iw$$iw$$iw.(:36)\n$line15.$read$$iw$$iw$$iw.(:38)\n$line15.$read$$iw$$iw.(:40)\n$line15.$read$$iw.(:42)\n$line15.$read.(:44)\n$line15.$read$.(:48)\n$line15.$read$.()\n$line15.$eval$.$print$lzycompute(:7)\n$line15.$eval$.$print(:6)\n$line15.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Submission Time":1629446117879,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/spark-warehouse/","spark.driver.host":"nartal.attlocal.net","spark.eventLog.enabled":"true","spark.driver.port":"40637","spark.repl.class.uri":"spark://nartal.attlocal.net:40637/classes","spark.jars":"file:///home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.repl.class.outputDir":"/tmp/spark-90f7e1c4-0b66-4c35-95ba-b0df2a943d36/repl-cf3d7009-52a2-41f6-8ee9-3bef87fa92d7","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"5\",\"name\":\"Execute InsertIntoHadoopFsRelationCommand\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.startTime":"1629446105772","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://nartal.attlocal.net:40637/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.submit.deployMode":"client","spark.master":"local[*]","spark.home":"/home/nartal/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","spark.eventLog.dir":"/home/nartal/data_format_eventlog","spark.sql.execution.id":"0","spark.sql.catalogImplementation":"hive","spark.repl.local.jars":"file:///home/nartal/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.app.id":"local-1629446106683"}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Launch Time":1629446117943,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":2,"Index":1,"Attempt":0,"Launch Time":1629446117946,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":3,"Index":2,"Attempt":0,"Launch Time":1629446117947,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":4,"Index":3,"Attempt":0,"Launch Time":1629446117947,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":5,"Index":4,"Attempt":0,"Launch Time":1629446117948,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":6,"Index":5,"Attempt":0,"Launch Time":1629446117949,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":6,"Index":5,"Attempt":0,"Launch Time":1629446117949,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1629446118768,"Failed":false,"Killed":false,"Accumulables":[{"ID":33,"Name":"internal.metrics.executorDeserializeTime","Update":132,"Value":132,"Internal":true,"Count Failed Values":true},{"ID":34,"Name":"internal.metrics.executorDeserializeCpuTime","Update":81260053,"Value":81260053,"Internal":true,"Count Failed Values":true},{"ID":35,"Name":"internal.metrics.executorRunTime","Update":655,"Value":655,"Internal":true,"Count Failed Values":true},{"ID":36,"Name":"internal.metrics.executorCpuTime","Update":211243308,"Value":211243308,"Internal":true,"Count Failed Values":true},{"ID":37,"Name":"internal.metrics.resultSize","Update":2561,"Value":2561,"Internal":true,"Count Failed Values":true},{"ID":38,"Name":"internal.metrics.jvmGCTime","Update":21,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":39,"Name":"internal.metrics.resultSerializationTime","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":54,"Name":"internal.metrics.input.bytesRead","Update":3844,"Value":3844,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":349685272,"JVMOffHeapMemory":178144768,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":593163,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":593163,"OffHeapUnifiedMemory":0,"DirectPoolMemory":8193,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":10,"MinorGCTime":148,"MajorGCCount":4,"MajorGCTime":423},"Task Metrics":{"Executor Deserialize Time":132,"Executor Deserialize CPU Time":81260053,"Executor Run Time":655,"Executor CPU Time":211243308,"Peak Execution Memory":0,"Result Size":2561,"JVM GC Time":21,"Result Serialization Time":5,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":3844,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":4,"Index":3,"Attempt":0,"Launch Time":1629446117947,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1629446119075,"Failed":false,"Killed":false,"Accumulables":[{"ID":29,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"internal.metrics.executorDeserializeTime","Update":150,"Value":282,"Internal":true,"Count Failed Values":true},{"ID":34,"Name":"internal.metrics.executorDeserializeCpuTime","Update":83433184,"Value":164693237,"Internal":true,"Count Failed Values":true},{"ID":35,"Name":"internal.metrics.executorRunTime","Update":966,"Value":1621,"Internal":true,"Count Failed Values":true},{"ID":36,"Name":"internal.metrics.executorCpuTime","Update":433060372,"Value":644303680,"Internal":true,"Count Failed Values":true},{"ID":37,"Name":"internal.metrics.resultSize","Update":2647,"Value":5208,"Internal":true,"Count Failed Values":true},{"ID":38,"Name":"internal.metrics.jvmGCTime","Update":21,"Value":42,"Internal":true,"Count Failed Values":true},{"ID":54,"Name":"internal.metrics.input.bytesRead","Update":9279,"Value":13123,"Internal":true,"Count Failed Values":true},{"ID":55,"Name":"internal.metrics.input.recordsRead","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":56,"Name":"internal.metrics.output.bytesWritten","Update":1947,"Value":1947,"Internal":true,"Count Failed Values":true},{"ID":57,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":349685272,"JVMOffHeapMemory":178144768,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":593163,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":593163,"OffHeapUnifiedMemory":0,"DirectPoolMemory":8193,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":10,"MinorGCTime":148,"MajorGCCount":4,"MajorGCTime":423},"Task Metrics":{"Executor Deserialize Time":150,"Executor Deserialize CPU Time":83433184,"Executor Run Time":966,"Executor CPU Time":433060372,"Peak Execution Memory":0,"Result Size":2647,"JVM GC Time":21,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":9279,"Records Read":1},"Output Metrics":{"Bytes Written":1947,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":2,"Index":1,"Attempt":0,"Launch Time":1629446117946,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1629446119076,"Failed":false,"Killed":false,"Accumulables":[{"ID":29,"Name":"number of output rows","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"internal.metrics.executorDeserializeTime","Update":150,"Value":432,"Internal":true,"Count Failed Values":true},{"ID":34,"Name":"internal.metrics.executorDeserializeCpuTime","Update":80015368,"Value":244708605,"Internal":true,"Count Failed Values":true},{"ID":35,"Name":"internal.metrics.executorRunTime","Update":968,"Value":2589,"Internal":true,"Count Failed Values":true},{"ID":36,"Name":"internal.metrics.executorCpuTime","Update":129053723,"Value":773357403,"Internal":true,"Count Failed Values":true},{"ID":37,"Name":"internal.metrics.resultSize","Update":2647,"Value":7855,"Internal":true,"Count Failed Values":true},{"ID":38,"Name":"internal.metrics.jvmGCTime","Update":21,"Value":63,"Internal":true,"Count Failed Values":true},{"ID":54,"Name":"internal.metrics.input.bytesRead","Update":9309,"Value":22432,"Internal":true,"Count Failed Values":true},{"ID":55,"Name":"internal.metrics.input.recordsRead","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":56,"Name":"internal.metrics.output.bytesWritten","Update":1964,"Value":3911,"Internal":true,"Count Failed Values":true},{"ID":57,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":2,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":349685272,"JVMOffHeapMemory":178144768,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":593163,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":593163,"OffHeapUnifiedMemory":0,"DirectPoolMemory":8193,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":10,"MinorGCTime":148,"MajorGCCount":4,"MajorGCTime":423},"Task Metrics":{"Executor Deserialize Time":150,"Executor Deserialize CPU Time":80015368,"Executor Run Time":968,"Executor CPU Time":129053723,"Peak Execution Memory":0,"Result Size":2647,"JVM GC Time":21,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":9309,"Records Read":1},"Output Metrics":{"Bytes Written":1964,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":3,"Index":2,"Attempt":0,"Launch Time":1629446117947,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1629446119076,"Failed":false,"Killed":false,"Accumulables":[{"ID":29,"Name":"number of output rows","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"internal.metrics.executorDeserializeTime","Update":142,"Value":574,"Internal":true,"Count Failed Values":true},{"ID":34,"Name":"internal.metrics.executorDeserializeCpuTime","Update":65932495,"Value":310641100,"Internal":true,"Count Failed Values":true},{"ID":35,"Name":"internal.metrics.executorRunTime","Update":974,"Value":3563,"Internal":true,"Count Failed Values":true},{"ID":36,"Name":"internal.metrics.executorCpuTime","Update":175859803,"Value":949217206,"Internal":true,"Count Failed Values":true},{"ID":37,"Name":"internal.metrics.resultSize","Update":2647,"Value":10502,"Internal":true,"Count Failed Values":true},{"ID":38,"Name":"internal.metrics.jvmGCTime","Update":21,"Value":84,"Internal":true,"Count Failed Values":true},{"ID":54,"Name":"internal.metrics.input.bytesRead","Update":9289,"Value":31721,"Internal":true,"Count Failed Values":true},{"ID":55,"Name":"internal.metrics.input.recordsRead","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":56,"Name":"internal.metrics.output.bytesWritten","Update":1968,"Value":5879,"Internal":true,"Count Failed Values":true},{"ID":57,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":3,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":349685272,"JVMOffHeapMemory":178144768,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":593163,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":593163,"OffHeapUnifiedMemory":0,"DirectPoolMemory":8193,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":10,"MinorGCTime":148,"MajorGCCount":4,"MajorGCTime":423},"Task Metrics":{"Executor Deserialize Time":142,"Executor Deserialize CPU Time":65932495,"Executor Run Time":974,"Executor CPU Time":175859803,"Peak Execution Memory":0,"Result Size":2647,"JVM GC Time":21,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":9289,"Records Read":1},"Output Metrics":{"Bytes Written":1968,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":5,"Index":4,"Attempt":0,"Launch Time":1629446117948,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1629446119079,"Failed":false,"Killed":false,"Accumulables":[{"ID":29,"Name":"number of output rows","Update":"1","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"internal.metrics.executorDeserializeTime","Update":145,"Value":719,"Internal":true,"Count Failed Values":true},{"ID":34,"Name":"internal.metrics.executorDeserializeCpuTime","Update":65091488,"Value":375732588,"Internal":true,"Count Failed Values":true},{"ID":35,"Name":"internal.metrics.executorRunTime","Update":973,"Value":4536,"Internal":true,"Count Failed Values":true},{"ID":36,"Name":"internal.metrics.executorCpuTime","Update":143544511,"Value":1092761717,"Internal":true,"Count Failed Values":true},{"ID":37,"Name":"internal.metrics.resultSize","Update":2647,"Value":13149,"Internal":true,"Count Failed Values":true},{"ID":38,"Name":"internal.metrics.jvmGCTime","Update":21,"Value":105,"Internal":true,"Count Failed Values":true},{"ID":54,"Name":"internal.metrics.input.bytesRead","Update":8624,"Value":40345,"Internal":true,"Count Failed Values":true},{"ID":55,"Name":"internal.metrics.input.recordsRead","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":56,"Name":"internal.metrics.output.bytesWritten","Update":1824,"Value":7703,"Internal":true,"Count Failed Values":true},{"ID":57,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":4,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":349685272,"JVMOffHeapMemory":178144768,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":593163,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":593163,"OffHeapUnifiedMemory":0,"DirectPoolMemory":8193,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":10,"MinorGCTime":148,"MajorGCCount":4,"MajorGCTime":423},"Task Metrics":{"Executor Deserialize Time":145,"Executor Deserialize CPU Time":65091488,"Executor Run Time":973,"Executor CPU Time":143544511,"Peak Execution Memory":0,"Result Size":2647,"JVM GC Time":21,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":8624,"Records Read":1},"Output Metrics":{"Bytes Written":1824,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Launch Time":1629446117943,"Executor ID":"driver","Host":"nartal.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1629446119081,"Failed":false,"Killed":false,"Accumulables":[{"ID":29,"Name":"number of output rows","Update":"1","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"internal.metrics.executorDeserializeTime","Update":151,"Value":870,"Internal":true,"Count Failed Values":true},{"ID":34,"Name":"internal.metrics.executorDeserializeCpuTime","Update":70851949,"Value":446584537,"Internal":true,"Count Failed Values":true},{"ID":35,"Name":"internal.metrics.executorRunTime","Update":968,"Value":5504,"Internal":true,"Count Failed Values":true},{"ID":36,"Name":"internal.metrics.executorCpuTime","Update":437622948,"Value":1530384665,"Internal":true,"Count Failed Values":true},{"ID":37,"Name":"internal.metrics.resultSize","Update":2647,"Value":15796,"Internal":true,"Count Failed Values":true},{"ID":38,"Name":"internal.metrics.jvmGCTime","Update":21,"Value":126,"Internal":true,"Count Failed Values":true},{"ID":54,"Name":"internal.metrics.input.bytesRead","Update":9329,"Value":49674,"Internal":true,"Count Failed Values":true},{"ID":55,"Name":"internal.metrics.input.recordsRead","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":56,"Name":"internal.metrics.output.bytesWritten","Update":1910,"Value":9613,"Internal":true,"Count Failed Values":true},{"ID":57,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":5,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":349685272,"JVMOffHeapMemory":178144768,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":593163,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":593163,"OffHeapUnifiedMemory":0,"DirectPoolMemory":8193,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":10,"MinorGCTime":148,"MajorGCCount":4,"MajorGCTime":423},"Task Metrics":{"Executor Deserialize Time":151,"Executor Deserialize CPU Time":70851949,"Executor Run Time":968,"Executor CPU Time":437622948,"Peak Execution Memory":0,"Result Size":2647,"JVM GC Time":21,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":9329,"Records Read":1},"Output Metrics":{"Bytes Written":1910,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"save at :26","Number of Tasks":6,"RDD Info":[{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"6\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"FileScanRDD","Scope":"{\"id\":\"6\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:30)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line15.$read$$iw$$iw$$iw$$iw$$iw.(:34)\n$line15.$read$$iw$$iw$$iw$$iw.(:36)\n$line15.$read$$iw$$iw$$iw.(:38)\n$line15.$read$$iw$$iw.(:40)\n$line15.$read$$iw.(:42)\n$line15.$read.(:44)\n$line15.$read$.(:48)\n$line15.$read$.()\n$line15.$eval$.$print$lzycompute(:7)\n$line15.$eval$.$print(:6)\n$line15.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Submission Time":1629446117879,"Completion Time":1629446119082,"Accumulables":[{"ID":29,"Name":"number of output rows","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"internal.metrics.executorDeserializeTime","Value":870,"Internal":true,"Count Failed Values":true},{"ID":34,"Name":"internal.metrics.executorDeserializeCpuTime","Value":446584537,"Internal":true,"Count Failed Values":true},{"ID":35,"Name":"internal.metrics.executorRunTime","Value":5504,"Internal":true,"Count Failed Values":true},{"ID":36,"Name":"internal.metrics.executorCpuTime","Value":1530384665,"Internal":true,"Count Failed Values":true},{"ID":37,"Name":"internal.metrics.resultSize","Value":15796,"Internal":true,"Count Failed Values":true},{"ID":38,"Name":"internal.metrics.jvmGCTime","Value":126,"Internal":true,"Count Failed Values":true},{"ID":39,"Name":"internal.metrics.resultSerializationTime","Value":5,"Internal":true,"Count Failed Values":true},{"ID":54,"Name":"internal.metrics.input.bytesRead","Value":49674,"Internal":true,"Count Failed Values":true},{"ID":55,"Name":"internal.metrics.input.recordsRead","Value":5,"Internal":true,"Count Failed Values":true},{"ID":56,"Name":"internal.metrics.output.bytesWritten","Value":9613,"Internal":true,"Count Failed Values":true},{"ID":57,"Name":"internal.metrics.output.recordsWritten","Value":5,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":1,"Completion Time":1629446119082,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[25,5],[26,9613],[27,5],[28,0]]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":0,"time":1629446119110} +{"Event":"SparkListenerApplicationEnd","Timestamp":1629446123470} diff --git a/tools/src/test/resources/spark-events-qualification/writeformat_eventlog b/tools/src/test/resources/spark-events-qualification/writeformat_eventlog new file mode 100644 index 00000000000..2239c98256e --- /dev/null +++ b/tools/src/test/resources/spark-events-qualification/writeformat_eventlog @@ -0,0 +1,53 @@ +{"Event":"SparkListenerLogStart","Spark Version":"3.1.1"} +{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"cores":{"Resource Name":"cores","Amount":1,"Discovery Script":"","Vendor":""},"memory":{"Resource Name":"memory","Amount":1024,"Discovery Script":"","Vendor":""},"offHeap":{"Resource Name":"offHeap","Amount":0,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1629442300205,"Executor ID":"driver","Executor Info":{"Host":"user1.attlocal.net","Total Cores":8,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"user1.attlocal.net","Port":46089},"Maximum Memory":384093388,"Timestamp":1629442300232,"Maximum Onheap Memory":384093388,"Maximum Offheap Memory":0} +{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-8-openjdk-amd64/jre","Java Version":"1.8.0_292 (Private Build)","Scala Version":"version 2.12.10"},"Spark Properties":{"spark.driver.host":"user1.attlocal.net","spark.eventLog.enabled":"true","spark.driver.port":"39937","spark.repl.class.uri":"spark://user1.attlocal.net:39937/classes","spark.jars":"file:///home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.repl.class.outputDir":"/tmp/spark-655d1756-32df-40e3-81e9-42ce076c6c55/repl-7d342e8a-9aae-463e-be0c-94065166b845","spark.app.name":"Spark shell","spark.scheduler.mode":"FIFO","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.startTime":"1629442298969","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://user1.attlocal.net:39937/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.submit.deployMode":"client","spark.master":"local[*]","spark.home":"/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","spark.eventLog.dir":"/home/user1/data_format_eventlog","spark.sql.catalogImplementation":"hive","spark.repl.local.jars":"file:///home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.app.id":"local-1629442299891"},"Hadoop Properties":{"hadoop.service.shutdown.timeout":"30s","yarn.resourcemanager.amlauncher.thread-count":"50","yarn.sharedcache.enabled":"false","fs.s3a.connection.maximum":"15","yarn.nodemanager.numa-awareness.numactl.cmd":"/usr/bin/numactl","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","yarn.timeline-service.timeline-client.number-of-async-entities-to-merge":"10","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.jobhistory.loadedjob.tasks.max":"-1","mapreduce.framework.name":"local","yarn.sharedcache.uploader.server.thread-count":"50","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","hadoop.security.groups.cache.background.reload.threads":"3","yarn.resourcemanager.webapp.cross-origin.enabled":"false","fs.AbstractFileSystem.ftp.impl":"org.apache.hadoop.fs.ftp.FtpFs","hadoop.registry.secure":"false","hadoop.shell.safely.delete.limit.num.files":"100","dfs.bytes-per-checksum":"512","mapreduce.job.acl-view-job":" ","fs.s3a.s3guard.ddb.background.sleep":"25ms","fs.s3a.retry.limit":"${fs.s3a.attempts.maximum}","mapreduce.jobhistory.loadedjobs.cache.size":"5","fs.s3a.s3guard.ddb.table.create":"false","yarn.nodemanager.amrmproxy.enabled":"false","yarn.timeline-service.entity-group-fs-store.with-user-dir":"false","mapreduce.input.fileinputformat.split.minsize":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","yarn.sharedcache.checksum.algo.impl":"org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl","yarn.nodemanager.amrmproxy.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor","yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size":"10485760","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","mapreduce.task.profile.maps":"0-2","yarn.scheduler.include-port-in-node-name":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","yarn.resourcemanager.node-removal-untracked.timeout-ms":"60000","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","mapreduce.jobhistory.webapp.https.address":"0.0.0.0:19890","yarn.node-labels.fs-store.impl.class":"org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore","yarn.nodemanager.collector-service.address":"${yarn.nodemanager.hostname}:8048","fs.trash.checkpoint.interval":"0","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","yarn.resourcemanager.node-ip-cache.expiry-interval-secs":"-1","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","yarn.resourcemanager.placement-constraints.handler":"disabled","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"1000","yarn.resourcemanager.system-metrics-publisher.enabled":"false","yarn.resourcemanager.placement-constraints.algorithm.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm","yarn.sharedcache.webapp.address":"0.0.0.0:8788","yarn.resourcemanager.delegation.token.renew-interval":"*********(redacted)","yarn.sharedcache.nm.uploader.replication.factor":"10","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","mapreduce.reduce.skip.proc-count.auto-incr":"true","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","ipc.client.fallback-to-simple-auth-allowed":"false","yarn.nodemanager.resource.memory.enforced":"true","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","yarn.acl.reservation-enable":"false","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","yarn.app.mapreduce.am.hard-kill-timeout-ms":"10000","fs.s3a.etag.checksum.enabled":"false","yarn.nodemanager.container-metrics.enable":"true","yarn.timeline-service.client.fd-clean-interval-secs":"60","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","hadoop.common.configuration.version":"3.0.0","fs.s3a.s3guard.ddb.table.capacity.read":"500","yarn.nodemanager.remote-app-log-dir-suffix":"logs","yarn.nodemanager.windows-container.cpu-limit.enabled":"false","yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed":"false","file.blocksize":"67108864","hadoop.registry.zk.retry.ceiling.ms":"60000","yarn.scheduler.configuration.leveldb-store.path":"${hadoop.tmp.dir}/yarn/system/confstore","yarn.sharedcache.store.in-memory.initial-delay-mins":"10","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","mapreduce.map.skip.proc-count.auto-incr":"true","fs.s3a.committer.name":"file","mapreduce.task.profile.reduces":"0-2","hadoop.zk.num-retries":"1000","yarn.webapp.xfs-filter.enabled":"true","seq.io.sort.mb":"100","yarn.scheduler.configuration.max.version":"100","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"${yarn.resourcemanager.hostname}:8030","yarn.node-labels.enabled":"false","yarn.resourcemanager.webapp.ui-actions.enabled":"true","mapreduce.task.timeout":"600000","yarn.sharedcache.client-server.thread-count":"50","hadoop.security.groups.shell.command.timeout":"0s","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.nodemanager.elastic-memory-control.oom-handler":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.DefaultOOMHandler","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"file:///","yarn.minicluster.use-rpc":"false","fs.har.impl.disable.cache":"true","yarn.webapp.ui2.enable":"false","io.compression.codec.bzip2.library":"system-native","yarn.nodemanager.distributed-scheduling.enabled":"false","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"${yarn.resourcemanager.hostname}:8090","mapreduce.jobhistory.address":"0.0.0.0:10020","yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.is.minicluster":"false","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","mapreduce.task.combine.progress.records":"10000","yarn.resourcemanager.epoch.range":"0","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","fs.AbstractFileSystem.wasbs.impl":"org.apache.hadoop.fs.azure.Wasbs","yarn.timeline-service.entity-group-fs-store.cache-store-class":"org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore","fs.ftp.transfer.mode":"BLOCK_TRANSFER_MODE","ipc.server.log.slow.rpc":"false","yarn.resourcemanager.node-labels.provider.fetch-interval-ms":"1800000","yarn.router.webapp.https.address":"0.0.0.0:8091","yarn.nodemanager.webapp.cross-origin.enabled":"false","fs.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","yarn.resourcemanager.auto-update.containers":"false","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","yarn.scheduler.configuration.zk-store.parent-path":"/confstore","yarn.nodemanager.default-container-executor.log-dirs.permissions":"710","yarn.app.attempt.diagnostics.limit.kc":"64","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"-1","fs.AbstractFileSystem.abfs.impl":"org.apache.hadoop.fs.azurebfs.Abfs","yarn.timeline-service.writer.flush-interval-seconds":"60","fs.s3a.fast.upload.active.blocks":"4","hadoop.security.credential.clear-text-fallback":"true","yarn.nodemanager.collector-service.thread-count":"5","fs.azure.secure.mode":"false","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","yarn.resourcemanager.fs.state-store.num-retries":"0","yarn.resourcemanager.nodemanager-connect-retries":"10","yarn.nodemanager.log-aggregation.num-log-files-per-app":"30","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","fs.s3a.committer.magic.enabled":"false","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","yarn.system-metrics-publisher.enabled":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","mapreduce.task.io.sort.factor":"10","yarn.nodemanager.amrmproxy.client.thread-count":"25","ha.failover-controller.new-active.rpc-timeout.ms":"60000","yarn.nodemanager.container-localizer.java.opts":"-Xmx256m","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","yarn.nodemanager.windows-container.memory-limit.enabled":"false","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.minicluster.fixed.ports":"false","hadoop.tags.system":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.cluster.max-application-priority":"0","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","hadoop.caller.context.signature.max.size":"40","yarn.client.load.resource-types.from-server":"false","ha.zookeeper.session-timeout.ms":"10000","tfile.io.chunk.size":"1048576","fs.s3a.s3guard.ddb.table.capacity.write":"100","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","yarn.scheduler.configuration.store.max-logs":"1000","yarn.nodemanager.node-attributes.provider.fetch-interval-ms":"600000","fs.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","yarn.nodemanager.local-cache.max-files-per-directory":"8192","hadoop.http.cross-origin.enabled":"false","hadoop.zk.acl":"world:anyone:rwcda","mapreduce.map.sort.spill.percent":"0.80","yarn.timeline-service.entity-group-fs-store.scan-interval-seconds":"60","yarn.node-attribute.fs-store.impl.class":"org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore","fs.s3a.retry.interval":"500ms","yarn.timeline-service.client.best-effort":"false","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","fs.AbstractFileSystem.swebhdfs.impl":"org.apache.hadoop.fs.SWebHdfs","yarn.nodemanager.elastic-memory-control.timeout-sec":"5","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","yarn.timeline-service.reader.webapp.address":"${yarn.timeline-service.webapp.address}","yarn.resourcemanager.placement-constraints.algorithm.pool-size":"1","yarn.timeline-service.hbase.coprocessor.jar.hdfs.location":"/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler","yarn.app.mapreduce.am.command-opts":"-Xmx1024m","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","io.mapfile.bloom.error.rate":"0.005","fs.client.resolve.topology.enabled":"false","yarn.nodemanager.runtime.linux.allowed-runtimes":"default","yarn.sharedcache.store.class":"org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","fs.s3a.metadatastore.impl":"org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore","io.skip.checksum.errors":"false","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","yarn.timeline-service.webapp.xfs-filter.xframe-options":"SAMEORIGIN","fs.s3a.connection.timeout":"200000","mapreduce.job.max.split.locations":"15","yarn.resourcemanager.nm-container-queuing.max-queue-length":"15","hadoop.registry.zk.session.timeout.ms":"60000","yarn.federation.cache-ttl.secs":"300","mapreduce.jvm.system-properties-to-log":"os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name","yarn.resourcemanager.opportunistic-container-allocation.nodes-used":"10","yarn.timeline-service.entity-group-fs-store.active-dir":"/tmp/entity-file-history/active","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","fs.AbstractFileSystem.abfss.impl":"org.apache.hadoop.fs.azurebfs.Abfss","yarn.sharedcache.uploader.server.address":"0.0.0.0:8046","yarn.resourcemanager.delegation-token.max-conf-size-bytes":"*********(redacted)","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","yarn.minicluster.yarn.nodemanager.resource.memory-mb":"4096","mapreduce.job.emit-timeline-data":"false","yarn.nodemanager.resource.system-reserved-memory-mb":"-1","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","yarn.resourcemanager.admin.client.thread-count":"1","yarn.dispatcher.drain-events.timeout":"300000","fs.s3a.buffer.dir":"${hadoop.tmp.dir}/s3a","hadoop.ssl.enabled.protocols":"TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2","mapreduce.jobhistory.admin.address":"0.0.0.0:10033","yarn.log-aggregation-status.time-out.ms":"600000","fs.s3a.assumed.role.sts.endpoint.region":"us-west-1","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","yarn.router.clientrm.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor","yarn.resourcemanager.zk-appid-node.split-index":"0","ftp.blocksize":"67108864","yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions":"read","yarn.router.rmadmin.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor","yarn.nodemanager.log-container-debug-info.enabled":"true","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms":"20","yarn.nodemanager.delete.debug-delay-sec":"0","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","yarn.nodemanager.container-retry-minimum-interval-ms":"1000","hadoop.security.groups.cache.secs":"300","yarn.federation.enabled":"false","fs.azure.local.sas.key.mode":"false","ipc.maximum.data.length":"67108864","mapreduce.shuffle.max.threads":"0","yarn.router.pipeline.cache-max-size":"25","yarn.resourcemanager.nm-container-queuing.load-comparator":"QUEUE_LENGTH","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","yarn.nodemanager.amrmproxy.ha.enable":"false","mapreduce.jobhistory.http.policy":"HTTP_ONLY","yarn.sharedcache.store.in-memory.check-period-mins":"720","hadoop.security.group.mapping.ldap.ssl":"false","yarn.client.application-client-protocol.poll-interval-ms":"200","yarn.scheduler.configuration.leveldb-store.compaction-interval-secs":"86400","yarn.timeline-service.writer.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl","ha.zookeeper.parent-znode":"/hadoop-ha","yarn.nodemanager.log-aggregation.policy.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","yarn.resourcemanager.placement-constraints.scheduler.pool-size":"1","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","yarn.nodemanager.recovery.supervised":"false","yarn.sharedcache.admin.thread-count":"1","yarn.resourcemanager.ha.automatic-failover.enabled":"true","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.resourcemanager.address":"${yarn.resourcemanager.hostname}:8032","ipc.client.ping":"true","mapreduce.task.local-fs.write-limit.bytes":"-1","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","yarn.nodemanager.resource-plugins.gpu.docker-plugin":"nvidia-docker-v1","hadoop.ssl.enabled":"false","fs.s3a.multipart.purge":"false","yarn.scheduler.configuration.store.class":"file","yarn.resourcemanager.nm-container-queuing.queue-limit-stdev":"1.0f","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","ipc.client.bind.wildcard.addr":"false","yarn.resourcemanager.webapp.rest-csrf.enabled":"false","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"10","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","yarn.nodemanager.runtime.linux.docker.allowed-container-networks":"host,none,bridge","yarn.nodemanager.node-labels.resync-interval-ms":"120000","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","mapreduce.job.maps":"2","mapreduce.jobhistory.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","yarn.rm.system-metrics-publisher.emit-container-events":"false","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","fs.s3a.s3guard.ddb.throttle.retry.interval":"100ms","fs.AbstractFileSystem.wasb.impl":"org.apache.hadoop.fs.azure.Wasb","mapreduce.client.submit.file.replication":"10","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"2147483647","yarn.resourcemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold":"1","yarn.nodemanager.webapp.rest-csrf.enabled":"false","fs.ftp.host.port":"21","ipc.ping.interval":"60000","yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size":"10","yarn.resourcemanager.admin.address":"${yarn.resourcemanager.hostname}:8033","file.client-write-packet-size":"65536","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"true","hadoop.security.key.default.bitlength":"128","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","yarn.nodemanager.log.deletion-threads-count":"4","yarn.webapp.filter-entity-list-by-user":"false","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"100","yarn.nodemanager.localizer.client.thread-count":"5","io.erasurecode.codec.rs.rawcoders":"rs_native,rs_java","io.erasurecode.codec.rs-legacy.rawcoders":"rs-legacy_java","yarn.sharedcache.admin.address":"0.0.0.0:8047","yarn.resourcemanager.placement-constraints.algorithm.iterator":"SERIAL","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec","mapreduce.job.cache.limit.max-resources-mb":"0","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"5000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","hadoop.workaround.non.threadsafe.getpwuid":"true","fs.df.interval":"60000","fs.s3a.multiobjectdelete.enable":"true","yarn.sharedcache.cleaner.resource-sleep-ms":"0","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"false","io.file.buffer.size":"65536","hadoop.security.group.mapping.ldap.search.attr.member":"member","hadoop.security.random.device.file.path":"/dev/urandom","hadoop.security.sensitive-config-keys":"*********(redacted)","fs.s3a.s3guard.ddb.max.retries":"9","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","yarn.intermediate-data-encryption.enable":"false","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.nodemanager.container.stderr.pattern":"{*stderr*,*STDERR*}","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/tmp/hadoop-yarn/staging","mapreduce.reduce.shuffle.read.timeout":"180000","hadoop.http.cross-origin.max-age":"1800","io.erasurecode.codec.xor.rawcoders":"xor_native,xor_java","fs.s3a.connection.establish.timeout":"5000","mapreduce.job.running.map.limit":"0","yarn.minicluster.control-resource-monitoring":"false","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","yarn.federation.state-store.class":"org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","adl.http.timeout":"-1","yarn.resourcemanager.placement-constraints.retry-attempts":"3","hadoop.caller.context.enabled":"false","yarn.nodemanager.vmem-pmem-ratio":"2.1","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","hadoop.zk.timeout-ms":"10000","fs.s3a.s3guard.cli.prune.age":"86400000","yarn.nodemanager.resource.pcores-vcores-multiplier":"1.0","yarn.nodemanager.runtime.linux.sandbox-mode":"disabled","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.s3a.committer.threads":"8","hadoop.zk.retry-interval-ms":"1000","hadoop.security.crypto.buffer.size":"8192","yarn.nodemanager.node-labels.provider.fetch-interval-ms":"600000","mapreduce.jobhistory.recovery.store.leveldb.path":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.client.failover-retries-on-socket-timeouts":"0","yarn.nodemanager.resource.memory.enabled":"false","fs.azure.authorization.caching.enable":"true","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","mapreduce.job.finish-when-all-reducers-done":"true","hadoop.registry.jaas.context":"Client","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","io.map.index.interval":"128","yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms":"100","fs.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","mapreduce.job.counters.max":"120","mapreduce.jobhistory.webapp.rest-csrf.enabled":"false","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","yarn.timeline-service.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/timeline","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","hadoop.security.auth_to_local.mechanism":"hadoop","yarn.timeline-service.app-collector.linger-period.ms":"60000","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.reservation-system.planfollower.time-step":"1000","yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed":"true","yarn.webapp.api-service.enable":"false","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","fs.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","yarn.nodemanager.container.stderr.tail.bytes":"4096","hadoop.security.group.mapping.ldap.read.timeout.ms":"60000","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","mapreduce.outputcommitter.factory.scheme.s3a":"org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory","hadoop.security.groups.cache.background.reload":"false","yarn.nodemanager.container-monitor.enabled":"true","yarn.nodemanager.elastic-memory-control.enabled":"false","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","yarn.resourcemanager.rm.container-allocation.expiry-interval-ms":"600000","mapreduce.fileoutputcommitter.algorithm.version":"1","yarn.resourcemanager.work-preserving-recovery.enabled":"true","mapreduce.map.skip.maxrecords":"0","yarn.sharedcache.root-dir":"/sharedcache","fs.s3a.retry.throttle.limit":"${fs.s3a.attempts.maximum}","hadoop.http.authentication.type":"simple","mapreduce.job.cache.limit.max-resources":"0","mapreduce.task.userlog.limit.kb":"0","yarn.resourcemanager.scheduler.monitor.enable":"false","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","yarn.nodemanager.resource-monitor.interval-ms":"3000","yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices":"auto","mapreduce.job.sharedcache.mode":"disabled","yarn.nodemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.shuffle.listen.queue.size":"128","yarn.scheduler.configuration.mutation.acl-policy.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy","mapreduce.map.cpu.vcores":"1","yarn.log-aggregation.file-formats":"TFile","yarn.timeline-service.client.fd-retain-secs":"300","hadoop.user.group.static.mapping.overrides":"dr.who=;","fs.azure.sas.expiry.period":"90d","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","yarn.router.webapp.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST","yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage":"90.0","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","yarn.nodemanager.disk-validator":"basic","yarn.app.mapreduce.client.job.max-retries":"3","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","hadoop.security.group.mapping.ldap.connection.timeout.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","yarn.nodemanager.container-diagnostics-maximum-size":"10000","hadoop.registry.zk.retry.interval.ms":"1000","yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms":"1000","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","fs.s3a.assumed.role.session.duration":"30m","hadoop.security.group.mapping.ldap.conversion.rule":"none","hadoop.ssl.server.conf":"ssl-server.xml","fs.s3a.retry.throttle.interval":"1000ms","seq.io.sort.factor":"100","yarn.sharedcache.cleaner.initial-delay-mins":"10","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.nodemanager.runtime.linux.docker.capabilities":"CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE","yarn.acl.enable":"false","yarn.timeline-service.entity-group-fs-store.done-dir":"/tmp/entity-file-history/done/","mapreduce.task.profile":"false","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","mapreduce.jobhistory.always-scan-user-dir":"false","yarn.nodemanager.opportunistic-containers-use-pause-for-preemption":"false","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","yarn.timeline-service.reader.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold":"1","yarn.resourcemanager.configuration.file-system-based-store":"/yarn/conf","mapreduce.job.cache.limit.max-single-resource-mb":"0","yarn.nodemanager.runtime.linux.docker.stop.grace-period":"10","yarn.resourcemanager.resource-profiles.source-file":"resource-profiles.json","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","fs.automatic.close":"true","yarn.nodemanager.hostname":"0.0.0.0","yarn.nodemanager.resource.memory.cgroups.swappiness":"0","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","yarn.timeline-service.app-aggregation-interval-secs":"15","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","yarn.nodemanager.container-localizer.log.level":"INFO","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60","mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","yarn.resourcemanager.delegation.token.max-lifetime":"*********(redacted)","yarn.timeline-service.client.drain-entities.timeout.ms":"2000","yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin","yarn.timeline-service.entity-group-fs-store.summary-store":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.reduce.cpu.vcores":"1","mapreduce.job.encrypted-intermediate-data.buffer.kb":"128","fs.client.resolve.remote.symlinks":"true","yarn.nodemanager.webapp.https.address":"0.0.0.0:8044","hadoop.http.cross-origin.allowed-origins":"*","mapreduce.job.encrypted-intermediate-data":"false","yarn.timeline-service.entity-group-fs-store.retain-seconds":"604800","yarn.resourcemanager.metrics.runtime.buckets":"60,300,1440","yarn.timeline-service.generic-application-history.max-applications":"10000","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","yarn.node-labels.configuration-type":"centralized","fs.s3a.path.style.access":"false","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","yarn.sharedcache.store.in-memory.staleness-period-mins":"10080","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.nodemanager.minimum.version":"NONE","mapreduce.jobhistory.webapp.xfs-filter.xframe-options":"SAMEORIGIN","yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled":"false","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","yarn.timeline-service.reader.webapp.https.address":"${yarn.timeline-service.webapp.https.address}","fs.ftp.data.connection.mode":"ACTIVE_LOCAL_DATA_CONNECTION_MODE","mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed":"true","yarn.scheduler.maximum-allocation-vcores":"4","hadoop.http.cross-origin.allowed-headers":"X-Requested-With,Content-Type,Accept,Origin","yarn.nodemanager.log-aggregation.compression-type":"none","yarn.timeline-service.version":"1.0f","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","mapreduce.reduce.maxattempts":"4","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","mapreduce.job.running.reduce.limit":"0","ipc.maximum.response.length":"134217728","yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","hadoop.caller.context.max.size":"128","yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed":"false","yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed":"false","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","fs.s3a.max.total.tasks":"5","fs.s3a.readahead.range":"64K","hadoop.http.authentication.simple.anonymous.allowed":"true","fs.s3a.attempts.maximum":"20","hadoop.registry.zk.connection.timeout.ms":"15000","yarn.resourcemanager.delegation-token-renewer.thread-count":"*********(redacted)","yarn.nodemanager.health-checker.script.timeout-ms":"1200000","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","yarn.resourcemanager.resource-profiles.enabled":"false","yarn.timeline-service.hbase-schema.prefix":"prod.","fs.azure.authorization":"false","mapreduce.map.log.level":"INFO","yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs":"20","mapreduce.output.fileoutputformat.compress.type":"RECORD","yarn.resourcemanager.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.timeline-service.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.ifile.readahead.bytes":"4194304","yarn.sharedcache.app-checker.class":"org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"true","yarn.nodemanager.resource.detect-hardware-capabilities":"false","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","hadoop.security.group.mapping.ldap.search.group.hierarchy.levels":"0","yarn.resourcemanager.fs.state-store.retry-interval-ms":"1000","file.stream-buffer-size":"4096","yarn.resourcemanager.application-timeouts.monitor.interval-ms":"3000","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.map.speculative":"true","mapreduce.job.speculative.retry-after-speculate":"15000","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","yarn.app.mapreduce.am.log.level":"INFO","mapreduce.job.reduce.slowstart.completedmaps":"0.05","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices":"0,1","yarn.timeline-service.client.internal-timers-ttl-secs":"420","hadoop.http.logs.enabled":"true","fs.s3a.block.size":"32M","yarn.sharedcache.client-server.address":"0.0.0.0:8045","yarn.nodemanager.logaggregation.threadpool-size-max":"100","yarn.resourcemanager.hostname":"0.0.0.0","yarn.resourcemanager.delegation.key.update-interval":"86400000","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"-1","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","yarn.nodemanager.disk-health-checker.enable":"true","ipc.client.tcpnodelay":"true","ipc.client.rpc-timeout.ms":"0","yarn.nodemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","ipc.client.low-latency":"false","mapreduce.input.lineinputformat.linespermap":"1","yarn.router.interceptor.user.threadpool-size":"5","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.timeline-service.entity-group-fs-store.app-cache-size":"10","fs.s3a.socket.recv.buffer":"8192","yarn.resourcemanager.resource-tracker.address":"${yarn.resourcemanager.hostname}:8031","yarn.nodemanager.node-labels.provider.fetch-timeout-ms":"1200000","mapreduce.job.heap.memory-mb.ratio":"0.8","yarn.resourcemanager.leveldb-state-store.compaction-interval-secs":"3600","yarn.resourcemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","yarn.scheduler.configuration.fs.path":"file://${hadoop.tmp.dir}/yarn/system/schedconf","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.http.lib.StaticUserWebFilter","mapreduce.reduce.memory.mb":"-1","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","yarn.nodemanager.container-metrics.period-ms":"-1","mapreduce.fileoutputcommitter.task.cleanup.enabled":"false","yarn.nodemanager.log.retain-seconds":"10800","yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds":"3600","yarn.resourcemanager.keytab":"/etc/krb5.keytab","hadoop.security.group.mapping.providers.combined":"true","mapreduce.reduce.merge.inmem.threshold":"1000","yarn.timeline-service.recovery.enabled":"false","fs.azure.saskey.usecontainersaskeyforallaccess":"true","yarn.sharedcache.nm.uploader.thread-count":"20","yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs":"3600","mapreduce.shuffle.ssl.enabled":"false","yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds":"259200000","fs.s3a.committer.staging.abort.pending.uploads":"true","yarn.nodemanager.opportunistic-containers-max-queue-length":"0","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","mapreduce.job.speculative.minimum-allowed-tasks":"10","yarn.log-aggregation.retain-seconds":"-1","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.http.cross-origin.allowed-methods":"GET,POST,HEAD","yarn.resourcemanager.opportunistic-container-allocation.enabled":"false","mapreduce.jobhistory.webapp.address":"0.0.0.0:19888","hadoop.system.tags":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.log-aggregation.file-controller.TFile.class":"org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"${yarn.resourcemanager.hostname}:8088","mapreduce.jobhistory.recovery.enable":"false","mapreduce.reduce.shuffle.parallelcopies":"5","fs.AbstractFileSystem.webhdfs.impl":"org.apache.hadoop.fs.WebHdfs","fs.trash.interval":"0","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","yarn.app.mapreduce.am.resource.mb":"1536","mapreduce.input.fileinputformat.list-status.num-threads":"1","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.resourcemanager.nm-container-queuing.min-queue-length":"5","yarn.nodemanager.resource.cpu-vcores":"-1","mapreduce.job.reduces":"1","fs.s3a.multipart.size":"100M","yarn.scheduler.minimum-allocation-vcores":"1","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","mapreduce.job.encrypted-intermediate-data-key-size-bits":"128","fs.s3a.metadatastore.authoritative":"false","yarn.nodemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","ha.health-monitor.sleep-after-disconnect.ms":"1000","yarn.app.mapreduce.shuffle.log.limit.kb":"0","hadoop.security.group.mapping":"org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback","yarn.client.application-client-protocol.poll-timeout-ms":"-1","mapreduce.jobhistory.jhist.format":"binary","yarn.resourcemanager.ha.enabled":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.jobhistory.intermediate-user-done-dir.permissions":"770","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","yarn.resourcemanager.reservation-system.enable":"false","mapreduce.map.output.compress":"false","ha.zookeeper.acl":"world:anyone:rwcda","ipc.server.max.connections":"0","yarn.nodemanager.runtime.linux.docker.default-container-network":"host","yarn.router.webapp.address":"0.0.0.0:8089","yarn.scheduler.maximum-allocation-mb":"8192","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.sharedcache.cleaner.period-mins":"1440","yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint":"http://localhost:3476/v1.0/docker/cli","yarn.app.mapreduce.am.container.log.limit.kb":"0","ipc.client.connect.retry.interval":"1000","yarn.timeline-service.http-cross-origin.enabled":"false","fs.wasbs.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure","yarn.federation.subcluster-resolver.class":"org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","mapreduce.jobhistory.cleaner.enable":"true","yarn.timeline-service.client.fd-flush-interval-secs":"10","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms":"1000","fs.s3a.committer.staging.tmp.path":"tmp/staging","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms":"10","mapreduce.job.end-notification.retry.attempts":"0","yarn.nodemanager.resource.count-logical-processors-as-cores":"false","hadoop.registry.zk.root":"/registry","adl.feature.ownerandgroup.enableupn":"false","yarn.resourcemanager.zk-max-znode-size.bytes":"1048576","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","yarn.nodemanager.localizer.cache.target-size-mb":"10240","fs.s3a.committer.staging.conflict-mode":"fail","mapreduce.client.libjars.wildcard":"true","fs.s3a.committer.staging.unique-filenames":"true","yarn.nodemanager.node-attributes.provider.fetch-timeout-ms":"1200000","fs.s3a.list.version":"2","ftp.client-write-packet-size":"65536","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","hadoop.security.key.default.cipher":"AES/CTR/NoPadding","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"86400","mapreduce.job.local-fs.single-disk-limit.check.interval-ms":"5000","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","yarn.nodemanager.amrmproxy.address":"0.0.0.0:8049","ipc.server.listen.queue.size":"128","map.sort.class":"org.apache.hadoop.util.QuickSort","fs.viewfs.rename.strategy":"SAME_MOUNTPOINT","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","fs.s3a.assumed.role.credentials.provider":"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider","yarn.nodemanager.vmem-check-enabled":"true","yarn.nodemanager.numa-awareness.enabled":"false","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","yarn.federation.registry.base-dir":"yarnfederation/","mapreduce.job.max.map":"-1","mapreduce.job.local-fs.single-disk-limit.bytes":"-1","mapreduce.job.ubertask.maxreduces":"1","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.jobhistory.jobname.limit":"50","yarn.client.nodemanager-connect.retry-interval-ms":"10000","yarn.timeline-service.state-store-class":"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ","yarn.sharedcache.nested-level":"3","yarn.timeline-service.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","fs.azure.user.agent.prefix":"unknown","yarn.resourcemanager.zk-delegation-token-node.split-index":"*********(redacted)","yarn.nodemanager.numa-awareness.read-topology":"false","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","yarn.registry.class":"org.apache.hadoop.registry.client.impl.FSRegistryOperationsService","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","yarn.scheduler.queue-placement-rules":"user-group","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","yarn.resourcemanager.recovery.enabled":"false","yarn.timeline-service.webapp.rest-csrf.enabled":"false"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.version":"1.8","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Private Build","java.vm.specification.version":"1.8","user.home":"/home/user1","file.encoding.pkg":"sun.io","sun.nio.ch.bugLevel":"","sun.arch.data.model":"64","sun.boot.library.path":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64","user.dir":"/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","java.library.path":"/snap/bin/cmake:/usr/local/cuda-11.2/lib64::/usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib","sun.cpu.isalist":"","sun.desktop":"gnome","os.arch":"amd64","java.vm.version":"25.292-b10","jetty.git.hash":"238ec6997c7806b055319a6d11f8ae7564adc0de","java.endorsed.dirs":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/endorsed","java.runtime.version":"1.8.0_292-8u292-b10-0ubuntu1~18.04-b10","java.vm.info":"mixed mode","java.ext.dirs":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/ext:/usr/java/packages/lib/ext","java.runtime.name":"OpenJDK Runtime Environment","file.separator":"/","java.class.version":"52.0","scala.usejavacp":"true","java.specification.name":"Java Platform API Specification","sun.boot.class.path":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/resources.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/rt.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/sunrsasign.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jsse.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jce.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/charsets.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jfr.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/classes","file.encoding":"UTF-8","user.timezone":"America/Los_Angeles","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","os.version":"5.4.0-80-generic","sun.os.patch.level":"unknown","java.vm.specification.vendor":"Oracle Corporation","user.country":"US","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.url":"http://java.oracle.com/","java.awt.printerjob":"sun.print.PSPrinterJob","java.awt.graphicsenv":"sun.awt.X11GraphicsEnvironment","awt.toolkit":"sun.awt.X11.XToolkit","os.name":"Linux","java.vm.vendor":"Private Build","java.vendor.url.bug":"http://bugreport.sun.com/bugreport/","user.name":"user1","java.vm.name":"OpenJDK 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --conf spark.eventLog.enabled=true --conf spark.eventLog.dir=/home/user1/data_format_eventlog --class org.apache.spark.repl.Main --name Spark shell --jars /home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar spark-shell","java.home":"/usr/lib/jvm/java-8-openjdk-amd64/jre","java.version":"1.8.0_292","sun.io.unicode.encoding":"UnicodeLittle"},"Classpath Entries":{"/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/shapeless_2.12-2.3.3.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/metrics-jvm-4.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-sql_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spire-macros_2.12-0.17.0-M1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/antlr4-runtime-4.8-1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/arpack_combined_all-0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/htrace-core4-4.1.0-incubating.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/machinist_2.12-0.6.8.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jline-2.14.6.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-client-2.30.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jaxb-api-2.2.11.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-yarn-registry-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-common-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/stax2-api-3.1.4.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/osgi-resource-locator-1.0.3.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/aopalliance-1.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/janino-3.0.16.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-dataformat-yaml-2.10.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-module-scala_2.12-2.10.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/scala-collection-compat_2.12-2.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/scala-xml_2.12-1.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/leveldbjni-all-1.8.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spire-util_2.12-0.17.0-M1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/curator-recipes-2.13.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-repl_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/json4s-core_2.12-3.7.0-M5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-core-2.10.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jcl-over-slf4j-1.7.30.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/httpcore-4.4.12.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-metrics-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/generex-1.0.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jpam-1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/JLargeArrays-1.5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-core_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-yarn-server-web-proxy-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/mesos-1.4.0-shaded-protobuf.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jul-to-slf4j-1.7.30.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerby-pkix-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-apiextensions-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/json4s-ast_2.12-3.7.0-M5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/protobuf-java-2.5.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-io-2.5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jta-1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-catalyst_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/activation-1.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-common-2.3.7.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/ivy-2.4.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-container-servlet-2.30.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/py4j-0.10.9.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-codec-1.10.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-admin-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-simplekdc-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/threeten-extra-1.5.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/cats-kernel_2.12-2.0.0-M4.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hk2-locator-2.6.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-hive_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spire-platform_2.12-0.17.0-M1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-network-common_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/minlog-1.3.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/token-provider-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/aopalliance-repackaged-2.6.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/conf/":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/stax-api-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerby-config-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/javassist-3.25.0-GA.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.inject-2.6.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/opencsv-2.3.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-mapreduce-client-core-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/zjsonpatch-0.3.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/univocity-parsers-2.9.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-yarn-api-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-yarn_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-text-1.6.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jsp-api-2.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-kvstore_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-media-jaxb-2.30.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/bonecp-0.8.0.RELEASE.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-networking-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/transaction-api-1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-annotations-2.10.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/datanucleus-api-jdo-4.2.4.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-autoscaling-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/velocity-1.5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-batch-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/okio-1.14.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-streaming_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/arrow-memory-netty-2.0.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-yarn-server-common-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.servlet-api-4.0.3.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-llap-common-2.3.7.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-identity-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/okhttp-3.12.12.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/xz-1.5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/compress-lzf-1.0.3.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/arrow-format-2.0.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-jdbc-2.3.7.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-launcher_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/parquet-column-1.10.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-mapreduce-client-common-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/avro-ipc-1.8.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-mllib_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/shims-0.9.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/parquet-format-2.4.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-crypto-1.1.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/re2j-1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/automaton-1.11-8.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-policy-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/flatbuffers-java-1.9.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/parquet-jackson-1.10.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-tags_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-core-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-annotations-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-auth-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/metrics-json-4.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/parquet-encoding-1.10.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/oro-2.0.8.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/avro-1.8.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jodd-core-3.5.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-admissionregistration-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/metrics-core-4.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/parquet-common-1.10.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/istack-commons-runtime-3.0.8.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/gson-2.2.4.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-module-jaxb-annotations-2.10.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-beanutils-1.9.4.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-mesos_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/snappy-java-1.1.8.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jcip-annotations-1.0-1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/log4j-1.2.17.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerby-asn1-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/accessors-smart-1.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-module-paranamer-2.10.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/xbean-asm7-shaded-4.15.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/scala-compiler-2.12.10.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-daemon-1.0.13.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-dbcp-1.4.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/orc-core-1.5.12.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/aircompressor-0.10.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/okhttp-2.7.5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-jaxrs-json-provider-2.9.5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/metrics-graphite-4.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/stream-2.9.6.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/algebra_2.12-2.0.0-M2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-mllib-local_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-service-rpc-3.1.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-rbac-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/metrics-jmx-4.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/nimbus-jose-jwt-4.41.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-kubernetes_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/slf4j-api-1.7.30.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-common-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-compiler-3.0.16.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-metastore-2.3.7.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/core-1.1.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jaxb-runtime-2.3.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/paranamer-2.8.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/guice-4.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-pool-1.5.4.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/super-csv-2.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-exec-2.3.7-core.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-logging-1.1.3.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-settings-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-network-shuffle_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-tags_2.12-3.1.1-tests.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-configuration2-2.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-common-2.30.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-certificates-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-datatype-jsr310-2.11.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/antlr-runtime-3.5.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/curator-framework-2.13.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/parquet-hadoop-1.10.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/json-1.8.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-core-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/scala-library-2.12.10.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-cli-1.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerby-util-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/JTransforms-3.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/macro-compat_2.12-1.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-databind-2.10.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-hive-thriftserver_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hk2-api-2.6.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-shims-common-2.3.7.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/json4s-scalap_2.12-3.7.0-M5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-server-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/ST4-4.0.4.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.xml.bind-api-2.3.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-graphx_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/slf4j-log4j12-1.7.30.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jdo-api-3.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.ws.rs-api-2.1.6.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-client-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-coordination-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/orc-mapreduce-1.5.12.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-collections-3.2.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-shims-2.3.7.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-httpclient-3.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/breeze-macros_2.12-1.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-storage-api-2.7.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-crypto-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/objenesis-2.6.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/orc-shims-1.5.12.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/libthrift-0.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/snakeyaml-1.24.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-mapreduce-client-jobclient-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/zstd-jni-1.4.8-1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-serde-2.3.7.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/arrow-memory-core-2.0.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/guice-servlet-4.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jackson-jaxrs-base-2.9.5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/javax.inject-1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/datanucleus-rdbms-4.1.19.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/derby-10.12.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kryo-shaded-4.0.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hk2-utils-2.6.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-beeline-2.3.7.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-yarn-client-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/audience-annotations-0.5.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/scala-parser-combinators_2.12-1.1.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-vector-code-gen-2.3.7.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.activation-api-1.2.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-server-2.30.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-events-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-sketch_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/logging-interceptor-3.12.12.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-client-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-hk2-2.30.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-hdfs-client-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.annotation-api-1.3.5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/netty-all-4.1.51.Final.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/pyrolite-4.30.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/httpclient-4.5.6.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/javolution-5.5.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/joda-time-2.10.5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-net-3.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/avro-mapred-1.8.2-hadoop2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-apps-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/dnsjava-2.1.7.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-storageclass-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jsr305-3.0.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/RoaringBitmap-0.9.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/zookeeper-3.4.14.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/chill_2.12-0.9.5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-math3-3.4.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/ehcache-3.3.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/chill-java-0.9.5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/guava-14.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/woodstox-core-5.0.3.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-cli-2.3.7.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-yarn-common-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/json4s-jackson_2.12-3.7.0-M5.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-compress-1.20.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/javax.jdo-3.2.0-m3.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/lz4-java-1.7.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-client-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerb-util-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-discovery-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-scheduling-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/datanucleus-core-4.1.17.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/json-smart-2.3.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jersey-container-servlet-core-2.30.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kerby-xdr-1.0.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spire_2.12-0.17.0-M1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/jakarta.validation-api-2.0.2.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/geronimo-jcache_1.0_spec-1.0-alpha-1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/arrow-vector-2.0.0.jar":"System Classpath","spark://user1.attlocal.net:39937/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar":"Added By User","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-shims-0.23-2.3.7.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/kubernetes-model-extensions-4.12.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-lang-2.6.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/libfb303-0.9.3.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/HikariCP-2.5.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/spark-unsafe_2.12-3.1.1.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/breeze_2.12-1.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/commons-lang3-3.10.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hadoop-common-3.2.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/scala-reflect-2.12.10.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/curator-client-2.13.0.jar":"System Classpath","/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/jars/hive-shims-scheduler-2.3.7.jar":"System Classpath"}} +{"Event":"SparkListenerApplicationStart","App Name":"Spark shell","App ID":"local-1629442299891","Timestamp":1629442298969,"User":"user1"} +{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1629442310154,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"load at :23","Number of Tasks":1,"RDD Info":[{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"load at :23","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"load at :23","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:240)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:23)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:27)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:29)\n$line14.$read$$iw$$iw$$iw$$iw$$iw.(:31)\n$line14.$read$$iw$$iw$$iw$$iw.(:33)\n$line14.$read$$iw$$iw$$iw.(:35)\n$line14.$read$$iw$$iw.(:37)\n$line14.$read$$iw.(:39)\n$line14.$read.(:41)\n$line14.$read$.(:45)\n$line14.$read$.()\n$line14.$eval$.$print$lzycompute(:7)\n$line14.$eval$.$print(:6)\n$line14.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[0],"Properties":{"spark.rdd.scope.noOverride":"true","spark.rdd.scope":"{\"id\":\"2\",\"name\":\"collect\"}"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"load at :23","Number of Tasks":1,"RDD Info":[{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"load at :23","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"load at :23","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:240)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:23)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:27)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:29)\n$line14.$read$$iw$$iw$$iw$$iw$$iw.(:31)\n$line14.$read$$iw$$iw$$iw$$iw.(:33)\n$line14.$read$$iw$$iw$$iw.(:35)\n$line14.$read$$iw$$iw.(:37)\n$line14.$read$$iw.(:39)\n$line14.$read.(:41)\n$line14.$read$.(:45)\n$line14.$read$.()\n$line14.$eval$.$print$lzycompute(:7)\n$line14.$eval$.$print(:6)\n$line14.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Submission Time":1629442310181,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.rdd.scope.noOverride":"true","spark.rdd.scope":"{\"id\":\"2\",\"name\":\"collect\"}"}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1629442310377,"Executor ID":"driver","Host":"user1.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1629442310377,"Executor ID":"driver","Host":"user1.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1629442310978,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"internal.metrics.executorDeserializeTime","Update":282,"Value":282,"Internal":true,"Count Failed Values":true},{"ID":1,"Name":"internal.metrics.executorDeserializeCpuTime","Update":277613202,"Value":277613202,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"internal.metrics.executorRunTime","Update":230,"Value":230,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"internal.metrics.executorCpuTime","Update":46445578,"Value":46445578,"Internal":true,"Count Failed Values":true},{"ID":4,"Name":"internal.metrics.resultSize","Update":2012,"Value":2012,"Internal":true,"Count Failed Values":true},{"ID":6,"Name":"internal.metrics.resultSerializationTime","Update":5,"Value":5,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":282,"Executor Deserialize CPU Time":277613202,"Executor Run Time":230,"Executor CPU Time":46445578,"Peak Execution Memory":0,"Result Size":2012,"JVM GC Time":0,"Result Serialization Time":5,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"load at :23","Number of Tasks":1,"RDD Info":[{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"load at :23","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"load at :23","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:240)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:23)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:27)\n$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:29)\n$line14.$read$$iw$$iw$$iw$$iw$$iw.(:31)\n$line14.$read$$iw$$iw$$iw$$iw.(:33)\n$line14.$read$$iw$$iw$$iw.(:35)\n$line14.$read$$iw$$iw.(:37)\n$line14.$read$$iw.(:39)\n$line14.$read.(:41)\n$line14.$read$.(:45)\n$line14.$read$.()\n$line14.$eval$.$print$lzycompute(:7)\n$line14.$eval$.$print(:6)\n$line14.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Submission Time":1629442310181,"Completion Time":1629442310990,"Accumulables":[{"ID":0,"Name":"internal.metrics.executorDeserializeTime","Value":282,"Internal":true,"Count Failed Values":true},{"ID":1,"Name":"internal.metrics.executorDeserializeCpuTime","Value":277613202,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"internal.metrics.executorRunTime","Value":230,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"internal.metrics.executorCpuTime","Value":46445578,"Internal":true,"Count Failed Values":true},{"ID":4,"Name":"internal.metrics.resultSize","Value":2012,"Internal":true,"Count Failed Values":true},{"ID":6,"Name":"internal.metrics.resultSerializationTime","Value":5,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1629442310997,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":0,"description":"save at :26","details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:30)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line15.$read$$iw$$iw$$iw$$iw$$iw.(:34)\n$line15.$read$$iw$$iw$$iw$$iw.(:36)\n$line15.$read$$iw$$iw$$iw.(:38)\n$line15.$read$$iw$$iw.(:40)\n$line15.$read$$iw.(:42)\n$line15.$read.(:44)\n$line15.$read$.(:48)\n$line15.$read$.()\n$line15.$eval$.$print$lzycompute(:7)\n$line15.$eval$.$print(:6)\n$line15.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","physicalPlanDescription":"== Physical Plan ==\nExecute InsertIntoHadoopFsRelationCommand (3)\n+- * ColumnarToRow (2)\n +- Scan parquet (1)\n\n\n(1) Scan parquet \nOutput [1]: [value#0]\nBatched: true\nLocation: InMemoryFileIndex [file:/home/user1/event_logs_spark/nested_type/smalldec.parquet]\nReadSchema: struct\n\n(2) ColumnarToRow [codegen id : 1]\nInput [1]: [value#0]\n\n(3) Execute InsertIntoHadoopFsRelationCommand\nInput [1]: [value#0]\nArguments: file:/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/writeDF.csv, false, CSV, Map(path -> writeDF.csv), ErrorIfExists, [value]\n\n","sparkPlanInfo":{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand file:/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/writeDF.csv, false, CSV, Map(path -> writeDF.csv), ErrorIfExists, [value]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [value#0] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex[file:/home/user1/event_logs_spark/nested_type/smalldec.parquet], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[file:/home/user1/event_logs_spark/nested_type/smalldec.parquet]","ReadSchema":"struct","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":33,"metricType":"sum"},{"name":"scan time","accumulatorId":36,"metricType":"timing"},{"name":"metadata time","accumulatorId":34,"metricType":"timing"},{"name":"size of files read","accumulatorId":35,"metricType":"size"},{"name":"number of output rows","accumulatorId":32,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":30,"metricType":"sum"},{"name":"number of input batches","accumulatorId":31,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":29,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of written files","accumulatorId":25,"metricType":"sum"},{"name":"written output","accumulatorId":26,"metricType":"size"},{"name":"number of output rows","accumulatorId":27,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":28,"metricType":"sum"}]},"time":1629442312300} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[33,1],[34,3],[35,447]]} +{"Event":"SparkListenerJobStart","Job ID":1,"Submission Time":1629442312927,"Stage Infos":[{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"save at :26","Number of Tasks":1,"RDD Info":[{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"6\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at :26","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"FileScanRDD","Scope":"{\"id\":\"9\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:30)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line15.$read$$iw$$iw$$iw$$iw$$iw.(:34)\n$line15.$read$$iw$$iw$$iw$$iw.(:36)\n$line15.$read$$iw$$iw$$iw.(:38)\n$line15.$read$$iw$$iw.(:40)\n$line15.$read$$iw.(:42)\n$line15.$read.(:44)\n$line15.$read$.(:48)\n$line15.$read$.()\n$line15.$eval$.$print$lzycompute(:7)\n$line15.$eval$.$print(:6)\n$line15.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[1],"Properties":{"spark.sql.warehouse.dir":"file:/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/spark-warehouse/","spark.driver.host":"user1.attlocal.net","spark.eventLog.enabled":"true","spark.driver.port":"39937","spark.repl.class.uri":"spark://user1.attlocal.net:39937/classes","spark.jars":"file:///home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.repl.class.outputDir":"/tmp/spark-655d1756-32df-40e3-81e9-42ce076c6c55/repl-7d342e8a-9aae-463e-be0c-94065166b845","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"5\",\"name\":\"Execute InsertIntoHadoopFsRelationCommand\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.startTime":"1629442298969","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://user1.attlocal.net:39937/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.submit.deployMode":"client","spark.master":"local[*]","spark.home":"/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","spark.eventLog.dir":"/home/user1/data_format_eventlog","spark.sql.execution.id":"0","spark.sql.catalogImplementation":"hive","spark.repl.local.jars":"file:///home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.app.id":"local-1629442299891"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"save at :26","Number of Tasks":1,"RDD Info":[{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"6\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at :26","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"FileScanRDD","Scope":"{\"id\":\"9\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:30)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line15.$read$$iw$$iw$$iw$$iw$$iw.(:34)\n$line15.$read$$iw$$iw$$iw$$iw.(:36)\n$line15.$read$$iw$$iw$$iw.(:38)\n$line15.$read$$iw$$iw.(:40)\n$line15.$read$$iw.(:42)\n$line15.$read.(:44)\n$line15.$read$.(:48)\n$line15.$read$.()\n$line15.$eval$.$print$lzycompute(:7)\n$line15.$eval$.$print(:6)\n$line15.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Submission Time":1629442312931,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/spark-warehouse/","spark.driver.host":"user1.attlocal.net","spark.eventLog.enabled":"true","spark.driver.port":"39937","spark.repl.class.uri":"spark://user1.attlocal.net:39937/classes","spark.jars":"file:///home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.repl.class.outputDir":"/tmp/spark-655d1756-32df-40e3-81e9-42ce076c6c55/repl-7d342e8a-9aae-463e-be0c-94065166b845","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"5\",\"name\":\"Execute InsertIntoHadoopFsRelationCommand\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.startTime":"1629442298969","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://user1.attlocal.net:39937/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.submit.deployMode":"client","spark.master":"local[*]","spark.home":"/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","spark.eventLog.dir":"/home/user1/data_format_eventlog","spark.sql.execution.id":"0","spark.sql.catalogImplementation":"hive","spark.repl.local.jars":"file:///home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.app.id":"local-1629442299891"}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Launch Time":1629442312972,"Executor ID":"driver","Host":"user1.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Launch Time":1629442312972,"Executor ID":"driver","Host":"user1.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1629442313504,"Failed":false,"Killed":false,"Accumulables":[{"ID":29,"Name":"duration","Update":"418","Value":"418","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"number of input batches","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"scan time","Update":"357","Value":"357","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":37,"Name":"internal.metrics.executorDeserializeTime","Update":77,"Value":77,"Internal":true,"Count Failed Values":true},{"ID":38,"Name":"internal.metrics.executorDeserializeCpuTime","Update":76242156,"Value":76242156,"Internal":true,"Count Failed Values":true},{"ID":39,"Name":"internal.metrics.executorRunTime","Update":443,"Value":443,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.executorCpuTime","Update":408035307,"Value":408035307,"Internal":true,"Count Failed Values":true},{"ID":41,"Name":"internal.metrics.resultSize","Update":2739,"Value":2739,"Internal":true,"Count Failed Values":true},{"ID":42,"Name":"internal.metrics.jvmGCTime","Update":17,"Value":17,"Internal":true,"Count Failed Values":true},{"ID":43,"Name":"internal.metrics.resultSerializationTime","Update":4,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":58,"Name":"internal.metrics.input.bytesRead","Update":1377,"Value":1377,"Internal":true,"Count Failed Values":true},{"ID":59,"Name":"internal.metrics.input.recordsRead","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":60,"Name":"internal.metrics.output.bytesWritten","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":61,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":77,"Executor Deserialize CPU Time":76242156,"Executor Run Time":443,"Executor CPU Time":408035307,"Peak Execution Memory":0,"Result Size":2739,"JVM GC Time":17,"Result Serialization Time":4,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":1377,"Records Read":1},"Output Metrics":{"Bytes Written":5,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"save at :26","Number of Tasks":1,"RDD Info":[{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"6\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at :26","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"FileScanRDD","Scope":"{\"id\":\"9\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:30)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line15.$read$$iw$$iw$$iw$$iw$$iw.(:34)\n$line15.$read$$iw$$iw$$iw$$iw.(:36)\n$line15.$read$$iw$$iw$$iw.(:38)\n$line15.$read$$iw$$iw.(:40)\n$line15.$read$$iw.(:42)\n$line15.$read.(:44)\n$line15.$read$.(:48)\n$line15.$read$.()\n$line15.$eval$.$print$lzycompute(:7)\n$line15.$eval$.$print(:6)\n$line15.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Submission Time":1629442312931,"Completion Time":1629442313505,"Accumulables":[{"ID":29,"Name":"duration","Value":"418","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"number of input batches","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"scan time","Value":"357","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":37,"Name":"internal.metrics.executorDeserializeTime","Value":77,"Internal":true,"Count Failed Values":true},{"ID":38,"Name":"internal.metrics.executorDeserializeCpuTime","Value":76242156,"Internal":true,"Count Failed Values":true},{"ID":39,"Name":"internal.metrics.executorRunTime","Value":443,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.executorCpuTime","Value":408035307,"Internal":true,"Count Failed Values":true},{"ID":41,"Name":"internal.metrics.resultSize","Value":2739,"Internal":true,"Count Failed Values":true},{"ID":42,"Name":"internal.metrics.jvmGCTime","Value":17,"Internal":true,"Count Failed Values":true},{"ID":43,"Name":"internal.metrics.resultSerializationTime","Value":4,"Internal":true,"Count Failed Values":true},{"ID":58,"Name":"internal.metrics.input.bytesRead","Value":1377,"Internal":true,"Count Failed Values":true},{"ID":59,"Name":"internal.metrics.input.recordsRead","Value":1,"Internal":true,"Count Failed Values":true},{"ID":60,"Name":"internal.metrics.output.bytesWritten","Value":5,"Internal":true,"Count Failed Values":true},{"ID":61,"Name":"internal.metrics.output.recordsWritten","Value":1,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":1,"Completion Time":1629442313506,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[25,1],[26,5],[27,1],[28,0]]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":0,"time":1629442313535} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":1,"description":"save at :26","details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:30)\n$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line16.$read$$iw$$iw$$iw$$iw$$iw.(:34)\n$line16.$read$$iw$$iw$$iw$$iw.(:36)\n$line16.$read$$iw$$iw$$iw.(:38)\n$line16.$read$$iw$$iw.(:40)\n$line16.$read$$iw.(:42)\n$line16.$read.(:44)\n$line16.$read$.(:48)\n$line16.$read$.()\n$line16.$eval$.$print$lzycompute(:7)\n$line16.$eval$.$print(:6)\n$line16.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","physicalPlanDescription":"== Physical Plan ==\nExecute InsertIntoHadoopFsRelationCommand (3)\n+- * ColumnarToRow (2)\n +- Scan parquet (1)\n\n\n(1) Scan parquet \nOutput [1]: [value#0]\nBatched: true\nLocation: InMemoryFileIndex [file:/home/user1/event_logs_spark/nested_type/smalldec.parquet]\nReadSchema: struct\n\n(2) ColumnarToRow [codegen id : 1]\nInput [1]: [value#0]\n\n(3) Execute InsertIntoHadoopFsRelationCommand\nInput [1]: [value#0]\nArguments: file:/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/writeDF.parquet, false, Parquet, Map(path -> writeDF.parquet), ErrorIfExists, [value]\n\n","sparkPlanInfo":{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand file:/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/writeDF.parquet, false, Parquet, Map(path -> writeDF.parquet), ErrorIfExists, [value]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [value#0] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex[file:/home/user1/event_logs_spark/nested_type/smalldec.parquet], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[file:/home/user1/event_logs_spark/nested_type/smalldec.parquet]","ReadSchema":"struct","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":70,"metricType":"sum"},{"name":"scan time","accumulatorId":73,"metricType":"timing"},{"name":"metadata time","accumulatorId":71,"metricType":"timing"},{"name":"size of files read","accumulatorId":72,"metricType":"size"},{"name":"number of output rows","accumulatorId":69,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":67,"metricType":"sum"},{"name":"number of input batches","accumulatorId":68,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":66,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of written files","accumulatorId":62,"metricType":"sum"},{"name":"written output","accumulatorId":63,"metricType":"size"},{"name":"number of output rows","accumulatorId":64,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":65,"metricType":"sum"}]},"time":1629442313850} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[70,1],[71,0],[72,447]]} +{"Event":"SparkListenerJobStart","Job ID":2,"Submission Time":1629442313929,"Stage Infos":[{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"save at :26","Number of Tasks":1,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"14\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at :26","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"FileScanRDD","Scope":"{\"id\":\"17\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"17\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:30)\n$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line16.$read$$iw$$iw$$iw$$iw$$iw.(:34)\n$line16.$read$$iw$$iw$$iw$$iw.(:36)\n$line16.$read$$iw$$iw$$iw.(:38)\n$line16.$read$$iw$$iw.(:40)\n$line16.$read$$iw.(:42)\n$line16.$read.(:44)\n$line16.$read$.(:48)\n$line16.$read$.()\n$line16.$eval$.$print$lzycompute(:7)\n$line16.$eval$.$print(:6)\n$line16.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[2],"Properties":{"spark.sql.warehouse.dir":"file:/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/spark-warehouse/","spark.driver.host":"user1.attlocal.net","spark.eventLog.enabled":"true","spark.driver.port":"39937","spark.repl.class.uri":"spark://user1.attlocal.net:39937/classes","spark.jars":"file:///home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.repl.class.outputDir":"/tmp/spark-655d1756-32df-40e3-81e9-42ce076c6c55/repl-7d342e8a-9aae-463e-be0c-94065166b845","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"13\",\"name\":\"Execute InsertIntoHadoopFsRelationCommand\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.startTime":"1629442298969","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://user1.attlocal.net:39937/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.submit.deployMode":"client","spark.master":"local[*]","spark.home":"/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","spark.eventLog.dir":"/home/user1/data_format_eventlog","spark.sql.execution.id":"1","spark.sql.catalogImplementation":"hive","spark.repl.local.jars":"file:///home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.app.id":"local-1629442299891"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"save at :26","Number of Tasks":1,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"14\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at :26","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"FileScanRDD","Scope":"{\"id\":\"17\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"17\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:30)\n$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line16.$read$$iw$$iw$$iw$$iw$$iw.(:34)\n$line16.$read$$iw$$iw$$iw$$iw.(:36)\n$line16.$read$$iw$$iw$$iw.(:38)\n$line16.$read$$iw$$iw.(:40)\n$line16.$read$$iw.(:42)\n$line16.$read.(:44)\n$line16.$read$.(:48)\n$line16.$read$.()\n$line16.$eval$.$print$lzycompute(:7)\n$line16.$eval$.$print(:6)\n$line16.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Submission Time":1629442313931,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/spark-warehouse/","spark.driver.host":"user1.attlocal.net","spark.eventLog.enabled":"true","spark.driver.port":"39937","spark.repl.class.uri":"spark://user1.attlocal.net:39937/classes","spark.jars":"file:///home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.repl.class.outputDir":"/tmp/spark-655d1756-32df-40e3-81e9-42ce076c6c55/repl-7d342e8a-9aae-463e-be0c-94065166b845","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"13\",\"name\":\"Execute InsertIntoHadoopFsRelationCommand\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.startTime":"1629442298969","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://user1.attlocal.net:39937/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.submit.deployMode":"client","spark.master":"local[*]","spark.home":"/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","spark.eventLog.dir":"/home/user1/data_format_eventlog","spark.sql.execution.id":"1","spark.sql.catalogImplementation":"hive","spark.repl.local.jars":"file:///home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.app.id":"local-1629442299891"}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":2,"Index":0,"Attempt":0,"Launch Time":1629442313956,"Executor ID":"driver","Host":"user1.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":2,"Index":0,"Attempt":0,"Launch Time":1629442313956,"Executor ID":"driver","Host":"user1.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1629442314084,"Failed":false,"Killed":false,"Accumulables":[{"ID":66,"Name":"duration","Update":"89","Value":"89","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":68,"Name":"number of input batches","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":69,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":73,"Name":"scan time","Update":"6","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"internal.metrics.executorDeserializeTime","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":75,"Name":"internal.metrics.executorDeserializeCpuTime","Update":20081269,"Value":20081269,"Internal":true,"Count Failed Values":true},{"ID":76,"Name":"internal.metrics.executorRunTime","Update":103,"Value":103,"Internal":true,"Count Failed Values":true},{"ID":77,"Name":"internal.metrics.executorCpuTime","Update":91724024,"Value":91724024,"Internal":true,"Count Failed Values":true},{"ID":78,"Name":"internal.metrics.resultSize","Update":2653,"Value":2653,"Internal":true,"Count Failed Values":true},{"ID":95,"Name":"internal.metrics.input.bytesRead","Update":1377,"Value":1377,"Internal":true,"Count Failed Values":true},{"ID":96,"Name":"internal.metrics.input.recordsRead","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":97,"Name":"internal.metrics.output.bytesWritten","Update":447,"Value":447,"Internal":true,"Count Failed Values":true},{"ID":98,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":20,"Executor Deserialize CPU Time":20081269,"Executor Run Time":103,"Executor CPU Time":91724024,"Peak Execution Memory":0,"Result Size":2653,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":1377,"Records Read":1},"Output Metrics":{"Bytes Written":447,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"save at :26","Number of Tasks":1,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"14\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at :26","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"FileScanRDD","Scope":"{\"id\":\"17\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"17\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:30)\n$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line16.$read$$iw$$iw$$iw$$iw$$iw.(:34)\n$line16.$read$$iw$$iw$$iw$$iw.(:36)\n$line16.$read$$iw$$iw$$iw.(:38)\n$line16.$read$$iw$$iw.(:40)\n$line16.$read$$iw.(:42)\n$line16.$read.(:44)\n$line16.$read$.(:48)\n$line16.$read$.()\n$line16.$eval$.$print$lzycompute(:7)\n$line16.$eval$.$print(:6)\n$line16.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Submission Time":1629442313931,"Completion Time":1629442314085,"Accumulables":[{"ID":66,"Name":"duration","Value":"89","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":68,"Name":"number of input batches","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":69,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":73,"Name":"scan time","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"internal.metrics.executorDeserializeTime","Value":20,"Internal":true,"Count Failed Values":true},{"ID":75,"Name":"internal.metrics.executorDeserializeCpuTime","Value":20081269,"Internal":true,"Count Failed Values":true},{"ID":76,"Name":"internal.metrics.executorRunTime","Value":103,"Internal":true,"Count Failed Values":true},{"ID":77,"Name":"internal.metrics.executorCpuTime","Value":91724024,"Internal":true,"Count Failed Values":true},{"ID":78,"Name":"internal.metrics.resultSize","Value":2653,"Internal":true,"Count Failed Values":true},{"ID":95,"Name":"internal.metrics.input.bytesRead","Value":1377,"Internal":true,"Count Failed Values":true},{"ID":96,"Name":"internal.metrics.input.recordsRead","Value":1,"Internal":true,"Count Failed Values":true},{"ID":97,"Name":"internal.metrics.output.bytesWritten","Value":447,"Internal":true,"Count Failed Values":true},{"ID":98,"Name":"internal.metrics.output.recordsWritten","Value":1,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":2,"Completion Time":1629442314085,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[62,1],[63,447],[64,1],[65,0]]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":1,"time":1629442314099} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":2,"description":"save at :26","details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:30)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line17.$read$$iw$$iw$$iw$$iw$$iw.(:34)\n$line17.$read$$iw$$iw$$iw$$iw.(:36)\n$line17.$read$$iw$$iw$$iw.(:38)\n$line17.$read$$iw$$iw.(:40)\n$line17.$read$$iw.(:42)\n$line17.$read.(:44)\n$line17.$read$.(:48)\n$line17.$read$.()\n$line17.$eval$.$print$lzycompute(:7)\n$line17.$eval$.$print(:6)\n$line17.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","physicalPlanDescription":"== Physical Plan ==\nExecute InsertIntoHadoopFsRelationCommand (3)\n+- * ColumnarToRow (2)\n +- Scan parquet (1)\n\n\n(1) Scan parquet \nOutput [1]: [value#0]\nBatched: true\nLocation: InMemoryFileIndex [file:/home/user1/event_logs_spark/nested_type/smalldec.parquet]\nReadSchema: struct\n\n(2) ColumnarToRow [codegen id : 1]\nInput [1]: [value#0]\n\n(3) Execute InsertIntoHadoopFsRelationCommand\nInput [1]: [value#0]\nArguments: file:/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/writeDF.orc, false, ORC, Map(path -> writeDF.orc), ErrorIfExists, [value]\n\n","sparkPlanInfo":{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand file:/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/writeDF.orc, false, ORC, Map(path -> writeDF.orc), ErrorIfExists, [value]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [value#0] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex[file:/home/user1/event_logs_spark/nested_type/smalldec.parquet], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[file:/home/user1/event_logs_spark/nested_type/smalldec.parquet]","ReadSchema":"struct","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":107,"metricType":"sum"},{"name":"scan time","accumulatorId":110,"metricType":"timing"},{"name":"metadata time","accumulatorId":108,"metricType":"timing"},{"name":"size of files read","accumulatorId":109,"metricType":"size"},{"name":"number of output rows","accumulatorId":106,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":104,"metricType":"sum"},{"name":"number of input batches","accumulatorId":105,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":103,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of written files","accumulatorId":99,"metricType":"sum"},{"name":"written output","accumulatorId":100,"metricType":"size"},{"name":"number of output rows","accumulatorId":101,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":102,"metricType":"sum"}]},"time":1629442314383} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[107,1],[108,0],[109,447]]} +{"Event":"SparkListenerJobStart","Job ID":3,"Submission Time":1629442314445,"Stage Infos":[{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"save at :26","Number of Tasks":1,"RDD Info":[{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"22\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at :26","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"FileScanRDD","Scope":"{\"id\":\"25\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"25\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:30)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line17.$read$$iw$$iw$$iw$$iw$$iw.(:34)\n$line17.$read$$iw$$iw$$iw$$iw.(:36)\n$line17.$read$$iw$$iw$$iw.(:38)\n$line17.$read$$iw$$iw.(:40)\n$line17.$read$$iw.(:42)\n$line17.$read.(:44)\n$line17.$read$.(:48)\n$line17.$read$.()\n$line17.$eval$.$print$lzycompute(:7)\n$line17.$eval$.$print(:6)\n$line17.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[3],"Properties":{"spark.sql.warehouse.dir":"file:/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/spark-warehouse/","spark.driver.host":"user1.attlocal.net","spark.eventLog.enabled":"true","spark.driver.port":"39937","spark.repl.class.uri":"spark://user1.attlocal.net:39937/classes","spark.jars":"file:///home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.repl.class.outputDir":"/tmp/spark-655d1756-32df-40e3-81e9-42ce076c6c55/repl-7d342e8a-9aae-463e-be0c-94065166b845","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"21\",\"name\":\"Execute InsertIntoHadoopFsRelationCommand\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.startTime":"1629442298969","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://user1.attlocal.net:39937/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.submit.deployMode":"client","spark.master":"local[*]","spark.home":"/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","spark.eventLog.dir":"/home/user1/data_format_eventlog","spark.sql.execution.id":"2","spark.sql.catalogImplementation":"hive","spark.repl.local.jars":"file:///home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.app.id":"local-1629442299891"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"save at :26","Number of Tasks":1,"RDD Info":[{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"22\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at :26","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"FileScanRDD","Scope":"{\"id\":\"25\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"25\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:30)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line17.$read$$iw$$iw$$iw$$iw$$iw.(:34)\n$line17.$read$$iw$$iw$$iw$$iw.(:36)\n$line17.$read$$iw$$iw$$iw.(:38)\n$line17.$read$$iw$$iw.(:40)\n$line17.$read$$iw.(:42)\n$line17.$read.(:44)\n$line17.$read$.(:48)\n$line17.$read$.()\n$line17.$eval$.$print$lzycompute(:7)\n$line17.$eval$.$print(:6)\n$line17.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Submission Time":1629442314448,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/spark-warehouse/","spark.driver.host":"user1.attlocal.net","spark.eventLog.enabled":"true","spark.driver.port":"39937","spark.repl.class.uri":"spark://user1.attlocal.net:39937/classes","spark.jars":"file:///home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.repl.class.outputDir":"/tmp/spark-655d1756-32df-40e3-81e9-42ce076c6c55/repl-7d342e8a-9aae-463e-be0c-94065166b845","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"21\",\"name\":\"Execute InsertIntoHadoopFsRelationCommand\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.startTime":"1629442298969","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://user1.attlocal.net:39937/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.submit.deployMode":"client","spark.master":"local[*]","spark.home":"/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","spark.eventLog.dir":"/home/user1/data_format_eventlog","spark.sql.execution.id":"2","spark.sql.catalogImplementation":"hive","spark.repl.local.jars":"file:///home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.app.id":"local-1629442299891"}} +{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":3,"Index":0,"Attempt":0,"Launch Time":1629442314482,"Executor ID":"driver","Host":"user1.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":3,"Index":0,"Attempt":0,"Launch Time":1629442314482,"Executor ID":"driver","Host":"user1.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1629442314693,"Failed":false,"Killed":false,"Accumulables":[{"ID":103,"Name":"duration","Update":"130","Value":"130","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":104,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":105,"Name":"number of input batches","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":106,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":110,"Name":"scan time","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":111,"Name":"internal.metrics.executorDeserializeTime","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.executorDeserializeCpuTime","Update":20738373,"Value":20738373,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.executorRunTime","Update":185,"Value":185,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.executorCpuTime","Update":176884789,"Value":176884789,"Internal":true,"Count Failed Values":true},{"ID":115,"Name":"internal.metrics.resultSize","Update":2653,"Value":2653,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.input.bytesRead","Update":1377,"Value":1377,"Internal":true,"Count Failed Values":true},{"ID":133,"Name":"internal.metrics.input.recordsRead","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.output.bytesWritten","Update":288,"Value":288,"Internal":true,"Count Failed Values":true},{"ID":135,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":20,"Executor Deserialize CPU Time":20738373,"Executor Run Time":185,"Executor CPU Time":176884789,"Peak Execution Memory":0,"Result Size":2653,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":1377,"Records Read":1},"Output Metrics":{"Bytes Written":288,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"save at :26","Number of Tasks":1,"RDD Info":[{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"22\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at :26","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"FileScanRDD","Scope":"{\"id\":\"25\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"25\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:30)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line17.$read$$iw$$iw$$iw$$iw$$iw.(:34)\n$line17.$read$$iw$$iw$$iw$$iw.(:36)\n$line17.$read$$iw$$iw$$iw.(:38)\n$line17.$read$$iw$$iw.(:40)\n$line17.$read$$iw.(:42)\n$line17.$read.(:44)\n$line17.$read$.(:48)\n$line17.$read$.()\n$line17.$eval$.$print$lzycompute(:7)\n$line17.$eval$.$print(:6)\n$line17.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Submission Time":1629442314448,"Completion Time":1629442314694,"Accumulables":[{"ID":103,"Name":"duration","Value":"130","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":104,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":105,"Name":"number of input batches","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":106,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":110,"Name":"scan time","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":111,"Name":"internal.metrics.executorDeserializeTime","Value":20,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.executorDeserializeCpuTime","Value":20738373,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.executorRunTime","Value":185,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.executorCpuTime","Value":176884789,"Internal":true,"Count Failed Values":true},{"ID":115,"Name":"internal.metrics.resultSize","Value":2653,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.input.bytesRead","Value":1377,"Internal":true,"Count Failed Values":true},{"ID":133,"Name":"internal.metrics.input.recordsRead","Value":1,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.output.bytesWritten","Value":288,"Internal":true,"Count Failed Values":true},{"ID":135,"Name":"internal.metrics.output.recordsWritten","Value":1,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":3,"Completion Time":1629442314694,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[99,1],[100,288],[101,1],[102,0]]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":2,"time":1629442314709} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":3,"description":"save at :26","details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line18.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line18.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:30)\n$line18.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line18.$read$$iw$$iw$$iw$$iw$$iw.(:34)\n$line18.$read$$iw$$iw$$iw$$iw.(:36)\n$line18.$read$$iw$$iw$$iw.(:38)\n$line18.$read$$iw$$iw.(:40)\n$line18.$read$$iw.(:42)\n$line18.$read.(:44)\n$line18.$read$.(:48)\n$line18.$read$.()\n$line18.$eval$.$print$lzycompute(:7)\n$line18.$eval$.$print(:6)\n$line18.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","physicalPlanDescription":"== Physical Plan ==\nExecute InsertIntoHadoopFsRelationCommand (3)\n+- * ColumnarToRow (2)\n +- Scan parquet (1)\n\n\n(1) Scan parquet \nOutput [1]: [value#0]\nBatched: true\nLocation: InMemoryFileIndex [file:/home/user1/event_logs_spark/nested_type/smalldec.parquet]\nReadSchema: struct\n\n(2) ColumnarToRow [codegen id : 1]\nInput [1]: [value#0]\n\n(3) Execute InsertIntoHadoopFsRelationCommand\nInput [1]: [value#0]\nArguments: file:/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/writeDF.json, false, JSON, Map(path -> writeDF.json), ErrorIfExists, [value]\n\n","sparkPlanInfo":{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand file:/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/writeDF.json, false, JSON, Map(path -> writeDF.json), ErrorIfExists, [value]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [value#0] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex[file:/home/user1/event_logs_spark/nested_type/smalldec.parquet], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[file:/home/user1/event_logs_spark/nested_type/smalldec.parquet]","ReadSchema":"struct","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":144,"metricType":"sum"},{"name":"scan time","accumulatorId":147,"metricType":"timing"},{"name":"metadata time","accumulatorId":145,"metricType":"timing"},{"name":"size of files read","accumulatorId":146,"metricType":"size"},{"name":"number of output rows","accumulatorId":143,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":141,"metricType":"sum"},{"name":"number of input batches","accumulatorId":142,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":140,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of written files","accumulatorId":136,"metricType":"sum"},{"name":"written output","accumulatorId":137,"metricType":"size"},{"name":"number of output rows","accumulatorId":138,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":139,"metricType":"sum"}]},"time":1629442316427} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":3,"accumUpdates":[[144,1],[145,0],[146,447]]} +{"Event":"SparkListenerJobStart","Job ID":4,"Submission Time":1629442316520,"Stage Infos":[{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"save at :26","Number of Tasks":1,"RDD Info":[{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"30\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at :26","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":17,"Name":"FileScanRDD","Scope":"{\"id\":\"33\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"33\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line18.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line18.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:30)\n$line18.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line18.$read$$iw$$iw$$iw$$iw$$iw.(:34)\n$line18.$read$$iw$$iw$$iw$$iw.(:36)\n$line18.$read$$iw$$iw$$iw.(:38)\n$line18.$read$$iw$$iw.(:40)\n$line18.$read$$iw.(:42)\n$line18.$read.(:44)\n$line18.$read$.(:48)\n$line18.$read$.()\n$line18.$eval$.$print$lzycompute(:7)\n$line18.$eval$.$print(:6)\n$line18.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[4],"Properties":{"spark.sql.warehouse.dir":"file:/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/spark-warehouse/","spark.driver.host":"user1.attlocal.net","spark.eventLog.enabled":"true","spark.driver.port":"39937","spark.repl.class.uri":"spark://user1.attlocal.net:39937/classes","spark.jars":"file:///home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.repl.class.outputDir":"/tmp/spark-655d1756-32df-40e3-81e9-42ce076c6c55/repl-7d342e8a-9aae-463e-be0c-94065166b845","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"29\",\"name\":\"Execute InsertIntoHadoopFsRelationCommand\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.startTime":"1629442298969","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://user1.attlocal.net:39937/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.submit.deployMode":"client","spark.master":"local[*]","spark.home":"/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","spark.eventLog.dir":"/home/user1/data_format_eventlog","spark.sql.execution.id":"3","spark.sql.catalogImplementation":"hive","spark.repl.local.jars":"file:///home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.app.id":"local-1629442299891"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"save at :26","Number of Tasks":1,"RDD Info":[{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"30\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at :26","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":17,"Name":"FileScanRDD","Scope":"{\"id\":\"33\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"33\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line18.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line18.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:30)\n$line18.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line18.$read$$iw$$iw$$iw$$iw$$iw.(:34)\n$line18.$read$$iw$$iw$$iw$$iw.(:36)\n$line18.$read$$iw$$iw$$iw.(:38)\n$line18.$read$$iw$$iw.(:40)\n$line18.$read$$iw.(:42)\n$line18.$read.(:44)\n$line18.$read$.(:48)\n$line18.$read$.()\n$line18.$eval$.$print$lzycompute(:7)\n$line18.$eval$.$print(:6)\n$line18.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Submission Time":1629442316522,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2/spark-warehouse/","spark.driver.host":"user1.attlocal.net","spark.eventLog.enabled":"true","spark.driver.port":"39937","spark.repl.class.uri":"spark://user1.attlocal.net:39937/classes","spark.jars":"file:///home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.repl.class.outputDir":"/tmp/spark-655d1756-32df-40e3-81e9-42ce076c6c55/repl-7d342e8a-9aae-463e-be0c-94065166b845","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"29\",\"name\":\"Execute InsertIntoHadoopFsRelationCommand\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.app.startTime":"1629442298969","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://user1.attlocal.net:39937/jars/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.submit.deployMode":"client","spark.master":"local[*]","spark.home":"/home/user1/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","spark.eventLog.dir":"/home/user1/data_format_eventlog","spark.sql.execution.id":"3","spark.sql.catalogImplementation":"hive","spark.repl.local.jars":"file:///home/user1/spark-rapids-July19/spark-rapids/tools/target/rapids-4-spark-tools_2.12-21.10.0-SNAPSHOT.jar","spark.app.id":"local-1629442299891"}} +{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":4,"Index":0,"Attempt":0,"Launch Time":1629442316547,"Executor ID":"driver","Host":"user1.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":4,"Index":0,"Attempt":0,"Launch Time":1629442316547,"Executor ID":"driver","Host":"user1.attlocal.net","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1629442316596,"Failed":false,"Killed":false,"Accumulables":[{"ID":140,"Name":"duration","Update":"27","Value":"27","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":142,"Name":"number of input batches","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":143,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":147,"Name":"scan time","Update":"6","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":148,"Name":"internal.metrics.executorDeserializeTime","Update":14,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.executorDeserializeCpuTime","Update":14661400,"Value":14661400,"Internal":true,"Count Failed Values":true},{"ID":150,"Name":"internal.metrics.executorRunTime","Update":30,"Value":30,"Internal":true,"Count Failed Values":true},{"ID":151,"Name":"internal.metrics.executorCpuTime","Update":23473257,"Value":23473257,"Internal":true,"Count Failed Values":true},{"ID":152,"Name":"internal.metrics.resultSize","Update":2653,"Value":2653,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.input.bytesRead","Update":1377,"Value":1377,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.input.recordsRead","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.output.bytesWritten","Update":15,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.output.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":14,"Executor Deserialize CPU Time":14661400,"Executor Run Time":30,"Executor CPU Time":23473257,"Peak Execution Memory":0,"Result Size":2653,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":1377,"Records Read":1},"Output Metrics":{"Bytes Written":15,"Records Written":1},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"save at :26","Number of Tasks":1,"RDD Info":[{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"30\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at :26","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":17,"Name":"FileScanRDD","Scope":"{\"id\":\"33\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"33\",\"name\":\"Scan parquet \"}","Callsite":"save at :26","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)\n$line18.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line18.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:30)\n$line18.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:32)\n$line18.$read$$iw$$iw$$iw$$iw$$iw.(:34)\n$line18.$read$$iw$$iw$$iw$$iw.(:36)\n$line18.$read$$iw$$iw$$iw.(:38)\n$line18.$read$$iw$$iw.(:40)\n$line18.$read$$iw.(:42)\n$line18.$read.(:44)\n$line18.$read$.(:48)\n$line18.$read$.()\n$line18.$eval$.$print$lzycompute(:7)\n$line18.$eval$.$print(:6)\n$line18.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Submission Time":1629442316522,"Completion Time":1629442316597,"Accumulables":[{"ID":140,"Name":"duration","Value":"27","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":141,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":142,"Name":"number of input batches","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":143,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":147,"Name":"scan time","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":148,"Name":"internal.metrics.executorDeserializeTime","Value":14,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.executorDeserializeCpuTime","Value":14661400,"Internal":true,"Count Failed Values":true},{"ID":150,"Name":"internal.metrics.executorRunTime","Value":30,"Internal":true,"Count Failed Values":true},{"ID":151,"Name":"internal.metrics.executorCpuTime","Value":23473257,"Internal":true,"Count Failed Values":true},{"ID":152,"Name":"internal.metrics.resultSize","Value":2653,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.input.bytesRead","Value":1377,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.input.recordsRead","Value":1,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.output.bytesWritten","Value":15,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.output.recordsWritten","Value":1,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":4,"Completion Time":1629442316597,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":3,"accumUpdates":[[136,1],[137,15],[138,1],[139,0]]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":3,"time":1629442316609} +{"Event":"SparkListenerApplicationEnd","Timestamp":1629442318523} diff --git a/tools/src/test/scala/com/nvidia/spark/rapids/tool/qualification/QualificationSuite.scala b/tools/src/test/scala/com/nvidia/spark/rapids/tool/qualification/QualificationSuite.scala index fef33566a1d..6f6c8b1c195 100644 --- a/tools/src/test/scala/com/nvidia/spark/rapids/tool/qualification/QualificationSuite.scala +++ b/tools/src/test/scala/com/nvidia/spark/rapids/tool/qualification/QualificationSuite.scala @@ -19,10 +19,11 @@ package com.nvidia.spark.rapids.tool.qualification import java.io.File import java.util.concurrent.TimeUnit.NANOSECONDS -import scala.collection.mutable.ListBuffer +import scala.collection.mutable.{ArrayBuffer, ListBuffer} import scala.io.Source import com.nvidia.spark.rapids.tool.{EventLogPathProcessor, ToolTestUtils} +import org.apache.hadoop.conf.Configuration import org.scalatest.{BeforeAndAfterEach, FunSuite} import org.apache.spark.internal.Logging @@ -30,7 +31,7 @@ import org.apache.spark.scheduler.{SparkListener, SparkListenerStageCompleted, S import org.apache.spark.sql.{DataFrame, SparkSession, TrampolineUtil} import org.apache.spark.sql.functions.udf import org.apache.spark.sql.rapids.tool.{AppFilterImpl, ToolUtils} -import org.apache.spark.sql.rapids.tool.qualification.QualificationSummaryInfo +import org.apache.spark.sql.rapids.tool.qualification.{QualAppInfo, QualificationSummaryInfo} import org.apache.spark.sql.types._ class QualificationSuite extends FunSuite with BeforeAndAfterEach with Logging { @@ -64,6 +65,9 @@ class QualificationSuite extends FunSuite with BeforeAndAfterEach with Logging { .add("Read Score Percent", IntegerType, true) .add("Read File Format Score", DoubleType, true) .add("Unsupported Read File Formats and Types", StringType, true) + .add("Unsupported Write Data Format", StringType, true) + .add("Complex Types", StringType, true) + .add("Unsupported Nested Complex Types", StringType, true) def readExpectedFile(expected: File): DataFrame = { ToolTestUtils.readExpectationCSV(sparkSession, expected.getPath(), @@ -338,6 +342,66 @@ class QualificationSuite extends FunSuite with BeforeAndAfterEach with Logging { runQualificationTest(logFiles, "nds_q86_fail_test_expectation.csv") } + test("test event log write format") { + val logFiles = Array(s"$logDir/writeformat_eventlog") + runQualificationTest(logFiles, "write_format_expectation.csv") + } + + test("test event log nested types in ReadSchema") { + val logFiles = Array(s"$logDir/nested_type_eventlog") + runQualificationTest(logFiles, "nested_type_expectation.csv") + } + + // this tests parseReadSchema by passing different schemas as strings. Schemas + // with complex types, complex nested types, decimals and simple types + test("test different types in ReadSchema") { + val testSchemas: ArrayBuffer[ArrayBuffer[String]] = ArrayBuffer( + ArrayBuffer(""), + ArrayBuffer("firstName:string,lastName:string"), + ArrayBuffer("properties:map"), + ArrayBuffer("name:array"), + ArrayBuffer("name:string,booksInterested:array>,authbook:array>, " + + "pages:array>>,name:string,subject:string"), + ArrayBuffer("name:struct,ln:string>," + + "add:struct," + + "previous:struct,city:string>>," + + "next:struct"), + ArrayBuffer("name:map>, " + + "address:map>," + + "orders:map>>," + + "status:map") + ) + + var index = 0 + val expectedResult = List( + ("", ""), + ("", ""), + ("map", ""), + ("array", ""), + ("array>;" + + "array>;array>>", + "array>;" + + "array>;array>>"), + ("struct;ln:string>;" + + "struct;previous:struct;" + + "city:string>>;struct", + "struct;ln:string>;" + + "struct;previous:struct;" + + "city:string>>"), + ("map>;map>;" + + "map>>;map", + "map>;map>;" + + "map>>")) + + val result = testSchemas.map(x => QualAppInfo.parseReadSchemaForNestedTypes(x)) + result.foreach { actualResult => + assert(actualResult._1.equals(expectedResult(index)._1)) + assert(actualResult._2.equals(expectedResult(index)._2)) + index += 1 + } + } + // this event log has both decimal and non-decimal so comes out partial // it has both reading decimal, multiplication and join on decimal test("test decimal problematic") { @@ -441,6 +505,11 @@ class QualificationSuite extends FunSuite with BeforeAndAfterEach with Logging { runQualificationTest(logFiles, "complex_dec_expectation.csv") } + test("test dsv2 nested complex") { + val logFiles = Array(s"$logDir/eventlog_nested_dsv2") + runQualificationTest(logFiles, "nested_dsv2_expectation.csv") + } + test("sql metric agg") { TrampolineUtil.withTempDir { eventLogDir => val listener = new ToolTestListener