Improve spark integration: turn off logging, add magic for class dumping

Kotlin · Nov 29, 2019 · 09e5766 · 09e5766
1 parent 9df879d
commit 09e5766
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/libraries.json b/libraries.json
@@ -117,9 +117,11 @@
         "org.apache.spark.ml.feature.*"
       ],
       "init": [
+        "org.apache.log4j.Logger.getLogger(\"org\").setLevel(org.apache.log4j.Level.OFF)",
+        "org.apache.log4j.Logger.getLogger(\"akka\").setLevel(org.apache.log4j.Level.OFF)",
         "val spark = SparkSession\n        .builder()\n        .appName(\"Spark example\")\n        .master(\"local\")\n        .getOrCreate()",
         "val sc = spark.sparkContext()",
-        "spark.sparkContext().setLogLevel(\"ERROR\")"
+        "%dumpClassesForSpark"
       ],
       "initCell": [
         "scala.Console.setOut(System.out)",