Update PlainTextExtractor to output a single column; text. (#453)

- Resolves #452
- PlainTextExtractor runs ExtractBoilerplate on `content`
- Update test

src/main/scala/io/archivesunleashed/app/PlainTextExtractor.scala

@@ -17,8 +17,7 @@
 package io.archivesunleashed.app
 
 import io.archivesunleashed.ArchiveRecord
-import io.archivesunleashed.df.{ExtractDomainDF, RemoveHTMLDF,
-                                RemoveHTTPHeaderDF}
+import io.archivesunleashed.df.{ExtractBoilerpipeTextDF}
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
 
 object PlainTextExtractor {
@@ -32,7 +31,6 @@ object PlainTextExtractor {
     // scalastyle:off
     import spark.implicits._
     // scalastyle:on
-    d.select($"crawl_date", ExtractDomainDF($"url").as("domain"),
-      $"url", RemoveHTMLDF(RemoveHTTPHeaderDF($"content")).as("text"))
+    d.select(ExtractBoilerpipeTextDF($"content").as("content"))
   }
 }

src/test/scala/io/archivesunleashed/app/PlainTextExtractorTest.scala

@@ -38,16 +38,19 @@ class PlainTextExtractorTest extends FunSuite with BeforeAndAfter {
     sc = new SparkContext(conf)
   }
 
-  test("Plain text extractor RDD & DF") {
+  test("Plain text extractor") {
     val df = RecordLoader.loadArchives(arcPath, sc).webpages()
     val dfResults = PlainTextExtractor(df).collect()
     val RESULTSLENGTH = 94
 
     assert(dfResults.length == RESULTSLENGTH)
-    assert(dfResults(0).get(0) == "20080430")
-    assert(dfResults(0).get(1) == "www.archive.org")
-    assert(dfResults(0).get(2) == "http://www.archive.org/")
-    assert(dfResults(0).get(3) == "Please visit our website at: http://www.archive.org")
+    assert(dfResults(0).get(0) == "")
+    assert(dfResults(4).get(0)
+      .toString
+      .startsWith("Author: Spivak, John L. (John Louis), b. 1897 Published: 1939"))
+    assert(dfResults(50).get(0)
+      .toString
+      .startsWith("How many hours in a day They tell me 24 "))
   }
 
   after {