Add doc comments for webpages and webgraph; resolves #392. (#394)

archivesunleashed · Dec 18, 2019 · 99e9d06 · 99e9d06
1 parent 8eb43ff
commit 99e9d06
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/src/main/scala/io/archivesunleashed/package.scala b/src/main/scala/io/archivesunleashed/package.scala
@@ -185,6 +185,7 @@ package object archivesunleashed {
           && r.getHttpStatus == "200")
     }
 
+    /** Extracts webpages with columns for crawl data, url, MIME type, and content. */
     def webpages(): DataFrame = {
       val records = rdd.keepValidPages()
         .map(r => Row(r.getCrawlDate, r.getUrl, r.getMimeType,
@@ -201,6 +202,7 @@ package object archivesunleashed {
       sqlContext.getOrCreate().createDataFrame(records, schema)
     }
 
+    /** Extracts a webgraph with columns for crawl date, source url, destination url, and anchor text. */
     def webgraph(): DataFrame = {
       val records = rdd
         .keepValidPages()