DataFrame Implementation - Serializable APIs

[SinghGursimran]

DataFrame implementations for all the remaining serializable APIs.

#223

For Testing:

discardContentDF

import io.archivesunleashed._
import io.archivesunleashed.df._

RecordLoader.loadArchives("./src/test/resources/arc/example.arc.gz",sc)
			.all()
			.select($"content")
			.discardContentDF(Set("Content-Length: [0-9]{4}".r))
			.write
  			.option("header","true")
  			.csv("content_after_discard")

keepContentDF

import io.archivesunleashed._
import io.archivesunleashed.df._

RecordLoader.loadArchives("./src/test/resources/arc/example.arc.gz",sc)
			.all()
			.select($"content")
			.keepContentDF(Set("Content-Length: [0-9]{4}".r))
			.write
  			.option("header","true")
  			.csv("content_kept")

discardUrlPatternsDF

import io.archivesunleashed._
import io.archivesunleashed.df._

RecordLoader.loadArchives("./src/test/resources/arc/example.arc.gz",sc)
			.all()
			.select($"url")
			.discardUrlPatternsDF(Set(".*images.*".r))
			.show(10,false)

keepUrlPatternsDF

import io.archivesunleashed._
import io.archivesunleashed.df._

RecordLoader.loadArchives("./src/test/resources/arc/example.arc.gz",sc)
			.all()
			.select($"url")
			.keepUrlPatternsDF(Set(".*images.*".r))
			.show(10,false)

discardLanguagesDF

import io.archivesunleashed._
import io.archivesunleashed.df._

RecordLoader.loadArchives("./src/test/resources/arc/example.arc.gz",sc)
			.all()
			.select(DetectLanguageDF(RemoveHTMLDF($"content")).as("language"))
			.discardLanguagesDF(Set("th","de","ht"))
			.show(10,false)

keepLanguagesDF

import io.archivesunleashed._
import io.archivesunleashed.df._

RecordLoader.loadArchives("./src/test/resources/arc/example.arc.gz",sc)
			.all()
			.select(DetectLanguageDF(RemoveHTMLDF($"content")).as("language"))
			.keepLanguagesDF(Set("th","de","ht"))
			.show(10,false)

keepImagesDF

import io.archivesunleashed._
import io.archivesunleashed.df._

RecordLoader.loadArchives("./src/test/resources/arc/example.arc.gz",sc)
			.all()
			.keepImagesDF()
			.select($"mime_type_tika")
			.show(10,false)

[codecov]

Codecov Report

❗ No coverage uploaded for pull request base (master@7960472). Click here to learn what that means.
The diff coverage is 88.63%.

@@            Coverage Diff            @@
##             master     #401   +/-   ##
=========================================
  Coverage          ?   77.48%           
=========================================
  Files             ?       40           
  Lines             ?     1537           
  Branches          ?      292           
=========================================
  Hits              ?     1191           
  Misses            ?      218           
  Partials          ?      128