Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[jvm-packages] remove APIs with DMatrix from xgboost-spark #1519

Merged
merged 4 commits into from
Aug 29, 2016
Merged

[jvm-packages] remove APIs with DMatrix from xgboost-spark #1519

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e58ed7f
test consistency of prediction functions between DMatrix and RDD
CodingCat Aug 28, 2016
406004a
remove APIs with DMatrix from xgboost-spark
CodingCat Aug 29, 2016
ab5bfde
fix compilation error in xgboost4j-example
CodingCat Aug 29, 2016
0c0e29d
fix test cases
CodingCat Aug 29, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...t4j-example/src/main/scala/ml/dmlc/xgboost4j/scala/example/spark/DistTrainWithSpark.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ object DistTrainWithSpark {
"objective" -> "binary:logistic").toMap
val xgboostModel = XGBoost.train(trainRDD, paramMap, numRound, nWorkers = args(1).toInt,
useExternalMemory = true)
xgboostModel.predict(new DMatrix(testSet))
xgboostModel.booster.predict(new DMatrix(testSet))
// save model to HDFS path
xgboostModel.saveModelAsHadoopFile(outputModelPath)
}
Expand Down
18 changes: 0 additions & 18 deletions jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostModel.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,15 +122,6 @@ class XGBoostModel(_booster: Booster) extends Serializable {
}
}

/**
* Predict result with the given test set (represented as DMatrix)
*
* @param testSet test set represented as DMatrix
*/
def predict(testSet: DMatrix): Array[Array[Float]] = {
_booster.predict(testSet)
}

/**
* Predict leaf instances with the given test set (represented as RDD)
*
Expand All @@ -149,15 +140,6 @@ class XGBoostModel(_booster: Booster) extends Serializable {
}
}

/**
* Predict leaf instances with the given test set (represented as DMatrix)
*
* @param testSet test set represented as DMatrix
*/
def predictLeaves(testSet: DMatrix): Array[Array[Float]] = {
_booster.predictLeaf(testSet, 0)
}

/**
* Save the model as to HDFS-compatible file system.
*
Expand Down
8 changes: 4 additions & 4 deletions jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ class XGBoostSuite extends FunSuite with BeforeAndAfter {
val testSetDMatrix = new DMatrix(new JDMatrix(testSet, null))
val boosterRDD = XGBoost.buildDistributedBoosters(
trainingRDD,
List("eta" -> "1", "max_depth" -> "2", "silent" -> "0",
List("eta" -> "1", "max_depth" -> "6", "silent" -> "0",
"objective" -> "binary:logistic").toMap,
new scala.collection.mutable.HashMap[String, String],
numWorkers = 2, round = 5, null, null, useExternalMemory = false)
Expand All @@ -134,8 +134,9 @@ class XGBoostSuite extends FunSuite with BeforeAndAfter {
val boosters = boosterRDD.collect()
val eval = new EvalError()
for (booster <- boosters) {
// the threshold is 0.11 because it does not sync boosters with AllReduce
val predicts = booster.predict(testSetDMatrix, outPutMargin = true)
assert(eval.eval(predicts, testSetDMatrix) < 0.17)
assert(eval.eval(predicts, testSetDMatrix) < 0.11)
}
}

Expand Down Expand Up @@ -211,7 +212,7 @@ class XGBoostSuite extends FunSuite with BeforeAndAfter {
val predRDD = xgBoostModel.predict(testRDD)
val predResult1 = predRDD.collect()(0)
import DataUtils._
val predResult2 = xgBoostModel.predict(new DMatrix(testSet.iterator))
val predResult2 = xgBoostModel.booster.predict(new DMatrix(testSet.iterator))
for (i <- predResult1.indices; j <- predResult1(i).indices) {
assert(predResult1(i)(j) === predResult2(i)(j))
}
Expand All @@ -222,7 +223,6 @@ class XGBoostSuite extends FunSuite with BeforeAndAfter {
val sampleList = new ListBuffer[SparkVector]
sparkContext.getOrElse(sc).parallelize(sampleList, numWorkers)
}

val trainingRDD = buildTrainingRDD()
val testRDD = buildEmptyRDD()
import DataUtils._
Expand Down