Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Temporary] Multi-GPU predictor #3819

Closed
wants to merge 13 commits into from
Closed

[Temporary] Multi-GPU predictor #3819

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
9f50ad7
Multi-GPU support in GPUPredictor.
canonizer Sep 28, 2018
47d764f
Merge branch 'master' into mgpu-predictor-2
canonizer Oct 21, 2018
801f671
Replaced pointers with spans in GPUPredictor.
canonizer Oct 21, 2018
55980e4
Added a multi-GPU predictor test.
canonizer Oct 21, 2018
10bc710
Fix multi-gpu test.
trivialfis Oct 23, 2018
5550da7
Set device.
trivialfis Oct 23, 2018
cd3e81a
Can't reproduce, log everything for Jenkins.
trivialfis Oct 23, 2018
48a3fae
Log CPU functions.
trivialfis Oct 23, 2018
e1ac03d
Use span instead of pointer in DeviceOffsets.
trivialfis Oct 23, 2018
b6bef1c
Change back the number of rows.
trivialfis Oct 23, 2018
1182732
Turn test data into range.
trivialfis Oct 23, 2018
8e18357
Configure shards.
trivialfis Oct 23, 2018
ce45169
Reinitialize shards when GPUSet is changed.
trivialfis Oct 23, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/common/span.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ class SpanIterator {

using reference = typename std::conditional< // NOLINT
IsConst, const ElementType, ElementType>::type&;
using pointer = typename std::add_pointer<reference>::type&; // NOLINT
using pointer = typename std::add_pointer<reference>::type; // NOLINT

XGBOOST_DEVICE constexpr SpanIterator() : span_{nullptr}, index_{0} {}

Expand Down
5 changes: 3 additions & 2 deletions src/gbm/gbtree.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,8 +194,9 @@ class GBTree : public GradientBooster {
CHECK_EQ(in_gpair->Size() % ngroup, 0U)
<< "must have exactly ngroup*nrow gpairs";
// TODO(canonizer): perform this on GPU if HostDeviceVector has device set.
HostDeviceVector<GradientPair> tmp(in_gpair->Size() / ngroup,
GradientPair(), in_gpair->Distribution());
HostDeviceVector<GradientPair> tmp
(in_gpair->Size() / ngroup, GradientPair(),
GPUDistribution::Block(in_gpair->Distribution().Devices()));
const auto& gpair_h = in_gpair->ConstHostVector();
auto nsize = static_cast<bst_omp_uint>(tmp.Size());
for (int gid = 0; gid < ngroup; ++gid) {
Expand Down
2 changes: 1 addition & 1 deletion src/objective/hinge.cu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ struct HingeObjParam : public dmlc::Parameter<HingeObjParam> {
int n_gpus;
int gpu_id;
DMLC_DECLARE_PARAMETER(HingeObjParam) {
DMLC_DECLARE_FIELD(n_gpus).set_default(0).set_lower_bound(0)
DMLC_DECLARE_FIELD(n_gpus).set_default(1).set_lower_bound(-1)
.describe("Number of GPUs to use for multi-gpu algorithms.");
DMLC_DECLARE_FIELD(gpu_id)
.set_lower_bound(0)
Expand Down
15 changes: 4 additions & 11 deletions src/objective/multiclass_obj.cu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ struct SoftmaxMultiClassParam : public dmlc::Parameter<SoftmaxMultiClassParam> {
DMLC_DECLARE_PARAMETER(SoftmaxMultiClassParam) {
DMLC_DECLARE_FIELD(num_class).set_lower_bound(1)
.describe("Number of output class in the multi-class classification.");
DMLC_DECLARE_FIELD(n_gpus).set_default(-1).set_lower_bound(-1)
DMLC_DECLARE_FIELD(n_gpus).set_default(1).set_lower_bound(-1)
.describe("Number of GPUs to use for multi-gpu algorithms.");
DMLC_DECLARE_FIELD(gpu_id)
.set_lower_bound(0)
Expand Down Expand Up @@ -65,8 +65,8 @@ class SoftmaxMultiClassObj : public ObjFunction {
const auto ndata = static_cast<int64_t>(preds.Size() / nclass);

// clear out device memory;
out_gpair->Reshard(GPUSet::Empty());
preds.Reshard(GPUSet::Empty());
// out_gpair->Reshard(GPUSet::Empty());
// preds.Reshard(GPUSet::Empty());

out_gpair->Reshard(GPUDistribution::Granular(devices_, nclass));
info.labels_.Reshard(GPUDistribution::Block(devices_));
Expand Down Expand Up @@ -109,11 +109,6 @@ class SoftmaxMultiClassObj : public ObjFunction {
}, common::Range{0, ndata}, devices_, false)
.Eval(out_gpair, &info.labels_, &preds, &info.weights_, &label_correct_);

out_gpair->Reshard(GPUSet::Empty());
out_gpair->Reshard(GPUDistribution::Block(devices_));
preds.Reshard(GPUSet::Empty());
preds.Reshard(GPUDistribution::Block(devices_));

std::vector<int>& label_correct_h = label_correct_.HostVector();
for (auto const flag : label_correct_h) {
if (flag != 1) {
Expand All @@ -136,7 +131,7 @@ class SoftmaxMultiClassObj : public ObjFunction {
const auto ndata = static_cast<int64_t>(io_preds->Size() / nclass);
max_preds_.Resize(ndata);

io_preds->Reshard(GPUSet::Empty()); // clear out device memory
// io_preds->Reshard(GPUSet::Empty()); // clear out device memory
if (prob) {
common::Transform<>::Init(
[=] XGBOOST_DEVICE(size_t _idx, common::Span<bst_float> _preds) {
Expand Down Expand Up @@ -166,8 +161,6 @@ class SoftmaxMultiClassObj : public ObjFunction {
io_preds->Resize(max_preds_.Size());
io_preds->Copy(max_preds_);
}
io_preds->Reshard(GPUSet::Empty()); // clear out device memory
io_preds->Reshard(GPUDistribution::Block(devices_));
}

private:
Expand Down
2 changes: 1 addition & 1 deletion src/objective/regression_obj.cu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ struct RegLossParam : public dmlc::Parameter<RegLossParam> {
DMLC_DECLARE_PARAMETER(RegLossParam) {
DMLC_DECLARE_FIELD(scale_pos_weight).set_default(1.0f).set_lower_bound(0.0f)
.describe("Scale the weight of positive examples by this factor");
DMLC_DECLARE_FIELD(n_gpus).set_default(-1).set_lower_bound(-1)
DMLC_DECLARE_FIELD(n_gpus).set_default(1).set_lower_bound(-1)
.describe("Number of GPUs to use for multi-gpu algorithms.");
DMLC_DECLARE_FIELD(gpu_id)
.set_lower_bound(0)
Expand Down
Loading