From 9068c5c1570150de31a871bccf8271950fa5b5a3 Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Sun, 6 Apr 2014 18:55:58 -0700 Subject: [PATCH 01/17] add test demonstrating weird boost RNG issue when sampling from a gaussian followed by a bernoulli --- .../test/test_random_number_generator.cpp | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/src/caffe/test/test_random_number_generator.cpp b/src/caffe/test/test_random_number_generator.cpp index 267e7731475..841401ddd1b 100644 --- a/src/caffe/test/test_random_number_generator.cpp +++ b/src/caffe/test/test_random_number_generator.cpp @@ -95,4 +95,115 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngBernoulli) { } +TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussianTimesBernoulli) { + size_t sample_size = 10000; + SyncedMemory gaussian_data(sample_size * sizeof(TypeParam)); + SyncedMemory bernoulli_data(sample_size * sizeof(int)); + Caffe::set_random_seed(1701); + // Sample from 0 mean Gaussian + TypeParam mu = 0; + TypeParam sigma = 1; + caffe_vRngGaussian(sample_size, reinterpret_cast( + gaussian_data.mutable_cpu_data()), mu, sigma); + TypeParam true_mean = mu; + TypeParam true_std = sigma; + TypeParam bound = this->mean_bound(true_std, sample_size); + TypeParam empirical_mean = this->sample_mean( + reinterpret_cast(gaussian_data.cpu_data()), + sample_size); + EXPECT_NEAR(empirical_mean, true_mean, bound); + int num_pos = 0; + int num_neg = 0; + int num_zeros = 0; + TypeParam* samples = + static_cast(gaussian_data.mutable_cpu_data()); + for (int i = 0; i < sample_size; ++i) { + if (samples[i] == TypeParam(0)) { + ++num_zeros; + } else if (samples[i] > TypeParam(0)) { + ++num_pos; + } else if (samples[i] < TypeParam(0)) { + ++num_neg; + } + } + // Check that we have no zeros (possible to generate 0s, but highly + // improbable), and roughly half positives and half negatives (with bound + // computed from a Bernoulli with p = 0.5). + EXPECT_EQ(0, num_zeros); + double p = 0.5; + true_mean = p; + true_std = sqrt(p * (1 - p)); + bound = this->mean_bound(true_std, sample_size); + TypeParam expected_num_each_sign = sample_size * p; + LOG(INFO) << "Gaussian: Expected " << expected_num_each_sign << " positives" + << "; got " << num_pos; + LOG(INFO) << "Gaussian: Expected " << expected_num_each_sign << " negatives" + << "; got " << num_neg; + EXPECT_NEAR(expected_num_each_sign, num_pos, sample_size * bound); + EXPECT_NEAR(expected_num_each_sign, num_neg, sample_size * bound); + // Sample from Bernoulli with p = 0.3 + p = 0.3; + caffe_vRngBernoulli(sample_size, + reinterpret_cast(bernoulli_data.mutable_cpu_data()), p); + true_mean = p; + true_std = sqrt(p * (1 - p)); + bound = this->mean_bound(true_std, sample_size); + empirical_mean = + this->sample_mean((const int *)bernoulli_data.cpu_data(), sample_size); + LOG(INFO) << "Bernoulli: Expected mean = " << true_mean + << "; sample mean = " << empirical_mean; + EXPECT_NEAR(empirical_mean, true_mean, bound); + int bernoulli_num_zeros = 0; + int num_ones = 0; + int num_other = 0; + const int* bernoulli_samples = + reinterpret_cast(bernoulli_data.cpu_data()); + for (int i = 0; i < sample_size; ++i) { + if (bernoulli_samples[i] == 0) { + ++bernoulli_num_zeros; + } else if (bernoulli_samples[i] == 1) { + ++num_ones; + } else { + ++num_other; + } + } + LOG(INFO) << "Bernoulli: zeros: " << bernoulli_num_zeros + << "; ones: " << num_ones << "; other: " << num_other; + EXPECT_EQ(0, num_other); + EXPECT_EQ(sample_size * empirical_mean, num_ones); + EXPECT_EQ(sample_size * (1.0 - empirical_mean), bernoulli_num_zeros); + // Multiply Gaussian by Bernoulli + for (int i = 0; i < sample_size; ++i) { + samples[i] *= bernoulli_samples[i]; + } + num_pos = 0; + num_neg = 0; + num_zeros = 0; + for (int i = 0; i < sample_size; ++i) { + if (samples[i] == TypeParam(0)) { + ++num_zeros; + } else if (samples[i] > TypeParam(0)) { + ++num_pos; + } else if (samples[i] < TypeParam(0)) { + ++num_neg; + } + } + // Check that we have as many zeros as Bernoulli, and roughly half positives + // and half negatives (with bound computed from a Bernoulli with p = 0.5). + EXPECT_EQ(bernoulli_num_zeros, num_zeros); + p = 0.5; + true_mean = p; + true_std = sqrt(p * (1 - p)); + int sub_sample_size = sample_size - bernoulli_num_zeros; + bound = this->mean_bound(true_std, sub_sample_size); + expected_num_each_sign = sub_sample_size * p; + LOG(INFO) << "Gaussian: Expected " << expected_num_each_sign << " positives" + << "; got " << num_pos; + LOG(INFO) << "Gaussian: Expected " << expected_num_each_sign << " negatives" + << "; got " << num_neg; + EXPECT_NEAR(expected_num_each_sign, num_pos, sample_size * bound); + EXPECT_NEAR(expected_num_each_sign, num_neg, sample_size * bound); +} + + } // namespace caffe From 3fe018c7a043f95dc87e3d8b2772247062ebed8e Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Sun, 6 Apr 2014 19:41:14 -0700 Subject: [PATCH 02/17] add analogous test for uniform instead of gaussian --- .../test/test_random_number_generator.cpp | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/src/caffe/test/test_random_number_generator.cpp b/src/caffe/test/test_random_number_generator.cpp index 841401ddd1b..4572d770ed2 100644 --- a/src/caffe/test/test_random_number_generator.cpp +++ b/src/caffe/test/test_random_number_generator.cpp @@ -206,4 +206,115 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussianTimesBernoulli) { } +TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformTimesBernoulli) { + size_t sample_size = 10000; + SyncedMemory uniform_data(sample_size * sizeof(TypeParam)); + SyncedMemory bernoulli_data(sample_size * sizeof(int)); + Caffe::set_random_seed(1701); + // Sample from Uniform on [-1, 1] + TypeParam a = -1; + TypeParam b = 1; + caffe_vRngUniform(sample_size, reinterpret_cast( + uniform_data.mutable_cpu_data()), a, b); + TypeParam true_mean = (a + b) / 2; + TypeParam true_std = (b - a) / sqrt(12); + TypeParam bound = this->mean_bound(true_std, sample_size); + TypeParam empirical_mean = this->sample_mean( + reinterpret_cast(uniform_data.cpu_data()), + sample_size); + EXPECT_NEAR(empirical_mean, true_mean, bound); + int num_pos = 0; + int num_neg = 0; + int num_zeros = 0; + TypeParam* samples = + static_cast(uniform_data.mutable_cpu_data()); + for (int i = 0; i < sample_size; ++i) { + if (samples[i] == TypeParam(0)) { + ++num_zeros; + } else if (samples[i] > TypeParam(0)) { + ++num_pos; + } else if (samples[i] < TypeParam(0)) { + ++num_neg; + } + } + // Check that we have no zeros (possible to generate 0s, but highly + // improbable), and roughly half positives and half negatives (with bound + // computed from a Bernoulli with p = 0.5). + EXPECT_EQ(0, num_zeros); + double p = 0.5; + true_mean = p; + true_std = sqrt(p * (1 - p)); + bound = this->mean_bound(true_std, sample_size); + TypeParam expected_num_each_sign = sample_size * p; + LOG(INFO) << "Uniform: Expected " << expected_num_each_sign << " positives" + << "; got " << num_pos; + LOG(INFO) << "Uniform: Expected " << expected_num_each_sign << " negatives" + << "; got " << num_neg; + EXPECT_NEAR(expected_num_each_sign, num_pos, sample_size * bound); + EXPECT_NEAR(expected_num_each_sign, num_neg, sample_size * bound); + // Sample from Bernoulli with p = 0.3 + p = 0.3; + caffe_vRngBernoulli(sample_size, + reinterpret_cast(bernoulli_data.mutable_cpu_data()), p); + true_mean = p; + true_std = sqrt(p * (1 - p)); + bound = this->mean_bound(true_std, sample_size); + empirical_mean = + this->sample_mean((const int *)bernoulli_data.cpu_data(), sample_size); + LOG(INFO) << "Bernoulli: Expected mean = " << true_mean + << "; sample mean = " << empirical_mean; + EXPECT_NEAR(empirical_mean, true_mean, bound); + int bernoulli_num_zeros = 0; + int num_ones = 0; + int num_other = 0; + const int* bernoulli_samples = + reinterpret_cast(bernoulli_data.cpu_data()); + for (int i = 0; i < sample_size; ++i) { + if (bernoulli_samples[i] == 0) { + ++bernoulli_num_zeros; + } else if (bernoulli_samples[i] == 1) { + ++num_ones; + } else { + ++num_other; + } + } + LOG(INFO) << "Bernoulli: zeros: " << bernoulli_num_zeros + << "; ones: " << num_ones << "; other: " << num_other; + EXPECT_EQ(0, num_other); + EXPECT_EQ(sample_size * empirical_mean, num_ones); + EXPECT_EQ(sample_size * (1.0 - empirical_mean), bernoulli_num_zeros); + // Multiply Uniform by Bernoulli + for (int i = 0; i < sample_size; ++i) { + samples[i] *= bernoulli_samples[i]; + } + num_pos = 0; + num_neg = 0; + num_zeros = 0; + for (int i = 0; i < sample_size; ++i) { + if (samples[i] == TypeParam(0)) { + ++num_zeros; + } else if (samples[i] > TypeParam(0)) { + ++num_pos; + } else if (samples[i] < TypeParam(0)) { + ++num_neg; + } + } + // Check that we have as many zeros as Bernoulli, and roughly half positives + // and half negatives (with bound computed from a Bernoulli with p = 0.5). + EXPECT_EQ(bernoulli_num_zeros, num_zeros); + p = 0.5; + true_mean = p; + true_std = sqrt(p * (1 - p)); + int sub_sample_size = sample_size - bernoulli_num_zeros; + bound = this->mean_bound(true_std, sub_sample_size); + expected_num_each_sign = sub_sample_size * p; + LOG(INFO) << "Uniform: Expected " << expected_num_each_sign << " positives" + << "; got " << num_pos; + LOG(INFO) << "Uniform: Expected " << expected_num_each_sign << " negatives" + << "; got " << num_neg; + EXPECT_NEAR(expected_num_each_sign, num_pos, sample_size * bound); + EXPECT_NEAR(expected_num_each_sign, num_neg, sample_size * bound); +} + + } // namespace caffe From fca7eaf17aea9709045d54273a48bcbc7100bcd3 Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Sun, 6 Apr 2014 19:57:44 -0700 Subject: [PATCH 03/17] cleanup log messages --- src/caffe/test/test_random_number_generator.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/caffe/test/test_random_number_generator.cpp b/src/caffe/test/test_random_number_generator.cpp index 4572d770ed2..a7588fbaa46 100644 --- a/src/caffe/test/test_random_number_generator.cpp +++ b/src/caffe/test/test_random_number_generator.cpp @@ -197,10 +197,10 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussianTimesBernoulli) { int sub_sample_size = sample_size - bernoulli_num_zeros; bound = this->mean_bound(true_std, sub_sample_size); expected_num_each_sign = sub_sample_size * p; - LOG(INFO) << "Gaussian: Expected " << expected_num_each_sign << " positives" - << "; got " << num_pos; - LOG(INFO) << "Gaussian: Expected " << expected_num_each_sign << " negatives" - << "; got " << num_neg; + LOG(INFO) << "Gaussian*Bernoulli: Expected " << expected_num_each_sign + << " positives; got " << num_pos; + LOG(INFO) << "Gaussian*Bernoulli: Expected " << expected_num_each_sign + << " negatives; got " << num_neg; EXPECT_NEAR(expected_num_each_sign, num_pos, sample_size * bound); EXPECT_NEAR(expected_num_each_sign, num_neg, sample_size * bound); } @@ -308,10 +308,10 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformTimesBernoulli) { int sub_sample_size = sample_size - bernoulli_num_zeros; bound = this->mean_bound(true_std, sub_sample_size); expected_num_each_sign = sub_sample_size * p; - LOG(INFO) << "Uniform: Expected " << expected_num_each_sign << " positives" - << "; got " << num_pos; - LOG(INFO) << "Uniform: Expected " << expected_num_each_sign << " negatives" - << "; got " << num_neg; + LOG(INFO) << "Uniform*Bernoulli: Expected " << expected_num_each_sign + << " positives; got " << num_pos; + LOG(INFO) << "Uniform*Bernoulli: Expected " << expected_num_each_sign + << " negatives; got " << num_neg; EXPECT_NEAR(expected_num_each_sign, num_pos, sample_size * bound); EXPECT_NEAR(expected_num_each_sign, num_neg, sample_size * bound); } From 7d542294f8e1ad7486b7ad8758f8369a0825077e Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Mon, 7 Apr 2014 12:24:21 -0700 Subject: [PATCH 04/17] add bernoulli*bernoulli test --- .../test/test_random_number_generator.cpp | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/src/caffe/test/test_random_number_generator.cpp b/src/caffe/test/test_random_number_generator.cpp index a7588fbaa46..69a19aadcb8 100644 --- a/src/caffe/test/test_random_number_generator.cpp +++ b/src/caffe/test/test_random_number_generator.cpp @@ -317,4 +317,103 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformTimesBernoulli) { } +TYPED_TEST(RandomNumberGeneratorTest, TestRngBernoulliTimesBernoulli) { + size_t sample_size = 10000; + SyncedMemory bernoulli1_data(sample_size * sizeof(int)); + SyncedMemory bernoulli2_data(sample_size * sizeof(int)); + Caffe::set_random_seed(1701); + double p1 = 0.5; + caffe_vRngBernoulli(sample_size, reinterpret_cast( + bernoulli1_data.mutable_cpu_data()), p1); + TypeParam empirical_mean = this->sample_mean( + reinterpret_cast(bernoulli1_data.cpu_data()), + sample_size); + int bernoulli1_num_zeros = 0; + int num_ones = 0; + int num_other = 0; + int* bernoulli_samples = + reinterpret_cast(bernoulli1_data.mutable_cpu_data()); + for (int i = 0; i < sample_size; ++i) { + if (bernoulli_samples[i] == 0) { + ++bernoulli1_num_zeros; + } else if (bernoulli_samples[i] == 1) { + ++num_ones; + } else { + ++num_other; + } + } + TypeParam true_mean = p1; + TypeParam true_std = sqrt(p1 * (1 - p1)); + TypeParam bound = this->mean_bound(true_std, sample_size); + TypeParam expected_num_zeros = sample_size * (1 - true_mean); + TypeParam expected_num_ones = sample_size * true_mean; + LOG(INFO) << "Bernoulli1: Expected mean = " << true_mean + << "; sample mean = " << empirical_mean; + LOG(INFO) << "Bernoulli1: zeros: " << bernoulli1_num_zeros + << "; ones: " << num_ones << "; other: " << num_other; + empirical_mean = + this->sample_mean((const int *)bernoulli2_data.cpu_data(), sample_size); + EXPECT_NEAR(empirical_mean, true_mean, bound); + EXPECT_EQ(num_other, 0); + // Sample from Bernoulli with p = 0.3 + double p = 0.3; + caffe_vRngBernoulli(sample_size, + reinterpret_cast(bernoulli2_data.mutable_cpu_data()), p); + true_mean = p; + true_std = sqrt(p * (1 - p)); + bound = this->mean_bound(true_std, sample_size); + empirical_mean = + this->sample_mean((const int *)bernoulli2_data.cpu_data(), sample_size); + LOG(INFO) << "Bernoulli2: Expected mean = " << true_mean + << "; sample mean = " << empirical_mean; + EXPECT_NEAR(empirical_mean, true_mean, bound); + int bernoulli2_num_zeros = 0; + num_ones = 0; + num_other = 0; + const int* bernoulli2_samples = + reinterpret_cast(bernoulli2_data.cpu_data()); + for (int i = 0; i < sample_size; ++i) { + if (bernoulli2_samples[i] == 0) { + ++bernoulli2_num_zeros; + } else if (bernoulli2_samples[i] == 1) { + ++num_ones; + } else { + ++num_other; + } + } + LOG(INFO) << "Bernoulli2: zeros: " << bernoulli2_num_zeros + << "; ones: " << num_ones << "; other: " << num_other; + EXPECT_EQ(0, num_other); + EXPECT_EQ(sample_size * empirical_mean, num_ones); + EXPECT_EQ(sample_size * (1.0 - empirical_mean), bernoulli2_num_zeros); + // Multiply Bernoulli1 by Bernoulli2 + for (int i = 0; i < sample_size; ++i) { + bernoulli_samples[i] *= bernoulli2_samples[i]; + } + bernoulli1_num_zeros = 0; + num_ones = 0; + num_other = 0; + for (int i = 0; i < sample_size; ++i) { + if (bernoulli_samples[i] == 0) { + ++bernoulli1_num_zeros; + } else if (bernoulli_samples[i] == 1) { + ++num_ones; + } else { + ++num_other; + } + } + // Check that we have as many zeros as Bernoulli, and roughly half positives + // and half negatives (with bound computed from a Bernoulli with p = 0.5). + p *= p1; + true_mean = p; + true_std = sqrt(p * (1 - p)); + empirical_mean = + this->sample_mean((const int *)bernoulli2_data.cpu_data(), sample_size); + bound = this->mean_bound(true_std, sample_size); + LOG(INFO) << "Bernoulli1*Bernoulli2: Expected mean = " << true_mean + << "; sample mean = " << empirical_mean; + EXPECT_NEAR(empirical_mean, true_mean, bound); +} + + } // namespace caffe From db4d3b0f30515f539d41b033917446f84130a3e9 Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Mon, 7 Apr 2014 17:14:23 -0700 Subject: [PATCH 05/17] call caffe_set_rng at the end of each vRng function to maintain state --- include/caffe/common.hpp | 15 ++-- include/caffe/util/math_functions.hpp | 2 +- include/caffe/util/rng.hpp | 10 ++- include/caffe/vision_layers.hpp | 4 +- src/caffe/common.cpp | 40 +++++------ src/caffe/layers/dropout_layer.cpp | 2 +- .../test/test_random_number_generator.cpp | 72 ++++++++++--------- src/caffe/util/math_functions.cpp | 34 +++++---- 8 files changed, 94 insertions(+), 85 deletions(-) diff --git a/include/caffe/common.hpp b/include/caffe/common.hpp index c1bc3954df9..908deeecbb5 100644 --- a/include/caffe/common.hpp +++ b/include/caffe/common.hpp @@ -85,19 +85,18 @@ class Caffe { public: RNG(); explicit RNG(unsigned int seed); - ~RNG(); - RNG(const RNG&); + explicit RNG(const RNG&); RNG& operator=(const RNG&); const void* generator() const; - void* generator(); + void set_generator(const void* other_rng); private: class Generator; - Generator* generator_; + shared_ptr generator_; }; // Getters for boost rng, curand, and cublas handles - inline static RNG &rng_stream() { - return Get().random_generator_; + inline static const RNG& rng_stream() { + return *(Get().random_generator_); } inline static cublasHandle_t cublas_handle() { return Get().cublas_handle_; } inline static curandGenerator_t curand_generator() { @@ -118,6 +117,8 @@ class Caffe { inline static void set_phase(Phase phase) { Get().phase_ = phase; } // Sets the random seed of both boost and curand static void set_random_seed(const unsigned int seed); + // Sets the boost RNG engine from another RNG engine + static void set_generator(const void* other_rng); // Sets the device. Since we have cublas and curand stuff, set device also // requires us to reset those values. static void SetDevice(const int device_id); @@ -127,7 +128,7 @@ class Caffe { protected: cublasHandle_t cublas_handle_; curandGenerator_t curand_generator_; - RNG random_generator_; + shared_ptr random_generator_; Brew mode_; Phase phase_; diff --git a/include/caffe/util/math_functions.hpp b/include/caffe/util/math_functions.hpp index c501f2383a1..77e3234cf77 100644 --- a/include/caffe/util/math_functions.hpp +++ b/include/caffe/util/math_functions.hpp @@ -116,7 +116,7 @@ void caffe_vRngGaussian(const int n, Dtype* r, const Dtype a, const Dtype sigma); template -void caffe_vRngBernoulli(const int n, Dtype* r, const double p); +void caffe_vRngBernoulli(const int n, int* r, const Dtype p); template void caffe_exp(const int n, const Dtype* a, Dtype* y); diff --git a/include/caffe/util/rng.hpp b/include/caffe/util/rng.hpp index 8151a9a6f67..e684b3515de 100644 --- a/include/caffe/util/rng.hpp +++ b/include/caffe/util/rng.hpp @@ -9,9 +9,13 @@ namespace caffe { typedef boost::mt19937 rng_t; - inline rng_t& caffe_rng() { - Caffe::RNG &generator = Caffe::rng_stream(); - return *(caffe::rng_t*) generator.generator(); + + inline const rng_t& caffe_rng() { + return *static_cast(Caffe::rng_stream().generator()); + } + + inline void caffe_set_rng(const caffe::rng_t& other) { + Caffe::set_generator(static_cast(&other)); } } // namespace caffe diff --git a/include/caffe/vision_layers.hpp b/include/caffe/vision_layers.hpp index feda44d85df..0bfc677c043 100644 --- a/include/caffe/vision_layers.hpp +++ b/include/caffe/vision_layers.hpp @@ -70,8 +70,8 @@ class DropoutLayer : public NeuronLayer { const bool propagate_down, vector*>* bottom); shared_ptr rand_vec_; - float threshold_; - float scale_; + Dtype threshold_; + Dtype scale_; unsigned int uint_thres_; }; diff --git a/src/caffe/common.cpp b/src/caffe/common.cpp index 17e4ca5e508..65c68005042 100644 --- a/src/caffe/common.cpp +++ b/src/caffe/common.cpp @@ -60,7 +60,11 @@ void Caffe::set_random_seed(const unsigned int seed) { LOG(ERROR) << "Curand not available. Skipping setting the curand seed."; } // RNG seed - Get().random_generator_ = RNG(seed); + Get().random_generator_.reset(new RNG(seed)); +} + +void Caffe::set_generator(const void* other_rng) { + Get().random_generator_->set_generator(other_rng); } void Caffe::SetDevice(const int device_id) { @@ -117,36 +121,32 @@ void Caffe::DeviceQuery() { class Caffe::RNG::Generator { public: - caffe::rng_t rng; + Generator() : rng_(new caffe::rng_t(cluster_seedgen())) {} + explicit Generator(unsigned int seed) : rng_(new caffe::rng_t(seed)) {} + explicit Generator(const caffe::rng_t& other) : + rng_(new caffe::rng_t(other)) {} + shared_ptr rng_; }; -Caffe::RNG::RNG() -: generator_(new Generator) { - generator_->rng = caffe::rng_t(cluster_seedgen()); -} - -Caffe::RNG::RNG(unsigned int seed) -: generator_(new Generator) { - generator_->rng = caffe::rng_t(seed); -} +Caffe::RNG::RNG() : generator_(new Generator) { } -Caffe::RNG::~RNG() { delete generator_; } +Caffe::RNG::RNG(unsigned int seed) : generator_(new Generator(seed)) { } -Caffe::RNG::RNG(const RNG& other) : generator_(new Generator) { - *generator_ = *other.generator_; -} +Caffe::RNG::RNG(const RNG& other) : generator_(new Generator(*other.generator_)) + { } Caffe::RNG& Caffe::RNG::operator=(const RNG& other) { - *generator_ = *other.generator_; + generator_.reset(other.generator_.get()); return *this; } -void* Caffe::RNG::generator() { - return &generator_->rng; +const void* Caffe::RNG::generator() const { + return static_cast(generator_->rng_.get()); } -const void* Caffe::RNG::generator() const { - return &generator_->rng; +void Caffe::RNG::set_generator(const void* other_rng) { + const caffe::rng_t& rng = *static_cast(other_rng); + return generator_.reset(new Generator(rng)); } const char* cublasGetErrorString(cublasStatus_t error) { diff --git a/src/caffe/layers/dropout_layer.cpp b/src/caffe/layers/dropout_layer.cpp index 5dbba5d550c..a57999c2436 100644 --- a/src/caffe/layers/dropout_layer.cpp +++ b/src/caffe/layers/dropout_layer.cpp @@ -32,7 +32,7 @@ Dtype DropoutLayer::Forward_cpu(const vector*>& bottom, const int count = bottom[0]->count(); if (Caffe::phase() == Caffe::TRAIN) { // Create random numbers - caffe_vRngBernoulli(count, mask, 1. - threshold_); + caffe_vRngBernoulli(count, mask, 1. - threshold_); for (int i = 0; i < count; ++i) { top_data[i] = bottom_data[i] * mask[i] * scale_; } diff --git a/src/caffe/test/test_random_number_generator.cpp b/src/caffe/test/test_random_number_generator.cpp index 69a19aadcb8..25335c0eeb6 100644 --- a/src/caffe/test/test_random_number_generator.cpp +++ b/src/caffe/test/test_random_number_generator.cpp @@ -50,12 +50,12 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussian) { TypeParam mu = 0; TypeParam sigma = 1; caffe_vRngGaussian(sample_size, - reinterpret_cast(data_a.mutable_cpu_data()), mu, sigma); + static_cast(data_a.mutable_cpu_data()), mu, sigma); TypeParam true_mean = mu; TypeParam true_std = sigma; TypeParam bound = this->mean_bound(true_std, sample_size); TypeParam empirical_mean = - this->sample_mean(reinterpret_cast(data_a.cpu_data()), + this->sample_mean(static_cast(data_a.cpu_data()), sample_size); EXPECT_NEAR(empirical_mean, true_mean, bound); } @@ -68,12 +68,12 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngUniform) { TypeParam lower = 0; TypeParam upper = 1; caffe_vRngUniform(sample_size, - reinterpret_cast(data_a.mutable_cpu_data()), lower, upper); + static_cast(data_a.mutable_cpu_data()), lower, upper); TypeParam true_mean = (lower + upper) / 2; TypeParam true_std = (upper - lower) / sqrt(12); TypeParam bound = this->mean_bound(true_std, sample_size); TypeParam empirical_mean = - this->sample_mean(reinterpret_cast(data_a.cpu_data()), + this->sample_mean(static_cast(data_a.cpu_data()), sample_size); EXPECT_NEAR(empirical_mean, true_mean, bound); } @@ -103,13 +103,13 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussianTimesBernoulli) { // Sample from 0 mean Gaussian TypeParam mu = 0; TypeParam sigma = 1; - caffe_vRngGaussian(sample_size, reinterpret_cast( + caffe_vRngGaussian(sample_size, static_cast( gaussian_data.mutable_cpu_data()), mu, sigma); TypeParam true_mean = mu; TypeParam true_std = sigma; TypeParam bound = this->mean_bound(true_std, sample_size); TypeParam empirical_mean = this->sample_mean( - reinterpret_cast(gaussian_data.cpu_data()), + static_cast(gaussian_data.cpu_data()), sample_size); EXPECT_NEAR(empirical_mean, true_mean, bound); int num_pos = 0; @@ -144,7 +144,7 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussianTimesBernoulli) { // Sample from Bernoulli with p = 0.3 p = 0.3; caffe_vRngBernoulli(sample_size, - reinterpret_cast(bernoulli_data.mutable_cpu_data()), p); + static_cast(bernoulli_data.mutable_cpu_data()), p); true_mean = p; true_std = sqrt(p * (1 - p)); bound = this->mean_bound(true_std, sample_size); @@ -157,7 +157,7 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussianTimesBernoulli) { int num_ones = 0; int num_other = 0; const int* bernoulli_samples = - reinterpret_cast(bernoulli_data.cpu_data()); + static_cast(bernoulli_data.cpu_data()); for (int i = 0; i < sample_size; ++i) { if (bernoulli_samples[i] == 0) { ++bernoulli_num_zeros; @@ -170,8 +170,10 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussianTimesBernoulli) { LOG(INFO) << "Bernoulli: zeros: " << bernoulli_num_zeros << "; ones: " << num_ones << "; other: " << num_other; EXPECT_EQ(0, num_other); - EXPECT_EQ(sample_size * empirical_mean, num_ones); - EXPECT_EQ(sample_size * (1.0 - empirical_mean), bernoulli_num_zeros); + TypeParam epsilon = 1e-4; + EXPECT_NEAR(sample_size * empirical_mean, num_ones, epsilon); + EXPECT_NEAR(sample_size * (1.0 - empirical_mean), bernoulli_num_zeros, + epsilon); // Multiply Gaussian by Bernoulli for (int i = 0; i < sample_size; ++i) { samples[i] *= bernoulli_samples[i]; @@ -214,13 +216,13 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformTimesBernoulli) { // Sample from Uniform on [-1, 1] TypeParam a = -1; TypeParam b = 1; - caffe_vRngUniform(sample_size, reinterpret_cast( + caffe_vRngUniform(sample_size, static_cast( uniform_data.mutable_cpu_data()), a, b); TypeParam true_mean = (a + b) / 2; TypeParam true_std = (b - a) / sqrt(12); TypeParam bound = this->mean_bound(true_std, sample_size); TypeParam empirical_mean = this->sample_mean( - reinterpret_cast(uniform_data.cpu_data()), + static_cast(uniform_data.cpu_data()), sample_size); EXPECT_NEAR(empirical_mean, true_mean, bound); int num_pos = 0; @@ -241,7 +243,7 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformTimesBernoulli) { // improbable), and roughly half positives and half negatives (with bound // computed from a Bernoulli with p = 0.5). EXPECT_EQ(0, num_zeros); - double p = 0.5; + TypeParam p = 0.5; true_mean = p; true_std = sqrt(p * (1 - p)); bound = this->mean_bound(true_std, sample_size); @@ -255,7 +257,7 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformTimesBernoulli) { // Sample from Bernoulli with p = 0.3 p = 0.3; caffe_vRngBernoulli(sample_size, - reinterpret_cast(bernoulli_data.mutable_cpu_data()), p); + static_cast(bernoulli_data.mutable_cpu_data()), p); true_mean = p; true_std = sqrt(p * (1 - p)); bound = this->mean_bound(true_std, sample_size); @@ -268,7 +270,7 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformTimesBernoulli) { int num_ones = 0; int num_other = 0; const int* bernoulli_samples = - reinterpret_cast(bernoulli_data.cpu_data()); + static_cast(bernoulli_data.cpu_data()); for (int i = 0; i < sample_size; ++i) { if (bernoulli_samples[i] == 0) { ++bernoulli_num_zeros; @@ -281,8 +283,10 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformTimesBernoulli) { LOG(INFO) << "Bernoulli: zeros: " << bernoulli_num_zeros << "; ones: " << num_ones << "; other: " << num_other; EXPECT_EQ(0, num_other); - EXPECT_EQ(sample_size * empirical_mean, num_ones); - EXPECT_EQ(sample_size * (1.0 - empirical_mean), bernoulli_num_zeros); + TypeParam epsilon = 1e-4; + EXPECT_NEAR(sample_size * empirical_mean, num_ones, epsilon); + EXPECT_NEAR(sample_size * (1.0 - empirical_mean), bernoulli_num_zeros, + epsilon); // Multiply Uniform by Bernoulli for (int i = 0; i < sample_size; ++i) { samples[i] *= bernoulli_samples[i]; @@ -322,17 +326,17 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngBernoulliTimesBernoulli) { SyncedMemory bernoulli1_data(sample_size * sizeof(int)); SyncedMemory bernoulli2_data(sample_size * sizeof(int)); Caffe::set_random_seed(1701); - double p1 = 0.5; - caffe_vRngBernoulli(sample_size, reinterpret_cast( + TypeParam p1 = 0.5; + caffe_vRngBernoulli(sample_size, static_cast( bernoulli1_data.mutable_cpu_data()), p1); TypeParam empirical_mean = this->sample_mean( - reinterpret_cast(bernoulli1_data.cpu_data()), + static_cast(bernoulli1_data.cpu_data()), sample_size); int bernoulli1_num_zeros = 0; int num_ones = 0; int num_other = 0; int* bernoulli_samples = - reinterpret_cast(bernoulli1_data.mutable_cpu_data()); + static_cast(bernoulli1_data.mutable_cpu_data()); for (int i = 0; i < sample_size; ++i) { if (bernoulli_samples[i] == 0) { ++bernoulli1_num_zeros; @@ -351,19 +355,19 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngBernoulliTimesBernoulli) { << "; sample mean = " << empirical_mean; LOG(INFO) << "Bernoulli1: zeros: " << bernoulli1_num_zeros << "; ones: " << num_ones << "; other: " << num_other; - empirical_mean = - this->sample_mean((const int *)bernoulli2_data.cpu_data(), sample_size); + empirical_mean = this->sample_mean( + static_cast(bernoulli1_data.cpu_data()), sample_size); EXPECT_NEAR(empirical_mean, true_mean, bound); EXPECT_EQ(num_other, 0); // Sample from Bernoulli with p = 0.3 - double p = 0.3; - caffe_vRngBernoulli(sample_size, - reinterpret_cast(bernoulli2_data.mutable_cpu_data()), p); + TypeParam p = 0.3; + caffe_vRngBernoulli(sample_size, static_cast( + bernoulli2_data.mutable_cpu_data()), p); true_mean = p; true_std = sqrt(p * (1 - p)); bound = this->mean_bound(true_std, sample_size); - empirical_mean = - this->sample_mean((const int *)bernoulli2_data.cpu_data(), sample_size); + empirical_mean = this->sample_mean( + static_cast(bernoulli2_data.cpu_data()), sample_size); LOG(INFO) << "Bernoulli2: Expected mean = " << true_mean << "; sample mean = " << empirical_mean; EXPECT_NEAR(empirical_mean, true_mean, bound); @@ -371,7 +375,7 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngBernoulliTimesBernoulli) { num_ones = 0; num_other = 0; const int* bernoulli2_samples = - reinterpret_cast(bernoulli2_data.cpu_data()); + static_cast(bernoulli2_data.cpu_data()); for (int i = 0; i < sample_size; ++i) { if (bernoulli2_samples[i] == 0) { ++bernoulli2_num_zeros; @@ -384,8 +388,10 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngBernoulliTimesBernoulli) { LOG(INFO) << "Bernoulli2: zeros: " << bernoulli2_num_zeros << "; ones: " << num_ones << "; other: " << num_other; EXPECT_EQ(0, num_other); - EXPECT_EQ(sample_size * empirical_mean, num_ones); - EXPECT_EQ(sample_size * (1.0 - empirical_mean), bernoulli2_num_zeros); + TypeParam epsilon = 1e-4; + EXPECT_NEAR(sample_size * empirical_mean, num_ones, epsilon); + EXPECT_NEAR(sample_size * (1.0 - empirical_mean), bernoulli2_num_zeros, + epsilon); // Multiply Bernoulli1 by Bernoulli2 for (int i = 0; i < sample_size; ++i) { bernoulli_samples[i] *= bernoulli2_samples[i]; @@ -407,8 +413,8 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngBernoulliTimesBernoulli) { p *= p1; true_mean = p; true_std = sqrt(p * (1 - p)); - empirical_mean = - this->sample_mean((const int *)bernoulli2_data.cpu_data(), sample_size); + empirical_mean = this->sample_mean( + static_cast(bernoulli2_data.cpu_data()), sample_size); bound = this->mean_bound(true_std, sample_size); LOG(INFO) << "Bernoulli1*Bernoulli2: Expected mean = " << true_mean << "; sample mean = " << empirical_mean; diff --git a/src/caffe/util/math_functions.cpp b/src/caffe/util/math_functions.cpp index 3062dbfde74..0791a86c55a 100644 --- a/src/caffe/util/math_functions.cpp +++ b/src/caffe/util/math_functions.cpp @@ -320,16 +320,13 @@ void caffe_vRngUniform(const int n, Dtype* r, CHECK_GE(n, 0); CHECK(r); CHECK_LE(a, b); - - boost::uniform_real random_distribution( - a, caffe_nextafter(b)); - boost::variate_generator > variate_generator( - caffe_rng(), random_distribution); - + boost::uniform_real random_distribution(a, caffe_nextafter(b)); + boost::variate_generator > + variate_generator(caffe_rng(), random_distribution); for (int i = 0; i < n; ++i) { r[i] = variate_generator(); } + caffe_set_rng(variate_generator.engine()); } template @@ -346,13 +343,12 @@ void caffe_vRngGaussian(const int n, Dtype* r, const Dtype a, CHECK(r); CHECK_GT(sigma, 0); boost::normal_distribution random_distribution(a, sigma); - boost::variate_generator > variate_generator( - caffe_rng(), random_distribution); - + boost::variate_generator > + variate_generator(caffe_rng(), random_distribution); for (int i = 0; i < n; ++i) { r[i] = variate_generator(); } + caffe_set_rng(variate_generator.engine()); } template @@ -364,23 +360,25 @@ void caffe_vRngGaussian(const int n, double* r, const double a, const double sigma); template -void caffe_vRngBernoulli(const int n, Dtype* r, const double p) { +void caffe_vRngBernoulli(const int n, int* r, const Dtype p) { CHECK_GE(n, 0); CHECK(r); CHECK_GE(p, 0); CHECK_LE(p, 1); - boost::bernoulli_distribution random_distribution(p); - boost::variate_generator > variate_generator( - caffe_rng(), random_distribution); - + boost::bernoulli_distribution random_distribution(p); + boost::variate_generator > + variate_generator(caffe_rng(), random_distribution); for (int i = 0; i < n; ++i) { r[i] = variate_generator(); } + caffe_set_rng(variate_generator.engine()); } template -void caffe_vRngBernoulli(const int n, int* r, const double p); +void caffe_vRngBernoulli(const int n, int* r, const double p); + +template +void caffe_vRngBernoulli(const int n, int* r, const float p); template <> float caffe_cpu_dot(const int n, const float* x, const float* y) { From a52f65426e234a3a0cba5a7245696faf6aba6d6d Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Mon, 7 Apr 2014 17:58:11 -0700 Subject: [PATCH 06/17] fix bernoulli*bernoulli test, now all pass --- src/caffe/test/test_random_number_generator.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/caffe/test/test_random_number_generator.cpp b/src/caffe/test/test_random_number_generator.cpp index 25335c0eeb6..f33b4ce79c3 100644 --- a/src/caffe/test/test_random_number_generator.cpp +++ b/src/caffe/test/test_random_number_generator.cpp @@ -326,6 +326,7 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngBernoulliTimesBernoulli) { SyncedMemory bernoulli1_data(sample_size * sizeof(int)); SyncedMemory bernoulli2_data(sample_size * sizeof(int)); Caffe::set_random_seed(1701); + // Sample from Bernoulli with p = 0.5 TypeParam p1 = 0.5; caffe_vRngBernoulli(sample_size, static_cast( bernoulli1_data.mutable_cpu_data()), p1); @@ -414,7 +415,7 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngBernoulliTimesBernoulli) { true_mean = p; true_std = sqrt(p * (1 - p)); empirical_mean = this->sample_mean( - static_cast(bernoulli2_data.cpu_data()), sample_size); + static_cast(bernoulli1_data.cpu_data()), sample_size); bound = this->mean_bound(true_std, sample_size); LOG(INFO) << "Bernoulli1*Bernoulli2: Expected mean = " << true_mean << "; sample mean = " << empirical_mean; From 435cbf4703b399fe7f5f9218e5ea9c98cd772b11 Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Mon, 7 Apr 2014 19:08:27 -0700 Subject: [PATCH 07/17] comment to explain the purpose of Caffe::set_generator --- include/caffe/common.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/caffe/common.hpp b/include/caffe/common.hpp index 908deeecbb5..18290262e32 100644 --- a/include/caffe/common.hpp +++ b/include/caffe/common.hpp @@ -117,7 +117,8 @@ class Caffe { inline static void set_phase(Phase phase) { Get().phase_ = phase; } // Sets the random seed of both boost and curand static void set_random_seed(const unsigned int seed); - // Sets the boost RNG engine from another RNG engine + // Sets the boost RNG engine from another RNG engine to maintain state across + // variate_generator calls. static void set_generator(const void* other_rng); // Sets the device. Since we have cublas and curand stuff, set device also // requires us to reset those values. From d6776f04cce164dd51aec137a98892333c18760e Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Mon, 7 Apr 2014 19:05:59 -0700 Subject: [PATCH 08/17] cleanup test_math_functions --- src/caffe/test/test_math_functions.cpp | 79 +++++++++++++------------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/src/caffe/test/test_math_functions.cpp b/src/caffe/test/test_math_functions.cpp index 7e2c71d066c..2c2c062d748 100644 --- a/src/caffe/test/test_math_functions.cpp +++ b/src/caffe/test/test_math_functions.cpp @@ -39,34 +39,32 @@ class MathFunctionsTest : public ::testing::Test { delete blob_bottom_; delete blob_top_; } + // http://en.wikipedia.org/wiki/Hamming_distance - int ReferenceHammingDistance(const int n, const Dtype* x, const Dtype* y); + int ReferenceHammingDistance(const int n, const Dtype* x, const Dtype* y) { + int dist = 0; + uint64_t val; + for (int i = 0; i < n; ++i) { + if (sizeof(Dtype) == 8) { + val = static_cast(x[i]) ^ static_cast(y[i]); + } else if (sizeof(Dtype) == 4) { + val = static_cast(x[i]) ^ static_cast(y[i]); + } else { + LOG(FATAL) << "Unrecognized Dtype size: " << sizeof(Dtype); + } + // Count the number of set bits + while (val) { + ++dist; + val &= val - 1; + } + } + return dist; + } Blob* const blob_bottom_; Blob* const blob_top_; }; -#define REF_HAMMING_DIST(float_type, int_type) \ -template<> \ -int MathFunctionsTest::ReferenceHammingDistance(const int n, \ - const float_type* x, \ - const float_type* y) { \ - int dist = 0; \ - int_type val; \ - for (int i = 0; i < n; ++i) { \ - val = static_cast(x[i]) ^ static_cast(y[i]); \ - /* Count the number of set bits */ \ - while (val) { \ - ++dist; \ - val &= val - 1; \ - } \ - } \ - return dist; \ -} - -REF_HAMMING_DIST(float, uint32_t); -REF_HAMMING_DIST(double, uint64_t); - typedef ::testing::Types Dtypes; TYPED_TEST_CASE(MathFunctionsTest, Dtypes); @@ -79,18 +77,19 @@ TYPED_TEST(MathFunctionsTest, TestHammingDistanceCPU) { int n = this->blob_bottom_->count(); const TypeParam* x = this->blob_bottom_->cpu_data(); const TypeParam* y = this->blob_top_->cpu_data(); - CHECK_EQ(this->ReferenceHammingDistance(n, x, y), - caffe_cpu_hamming_distance(n, x, y)); + EXPECT_EQ(this->ReferenceHammingDistance(n, x, y), + caffe_cpu_hamming_distance(n, x, y)); } TYPED_TEST(MathFunctionsTest, TestHammingDistanceGPU) { int n = this->blob_bottom_->count(); - const TypeParam* x = this->blob_bottom_->gpu_data(); - const TypeParam* y = this->blob_top_->gpu_data(); - const TypeParam* cpu_x = this->blob_bottom_->cpu_data(); - const TypeParam* cpu_y = this->blob_top_->cpu_data(); - CHECK_EQ(this->ReferenceHammingDistance(n, cpu_x, cpu_y), - caffe_gpu_hamming_distance(n, x, y)); + const TypeParam* x = this->blob_bottom_->cpu_data(); + const TypeParam* y = this->blob_top_->cpu_data(); + int reference_distance = this->ReferenceHammingDistance(n, x, y); + x = this->blob_bottom_->gpu_data(); + y = this->blob_top_->gpu_data(); + int computed_distance = caffe_gpu_hamming_distance(n, x, y); + EXPECT_EQ(reference_distance, computed_distance); } TYPED_TEST(MathFunctionsTest, TestAsumCPU) { @@ -101,7 +100,7 @@ TYPED_TEST(MathFunctionsTest, TestAsumCPU) { std_asum += std::fabs(x[i]); } TypeParam cpu_asum = caffe_cpu_asum(n, x); - CHECK_LT((cpu_asum - std_asum) / std_asum, 1e-2); + EXPECT_LT((cpu_asum - std_asum) / std_asum, 1e-2); } TYPED_TEST(MathFunctionsTest, TestAsumGPU) { @@ -113,7 +112,7 @@ TYPED_TEST(MathFunctionsTest, TestAsumGPU) { } TypeParam gpu_asum; caffe_gpu_asum(n, this->blob_bottom_->gpu_data(), &gpu_asum); - CHECK_LT((gpu_asum - std_asum) / std_asum, 1e-2); + EXPECT_LT((gpu_asum - std_asum) / std_asum, 1e-2); } TYPED_TEST(MathFunctionsTest, TestSignCPU) { @@ -122,7 +121,7 @@ TYPED_TEST(MathFunctionsTest, TestSignCPU) { caffe_cpu_sign(n, x, this->blob_bottom_->mutable_cpu_diff()); const TypeParam* signs = this->blob_bottom_->cpu_diff(); for (int i = 0; i < n; ++i) { - CHECK_EQ(signs[i], x[i] > 0 ? 1 : (x[i] < 0 ? -1 : 0)); + EXPECT_EQ(signs[i], x[i] > 0 ? 1 : (x[i] < 0 ? -1 : 0)); } } @@ -133,7 +132,7 @@ TYPED_TEST(MathFunctionsTest, TestSignGPU) { const TypeParam* signs = this->blob_bottom_->cpu_diff(); const TypeParam* x = this->blob_bottom_->cpu_data(); for (int i = 0; i < n; ++i) { - CHECK_EQ(signs[i], x[i] > 0 ? 1 : (x[i] < 0 ? -1 : 0)); + EXPECT_EQ(signs[i], x[i] > 0 ? 1 : (x[i] < 0 ? -1 : 0)); } } @@ -143,7 +142,7 @@ TYPED_TEST(MathFunctionsTest, TestSgnbitCPU) { caffe_cpu_sgnbit(n, x, this->blob_bottom_->mutable_cpu_diff()); const TypeParam* signbits = this->blob_bottom_->cpu_diff(); for (int i = 0; i < n; ++i) { - CHECK_EQ(signbits[i], x[i] < 0 ? 1 : 0); + EXPECT_EQ(signbits[i], x[i] < 0 ? 1 : 0); } } @@ -154,7 +153,7 @@ TYPED_TEST(MathFunctionsTest, TestSgnbitGPU) { const TypeParam* signbits = this->blob_bottom_->cpu_diff(); const TypeParam* x = this->blob_bottom_->cpu_data(); for (int i = 0; i < n; ++i) { - CHECK_EQ(signbits[i], x[i] < 0 ? 1 : 0); + EXPECT_EQ(signbits[i], x[i] < 0 ? 1 : 0); } } @@ -164,7 +163,7 @@ TYPED_TEST(MathFunctionsTest, TestFabsCPU) { caffe_cpu_fabs(n, x, this->blob_bottom_->mutable_cpu_diff()); const TypeParam* abs_val = this->blob_bottom_->cpu_diff(); for (int i = 0; i < n; ++i) { - CHECK_EQ(abs_val[i], x[i] > 0 ? x[i] : -x[i]); + EXPECT_EQ(abs_val[i], x[i] > 0 ? x[i] : -x[i]); } } @@ -175,7 +174,7 @@ TYPED_TEST(MathFunctionsTest, TestFabsGPU) { const TypeParam* abs_val = this->blob_bottom_->cpu_diff(); const TypeParam* x = this->blob_bottom_->cpu_data(); for (int i = 0; i < n; ++i) { - CHECK_EQ(abs_val[i], x[i] > 0 ? x[i] : -x[i]); + EXPECT_EQ(abs_val[i], x[i] > 0 ? x[i] : -x[i]); } } @@ -189,7 +188,7 @@ TYPED_TEST(MathFunctionsTest, TestScaleCPU) { const TypeParam* scaled = this->blob_bottom_->cpu_diff(); const TypeParam* x = this->blob_bottom_->cpu_data(); for (int i = 0; i < n; ++i) { - CHECK_EQ(scaled[i], x[i] * alpha); + EXPECT_EQ(scaled[i], x[i] * alpha); } } @@ -203,7 +202,7 @@ TYPED_TEST(MathFunctionsTest, TestScaleGPU) { const TypeParam* scaled = this->blob_bottom_->cpu_diff(); const TypeParam* x = this->blob_bottom_->cpu_data(); for (int i = 0; i < n; ++i) { - CHECK_EQ(scaled[i], x[i] * alpha); + EXPECT_EQ(scaled[i], x[i] * alpha); } } From 95783f934ca630ff2baf12ab39519f12af8c4ecf Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Mon, 7 Apr 2014 21:16:00 -0700 Subject: [PATCH 09/17] make rng_ a private member of Generator --- src/caffe/common.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/caffe/common.cpp b/src/caffe/common.cpp index 65c68005042..1ea329f23f1 100644 --- a/src/caffe/common.cpp +++ b/src/caffe/common.cpp @@ -125,6 +125,8 @@ class Caffe::RNG::Generator { explicit Generator(unsigned int seed) : rng_(new caffe::rng_t(seed)) {} explicit Generator(const caffe::rng_t& other) : rng_(new caffe::rng_t(other)) {} + const caffe::rng_t& rng() const { return *rng_; } + private: shared_ptr rng_; }; @@ -141,7 +143,7 @@ Caffe::RNG& Caffe::RNG::operator=(const RNG& other) { } const void* Caffe::RNG::generator() const { - return static_cast(generator_->rng_.get()); + return static_cast(&generator_->rng()); } void Caffe::RNG::set_generator(const void* other_rng) { From 5491f58e49f9b8f4255911aebd987efcccda4081 Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Tue, 8 Apr 2014 03:54:27 -0700 Subject: [PATCH 10/17] cleanup RNG unit tests --- .../test/test_random_number_generator.cpp | 611 ++++++++---------- 1 file changed, 255 insertions(+), 356 deletions(-) diff --git a/src/caffe/test/test_random_number_generator.cpp b/src/caffe/test/test_random_number_generator.cpp index f33b4ce79c3..c1425f27ab9 100644 --- a/src/caffe/test/test_random_number_generator.cpp +++ b/src/caffe/test/test_random_number_generator.cpp @@ -14,28 +14,137 @@ namespace caffe { template class RandomNumberGeneratorTest : public ::testing::Test { - public: - virtual ~RandomNumberGeneratorTest() {} + protected: + RandomNumberGeneratorTest() + : sample_size_(10000), + seed_(1701), + data_(new SyncedMemory(sample_size_ * sizeof(Dtype))), + int_data_(new SyncedMemory(sample_size_ * sizeof(int))), + int_data_2_(new SyncedMemory(sample_size_ * sizeof(int))) {} - Dtype sample_mean(const Dtype* const seqs, const size_t sample_size) { - double sum = 0; - for (int i = 0; i < sample_size; ++i) { - sum += seqs[i]; + virtual void SetUp() { + Caffe::set_random_seed(this->seed_); + } + + Dtype sample_mean(const Dtype* const seqs, const int sample_size) { + Dtype sum = 0; + for (int i = 0; i < sample_size; ++i) { + sum += seqs[i]; + } + return sum / sample_size; + } + + Dtype sample_mean(const Dtype* const seqs) { + return sample_mean(seqs, sample_size_); + } + + Dtype sample_mean(const int* const seqs, const int sample_size) { + Dtype sum = 0; + for (int i = 0; i < sample_size; ++i) { + sum += Dtype(seqs[i]); + } + return sum / sample_size; + } + + Dtype sample_mean(const int* const seqs) { + return sample_mean(seqs, sample_size_); + } + + Dtype mean_bound(const Dtype std, const int sample_size) { + return std / sqrt(static_cast(sample_size)); + } + + Dtype mean_bound(const Dtype std) { + return mean_bound(std, sample_size_); + } + + void RngGaussianTest(const Dtype mu, const Dtype sigma, void* cpu_data) { + Dtype* rng_data = static_cast(cpu_data); + caffe_vRngGaussian(sample_size_, rng_data, mu, sigma); + const Dtype true_mean = mu; + const Dtype true_std = sigma; + // Check that sample mean roughly matches true mean. + const Dtype bound = this->mean_bound(true_std); + const Dtype sample_mean = this->sample_mean( + static_cast(cpu_data)); + EXPECT_NEAR(sample_mean, true_mean, bound); + // Check that roughly half the samples are above the true mean. + int num_above_mean = 0; + int num_below_mean = 0; + for (int i = 0; i < sample_size_; ++i) { + if (rng_data[i] > true_mean) { + ++num_above_mean; + } else if (rng_data[i] < true_mean) { + ++num_below_mean; } - return sum / sample_size; + } + EXPECT_EQ(sample_size_, num_above_mean + num_below_mean); + const Dtype sample_p_above_mean = + static_cast(num_above_mean) / sample_size_; + const Dtype bernoulli_p = 0.5; + const Dtype bernoulli_std = sqrt(bernoulli_p * (1 - bernoulli_p)); + const Dtype bernoulli_bound = this->mean_bound(true_std); + EXPECT_NEAR(bernoulli_p, sample_p_above_mean, bernoulli_bound); } - Dtype sample_mean(const int* const seqs, const size_t sample_size) { - Dtype sum = 0; - for (int i = 0; i < sample_size; ++i) { - sum += Dtype(seqs[i]); + void RngUniformTest(const Dtype lower, const Dtype upper, void* cpu_data) { + CHECK_GE(upper, lower); + Dtype* rng_data = static_cast(cpu_data); + caffe_vRngUniform(sample_size_, rng_data, lower, upper); + const Dtype true_mean = (lower + upper) / 2; + const Dtype true_std = (upper - lower) / sqrt(12); + // Check that sample mean roughly matches true mean. + const Dtype bound = this->mean_bound(true_std); + const Dtype sample_mean = this->sample_mean(rng_data); + EXPECT_NEAR(sample_mean, true_mean, bound); + // Check that roughly half the samples are above the true mean, and none are + // above upper or below lower. + int num_above_mean = 0; + int num_below_mean = 0; + int num_above_upper = 0; + int num_below_lower = 0; + for (int i = 0; i < sample_size_; ++i) { + if (rng_data[i] > true_mean) { + ++num_above_mean; + } else if (rng_data[i] < true_mean) { + ++num_below_mean; + } + if (rng_data[i] > upper) { + ++num_above_upper; + } else if (rng_data[i] < lower) { + ++num_below_lower; } - return sum / sample_size; + } + EXPECT_EQ(0, num_above_upper); + EXPECT_EQ(0, num_below_lower); + EXPECT_EQ(sample_size_, num_above_mean + num_below_mean); + const Dtype sample_p_above_mean = + static_cast(num_above_mean) / sample_size_; + const Dtype bernoulli_p = 0.5; + const Dtype bernoulli_std = sqrt(bernoulli_p * (1 - bernoulli_p)); + const Dtype bernoulli_bound = this->mean_bound(true_std); + EXPECT_NEAR(bernoulli_p, sample_p_above_mean, bernoulli_bound); } - Dtype mean_bound(const Dtype std, const size_t sample_size) { - return std/sqrt(static_cast(sample_size)); + void RngBernoulliTest(const Dtype p, void* cpu_data) { + int* rng_data = static_cast(cpu_data); + caffe_vRngBernoulli(sample_size_, rng_data, p); + const Dtype true_mean = p; + const Dtype true_std = sqrt(p * (1 - p)); + const Dtype bound = this->mean_bound(true_std); + const Dtype sample_mean = this->sample_mean(rng_data); + EXPECT_NEAR(sample_mean, true_mean, bound); } + + int num_above_mean; + int num_below_mean; + + size_t sample_size_; + uint32_t seed_; + + shared_ptr data_; + shared_ptr int_data_; + shared_ptr int_data_2_; }; @@ -44,382 +153,172 @@ TYPED_TEST_CASE(RandomNumberGeneratorTest, Dtypes); TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussian) { - size_t sample_size = 10000; - SyncedMemory data_a(sample_size * sizeof(TypeParam)); - Caffe::set_random_seed(1701); - TypeParam mu = 0; - TypeParam sigma = 1; - caffe_vRngGaussian(sample_size, - static_cast(data_a.mutable_cpu_data()), mu, sigma); - TypeParam true_mean = mu; - TypeParam true_std = sigma; - TypeParam bound = this->mean_bound(true_std, sample_size); - TypeParam empirical_mean = - this->sample_mean(static_cast(data_a.cpu_data()), - sample_size); - EXPECT_NEAR(empirical_mean, true_mean, bound); + const TypeParam mu = 0; + const TypeParam sigma = 1; + void* gaussian_data = this->data_->mutable_cpu_data(); + this->RngGaussianTest(mu, sigma, gaussian_data); +} + + +TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussian2) { + const TypeParam mu = -2; + const TypeParam sigma = 3; + void* gaussian_data = this->data_->mutable_cpu_data(); + this->RngGaussianTest(mu, sigma, gaussian_data); } TYPED_TEST(RandomNumberGeneratorTest, TestRngUniform) { - size_t sample_size = 10000; - SyncedMemory data_a(sample_size * sizeof(TypeParam)); - Caffe::set_random_seed(1701); - TypeParam lower = 0; - TypeParam upper = 1; - caffe_vRngUniform(sample_size, - static_cast(data_a.mutable_cpu_data()), lower, upper); - TypeParam true_mean = (lower + upper) / 2; - TypeParam true_std = (upper - lower) / sqrt(12); - TypeParam bound = this->mean_bound(true_std, sample_size); - TypeParam empirical_mean = - this->sample_mean(static_cast(data_a.cpu_data()), - sample_size); - EXPECT_NEAR(empirical_mean, true_mean, bound); + const TypeParam lower = 0; + const TypeParam upper = 1; + void* uniform_data = this->data_->mutable_cpu_data(); + this->RngUniformTest(lower, upper, uniform_data); +} + + +TYPED_TEST(RandomNumberGeneratorTest, TestRngUniform2) { + const TypeParam lower = -7.3; + const TypeParam upper = -2.3; + void* uniform_data = this->data_->mutable_cpu_data(); + this->RngUniformTest(lower, upper, uniform_data); } TYPED_TEST(RandomNumberGeneratorTest, TestRngBernoulli) { - size_t sample_size = 10000; - SyncedMemory data_a(sample_size * sizeof(int)); - Caffe::set_random_seed(1701); - double p = 0.3; - caffe_vRngBernoulli(sample_size, - static_cast(data_a.mutable_cpu_data()), p); - TypeParam true_mean = p; - TypeParam true_std = sqrt(p * (1 - p)); - TypeParam bound = this->mean_bound(true_std, sample_size); - TypeParam empirical_mean = - this->sample_mean((const int *)data_a.cpu_data(), sample_size); - EXPECT_NEAR(empirical_mean, true_mean, bound); + const TypeParam p = 0.3; + void* bernoulli_data = this->int_data_->mutable_cpu_data(); + this->RngBernoulliTest(p, bernoulli_data); +} + + +TYPED_TEST(RandomNumberGeneratorTest, TestRngBernoulli2) { + const TypeParam p = 0.9; + void* bernoulli_data = this->int_data_->mutable_cpu_data(); + this->RngBernoulliTest(p, bernoulli_data); } TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussianTimesBernoulli) { - size_t sample_size = 10000; - SyncedMemory gaussian_data(sample_size * sizeof(TypeParam)); - SyncedMemory bernoulli_data(sample_size * sizeof(int)); - Caffe::set_random_seed(1701); - // Sample from 0 mean Gaussian - TypeParam mu = 0; - TypeParam sigma = 1; - caffe_vRngGaussian(sample_size, static_cast( - gaussian_data.mutable_cpu_data()), mu, sigma); - TypeParam true_mean = mu; - TypeParam true_std = sigma; - TypeParam bound = this->mean_bound(true_std, sample_size); - TypeParam empirical_mean = this->sample_mean( - static_cast(gaussian_data.cpu_data()), - sample_size); - EXPECT_NEAR(empirical_mean, true_mean, bound); + // Sample from 0 mean Gaussian. + const TypeParam mu = 0; + const TypeParam sigma = 1; + TypeParam* gaussian_data = + static_cast(this->data_->mutable_cpu_data()); + this->RngGaussianTest(mu, sigma, gaussian_data); + + // Sample from Bernoulli with p = 0.3. + const TypeParam bernoulli_p = 0.3; + int* bernoulli_data = + static_cast(this->int_data_->mutable_cpu_data()); + this->RngBernoulliTest(bernoulli_p, bernoulli_data); + + // Multiply Gaussian by Bernoulli. + for (int i = 0; i < this->sample_size_; ++i) { + gaussian_data[i] *= bernoulli_data[i]; + } int num_pos = 0; int num_neg = 0; - int num_zeros = 0; - TypeParam* samples = - static_cast(gaussian_data.mutable_cpu_data()); - for (int i = 0; i < sample_size; ++i) { - if (samples[i] == TypeParam(0)) { - ++num_zeros; - } else if (samples[i] > TypeParam(0)) { - ++num_pos; - } else if (samples[i] < TypeParam(0)) { - ++num_neg; - } - } - // Check that we have no zeros (possible to generate 0s, but highly - // improbable), and roughly half positives and half negatives (with bound - // computed from a Bernoulli with p = 0.5). - EXPECT_EQ(0, num_zeros); - double p = 0.5; - true_mean = p; - true_std = sqrt(p * (1 - p)); - bound = this->mean_bound(true_std, sample_size); - TypeParam expected_num_each_sign = sample_size * p; - LOG(INFO) << "Gaussian: Expected " << expected_num_each_sign << " positives" - << "; got " << num_pos; - LOG(INFO) << "Gaussian: Expected " << expected_num_each_sign << " negatives" - << "; got " << num_neg; - EXPECT_NEAR(expected_num_each_sign, num_pos, sample_size * bound); - EXPECT_NEAR(expected_num_each_sign, num_neg, sample_size * bound); - // Sample from Bernoulli with p = 0.3 - p = 0.3; - caffe_vRngBernoulli(sample_size, - static_cast(bernoulli_data.mutable_cpu_data()), p); - true_mean = p; - true_std = sqrt(p * (1 - p)); - bound = this->mean_bound(true_std, sample_size); - empirical_mean = - this->sample_mean((const int *)bernoulli_data.cpu_data(), sample_size); - LOG(INFO) << "Bernoulli: Expected mean = " << true_mean - << "; sample mean = " << empirical_mean; - EXPECT_NEAR(empirical_mean, true_mean, bound); - int bernoulli_num_zeros = 0; - int num_ones = 0; - int num_other = 0; - const int* bernoulli_samples = - static_cast(bernoulli_data.cpu_data()); - for (int i = 0; i < sample_size; ++i) { - if (bernoulli_samples[i] == 0) { - ++bernoulli_num_zeros; - } else if (bernoulli_samples[i] == 1) { - ++num_ones; + for (int i = 0; i < this->sample_size_; ++i) { + if (gaussian_data[i] == TypeParam(0)) { + EXPECT_EQ(TypeParam(0), bernoulli_data[i]); } else { - ++num_other; - } - } - LOG(INFO) << "Bernoulli: zeros: " << bernoulli_num_zeros - << "; ones: " << num_ones << "; other: " << num_other; - EXPECT_EQ(0, num_other); - TypeParam epsilon = 1e-4; - EXPECT_NEAR(sample_size * empirical_mean, num_ones, epsilon); - EXPECT_NEAR(sample_size * (1.0 - empirical_mean), bernoulli_num_zeros, - epsilon); - // Multiply Gaussian by Bernoulli - for (int i = 0; i < sample_size; ++i) { - samples[i] *= bernoulli_samples[i]; - } - num_pos = 0; - num_neg = 0; - num_zeros = 0; - for (int i = 0; i < sample_size; ++i) { - if (samples[i] == TypeParam(0)) { - ++num_zeros; - } else if (samples[i] > TypeParam(0)) { - ++num_pos; - } else if (samples[i] < TypeParam(0)) { - ++num_neg; + EXPECT_EQ(TypeParam(1), bernoulli_data[i]); + if (gaussian_data[i] > TypeParam(0)) { + ++num_pos; + } else if (gaussian_data[i] < TypeParam(0)) { + ++num_neg; + } } } - // Check that we have as many zeros as Bernoulli, and roughly half positives - // and half negatives (with bound computed from a Bernoulli with p = 0.5). - EXPECT_EQ(bernoulli_num_zeros, num_zeros); - p = 0.5; - true_mean = p; - true_std = sqrt(p * (1 - p)); - int sub_sample_size = sample_size - bernoulli_num_zeros; - bound = this->mean_bound(true_std, sub_sample_size); - expected_num_each_sign = sub_sample_size * p; - LOG(INFO) << "Gaussian*Bernoulli: Expected " << expected_num_each_sign - << " positives; got " << num_pos; - LOG(INFO) << "Gaussian*Bernoulli: Expected " << expected_num_each_sign - << " negatives; got " << num_neg; - EXPECT_NEAR(expected_num_each_sign, num_pos, sample_size * bound); - EXPECT_NEAR(expected_num_each_sign, num_neg, sample_size * bound); + + // Check that Gaussian still has roughly half positives and half negatives + // (with bound computed from a Bernoulli with p = 0.5). + const int num_non_zero = num_pos + num_neg; + const TypeParam sample_p = num_pos / static_cast(num_non_zero); + const TypeParam p = 0.5; + const TypeParam true_mean = p; + const TypeParam true_std = sqrt(p * (1 - p)); + const TypeParam bound = this->mean_bound(true_std, num_non_zero); + EXPECT_NEAR(true_mean, sample_p, bound); } TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformTimesBernoulli) { - size_t sample_size = 10000; - SyncedMemory uniform_data(sample_size * sizeof(TypeParam)); - SyncedMemory bernoulli_data(sample_size * sizeof(int)); - Caffe::set_random_seed(1701); - // Sample from Uniform on [-1, 1] - TypeParam a = -1; - TypeParam b = 1; - caffe_vRngUniform(sample_size, static_cast( - uniform_data.mutable_cpu_data()), a, b); - TypeParam true_mean = (a + b) / 2; - TypeParam true_std = (b - a) / sqrt(12); - TypeParam bound = this->mean_bound(true_std, sample_size); - TypeParam empirical_mean = this->sample_mean( - static_cast(uniform_data.cpu_data()), - sample_size); - EXPECT_NEAR(empirical_mean, true_mean, bound); + // Sample from Uniform on [-1, 1]. + const TypeParam lower = -1; + const TypeParam upper = 1; + TypeParam* uniform_data = + static_cast(this->data_->mutable_cpu_data()); + this->RngUniformTest(lower, upper, uniform_data); + + // Sample from Bernoulli with p = 0.3. + const TypeParam bernoulli_p = 0.3; + int* bernoulli_data = + static_cast(this->int_data_->mutable_cpu_data()); + this->RngBernoulliTest(bernoulli_p, bernoulli_data); + + // Multiply Uniform by Bernoulli. + for (int i = 0; i < this->sample_size_; ++i) { + uniform_data[i] *= bernoulli_data[i]; + } int num_pos = 0; int num_neg = 0; - int num_zeros = 0; - TypeParam* samples = - static_cast(uniform_data.mutable_cpu_data()); - for (int i = 0; i < sample_size; ++i) { - if (samples[i] == TypeParam(0)) { - ++num_zeros; - } else if (samples[i] > TypeParam(0)) { - ++num_pos; - } else if (samples[i] < TypeParam(0)) { - ++num_neg; - } - } - // Check that we have no zeros (possible to generate 0s, but highly - // improbable), and roughly half positives and half negatives (with bound - // computed from a Bernoulli with p = 0.5). - EXPECT_EQ(0, num_zeros); - TypeParam p = 0.5; - true_mean = p; - true_std = sqrt(p * (1 - p)); - bound = this->mean_bound(true_std, sample_size); - TypeParam expected_num_each_sign = sample_size * p; - LOG(INFO) << "Uniform: Expected " << expected_num_each_sign << " positives" - << "; got " << num_pos; - LOG(INFO) << "Uniform: Expected " << expected_num_each_sign << " negatives" - << "; got " << num_neg; - EXPECT_NEAR(expected_num_each_sign, num_pos, sample_size * bound); - EXPECT_NEAR(expected_num_each_sign, num_neg, sample_size * bound); - // Sample from Bernoulli with p = 0.3 - p = 0.3; - caffe_vRngBernoulli(sample_size, - static_cast(bernoulli_data.mutable_cpu_data()), p); - true_mean = p; - true_std = sqrt(p * (1 - p)); - bound = this->mean_bound(true_std, sample_size); - empirical_mean = - this->sample_mean((const int *)bernoulli_data.cpu_data(), sample_size); - LOG(INFO) << "Bernoulli: Expected mean = " << true_mean - << "; sample mean = " << empirical_mean; - EXPECT_NEAR(empirical_mean, true_mean, bound); - int bernoulli_num_zeros = 0; - int num_ones = 0; - int num_other = 0; - const int* bernoulli_samples = - static_cast(bernoulli_data.cpu_data()); - for (int i = 0; i < sample_size; ++i) { - if (bernoulli_samples[i] == 0) { - ++bernoulli_num_zeros; - } else if (bernoulli_samples[i] == 1) { - ++num_ones; + for (int i = 0; i < this->sample_size_; ++i) { + if (uniform_data[i] == TypeParam(0)) { + EXPECT_EQ(TypeParam(0), bernoulli_data[i]); } else { - ++num_other; - } - } - LOG(INFO) << "Bernoulli: zeros: " << bernoulli_num_zeros - << "; ones: " << num_ones << "; other: " << num_other; - EXPECT_EQ(0, num_other); - TypeParam epsilon = 1e-4; - EXPECT_NEAR(sample_size * empirical_mean, num_ones, epsilon); - EXPECT_NEAR(sample_size * (1.0 - empirical_mean), bernoulli_num_zeros, - epsilon); - // Multiply Uniform by Bernoulli - for (int i = 0; i < sample_size; ++i) { - samples[i] *= bernoulli_samples[i]; - } - num_pos = 0; - num_neg = 0; - num_zeros = 0; - for (int i = 0; i < sample_size; ++i) { - if (samples[i] == TypeParam(0)) { - ++num_zeros; - } else if (samples[i] > TypeParam(0)) { - ++num_pos; - } else if (samples[i] < TypeParam(0)) { - ++num_neg; + EXPECT_EQ(TypeParam(1), bernoulli_data[i]); + if (uniform_data[i] > TypeParam(0)) { + ++num_pos; + } else if (uniform_data[i] < TypeParam(0)) { + ++num_neg; + } } } - // Check that we have as many zeros as Bernoulli, and roughly half positives - // and half negatives (with bound computed from a Bernoulli with p = 0.5). - EXPECT_EQ(bernoulli_num_zeros, num_zeros); - p = 0.5; - true_mean = p; - true_std = sqrt(p * (1 - p)); - int sub_sample_size = sample_size - bernoulli_num_zeros; - bound = this->mean_bound(true_std, sub_sample_size); - expected_num_each_sign = sub_sample_size * p; - LOG(INFO) << "Uniform*Bernoulli: Expected " << expected_num_each_sign - << " positives; got " << num_pos; - LOG(INFO) << "Uniform*Bernoulli: Expected " << expected_num_each_sign - << " negatives; got " << num_neg; - EXPECT_NEAR(expected_num_each_sign, num_pos, sample_size * bound); - EXPECT_NEAR(expected_num_each_sign, num_neg, sample_size * bound); + + // Check that Uniform still has roughly half positives and half negatives + // (with bound computed from a Bernoulli with p = 0.5). + const int num_non_zero = num_pos + num_neg; + const TypeParam sample_p = num_pos / static_cast(num_non_zero); + const TypeParam p = 0.5; + const TypeParam true_mean = p; + const TypeParam true_std = sqrt(p * (1 - p)); + const TypeParam bound = this->mean_bound(true_std, num_non_zero); + EXPECT_NEAR(true_mean, sample_p, bound); } TYPED_TEST(RandomNumberGeneratorTest, TestRngBernoulliTimesBernoulli) { - size_t sample_size = 10000; - SyncedMemory bernoulli1_data(sample_size * sizeof(int)); - SyncedMemory bernoulli2_data(sample_size * sizeof(int)); - Caffe::set_random_seed(1701); - // Sample from Bernoulli with p = 0.5 - TypeParam p1 = 0.5; - caffe_vRngBernoulli(sample_size, static_cast( - bernoulli1_data.mutable_cpu_data()), p1); - TypeParam empirical_mean = this->sample_mean( - static_cast(bernoulli1_data.cpu_data()), - sample_size); - int bernoulli1_num_zeros = 0; - int num_ones = 0; - int num_other = 0; - int* bernoulli_samples = - static_cast(bernoulli1_data.mutable_cpu_data()); - for (int i = 0; i < sample_size; ++i) { - if (bernoulli_samples[i] == 0) { - ++bernoulli1_num_zeros; - } else if (bernoulli_samples[i] == 1) { - ++num_ones; - } else { - ++num_other; - } - } - TypeParam true_mean = p1; - TypeParam true_std = sqrt(p1 * (1 - p1)); - TypeParam bound = this->mean_bound(true_std, sample_size); - TypeParam expected_num_zeros = sample_size * (1 - true_mean); - TypeParam expected_num_ones = sample_size * true_mean; - LOG(INFO) << "Bernoulli1: Expected mean = " << true_mean - << "; sample mean = " << empirical_mean; - LOG(INFO) << "Bernoulli1: zeros: " << bernoulli1_num_zeros - << "; ones: " << num_ones << "; other: " << num_other; - empirical_mean = this->sample_mean( - static_cast(bernoulli1_data.cpu_data()), sample_size); - EXPECT_NEAR(empirical_mean, true_mean, bound); - EXPECT_EQ(num_other, 0); - // Sample from Bernoulli with p = 0.3 - TypeParam p = 0.3; - caffe_vRngBernoulli(sample_size, static_cast( - bernoulli2_data.mutable_cpu_data()), p); - true_mean = p; - true_std = sqrt(p * (1 - p)); - bound = this->mean_bound(true_std, sample_size); - empirical_mean = this->sample_mean( - static_cast(bernoulli2_data.cpu_data()), sample_size); - LOG(INFO) << "Bernoulli2: Expected mean = " << true_mean - << "; sample mean = " << empirical_mean; - EXPECT_NEAR(empirical_mean, true_mean, bound); - int bernoulli2_num_zeros = 0; - num_ones = 0; - num_other = 0; - const int* bernoulli2_samples = - static_cast(bernoulli2_data.cpu_data()); - for (int i = 0; i < sample_size; ++i) { - if (bernoulli2_samples[i] == 0) { - ++bernoulli2_num_zeros; - } else if (bernoulli2_samples[i] == 1) { - ++num_ones; - } else { - ++num_other; - } - } - LOG(INFO) << "Bernoulli2: zeros: " << bernoulli2_num_zeros - << "; ones: " << num_ones << "; other: " << num_other; - EXPECT_EQ(0, num_other); - TypeParam epsilon = 1e-4; - EXPECT_NEAR(sample_size * empirical_mean, num_ones, epsilon); - EXPECT_NEAR(sample_size * (1.0 - empirical_mean), bernoulli2_num_zeros, - epsilon); - // Multiply Bernoulli1 by Bernoulli2 - for (int i = 0; i < sample_size; ++i) { - bernoulli_samples[i] *= bernoulli2_samples[i]; + // Sample from Bernoulli with p = 0.5. + const TypeParam p_a = 0.5; + int* bernoulli_data_a = + static_cast(this->int_data_->mutable_cpu_data()); + this->RngBernoulliTest(p_a, bernoulli_data_a); + + // Sample from Bernoulli with p = 0.3. + const TypeParam p_b = 0.3; + int* bernoulli_data_b = + static_cast(this->int_data_2_->mutable_cpu_data()); + this->RngBernoulliTest(p_b, bernoulli_data_b); + + // Multiply Bernoullis. + for (int i = 0; i < this->sample_size_; ++i) { + bernoulli_data_a[i] *= bernoulli_data_b[i]; } - bernoulli1_num_zeros = 0; - num_ones = 0; - num_other = 0; - for (int i = 0; i < sample_size; ++i) { - if (bernoulli_samples[i] == 0) { - ++bernoulli1_num_zeros; - } else if (bernoulli_samples[i] == 1) { + int num_ones = 0; + for (int i = 0; i < this->sample_size_; ++i) { + if (bernoulli_data_a[i] != TypeParam(0)) { + EXPECT_EQ(TypeParam(1), bernoulli_data_a[i]); ++num_ones; - } else { - ++num_other; } } - // Check that we have as many zeros as Bernoulli, and roughly half positives - // and half negatives (with bound computed from a Bernoulli with p = 0.5). - p *= p1; - true_mean = p; - true_std = sqrt(p * (1 - p)); - empirical_mean = this->sample_mean( - static_cast(bernoulli1_data.cpu_data()), sample_size); - bound = this->mean_bound(true_std, sample_size); - LOG(INFO) << "Bernoulli1*Bernoulli2: Expected mean = " << true_mean - << "; sample mean = " << empirical_mean; - EXPECT_NEAR(empirical_mean, true_mean, bound); + + // Check that resulting product has roughly p_a * p_b ones. + const TypeParam sample_p = this->sample_mean(bernoulli_data_a); + const TypeParam true_mean = p_a * p_b; + const TypeParam true_std = sqrt(true_mean * (1 - true_mean)); + const TypeParam bound = this->mean_bound(true_std); + EXPECT_NEAR(true_mean, sample_p, bound); } From 81db75c6aef06ab0699690ad290a6853f2088711 Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Tue, 8 Apr 2014 04:25:20 -0700 Subject: [PATCH 11/17] gpu_hamming_distance fails unit test with fixed RNG; mark it NOT_IMPLEMENTED with TODO to fix and disable its unit test --- src/caffe/test/test_math_functions.cpp | 3 ++- src/caffe/util/math_functions.cu | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/caffe/test/test_math_functions.cpp b/src/caffe/test/test_math_functions.cpp index 2c2c062d748..3ddd320835d 100644 --- a/src/caffe/test/test_math_functions.cpp +++ b/src/caffe/test/test_math_functions.cpp @@ -81,7 +81,8 @@ TYPED_TEST(MathFunctionsTest, TestHammingDistanceCPU) { caffe_cpu_hamming_distance(n, x, y)); } -TYPED_TEST(MathFunctionsTest, TestHammingDistanceGPU) { +// TODO: Fix caffe_gpu_hamming_distance and re-enable this test. +TYPED_TEST(MathFunctionsTest, DISABLED_TestHammingDistanceGPU) { int n = this->blob_bottom_->count(); const TypeParam* x = this->blob_bottom_->cpu_data(); const TypeParam* y = this->blob_top_->cpu_data(); diff --git a/src/caffe/util/math_functions.cu b/src/caffe/util/math_functions.cu index 0ac10d966fd..40c4f507ef4 100644 --- a/src/caffe/util/math_functions.cu +++ b/src/caffe/util/math_functions.cu @@ -159,6 +159,9 @@ __global__ void popcll_kernel(const int n, const double* a, template <> uint32_t caffe_gpu_hamming_distance(const int n, const float* x, const float* y) { + // TODO: Fix caffe_gpu_hamming_distance (see failing unit test + // TestHammingDistanceGPU in test_math_functions.cpp). + NOT_IMPLEMENTED; thrust::device_vector popcounts(n); // NOLINT_NEXT_LINE(whitespace/operators) popc_kernel<<>>( @@ -170,6 +173,9 @@ uint32_t caffe_gpu_hamming_distance(const int n, const float* x, template <> uint32_t caffe_gpu_hamming_distance(const int n, const double* x, const double* y) { + // TODO: Fix caffe_gpu_hamming_distance (see failing unit test + // TestHammingDistanceGPU in test_math_functions.cpp). + NOT_IMPLEMENTED; thrust::device_vector popcounts(n); // NOLINT_NEXT_LINE(whitespace/operators) popcll_kernel<<>>( From 078e0bf713bcc4f9178f6694c66b368302448484 Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Tue, 8 Apr 2014 11:57:25 -0700 Subject: [PATCH 12/17] make RNG function names more similar to other caffe math function names --- include/caffe/filler.hpp | 8 ++++---- include/caffe/util/math_functions.hpp | 6 +++--- src/caffe/layers/dropout_layer.cpp | 2 +- src/caffe/test/test_common.cpp | 4 ++-- .../test/test_random_number_generator.cpp | 6 +++--- src/caffe/util/math_functions.cpp | 18 +++++++++--------- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/include/caffe/filler.hpp b/include/caffe/filler.hpp index ba473e13632..256a03be62c 100644 --- a/include/caffe/filler.hpp +++ b/include/caffe/filler.hpp @@ -51,7 +51,7 @@ class UniformFiller : public Filler { : Filler(param) {} virtual void Fill(Blob* blob) { CHECK(blob->count()); - caffe_vRngUniform(blob->count(), blob->mutable_cpu_data(), + caffe_rng_uniform(blob->count(), blob->mutable_cpu_data(), Dtype(this->filler_param_.min()), Dtype(this->filler_param_.max())); } @@ -65,7 +65,7 @@ class GaussianFiller : public Filler { virtual void Fill(Blob* blob) { Dtype* data = blob->mutable_cpu_data(); CHECK(blob->count()); - caffe_vRngGaussian(blob->count(), blob->mutable_cpu_data(), + caffe_rng_gaussian(blob->count(), blob->mutable_cpu_data(), Dtype(this->filler_param_.mean()), Dtype(this->filler_param_.std())); } @@ -79,7 +79,7 @@ class PositiveUnitballFiller : public Filler { virtual void Fill(Blob* blob) { Dtype* data = blob->mutable_cpu_data(); DCHECK(blob->count()); - caffe_vRngUniform(blob->count(), blob->mutable_cpu_data(), 0, 1); + caffe_rng_uniform(blob->count(), blob->mutable_cpu_data(), 0, 1); // We expect the filler to not be called very frequently, so we will // just use a simple implementation int dim = blob->count() / blob->num(); @@ -113,7 +113,7 @@ class XavierFiller : public Filler { CHECK(blob->count()); int fan_in = blob->count() / blob->num(); Dtype scale = sqrt(Dtype(3) / fan_in); - caffe_vRngUniform(blob->count(), blob->mutable_cpu_data(), + caffe_rng_uniform(blob->count(), blob->mutable_cpu_data(), -scale, scale); } }; diff --git a/include/caffe/util/math_functions.hpp b/include/caffe/util/math_functions.hpp index 77e3234cf77..23aa2654488 100644 --- a/include/caffe/util/math_functions.hpp +++ b/include/caffe/util/math_functions.hpp @@ -109,14 +109,14 @@ template Dtype caffe_nextafter(const Dtype b); template -void caffe_vRngUniform(const int n, Dtype* r, const Dtype a, const Dtype b); +void caffe_rng_uniform(const int n, Dtype* r, const Dtype a, const Dtype b); template -void caffe_vRngGaussian(const int n, Dtype* r, const Dtype a, +void caffe_rng_gaussian(const int n, Dtype* r, const Dtype a, const Dtype sigma); template -void caffe_vRngBernoulli(const int n, int* r, const Dtype p); +void caffe_rng_bernoulli(const int n, int* r, const Dtype p); template void caffe_exp(const int n, const Dtype* a, Dtype* y); diff --git a/src/caffe/layers/dropout_layer.cpp b/src/caffe/layers/dropout_layer.cpp index a57999c2436..a9dd842e97d 100644 --- a/src/caffe/layers/dropout_layer.cpp +++ b/src/caffe/layers/dropout_layer.cpp @@ -32,7 +32,7 @@ Dtype DropoutLayer::Forward_cpu(const vector*>& bottom, const int count = bottom[0]->count(); if (Caffe::phase() == Caffe::TRAIN) { // Create random numbers - caffe_vRngBernoulli(count, mask, 1. - threshold_); + caffe_rng_bernoulli(count, mask, 1. - threshold_); for (int i = 0; i < count; ++i) { top_data[i] = bottom_data[i] * mask[i] * scale_; } diff --git a/src/caffe/test/test_common.cpp b/src/caffe/test/test_common.cpp index 7839c377bde..a043db16624 100644 --- a/src/caffe/test/test_common.cpp +++ b/src/caffe/test/test_common.cpp @@ -36,11 +36,11 @@ TEST_F(CommonTest, TestRandSeedCPU) { SyncedMemory data_a(10 * sizeof(int)); SyncedMemory data_b(10 * sizeof(int)); Caffe::set_random_seed(1701); - caffe_vRngBernoulli(10, + caffe_rng_bernoulli(10, reinterpret_cast(data_a.mutable_cpu_data()), 0.5); Caffe::set_random_seed(1701); - caffe_vRngBernoulli(10, + caffe_rng_bernoulli(10, reinterpret_cast(data_b.mutable_cpu_data()), 0.5); for (int i = 0; i < 10; ++i) { diff --git a/src/caffe/test/test_random_number_generator.cpp b/src/caffe/test/test_random_number_generator.cpp index c1425f27ab9..664f82f154c 100644 --- a/src/caffe/test/test_random_number_generator.cpp +++ b/src/caffe/test/test_random_number_generator.cpp @@ -60,7 +60,7 @@ class RandomNumberGeneratorTest : public ::testing::Test { void RngGaussianTest(const Dtype mu, const Dtype sigma, void* cpu_data) { Dtype* rng_data = static_cast(cpu_data); - caffe_vRngGaussian(sample_size_, rng_data, mu, sigma); + caffe_rng_gaussian(sample_size_, rng_data, mu, sigma); const Dtype true_mean = mu; const Dtype true_std = sigma; // Check that sample mean roughly matches true mean. @@ -90,7 +90,7 @@ class RandomNumberGeneratorTest : public ::testing::Test { void RngUniformTest(const Dtype lower, const Dtype upper, void* cpu_data) { CHECK_GE(upper, lower); Dtype* rng_data = static_cast(cpu_data); - caffe_vRngUniform(sample_size_, rng_data, lower, upper); + caffe_rng_uniform(sample_size_, rng_data, lower, upper); const Dtype true_mean = (lower + upper) / 2; const Dtype true_std = (upper - lower) / sqrt(12); // Check that sample mean roughly matches true mean. @@ -128,7 +128,7 @@ class RandomNumberGeneratorTest : public ::testing::Test { void RngBernoulliTest(const Dtype p, void* cpu_data) { int* rng_data = static_cast(cpu_data); - caffe_vRngBernoulli(sample_size_, rng_data, p); + caffe_rng_bernoulli(sample_size_, rng_data, p); const Dtype true_mean = p; const Dtype true_std = sqrt(p * (1 - p)); const Dtype bound = this->mean_bound(true_std); diff --git a/src/caffe/util/math_functions.cpp b/src/caffe/util/math_functions.cpp index 0791a86c55a..a524ed3d838 100644 --- a/src/caffe/util/math_functions.cpp +++ b/src/caffe/util/math_functions.cpp @@ -315,7 +315,7 @@ template double caffe_nextafter(const double b); template -void caffe_vRngUniform(const int n, Dtype* r, +void caffe_rng_uniform(const int n, Dtype* r, const Dtype a, const Dtype b) { CHECK_GE(n, 0); CHECK(r); @@ -330,14 +330,14 @@ void caffe_vRngUniform(const int n, Dtype* r, } template -void caffe_vRngUniform(const int n, float* r, +void caffe_rng_uniform(const int n, float* r, const float a, const float b); template -void caffe_vRngUniform(const int n, double* r, +void caffe_rng_uniform(const int n, double* r, const double a, const double b); template -void caffe_vRngGaussian(const int n, Dtype* r, const Dtype a, +void caffe_rng_gaussian(const int n, Dtype* r, const Dtype a, const Dtype sigma) { CHECK_GE(n, 0); CHECK(r); @@ -352,15 +352,15 @@ void caffe_vRngGaussian(const int n, Dtype* r, const Dtype a, } template -void caffe_vRngGaussian(const int n, float* r, const float a, +void caffe_rng_gaussian(const int n, float* r, const float a, const float sigma); template -void caffe_vRngGaussian(const int n, double* r, const double a, +void caffe_rng_gaussian(const int n, double* r, const double a, const double sigma); template -void caffe_vRngBernoulli(const int n, int* r, const Dtype p) { +void caffe_rng_bernoulli(const int n, int* r, const Dtype p) { CHECK_GE(n, 0); CHECK(r); CHECK_GE(p, 0); @@ -375,10 +375,10 @@ void caffe_vRngBernoulli(const int n, int* r, const Dtype p) { } template -void caffe_vRngBernoulli(const int n, int* r, const double p); +void caffe_rng_bernoulli(const int n, int* r, const double p); template -void caffe_vRngBernoulli(const int n, int* r, const float p); +void caffe_rng_bernoulli(const int n, int* r, const float p); template <> float caffe_cpu_dot(const int n, const float* x, const float* y) { From d00279917a5282e8ce6194c87e443623e545033a Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Tue, 8 Apr 2014 13:18:29 -0700 Subject: [PATCH 13/17] make RNG function outputs the last argument per Google C++ style guidelines --- include/caffe/filler.hpp | 16 +++++----- include/caffe/util/math_functions.hpp | 8 ++--- src/caffe/layers/dropout_layer.cpp | 2 +- src/caffe/test/test_common.cpp | 10 +++---- .../test/test_random_number_generator.cpp | 6 ++-- src/caffe/util/math_functions.cpp | 30 +++++++++---------- 6 files changed, 34 insertions(+), 38 deletions(-) diff --git a/include/caffe/filler.hpp b/include/caffe/filler.hpp index 256a03be62c..50a397e1ee1 100644 --- a/include/caffe/filler.hpp +++ b/include/caffe/filler.hpp @@ -51,9 +51,8 @@ class UniformFiller : public Filler { : Filler(param) {} virtual void Fill(Blob* blob) { CHECK(blob->count()); - caffe_rng_uniform(blob->count(), blob->mutable_cpu_data(), - Dtype(this->filler_param_.min()), - Dtype(this->filler_param_.max())); + caffe_rng_uniform(blob->count(), Dtype(this->filler_param_.min()), + Dtype(this->filler_param_.max()), blob->mutable_cpu_data()); } }; @@ -65,9 +64,8 @@ class GaussianFiller : public Filler { virtual void Fill(Blob* blob) { Dtype* data = blob->mutable_cpu_data(); CHECK(blob->count()); - caffe_rng_gaussian(blob->count(), blob->mutable_cpu_data(), - Dtype(this->filler_param_.mean()), - Dtype(this->filler_param_.std())); + caffe_rng_gaussian(blob->count(), Dtype(this->filler_param_.mean()), + Dtype(this->filler_param_.std()), blob->mutable_cpu_data()); } }; @@ -79,7 +77,7 @@ class PositiveUnitballFiller : public Filler { virtual void Fill(Blob* blob) { Dtype* data = blob->mutable_cpu_data(); DCHECK(blob->count()); - caffe_rng_uniform(blob->count(), blob->mutable_cpu_data(), 0, 1); + caffe_rng_uniform(blob->count(), 0, 1, blob->mutable_cpu_data()); // We expect the filler to not be called very frequently, so we will // just use a simple implementation int dim = blob->count() / blob->num(); @@ -113,8 +111,8 @@ class XavierFiller : public Filler { CHECK(blob->count()); int fan_in = blob->count() / blob->num(); Dtype scale = sqrt(Dtype(3) / fan_in); - caffe_rng_uniform(blob->count(), blob->mutable_cpu_data(), - -scale, scale); + caffe_rng_uniform(blob->count(), -scale, scale, + blob->mutable_cpu_data()); } }; diff --git a/include/caffe/util/math_functions.hpp b/include/caffe/util/math_functions.hpp index 23aa2654488..7129cf9c795 100644 --- a/include/caffe/util/math_functions.hpp +++ b/include/caffe/util/math_functions.hpp @@ -109,14 +109,14 @@ template Dtype caffe_nextafter(const Dtype b); template -void caffe_rng_uniform(const int n, Dtype* r, const Dtype a, const Dtype b); +void caffe_rng_uniform(const int n, const Dtype a, const Dtype b, Dtype* r); template -void caffe_rng_gaussian(const int n, Dtype* r, const Dtype a, - const Dtype sigma); +void caffe_rng_gaussian(const int n, const Dtype mu, const Dtype sigma, + Dtype* r); template -void caffe_rng_bernoulli(const int n, int* r, const Dtype p); +void caffe_rng_bernoulli(const int n, const Dtype p, int* r); template void caffe_exp(const int n, const Dtype* a, Dtype* y); diff --git a/src/caffe/layers/dropout_layer.cpp b/src/caffe/layers/dropout_layer.cpp index a9dd842e97d..e0068fedfb6 100644 --- a/src/caffe/layers/dropout_layer.cpp +++ b/src/caffe/layers/dropout_layer.cpp @@ -32,7 +32,7 @@ Dtype DropoutLayer::Forward_cpu(const vector*>& bottom, const int count = bottom[0]->count(); if (Caffe::phase() == Caffe::TRAIN) { // Create random numbers - caffe_rng_bernoulli(count, mask, 1. - threshold_); + caffe_rng_bernoulli(count, 1. - threshold_, mask); for (int i = 0; i < count; ++i) { top_data[i] = bottom_data[i] * mask[i] * scale_; } diff --git a/src/caffe/test/test_common.cpp b/src/caffe/test/test_common.cpp index a043db16624..f236d1253ed 100644 --- a/src/caffe/test/test_common.cpp +++ b/src/caffe/test/test_common.cpp @@ -36,16 +36,14 @@ TEST_F(CommonTest, TestRandSeedCPU) { SyncedMemory data_a(10 * sizeof(int)); SyncedMemory data_b(10 * sizeof(int)); Caffe::set_random_seed(1701); - caffe_rng_bernoulli(10, - reinterpret_cast(data_a.mutable_cpu_data()), 0.5); + caffe_rng_bernoulli(10, 0.5, static_cast(data_a.mutable_cpu_data())); Caffe::set_random_seed(1701); - caffe_rng_bernoulli(10, - reinterpret_cast(data_b.mutable_cpu_data()), 0.5); + caffe_rng_bernoulli(10, 0.5, static_cast(data_b.mutable_cpu_data())); for (int i = 0; i < 10; ++i) { - EXPECT_EQ(((const int*)(data_a.cpu_data()))[i], - ((const int*)(data_b.cpu_data()))[i]); + EXPECT_EQ(static_cast(data_a.cpu_data())[i], + static_cast(data_b.cpu_data())[i]); } } diff --git a/src/caffe/test/test_random_number_generator.cpp b/src/caffe/test/test_random_number_generator.cpp index 664f82f154c..d068d03cf35 100644 --- a/src/caffe/test/test_random_number_generator.cpp +++ b/src/caffe/test/test_random_number_generator.cpp @@ -60,7 +60,7 @@ class RandomNumberGeneratorTest : public ::testing::Test { void RngGaussianTest(const Dtype mu, const Dtype sigma, void* cpu_data) { Dtype* rng_data = static_cast(cpu_data); - caffe_rng_gaussian(sample_size_, rng_data, mu, sigma); + caffe_rng_gaussian(sample_size_, mu, sigma, rng_data); const Dtype true_mean = mu; const Dtype true_std = sigma; // Check that sample mean roughly matches true mean. @@ -90,7 +90,7 @@ class RandomNumberGeneratorTest : public ::testing::Test { void RngUniformTest(const Dtype lower, const Dtype upper, void* cpu_data) { CHECK_GE(upper, lower); Dtype* rng_data = static_cast(cpu_data); - caffe_rng_uniform(sample_size_, rng_data, lower, upper); + caffe_rng_uniform(sample_size_, lower, upper, rng_data); const Dtype true_mean = (lower + upper) / 2; const Dtype true_std = (upper - lower) / sqrt(12); // Check that sample mean roughly matches true mean. @@ -128,7 +128,7 @@ class RandomNumberGeneratorTest : public ::testing::Test { void RngBernoulliTest(const Dtype p, void* cpu_data) { int* rng_data = static_cast(cpu_data); - caffe_rng_bernoulli(sample_size_, rng_data, p); + caffe_rng_bernoulli(sample_size_, p, rng_data); const Dtype true_mean = p; const Dtype true_std = sqrt(p * (1 - p)); const Dtype bound = this->mean_bound(true_std); diff --git a/src/caffe/util/math_functions.cpp b/src/caffe/util/math_functions.cpp index a524ed3d838..c26675f8917 100644 --- a/src/caffe/util/math_functions.cpp +++ b/src/caffe/util/math_functions.cpp @@ -315,8 +315,7 @@ template double caffe_nextafter(const double b); template -void caffe_rng_uniform(const int n, Dtype* r, - const Dtype a, const Dtype b) { +void caffe_rng_uniform(const int n, const Dtype a, const Dtype b, Dtype* r) { CHECK_GE(n, 0); CHECK(r); CHECK_LE(a, b); @@ -330,15 +329,16 @@ void caffe_rng_uniform(const int n, Dtype* r, } template -void caffe_rng_uniform(const int n, float* r, - const float a, const float b); +void caffe_rng_uniform(const int n, const float a, const float b, + float* r); + template -void caffe_rng_uniform(const int n, double* r, - const double a, const double b); +void caffe_rng_uniform(const int n, const double a, const double b, + double* r); template -void caffe_rng_gaussian(const int n, Dtype* r, const Dtype a, - const Dtype sigma) { +void caffe_rng_gaussian(const int n, const Dtype a, + const Dtype sigma, Dtype* r) { CHECK_GE(n, 0); CHECK(r); CHECK_GT(sigma, 0); @@ -352,15 +352,15 @@ void caffe_rng_gaussian(const int n, Dtype* r, const Dtype a, } template -void caffe_rng_gaussian(const int n, float* r, const float a, - const float sigma); +void caffe_rng_gaussian(const int n, const float mu, + const float sigma, float* r); template -void caffe_rng_gaussian(const int n, double* r, const double a, - const double sigma); +void caffe_rng_gaussian(const int n, const double mu, + const double sigma, double* r); template -void caffe_rng_bernoulli(const int n, int* r, const Dtype p) { +void caffe_rng_bernoulli(const int n, const Dtype p, int* r) { CHECK_GE(n, 0); CHECK(r); CHECK_GE(p, 0); @@ -375,10 +375,10 @@ void caffe_rng_bernoulli(const int n, int* r, const Dtype p) { } template -void caffe_rng_bernoulli(const int n, int* r, const double p); +void caffe_rng_bernoulli(const int n, const double p, int* r); template -void caffe_rng_bernoulli(const int n, int* r, const float p); +void caffe_rng_bernoulli(const int n, const float p, int* r); template <> float caffe_cpu_dot(const int n, const float* x, const float* y) { From acfafb2f3940f05b95daeaaada7f13ecf66b4699 Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Tue, 8 Apr 2014 14:24:48 -0700 Subject: [PATCH 14/17] have rng_stream initialize RNG if not already initialized --- include/caffe/common.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/caffe/common.hpp b/include/caffe/common.hpp index 18290262e32..af7dcc408cf 100644 --- a/include/caffe/common.hpp +++ b/include/caffe/common.hpp @@ -96,6 +96,9 @@ class Caffe { // Getters for boost rng, curand, and cublas handles inline static const RNG& rng_stream() { + if (!Get().random_generator_) { + Get().random_generator_.reset(new RNG()); + } return *(Get().random_generator_); } inline static cublasHandle_t cublas_handle() { return Get().cublas_handle_; } From 2ebe94eca73ad45ba7dcc1014c9eb3d46b136501 Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Tue, 8 Apr 2014 16:24:50 -0700 Subject: [PATCH 15/17] add analogous caffe_gpu_rng_* functions for gaussian and uniform, and add test cases --- include/caffe/util/math_functions.hpp | 19 ++ include/caffe/vision_layers.hpp | 2 +- src/caffe/layers/dropout_layer.cpp | 4 +- src/caffe/layers/dropout_layer.cu | 15 +- src/caffe/layers/pooling_layer.cu | 8 +- .../test/test_random_number_generator.cpp | 248 ++++++++++++++++-- src/caffe/util/math_functions.cu | 44 ++++ 7 files changed, 310 insertions(+), 30 deletions(-) diff --git a/include/caffe/util/math_functions.hpp b/include/caffe/util/math_functions.hpp index 7129cf9c795..62290123b8b 100644 --- a/include/caffe/util/math_functions.hpp +++ b/include/caffe/util/math_functions.hpp @@ -111,13 +111,32 @@ Dtype caffe_nextafter(const Dtype b); template void caffe_rng_uniform(const int n, const Dtype a, const Dtype b, Dtype* r); +// caffe_gpu_rng_uniform with two arguments generates integers in the range +// [0, UINT_MAX]. +void caffe_gpu_rng_uniform(const int n, unsigned int* r); + +// caffe_gpu_rng_uniform with four arguments generates floats in the range +// (a, b] (strictly greater than a, less than or equal to b) due to the +// specification of curandGenerateUniform. With a = 0, b = 1, just calls +// curandGenerateUniform; with other limits will shift and scale the outputs +// appropriately after calling curandGenerateUniform. +template +void caffe_gpu_rng_uniform(const int n, const Dtype a, const Dtype b, Dtype* r); + template void caffe_rng_gaussian(const int n, const Dtype mu, const Dtype sigma, Dtype* r); +template +void caffe_gpu_rng_gaussian(const int n, const Dtype mu, const Dtype sigma, + Dtype* r); + template void caffe_rng_bernoulli(const int n, const Dtype p, int* r); +template +void caffe_gpu_rng_bernoulli(const int n, const Dtype p, int* r); + template void caffe_exp(const int n, const Dtype* a, Dtype* y); diff --git a/include/caffe/vision_layers.hpp b/include/caffe/vision_layers.hpp index 0bfc677c043..5af7b28df3c 100644 --- a/include/caffe/vision_layers.hpp +++ b/include/caffe/vision_layers.hpp @@ -607,7 +607,7 @@ class PoolingLayer : public Layer { int width_; int pooled_height_; int pooled_width_; - Blob rand_idx_; + Blob rand_idx_; }; template diff --git a/src/caffe/layers/dropout_layer.cpp b/src/caffe/layers/dropout_layer.cpp index e0068fedfb6..e28cab33fef 100644 --- a/src/caffe/layers/dropout_layer.cpp +++ b/src/caffe/layers/dropout_layer.cpp @@ -20,7 +20,7 @@ void DropoutLayer::SetUp(const vector*>& bottom, DCHECK(threshold_ > 0.); DCHECK(threshold_ < 1.); scale_ = 1. / (1. - threshold_); - uint_thres_ = (unsigned int)(UINT_MAX * threshold_); + uint_thres_ = static_cast(UINT_MAX * threshold_); } template @@ -37,7 +37,7 @@ Dtype DropoutLayer::Forward_cpu(const vector*>& bottom, top_data[i] = bottom_data[i] * mask[i] * scale_; } } else { - memcpy(top_data, bottom_data, bottom[0]->count() * sizeof(Dtype)); + caffe_copy(bottom[0]->count(), bottom_data, top_data); } return Dtype(0); } diff --git a/src/caffe/layers/dropout_layer.cu b/src/caffe/layers/dropout_layer.cu index 6d995d1f195..3c25d6a1292 100644 --- a/src/caffe/layers/dropout_layer.cu +++ b/src/caffe/layers/dropout_layer.cu @@ -8,6 +8,7 @@ #include "caffe/layer.hpp" #include "caffe/syncedmem.hpp" #include "caffe/vision_layers.hpp" +#include "caffe/util/math_functions.hpp" using std::max; @@ -30,17 +31,16 @@ Dtype DropoutLayer::Forward_gpu(const vector*>& bottom, Dtype* top_data = (*top)[0]->mutable_gpu_data(); const int count = bottom[0]->count(); if (Caffe::phase() == Caffe::TRAIN) { - CURAND_CHECK(curandGenerate(Caffe::curand_generator(), - (unsigned int*)(rand_vec_->mutable_gpu_data()), count)); + unsigned int* mask = + static_cast(rand_vec_->mutable_gpu_data()); + caffe_gpu_rng_uniform(count, mask); // set thresholds // NOLINT_NEXT_LINE(whitespace/operators) DropoutForward<<>>( - count, bottom_data, (unsigned int*)rand_vec_->gpu_data(), uint_thres_, - scale_, top_data); + count, bottom_data, mask, uint_thres_, scale_, top_data); CUDA_POST_KERNEL_CHECK; } else { - CUDA_CHECK(cudaMemcpy(top_data, bottom_data, - count * sizeof(Dtype), cudaMemcpyDeviceToDevice)); + caffe_gpu_copy(count, bottom_data, top_data); } return Dtype(0); } @@ -62,7 +62,8 @@ void DropoutLayer::Backward_gpu(const vector*>& top, if (propagate_down) { const Dtype* top_diff = top[0]->gpu_diff(); Dtype* bottom_diff = (*bottom)[0]->mutable_gpu_diff(); - const unsigned int* mask = (unsigned int*)rand_vec_->gpu_data(); + const unsigned int* mask = + static_cast(rand_vec_->gpu_data()); const int count = (*bottom)[0]->count(); // NOLINT_NEXT_LINE(whitespace/operators) DropoutBackward<<>>( diff --git a/src/caffe/layers/pooling_layer.cu b/src/caffe/layers/pooling_layer.cu index 74150df2493..95bfaefc951 100644 --- a/src/caffe/layers/pooling_layer.cu +++ b/src/caffe/layers/pooling_layer.cu @@ -73,7 +73,7 @@ __global__ void StoPoolForwardTrain(const int nthreads, const Dtype* bottom_data, const int num, const int channels, const int height, const int width, const int pooled_height, const int pooled_width, - const int kernel_size, const int stride, float* rand_idx, Dtype* top_data) { + const int kernel_size, const int stride, Dtype* rand_idx, Dtype* top_data) { CUDA_KERNEL_LOOP(index, nthreads) { int pw = index % pooled_width; int ph = (index / pooled_width) % pooled_height; @@ -163,8 +163,8 @@ Dtype PoolingLayer::Forward_gpu(const vector*>& bottom, case PoolingParameter_PoolMethod_STOCHASTIC: if (Caffe::phase() == Caffe::TRAIN) { // We need to create the random index as well. - CURAND_CHECK(curandGenerateUniform(Caffe::curand_generator(), - rand_idx_.mutable_gpu_data(), count)); + caffe_gpu_rng_uniform(count, Dtype(0), Dtype(1), + rand_idx_.mutable_gpu_data()); // NOLINT_NEXT_LINE(whitespace/operators) StoPoolForwardTrain<<>>( @@ -257,7 +257,7 @@ __global__ void AvePoolBackward(const int nthreads, const Dtype* top_diff, template __global__ void StoPoolBackward(const int nthreads, - const float* rand_idx, const Dtype* top_diff, + const Dtype* rand_idx, const Dtype* top_diff, const int num, const int channels, const int height, const int width, const int pooled_height, const int pooled_width, const int kernel_size, const int stride, Dtype* bottom_diff) { diff --git a/src/caffe/test/test_random_number_generator.cpp b/src/caffe/test/test_random_number_generator.cpp index d068d03cf35..d3a40a51e81 100644 --- a/src/caffe/test/test_random_number_generator.cpp +++ b/src/caffe/test/test_random_number_generator.cpp @@ -18,7 +18,9 @@ class RandomNumberGeneratorTest : public ::testing::Test { RandomNumberGeneratorTest() : sample_size_(10000), seed_(1701), + mean_bound_multiplier_(3.8), // ~99.99% confidence for test failure. data_(new SyncedMemory(sample_size_ * sizeof(Dtype))), + data_2_(new SyncedMemory(sample_size_ * sizeof(Dtype))), int_data_(new SyncedMemory(sample_size_ * sizeof(int))), int_data_2_(new SyncedMemory(sample_size_ * sizeof(int))) {} @@ -51,16 +53,26 @@ class RandomNumberGeneratorTest : public ::testing::Test { } Dtype mean_bound(const Dtype std, const int sample_size) { - return std / sqrt(static_cast(sample_size)); + return mean_bound_multiplier_ * std / sqrt(static_cast(sample_size)); } Dtype mean_bound(const Dtype std) { return mean_bound(std, sample_size_); } - void RngGaussianTest(const Dtype mu, const Dtype sigma, void* cpu_data) { + void RngGaussianFill(const Dtype mu, const Dtype sigma, void* cpu_data) { Dtype* rng_data = static_cast(cpu_data); caffe_rng_gaussian(sample_size_, mu, sigma, rng_data); + } + + void RngGaussianFillGPU(const Dtype mu, const Dtype sigma, void* gpu_data) { + Dtype* rng_data = static_cast(gpu_data); + caffe_gpu_rng_gaussian(sample_size_, mu, sigma, rng_data); + } + + void RngGaussianChecks(const Dtype mu, const Dtype sigma, + const void* cpu_data) { + const Dtype* rng_data = static_cast(cpu_data); const Dtype true_mean = mu; const Dtype true_std = sigma; // Check that sample mean roughly matches true mean. @@ -87,10 +99,28 @@ class RandomNumberGeneratorTest : public ::testing::Test { EXPECT_NEAR(bernoulli_p, sample_p_above_mean, bernoulli_bound); } - void RngUniformTest(const Dtype lower, const Dtype upper, void* cpu_data) { + void RngUniformFill(const Dtype lower, const Dtype upper, void* cpu_data) { CHECK_GE(upper, lower); Dtype* rng_data = static_cast(cpu_data); caffe_rng_uniform(sample_size_, lower, upper, rng_data); + } + + void RngUniformFillGPU(const Dtype lower, const Dtype upper, void* gpu_data) { + CHECK_GE(upper, lower); + Dtype* rng_data = static_cast(gpu_data); + caffe_gpu_rng_uniform(sample_size_, lower, upper, rng_data); + } + + // Fills with uniform integers in [0, UINT_MAX] using 2 argument form of + // caffe_gpu_rng_uniform. + void RngUniformIntFillGPU(void* gpu_data) { + unsigned int* rng_data = static_cast(gpu_data); + caffe_gpu_rng_uniform(sample_size_, rng_data); + } + + void RngUniformChecks(const Dtype lower, const Dtype upper, + const void* cpu_data) { + const Dtype* rng_data = static_cast(cpu_data); const Dtype true_mean = (lower + upper) / 2; const Dtype true_std = (upper - lower) / sqrt(12); // Check that sample mean roughly matches true mean. @@ -126,9 +156,13 @@ class RandomNumberGeneratorTest : public ::testing::Test { EXPECT_NEAR(bernoulli_p, sample_p_above_mean, bernoulli_bound); } - void RngBernoulliTest(const Dtype p, void* cpu_data) { + void RngBernoulliFill(const Dtype p, void* cpu_data) { int* rng_data = static_cast(cpu_data); caffe_rng_bernoulli(sample_size_, p, rng_data); + } + + void RngBernoulliChecks(const Dtype p, const void* cpu_data) { + const int* rng_data = static_cast(cpu_data); const Dtype true_mean = p; const Dtype true_std = sqrt(p * (1 - p)); const Dtype bound = this->mean_bound(true_std); @@ -139,10 +173,13 @@ class RandomNumberGeneratorTest : public ::testing::Test { int num_above_mean; int num_below_mean; + Dtype mean_bound_multiplier_; + size_t sample_size_; uint32_t seed_; shared_ptr data_; + shared_ptr data_2_; shared_ptr int_data_; shared_ptr int_data_2_; }; @@ -156,7 +193,8 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussian) { const TypeParam mu = 0; const TypeParam sigma = 1; void* gaussian_data = this->data_->mutable_cpu_data(); - this->RngGaussianTest(mu, sigma, gaussian_data); + this->RngGaussianFill(mu, sigma, gaussian_data); + this->RngGaussianChecks(mu, sigma, gaussian_data); } @@ -164,7 +202,8 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussian2) { const TypeParam mu = -2; const TypeParam sigma = 3; void* gaussian_data = this->data_->mutable_cpu_data(); - this->RngGaussianTest(mu, sigma, gaussian_data); + this->RngGaussianFill(mu, sigma, gaussian_data); + this->RngGaussianChecks(mu, sigma, gaussian_data); } @@ -172,7 +211,8 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngUniform) { const TypeParam lower = 0; const TypeParam upper = 1; void* uniform_data = this->data_->mutable_cpu_data(); - this->RngUniformTest(lower, upper, uniform_data); + this->RngUniformFill(lower, upper, uniform_data); + this->RngUniformChecks(lower, upper, uniform_data); } @@ -180,21 +220,78 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngUniform2) { const TypeParam lower = -7.3; const TypeParam upper = -2.3; void* uniform_data = this->data_->mutable_cpu_data(); - this->RngUniformTest(lower, upper, uniform_data); + this->RngUniformFill(lower, upper, uniform_data); + this->RngUniformChecks(lower, upper, uniform_data); } TYPED_TEST(RandomNumberGeneratorTest, TestRngBernoulli) { const TypeParam p = 0.3; void* bernoulli_data = this->int_data_->mutable_cpu_data(); - this->RngBernoulliTest(p, bernoulli_data); + this->RngBernoulliFill(p, bernoulli_data); + this->RngBernoulliChecks(p, bernoulli_data); } TYPED_TEST(RandomNumberGeneratorTest, TestRngBernoulli2) { const TypeParam p = 0.9; void* bernoulli_data = this->int_data_->mutable_cpu_data(); - this->RngBernoulliTest(p, bernoulli_data); + this->RngBernoulliFill(p, bernoulli_data); + this->RngBernoulliChecks(p, bernoulli_data); +} + + +TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussianPlusGaussian) { + const TypeParam sigma = 1; + + // Sample from -3 mean Gaussian. + const TypeParam mu_1 = -3; + TypeParam* gaussian_data_1 = + static_cast(this->data_->mutable_cpu_data()); + this->RngGaussianFill(mu_1, sigma, gaussian_data_1); + + // Sample from -2 mean Gaussian. + const TypeParam mu_2 = -2; + TypeParam* gaussian_data_2 = + static_cast(this->data_2_->mutable_cpu_data()); + this->RngGaussianFill(mu_2, sigma, gaussian_data_2); + + // Add Gaussians. + for (int i = 0; i < this->sample_size_; ++i) { + gaussian_data_1[i] += gaussian_data_2[i]; + } + + // Check that result is Gaussian with mean mu_1 + mu_2. + this->RngGaussianChecks(mu_1 + mu_2, sqrt(2 * pow(sigma, 2)), + gaussian_data_1); +} + + +TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformPlusUniform) { + const TypeParam sigma = 1; + + // Sample from Uniform on [-4, -2]. + const TypeParam lower_1 = -4; + const TypeParam upper_1 = -2; + TypeParam* uniform_data_1 = + static_cast(this->data_->mutable_cpu_data()); + this->RngUniformFill(lower_1, upper_1, uniform_data_1); + + // Sample from Uniform on [-3, -1]. + const TypeParam lower_2 = -3; + const TypeParam upper_2 = -1; + TypeParam* uniform_data_2 = + static_cast(this->data_2_->mutable_cpu_data()); + this->RngUniformFill(lower_2, upper_2, uniform_data_2); + + // Add Uniforms. + for (int i = 0; i < this->sample_size_; ++i) { + uniform_data_1[i] += uniform_data_2[i]; + } + + // Check that result does not violate properties of Uniform on [-7, -3]. + this->RngUniformChecks(lower_1 + lower_2, upper_1 + upper_2, + uniform_data_1); } @@ -204,13 +301,13 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussianTimesBernoulli) { const TypeParam sigma = 1; TypeParam* gaussian_data = static_cast(this->data_->mutable_cpu_data()); - this->RngGaussianTest(mu, sigma, gaussian_data); + this->RngGaussianFill(mu, sigma, gaussian_data); // Sample from Bernoulli with p = 0.3. const TypeParam bernoulli_p = 0.3; int* bernoulli_data = static_cast(this->int_data_->mutable_cpu_data()); - this->RngBernoulliTest(bernoulli_p, bernoulli_data); + this->RngBernoulliFill(bernoulli_p, bernoulli_data); // Multiply Gaussian by Bernoulli. for (int i = 0; i < this->sample_size_; ++i) { @@ -249,13 +346,13 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformTimesBernoulli) { const TypeParam upper = 1; TypeParam* uniform_data = static_cast(this->data_->mutable_cpu_data()); - this->RngUniformTest(lower, upper, uniform_data); + this->RngUniformFill(lower, upper, uniform_data); // Sample from Bernoulli with p = 0.3. const TypeParam bernoulli_p = 0.3; int* bernoulli_data = static_cast(this->int_data_->mutable_cpu_data()); - this->RngBernoulliTest(bernoulli_p, bernoulli_data); + this->RngBernoulliFill(bernoulli_p, bernoulli_data); // Multiply Uniform by Bernoulli. for (int i = 0; i < this->sample_size_; ++i) { @@ -293,13 +390,13 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngBernoulliTimesBernoulli) { const TypeParam p_a = 0.5; int* bernoulli_data_a = static_cast(this->int_data_->mutable_cpu_data()); - this->RngBernoulliTest(p_a, bernoulli_data_a); + this->RngBernoulliFill(p_a, bernoulli_data_a); // Sample from Bernoulli with p = 0.3. const TypeParam p_b = 0.3; int* bernoulli_data_b = static_cast(this->int_data_2_->mutable_cpu_data()); - this->RngBernoulliTest(p_b, bernoulli_data_b); + this->RngBernoulliFill(p_b, bernoulli_data_b); // Multiply Bernoullis. for (int i = 0; i < this->sample_size_; ++i) { @@ -322,4 +419,123 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngBernoulliTimesBernoulli) { } +TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussianGPU) { + const TypeParam mu = 0; + const TypeParam sigma = 1; + void* gaussian_gpu_data = this->data_->mutable_gpu_data(); + this->RngGaussianFillGPU(mu, sigma, gaussian_gpu_data); + const void* gaussian_data = this->data_->cpu_data(); + this->RngGaussianChecks(mu, sigma, gaussian_data); +} + + +TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussian2GPU) { + const TypeParam mu = -2; + const TypeParam sigma = 3; + void* gaussian_gpu_data = this->data_->mutable_gpu_data(); + this->RngGaussianFillGPU(mu, sigma, gaussian_gpu_data); + const void* gaussian_data = this->data_->cpu_data(); + this->RngGaussianChecks(mu, sigma, gaussian_data); +} + + +TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformGPU) { + const TypeParam lower = 0; + const TypeParam upper = 1; + void* uniform_gpu_data = this->data_->mutable_gpu_data(); + this->RngUniformFillGPU(lower, upper, uniform_gpu_data); + const void* uniform_data = this->data_->cpu_data(); + this->RngUniformChecks(lower, upper, uniform_data); +} + + +TYPED_TEST(RandomNumberGeneratorTest, TestRngUniform2GPU) { + const TypeParam lower = -7.3; + const TypeParam upper = -2.3; + void* uniform_gpu_data = this->data_->mutable_gpu_data(); + this->RngUniformFillGPU(lower, upper, uniform_gpu_data); + const void* uniform_data = this->data_->cpu_data(); + this->RngUniformChecks(lower, upper, uniform_data); +} + + +TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformIntGPU) { + unsigned int* uniform_uint_gpu_data = + static_cast(this->int_data_->mutable_gpu_data()); + this->RngUniformIntFillGPU(uniform_uint_gpu_data); + const unsigned int* uniform_uint_data = + static_cast(this->int_data_->cpu_data()); + TypeParam* uniform_data = + static_cast(this->data_->mutable_cpu_data()); + for (int i = 0; i < this->sample_size_; ++i) { + uniform_data[i] = static_cast(uniform_uint_data[i]); + } + const TypeParam lower = 0; + const TypeParam upper = UINT_MAX; + this->RngUniformChecks(lower, upper, uniform_data); +} + + +TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussianPlusGaussianGPU) { + const TypeParam sigma = 1; + + // Sample from -3 mean Gaussian. + const TypeParam mu_1 = -3; + TypeParam* gaussian_gpu_data_1 = + static_cast(this->data_->mutable_gpu_data()); + this->RngGaussianFillGPU(mu_1, sigma, gaussian_gpu_data_1); + + // Sample from -2 mean Gaussian. + const TypeParam mu_2 = -2; + TypeParam* gaussian_gpu_data_2 = + static_cast(this->data_2_->mutable_gpu_data()); + this->RngGaussianFillGPU(mu_2, sigma, gaussian_gpu_data_2); + + // Add Gaussians. + TypeParam* gaussian_data_1 = + static_cast(this->data_->mutable_cpu_data()); + const TypeParam* gaussian_data_2 = + static_cast(this->data_2_->cpu_data()); + for (int i = 0; i < this->sample_size_; ++i) { + gaussian_data_1[i] += gaussian_data_2[i]; + } + + // Check that result is Gaussian with mean mu_1 + mu_2. + this->RngGaussianChecks(mu_1 + mu_2, sqrt(2 * pow(sigma, 2)), + gaussian_data_1); +} + + +TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformPlusUniformGPU) { + const TypeParam sigma = 1; + + // Sample from Uniform on [-4, -2]. + const TypeParam lower_1 = -4; + const TypeParam upper_1 = -2; + TypeParam* uniform_gpu_data_1 = + static_cast(this->data_->mutable_gpu_data()); + this->RngUniformFillGPU(lower_1, upper_1, uniform_gpu_data_1); + + // Sample from Uniform on [-3, -1]. + const TypeParam lower_2 = -3; + const TypeParam upper_2 = -1; + TypeParam* uniform_gpu_data_2 = + static_cast(this->data_2_->mutable_gpu_data()); + this->RngUniformFillGPU(lower_2, upper_2, uniform_gpu_data_2); + + // Add Uniforms. + TypeParam* uniform_data_1 = + static_cast(this->data_->mutable_cpu_data()); + const TypeParam* uniform_data_2 = + static_cast(this->data_2_->cpu_data()); + for (int i = 0; i < this->sample_size_; ++i) { + uniform_data_1[i] += uniform_data_2[i]; + } + + // Check that result does not violate properties of Uniform on [-7, -3]. + this->RngUniformChecks(lower_1 + lower_2, upper_1 + upper_2, + uniform_data_1); +} + + } // namespace caffe diff --git a/src/caffe/util/math_functions.cu b/src/caffe/util/math_functions.cu index 40c4f507ef4..184613c0a84 100644 --- a/src/caffe/util/math_functions.cu +++ b/src/caffe/util/math_functions.cu @@ -185,4 +185,48 @@ uint32_t caffe_gpu_hamming_distance(const int n, const double* x, (uint32_t) 0, thrust::plus()); } +void caffe_gpu_rng_uniform(const int n, unsigned int* r) { + CURAND_CHECK(curandGenerate(Caffe::curand_generator(), r, n)); +} + +template <> +void caffe_gpu_rng_uniform(const int n, const float a, const float b, + float* r) { + CURAND_CHECK(curandGenerateUniform(Caffe::curand_generator(), r, n)); + const float range = b - a; + if (range != static_cast(1)) { + caffe_gpu_scal(n, range, r); + } + if (a != static_cast(0)) { + caffe_gpu_add_scalar(n, a, r); + } +} + +template <> +void caffe_gpu_rng_uniform(const int n, const double a, const double b, + double* r) { + CURAND_CHECK(curandGenerateUniformDouble(Caffe::curand_generator(), r, n)); + const double range = b - a; + if (range != static_cast(1)) { + caffe_gpu_scal(n, range, r); + } + if (a != static_cast(0)) { + caffe_gpu_add_scalar(n, a, r); + } +} + +template <> +void caffe_gpu_rng_gaussian(const int n, const float mu, const float sigma, + float* r) { + CURAND_CHECK( + curandGenerateNormal(Caffe::curand_generator(), r, n, mu, sigma)); +} + +template <> +void caffe_gpu_rng_gaussian(const int n, const double mu, const double sigma, + double* r) { + CURAND_CHECK( + curandGenerateNormalDouble(Caffe::curand_generator(), r, n, mu, sigma)); +} + } // namespace caffe From 61c5653d2290eff313fdcb3656f9d9ad7cb17983 Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Tue, 8 Apr 2014 17:15:28 -0700 Subject: [PATCH 16/17] change *Plus* tests to *Times* tests because the Plus tests don't actually check for uncorrelated RNG results --- .../test/test_random_number_generator.cpp | 96 +++++++++---------- 1 file changed, 47 insertions(+), 49 deletions(-) diff --git a/src/caffe/test/test_random_number_generator.cpp b/src/caffe/test/test_random_number_generator.cpp index d3a40a51e81..19734fe3b1d 100644 --- a/src/caffe/test/test_random_number_generator.cpp +++ b/src/caffe/test/test_random_number_generator.cpp @@ -241,57 +241,56 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngBernoulli2) { } -TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussianPlusGaussian) { +TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussianTimesGaussian) { + const TypeParam mu = 0; const TypeParam sigma = 1; - // Sample from -3 mean Gaussian. - const TypeParam mu_1 = -3; + // Sample from 0 mean Gaussian. TypeParam* gaussian_data_1 = static_cast(this->data_->mutable_cpu_data()); - this->RngGaussianFill(mu_1, sigma, gaussian_data_1); + this->RngGaussianFill(mu, sigma, gaussian_data_1); - // Sample from -2 mean Gaussian. - const TypeParam mu_2 = -2; + // Sample from 0 mean Gaussian again. TypeParam* gaussian_data_2 = static_cast(this->data_2_->mutable_cpu_data()); - this->RngGaussianFill(mu_2, sigma, gaussian_data_2); + this->RngGaussianFill(mu, sigma, gaussian_data_2); - // Add Gaussians. + // Multiply Gaussians. for (int i = 0; i < this->sample_size_; ++i) { - gaussian_data_1[i] += gaussian_data_2[i]; + gaussian_data_1[i] *= gaussian_data_2[i]; } - // Check that result is Gaussian with mean mu_1 + mu_2. - this->RngGaussianChecks(mu_1 + mu_2, sqrt(2 * pow(sigma, 2)), - gaussian_data_1); + // Check that result has mean 0. + TypeParam mu_product = pow(mu, 2); + TypeParam sigma_product = sqrt(pow(sigma, 2) / 2); + this->RngGaussianChecks(mu_product, sigma_product, gaussian_data_1); } -TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformPlusUniform) { - const TypeParam sigma = 1; - - // Sample from Uniform on [-4, -2]. - const TypeParam lower_1 = -4; - const TypeParam upper_1 = -2; +TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformTimesUniform) { + // Sample from Uniform on [-2, 2]. + const TypeParam lower_1 = -2; + const TypeParam upper_1 = -lower_1; TypeParam* uniform_data_1 = static_cast(this->data_->mutable_cpu_data()); this->RngUniformFill(lower_1, upper_1, uniform_data_1); - // Sample from Uniform on [-3, -1]. + // Sample from Uniform on [-3, 3]. const TypeParam lower_2 = -3; - const TypeParam upper_2 = -1; + const TypeParam upper_2 = -lower_2; TypeParam* uniform_data_2 = static_cast(this->data_2_->mutable_cpu_data()); this->RngUniformFill(lower_2, upper_2, uniform_data_2); - // Add Uniforms. + // Multiply Uniforms. for (int i = 0; i < this->sample_size_; ++i) { - uniform_data_1[i] += uniform_data_2[i]; + uniform_data_1[i] *= uniform_data_2[i]; } - // Check that result does not violate properties of Uniform on [-7, -3]. - this->RngUniformChecks(lower_1 + lower_2, upper_1 + upper_2, - uniform_data_1); + // Check that result does not violate properties of Uniform on [-6, 6]. + const TypeParam lower_prod = lower_1 * upper_2; + const TypeParam upper_prod = -lower_prod; + this->RngUniformChecks(lower_prod, upper_prod, uniform_data_1); } @@ -476,65 +475,64 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformIntGPU) { } -TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussianPlusGaussianGPU) { +TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussianTimesGaussianGPU) { + const TypeParam mu = 0; const TypeParam sigma = 1; - // Sample from -3 mean Gaussian. - const TypeParam mu_1 = -3; + // Sample from 0 mean Gaussian. TypeParam* gaussian_gpu_data_1 = static_cast(this->data_->mutable_gpu_data()); - this->RngGaussianFillGPU(mu_1, sigma, gaussian_gpu_data_1); + this->RngGaussianFillGPU(mu, sigma, gaussian_gpu_data_1); - // Sample from -2 mean Gaussian. - const TypeParam mu_2 = -2; + // Sample from 0 mean Gaussian again. TypeParam* gaussian_gpu_data_2 = static_cast(this->data_2_->mutable_gpu_data()); - this->RngGaussianFillGPU(mu_2, sigma, gaussian_gpu_data_2); + this->RngGaussianFillGPU(mu, sigma, gaussian_gpu_data_2); - // Add Gaussians. + // Multiply Gaussians. TypeParam* gaussian_data_1 = static_cast(this->data_->mutable_cpu_data()); const TypeParam* gaussian_data_2 = static_cast(this->data_2_->cpu_data()); for (int i = 0; i < this->sample_size_; ++i) { - gaussian_data_1[i] += gaussian_data_2[i]; + gaussian_data_1[i] *= gaussian_data_2[i]; } - // Check that result is Gaussian with mean mu_1 + mu_2. - this->RngGaussianChecks(mu_1 + mu_2, sqrt(2 * pow(sigma, 2)), - gaussian_data_1); + // Check that result has mean 0. + TypeParam mu_product = pow(mu, 2); + TypeParam sigma_product = sqrt(pow(sigma, 2) / 2); + this->RngGaussianChecks(mu_product, sigma_product, gaussian_data_1); } -TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformPlusUniformGPU) { - const TypeParam sigma = 1; - - // Sample from Uniform on [-4, -2]. - const TypeParam lower_1 = -4; - const TypeParam upper_1 = -2; +TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformTimesUniformGPU) { + // Sample from Uniform on [-2, 2]. + const TypeParam lower_1 = -2; + const TypeParam upper_1 = -lower_1; TypeParam* uniform_gpu_data_1 = static_cast(this->data_->mutable_gpu_data()); this->RngUniformFillGPU(lower_1, upper_1, uniform_gpu_data_1); - // Sample from Uniform on [-3, -1]. + // Sample from Uniform on [-3, 3]. const TypeParam lower_2 = -3; - const TypeParam upper_2 = -1; + const TypeParam upper_2 = -lower_2; TypeParam* uniform_gpu_data_2 = static_cast(this->data_2_->mutable_gpu_data()); this->RngUniformFillGPU(lower_2, upper_2, uniform_gpu_data_2); - // Add Uniforms. + // Multiply Uniforms. TypeParam* uniform_data_1 = static_cast(this->data_->mutable_cpu_data()); const TypeParam* uniform_data_2 = static_cast(this->data_2_->cpu_data()); for (int i = 0; i < this->sample_size_; ++i) { - uniform_data_1[i] += uniform_data_2[i]; + uniform_data_1[i] *= uniform_data_2[i]; } // Check that result does not violate properties of Uniform on [-7, -3]. - this->RngUniformChecks(lower_1 + lower_2, upper_1 + upper_2, - uniform_data_1); + const TypeParam lower_prod = lower_1 * upper_2; + const TypeParam upper_prod = -lower_prod; + this->RngUniformChecks(lower_prod, upper_prod, uniform_data_1); } From 1f5b9a52c600594026cdd455e53d0a7fa5a2aa05 Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Tue, 8 Apr 2014 19:45:31 -0700 Subject: [PATCH 17/17] more rng test cleanup --- .../test/test_random_number_generator.cpp | 90 ++++++++----------- 1 file changed, 35 insertions(+), 55 deletions(-) diff --git a/src/caffe/test/test_random_number_generator.cpp b/src/caffe/test/test_random_number_generator.cpp index 19734fe3b1d..2bb94a7d517 100644 --- a/src/caffe/test/test_random_number_generator.cpp +++ b/src/caffe/test/test_random_number_generator.cpp @@ -18,7 +18,7 @@ class RandomNumberGeneratorTest : public ::testing::Test { RandomNumberGeneratorTest() : sample_size_(10000), seed_(1701), - mean_bound_multiplier_(3.8), // ~99.99% confidence for test failure. + mean_bound_multiplier_(3.8), // ~99.99% confidence for test failure. data_(new SyncedMemory(sample_size_ * sizeof(Dtype))), data_2_(new SyncedMemory(sample_size_ * sizeof(Dtype))), int_data_(new SyncedMemory(sample_size_ * sizeof(int))), @@ -71,7 +71,7 @@ class RandomNumberGeneratorTest : public ::testing::Test { } void RngGaussianChecks(const Dtype mu, const Dtype sigma, - const void* cpu_data) { + const void* cpu_data, const Dtype sparse_p = 0) { const Dtype* rng_data = static_cast(cpu_data); const Dtype true_mean = mu; const Dtype true_std = sigma; @@ -83,17 +83,26 @@ class RandomNumberGeneratorTest : public ::testing::Test { // Check that roughly half the samples are above the true mean. int num_above_mean = 0; int num_below_mean = 0; + int num_mean = 0; + int num_nan = 0; for (int i = 0; i < sample_size_; ++i) { if (rng_data[i] > true_mean) { ++num_above_mean; } else if (rng_data[i] < true_mean) { ++num_below_mean; + } else if (rng_data[i] == true_mean) { + ++num_mean; + } else { + ++num_nan; } } - EXPECT_EQ(sample_size_, num_above_mean + num_below_mean); + EXPECT_EQ(0, num_nan); + if (sparse_p == Dtype(0)) { + EXPECT_EQ(0, num_mean); + } const Dtype sample_p_above_mean = static_cast(num_above_mean) / sample_size_; - const Dtype bernoulli_p = 0.5; + const Dtype bernoulli_p = (1 - sparse_p) * 0.5; const Dtype bernoulli_std = sqrt(bernoulli_p * (1 - bernoulli_p)); const Dtype bernoulli_bound = this->mean_bound(true_std); EXPECT_NEAR(bernoulli_p, sample_p_above_mean, bernoulli_bound); @@ -119,7 +128,7 @@ class RandomNumberGeneratorTest : public ::testing::Test { } void RngUniformChecks(const Dtype lower, const Dtype upper, - const void* cpu_data) { + const void* cpu_data, const Dtype sparse_p = 0) { const Dtype* rng_data = static_cast(cpu_data); const Dtype true_mean = (lower + upper) / 2; const Dtype true_std = (upper - lower) / sqrt(12); @@ -131,6 +140,8 @@ class RandomNumberGeneratorTest : public ::testing::Test { // above upper or below lower. int num_above_mean = 0; int num_below_mean = 0; + int num_mean = 0; + int num_nan = 0; int num_above_upper = 0; int num_below_lower = 0; for (int i = 0; i < sample_size_; ++i) { @@ -138,6 +149,10 @@ class RandomNumberGeneratorTest : public ::testing::Test { ++num_above_mean; } else if (rng_data[i] < true_mean) { ++num_below_mean; + } else if (rng_data[i] == true_mean) { + ++num_mean; + } else { + ++num_nan; } if (rng_data[i] > upper) { ++num_above_upper; @@ -145,12 +160,15 @@ class RandomNumberGeneratorTest : public ::testing::Test { ++num_below_lower; } } + EXPECT_EQ(0, num_nan); EXPECT_EQ(0, num_above_upper); EXPECT_EQ(0, num_below_lower); - EXPECT_EQ(sample_size_, num_above_mean + num_below_mean); + if (sparse_p == Dtype(0)) { + EXPECT_EQ(0, num_mean); + } const Dtype sample_p_above_mean = static_cast(num_above_mean) / sample_size_; - const Dtype bernoulli_p = 0.5; + const Dtype bernoulli_p = (1 - sparse_p) * 0.5; const Dtype bernoulli_std = sqrt(bernoulli_p * (1 - bernoulli_p)); const Dtype bernoulli_bound = this->mean_bound(true_std); EXPECT_NEAR(bernoulli_p, sample_p_above_mean, bernoulli_bound); @@ -287,7 +305,8 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformTimesUniform) { uniform_data_1[i] *= uniform_data_2[i]; } - // Check that result does not violate properties of Uniform on [-6, 6]. + // Check that result does not violate checked properties of Uniform on [-6, 6] + // (though it is not actually uniformly distributed). const TypeParam lower_prod = lower_1 * upper_2; const TypeParam upper_prod = -lower_prod; this->RngUniformChecks(lower_prod, upper_prod, uniform_data_1); @@ -312,30 +331,10 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussianTimesBernoulli) { for (int i = 0; i < this->sample_size_; ++i) { gaussian_data[i] *= bernoulli_data[i]; } - int num_pos = 0; - int num_neg = 0; - for (int i = 0; i < this->sample_size_; ++i) { - if (gaussian_data[i] == TypeParam(0)) { - EXPECT_EQ(TypeParam(0), bernoulli_data[i]); - } else { - EXPECT_EQ(TypeParam(1), bernoulli_data[i]); - if (gaussian_data[i] > TypeParam(0)) { - ++num_pos; - } else if (gaussian_data[i] < TypeParam(0)) { - ++num_neg; - } - } - } - // Check that Gaussian still has roughly half positives and half negatives - // (with bound computed from a Bernoulli with p = 0.5). - const int num_non_zero = num_pos + num_neg; - const TypeParam sample_p = num_pos / static_cast(num_non_zero); - const TypeParam p = 0.5; - const TypeParam true_mean = p; - const TypeParam true_std = sqrt(p * (1 - p)); - const TypeParam bound = this->mean_bound(true_std, num_non_zero); - EXPECT_NEAR(true_mean, sample_p, bound); + // Check that result does not violate checked properties of sparsified + // Gaussian (though it is not actually a Gaussian). + this->RngGaussianChecks(mu, sigma, gaussian_data, 1 - bernoulli_p); } @@ -357,30 +356,10 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformTimesBernoulli) { for (int i = 0; i < this->sample_size_; ++i) { uniform_data[i] *= bernoulli_data[i]; } - int num_pos = 0; - int num_neg = 0; - for (int i = 0; i < this->sample_size_; ++i) { - if (uniform_data[i] == TypeParam(0)) { - EXPECT_EQ(TypeParam(0), bernoulli_data[i]); - } else { - EXPECT_EQ(TypeParam(1), bernoulli_data[i]); - if (uniform_data[i] > TypeParam(0)) { - ++num_pos; - } else if (uniform_data[i] < TypeParam(0)) { - ++num_neg; - } - } - } - // Check that Uniform still has roughly half positives and half negatives - // (with bound computed from a Bernoulli with p = 0.5). - const int num_non_zero = num_pos + num_neg; - const TypeParam sample_p = num_pos / static_cast(num_non_zero); - const TypeParam p = 0.5; - const TypeParam true_mean = p; - const TypeParam true_std = sqrt(p * (1 - p)); - const TypeParam bound = this->mean_bound(true_std, num_non_zero); - EXPECT_NEAR(true_mean, sample_p, bound); + // Check that result does not violate checked properties of sparsified + // Uniform on [-1, 1] (though it is not actually uniformly distributed). + this->RngUniformChecks(lower, upper, uniform_data, 1 - bernoulli_p); } @@ -498,7 +477,8 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussianTimesGaussianGPU) { gaussian_data_1[i] *= gaussian_data_2[i]; } - // Check that result has mean 0. + // Check that result does not violate checked properties of Gaussian + // (though it is not actually a Gaussian). TypeParam mu_product = pow(mu, 2); TypeParam sigma_product = sqrt(pow(sigma, 2) / 2); this->RngGaussianChecks(mu_product, sigma_product, gaussian_data_1);