merge with lasted develop branch. Optimizer lib2

paddle/CMakeLists.txt

@@ -8,6 +8,7 @@ add_subdirectory(gserver)
 add_subdirectory(pserver)
 add_subdirectory(trainer)
 add_subdirectory(scripts)
+add_subdirectory(optimizer)
 
 # Do not build go directory until go cmake is working smoothly.
 # if(CMAKE_Go_COMPILER)

paddle/optimizer/CMakeLists.txt

@@ -0,0 +1,17 @@
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
+
+set(OPITMIZER_SRCS
+    adadelta_optimizer.cc
+    adagrad_optimizer.cc
+    adam_optimizer.cc
+    optimizer.cc
+    parameter_optimizer.cc
+    sgd_optmizer.cc
+  )
+
+add_library(optimizer STATIC ${OPITMIZER_SRCS})
+add_dependencies(optimizer gen_proto_cpp)
+
+add_simple_unittest(tensor_test)
+add_simple_unittest(parameter_optimizer_test)
+add_dependencies(parameter_optimizer_test optimizer)

paddle/optimizer/Tensor.h

@@ -0,0 +1,52 @@
+#pragma once
+/**
+ * @brief tensor used by optimizer
+ */
+
+#include <string.h>
+#include <memory>
+#include "paddle/utils/Common.h"
+#include "paddle/utils/Logging.h"
+
+namespace paddle {
+namespace optimizer {
+
+template <class T>
+class TensorT {
+public:
+  TensorT(size_t size) : height_(1), width_(size) { data_ = new T[size]; }
+  TensorT(T* data, size_t size) : height_(1), width_(size), data_(data) {}
+  TensorT(T* data, size_t h, size_t w) : height_(h), width_(w), data_(data_) {}
+  TensorT(const TensorT& t)
+      : TensorT(1, t.size(), 0, t.get_buffer(), false, false) {}
+  ~TensorT() {
+    if (data_) delete data_;
+  }
+
+  TensorT& operator=(const TensorT& t) {
+    this->width_ = t.size();
+    this->data_ = t.get_buffer();
+  }
+  T* get_buffer() { return this->data_; }
+  T& operator[](const size_t idx) {
+    CHECK(idx >= 0 && idx < this->width_) << "out of index range";
+    return data_[idx];
+  }
+  T& operator[](const size_t idx) const {
+    CHECK(idx >= 0 && idx < this->width_) << "out of index range";
+    return data_[idx];
+  }
+  // TODO: replace with tensorshape
+  size_t size() const { return this->width_ * this->height_; }
+
+protected:
+  size_t height_;
+  size_t width_;
+  T* data_;
+};
+
+// TODO(zhihong): design problem of dynamic datatype, need to fix it
+typedef TensorT<real> Tensor;
+
+}  // namespace optimizer
+}  // namespace paddle

paddle/optimizer/Tensor_test.cpp

@@ -0,0 +1,20 @@
+#include <iostream>
+#include "gtest/gtest.h"
+#include "tensor.h"
+
+using namespace paddle;
+using namespace paddle::optimizer;
+
+TEST(Tensor, indexer) {
+  Tensor t(3);
+  for (auto i = 0; i < t.size(); ++i) {
+    t[i] = i;
+  }
+  ASSERT_EQ(t[2], 2);
+  ASSERT_EQ(t[1], 1);
+}
+
+int main(int argc, char** argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

paddle/optimizer/adadelta_optimizer.cc

@@ -0,0 +1,36 @@
+#include "adadelta_optimizer.h"
+#include <algorithm>
+#include <cmath>
+
+namespace paddle {
+namespace optimizer {
+
+void AdadeltaOptimizer::set_weight(Tensor* p) {
+  parameter_ = p;
+  size_t size = p->size();
+  accum_gradient_ = new Tensor(size);
+  accum_delta_ = new Tensor(size);
+  update_delta_ = new Tensor(size);
+}
+
+void AdadeltaOptimizer::Update(const Tensor* gradient) {
+  num_sample_passed_ += 1;
+  double learning_rate = lr_policy_->LearningRate(num_sample_passed_);
+  Tensor& param = *parameter_;
+  const Tensor& grad = *gradient;
+  Tensor& accum_g = *accum_gradient_;
+  Tensor& accum_d = *accum_delta_;
+  Tensor& update_d = *update_delta_;
+  for (size_t i = 0; i < param.size(); ++i) {
+    accum_g[i] = rho_ * accum_g[i] + (1.0 - rho_) * grad[i] * grad[i];
+
+    update_d[i] = std::sqrt(accum_d[i] + epsilon_) /
+                  std::sqrt(accum_g[i] + epsilon_) * grad[i];
+
+    accum_d[i] = rho_ * accum_d[i] + (1.0 - rho_) * update_d[i] * update_d[i];
+
+    param[i] -= learning_rate * update_d[i] + learning_rate * decay_ * param[i];
+  }
+}
+}  // namespace optimizer
+}  // namespace paddle

paddle/optimizer/adadelta_optimizer.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include "parameter_optimizer.h"
+
+namespace paddle {
+namespace optimizer {
+
+class AdadeltaOptimizer : public ParameterOptimizer {
+public:
+  AdadeltaOptimizer(double rho, double epsilon, double decay, LrPolicy *lr)
+      : ParameterOptimizer(lr), rho_(rho), epsilon_(epsilon), decay_(decay) {}
+  ~AdadeltaOptimizer() {
+    if (accum_gradient_) delete accum_gradient_;
+    if (accum_delta_) delete accum_delta_;
+    if (update_delta_) delete update_delta_;
+  }
+  void Update(const Tensor *gradient);
+  void set_weight(Tensor *p);
+  real *get_weight() const;
+
+private:
+  Tensor *accum_gradient_;
+  Tensor *accum_delta_;
+  Tensor *update_delta_;
+
+  double rho_;
+  double epsilon_;
+  double decay_;
+};
+
+}  // namespace optimizer
+}  // namespace paddle

paddle/optimizer/adagrad_optimizer.cc

@@ -0,0 +1,28 @@
+#include <cmath>
+
+#include "adagrad_optimizer.h"
+
+namespace paddle {
+namespace optimizer {
+
+void AdagradOptimizer::set_weight(Tensor* p) {
+  parameter_ = p;
+  size_t size = p->size();
+  accum_gradient_ = new Tensor(size);
+}
+
+void AdagradOptimizer::Update(const Tensor* gradient) {
+  num_sample_passed_ += 1;
+  double learning_rate = lr_policy_->LearningRate(num_sample_passed_);
+  Tensor& param = *parameter_;
+  Tensor& accum_g = *accum_gradient_;
+  const Tensor& grad = *gradient;
+  for (size_t i = 0; i < param.size(); ++i) {
+    accum_g[i] += grad[i] * grad[i];
+    param[i] += learning_rate * grad[i] / std::sqrt(accum_g[i] + epsilon_) +
+                learning_rate * decay_ * param[i];
+  }
+}
+
+}  // namespace optimizer
+}  // namespace paddle

paddle/optimizer/adagrad_optimizer.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "parameter_optimizer.h"
+
+namespace paddle {
+namespace optimizer {
+
+class AdagradOptimizer : public ParameterOptimizer {
+public:
+  AdagradOptimizer(double epsilon, double decay, LrPolicy *lr)
+      : ParameterOptimizer(lr), epsilon_(epsilon), decay_(decay) {}
+  ~AdagradOptimizer() {
+    if (accum_gradient_) delete accum_gradient_;
+  }
+  void Update(const Tensor *gradient);
+  void set_weight(Tensor *p);
+  real *get_weight() const;
+
+private:
+  Tensor *accum_gradient_;
+  double epsilon_;
+  double decay_;
+};
+
+}  // namespace optimizer
+}  // namespace paddle

paddle/optimizer/adam_optimizer.cc

@@ -0,0 +1,32 @@
+#include "adam_optimizer.h"
+#include <cmath>
+
+namespace paddle {
+namespace optimizer {
+
+void AdamOptimizer::set_weight(Tensor *p) {
+  parameter_ = p;
+  size_t size = p->size();
+  momentums_ = new Tensor(size);
+  velocitys_ = new Tensor(size);
+}
+
+void AdamOptimizer::Update(const Tensor *gradient) {
+  num_sample_passed_ += 1;
+  double learning_rate = lr_policy_->LearningRate(num_sample_passed_);
+  double coef1 = 1.0 - std::pow(beta_1_, num_sample_passed_);
+  double coef2 = 1.0 - std::pow(beta_2_, num_sample_passed_);
+  learning_rate *= std::sqrt(coef2) / coef1;
+  Tensor &param = *parameter_;
+  const Tensor &grad = *gradient;
+  Tensor &m = *momentums_;
+  Tensor &v = *velocitys_;
+  for (size_t i = 0; i < param.size(); ++i) {
+    m[i] = beta_1_ * m[i] + (1.0 - beta_1_) * grad[i];
+    v[i] = beta_2_ * v[i] + (1.0 - beta_2_) * grad[i] * grad[i];
+    param[i] -=
+        learning_rate * (m[i] / std::sqrt(v[i] + epsilon_) + decay_ * param[i]);
+  }
+}
+}  // namespace optimizer
+}  // namespace paddle

paddle/optimizer/adam_optimizer.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include "parameter_optimizer.h"
+
+namespace paddle {
+namespace optimizer {
+
+class AdamOptimizer : public ParameterOptimizer {
+public:
+  AdamOptimizer(
+      double beta_1, double beta_2, double epsilon, double decay, LrPolicy *lr)
+      : ParameterOptimizer(lr),
+        beta_1_(beta_1),
+        beta_2_(beta_2),
+        epsilon_(epsilon),
+        decay_(decay) {}
+  ~AdamOptimizer() {
+    if (momentums_) delete momentums_;
+    if (velocitys_) delete velocitys_;
+  }
+  void Update(const Tensor *gradient);
+  void set_weight(Tensor *p);
+  real *get_weight() const;
+
+private:
+  Tensor *momentums_;
+  Tensor *velocitys_;
+  double beta_1_;
+  double beta_2_;
+  double epsilon_;
+  double decay_;
+};
+
+}  // namespace optimizer
+}  // namespace paddle

paddle/optimizer/lr_policy.h

@@ -0,0 +1,42 @@
+#pragma once
+
+#include <algorithm>
+#include "OptimizerConfig.pb.h"
+
+namespace paddle {
+namespace optimizer {
+
+class LrPolicy {
+public:
+  virtual ~LrPolicy() {}
+  virtual double LearningRate(const uint64_t num_sample_passed) = 0;
+};
+
+// constant learning rate policy
+class ConstLr final : public LrPolicy {
+public:
+  ConstLr(double lr) : learning_rate(lr){};
+  double LearningRate(const uint64_t num_sample_passed) {
+    return learning_rate;
+  }
+
+protected:
+  double learning_rate;
+};
+
+class LinearLr final : public LrPolicy {
+public:
+  LinearLr(double lr, double lr_decay_a, double lr_decay_b)
+      : learning_rate(lr), lr_decay_a(lr_decay_a), lr_decay_b(lr_decay_b) {}
+  double LearningRate(const uint64_t num_sample_passed) {
+    return std::max(learning_rate - lr_decay_a * num_sample_passed, lr_decay_b);
+  }
+
+private:
+  double learning_rate;
+  double lr_decay_a;
+  double lr_decay_b;
+};
+
+}  // namespace optimizer
+}  // namespace paddle