merge with lasted develop branch. Optimizer lib2

paddle/CMakeLists.txt

@@ -8,6 +8,7 @@ add_subdirectory(gserver)
 add_subdirectory(pserver)
 add_subdirectory(trainer)
 add_subdirectory(scripts)
+add_subdirectory(optimizer)
 
 # Do not build go directory until go cmake is working smoothly.
 # if(CMAKE_Go_COMPILER)

paddle/optimizer/CMakeLists.txt

@@ -0,0 +1,28 @@
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
+
+set(OPITMIZER_SRCS
+    adadelta_optimizer.cc
+    adagrad_optimizer.cc
+    adam_optimizer.cc
+    optimizer.cc
+    parameter_optimizer.cc
+    sgd_optmizer.cc
+  )
+
+set(OPITMIZER_Headers
+    adadelta_optimizer.h
+    adagrad_optimizer.h
+    adam_optimizer.h
+    lr_policy.h
+    optimizer.h
+    parameter_optimizer.h
+    sgd_optimizer.h
+    Tensor.h
+  )
+
+add_library(optimizer STATIC ${OPITMIZER_SRCS})
+add_dependencies(optimizer gen_proto_cpp)
+
+add_simple_unittest(Tensor_test)
+add_simple_unittest(parameter_optimizer_test)
+add_dependencies(parameter_optimizer_test optimizer)

paddle/optimizer/Tensor.h

@@ -0,0 +1,49 @@
+#ifndef PADDLE_OPTIMIZER_TENSOR_H_
+#define PADDLE_OPTIMIZER_TENSOR_H_
+/**
+ * @brief tensor used by optimizer
+ */
+
+#include <string.h>
+#include "paddle/utils/Common.h"
+#include "paddle/utils/Logging.h"
+
+namespace paddle {
+namespace optimizer {
+
+template <class T>
+class TensorT {
+public:
+  TensorT(size_t h, size_t w, T* data) : height_(h), width_(w), data_(data_) {}
+  TensorT(T* data, int size) : height_(1), width_(size), data_(data) {}
+  TensorT(const TensorT& t)
+      : TensorT(1, t.size(), 0, t.get_buffer(), false, false) {}
+  TensorT& operator=(const TensorT& t) {
+    this->width_ = t.size();
+    this->data_ = t.get_buffer();
+  }
+  T* get_buffer() { return this->data_; }
+  T& operator[](const size_t idx) {
+    CHECK(idx >= 0 && idx < this->width_) << "out of index range";
+    return data_[idx];
+  }
+  T& operator[](const size_t idx) const {
+    CHECK(idx >= 0 && idx < this->width_) << "out of index range";
+    return data_[idx];
+  }
+  // TODO: replace with tensorshape
+  size_t size() const { return this->width_; }
+
+protected:
+  size_t height_;
+  size_t width_;
+  T* data_;
+};
+
+// TODO(zhihong): design problem of dynamic datatype, need to fix it
+typedef TensorT<real> Tensor;
+
+}  // namespace optimizer
+}  // namespace paddle
+
+#endif

paddle/optimizer/Tensor_test.cpp

@@ -0,0 +1,21 @@
+#include "Tensor.h"
+#include <iostream>
+#include "gtest/gtest.h"
+
+using namespace paddle;
+using namespace paddle::optimizer;
+
+TEST(Tensor, indexer) {
+  real* ptr = new real[3];
+  Tensor t(ptr, 3);
+  for (auto i = 0; i < t.size(); ++i) {
+    t[i] = i;
+  }
+  ASSERT_EQ(t[2], 2);
+  ASSERT_EQ(t[1], 1);
+}
+
+int main(int argc, char** argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

paddle/optimizer/adadelta_optimizer.cc

@@ -0,0 +1,38 @@
+#include "adadelta_optimizer.h"
+#include <algorithm>
+#include <cmath>
+
+namespace paddle {
+namespace optimizer {
+
+void AdadeltaOptimizer::set_weight(Tensor* p) {
+  size_t size = p->size();
+  real* gptr = new real[size];
+  accum_gradient = new Tensor(gptr, size);
+  real* dptr = new real[size];
+  accum_delta = new Tensor(dptr, size);
+  real* dptr_current = new real[size];
+  update_delta = new Tensor(dptr_current, size);
+}
+
+void AdadeltaOptimizer::update(const Tensor* gradient) {
+  num_sample_passed += 1;
+  double learning_rate = lr_policy->get_learning_rate(num_sample_passed);
+  Tensor& param = *parameter_;
+  const Tensor& grad = *gradient;
+  Tensor& accum_g = *accum_gradient;
+  Tensor& accum_d = *accum_delta;
+  Tensor& update_d = *update_delta;
+  for (size_t i = 0; i < param.size(); ++i) {
+    accum_g[i] = rho * accum_g[i] + (1.0 - rho) * grad[i] * grad[i];
+
+    update_d[i] = std::sqrt(accum_d[i] + epsilon) /
+                  std::sqrt(accum_g[i] + epsilon) * grad[i];
+
+    accum_d[i] = rho * accum_d[i] + (1.0 - rho) * update_d[i] * update_d[i];
+
+    param[i] -= learning_rate * update_d[i] + learning_rate * decay * param[i];
+  }
+}
+}  // namespace optimizer
+}  // namespace paddle

paddle/optimizer/adadelta_optimizer.h

@@ -0,0 +1,39 @@
+#ifndef PADDLE_ADADELTA_OPTIMIZER_H_
+#define PADDLE_ADADELTA_OPTIMIZER_H_
+
+#include "parameter_optimizer.h"
+
+namespace paddle {
+namespace optimizer {
+
+class AdadeltaOptimizer : public ParameterOptimizer {
+public:
+  using ParameterOptimizer::parameter_;
+  using ParameterOptimizer::num_sample_passed;
+  using ParameterOptimizer::lr_policy;
+
+  AdadeltaOptimizer(double rho, double epsilon, double decay, BaseLr *lr)
+      : ParameterOptimizer(lr), rho(rho), epsilon(epsilon), decay(decay) {}
+  ~AdadeltaOptimizer() {
+    if (accum_gradient) delete accum_gradient;
+    if (accum_delta) delete accum_delta;
+    if (update_delta) delete update_delta;
+  }
+  void update(const Tensor *gradient);
+  void set_weight(Tensor *p);
+  real *get_weight() const;
+
+private:
+  Tensor *accum_gradient;
+  Tensor *accum_delta;
+  Tensor *update_delta;
+
+  double rho;
+  double epsilon;
+  double decay;
+};
+
+}  // namespace optimizer
+}  // namespace paddle
+
+#endif

paddle/optimizer/adagrad_optimizer.cc

@@ -0,0 +1,28 @@
+#include <cmath>
+
+#include "adagrad_optimizer.h"
+
+namespace paddle {
+namespace optimizer {
+
+void AdagradOptimizer::set_weight(Tensor* p) {
+  size_t size = p->size();
+  real* gptr = new real[size];
+  accum_gradient = new Tensor(gptr, size);
+}
+
+void AdagradOptimizer::update(const Tensor* gradient) {
+  num_sample_passed += 1;
+  double learning_rate = lr_policy->get_learning_rate(num_sample_passed);
+  Tensor& param = *parameter_;
+  const Tensor& grad = *gradient;
+  Tensor& accum_g = *accum_gradient;
+  for (size_t i = 0; i < param.size(); ++i) {
+    accum_g[i] += grad[i] * grad[i];
+    param[i] += learning_rate * grad[i] / std::sqrt(accum_g[i] + epsilon) +
+                learning_rate * decay * param[i];
+  }
+}
+
+}  // namespace optimizer
+}  // namespace paddle

paddle/optimizer/adagrad_optimizer.h

@@ -0,0 +1,29 @@
+#ifndef PADDLE_ADAGRAD_OPTIMIZER_H_
+#define PADDLE_ADAGRAD_OPTIMIZER_H_
+
+#include "parameter_optimizer.h"
+
+namespace paddle {
+namespace optimizer {
+
+class AdagradOptimizer : public ParameterOptimizer {
+public:
+  AdagradOptimizer(double epsilon, double decay, BaseLr *lr)
+      : ParameterOptimizer(lr), epsilon(epsilon), decay(decay) {}
+  ~AdagradOptimizer() {
+    if (accum_gradient) delete accum_gradient;
+  }
+  void update(const Tensor *gradient);
+  void set_weight(Tensor *p);
+  real *get_weight() const;
+
+private:
+  Tensor *accum_gradient;
+  double epsilon;
+  double decay;
+};
+
+}  // namespace optimizer
+}  // namespace paddle
+
+#endif

paddle/optimizer/adam_optimizer.cc

@@ -0,0 +1,33 @@
+#include "adam_optimizer.h"
+#include <cmath>
+
+namespace paddle {
+namespace optimizer {
+
+void AdamOptimizer::set_weight(Tensor *p) {
+  size_t size = p->size();
+  real *mptr = new real[size];
+  momentums_ = new Tensor(mptr, size);
+  real *vptr = new real[size];
+  velocitys_ = new Tensor(vptr, size);
+}
+
+void AdamOptimizer::update(const Tensor *gradient) {
+  num_sample_passed += 1;
+  double learning_rate = lr_policy->get_learning_rate(num_sample_passed);
+  double coef1 = 1.0 - std::pow(beta_1, num_sample_passed);
+  double coef2 = 1.0 - std::pow(beta_2, num_sample_passed);
+  learning_rate *= std::sqrt(coef2) / coef1;
+  Tensor &param = *parameter_;
+  const Tensor &grad = *gradient;
+  Tensor &m = *momentums_;
+  Tensor &v = *velocitys_;
+  for (size_t i = 0; i < param.size(); ++i) {
+    m[i] = beta_1 * m[i] + (1.0 - beta_1) * grad[i];
+    v[i] = beta_2 * v[i] + (1.0 - beta_2) * grad[i] * grad[i];
+    param[i] -=
+        learning_rate * (m[i] / std::sqrt(v[i] + epsilon) + decay * param[i]);
+  }
+}
+}  // namespace optimizer
+}  // namespace paddle

paddle/optimizer/adam_optimizer.h

@@ -0,0 +1,37 @@
+#ifndef PADDLE_ADAM_OPTIMIZER_H_
+#define PADDLE_ADAM_OPTIMIZER_H_
+
+#include "parameter_optimizer.h"
+
+namespace paddle {
+namespace optimizer {
+
+class AdamOptimizer : public ParameterOptimizer {
+public:
+  AdamOptimizer(
+      double beta_1, double beta_2, double epsilon, double decay, BaseLr *lr)
+      : ParameterOptimizer(lr),
+        beta_1(beta_1),
+        beta_2(beta_2),
+        epsilon(epsilon),
+        decay(decay) {}
+  ~AdamOptimizer() {
+    if (momentums_) delete momentums_;
+    if (velocitys_) delete velocitys_;
+  }
+  void update(const Tensor *gradient);
+  void set_weight(Tensor *p);
+  real *get_weight() const;
+
+private:
+  Tensor *momentums_;
+  Tensor *velocitys_;
+  double beta_1;
+  double beta_2;
+  double epsilon;
+  double decay;
+};
+
+}  // namespace optimizer
+}  // namespace paddle
+#endif

paddle/optimizer/lr_policy.h

@@ -0,0 +1,45 @@
+#ifndef PADDLE_OPTIMIZER_LR_POLICY_H_
+#define PADDLE_OPTIMIZER_LR_POLICY_H_
+
+#include <algorithm>
+#include "OptimizerConfig.pb.h"
+
+namespace paddle {
+namespace optimizer {
+
+class BaseLr {
+public:
+  BaseLr(double lr) : learning_rate(lr) {}
+  virtual ~BaseLr() {}
+  virtual double get_learning_rate(const uint64_t num_sample_passed) = 0;
+
+protected:
+  double learning_rate;
+};
+
+// constant learning rate policy
+class ConstLr final : public BaseLr {
+public:
+  ConstLr(double lr) : BaseLr(lr){};
+  double get_learning_rate(const uint64_t num_sample_passed) {
+    return learning_rate;
+  }
+};
+
+class LinearLr final : public BaseLr {
+public:
+  LinearLr(double lr, double lr_decay_a, double lr_decay_b)
+      : BaseLr(lr), lr_decay_a(lr_decay_a), lr_decay_b(lr_decay_b) {}
+  double get_learning_rate(const uint64_t num_sample_passed) {
+    return std::max(learning_rate - lr_decay_a * num_sample_passed, lr_decay_b);
+  }
+
+private:
+  double lr_decay_a;
+  double lr_decay_b;
+};
+
+}  // namespace optimizer
+}  // namespace paddle
+
+#endif