fix doc and unittest of 2.0 lr_scheduler

python/paddle/__init__.py

@@ -237,13 +237,6 @@
 from .framework import SaveLoadConfig  #DEFINE_ALIAS
 from .framework import DataParallel  #DEFINE_ALIAS
 
-from .framework import NoamDecay  #DEFINE_ALIAS
-from .framework import PiecewiseDecay  #DEFINE_ALIAS
-from .framework import NaturalExpDecay  #DEFINE_ALIAS
-from .framework import ExponentialDecay  #DEFINE_ALIAS
-from .framework import InverseTimeDecay  #DEFINE_ALIAS
-from .framework import PolynomialDecay  #DEFINE_ALIAS
-from .framework import CosineDecay  #DEFINE_ALIAS
 from .framework import set_default_dtype  #DEFINE_ALIAS
 from .framework import get_default_dtype  #DEFINE_ALIAS
 

python/paddle/fluid/tests/unittests/test_imperative_optimizer_v2.py

@@ -23,7 +23,7 @@
 import paddle
 import paddle.fluid as fluid
 from paddle.fluid import core
-from paddle.fluid.optimizer import SGDOptimizer, Adam, MomentumOptimizer, LarsMomentumOptimizer, AdagradOptimizer, AdamaxOptimizer, DpsgdOptimizer, DecayedAdagradOptimizer, AdadeltaOptimizer, RMSPropOptimizer, FtrlOptimizer, LambOptimizer
+from paddle.fluid.optimizer import MomentumOptimizer, LarsMomentumOptimizer, AdagradOptimizer, AdamaxOptimizer, DpsgdOptimizer, DecayedAdagradOptimizer, AdadeltaOptimizer, RMSPropOptimizer, FtrlOptimizer, LambOptimizer
 from paddle.fluid.optimizer import ModelAverage, DGCMomentumOptimizer, ExponentialMovingAverage, PipelineOptimizer, LookaheadOptimizer, RecomputeOptimizer
 from paddle.fluid.dygraph import Linear
 from paddle.fluid.dygraph.base import to_variable
@@ -126,6 +126,9 @@ def _check_mlp(self, place=None):
 
                 avg_loss.backward()
                 optimizer.minimize(avg_loss)
+                if isinstance(optimizer._learning_rate,
+                              paddle.optimizer._LRScheduler):
+                    optimizer._learning_rate.step()
                 mlp.clear_gradients()
                 dy_param_value = {}
                 for param in mlp.parameters():
@@ -181,6 +184,9 @@ def _check_mlp(self, place=None):
                               feed={"pixel": static_x_data,
                                     "label": y_data},
                               fetch_list=fetch_list)
+                if isinstance(optimizer._learning_rate,
+                              paddle.optimizer._LRScheduler):
+                    optimizer._learning_rate.step()
 
                 static_param_value = {}
                 static_out = out[0]
@@ -196,106 +202,93 @@ def _check_mlp(self, place=None):
             self.assertTrue(np.allclose(value, dy_param_value[key]))
 
 
-class TestImperativeOptimizerPiecewiseDecay(TestImperativeOptimizerBase):
+class TestImperativeOptimizerPiecewiseLR(TestImperativeOptimizerBase):
     def get_optimizer_dygraph(self, parameter_list):
         bd = [3, 6, 9]
-        optimizer = SGDOptimizer(
-            learning_rate=paddle.optimizer.PiecewiseLR(
+        optimizer = paddle.optimizer.SGD(
+            learning_rate=paddle.optimizer.lr_scheduler.PiecewiseLR(
                 boundaries=bd,
                 values=[0.1 * (0.1**i) for i in range(len(bd) + 1)]),
-            parameter_list=parameter_list)
+            parameters=parameter_list)
         return optimizer
 
     def get_optimizer(self):
         bd = [3, 6, 9]
-        optimizer = SGDOptimizer(learning_rate=paddle.optimizer.PiecewiseLR(
-            boundaries=bd, values=[0.1 * (0.1**i) for i in range(len(bd) + 1)]))
+        optimizer = paddle.optimizer.SGD(
+            learning_rate=paddle.optimizer.lr_scheduler.PiecewiseLR(
+                boundaries=bd,
+                values=[0.1 * (0.1**i) for i in range(len(bd) + 1)]))
         return optimizer
 
     def test_sgd(self):
         self._check_mlp()
 
 
-class TestImperativeOptimizerNaturalExpDecay(TestImperativeOptimizerBase):
+class TestImperativeOptimizerNaturalExpLR(TestImperativeOptimizerBase):
     def get_optimizer_dygraph(self, parameter_list):
-        optimizer = SGDOptimizer(
-            learning_rate=fluid.layers.natural_exp_decay(
-                learning_rate=0.1,
-                decay_steps=10000,
-                decay_rate=0.5,
-                staircase=True),
-            parameter_list=parameter_list)
+        optimizer = paddle.optimizer.SGD(
+            learning_rate=paddle.optimizer.lr_scheduler.NaturalExpLR(
+                learning_rate=0.1, gamma=0.5, parameters=parameter_list))
         return optimizer
 
     def get_optimizer(self):
-        optimizer = SGDOptimizer(learning_rate=fluid.layers.natural_exp_decay(
-            learning_rate=0.1,
-            decay_steps=10000,
-            decay_rate=0.5,
-            staircase=True))
+        optimizer = paddle.optimizer.SGD(
+            learning_rate=paddle.optimizer.lr_scheduler.NaturalExpLR(
+                learning_rate=0.1, gamma=0.5))
         return optimizer
 
     def test_sgd(self):
         self._check_mlp()
 
 
-class TestImperativeOptimizerExponentialDecay(TestImperativeOptimizerBase):
+class TestImperativeOptimizerExponentialLR(TestImperativeOptimizerBase):
     def get_optimizer_dygraph(self, parameter_list):
-        optimizer = SGDOptimizer(
-            learning_rate=fluid.layers.exponential_decay(
-                learning_rate=0.1,
-                decay_steps=10000,
-                decay_rate=0.5,
-                staircase=True),
-            parameter_list=parameter_list)
+        optimizer = paddle.optimizer.SGD(
+            learning_rate=paddle.optimizer.lr_scheduler.ExponentialLR(
+                learning_rate=0.1, gamma=0.5),
+            parameters=parameter_list)
         return optimizer
 
     def get_optimizer(self):
-        optimizer = SGDOptimizer(learning_rate=fluid.layers.exponential_decay(
-            learning_rate=0.1,
-            decay_steps=10000,
-            decay_rate=0.5,
-            staircase=True))
+        optimizer = paddle.optimizer.SGD(
+            learning_rate=paddle.optimizer.lr_scheduler.ExponentialLR(
+                learning_rate=0.1, gamma=0.05))
         return optimizer
 
     def test_sgd(self):
         self._check_mlp()
 
 
-class TestImperativeOptimizerInverseTimeDecay(TestImperativeOptimizerBase):
+class TestImperativeOptimizerInverseTimeLR(TestImperativeOptimizerBase):
     def get_optimizer_dygraph(self, parameter_list):
-        optimizer = Adam(
-            learning_rate=fluid.layers.inverse_time_decay(
-                learning_rate=0.1,
-                decay_steps=10000,
-                decay_rate=0.5,
-                staircase=True),
-            parameter_list=parameter_list)
+        optimizer = paddle.optimizer.Adam(
+            learning_rate=paddle.optimizer.lr_scheduler.InverseTimeLR(
+                learning_rate=0.1, gamma=0.5),
+            parameters=parameter_list)
         return optimizer
 
     def get_optimizer(self):
-        optimizer = Adam(learning_rate=fluid.layers.inverse_time_decay(
-            learning_rate=0.1,
-            decay_steps=10000,
-            decay_rate=0.5,
-            staircase=True))
+        optimizer = paddle.optimizer.Adam(
+            learning_rate=paddle.optimizer.lr_scheduler.InverseTimeLR(
+                learning_rate=0.1, gamma=0.5))
         return optimizer
 
     def test_adam(self):
         self._check_mlp()
 
 
-class TestImperativeOptimizerPolynomialDecay(TestImperativeOptimizerBase):
+class TestImperativeOptimizerPolynomialLR(TestImperativeOptimizerBase):
     def get_optimizer_dygraph(self, parameter_list):
-        optimizer = SGDOptimizer(
-            learning_rate=fluid.layers.polynomial_decay(
+        optimizer = paddle.optimizer.SGD(
+            learning_rate=paddle.optimizer.lr_scheduler.PolynomialLR(
                 learning_rate=0.1, decay_steps=5, cycle=self.cycle),
-            parameter_list=parameter_list)
+            parameters=parameter_list)
         return optimizer
 
     def get_optimizer(self):
-        optimizer = SGDOptimizer(learning_rate=fluid.layers.polynomial_decay(
-            learning_rate=0.1, decay_steps=5, cycle=self.cycle))
+        optimizer = paddle.optimizer.SGD(
+            learning_rate=paddle.optimizer.lr_scheduler.PolynomialLR(
+                learning_rate=0.1, decay_steps=5, cycle=self.cycle))
         return optimizer
 
     def test_sgd_cycle(self):
@@ -307,17 +300,18 @@ def test_sgd(self):
         self._check_mlp()
 
 
-class TestImperativeOptimizerCosineDecay(TestImperativeOptimizerBase):
+class TestImperativeOptimizerCosineLR(TestImperativeOptimizerBase):
     def get_optimizer_dygraph(self, parameter_list):
-        optimizer = SGDOptimizer(
-            learning_rate=fluid.layers.cosine_decay(
-                learning_rate=0.1, step_each_epoch=10000, epochs=120),
-            parameter_list=parameter_list)
+        optimizer = paddle.optimizer.SGD(
+            learning_rate=paddle.optimizer.lr_scheduler.CosineAnnealingLR(
+                learning_rate=0.1, T_max=120),
+            parameter=parameter_list)
         return optimizer
 
     def get_optimizer(self):
-        optimizer = SGDOptimizer(learning_rate=fluid.layers.cosine_decay(
-            learning_rate=0.1, step_each_epoch=10000, epochs=120))
+        optimizer = paddle.optimizer.SGD(
+            learning_rate=paddle.optimizer.lr_scheduler.CosineAnnealingLR(
+                learning_rate=0.1, T_max=120))
         return optimizer
 
     def test_sgd(self):
@@ -326,15 +320,16 @@ def test_sgd(self):
 
 class TestImperativeOptimizerNoamDecay(TestImperativeOptimizerBase):
     def get_optimizer_dygraph(self, parameter_list):
-        optimizer = SGDOptimizer(
-            learning_rate=fluid.layers.noam_decay(
+        optimizer = paddle.optimizer.SGD(
+            learning_rate=paddle.optimizer.lr_scheduler.NoamLR(
                 d_model=512, warmup_steps=8000),
-            parameter_list=parameter_list)
+            parameter=parameter_list)
         return optimizer
 
     def get_optimizer(self):
-        optimizer = SGDOptimizer(learning_rate=fluid.layers.noam_decay(
-            d_model=512, warmup_steps=8000))
+        optimizer = paddle.optimizer.SGD(
+            learning_rate=paddle.optimizer.lr_scheduler.NoamLR(
+                d_model=512, warmup_steps=8000))
         return optimizer
 
     def test_sgd(self):

python/paddle/framework/__init__.py

@@ -24,11 +24,6 @@
     'no_grad', 'DataParallel'
 ]
 
-__all__ += [
-    'NoamDecay', 'PiecewiseDecay', 'NaturalExpDecay', 'ExponentialDecay',
-    'InverseTimeDecay', 'PolynomialDecay', 'CosineDecay'
-]
-
 from . import random
 from .random import manual_seed
 from .framework import get_default_dtype
@@ -52,11 +47,3 @@
 from .io import load
 from ..fluid.dygraph.jit import SaveLoadConfig  #DEFINE_ALIAS
 from ..fluid.dygraph.parallel import DataParallel  #DEFINE_ALIAS
-
-from ..fluid.dygraph.learning_rate_scheduler import NoamDecay  #DEFINE_ALIAS
-from ..fluid.dygraph.learning_rate_scheduler import PiecewiseDecay  #DEFINE_ALIAS
-from ..fluid.dygraph.learning_rate_scheduler import NaturalExpDecay  #DEFINE_ALIAS
-from ..fluid.dygraph.learning_rate_scheduler import ExponentialDecay  #DEFINE_ALIAS
-from ..fluid.dygraph.learning_rate_scheduler import InverseTimeDecay  #DEFINE_ALIAS
-from ..fluid.dygraph.learning_rate_scheduler import PolynomialDecay  #DEFINE_ALIAS
-from ..fluid.dygraph.learning_rate_scheduler import CosineDecay  #DEFINE_ALIAS

python/paddle/nn/__init__.py

@@ -120,13 +120,6 @@
 # from .layer.conv import TreeConv        #DEFINE_ALIAS
 # from .layer.conv import Conv1D        #DEFINE_ALIAS
 from .layer.extension import RowConv  #DEFINE_ALIAS
-# from .layer.learning_rate import CosineDecay        #DEFINE_ALIAS
-# from .layer.learning_rate import ExponentialDecay        #DEFINE_ALIAS
-# from .layer.learning_rate import InverseTimeDecay        #DEFINE_ALIAS
-# from .layer.learning_rate import NaturalExpDecay        #DEFINE_ALIAS
-# from .layer.learning_rate import NoamDecay        #DEFINE_ALIAS
-# from .layer.learning_rate import PiecewiseDecay        #DEFINE_ALIAS
-# from .layer.learning_rate import PolynomialDecay        #DEFINE_ALIAS
 from .layer.common import Linear
 # from .layer.loss import NCELoss        #DEFINE_ALIAS
 from .layer.loss import BCEWithLogitsLoss  #DEFINE_ALIAS

python/paddle/nn/functional/__init__.py

@@ -97,14 +97,6 @@
 from .extension import temporal_shift  #DEFINE_ALIAS
 from .extension import warpctc  #DEFINE_ALIAS
 from .extension import diag_embed  #DEFINE_ALIAS
-from .learning_rate import cosine_decay  #DEFINE_ALIAS
-from .learning_rate import exponential_decay  #DEFINE_ALIAS
-from .learning_rate import inverse_time_decay  #DEFINE_ALIAS
-from .learning_rate import natural_exp_decay  #DEFINE_ALIAS
-from .learning_rate import noam_decay  #DEFINE_ALIAS
-from .learning_rate import piecewise_decay  #DEFINE_ALIAS
-from .learning_rate import polynomial_decay  #DEFINE_ALIAS
-from .learning_rate import linear_lr_warmup  #DEFINE_ALIAS
 # from .lod import sequence_concat        #DEFINE_ALIAS
 # from .lod import sequence_conv        #DEFINE_ALIAS
 # from .lod import sequence_enumerate        #DEFINE_ALIAS

python/paddle/nn/layer/__init__.py

@@ -87,13 +87,6 @@
 # from .conv import TreeConv        #DEFINE_ALIAS
 # from .conv import Conv1D        #DEFINE_ALIAS
 from .extension import RowConv  #DEFINE_ALIAS
-# from .learning_rate import CosineDecay        #DEFINE_ALIAS
-# from .learning_rate import ExponentialDecay        #DEFINE_ALIAS
-# from .learning_rate import InverseTimeDecay        #DEFINE_ALIAS
-# from .learning_rate import NaturalExpDecay        #DEFINE_ALIAS
-# from .learning_rate import NoamDecay        #DEFINE_ALIAS
-# from .learning_rate import PiecewiseDecay        #DEFINE_ALIAS
-# from .learning_rate import PolynomialDecay        #DEFINE_ALIAS
 # from .loss import NCELoss        #DEFINE_ALIAS
 from .loss import BCEWithLogitsLoss  #DEFINE_ALIAS
 from .loss import CrossEntropyLoss  #DEFINE_ALIAS