fix learning rate

Jackwaterveg · May 21, 2019 · f14fe81 · f14fe81
1 parent cb819cf
commit f14fe81
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/Mask-RCNN/paddle/rcnn/train.py b/Mask-RCNN/paddle/rcnn/train.py
@@ -47,6 +47,10 @@ def get_device_num():
 def update_lr(args):
     num_trainers = int(os.environ.get('PADDLE_TRAINERS_NUM', 1))
     args.learning_rate = args.learning_rate / num_trainers
+    # TODO(zcd): The loss_cls or loss maybe NAN, so we decreate the learning rate here.
+    # The reasons for this should be analyzed in depth.
+    if num_trainers > 1:
+        args.learning_rate = args.learning_rate / 10
 
 def train():
     update_lr(cfg)