Throw error for NVCC lazy in 1F1B pipeline (#56725)

PaddlePaddle · Aug 31, 2023 · 2582021 · 2582021
1 parent 0a15b0d
commit 2582021
Showing 1 changed file with 8 additions and 0 deletions.
diff --git a/python/paddle/distributed/auto_parallel/static/engine.py b/python/paddle/distributed/auto_parallel/static/engine.py
@@ -242,6 +242,14 @@ def __init__(
         elif self._strategy.pipeline.enable:
             self._acc_steps = self._strategy.pipeline.accumulate_steps
 
+        if (
+            self._strategy.pipeline.enable
+            and self._strategy.pipeline.schedule_mode == "1F1B"
+        ):
+            assert (
+                os.getenv("CUDA_MODULE_LOADING") != "LAZY"
+            ), "EXP_CUDA_MODULE_LOADING_LAZY not supported in 1F1B pipeline."
+
         self.history = None
 
         paddle.framework.set_flags({'FLAGS_new_executor_sequential_run': 1})