[AutoParallel] Distributed Unit Test Handling - Part 3 (#67872)

* deprecated test_pass_1F1B * deprecated test_amp_o2_pass * deprecated test_pass_amp * deprecated auto_parallel_relaunch_model * deprecated auto_parallel_relaunch_with_planner * deprecated test_pass_grad_clip * deprecated engine_api_dp * deprecated engine_api * deprecated generation_pipeline_pass_unittest * deprecated gpt_with_prim * deprecated gradient_merge_pass_unittest * deprecated mp_allreduce_matmul_grad_overlapping_unittest * deprecated optimization_tuner_api * deprecated pipeline_scheduler_vpp_unittest * deprecated pipeline_scheduler_zb_unittest * deprecated recompute_pass_unittest * Add sys.path.append("../../auto_parallel") to deprecated test files * fix path * fix * fix * fix * fix * fix * fix * fix launch module * fix merge * fix merge * fix merge * fix merge * fix merge * fix merge * fix merge * fix merge * fix merge * fix merge
PaddlePaddle · Sep 16, 2024 · 3f2fbae · 3f2fbae
1 parent 7f78e0a
commit 3f2fbae
Show file tree

Hide file tree

Showing 17 changed files with 67 additions and 30 deletions.
diff --git a/test/auto_parallel/CMakeLists.txt b/test/auto_parallel/CMakeLists.txt
@@ -9,20 +9,6 @@ add_subdirectory(pir)
 if(WITH_DISTRIBUTE AND WITH_GPU)
 
   # NOTE(zyl): unittests WITH multi cards and timeout
-  py_test_modules(test_auto_parallel_relaunch MODULES
-                  test_auto_parallel_relaunch)
-  set_tests_properties(test_auto_parallel_relaunch
-                       PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 120)
-  py_test_modules(test_engine_api MODULES test_engine_api)
-  set_tests_properties(test_engine_api PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE"
-                                                  TIMEOUT 80)
-  py_test_modules(test_engine_api_dp MODULES test_engine_api_dp)
-  set_tests_properties(test_engine_api_dp
-                       PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 80)
-  py_test_modules(test_optimization_tuner_api MODULES
-                  test_optimization_tuner_api)
-  set_tests_properties(test_optimization_tuner_api
-                       PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 120)
   py_test_modules(test_converter MODULES test_converter)
   set_tests_properties(test_converter PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE"
                                                  TIMEOUT 50)
@@ -33,13 +19,6 @@ if(WITH_DISTRIBUTE AND WITH_GPU)
   set_tests_properties(test_iterable_dataset
                        PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 80)
 
-  py_test_modules(test_auto_tuner MODULES test_auto_tuner)
-  set_tests_properties(test_auto_tuner PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE"
-                                                  TIMEOUT 100)
-  py_test_modules(test_auto_tuner_compare MODULES test_auto_tuner_compare)
-  set_tests_properties(test_auto_tuner_compare
-                       PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 100)
-
   py_test_modules(test_reshard_api MODULES test_reshard_api)
   set_tests_properties(test_reshard_api PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE"
                                                    TIMEOUT 150)

diff --git a/test/deprecated/auto_parallel/CMakeLists.txt b/test/deprecated/auto_parallel/CMakeLists.txt
@@ -7,7 +7,28 @@
 # add_subdirectory(pir)
 
 if(WITH_DISTRIBUTE AND WITH_GPU)
-
+  py_test_modules(test_auto_parallel_relaunch_deprecated MODULES
+                  test_auto_parallel_relaunch_deprecated)
+  set_tests_properties(test_auto_parallel_relaunch_deprecated
+                       PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 120)
+  py_test_modules(test_engine_api_dp_deprecated MODULES
+                  test_engine_api_dp_deprecated)
+  set_tests_properties(test_engine_api_dp_deprecated
+                       PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 80)
+  py_test_modules(test_engine_api_deprecated MODULES test_engine_api_deprecated)
+  set_tests_properties(test_engine_api_deprecated
+                       PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 80)
+  py_test_modules(test_auto_tuner_compare_deprecated MODULES
+                  test_auto_tuner_compare_deprecated)
+  set_tests_properties(test_auto_tuner_compare_deprecated
+                       PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 100)
+  py_test_modules(test_auto_tuner_deprecated MODULES test_auto_tuner_deprecated)
+  set_tests_properties(test_auto_tuner_deprecated
+                       PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 100)
+  py_test_modules(test_optimization_tuner_api_deprecated MODULES
+                  test_optimization_tuner_api_deprecated)
+  set_tests_properties(test_optimization_tuner_api_deprecated
+                       PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 120)
   py_test_modules(test_relaunch_with_planner_deprecated MODULES
                   test_relaunch_with_planner_deprecated)
   set_tests_properties(test_relaunch_with_planner_deprecated

diff --git a/..._parallel/auto_parallel_relaunch_model.py → ..._parallel/auto_parallel_relaunch_model.py b/..._parallel/auto_parallel_relaunch_model.py → ..._parallel/auto_parallel_relaunch_model.py
diff --git a/test/auto_parallel/engine_api.py → ...ed/auto_parallel/engine_api_deprecated.py b/test/auto_parallel/engine_api.py → ...ed/auto_parallel/engine_api_deprecated.py
diff --git a/test/auto_parallel/engine_api_dp.py → ...auto_parallel/engine_api_dp_deprecated.py b/test/auto_parallel/engine_api_dp.py → ...auto_parallel/engine_api_dp_deprecated.py
diff --git a/test/deprecated/auto_parallel/launch.py b/test/deprecated/auto_parallel/launch.py
@@ -0,0 +1,23 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from paddle.distributed.fleet import launch
+from paddle.distributed.fleet.launch_utils import run_with_coverage
+
+if __name__ == "__main__":
+    if os.environ.get("WITH_COVERAGE", "OFF") == "ON":
+        run_with_coverage(True)
+    launch.launch()
diff --git a/test/auto_parallel/optimization_tuner_api.py → ...llel/optimization_tuner_api_deprecated.py b/test/auto_parallel/optimization_tuner_api.py → ...llel/optimization_tuner_api_deprecated.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from engine_api_dp import MyDataset
+from engine_api_dp_deprecated import MyDataset
 
 import paddle
 import paddle.nn.functional as F

diff --git a/test/deprecated/auto_parallel/pipeline_scheduler_zb_unittest_deprecated.py b/test/deprecated/auto_parallel/pipeline_scheduler_zb_unittest_deprecated.py
@@ -19,6 +19,7 @@
 import numpy as np
 
 sys.path.append("../../auto_parallel")
+
 from get_gpt_model import FakeDataset, generate_model
 
 import paddle

diff --git a/test/deprecated/auto_parallel/recompute_pass_unittest_deprecated.py b/test/deprecated/auto_parallel/recompute_pass_unittest_deprecated.py
@@ -19,6 +19,7 @@
 import numpy as np
 
 sys.path.append("../../auto_parallel")
+
 from get_gpt_model import FakeDataset, generate_model
 
 import paddle

diff --git a/...o_parallel/test_auto_parallel_relaunch.py → ...test_auto_parallel_relaunch_deprecated.py b/...o_parallel/test_auto_parallel_relaunch.py → ...test_auto_parallel_relaunch_deprecated.py
diff --git a/.../auto_parallel/test_auto_tuner_compare.py → ...lel/test_auto_tuner_compare_deprecated.py b/.../auto_parallel/test_auto_tuner_compare.py → ...lel/test_auto_tuner_compare_deprecated.py
@@ -23,7 +23,9 @@
 class TestEngineAPI(unittest.TestCase):
     def test_auto_tuner_compare(self):
         file_dir = os.path.dirname(os.path.abspath(__file__))
-        launch_model_path = os.path.join(file_dir, "engine_api_dp.py")
+        launch_model_path = os.path.join(
+            file_dir, "engine_api_dp_deprecated.py"
+        )
 
         if os.environ.get("WITH_COVERAGE", "OFF") == "ON":
             coverage_args = ["-m", "coverage", "run", "--branch", "-p"]

diff --git a/test/auto_parallel/test_auto_tuner.py → ...to_parallel/test_auto_tuner_deprecated.py b/test/auto_parallel/test_auto_tuner.py → ...to_parallel/test_auto_tuner_deprecated.py
@@ -23,7 +23,9 @@
 class TestEngineAPI(unittest.TestCase):
     def test_auto_tuner(self):
         file_dir = os.path.dirname(os.path.abspath(__file__))
-        launch_model_path = os.path.join(file_dir, "engine_api_dp.py")
+        launch_model_path = os.path.join(
+            file_dir, "engine_api_dp_deprecated.py"
+        )
 
         if os.environ.get("WITH_COVERAGE", "OFF") == "ON":
             coverage_args = ["-m", "coverage", "run", "--branch", "-p"]

diff --git a/test/auto_parallel/test_engine_api_dp.py → ...to_parallel/test_engine_api_deprecated.py b/test/auto_parallel/test_engine_api_dp.py → ...to_parallel/test_engine_api_deprecated.py
@@ -22,7 +22,7 @@
 class TestEngineAPI(unittest.TestCase):
     def test_engine_api(self):
         file_dir = os.path.dirname(os.path.abspath(__file__))
-        launch_model_path = os.path.join(file_dir, "engine_api_dp.py")
+        launch_model_path = os.path.join(file_dir, "engine_api_deprecated.py")
 
         if os.environ.get("WITH_COVERAGE", "OFF") == "ON":
             coverage_args = ["-m", "coverage", "run", "--branch", "-p"]

diff --git a/test/auto_parallel/test_engine_api.py → ...parallel/test_engine_api_dp_deprecated.py b/test/auto_parallel/test_engine_api.py → ...parallel/test_engine_api_dp_deprecated.py
@@ -22,7 +22,9 @@
 class TestEngineAPI(unittest.TestCase):
     def test_engine_api(self):
         file_dir = os.path.dirname(os.path.abspath(__file__))
-        launch_model_path = os.path.join(file_dir, "engine_api.py")
+        launch_model_path = os.path.join(
+            file_dir, "engine_api_dp_deprecated.py"
+        )
 
         if os.environ.get("WITH_COVERAGE", "OFF") == "ON":
             coverage_args = ["-m", "coverage", "run", "--branch", "-p"]

diff --git a/test/deprecated/auto_parallel/test_mp_allreduce_matmul_grad_overlapping_deprecated.py b/test/deprecated/auto_parallel/test_mp_allreduce_matmul_grad_overlapping_deprecated.py
@@ -23,7 +23,8 @@ class TestMPAllreduceMatmulGradOverlapping(unittest.TestCase):
     def test_mp_allreduce_matmul_grad_overlapping(self):
         file_dir = os.path.dirname(os.path.abspath(__file__))
         launch_model_path = os.path.join(
-            file_dir, "mp_allreduce_matmul_grad_overlapping_unittest.py"
+            file_dir,
+            "mp_allreduce_matmul_grad_overlapping_unittest.py",
         )
 
         if os.environ.get("WITH_COVERAGE", "OFF") == "ON":

diff --git a/...o_parallel/test_optimization_tuner_api.py → ...test_optimization_tuner_api_deprecated.py b/...o_parallel/test_optimization_tuner_api.py → ...test_optimization_tuner_api_deprecated.py
@@ -23,7 +23,9 @@
 class TestOptimizationTunerAPI(unittest.TestCase):
     def test_engine_api(self):
         file_dir = os.path.dirname(os.path.abspath(__file__))
-        launch_model_path = os.path.join(file_dir, "optimization_tuner_api.py")
+        launch_model_path = os.path.join(
+            file_dir, "optimization_tuner_api_deprecated.py"
+        )
 
         if os.environ.get("WITH_COVERAGE", "OFF") == "ON":
             coverage_args = ["-m", "coverage", "run", "--branch", "-p"]

diff --git a/test/deprecated/auto_parallel/test_relaunch_with_planner_deprecated.py b/test/deprecated/auto_parallel/test_relaunch_with_planner_deprecated.py
@@ -30,7 +30,10 @@ def tearDown(self):
         self.temp_dir.cleanup()
 
     def test_relaunch_with_planner(self):
-        from test_auto_parallel_relaunch import cluster_json, mapping_json
+        from test_auto_parallel_relaunch_deprecated import (
+            cluster_json,
+            mapping_json,
+        )
 
         cluster_json_path = os.path.join(
             self.temp_dir.name, "auto_parallel_cluster.json"