From 7a11d22a9365b24ab3d055354b6863eaa6fd1c89 Mon Sep 17 00:00:00 2001 From: zhaoyingli Date: Wed, 30 Aug 2023 13:26:51 +0800 Subject: [PATCH 01/12] [NewIR]Update send recv infermeta and add unittest --- .../ir/dialect/op_generator/python_c_gen.py | 3 +- .../ir/dialect/paddle_dialect/ir/pd_op.yaml | 3 +- paddle/phi/infermeta/nullary.cc | 42 +++++++ paddle/phi/infermeta/nullary.h | 7 ++ paddle/phi/infermeta/unary.cc | 57 +++------- paddle/phi/infermeta/unary.h | 9 +- .../auto_parallel/static/engine.py | 12 ++ test/auto_parallel/CMakeLists.txt | 3 + test/auto_parallel/gpt_with_newir.py | 104 ++++++++++++++++++ test/auto_parallel/test_gpt_with_newir.py | 55 +++++++++ 10 files changed, 242 insertions(+), 53 deletions(-) create mode 100644 test/auto_parallel/gpt_with_newir.py create mode 100644 test/auto_parallel/test_gpt_with_newir.py diff --git a/paddle/fluid/ir/dialect/op_generator/python_c_gen.py b/paddle/fluid/ir/dialect/op_generator/python_c_gen.py index 340aa9569818a..a890a8db5d249 100644 --- a/paddle/fluid/ir/dialect/op_generator/python_c_gen.py +++ b/paddle/fluid/ir/dialect/op_generator/python_c_gen.py @@ -283,11 +283,12 @@ def _gen_cast_attrs(self, op_info, op_name, with_mutable): def _gen_one_impl(self, op_info, op_name): input_name_list = op_info.input_name_list + output_name_list = op_info.output_name_list attr_name_list = op_info.attribute_name_list mutable_attr_name_list = op_info.mutable_attribute_name_list no_mutable_attr_name_list = op_info.non_mutable_attribute_name_list - if op_name == "send_v2": + if len(output_name_list) == 0: ret = NO_OUTPUT_API_IMPL_TEMPLATE.format( api_name=op_name, inputs=self._gen_inputs(op_info, op_name), diff --git a/paddle/fluid/ir/dialect/paddle_dialect/ir/pd_op.yaml b/paddle/fluid/ir/dialect/paddle_dialect/ir/pd_op.yaml index 942c3b8703c7e..efc84a66dcb98 100644 --- a/paddle/fluid/ir/dialect/paddle_dialect/ir/pd_op.yaml +++ b/paddle/fluid/ir/dialect/paddle_dialect/ir/pd_op.yaml @@ -400,7 +400,7 @@ data_transform: null infer_meta: func: RecvV2InferMeta - param: [peer, dtype, out_shape] + param: [ring_id, dynamic_shape, peer, out_shape, dtype] kernel: func: [recv_v2] param: [ring_id, dynamic_shape, peer, out_shape, dtype, use_calc_stream] @@ -414,6 +414,7 @@ force_backend: null inplace: null view: null + backward: null - name : set_value inputs: diff --git a/paddle/phi/infermeta/nullary.cc b/paddle/phi/infermeta/nullary.cc index 1526ba2ec021b..d5da3a2f8bc87 100644 --- a/paddle/phi/infermeta/nullary.cc +++ b/paddle/phi/infermeta/nullary.cc @@ -181,6 +181,48 @@ void PRecvArrayInferMeta(int peer, out->set_dtype(dtype); } +void RecvV2InferMeta(const int ring_id, + const bool dynamic_shape, + const int peer, + const std::vector& out_shape, + DataType dtype, + MetaTensor* out) { + PADDLE_ENFORCE_GE( + peer, + 0, + errors::InvalidArgument( + "The peer (%d) for recv_v2 op must be non-negative.", peer)); + + PADDLE_ENFORCE_GE( + ring_id, + 0, + errors::InvalidArgument( + "The ring_id (%d) for recv_v2 op must be non-negative.", ring_id)); + + PADDLE_ENFORCE_GE(out_shape.size(), + 1, + errors::InvalidArgument( + "The size of the output shape must be greater than 0 " + "but the value given is %d.", + out_shape.size())); + + if (!dynamic_shape) { + for (size_t i = 0; i < out_shape.size(); ++i) { + PADDLE_ENFORCE_GE(out_shape[i], + 1, + errors::InvalidArgument( + "The shape attribute for recv_v2 must be set " + "explicitly, but the %dth element is %d which " + "is less than 1. Or dynamic_shape should be " + "set to True for both send_v2 and recv_v2.", + i, + out_shape[i])); + } + out->set_dims(phi::make_ddim(out_shape)); + } + out->set_dtype(dtype); +} + void TruncatedGaussianRandomInferMeta(const std::vector& shape, float mean, float std, diff --git a/paddle/phi/infermeta/nullary.h b/paddle/phi/infermeta/nullary.h index 775df0cc6ab47..bc73942c8ec1c 100644 --- a/paddle/phi/infermeta/nullary.h +++ b/paddle/phi/infermeta/nullary.h @@ -76,6 +76,13 @@ void PRecvArrayInferMeta(int peer, const std::vector& out_shape, MetaTensor* out); +void RecvV2InferMeta(const int ring_id, + const bool dynamic_shape, + const int peer, + const std::vector& out_shape, + DataType dtype, + MetaTensor* out); + void TruncatedGaussianRandomInferMeta(const std::vector& shape, float mean, float std, diff --git a/paddle/phi/infermeta/unary.cc b/paddle/phi/infermeta/unary.cc index 6e04a149266ac..5249cd601ffe7 100644 --- a/paddle/phi/infermeta/unary.cc +++ b/paddle/phi/infermeta/unary.cc @@ -403,50 +403,6 @@ void CConcatInferMeta(const MetaTensor& x, int nranks, MetaTensor* out) { out->set_dtype(x.dtype()); } -void SendV2InferMeta(const int peer, const int ring_id) { - PADDLE_ENFORCE_GE( - peer, - 0, - errors::InvalidArgument( - "The peer (%d) for send_v2 op must be non-negative.", peer)); - PADDLE_ENFORCE_GE( - ring_id, - 0, - errors::InvalidArgument( - "The ring_id (%d) for send_v2 op must be non-negative.", ring_id)); -} - -void RecvV2InferMeta(int peer, - DataType dtype, - const std::vector& out_shape, - MetaTensor* out) { - PADDLE_ENFORCE_GE( - peer, - 0, - errors::InvalidArgument( - "The peer (%d) for p_recv op must be non-negative.", peer)); - - PADDLE_ENFORCE_GE(out_shape.size(), - 1, - errors::InvalidArgument( - "The size of the output shape must be greater than 0 " - "but the value given is %d.", - out_shape.size())); - - for (size_t i = 0; i < out_shape.size(); ++i) { - PADDLE_ENFORCE_GE( - out_shape[i], - 1, - errors::InvalidArgument("The shape attribute for recv must be set " - "explicitly, but the %dth element is %d which " - "is less than 1. Or dynamic_shape should be " - "set to True for both send_v2 and recv_v2.", - i, - out_shape[i])); - } - out->set_dtype(dtype); -} - void CholeskyInferMeta(const MetaTensor& x, bool upper, MetaTensor* out) { auto dims = x.dims(); auto rank = dims.size(); @@ -3032,6 +2988,19 @@ void PSendArrayInferMeta(const MetaTensor& x, int peer) { "The peer (%d) for p_send op must be non-negative.", peer)); } +void SendV2InferMeta(const int peer, const int ring_id) { + PADDLE_ENFORCE_GE( + peer, + 0, + errors::InvalidArgument( + "The peer (%d) for send_v2 op must be non-negative.", peer)); + PADDLE_ENFORCE_GE( + ring_id, + 0, + errors::InvalidArgument( + "The ring_id (%d) for send_v2 op must be non-negative.", ring_id)); +} + void PoolInferMeta(const MetaTensor& x, const std::vector& kernel_size, const std::vector& strides, diff --git a/paddle/phi/infermeta/unary.h b/paddle/phi/infermeta/unary.h index 136b8c240e5f3..f0584abff6b78 100644 --- a/paddle/phi/infermeta/unary.h +++ b/paddle/phi/infermeta/unary.h @@ -73,13 +73,6 @@ void CastInferMeta(const MetaTensor& x, DataType out_dtype, MetaTensor* out); void CConcatInferMeta(const MetaTensor& x, int nranks, MetaTensor* out); -void SendV2InferMeta(const int peer, const int ring_id); - -void RecvV2InferMeta(int peer, - DataType dtype, - const std::vector& out_shape, - MetaTensor* out); - void ChannelShuffleInferMeta(const MetaTensor& x, int groups, const std::string& data_format, @@ -446,6 +439,8 @@ void PSendInferMeta(const MetaTensor& x, int peer); void PSendArrayInferMeta(const MetaTensor& x, int peer); +void SendV2InferMeta(const int peer, const int ring_id); + void QrInferMeta(const MetaTensor& x, const std::string& mode, MetaTensor* q, diff --git a/python/paddle/distributed/auto_parallel/static/engine.py b/python/paddle/distributed/auto_parallel/static/engine.py index 0e3c316a5470c..f49e4fae8304f 100644 --- a/python/paddle/distributed/auto_parallel/static/engine.py +++ b/python/paddle/distributed/auto_parallel/static/engine.py @@ -827,6 +827,13 @@ def _initialize(self, mode): dist_main_program, self._place, dist_context ) + use_new_ir = False + if os.environ.get('FLAGS_enable_new_ir_in_executor') == "True": + use_new_ir = True + paddle.framework.set_flags( + {"FLAGS_enable_new_ir_in_executor": False} + ) + if self._executor is None: self._executor = paddle.static.Executor(self._place) uninitialized = [] @@ -854,6 +861,11 @@ def _initialize(self, mode): ] self._executor.run(dist_startup_prog) + if use_new_ir: + paddle.framework.set_flags( + {"FLAGS_enable_new_ir_in_executor": True} + ) + def fit( self, train_data, diff --git a/test/auto_parallel/CMakeLists.txt b/test/auto_parallel/CMakeLists.txt index 817508e57034c..2d9a154008148 100644 --- a/test/auto_parallel/CMakeLists.txt +++ b/test/auto_parallel/CMakeLists.txt @@ -85,6 +85,9 @@ if(WITH_DISTRIBUTE AND WITH_GPU) py_test_modules(test_reshard_r_to_s MODULES test_reshard_r_to_s) set_tests_properties(test_reshard_r_to_s PROPERTIES LABELS "RUN_TYPE=EXECLUSIVE" TIMEOUT 100) + py_test_modules(test_gpt_with_newir MODULES test_gpt_with_newir) + set_tests_properties(test_gpt_with_newir + PROPERTIES LABELS "RUN_TYPE=EXECLUSIVE" TIMEOUT 60) # End of unittests WITH multi cards and timeout # NOTE(zyl): unittests WITH multi cards and WITHOUT timeout diff --git a/test/auto_parallel/gpt_with_newir.py b/test/auto_parallel/gpt_with_newir.py new file mode 100644 index 0000000000000..73fa1d619a55a --- /dev/null +++ b/test/auto_parallel/gpt_with_newir.py @@ -0,0 +1,104 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import random +import unittest + +import numpy as np +from get_gpt_model import FakeDataset, generate_model + +import paddle +from paddle.distributed import ParallelEnv +from paddle.distributed.fleet import auto + +paddle.enable_static() + + +def apply_pass(): + strategy = auto.Strategy() + strategy.auto_mode = "semi" + strategy.reinit = True + + return strategy + + +def reset_prog(): + paddle.fluid.framework.switch_main_program(paddle.static.Program()) + paddle.fluid.framework.switch_startup_program(paddle.static.Program()) + paddle.utils.unique_name.switch() + + +class Test1F1BPass(unittest.TestCase): + def setUp(self): + self.rtol = 1e-5 + self.atol = 1e-8 + self.batch_size = 2 + self.batch_num = 5 + self.clip_norm = 0.2 + self.dataset = FakeDataset(self.batch_size * self.batch_num) + + def init(self, engine): + paddle.seed(2021) + np.random.seed(2021) + random.seed(2021) + paddle.distributed.fleet.init(is_collective=True) + place = paddle.fluid.CUDAPlace(ParallelEnv().dev_id) + engine._executor = paddle.static.Executor(place) + + def get_engine(self, mode): + reset_prog() + + strategy = apply_pass() + clip = paddle.nn.ClipGradByGlobalNorm(self.clip_norm) + opt = paddle.optimizer.AdamW(learning_rate=0.00001, grad_clip=clip) + model, loss = generate_model(mode) + + engine = auto.Engine(model, loss, opt, strategy=strategy) + self.init(engine) + return engine + + def check_results(self, ref_losses, check_losses): + np.testing.assert_allclose( + ref_losses, + check_losses, + rtol=self.rtol, + atol=self.atol, + err_msg='pass {} has wrong results!, \nu={}\nv={}\ndiff={}'.format( + __class__, ref_losses, check_losses, ref_losses - check_losses + ), + ) + + def test_pp_pass(self): + os.environ['FLAGS_new_executor_micro_batching'] = 'True' + os.environ['FLAGS_enable_new_ir_in_executor'] = 'True' + + # data parallel + engine_dp = self.get_engine("dp") + outs = engine_dp.fit( + self.dataset, 3, batch_size=self.batch_size, log_freq=1 + ) + + # navie pipeline parallel without schedule + engine_pp = self.get_engine("pp") + outs = engine_pp.fit( + self.dataset, 3, batch_size=self.batch_size, log_freq=1 + ) + + assert os.environ.get('FLAGS_new_executor_micro_batching') == "True" + assert os.environ.get('FLAGS_enable_new_ir_in_executor') == "True" + + +if __name__ == "__main__": + unittest.main() diff --git a/test/auto_parallel/test_gpt_with_newir.py b/test/auto_parallel/test_gpt_with_newir.py new file mode 100644 index 0000000000000..2f736d8a3b297 --- /dev/null +++ b/test/auto_parallel/test_gpt_with_newir.py @@ -0,0 +1,55 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys +import tempfile +import unittest + + +class TestGPTNewIR(unittest.TestCase): + def test_gpt_newir(self): + file_dir = os.path.dirname(os.path.abspath(__file__)) + launch_model_path = os.path.join(file_dir, "gpt_with_newir.py") + + if os.environ.get("WITH_COVERAGE", "OFF") == "ON": + coverage_args = ["-m", "coverage", "run", "--branch", "-p"] + else: + coverage_args = [] + + tmp_dir = tempfile.TemporaryDirectory() + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--devices", + "0,1", + "--log_dir", + tmp_dir.name, + launch_model_path, + ] + ) + + process = subprocess.Popen(cmd) + process.wait() + self.assertEqual(process.returncode, 0) + + tmp_dir.cleanup() + + +if __name__ == "__main__": + unittest.main() From f6938c8edfa473ae9bc519a2a4462bcd7f864fdd Mon Sep 17 00:00:00 2001 From: zhaoyingli Date: Thu, 31 Aug 2023 19:30:31 +0800 Subject: [PATCH 02/12] rm new ir flag --- .../distributed/auto_parallel/static/engine.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/python/paddle/distributed/auto_parallel/static/engine.py b/python/paddle/distributed/auto_parallel/static/engine.py index f49e4fae8304f..0e3c316a5470c 100644 --- a/python/paddle/distributed/auto_parallel/static/engine.py +++ b/python/paddle/distributed/auto_parallel/static/engine.py @@ -827,13 +827,6 @@ def _initialize(self, mode): dist_main_program, self._place, dist_context ) - use_new_ir = False - if os.environ.get('FLAGS_enable_new_ir_in_executor') == "True": - use_new_ir = True - paddle.framework.set_flags( - {"FLAGS_enable_new_ir_in_executor": False} - ) - if self._executor is None: self._executor = paddle.static.Executor(self._place) uninitialized = [] @@ -861,11 +854,6 @@ def _initialize(self, mode): ] self._executor.run(dist_startup_prog) - if use_new_ir: - paddle.framework.set_flags( - {"FLAGS_enable_new_ir_in_executor": True} - ) - def fit( self, train_data, From 675605c6827b95b6eecffd50f01c886536e524a9 Mon Sep 17 00:00:00 2001 From: zhaoyingli Date: Fri, 1 Sep 2023 11:26:59 +0800 Subject: [PATCH 03/12] rm fluid api --- test/auto_parallel/gpt_with_newir.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/auto_parallel/gpt_with_newir.py b/test/auto_parallel/gpt_with_newir.py index 73fa1d619a55a..cbe3a248f3ed9 100644 --- a/test/auto_parallel/gpt_with_newir.py +++ b/test/auto_parallel/gpt_with_newir.py @@ -35,8 +35,8 @@ def apply_pass(): def reset_prog(): - paddle.fluid.framework.switch_main_program(paddle.static.Program()) - paddle.fluid.framework.switch_startup_program(paddle.static.Program()) + paddle.framework.switch_main_program(paddle.static.Program()) + paddle.framework.switch_startup_program(paddle.static.Program()) paddle.utils.unique_name.switch() @@ -54,7 +54,7 @@ def init(self, engine): np.random.seed(2021) random.seed(2021) paddle.distributed.fleet.init(is_collective=True) - place = paddle.fluid.CUDAPlace(ParallelEnv().dev_id) + place = paddle.CUDAPlace(ParallelEnv().dev_id) engine._executor = paddle.static.Executor(place) def get_engine(self, mode): From 18e2b8c7aa91ddb476c327cacca4652444e42b93 Mon Sep 17 00:00:00 2001 From: zhaoyingli Date: Fri, 1 Sep 2023 15:09:12 +0800 Subject: [PATCH 04/12] skip runing startup prog --- .../distributed/auto_parallel/static/engine.py | 13 +++++++++++++ .../distributed/auto_parallel/static/utils.py | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/python/paddle/distributed/auto_parallel/static/engine.py b/python/paddle/distributed/auto_parallel/static/engine.py index 2b66ac4e16a22..2428c5df13b9b 100644 --- a/python/paddle/distributed/auto_parallel/static/engine.py +++ b/python/paddle/distributed/auto_parallel/static/engine.py @@ -835,6 +835,14 @@ def _initialize(self, mode): dist_main_program, self._place, dist_context ) + # NOTE(zhaoyinglia): Skip startup program when use new ir temporarily. + use_new_ir = False + if auto_utils.use_new_ir(): + use_new_ir = True + paddle.framework.set_flags( + {"FLAGS_enable_new_ir_in_executor": False} + ) + if self._executor is None: self._executor = paddle.static.Executor(self._place) uninitialized = [] @@ -862,6 +870,11 @@ def _initialize(self, mode): ] self._executor.run(dist_startup_prog) + if use_new_ir: + paddle.framework.set_flags( + {"FLAGS_enable_new_ir_in_executor": True} + ) + def fit( self, train_data, diff --git a/python/paddle/distributed/auto_parallel/static/utils.py b/python/paddle/distributed/auto_parallel/static/utils.py index 3441914518822..d1906cfabb935 100644 --- a/python/paddle/distributed/auto_parallel/static/utils.py +++ b/python/paddle/distributed/auto_parallel/static/utils.py @@ -2423,6 +2423,19 @@ def use_new_executor(): ] +def use_new_ir(): + new_executor_micro_batching = os.environ.get( + 'FLAGS_enable_new_ir_in_executor', None + ) + return new_executor_micro_batching in [ + 1, + '1', + True, + 'True', + 'true', + ] + + def get_pp_stage(dist_context, rank): pp_idx = None for idx, process_mesh in enumerate(dist_context.process_meshes): From 5288d1695722059131fec92fc3bf0d097a322e33 Mon Sep 17 00:00:00 2001 From: zhaoyingli Date: Fri, 1 Sep 2023 15:32:51 +0800 Subject: [PATCH 05/12] update flag name --- python/paddle/distributed/auto_parallel/static/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/paddle/distributed/auto_parallel/static/utils.py b/python/paddle/distributed/auto_parallel/static/utils.py index d1906cfabb935..8ec0ba2e09f98 100644 --- a/python/paddle/distributed/auto_parallel/static/utils.py +++ b/python/paddle/distributed/auto_parallel/static/utils.py @@ -2424,10 +2424,10 @@ def use_new_executor(): def use_new_ir(): - new_executor_micro_batching = os.environ.get( + enable_new_ir_in_executor = os.environ.get( 'FLAGS_enable_new_ir_in_executor', None ) - return new_executor_micro_batching in [ + return enable_new_ir_in_executor in [ 1, '1', True, From cdb39ebad3a5fdf992a2cdce012ca7e93fb7ae4b Mon Sep 17 00:00:00 2001 From: zhaoyingli Date: Fri, 1 Sep 2023 15:39:02 +0800 Subject: [PATCH 06/12] update recv_v2 yaml --- paddle/fluid/ir/dialect/paddle_dialect/ir/pd_ops.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/ir/dialect/paddle_dialect/ir/pd_ops.yaml b/paddle/fluid/ir/dialect/paddle_dialect/ir/pd_ops.yaml index a5d2f42fc1ba1..da4c252af7217 100644 --- a/paddle/fluid/ir/dialect/paddle_dialect/ir/pd_ops.yaml +++ b/paddle/fluid/ir/dialect/paddle_dialect/ir/pd_ops.yaml @@ -92,7 +92,7 @@ output : Tensor(out) infer_meta: func: RecvV2InferMeta - param: [peer, dtype, out_shape] + param: [ring_id, dynamic_shape, peer, out_shape, dtype] kernel : func : recv_v2 param : [ring_id, dynamic_shape, peer, out_shape, dtype, use_calc_stream] From 8fd5aecc3db94c73826b4a4dc41bd54544888232 Mon Sep 17 00:00:00 2001 From: zhaoyingli Date: Sat, 2 Sep 2023 14:46:26 +0800 Subject: [PATCH 07/12] fix conflict --- .../ir/dialect/paddle_dialect/ir/pd_op.yaml | 486 ------------------ 1 file changed, 486 deletions(-) delete mode 100644 paddle/fluid/ir/dialect/paddle_dialect/ir/pd_op.yaml diff --git a/paddle/fluid/ir/dialect/paddle_dialect/ir/pd_op.yaml b/paddle/fluid/ir/dialect/paddle_dialect/ir/pd_op.yaml deleted file mode 100644 index efc84a66dcb98..0000000000000 --- a/paddle/fluid/ir/dialect/paddle_dialect/ir/pd_op.yaml +++ /dev/null @@ -1,486 +0,0 @@ -- name: assign_value - inputs: [] - attrs: - - {typename: 'int[]', name: shape} - - {typename: DataType, name: dtype} - - {typename: 'Scalar[]', name: values, data_type: 'std::vector'} - - {typename: Place, name: place, default_value: '{}'} - outputs: - - {typename: Tensor, name: out, optional: false, intermediate: false} - no_need_buffer: null - data_transform: null - infer_meta: - func: AssignValueInferMeta - param: [shape, dtype] - kernel: - func: [assign_value] - param: [shape, dtype, values] - backend: - ordered: true - candidates: [place] - layout: null - data_type: - ordered: false - candidates: [dtype] - to_complex_flag: [false] - dispatch: {assign_value: null} - force_backend: null - inplace: null - view: null - backward: null - -- name: feed - inputs: [] - attrs: - - {typename: str, name: name} - - {typename: int, name: col} - outputs: - - {typename: Tensor, name: out, optional: false, intermediate: false} - no_need_buffer: null - data_transform: null - inplace: null - backward: null - -- name: fetch - inputs: - - typename: Tensor - name: x - optional: false - no_need_buffer: false - data_transform: {} - attrs: - - {typename: str, name: name} - - {typename: int, name: col} - outputs: - - {typename: Tensor, name: out, optional: false, intermediate: false} - no_need_buffer: null - data_transform: null - infer_meta: - func: UnchangedInferMeta - param: [x] - kernel: - func: [fetch] - param: [x] - backend: null - layout: null - data_type: null - dispatch: {fetch: null} - force_backend: null - inplace: null - backward: null - -- name: save_combine - inputs: - - typename: Tensor[] - name: X - optional: false - no_need_buffer: false - data_transform: {} - attrs: - - {typename: str, name: file_path} - - {typename: bool, name: overwrite} - - {typename: bool, name: save_as_fp16} - - {typename: bool, name: save_to_memory} - outputs: - - {typename: Tensor, name: out, optional: true, intermediate: false} - no_need_buffer: null - data_transform: null - kernel: - func: [save_combine_tensor] - param: [X, file_path, overwrite, save_as_fp16, save_to_memory] - backend: null - layout: null - data_type: null - dispatch: {fetch: null} - force_backend: null - inplace: null - backward: null - -- name: load_combine - inputs: [] - attrs: - - {typename: str, name: file_path} - - {typename: bool, name: load_as_fp16} - - {typename: bool, name: model_from_memory} - outputs: - - {typename: 'Tensor[]', name: Out, optional: true, intermediate: false} - no_need_buffer: null - data_transform: null - kernel: - func: [load_combine] - param: [file_path, load_as_fp16, model_from_memory] - backend: null - layout: null - data_type: null - dispatch: {fetch: null} - force_backend: null - inplace: null - backward: null - -- name: share_buffer_ - inputs: - - typename: Tensor[] - name: x - optional: false - no_need_buffer: false - data_transform: {} - attrs: - - {typename: 'bool[]', name: share_dims_and_dtype, default_value: '{}'} - outputs: - - {typename: 'Tensor[]', name: out, size: x.size(), optional: false, intermediate: false} - - {typename: 'Tensor[]', name: xout, size: x.size(), optional: false, intermediate: false} - no_need_buffer: null - data_transform: null - inplace: null - backward: null - -- name: assert - inputs: - - typename: Tensor - name: cond - optional: false - no_need_buffer: false - data_transform: {} - - typename: Tensor[] - name: data - optional: false - no_need_buffer: false - data_transform: {} - attrs: - - {typename: 'int64_t', name: summarize, default_value: '-1'} - outputs: [] - no_need_buffer: null - data_transform: null - kernel: - func: [assert] - param: [cond, data, summarize] - backend: null - layout: null - data_type: - ordered: false - candidates: [cond] - to_complex_flag: [false] - dispatch: {assert: null} - force_backend: null - inplace: null - backward: null - -- name: print - inputs: - - typename: Tensor - name: in - optional: false - no_need_buffer: false - data_transform: {} - attrs: - - {typename: 'int', name: first_n} - - {typename: 'str', name: message} - - {typename: 'int', name: summarize} - - {typename: 'bool', name: print_tensor_name, default_value: 'true'} - - {typename: 'bool', name: print_tensor_type, default_value: 'true'} - - {typename: 'bool', name: print_tensor_shape, default_value: 'true'} - - {typename: 'bool', name: print_tensor_layout, default_value: 'true'} - - {typename: 'bool', name: print_tensor_lod, default_value: 'true'} - - {typename: 'str', name: print_phase, default_value: '"BOTH"'} - - {typename: 'bool', name: is_forward, default_value: 'true'} - outputs: - - typename: Tensor - name: out - optional: false - no_need_buffer: false - data_transform: {} - infer_meta: - func: UnchangedInferMeta - param: [in] - kernel: - func: [print_kernel] - param: [in, first_n, message, summarize, print_tensor_name, print_tensor_type, print_tensor_shape, print_tensor_layout, print_tensor_lod, print_phase, is_forward ] - backend: null - layout: null - data_type: null - dispatch: {print: null} - force_backend: null - no_need_buffer: null - data_transform: null - inplace: null - backward: null - -- name: add_n_ - inputs: - - typename: Tensor[] - name: inputs - optional: false - no_need_buffer: false - data_transform: {} - attrs: [] - outputs: - - {typename: Tensor, name: out, optional: false, intermediate: false} - no_need_buffer: null - data_transform: null - infer_meta: - func: AddNInferMeta - param: [inputs] - kernel: - func: [add_n] - param: [inputs] - backend: null - layout: null - data_type: null - dispatch: {fetch: null} - force_backend: null - backward: add_n_grad - -- name: add_n_with_kernel - inputs: - - typename: Tensor[] - name: inputs - optional: false - no_need_buffer: false - data_transform: {} - attrs: [] - outputs: - - {typename: Tensor, name: out, optional: false, intermediate: false} - no_need_buffer: null - data_transform: null - infer_meta: - func: AddNInferMeta - param: [inputs] - kernel: - func: [add_n] - param: [inputs] - backend: null - layout: null - data_type: null - dispatch: {fetch: null} - force_backend: null - backward: add_n_grad - -- name: write_to_array - inputs: - - typename: Tensor - name: i - optional: false - no_need_buffer: false - data_transform: {} - - typename: Tensor - name: x - optional: false - no_need_buffer: false - data_transform: {} - attrs: [] - outputs: - - {typename: 'Tensor[]', name: out, optional: false, intermediate: false} - no_need_buffer: null - data_transform: null - backward: write_to_array_grad - -- name: lod_array_length - inputs: - - typename: Tensor[] - name: x - optional: false - no_need_buffer: false - data_transform: {} - attrs: [] - outputs: - - {typename: 'Tensor', name: out, optional: false, intermediate: false} - no_need_buffer: null - data_transform: null - -- name: embedding_grad_sparse - inputs: - - typename: Tensor - name: x - optional: false - no_need_buffer: false - data_transform: {} - - typename: Tensor - name: weight - optional: false - no_need_buffer: false - data_transform: {} - - typename: Tensor - name: out_grad - optional: false - no_need_buffer: false - data_transform: {} - attrs: - - {typename: int64_t, name: padding_idx, default_value: '-1'} - - {typename: bool, name: sparse, default_value: 'false'} - outputs: - - {typename: SelectedRows, name: weight_grad, optional: false, intermediate: false} - no_need_buffer: null - data_transform: null - infer_meta: - func: EmbeddingGradSparseInferMeta - param: [weight] - kernel: - func: [embedding_sparse_grad] - param: [x, weight, out_grad, padding_idx, sparse] - backend: null - layout: null - data_type: - ordered: false - candidates: [weight] - to_complex_flag: [false] - dispatch: {embedding_sparse_grad: null} - force_backend: null - inplace: null - view: null - backward: null - -- name: shadow_feed - inputs: - - typename: Tensor - name: x - optional: false - no_need_buffer: false - data_transform: {} - attrs: [] - outputs: - - {typename: Tensor, name: out, optional: false, intermediate: false} - no_need_buffer: null - data_transform: null - infer_meta: - func: UnchangedInferMeta - param: [x] - kernel: - func: [shadow_feed] - param: [x] - backend: null - layout: null - data_type: null - dispatch: {fetch: null} - force_backend: null - inplace: null - backward: null - -- name: send_v2 - inputs: - - typename: Tensor - name: x - optional: false - no_need_buffer: false - data_transform: {} - attrs: - - {typename: int, name: ring_id, default_value: '0'} - - {typename: int, name: peer, default_value: '0'} - - {typename: bool, name: use_calc_stream, default_value: 'false'} - - {typename: bool, name: dynamic_shape, default_value: 'false'} - outputs: [] - no_need_buffer: null - data_transform: null - infer_meta: - func: SendV2InferMeta - param: [peer, ring_id] - kernel: - func: [send_v2] - param: [x, ring_id, dynamic_shape, peer, use_calc_stream] - backend: null - layout: null - data_type: null - dispatch: {send_v2: null} - force_backend: null - inplace: null - view: null - backward: null - -- name: recv_v2 - inputs: [] - attrs: - - {typename: 'int[]', name: out_shape, default_value: '{}'} - - {typename: DataType, name: dtype, default_value: 'DataType::FLOAT32'} - - {typename: int, name: peer, default_value: '0'} - - {typename: int, name: ring_id, default_value: '0'} - - {typename: bool, name: use_calc_stream, default_value: 'false'} - - {typename: bool, name: dynamic_shape, default_value: 'false'} - outputs: - - {typename: Tensor, name: out, optional: false, intermediate: false} - no_need_buffer: null - data_transform: null - infer_meta: - func: RecvV2InferMeta - param: [ring_id, dynamic_shape, peer, out_shape, dtype] - kernel: - func: [recv_v2] - param: [ring_id, dynamic_shape, peer, out_shape, dtype, use_calc_stream] - backend: null - layout: null - data_type: - ordered: false - candidates: [dtype] - to_complex_flag: [false] - dispatch: {recv_v2: null} - force_backend: null - inplace: null - view: null - backward: null - -- name : set_value - inputs: - - {typename: Tensor, name: x, optional: false, no_need_buffer: false, data_transform: {} } - attrs: - - {typename: 'int64_t[]', name: starts} - - {typename: 'int64_t[]', name: ends} - - {typename: 'int64_t[]', name: steps} - - {typename: 'int64_t[]', name: axes} - - {typename: 'int64_t[]', name: decrease_axes} - - {typename: 'int64_t[]', name: none_axes} - - {typename: 'int64_t[]', name: shape} - - {typename: 'Scalar[]', name: values} - outputs: - - {typename: Tensor, name: out, optional: false, intermediate: false} - infer_meta: - func: SetValueInferMeta - param: [x] - kernel: - func: [set_value] - param: [x, starts, ends, steps, axes, decrease_axes, none_axes, shape, values] - inplace: {out: x} - backward: set_value_grad - -- name : set_value_with_tensor - inputs: - - {typename: Tensor, name: x, optional: false, no_need_buffer: false, data_transform: {} } - - {typename: Tensor, name: values, optional: false, no_need_buffer: false, data_transform: {} } - attrs: - - {typename: 'int64_t[]', name: starts} - - {typename: 'int64_t[]', name: ends} - - {typename: 'int64_t[]', name: steps} - - {typename: 'int64_t[]', name: axes} - - {typename: 'int64_t[]', name: decrease_axes} - - {typename: 'int64_t[]', name: none_axes} - outputs: - - {typename: Tensor, name: out, optional: false, intermediate: false} - infer_meta: - func: SetValueInferMeta - param: [x] - kernel: - func: [set_value_with_tensor] - param: [x, values, starts, ends, steps, axes, decrease_axes, none_axes] - inplace: {out: x} - backward: set_value_grad - - -- name : set_value_grad - inputs: - - {typename: Tensor, name: out_grad, optional: false, no_need_buffer: false, data_transform: {} } - - {typename: Tensor, name: values, optional: false, no_need_buffer: false, data_transform: {} } - attrs: - - {typename: 'int64_t[]', name: starts} - - {typename: 'int64_t[]', name: ends} - - {typename: 'int64_t[]', name: steps} - - {typename: 'int64_t[]', name: axes} - - {typename: 'int64_t[]', name: decrease_axes} - - {typename: 'int64_t[]', name: none_axes} - outputs: - - {typename: Tensor, name: x_grad, optional: false, intermediate: false} - - {typename: Tensor, name: values_grad, optional: false, intermediate: false} - infer_meta: - func: SetValueGradInferMeta - param: [out_grad, values] - kernel: - func: [set_value_grad] - param: [out_grad, starts, ends, steps, axes, decrease_axes, none_axes] - inplace: null - backward: null From 70ec358393bd511a126216c9239fdd8ed826164b Mon Sep 17 00:00:00 2001 From: zhaoyingli Date: Mon, 4 Sep 2023 13:11:18 +0800 Subject: [PATCH 08/12] unittest only for pp --- test/auto_parallel/gpt_with_newir.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/auto_parallel/gpt_with_newir.py b/test/auto_parallel/gpt_with_newir.py index cbe3a248f3ed9..d3eaa51dd0988 100644 --- a/test/auto_parallel/gpt_with_newir.py +++ b/test/auto_parallel/gpt_with_newir.py @@ -62,7 +62,7 @@ def get_engine(self, mode): strategy = apply_pass() clip = paddle.nn.ClipGradByGlobalNorm(self.clip_norm) - opt = paddle.optimizer.AdamW(learning_rate=0.00001, grad_clip=clip) + opt = paddle.optimizer.AdamW(learning_rate=0.00001, grad_clip=None) model, loss = generate_model(mode) engine = auto.Engine(model, loss, opt, strategy=strategy) @@ -85,10 +85,10 @@ def test_pp_pass(self): os.environ['FLAGS_enable_new_ir_in_executor'] = 'True' # data parallel - engine_dp = self.get_engine("dp") - outs = engine_dp.fit( - self.dataset, 3, batch_size=self.batch_size, log_freq=1 - ) + # engine_dp = self.get_engine("dp") + # outs = engine_dp.fit( + # self.dataset, 3, batch_size=self.batch_size, log_freq=1 + # ) # navie pipeline parallel without schedule engine_pp = self.get_engine("pp") From c0c14ff9b756be14a416be06bdd31382ba96aaf3 Mon Sep 17 00:00:00 2001 From: zhaoyingli Date: Mon, 4 Sep 2023 14:54:46 +0800 Subject: [PATCH 09/12] fix cmakelist --- test/auto_parallel/CMakeLists.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/auto_parallel/CMakeLists.txt b/test/auto_parallel/CMakeLists.txt index e16387becf664..a47c5f11a6998 100644 --- a/test/auto_parallel/CMakeLists.txt +++ b/test/auto_parallel/CMakeLists.txt @@ -78,19 +78,19 @@ if(WITH_DISTRIBUTE AND WITH_GPU) PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 100) py_test_modules(test_pass_quantization MODULES test_pass_quantization) set_tests_properties(test_pass_quantization - PROPERTIES LABELS "RUN_TYPE=EXECLUSIVE" TIMEOUT 60) + PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 60) py_test_modules(test_reshard_s_to_r MODULES test_reshard_s_to_r) set_tests_properties(test_reshard_s_to_r - PROPERTIES LABELS "RUN_TYPE=EXECLUSIVE" TIMEOUT 100) + PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 100) py_test_modules(test_reshard_r_to_s MODULES test_reshard_r_to_s) set_tests_properties(test_reshard_r_to_s - PROPERTIES LABELS "RUN_TYPE=EXECLUSIVE" TIMEOUT 100) + PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 100) py_test_modules(test_reshard_r_to_p MODULES test_reshard_r_to_p) set_tests_properties(test_reshard_r_to_p - PROPERTIES LABELS "RUN_TYPE=EXECLUSIVE" TIMEOUT 100) + PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 100) py_test_modules(test_gpt_with_newir MODULES test_gpt_with_newir) set_tests_properties(test_gpt_with_newir - PROPERTIES LABELS "RUN_TYPE=EXECLUSIVE" TIMEOUT 100) + PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 100) # End of unittests WITH multi cards and timeout # NOTE(zyl): unittests WITH multi cards and WITHOUT timeout From fb38854619d482ca5bccd6085c20517c285828a5 Mon Sep 17 00:00:00 2001 From: zhaoyingli Date: Wed, 6 Sep 2023 11:29:49 +0800 Subject: [PATCH 10/12] unittest check precision --- test/auto_parallel/gpt_with_newir.py | 89 ++++++++++++++++++++++------ 1 file changed, 71 insertions(+), 18 deletions(-) diff --git a/test/auto_parallel/gpt_with_newir.py b/test/auto_parallel/gpt_with_newir.py index d3eaa51dd0988..e5ef86b4ae87f 100644 --- a/test/auto_parallel/gpt_with_newir.py +++ b/test/auto_parallel/gpt_with_newir.py @@ -30,7 +30,6 @@ def apply_pass(): strategy = auto.Strategy() strategy.auto_mode = "semi" strategy.reinit = True - return strategy @@ -40,14 +39,15 @@ def reset_prog(): paddle.utils.unique_name.switch() -class Test1F1BPass(unittest.TestCase): +class TestNewIR(unittest.TestCase): def setUp(self): - self.rtol = 1e-5 - self.atol = 1e-8 self.batch_size = 2 self.batch_num = 5 self.clip_norm = 0.2 self.dataset = FakeDataset(self.batch_size * self.batch_num) + os.environ['FLAGS_new_executor_micro_batching'] = 'True' + os.environ['FLAGS_embedding_deterministic'] = '1' + os.environ['FLAGS_cudnn_deterministic'] = '1' def init(self, engine): paddle.seed(2021) @@ -70,34 +70,87 @@ def get_engine(self, mode): return engine def check_results(self, ref_losses, check_losses): - np.testing.assert_allclose( + np.testing.assert_equal( ref_losses, check_losses, - rtol=self.rtol, - atol=self.atol, err_msg='pass {} has wrong results!, \nu={}\nv={}\ndiff={}'.format( __class__, ref_losses, check_losses, ref_losses - check_losses ), ) - def test_pp_pass(self): - os.environ['FLAGS_new_executor_micro_batching'] = 'True' + def test_dp(self): + os.environ['FLAGS_enable_new_ir_in_executor'] = 'False' + engine_dp_prog = self.get_engine("dp") + out_dp_prog = engine_dp_prog.fit( + self.dataset, 3, batch_size=self.batch_size, log_freq=1 + ) + os.environ['FLAGS_enable_new_ir_in_executor'] = 'True' + engine_dp_ir = self.get_engine("dp") + out_dp_ir = engine_dp_ir.fit( + self.dataset, 3, batch_size=self.batch_size, log_freq=1 + ) + + self.check_results( + out_dp_prog.history["loss"][0], out_dp_ir.history["loss"][0] + ) - # data parallel - # engine_dp = self.get_engine("dp") - # outs = engine_dp.fit( - # self.dataset, 3, batch_size=self.batch_size, log_freq=1 - # ) + def test_mp(self): + os.environ['FLAGS_enable_new_ir_in_executor'] = 'False' + engine_mp_prog = self.get_engine("mp") + out_mp_prog = engine_mp_prog.fit( + self.dataset, 3, batch_size=self.batch_size, log_freq=1 + ) + os.environ['FLAGS_enable_new_ir_in_executor'] = 'True' + engine_mp_ir = self.get_engine("mp") + out_mp_ir = engine_mp_ir.fit( + self.dataset, 3, batch_size=self.batch_size, log_freq=1 + ) + + self.check_results( + out_mp_prog.history["loss"][0], out_mp_ir.history["loss"][0] + ) + + def test_pp(self): # navie pipeline parallel without schedule - engine_pp = self.get_engine("pp") - outs = engine_pp.fit( + os.environ['FLAGS_enable_new_ir_in_executor'] = 'False' + engine_pp_prog = self.get_engine("pp") + out_pp_prog = engine_pp_prog.fit( self.dataset, 3, batch_size=self.batch_size, log_freq=1 ) - assert os.environ.get('FLAGS_new_executor_micro_batching') == "True" - assert os.environ.get('FLAGS_enable_new_ir_in_executor') == "True" + os.environ['FLAGS_enable_new_ir_in_executor'] = 'True' + # send_v2/recv_v2 dynamic_shape is True + engine_pp_ir = self.get_engine("pp") + out_pp_ir = engine_pp_ir.fit( + self.dataset, 3, batch_size=self.batch_size, log_freq=1 + ) + + if paddle.distributed.get_rank() == 1: + self.check_results( + out_pp_prog.history["loss"][0], out_pp_ir.history["loss"][0] + ) + + # send_v2/recv_v2 dynamic_shape is False + engine_pp_prog1 = self.get_engine("pp") + dataloader_pp_prog = engine_pp_prog1.dataloader( + self.dataset, + batch_size=self.batch_size, + sample_split=3, + mode="train", + ) + engine_pp_prog1.prepare(mode="train") + for op in engine_pp_prog1.main_program.global_block().ops: + if op.type in ["send_v2", "recv_v2"]: + op.desc._set_attr("dynamic_shape", False) + for data in dataloader_pp_prog: + out_pp_prog1 = engine_pp_prog1.run(data, mode="train") + + if paddle.distributed.get_rank() == 1: + self.check_results( + out_pp_prog1["loss"], out_pp_ir.history["loss"][0] + ) if __name__ == "__main__": From fef16f124ee017ff294effee4e5467d8eab08591 Mon Sep 17 00:00:00 2001 From: zhaoyingli Date: Wed, 6 Sep 2023 18:04:11 +0800 Subject: [PATCH 11/12] control random --- test/auto_parallel/gpt_with_newir.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/test/auto_parallel/gpt_with_newir.py b/test/auto_parallel/gpt_with_newir.py index e5ef86b4ae87f..4ddfd5a76ffe0 100644 --- a/test/auto_parallel/gpt_with_newir.py +++ b/test/auto_parallel/gpt_with_newir.py @@ -46,8 +46,8 @@ def setUp(self): self.clip_norm = 0.2 self.dataset = FakeDataset(self.batch_size * self.batch_num) os.environ['FLAGS_new_executor_micro_batching'] = 'True' - os.environ['FLAGS_embedding_deterministic'] = '1' - os.environ['FLAGS_cudnn_deterministic'] = '1' + paddle.set_flags({'FLAGS_embedding_deterministic': 1}) + paddle.set_flags({'FLAGS_cudnn_deterministic': 1}) def init(self, engine): paddle.seed(2021) @@ -78,14 +78,18 @@ def check_results(self, ref_losses, check_losses): ), ) + def enable_new_ir(self, flag): + paddle.set_flags({'FLAGS_enable_new_ir_in_executor': flag}) # for c++ + os.environ['FLAGS_enable_new_ir_in_executor'] = str(flag) # for python + def test_dp(self): - os.environ['FLAGS_enable_new_ir_in_executor'] = 'False' + self.enable_new_ir(False) engine_dp_prog = self.get_engine("dp") out_dp_prog = engine_dp_prog.fit( self.dataset, 3, batch_size=self.batch_size, log_freq=1 ) - os.environ['FLAGS_enable_new_ir_in_executor'] = 'True' + self.enable_new_ir(True) engine_dp_ir = self.get_engine("dp") out_dp_ir = engine_dp_ir.fit( self.dataset, 3, batch_size=self.batch_size, log_freq=1 @@ -96,13 +100,13 @@ def test_dp(self): ) def test_mp(self): - os.environ['FLAGS_enable_new_ir_in_executor'] = 'False' + self.enable_new_ir(False) engine_mp_prog = self.get_engine("mp") out_mp_prog = engine_mp_prog.fit( self.dataset, 3, batch_size=self.batch_size, log_freq=1 ) - os.environ['FLAGS_enable_new_ir_in_executor'] = 'True' + self.enable_new_ir(True) engine_mp_ir = self.get_engine("mp") out_mp_ir = engine_mp_ir.fit( self.dataset, 3, batch_size=self.batch_size, log_freq=1 @@ -114,13 +118,13 @@ def test_mp(self): def test_pp(self): # navie pipeline parallel without schedule - os.environ['FLAGS_enable_new_ir_in_executor'] = 'False' + self.enable_new_ir(False) engine_pp_prog = self.get_engine("pp") out_pp_prog = engine_pp_prog.fit( self.dataset, 3, batch_size=self.batch_size, log_freq=1 ) - os.environ['FLAGS_enable_new_ir_in_executor'] = 'True' + self.enable_new_ir(True) # send_v2/recv_v2 dynamic_shape is True engine_pp_ir = self.get_engine("pp") out_pp_ir = engine_pp_ir.fit( From 2b3ca733b0c2464e573eea76a7e46370941c403f Mon Sep 17 00:00:00 2001 From: zhaoyingli Date: Wed, 6 Sep 2023 21:17:14 +0800 Subject: [PATCH 12/12] fix cmakelist --- test/auto_parallel/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/auto_parallel/CMakeLists.txt b/test/auto_parallel/CMakeLists.txt index d7c40804834c7..3af649a809cca 100644 --- a/test/auto_parallel/CMakeLists.txt +++ b/test/auto_parallel/CMakeLists.txt @@ -87,11 +87,11 @@ if(WITH_DISTRIBUTE AND WITH_GPU) PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 100) py_test_modules(test_reshard_r_to_p MODULES test_reshard_r_to_p) set_tests_properties(test_reshard_r_to_p - PROPERTIES LABELS "RUN_TYPE=EXECLUSIVE" TIMEOUT 100) + PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 100) py_test_modules(test_semi_auto_parallel_basic MODULES test_semi_auto_parallel_basic) set_tests_properties(test_semi_auto_parallel_basic - PROPERTIES LABELS "RUN_TYPE=EXECLUSIVE" TIMEOUT 100) + PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 100) py_test_modules(test_gpt_with_newir MODULES test_gpt_with_newir) set_tests_properties(test_gpt_with_newir PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 100)