From 3a7843632b54036dbec79042d6def14d1b1b8bba Mon Sep 17 00:00:00 2001 From: PommesPeter <434596665@qq.com> Date: Thu, 10 Aug 2023 21:21:13 +0800 Subject: [PATCH 01/12] fix: updated code examples. --- .../auto_parallel/static/engine.py | 250 +++++++++--------- .../auto_parallel/static/process_mesh_v2.py | 13 +- .../distributed/auto_parallel/static/utils.py | 159 ++++++----- .../distributed/auto_parallel/strategy.py | 29 +- python/paddle/distributed/collective.py | 15 +- 5 files changed, 246 insertions(+), 220 deletions(-) diff --git a/python/paddle/distributed/auto_parallel/static/engine.py b/python/paddle/distributed/auto_parallel/static/engine.py index 853ff7d0aedbf..39c0b6e9c4ce9 100644 --- a/python/paddle/distributed/auto_parallel/static/engine.py +++ b/python/paddle/distributed/auto_parallel/static/engine.py @@ -79,39 +79,39 @@ class Engine: .. code-block:: python - import paddle - import paddle.vision.transforms as T - from paddle.distributed.fleet import auto - from paddle.vision.datasets import MNIST - - transform = T.Compose([ - T.Transpose(), - T.Normalize([127.5], [127.5]) - ]) - train_dataset = MNIST(mode='train', transform=transform) - valid_dataset = MNIST(mode='test', transform=transform) - - model = paddle.vision.models.LeNet() - loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Adam( - learning_rate=0.001, parameters=model.parameters()) - metrics = paddle.metric.Accuracy(topk=(1, 2)) - - engine = auto.Engine(model, loss, optimizer, metrics) - # fit - engine.fit(train_dataset, - epochs=2, - batch_size=64) - # evaluate - engine.evaluate(valid_dataset, - batch_size=64) - # predict - engine.predict(valid_dataset, - batch_size=64) - # save - engine.save("./my_model") - # load - engine.load("./my_model") + >>> import paddle + >>> import paddle.vision.transforms as T + >>> from paddle.distributed.fleet import auto + >>> from paddle.vision.datasets import MNIST + + >>> transform = T.Compose([ + ... T.Transpose(), + ... T.Normalize([127.5], [127.5]) + >>> ]) + >>> train_dataset = MNIST(mode='train', transform=transform) + >>> valid_dataset = MNIST(mode='test', transform=transform) + + >>> model = paddle.vision.models.LeNet() + >>> loss = paddle.nn.CrossEntropyLoss() + >>> optimizer = paddle.optimizer.Adam( + ... learning_rate=0.001, parameters=model.parameters()) + >>> metrics = paddle.metric.Accuracy(topk=(1, 2)) + + >>> engine = auto.Engine(model, loss, optimizer, metrics) + >>> # fit + >>> engine.fit(train_dataset, + ... epochs=2, + ... batch_size=64) + >>> # evaluate + >>> engine.evaluate(valid_dataset, + ... batch_size=64) + >>> # predict + >>> engine.predict(valid_dataset, + ... batch_size=64) + >>> # save + >>> engine.save("./my_model") + >>> # load + >>> engine.load("./my_model") """ @@ -916,27 +916,27 @@ def fit( .. code-block:: python - import paddle - import paddle.vision.transforms as T - from paddle.distributed.fleet import auto - from paddle.vision.datasets import MNIST - - transform = T.Compose([ - T.Transpose(), - T.Normalize([127.5], [127.5]) - ]) - train_dataset = MNIST(mode='train', transform=transform) - - model = paddle.vision.models.LeNet() - loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Adam( - learning_rate=0.001, parameters=model.parameters()) - metrics = paddle.metric.Accuracy(topk=(1, 2)) - - engine = auto.Engine(model, loss, optimizer, metrics) - engine.fit(train_dataset, - epochs=2, - batch_size=64) + >>> import paddle + >>> import paddle.vision.transforms as T + >>> from paddle.distributed.fleet import auto + >>> from paddle.vision.datasets import MNIST + + >>> transform = T.Compose([ + ... T.Transpose(), + ... T.Normalize([127.5], [127.5]) + >>> ]) + >>> train_dataset = MNIST(mode='train', transform=transform) + + >>> model = paddle.vision.models.LeNet() + >>> loss = paddle.nn.CrossEntropyLoss() + >>> optimizer = paddle.optimizer.Adam( + ... learning_rate=0.001, parameters=model.parameters()) + >>> metrics = paddle.metric.Accuracy(topk=(1, 2)) + + >>> engine = auto.Engine(model, loss, optimizer, metrics) + >>> engine.fit(train_dataset, + ... epochs=2, + ... batch_size=64) """ self._mode = 'train' self._inputs_spec, self._labels_spec = self._prepare_data_spec( @@ -1069,23 +1069,23 @@ def evaluate( .. code-block:: python - import paddle - import paddle.vision.transforms as T - from paddle.distributed.fleet import auto - from paddle.vision.datasets import MNIST + >>> import paddle + >>> import paddle.vision.transforms as T + >>> from paddle.distributed.fleet import auto + >>> from paddle.vision.datasets import MNIST - transform = T.Compose([ - T.Transpose(), - T.Normalize([127.5], [127.5]) - ]) - valid_dataset = MNIST(mode='test', transform=transform) + >>> transform = T.Compose([ + ... T.Transpose(), + ... T.Normalize([127.5], [127.5]) + >>> ]) + >>> valid_dataset = MNIST(mode='test', transform=transform) - model = paddle.vision.models.LeNet() - loss = paddle.nn.CrossEntropyLoss() - metrics = paddle.metric.Accuracy(topk=(1, 2)) + >>> model = paddle.vision.models.LeNet() + >>> loss = paddle.nn.CrossEntropyLoss() + >>> metrics = paddle.metric.Accuracy(topk=(1, 2)) - engine = auto.Engine(model, loss, metrics=metrics) - engine.evaluate(valid_dataset, batch_size=64) + >>> engine = auto.Engine(model, loss, metrics=metrics) + >>> engine.evaluate(valid_dataset, batch_size=64) """ self._mode = 'eval' @@ -1179,21 +1179,21 @@ def predict( .. code-block:: python - import paddle - import paddle.vision.transforms as T - from paddle.distributed.fleet import auto - from paddle.vision.datasets import MNIST + >>> import paddle + >>> import paddle.vision.transforms as T + >>> from paddle.distributed.fleet import auto + >>> from paddle.vision.datasets import MNIST - transform = T.Compose([ - T.Transpose(), - T.Normalize([127.5], [127.5]) - ]) - valid_dataset = MNIST(mode='test', transform=transform) + >>> transform = T.Compose([ + ... T.Transpose(), + ... T.Normalize([127.5], [127.5]) + >>> ]) + >>> valid_dataset = MNIST(mode='test', transform=transform) - model = paddle.vision.models.LeNet() + >>> model = paddle.vision.models.LeNet() - engine = auto.Engine(model) - engine.predict(valid_dataset, batch_size=64) + >>> engine = auto.Engine(model) + >>> engine.predict(valid_dataset, batch_size=64) """ self._mode = 'predict' self._inputs_spec, self._labels_spec = self._prepare_data_spec( @@ -1648,28 +1648,28 @@ def save(self, path, training=True): Examples: .. code-block:: python - import paddle - import paddle.vision.transforms as T - from paddle.distributed.fleet import auto - from paddle.vision.datasets import MNIST - - transform = T.Compose([ - T.Transpose(), - T.Normalize([127.5], [127.5]) - ]) - train_dataset = MNIST(mode='train', transform=transform) - - model = paddle.vision.models.LeNet() - loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Adam( - learning_rate=0.001, parameters=model.parameters()) - metrics = paddle.metric.Accuracy(topk=(1, 2)) - - engine = auto.Engine(model, loss, optimizer, metrics) - engine.fit(train_dataset, - epochs=1, - batch_size=64) - engine.save("./my_model") + >>> import paddle + >>> import paddle.vision.transforms as T + >>> from paddle.distributed.fleet import auto + >>> from paddle.vision.datasets import MNIST + + >>> transform = T.Compose([ + ... T.Transpose(), + ... T.Normalize([127.5], [127.5]) + >>> ]) + >>> train_dataset = MNIST(mode='train', transform=transform) + + >>> model = paddle.vision.models.LeNet() + >>> loss = paddle.nn.CrossEntropyLoss() + >>> optimizer = paddle.optimizer.Adam( + ... learning_rate=0.001, parameters=model.parameters()) + >>> metrics = paddle.metric.Accuracy(topk=(1, 2)) + + >>> engine = auto.Engine(model, loss, optimizer, metrics) + >>> engine.fit(train_dataset, + ... epochs=1, + ... batch_size=64) + >>> engine.save("./my_model") """ if training: @@ -1732,29 +1732,29 @@ def load(self, path, strict=True, load_optimizer=True): Examples: .. code-block:: python - import paddle - import paddle.vision.transforms as T - from paddle.distributed.fleet import auto - from paddle.vision.datasets import MNIST - - transform = T.Compose([ - T.Transpose(), - T.Normalize([127.5], [127.5]) - ]) - train_dataset = MNIST(mode='train', transform=transform) - - model = paddle.vision.models.LeNet() - loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Adam( - learning_rate=0.001, parameters=model.parameters()) - metrics = paddle.metric.Accuracy(topk=(1, 2)) - - engine = auto.Engine(model, loss, optimizer, metrics) - engine.fit(train_dataset, - epochs=1, - batch_size=64) - engine.save("./my_model") - engine.load("./my_model") + >>> import paddle + >>> import paddle.vision.transforms as T + >>> from paddle.distributed.fleet import auto + >>> from paddle.vision.datasets import MNIST + + >>> transform = T.Compose([ + ... T.Transpose(), + ... T.Normalize([127.5], [127.5]) + >>> ]) + >>> train_dataset = MNIST(mode='train', transform=transform) + + >>> model = paddle.vision.models.LeNet() + >>> loss = paddle.nn.CrossEntropyLoss() + >>> optimizer = paddle.optimizer.Adam( + ... learning_rate=0.001, parameters=model.parameters()) + >>> metrics = paddle.metric.Accuracy(topk=(1, 2)) + + >>> engine = auto.Engine(model, loss, optimizer, metrics) + >>> engine.fit(train_dataset, + ... epochs=1, + ... batch_size=64) + >>> engine.save("./my_model") + >>> engine.load("./my_model") """ self._strict = strict diff --git a/python/paddle/distributed/auto_parallel/static/process_mesh_v2.py b/python/paddle/distributed/auto_parallel/static/process_mesh_v2.py index 23fe66ab4bd28..d601af7c44c8b 100644 --- a/python/paddle/distributed/auto_parallel/static/process_mesh_v2.py +++ b/python/paddle/distributed/auto_parallel/static/process_mesh_v2.py @@ -33,14 +33,13 @@ class ProcessMesh(core.ProcessMesh): Examples: .. code-block:: python - import paddle - import paddle.distributed as dist + >>> import paddle + >>> import paddle.distributed as dist + >>> paddle.enable_static() - paddle.enable_static() - - mesh = dist.ProcessMesh([[2, 4, 5], [0, 1, 3]]) - assert mesh.shape == [2, 3] - assert mesh.processe_ids == [2, 4, 5, 0, 1, 3] + >>> mesh = dist.ProcessMesh([[2, 4, 5], [0, 1, 3]]) + >>> assert mesh.shape == [2, 3] + >>> assert mesh.processe_ids == [2, 4, 5, 0, 1, 3] """ diff --git a/python/paddle/distributed/auto_parallel/static/utils.py b/python/paddle/distributed/auto_parallel/static/utils.py index a5e19baf7cf12..7ade2f90c1c62 100644 --- a/python/paddle/distributed/auto_parallel/static/utils.py +++ b/python/paddle/distributed/auto_parallel/static/utils.py @@ -610,11 +610,13 @@ def save_distributed_checkpoint( Examples: .. code-block:: python + >>> import os + >>> from paddle.distributed.auto_parallel.static.utils import save_distributed_checkpoint - path = os.path.join("./output", "step_%d" % step) - os.makedirs(path, exist_ok=True) - add_info = {'batch': step, "batch_size": global_batch_size} - save_distributed_checkpoint(program, path, path, add_info) + >>> path = os.path.join("./output", "step_%d" % step) + >>> os.makedirs(path, exist_ok=True) + >>> add_info = {'batch': step, "batch_size": global_batch_size} + >>> save_distributed_checkpoint(program, path, path, add_info) """ from .dist_context import get_default_distributed_context @@ -653,11 +655,13 @@ def load_distributed_checkpoint(checkpoint_path, dist_attr_path): Examples: .. code-block:: python - ckpt_path = ['./model_state_rank0.pdmodel', - './model_state_rank1.pdmodel'] - dist_attr_path = ['./dist_attr_rank0.pdattr', - './dist_attr_rank1.pdattr'] - param_dict, dist_attr, add_info = load_distributed_checkpoint(ckpt_path, dist_attr_path) + >>> from paddle.distributed.auto_parallel.static.utils import load_distributed_checkpoint + + >>> ckpt_path = ['./model_state_rank0.pdmodel', + ... './model_state_rank1.pdmodel'] + >>> dist_attr_path = ['./dist_attr_rank0.pdattr', + ... './dist_attr_rank1.pdattr'] + >>> param_dict, dist_attr, add_info = load_distributed_checkpoint(ckpt_path, dist_attr_path) """ assert _check_valid_path( checkpoint_path @@ -692,12 +696,14 @@ def load_checkpoint_into_program( Examples: .. code-block:: python - exe.run(startup_program) - ckpt_path = ['./model_state_rank0.pdmodel', - './model_state_rank1.pdmodel'] - dist_attr_path = ['./dist_attr_rank0.pdattr', - './dist_attr_rank1.pdattr'] - load_checkpoint_into_program(ckpt_path, dist_attr_path, main_program) + >>> from paddle.distributed.auto_parallel.static.utils import load_checkpoint_into_program + + >>> exe.run(startup_program) + >>> ckpt_path = ['./model_state_rank0.pdmodel', + ... './model_state_rank1.pdmodel'] + >>> dist_attr_path = ['./dist_attr_rank0.pdattr', + ... './dist_attr_rank1.pdattr'] + >>> load_checkpoint_into_program(ckpt_path, dist_attr_path, main_program) """ from .dist_context import get_default_distributed_context @@ -999,13 +1005,17 @@ def _merge_parameter( Examples: .. code-block:: python - import numpy as np - partition_param_list = [(np.array([[[1.11, 1.12]]]), [[0,1],[0,1],[0,2]])] - param = np.array([[[1.13, 1.14]]]) - partition_index = [[0,1],[0,1],[2,4]] + >>> import numpy as np + >>> from paddle.distributed.auto_parallel.static.utils import _merge_parameter + + >>> partition_param_list = [(np.array([[[1.11, 1.12]]]), [[0,1],[0,1],[0,2]])] + >>> param = np.array([[[1.13, 1.14]]]) + >>> partition_index = [[0,1],[0,1],[2,4]] + + >>> _merge_parameter(partition_param_list, param, partition_index) + >>> print(partition_param_list) + [(array([[[1.11, 1.12, 1.13, 1.14]]]), [[0, 1],[0, 1],[0, 4]])] - _merge_parameter(partition_param_list, param, partition_index) - # partition_param_list: [(np.array([[[1.11, 1.12, 1.13, 1.14]]]), [[0,1],[0,1],[0,4]])] """ from .reshard import Resharder @@ -1061,16 +1071,20 @@ def _slice_parameter(complete_param, partition_index_list, length): Examples: .. code-block:: python - import numpy as np - complete_param = np.array([[[1.11, 1.12, 1.13, 1.14, 1.15, 1.16]]]) - rank = 2 - complete_shape = [1, 1, 6] - dims_mapping = [-1, -1, 0] - process_shape = [3] - process_group = [0, 1, 2] + >>> import numpy as np + >>> from paddle.distributed.auto_parallel.static.utils import _slice_parameter + + >>> complete_param = np.array([[[1.11, 1.12, 1.13, 1.14, 1.15, 1.16]]]) + >>> rank = 2 + >>> complete_shape = [1, 1, 6] + >>> dims_mapping = [-1, -1, 0] + >>> process_shape = [3] + >>> process_group = [0, 1, 2] + + >>> sliced_param_list = _slice_parameter(complete_param, [[], [], [2, 4]], 3) + >>> print(sliced_param_list) + [array([[[1.11, 1.12]]]), array([[[1.13, 1.14]]]), array([[[1.15, 1.16]]])] - sliced_param_list = _slice_parameter(complete_param, [[], [], [2, 4]], 3) - # [array([[[1.11, 1.12]]]), array([[[1.13, 1.14]]]), array([[[1.15, 1.16]]])] """ sliced_param_list = [] axis = len(complete_param.shape) - length @@ -1098,21 +1112,24 @@ def _get_sliced_param_index( Examples: .. code-block:: python - import numpy as np - complete_param = np.array([[[1.11, 1.12, 1.13, 1.14, 1.15, 1.16]]]) - rank = 2 - complete_shape = [1, 1, 6] - dims_mapping = [-1, -1, 0] - process_shape = [3] - process_group = [0, 1, 2] - - slice_param = _slice_parameter(complete_param, [[], [], [2, 4]], 3) - # slice_param: - # [array([[[1.11, 1.12]]]), array([[[1.13, 1.14]]]), array([[[1.15, 1.16]]])] - - index = _get_sliced_param_index(rank, complete_shape, dims_mapping - process_shape, process_group) - # index: 2 + >>> import numpy as np + >>> from paddle.distributed.auto_parallel.static.utils import _get_sliced_param_index + + >>> complete_param = np.array([[[1.11, 1.12, 1.13, 1.14, 1.15, 1.16]]]) + >>> rank = 2 + >>> complete_shape = [1, 1, 6] + >>> dims_mapping = [-1, -1, 0] + >>> process_shape = [3] + >>> process_group = [0, 1, 2] + + >>> slice_param = _slice_parameter(complete_param, [[], [], [2, 4]], 3) + >>> print(slice_param) + [array([[[1.11, 1.12]]]), array([[[1.13, 1.14]]]), array([[[1.15, 1.16]]])] + + >>> index = _get_sliced_param_index(rank, complete_shape, dims_mapping + ... process_shape, process_group) + >>> print(index) + 2 """ from .reshard import Resharder @@ -1145,15 +1162,18 @@ def _get_split_indices( Examples: .. code-block:: python - import numpy as np - complete_param = np.array([[[1.11, 1.12, 1.13, 1.14, 1.15, 1.16]]]) - complete_shape = [1, 1, 6] - dims_mapping = [-1, -1, 0] - process_shape = [3] - process_group = [0, 1, 2] + >>> import numpy as np + >>> from paddle.distributed.auto_parallel.static.utils import _get_split_indices - index = _get_split_indices(complete_shape, dims_mapping, process_shape, process_group) - # index: [[], [], [2, 4]] + >>> complete_param = np.array([[[1.11, 1.12, 1.13, 1.14, 1.15, 1.16]]]) + >>> complete_shape = [1, 1, 6] + >>> dims_mapping = [-1, -1, 0] + >>> process_shape = [3] + >>> process_group = [0, 1, 2] + + >>> index = _get_split_indices(complete_shape, dims_mapping, process_shape, process_group) + >>> print(index) + [[], [], [2, 4]] """ from .reshard import Resharder @@ -2410,19 +2430,26 @@ def wrap_data_for_completion( output_specs: list, DistTensorSpec for each output tensor of the dist_op attrs: dict, attribute map of the dist op - Usage: - op_desc = dist_op.serial_op.desc - input_name_list = [] - output_name_list = [] - input_name_list.append(op_desc.input('X')[0]) # 'X' is the arg name for op - input_name_list.append(op_desc.input('Y')[0]) - output_name_list.append(op_desc.output('Out')[0]) - attr_name_list = ['trans_x', 'trans_y'] - input_specs, output_specs, attrs = wrap_data_for_completion( - dist_op, - input_name_list, - output_name_list, - attr_name_list) + Examples: + .. code-block:: python + + >>> from paddle.distributed.auto_parallel.static import dist_op + >>> from paddle.distributed.auto_parallel.static.utils import wrap_data_for_completion + + >>> op_desc = dist_op.serial_op.desc + >>> input_name_list = [] + >>> output_name_list = [] + + >>> input_name_list.append(op_desc.input('X')[0]) # 'X' is the arg name for op + >>> input_name_list.append(op_desc.input('Y')[0]) + >>> output_name_list.append(op_desc.output('Out')[0]) + + >>> attr_name_list = ['trans_x', 'trans_y'] + >>> input_specs, output_specs, attrs = wrap_data_for_completion( + ... dist_op, + ... input_name_list, + ... output_name_list, + ... attr_name_list) """ diff --git a/python/paddle/distributed/auto_parallel/strategy.py b/python/paddle/distributed/auto_parallel/strategy.py index 8392ef7f37633..d5f9c5d56e521 100644 --- a/python/paddle/distributed/auto_parallel/strategy.py +++ b/python/paddle/distributed/auto_parallel/strategy.py @@ -152,20 +152,21 @@ class Strategy(BaseConfig): Examples: .. code-block:: python - import paddle - from paddle.distributed.fleet import auto - - strategy = auto.Strategy() - sharding = strategy.sharding - self.assertEqual(sharding.enabled, False) - self.assertEqual(sharding.stage, 1) - self.assertEqual(sharding.degree, 8) - sharding.enabled = True - sharding.stage = 2 - sharding.degree = 2 - self.assertEqual(sharding.enabled, True) - self.assertEqual(sharding.stage, 2) - self.assertEqual(sharding.degree, 2) + >>> import paddle + >>> from paddle.distributed.fleet import auto + + >>> strategy = auto.Strategy() + >>> sharding = strategy.sharding + >>> self.assertEqual(sharding.enabled, False) + >>> self.assertEqual(sharding.stage, 1) + >>> self.assertEqual(sharding.degree, 8) + + >>> sharding.enabled = True + >>> sharding.stage = 2 + >>> sharding.degree = 2 + >>> self.assertEqual(sharding.enabled, True) + >>> self.assertEqual(sharding.stage, 2) + >>> self.assertEqual(sharding.degree, 2) """ diff --git a/python/paddle/distributed/collective.py b/python/paddle/distributed/collective.py index a14d67429e917..ba40238aeca04 100644 --- a/python/paddle/distributed/collective.py +++ b/python/paddle/distributed/collective.py @@ -188,12 +188,12 @@ def new_group(ranks=None, backend=None, timeout=_default_timeout): Examples: .. code-block:: python - import paddle + >>> import paddle - paddle.distributed.init_parallel_env() - tindata = paddle.randn(shape=[2, 3]) - gp = paddle.distributed.new_group([2,4,6]) - paddle.distributed.all_reduce(tindata, group=gp, sync_op=False) + >>> paddle.distributed.init_parallel_env() + >>> tindata = paddle.randn(shape=[2, 3]) + >>> gp = paddle.distributed.new_group([2,4,6]) + >>> paddle.distributed.all_reduce(tindata, group=gp, sync_op=False) """ global _custom_gid @@ -310,9 +310,8 @@ def is_available(): Examples: .. code-block:: python - import paddle - - print(paddle.distributed.is_available()) + >>> import paddle + >>> print(paddle.distributed.is_available()) """ return core.is_compiled_with_dist() From f6b343bf825052b2f13fe7a755790e0529af6b2a Mon Sep 17 00:00:00 2001 From: PommesPeter <434596665@qq.com> Date: Thu, 10 Aug 2023 21:27:42 +0800 Subject: [PATCH 02/12] fix: added paddle.seed --- python/paddle/distributed/collective.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/paddle/distributed/collective.py b/python/paddle/distributed/collective.py index ba40238aeca04..24dff68cc744b 100644 --- a/python/paddle/distributed/collective.py +++ b/python/paddle/distributed/collective.py @@ -189,6 +189,7 @@ def new_group(ranks=None, backend=None, timeout=_default_timeout): .. code-block:: python >>> import paddle + >>> paddle.seed(2023) >>> paddle.distributed.init_parallel_env() >>> tindata = paddle.randn(shape=[2, 3]) From ebae2d07821d06f98da16110c7f91c1045bcdab1 Mon Sep 17 00:00:00 2001 From: PommesPeter <434596665@qq.com> Date: Fri, 11 Aug 2023 10:22:57 +0800 Subject: [PATCH 03/12] fix: updated code style --- python/paddle/distributed/auto_parallel/static/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/distributed/auto_parallel/static/utils.py b/python/paddle/distributed/auto_parallel/static/utils.py index 7ade2f90c1c62..1735726f35aab 100644 --- a/python/paddle/distributed/auto_parallel/static/utils.py +++ b/python/paddle/distributed/auto_parallel/static/utils.py @@ -697,7 +697,7 @@ def load_checkpoint_into_program( .. code-block:: python >>> from paddle.distributed.auto_parallel.static.utils import load_checkpoint_into_program - + >>> exe.run(startup_program) >>> ckpt_path = ['./model_state_rank0.pdmodel', ... './model_state_rank1.pdmodel'] From fca95c707b60a3061342eafe054342bb9cf94cdc Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Mon, 14 Aug 2023 22:58:09 +0800 Subject: [PATCH 04/12] Apply suggestions from code review --- .../auto_parallel/static/engine.py | 6 ++--- .../distributed/auto_parallel/static/utils.py | 24 ++++++++++++------- python/paddle/distributed/collective.py | 4 ++-- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/python/paddle/distributed/auto_parallel/static/engine.py b/python/paddle/distributed/auto_parallel/static/engine.py index 39c0b6e9c4ce9..0e8ddcfcb8a7e 100644 --- a/python/paddle/distributed/auto_parallel/static/engine.py +++ b/python/paddle/distributed/auto_parallel/static/engine.py @@ -100,14 +100,14 @@ class Engine: >>> engine = auto.Engine(model, loss, optimizer, metrics) >>> # fit >>> engine.fit(train_dataset, - ... epochs=2, - ... batch_size=64) + ... epochs=2, + ... batch_size=64) >>> # evaluate >>> engine.evaluate(valid_dataset, ... batch_size=64) >>> # predict >>> engine.predict(valid_dataset, - ... batch_size=64) + ... batch_size=64) >>> # save >>> engine.save("./my_model") >>> # load diff --git a/python/paddle/distributed/auto_parallel/static/utils.py b/python/paddle/distributed/auto_parallel/static/utils.py index 1735726f35aab..dc5e6146606e3 100644 --- a/python/paddle/distributed/auto_parallel/static/utils.py +++ b/python/paddle/distributed/auto_parallel/static/utils.py @@ -657,10 +657,14 @@ def load_distributed_checkpoint(checkpoint_path, dist_attr_path): >>> from paddle.distributed.auto_parallel.static.utils import load_distributed_checkpoint - >>> ckpt_path = ['./model_state_rank0.pdmodel', - ... './model_state_rank1.pdmodel'] - >>> dist_attr_path = ['./dist_attr_rank0.pdattr', - ... './dist_attr_rank1.pdattr'] + >>> ckpt_path = [ + ... './model_state_rank0.pdmodel', + ... './model_state_rank1.pdmodel', + ... ] + >>> dist_attr_path = [ + ... './dist_attr_rank0.pdattr', + ... './dist_attr_rank1.pdattr', + ... ] >>> param_dict, dist_attr, add_info = load_distributed_checkpoint(ckpt_path, dist_attr_path) """ assert _check_valid_path( @@ -699,10 +703,14 @@ def load_checkpoint_into_program( >>> from paddle.distributed.auto_parallel.static.utils import load_checkpoint_into_program >>> exe.run(startup_program) - >>> ckpt_path = ['./model_state_rank0.pdmodel', - ... './model_state_rank1.pdmodel'] - >>> dist_attr_path = ['./dist_attr_rank0.pdattr', - ... './dist_attr_rank1.pdattr'] + >>> ckpt_path = [ + ... './model_state_rank0.pdmodel', + ... './model_state_rank1.pdmodel', + ... ] + >>> dist_attr_path = [ + ... './dist_attr_rank0.pdattr', + ... './dist_attr_rank1.pdattr', + ... ] >>> load_checkpoint_into_program(ckpt_path, dist_attr_path, main_program) """ from .dist_context import get_default_distributed_context diff --git a/python/paddle/distributed/collective.py b/python/paddle/distributed/collective.py index 24dff68cc744b..e27557df6af9c 100644 --- a/python/paddle/distributed/collective.py +++ b/python/paddle/distributed/collective.py @@ -188,12 +188,12 @@ def new_group(ranks=None, backend=None, timeout=_default_timeout): Examples: .. code-block:: python + >>> # doctest: +REQUIRES(env: DISTRIBUTED) >>> import paddle - >>> paddle.seed(2023) >>> paddle.distributed.init_parallel_env() >>> tindata = paddle.randn(shape=[2, 3]) - >>> gp = paddle.distributed.new_group([2,4,6]) + >>> gp = paddle.distributed.new_group([2, 4, 6]) >>> paddle.distributed.all_reduce(tindata, group=gp, sync_op=False) """ From 9038d0b3ad48799662987a084030a79670a50966 Mon Sep 17 00:00:00 2001 From: PommesPeter <434596665@qq.com> Date: Tue, 15 Aug 2023 19:31:22 +0800 Subject: [PATCH 05/12] refactor: refine detail of code examples --- .../distributed/auto_parallel/static/engine.py | 1 + .../auto_parallel/static/process_mesh_v2.py | 1 - .../distributed/auto_parallel/static/utils.py | 17 +++++++++++++---- .../distributed/auto_parallel/strategy.py | 12 ++++++------ 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/python/paddle/distributed/auto_parallel/static/engine.py b/python/paddle/distributed/auto_parallel/static/engine.py index 39c0b6e9c4ce9..f991a4de3175c 100644 --- a/python/paddle/distributed/auto_parallel/static/engine.py +++ b/python/paddle/distributed/auto_parallel/static/engine.py @@ -1648,6 +1648,7 @@ def save(self, path, training=True): Examples: .. code-block:: python + >>> import paddle >>> import paddle.vision.transforms as T >>> from paddle.distributed.fleet import auto diff --git a/python/paddle/distributed/auto_parallel/static/process_mesh_v2.py b/python/paddle/distributed/auto_parallel/static/process_mesh_v2.py index d601af7c44c8b..79f75a82c32dd 100644 --- a/python/paddle/distributed/auto_parallel/static/process_mesh_v2.py +++ b/python/paddle/distributed/auto_parallel/static/process_mesh_v2.py @@ -39,7 +39,6 @@ class ProcessMesh(core.ProcessMesh): >>> mesh = dist.ProcessMesh([[2, 4, 5], [0, 1, 3]]) >>> assert mesh.shape == [2, 3] - >>> assert mesh.processe_ids == [2, 4, 5, 0, 1, 3] """ diff --git a/python/paddle/distributed/auto_parallel/static/utils.py b/python/paddle/distributed/auto_parallel/static/utils.py index 1735726f35aab..d8209c7b354b2 100644 --- a/python/paddle/distributed/auto_parallel/static/utils.py +++ b/python/paddle/distributed/auto_parallel/static/utils.py @@ -610,13 +610,19 @@ def save_distributed_checkpoint( Examples: .. code-block:: python + >>> import os >>> from paddle.distributed.auto_parallel.static.utils import save_distributed_checkpoint + >>> step = 16000 + >>> global_batch_size = 32 >>> path = os.path.join("./output", "step_%d" % step) >>> os.makedirs(path, exist_ok=True) + >>> program = fluid.Program() + >>> add_info = {'batch': step, "batch_size": global_batch_size} >>> save_distributed_checkpoint(program, path, path, add_info) + """ from .dist_context import get_default_distributed_context @@ -655,6 +661,7 @@ def load_distributed_checkpoint(checkpoint_path, dist_attr_path): Examples: .. code-block:: python + >>> # doctest: +SKIP('Depends on external files.') >>> from paddle.distributed.auto_parallel.static.utils import load_distributed_checkpoint >>> ckpt_path = ['./model_state_rank0.pdmodel', @@ -696,6 +703,7 @@ def load_checkpoint_into_program( Examples: .. code-block:: python + >>> # doctest: +SKIP('Depends on external files.') >>> from paddle.distributed.auto_parallel.static.utils import load_checkpoint_into_program >>> exe.run(startup_program) @@ -1008,11 +1016,12 @@ def _merge_parameter( >>> import numpy as np >>> from paddle.distributed.auto_parallel.static.utils import _merge_parameter - >>> partition_param_list = [(np.array([[[1.11, 1.12]]]), [[0,1],[0,1],[0,2]])] + >>> partition_param_list = [(np.array([[[1.11, 1.12]]]), [[0, 1],[0, 1],[0, 2]])] >>> param = np.array([[[1.13, 1.14]]]) - >>> partition_index = [[0,1],[0,1],[2,4]] + >>> partition_index = [[0, 1],[0, 1],[2, 4]] + >>> complete_shape = [2, 2, 4] - >>> _merge_parameter(partition_param_list, param, partition_index) + >>> _merge_parameter(partition_param_list, param, partition_index, complete_shape) >>> print(partition_param_list) [(array([[[1.11, 1.12, 1.13, 1.14]]]), [[0, 1],[0, 1],[0, 4]])] @@ -1126,7 +1135,7 @@ def _get_sliced_param_index( >>> print(slice_param) [array([[[1.11, 1.12]]]), array([[[1.13, 1.14]]]), array([[[1.15, 1.16]]])] - >>> index = _get_sliced_param_index(rank, complete_shape, dims_mapping + >>> index = _get_sliced_param_index(rank, complete_shape, dims_mapping, ... process_shape, process_group) >>> print(index) 2 diff --git a/python/paddle/distributed/auto_parallel/strategy.py b/python/paddle/distributed/auto_parallel/strategy.py index d5f9c5d56e521..5e0e0a9822956 100644 --- a/python/paddle/distributed/auto_parallel/strategy.py +++ b/python/paddle/distributed/auto_parallel/strategy.py @@ -157,16 +157,16 @@ class Strategy(BaseConfig): >>> strategy = auto.Strategy() >>> sharding = strategy.sharding - >>> self.assertEqual(sharding.enabled, False) - >>> self.assertEqual(sharding.stage, 1) - >>> self.assertEqual(sharding.degree, 8) + >>> assert sharding.enabled == False + >>> assert sharding.stage == 1 + >>> assert sharding.degree == 8 >>> sharding.enabled = True >>> sharding.stage = 2 >>> sharding.degree = 2 - >>> self.assertEqual(sharding.enabled, True) - >>> self.assertEqual(sharding.stage, 2) - >>> self.assertEqual(sharding.degree, 2) + >>> assert sharding.enabled == True + >>> assert sharding.stage == 2 + >>> assert sharding.degree == 2 """ From 48f25779087f5839f7eca145d37d9e26a7a2307c Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Wed, 16 Aug 2023 01:42:26 +0800 Subject: [PATCH 06/12] Update python/paddle/distributed/auto_parallel/static/process_mesh_v2.py --- .../paddle/distributed/auto_parallel/static/process_mesh_v2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/paddle/distributed/auto_parallel/static/process_mesh_v2.py b/python/paddle/distributed/auto_parallel/static/process_mesh_v2.py index 79f75a82c32dd..76b3ead1e3bdd 100644 --- a/python/paddle/distributed/auto_parallel/static/process_mesh_v2.py +++ b/python/paddle/distributed/auto_parallel/static/process_mesh_v2.py @@ -39,6 +39,7 @@ class ProcessMesh(core.ProcessMesh): >>> mesh = dist.ProcessMesh([[2, 4, 5], [0, 1, 3]]) >>> assert mesh.shape == [2, 3] + >>> assert mesh.process_ids == [2, 4, 5, 0, 1, 3] """ From 5b950641d26d7117fe0e602b30dfeac986ae2208 Mon Sep 17 00:00:00 2001 From: PommesPeter <434596665@qq.com> Date: Wed, 16 Aug 2023 13:55:06 +0800 Subject: [PATCH 07/12] fix: refine detail --- python/paddle/distributed/auto_parallel/static/engine.py | 1 + python/paddle/distributed/auto_parallel/strategy.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/python/paddle/distributed/auto_parallel/static/engine.py b/python/paddle/distributed/auto_parallel/static/engine.py index 1d21de1e41658..1bae7ee701993 100644 --- a/python/paddle/distributed/auto_parallel/static/engine.py +++ b/python/paddle/distributed/auto_parallel/static/engine.py @@ -1733,6 +1733,7 @@ def load(self, path, strict=True, load_optimizer=True): Examples: .. code-block:: python + >>> import paddle >>> import paddle.vision.transforms as T >>> from paddle.distributed.fleet import auto diff --git a/python/paddle/distributed/auto_parallel/strategy.py b/python/paddle/distributed/auto_parallel/strategy.py index 5e0e0a9822956..7d84a45430dc5 100644 --- a/python/paddle/distributed/auto_parallel/strategy.py +++ b/python/paddle/distributed/auto_parallel/strategy.py @@ -164,7 +164,7 @@ class Strategy(BaseConfig): >>> sharding.enabled = True >>> sharding.stage = 2 >>> sharding.degree = 2 - >>> assert sharding.enabled == True + >>> assert sharding.enable == True >>> assert sharding.stage == 2 >>> assert sharding.degree == 2 From 3629722b0456d9519bda0a1812cb271876fc9597 Mon Sep 17 00:00:00 2001 From: PommesPeter <434596665@qq.com> Date: Wed, 16 Aug 2023 14:53:05 +0800 Subject: [PATCH 08/12] fix: refine detail --- python/paddle/distributed/auto_parallel/strategy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/paddle/distributed/auto_parallel/strategy.py b/python/paddle/distributed/auto_parallel/strategy.py index 7d84a45430dc5..36607f0903bc7 100644 --- a/python/paddle/distributed/auto_parallel/strategy.py +++ b/python/paddle/distributed/auto_parallel/strategy.py @@ -157,11 +157,11 @@ class Strategy(BaseConfig): >>> strategy = auto.Strategy() >>> sharding = strategy.sharding - >>> assert sharding.enabled == False + >>> assert sharding.enable == False >>> assert sharding.stage == 1 >>> assert sharding.degree == 8 - >>> sharding.enabled = True + >>> sharding.enable = True >>> sharding.stage = 2 >>> sharding.degree = 2 >>> assert sharding.enable == True From 0360a8cba5de0db79f5c259e1a4d4cd2f1cb0e62 Mon Sep 17 00:00:00 2001 From: PommesPeter <54879512+PommesPeter@users.noreply.github.com> Date: Thu, 17 Aug 2023 10:29:27 +0800 Subject: [PATCH 09/12] Update python/paddle/distributed/auto_parallel/static/process_mesh_v2.py Co-authored-by: Nyakku Shigure --- .../paddle/distributed/auto_parallel/static/process_mesh_v2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/paddle/distributed/auto_parallel/static/process_mesh_v2.py b/python/paddle/distributed/auto_parallel/static/process_mesh_v2.py index 76b3ead1e3bdd..be621d52b0891 100644 --- a/python/paddle/distributed/auto_parallel/static/process_mesh_v2.py +++ b/python/paddle/distributed/auto_parallel/static/process_mesh_v2.py @@ -40,6 +40,7 @@ class ProcessMesh(core.ProcessMesh): >>> mesh = dist.ProcessMesh([[2, 4, 5], [0, 1, 3]]) >>> assert mesh.shape == [2, 3] >>> assert mesh.process_ids == [2, 4, 5, 0, 1, 3] + >>> assert mesh.process_ids == [2, 4, 5, 0, 1, 3] """ From 5ea2e2ef55d56e87a366c8ebdc0311be98d1ff55 Mon Sep 17 00:00:00 2001 From: PommesPeter <434596665@qq.com> Date: Thu, 17 Aug 2023 10:34:25 +0800 Subject: [PATCH 10/12] refactor: refine detail --- python/paddle/distributed/auto_parallel/static/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/distributed/auto_parallel/static/utils.py b/python/paddle/distributed/auto_parallel/static/utils.py index bf127b9314334..d6fea1d2c910e 100644 --- a/python/paddle/distributed/auto_parallel/static/utils.py +++ b/python/paddle/distributed/auto_parallel/static/utils.py @@ -2450,7 +2450,7 @@ def wrap_data_for_completion( Examples: .. code-block:: python - >>> from paddle.distributed.auto_parallel.static import dist_op + >>> # doctest: +SKIP('Depends on other ops.') >>> from paddle.distributed.auto_parallel.static.utils import wrap_data_for_completion >>> op_desc = dist_op.serial_op.desc From 3ce6fffe568955e633eff1966abb9134fd8437d6 Mon Sep 17 00:00:00 2001 From: PommesPeter <434596665@qq.com> Date: Thu, 17 Aug 2023 10:47:16 +0800 Subject: [PATCH 11/12] refactor: refine detail --- .../paddle/distributed/auto_parallel/static/process_mesh_v2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/paddle/distributed/auto_parallel/static/process_mesh_v2.py b/python/paddle/distributed/auto_parallel/static/process_mesh_v2.py index be621d52b0891..76b3ead1e3bdd 100644 --- a/python/paddle/distributed/auto_parallel/static/process_mesh_v2.py +++ b/python/paddle/distributed/auto_parallel/static/process_mesh_v2.py @@ -40,7 +40,6 @@ class ProcessMesh(core.ProcessMesh): >>> mesh = dist.ProcessMesh([[2, 4, 5], [0, 1, 3]]) >>> assert mesh.shape == [2, 3] >>> assert mesh.process_ids == [2, 4, 5, 0, 1, 3] - >>> assert mesh.process_ids == [2, 4, 5, 0, 1, 3] """ From 09d384d5043b94b723f84ed3e7f9353b0735a226 Mon Sep 17 00:00:00 2001 From: PommesPeter <434596665@qq.com> Date: Tue, 22 Aug 2023 14:53:04 +0800 Subject: [PATCH 12/12] fix: refine doc --- python/paddle/distributed/auto_parallel/static/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/distributed/auto_parallel/static/utils.py b/python/paddle/distributed/auto_parallel/static/utils.py index d6fea1d2c910e..80aac2e257530 100644 --- a/python/paddle/distributed/auto_parallel/static/utils.py +++ b/python/paddle/distributed/auto_parallel/static/utils.py @@ -618,7 +618,7 @@ def save_distributed_checkpoint( >>> global_batch_size = 32 >>> path = os.path.join("./output", "step_%d" % step) >>> os.makedirs(path, exist_ok=True) - >>> program = fluid.Program() + >>> program = paddle.static.Program() >>> add_info = {'batch': step, "batch_size": global_batch_size} >>> save_distributed_checkpoint(program, path, path, add_info)