From d3bdbdcef03e17e8793fe7657fa95707a0fdbc5b Mon Sep 17 00:00:00 2001 From: jim19930609 Date: Wed, 17 Nov 2021 11:40:56 +0000 Subject: [PATCH 1/9] [Second Patch PR #1]Eager Dygraph Forward and Autograd --- paddle/fluid/eager/CMakeLists.txt | 6 + paddle/fluid/eager/api/CMakeLists.txt | 4 + paddle/fluid/eager/api/all.cc | 18 ++ paddle/fluid/eager/api/all.h | 19 ++ .../fluid/eager/api/generated/CMakeLists.txt | 1 + .../generated/eager_generated/CMakeLists.txt | 2 + .../eager_generated/backwards/CMakeLists.txt | 1 + .../eager_generated/backwards/scale_node.cc | 67 ++++ .../eager_generated/backwards/scale_node.h | 51 +++ .../eager_generated/forwards/CMakeLists.txt | 2 + .../eager_generated/forwards/function_api.cc | 278 ++++++++++++++++ .../eager_generated/forwards/function_api.h | 34 ++ .../eager_generated/forwards/scale.cc | 100 ++++++ .../eager_generated/forwards/scale.h | 23 ++ paddle/fluid/eager/api/utils/CMakeLists.txt | 1 + paddle/fluid/eager/api/utils/global_utils.cc | 18 ++ paddle/fluid/eager/api/utils/global_utils.h | 58 ++++ paddle/fluid/eager/tests/CMakeLists.txt | 2 + .../eager/tests/forward_autograd_test.cc | 302 +++++++++++++++++ paddle/fluid/eager/tests/forward_test.cc | 304 ++++++++++++++++++ 20 files changed, 1291 insertions(+) create mode 100644 paddle/fluid/eager/CMakeLists.txt create mode 100644 paddle/fluid/eager/api/CMakeLists.txt create mode 100644 paddle/fluid/eager/api/all.cc create mode 100644 paddle/fluid/eager/api/all.h create mode 100644 paddle/fluid/eager/api/generated/CMakeLists.txt create mode 100644 paddle/fluid/eager/api/generated/eager_generated/CMakeLists.txt create mode 100644 paddle/fluid/eager/api/generated/eager_generated/backwards/CMakeLists.txt create mode 100644 paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.cc create mode 100644 paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h create mode 100644 paddle/fluid/eager/api/generated/eager_generated/forwards/CMakeLists.txt create mode 100644 paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.cc create mode 100644 paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.h create mode 100644 paddle/fluid/eager/api/generated/eager_generated/forwards/scale.cc create mode 100644 paddle/fluid/eager/api/generated/eager_generated/forwards/scale.h create mode 100644 paddle/fluid/eager/api/utils/CMakeLists.txt create mode 100644 paddle/fluid/eager/api/utils/global_utils.cc create mode 100644 paddle/fluid/eager/api/utils/global_utils.h create mode 100644 paddle/fluid/eager/tests/CMakeLists.txt create mode 100644 paddle/fluid/eager/tests/forward_autograd_test.cc create mode 100644 paddle/fluid/eager/tests/forward_test.cc diff --git a/paddle/fluid/eager/CMakeLists.txt b/paddle/fluid/eager/CMakeLists.txt new file mode 100644 index 0000000000000..3a0f7dc95f2a5 --- /dev/null +++ b/paddle/fluid/eager/CMakeLists.txt @@ -0,0 +1,6 @@ +add_subdirectory(api) + +set(eager_deps eager_api utils function_api autograd_meta grad_node_info pten pten_api) +set(fluid_deps tracer layer proto_desc operator op_registry variable_helper memcpy) + +add_subdirectory(tests) diff --git a/paddle/fluid/eager/api/CMakeLists.txt b/paddle/fluid/eager/api/CMakeLists.txt new file mode 100644 index 0000000000000..fee7f27b2657e --- /dev/null +++ b/paddle/fluid/eager/api/CMakeLists.txt @@ -0,0 +1,4 @@ +add_subdirectory(utils) +add_subdirectory(generated) + +cc_library(eager_api SRCS all.cc DEPS global_utils function_api eager_scale) diff --git a/paddle/fluid/eager/api/all.cc b/paddle/fluid/eager/api/all.cc new file mode 100644 index 0000000000000..2308e9341772c --- /dev/null +++ b/paddle/fluid/eager/api/all.cc @@ -0,0 +1,18 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include "paddle/fluid/eager/api/all.h" + +namespace egr {} // namespace egr diff --git a/paddle/fluid/eager/api/all.h b/paddle/fluid/eager/api/all.h new file mode 100644 index 0000000000000..2f95c1aff6c6a --- /dev/null +++ b/paddle/fluid/eager/api/all.h @@ -0,0 +1,19 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#pragma once + +#include "paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.h" +#include "paddle/fluid/eager/api/generated/eager_generated/forwards/scale.h" +#include "paddle/fluid/eager/api/utils/global_utils.h" diff --git a/paddle/fluid/eager/api/generated/CMakeLists.txt b/paddle/fluid/eager/api/generated/CMakeLists.txt new file mode 100644 index 0000000000000..41fadef153bdd --- /dev/null +++ b/paddle/fluid/eager/api/generated/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(eager_generated) diff --git a/paddle/fluid/eager/api/generated/eager_generated/CMakeLists.txt b/paddle/fluid/eager/api/generated/eager_generated/CMakeLists.txt new file mode 100644 index 0000000000000..15871597cd976 --- /dev/null +++ b/paddle/fluid/eager/api/generated/eager_generated/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(backwards) +add_subdirectory(forwards) diff --git a/paddle/fluid/eager/api/generated/eager_generated/backwards/CMakeLists.txt b/paddle/fluid/eager/api/generated/eager_generated/backwards/CMakeLists.txt new file mode 100644 index 0000000000000..5fa414819f343 --- /dev/null +++ b/paddle/fluid/eager/api/generated/eager_generated/backwards/CMakeLists.txt @@ -0,0 +1 @@ +cc_library(scale_node SRCS scale_node.cc DEPS function_api pten pten_api grad_node_info) diff --git a/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.cc b/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.cc new file mode 100644 index 0000000000000..3994e9ddf8617 --- /dev/null +++ b/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.cc @@ -0,0 +1,67 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h" +#include "paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.h" + +#include "paddle/fluid/eager/eager_tensor.h" +#include "paddle/fluid/platform/device_context.h" +#include "paddle/pten/api/all.h" + +#include "paddle/fluid/platform/enforce.h" +#include "paddle/fluid/platform/errors.h" + +#include "glog/logging.h" + +namespace egr { + +void GradNodeScale::SetTensorWrappers_X( + const std::vector& tensors) { + // Does nothing for scale +} + +void GradNodeScale::SetAttributes_scale(float scale) { scale_ = scale; } + +std::vector> GradNodeScale::operator()( + const std::vector>& grads) { + // 1. Check Output Size + PADDLE_ENFORCE(((grads.size() == 1) && (grads[0].size() == 1)), + paddle::platform::errors::Fatal( + "ScaleGradNode should take exactly 1 grad tensor" + "However received: %d", + grads.size())); + std::vector> outs; + // 2. Create needed out parttern + egr::EagerTensor out; + // Apply Gradient Hooks + if (GradientHooksRegistered()) { + // TODO(jiabin): Shall we apply hook slot by slot here or accept + // vector> to apply all hooks? + std::vector> hooked_grads = + ApplyGradientHooks(grads); + ScaleAPI(/* slot by slot set */ hooked_grads[0][0], scale_, 0.0 /* bias */, + true /* bias_after_scale */, &out); + } else { + ScaleAPI(grads[0][0], scale_, 0.0 /* bias */, true /* bias_after_scale */, + &out); + } + + // Apply Reduce Hooks + if (ReduceHooksRegistered()) { + ApplyReduceHooks(); + } + return {{out}}; +} + +} // namespace egr diff --git a/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h b/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h new file mode 100644 index 0000000000000..8881be31df79d --- /dev/null +++ b/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h @@ -0,0 +1,51 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/fluid/eager/grad_node_info.h" +#include "paddle/fluid/eager/tensor_wrapper.h" + +/* + Each Operation has a specific GradNode inheritted from GradNodeBase + A specific GradNode defines + 1. Input Tensors + 2. overrides operator() to perform actual backward computations + + TODO: Generate GradNode via auto-code-generation +*/ +namespace egr { + +class GradNodeScale : public GradNodeBase { + public: + // Constructor: configure fwd input tensors to grad node + GradNodeScale(size_t bwd_in_slot_num, size_t bwd_out_slot_num) + : GradNodeBase(bwd_in_slot_num, bwd_out_slot_num) {} + ~GradNodeScale() override = default; + + // Functor: perform backward computations + virtual std::vector> operator()( + const std::vector>& grads) override; + + void SetTensorWrappers_X(const std::vector& tensors); + + void SetAttributes_scale(float scale); + + // Members: define fwd input tensors + // For Scale there is no fwd input tensor needed + private: + float scale_{1.0}; +}; + +} // namespace egr diff --git a/paddle/fluid/eager/api/generated/eager_generated/forwards/CMakeLists.txt b/paddle/fluid/eager/api/generated/eager_generated/forwards/CMakeLists.txt new file mode 100644 index 0000000000000..02837d20327a2 --- /dev/null +++ b/paddle/fluid/eager/api/generated/eager_generated/forwards/CMakeLists.txt @@ -0,0 +1,2 @@ +cc_library(function_api SRCS function_api.cc DEPS pten pten_api layer proto_desc operator op_registry variable_helper memcpy scale_op) +cc_library(eager_scale SRCS scale.cc DEPS function_api pten autograd_meta scale_node) diff --git a/paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.cc b/paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.cc new file mode 100644 index 0000000000000..bc0969a10af5d --- /dev/null +++ b/paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.cc @@ -0,0 +1,278 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.h" +#include "paddle/fluid/eager/api/utils/global_utils.h" + +#include "paddle/pten/api/all.h" +#include "paddle/pten/core/dense_tensor.h" +#include "paddle/pten/include/core.h" + +#include "paddle/fluid/memory/memcpy.h" +#include "paddle/fluid/operators/math/math_function.h" +#include "paddle/fluid/platform/device_context.h" + +namespace egr { + +template +static void ScaleDeviceDispatch(const pten::DenseTensor& dense_tensor, + const DeviceContext& dev_ctx, float scale, + float bias, bool bias_after_scale, + pten::DenseTensor* dense_out) { + switch (dense_tensor.data_type()) { + case pten::DataType::FLOAT64: { + pten::Scale(dev_ctx, dense_tensor /* tensor */, scale /* scale */, + bias /* bias */, + bias_after_scale /* bias_after_scale */, + dense_out /* out tensor */); + break; + } + case pten::DataType::FLOAT32: { + pten::Scale(dev_ctx, dense_tensor /* tensor */, scale /* scale */, + bias /* bias */, + bias_after_scale /* bias_after_scale */, + dense_out /* out tensor */); + break; + } + case pten::DataType::INT64: { + pten::Scale(dev_ctx, dense_tensor /* tensor */, + scale /* scale */, bias /* bias */, + bias_after_scale /* bias_after_scale */, + dense_out /* out tensor */); + break; + } + case pten::DataType::INT32: { + pten::Scale(dev_ctx, dense_tensor /* tensor */, + scale /* scale */, bias /* bias */, + bias_after_scale /* bias_after_scale */, + dense_out /* out tensor */); + break; + } + default: { + PADDLE_THROW(paddle::platform::errors::Fatal("Unsupported data type")); + break; + } + } +} + +// TODO(jiabin): This may have serious performance issue move code from +// gradient_accumulator.cc +static void FillConstCPUFunctor(pten::DenseTensor* tensor_dense, double value) { + PADDLE_ENFORCE(tensor_dense, paddle::platform::errors::Fatal( + "Receive nullptr of dense tensor")); + switch (tensor_dense->data_type()) { + case pten::DataType::INT64: { + int64_t* data_ptr = tensor_dense->mutable_data(); + for (int i = 0; i < tensor_dense->numel(); i++) { + data_ptr[i] = static_cast(value); + } + + break; + } + case pten::DataType::INT32: { + int32_t* data_ptr = tensor_dense->mutable_data(); + for (int i = 0; i < tensor_dense->numel(); i++) { + data_ptr[i] = static_cast(value); + } + + break; + } + case pten::DataType::FLOAT64: { + double* data_ptr = tensor_dense->mutable_data(); + for (int i = 0; i < tensor_dense->numel(); i++) { + data_ptr[i] = static_cast(value); + } + + break; + } + case pten::DataType::FLOAT32: { + float* data_ptr = tensor_dense->mutable_data(); + for (int i = 0; i < tensor_dense->numel(); i++) { + data_ptr[i] = static_cast(value); + } + break; + } + default: { + PADDLE_THROW(paddle::platform::errors::Fatal( + "Only supports tensor with fp32, fp64, int32, int64 datatypes for " + "now")); + break; + } + } +} + +// TODO(jiabin): This may have serious performance issue move code from +// gradient_accumulator.cc +static void FillConstCUDAFunctor(pten::DenseTensor* tensor_dense, + double value) { + paddle::platform::DeviceContextPool& pool = + paddle::platform::DeviceContextPool::Instance(); + auto* dev_ctx = dynamic_cast( + pool.Get(paddle::platform::CUDAPlace())); + auto stream = dev_ctx->stream(); + + switch (tensor_dense->data_type()) { + case pten::DataType::INT64: { + std::vector host_data(tensor_dense->numel(), + static_cast(value)); + int64_t* device_ptr = tensor_dense->mutable_data(); + paddle::memory::Copy(paddle::platform::CUDAPlace(), device_ptr, + paddle::platform::CPUPlace(), host_data.data(), + sizeof(int64_t) * tensor_dense->numel(), stream); + break; + } + case pten::DataType::INT32: { + std::vector host_data(tensor_dense->numel(), + static_cast(value)); + int32_t* device_ptr = tensor_dense->mutable_data(); + paddle::memory::Copy(paddle::platform::CUDAPlace(), device_ptr, + paddle::platform::CPUPlace(), host_data.data(), + sizeof(int32_t) * tensor_dense->numel(), stream); + break; + } + case pten::DataType::FLOAT64: { + std::vector host_data(tensor_dense->numel(), + static_cast(value)); + double* device_ptr = tensor_dense->mutable_data(); + paddle::memory::Copy(paddle::platform::CUDAPlace(), device_ptr, + paddle::platform::CPUPlace(), host_data.data(), + sizeof(double) * tensor_dense->numel(), stream); + break; + } + case pten::DataType::FLOAT32: { + std::vector host_data(tensor_dense->numel(), + static_cast(value)); + float* device_ptr = tensor_dense->mutable_data(); + paddle::memory::Copy(paddle::platform::CUDAPlace(), device_ptr, + paddle::platform::CPUPlace(), host_data.data(), + sizeof(float) * tensor_dense->numel(), stream); + break; + } + default: { + PADDLE_THROW(paddle::platform::errors::Fatal( + "Only supports tensor with fp32, fp64, int32, int64 datatypes for " + "now")); + break; + } + } +} + +void ScaleAPI(const egr::EagerTensor& x, float scale, float bias, + bool bias_after_scale, egr::EagerTensor* out) { + // TODO(jiabin): Support multiple tensor here, Create DenseTensor is not a + // proper way to Demo it + // Run Forward Function + auto dense_tensor = std::dynamic_pointer_cast(x.impl()); + // Init output tensor + auto tensor_meta = pten::DenseTensorMeta( + dense_tensor->data_type(), dense_tensor->dims(), dense_tensor->layout()); + auto place = dense_tensor->place(); + size_t bytes_size = paddle::framework::product(dense_tensor->dims()) * + SizeOf(dense_tensor->data_type()); + auto dense_out = std::make_shared( + pten::make_intrusive( + paddle::memory::Alloc(place, bytes_size), 0), + std::move(tensor_meta)); + // Handle Device Context + const paddle::platform::Place& expected_kernel_place = + Controller::Instance().GetExpectedPlace(); + paddle::platform::DeviceContextPool& pool = + paddle::platform::DeviceContextPool::Instance(); + + if (expected_kernel_place == paddle::platform::CPUPlace()) { + auto* dev_ctx = dynamic_cast( + pool.Get(expected_kernel_place)); + if (!dev_ctx) { + PADDLE_THROW(paddle::platform::errors::Fatal("Backend mismatch")); + } + ScaleDeviceDispatch( + *dense_tensor.get(), *dev_ctx, scale, bias, bias_after_scale, + dense_out.get()); + + } else if (expected_kernel_place == paddle::platform::CUDAPlace()) { + auto* dev_ctx = dynamic_cast( + pool.Get(expected_kernel_place)); + if (!dev_ctx) { + PADDLE_THROW(paddle::platform::errors::Fatal("Backend mismatch")); + } + ScaleDeviceDispatch( + *dense_tensor.get(), *dev_ctx, scale, bias, bias_after_scale, + dense_out.get()); + + } else { + PADDLE_THROW(paddle::platform::errors::Fatal( + "Only CPU and CUDA Backend are supported for now")); + } + + out->set_impl(dense_out); +} + +void FillConstAPI(double value, const pten::DDim& ddim, + const paddle::platform::Place& place, + const pten::DataType& dtype, const pten::DataLayout& layout, + egr::EagerTensor* target) { + // Create new tensor->impl and fill it with 1.0 + // Fill 1.0 + // TODO(jiabin): Refactor this with operators::math::set_constant + std::shared_ptr tensor_dense = nullptr; + if (!target->defined() || !target->initialized()) { + VLOG(6) << "Init undefined or uninitialized tensor in FillConstAPI"; + auto tensor_meta = pten::DenseTensorMeta(dtype, ddim, layout); + size_t bytes_size = paddle::framework::product(ddim) * SizeOf(dtype); + + tensor_dense = std::make_shared( + pten::make_intrusive( + paddle::memory::Alloc(place, bytes_size), 0), + std::move(tensor_meta)); + target->set_impl(tensor_dense); + + } else { + tensor_dense = std::dynamic_pointer_cast(target->impl()); + } + + if (!tensor_dense) { + PADDLE_THROW(paddle::platform::errors::Fatal( + "FillConstAPI Only supports InputBuffer with DenseTensor for now.")); + } + VLOG(6) << "Call FillConstKernel"; + auto t_place = tensor_dense->place(); + if (paddle::platform::is_cpu_place(t_place)) { + VLOG(8) << "Call FillConst CPU Kernel"; + FillConstCPUFunctor(tensor_dense.get(), value); + } else if (paddle::platform::is_gpu_place(t_place)) { + VLOG(8) << "Call FillConst CUDA Kernel"; + FillConstCUDAFunctor(tensor_dense.get(), value); + } else { + PADDLE_THROW(paddle::platform::errors::Fatal( + "Only CPU and CUDA Backend are supported for now")); + } +} + +void FillConstAPI(double value, const paddle::framework::DDim& ddim, + const paddle::platform::Place& place, + const paddle::framework::proto::VarType::Type& dtype, + egr::EagerTensor* target) { + auto* dst_tensor = + target->MutableVar()->GetMutable(); + auto* dev_ctx = paddle::platform::DeviceContextPool::Instance().Get(place); + dst_tensor->Resize(ddim); + // TOOD(jiabin): Ugly fix here we have fwd_data_type_ and data_type, since in + // grad mission + // we can't get data_type_ directly. We need to check if we can only use + // default data_type for now. + dst_tensor->mutable_data(place, dtype); + paddle::operators::math::set_constant(*dev_ctx, dst_tensor, value); +} + +} // namespace egr diff --git a/paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.h b/paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.h new file mode 100644 index 0000000000000..c9902376e5957 --- /dev/null +++ b/paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.h @@ -0,0 +1,34 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/fluid/eager/eager_tensor.h" +#include "paddle/fluid/platform/enforce.h" +#include "paddle/pten/api/all.h" +#include "paddle/pten/include/core.h" +namespace egr { + +// Public +void ScaleAPI(const egr::EagerTensor& x, float scale, float bias, + bool bias_after_scale, egr::EagerTensor* out); +void FillConstAPI(double value, const pten::DDim& ddim, + const paddle::platform::Place& place, + const pten::DataType& dtype, const pten::DataLayout& layout, + egr::EagerTensor* target); +void FillConstAPI(double value, const paddle::framework::DDim& ddim, + const paddle::platform::Place& place, + const paddle::framework::proto::VarType::Type& dtype, + egr::EagerTensor* target); +} // namespace egr diff --git a/paddle/fluid/eager/api/generated/eager_generated/forwards/scale.cc b/paddle/fluid/eager/api/generated/eager_generated/forwards/scale.cc new file mode 100644 index 0000000000000..8bd0a10b104e6 --- /dev/null +++ b/paddle/fluid/eager/api/generated/eager_generated/forwards/scale.cc @@ -0,0 +1,100 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * This File should be automatically generated by coding auto generator. + * All ops C++ autograd logic is defined here, in Python-C extension API + * system we try to avoid any autograd related code, and move them all to + * here. + * + * Currently, we just manually do some fwd autograd here. And we will replace + * them with auto code generator later. + * **/ + +#include "paddle/fluid/eager/api/generated/eager_generated/forwards/scale.h" + +#include "paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.h" +#include "paddle/fluid/eager/autograd_meta.h" + +#include "paddle/fluid/eager/eager_tensor.h" +#include "paddle/fluid/eager/function_api.h" +#include "paddle/pten/api/all.h" +#include "paddle/pten/include/core.h" + +namespace egr { + +egr::EagerTensor scale(const egr::EagerTensor& x, float scale, float bias, + bool bias_after_scale, bool trace_backward) { + // 1. Run Forward + // 1.1 Create outputs + egr::EagerTensor out; + // 1.2 Need by original op, we assemble ins, outs, attrs here + + // 1.3 Call forward C++ api + ScaleAPI(x, scale, bias, bias_after_scale, &out); + + // 2. Build Backward Depends + // 2.1 Get AutogradMetas for all ins and outs + auto p_autograd_in = EagerUtils::unsafe_autograd_meta(x); + // NOTE: Call EagerUtils::multi_autograd_meta when we have vector of outputs + auto p_autograd_out = EagerUtils::autograd_meta(&out); + + // 2.2 Add GradNode + // 2.2.1 ComputeRequireGrad + // TODO(jiabin) : make this function accept different kinds of input + // TODO(zhanlve): which one is more efficient: + // 1. construct a vector of pointers + // 2. call "ComputeRequireGrad" multiple times + bool require_any_grad = ComputeRequireGrad(trace_backward, p_autograd_in); + if (require_any_grad) { + PassStopGradient(false /*generate_grad*/, p_autograd_out); + + // 2.2.2 Set OutRankInfo for outputs this needs to be as same as Edges's + // input_rank_ + /** Note: + // 1. We provide EagerUtils::SetMultiOutRank(vector), + // since we have some of Operator has servel slot name with duplicate + outputs. + // 2. We call AutogradMeta's SetOutput Rank only when we have single output + with + // single slot name. + **/ + p_autograd_out->SetSingleOutRankWithSlot(0, 0); + + // Init GradNode + auto scale_node = std::make_shared(/* fwd_in_slot_num */ 1, + /* bwd_in_slot_num */ 1); + + // Pass Attributes to GradNode + scale_node->SetAttributes_scale(scale); + + // Set Next Edges + scale_node->AddEdges(*p_autograd_in, /*slot id*/ 0); + + // Set TensorWrappers + scale_node->SetTensorWrappers_X({x}); + + // Set Grad out rank as same as fwd input and set stop gradient to bwd + scale_node->SetGradOutMeta(*p_autograd_in, /*slot id*/ 0); + // Set Grad out rank as same as fwd input and set stop gradient to bwd + scale_node->SetGradInMeta(*p_autograd_out, /*slot id*/ 0); + + // Set History for output set current Grad Node for + EagerUtils::SetHistory(p_autograd_out, scale_node); + } + + return out; +} + +} // namespace egr diff --git a/paddle/fluid/eager/api/generated/eager_generated/forwards/scale.h b/paddle/fluid/eager/api/generated/eager_generated/forwards/scale.h new file mode 100644 index 0000000000000..13b03c3b760bf --- /dev/null +++ b/paddle/fluid/eager/api/generated/eager_generated/forwards/scale.h @@ -0,0 +1,23 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/fluid/eager/eager_tensor.h" +namespace egr { + +egr::EagerTensor scale(const egr::EagerTensor& x, float scale, float bias, + bool bias_after_scale, bool trace_backward); + +} // namespace egr diff --git a/paddle/fluid/eager/api/utils/CMakeLists.txt b/paddle/fluid/eager/api/utils/CMakeLists.txt new file mode 100644 index 0000000000000..5168f1fc02489 --- /dev/null +++ b/paddle/fluid/eager/api/utils/CMakeLists.txt @@ -0,0 +1 @@ +cc_library(global_utils SRCS global_utils.cc DEPS enforce) diff --git a/paddle/fluid/eager/api/utils/global_utils.cc b/paddle/fluid/eager/api/utils/global_utils.cc new file mode 100644 index 0000000000000..7f9e0d52d0b39 --- /dev/null +++ b/paddle/fluid/eager/api/utils/global_utils.cc @@ -0,0 +1,18 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include "paddle/fluid/eager/api/utils/global_utils.h" + +Controller* Controller::controller_ = new Controller(); diff --git a/paddle/fluid/eager/api/utils/global_utils.h b/paddle/fluid/eager/api/utils/global_utils.h new file mode 100644 index 0000000000000..839b8b1c413f4 --- /dev/null +++ b/paddle/fluid/eager/api/utils/global_utils.h @@ -0,0 +1,58 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#pragma once + +#include "paddle/fluid/eager/eager_tensor.h" +#include "paddle/fluid/platform/enforce.h" + +class UniqueNameGenerator { + public: + explicit UniqueNameGenerator(std::string prefix = "") : prefix_(prefix) {} + std::string Generate(std::string key = "eager_tmp") { + return prefix_ + key + "_" + std::to_string(id_++); + } + + private: + std::atomic id_{0}; + std::string prefix_; +}; + +// Global +class Controller { + public: + static Controller& Instance() { return *controller_; } + const paddle::platform::Place& GetExpectedPlace() const { + return *expected_place_.get(); + } + void SetExpectedPlace(const paddle::platform::Place& place) { + expected_place_ = std::make_shared(place); + } + void SetAMPLevel(int level) { amp_level_ = level; } + int GetAMPLevel() const { return amp_level_; } + bool HasGrad() const { return has_grad_; } + std::string GenerateUniqueName(std::string key = "eager_tmp") { + return generator_->Generate(key); + } + + private: + Controller() = default; + static Controller* controller_; + std::shared_ptr expected_place_ = nullptr; + int amp_level_ = 0; + bool has_grad_ = true; + std::unique_ptr generator_{new UniqueNameGenerator()}; + DISABLE_COPY_AND_ASSIGN(Controller); +}; diff --git a/paddle/fluid/eager/tests/CMakeLists.txt b/paddle/fluid/eager/tests/CMakeLists.txt new file mode 100644 index 0000000000000..f1ac3c9136bfd --- /dev/null +++ b/paddle/fluid/eager/tests/CMakeLists.txt @@ -0,0 +1,2 @@ +cc_test(test_egr_forward SRCS forward_test.cc DEPS ${eager_deps} ${fluid_deps}) +cc_test(test_egr_forward_autograd SRCS forward_autograd_test.cc DEPS ${eager_deps} ${fluid_deps}) diff --git a/paddle/fluid/eager/tests/forward_autograd_test.cc b/paddle/fluid/eager/tests/forward_autograd_test.cc new file mode 100644 index 0000000000000..03e34f24763b7 --- /dev/null +++ b/paddle/fluid/eager/tests/forward_autograd_test.cc @@ -0,0 +1,302 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "glog/logging.h" +#include "gtest/gtest.h" + +#include "paddle/fluid/eager/api/api.h" +#include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h" +#include "paddle/fluid/eager/autograd_meta.h" +#include "paddle/fluid/eager/grad_node_info.h" +#include "paddle/fluid/eager/tests/test_utils.h" + +#include "paddle/pten/core/dense_tensor.h" +#include "paddle/pten/core/tensor_meta.h" + +// TODO(jiabin): remove nolint here!!! +using namespace egr; // NOLINT + +TEST(Forward, SingleNode) { + // Prepare Device Contexts + InitEnv(paddle::platform::CPUPlace()); + + // Prepare Inputs + std::vector target_tensors; + paddle::framework::DDim ddim = paddle::framework::make_ddim({4, 16, 16, 32}); + + // Create Target Tensor + egr::EagerTensor t = EagerUtils::CreateTensorWithValue( + ddim, paddle::platform::CPUPlace(), pten::DataType::FLOAT32, + pten::DataLayout::NCHW, 5.0 /*value*/, false /*is_leaf*/); + target_tensors.emplace_back(std::move(t)); + egr::EagerTensor& tensor = target_tensors[0]; + EagerUtils::autograd_meta(&tensor)->SetStopGradient(false); + + // Run Forward + float scale = 2.0; + float bias = 3.0; + egr::EagerTensor out = egr::scale( + tensor, scale, bias, true /*bias_after_scale*/, true /*trace_backward*/); + + // Examine Forward Output + CompareTensorWithValue(out, 13.0); + + // Examine GradNode + { + // 1. GradNode + AutogradMeta* meta = EagerUtils::autograd_meta(&out); + GradNodeBase* grad_node = meta->GradNode(); + GradNodeScale* scale_node = dynamic_cast(grad_node); + PADDLE_ENFORCE( + scale_node != nullptr, + paddle::platform::errors::Fatal("AutogradMeta should have held " + "grad_node with type GradNodeScale*")); + PADDLE_ENFORCE( + (meta->OutRankInfo().first == 0) && (meta->OutRankInfo().second == 0), + paddle::platform::errors::Fatal( + "OutRankInfo in AutogradMeta should have been 0")); + + // 2. TensorWrapper: No TensorWrapper for ScaleNode + // 3. NextEdges: No NextEdges for Single Node Case + } +} + +/* + inp + | +Node0 + | +Node1 + | + out +*/ +TEST(Forward, LinearNodes) { + InitEnv(paddle::platform::CPUPlace()); + + // Prepare Inputs + std::vector target_tensors; + paddle::framework::DDim ddim = paddle::framework::make_ddim({4, 16, 16, 32}); + + // Create Target Tensor + egr::EagerTensor t = EagerUtils::CreateTensorWithValue( + ddim, paddle::platform::CPUPlace(), pten::DataType::FLOAT32, + pten::DataLayout::NCHW, 5.0 /*value*/, false /*is_leaf*/); + target_tensors.emplace_back(std::move(t)); + egr::EagerTensor& tensor = target_tensors[0]; + EagerUtils::autograd_meta(&tensor)->SetStopGradient(false); + + // Run Forward Node 0 + float scale0 = 2.0; + float bias0 = 3.0; + egr::EagerTensor out0 = + egr::scale(tensor, scale0, bias0, true /*bias_after_scale*/, + true /*trace_backward*/); + + // Run Forward Node 1 + float scale1 = 5.0; + float bias1 = 10.0; + egr::EagerTensor out1 = egr::scale( + out0, scale1, bias1, true /*bias_after_scale*/, true /*trace_backward*/); + + // Examine Forward Output 0 + CompareTensorWithValue(out0, 13.0); + + // Examine Forward Output 1 + CompareTensorWithValue(out1, 75.0); + + // Examine GradNode + { + // 1. GradNode + // Node 0 + AutogradMeta* meta0 = EagerUtils::autograd_meta(&out0); + GradNodeBase* grad_node0 = meta0->GradNode(); + GradNodeScale* scale_node0 = dynamic_cast(grad_node0); + PADDLE_ENFORCE( + scale_node0 != nullptr, + paddle::platform::errors::Fatal("AutogradMeta should have held " + "grad_node with type GradNodeScale*")); + PADDLE_ENFORCE( + (meta0->OutRankInfo().first == 0) && (meta0->OutRankInfo().second == 0), + paddle::platform::errors::Fatal( + "OutRankInfo in AutogradMeta should have been 0")); + + // Node 1 + AutogradMeta* meta1 = EagerUtils::autograd_meta(&out1); + GradNodeBase* grad_node1 = meta1->GradNode(); + GradNodeScale* scale_node1 = dynamic_cast(grad_node1); + PADDLE_ENFORCE( + scale_node1 != nullptr, + paddle::platform::errors::Fatal("AutogradMeta should have held " + "grad_node with type GradNodeScale*")); + PADDLE_ENFORCE( + (meta1->OutRankInfo().first == 0) && (meta1->OutRankInfo().second == 0), + paddle::platform::errors::Fatal( + "OutRankInfo in AutogradMeta should have been 0")); + + // 2. TensorWrapper: No TensorWrapper for ScaleNode + // 3. NextEdges: Node 1 -> Node 0 + const std::vector>& node1_edges = grad_node1->GetEdges(); + PADDLE_ENFORCE(node1_edges.size() == 1, + paddle::platform::errors::Fatal( + "Node 1 should have exactly 1 edge slot ")); + + const auto& node1_edge = node1_edges[0]; + PADDLE_ENFORCE(node1_edge.size() == 1, + paddle::platform::errors::Fatal( + "Node 1 should have exactly 1 edge in slot 0")); + PADDLE_ENFORCE((node1_edge[0].GetEdgeRankInfo().first == 0) && + (node1_edge[0].GetEdgeRankInfo().second == 0), + paddle::platform::errors::Fatal( + "Node1's edge should have input rank of 0")); + + PADDLE_ENFORCE( + node1_edge[0].GetGradNode() == grad_node0, + paddle::platform::errors::Fatal("Node1's edge should point to Node 0")); + } +} + +/* + inp + | + Node0 + ____|____ + | | + Node1 Node2 + | | + out1 out2 +*/ +TEST(Forward, BranchedNodes) { + InitEnv(paddle::platform::CPUPlace()); + + // Prepare Inputs + std::vector target_tensors; + paddle::framework::DDim ddim = paddle::framework::make_ddim({4, 16, 16, 32}); + + // Create Target Tensor + egr::EagerTensor t = EagerUtils::CreateTensorWithValue( + ddim, paddle::platform::CPUPlace(), pten::DataType::FLOAT32, + pten::DataLayout::NCHW, 5.0 /*value*/, false /*is_leaf*/); + target_tensors.emplace_back(std::move(t)); + egr::EagerTensor& tensor = target_tensors[0]; + EagerUtils::autograd_meta(&tensor)->SetStopGradient(false); + + // Run Forward Node 0 + float scale0 = 2.0; + float bias0 = 3.0; + egr::EagerTensor out0 = + egr::scale(tensor, scale0, bias0, true /*bias_after_scale*/, + true /*trace_backward*/); + + // Run Forward Node 1 + float scale1 = 5.0; + float bias1 = 10.0; + egr::EagerTensor out1 = egr::scale( + out0, scale1, bias1, true /*bias_after_scale*/, true /*trace_backward*/); + + // Run Forward Node 2 + float scale2 = 10.0; + float bias2 = 20.0; + egr::EagerTensor out2 = egr::scale( + out0, scale2, bias2, true /*bias_after_scale*/, true /*trace_backward*/); + + // Examine Forward Output 0 + CompareTensorWithValue(out0, 13.0); + + // Examine Forward Output 1 + CompareTensorWithValue(out1, 75.0); + + // Examine Forward Output 2 + CompareTensorWithValue(out2, 150.0); + + // Examine GradNode + { + // 1. GradNode + // Node 0 + AutogradMeta* meta0 = EagerUtils::autograd_meta(&out0); + GradNodeBase* grad_node0 = meta0->GradNode(); + GradNodeScale* scale_node0 = dynamic_cast(grad_node0); + PADDLE_ENFORCE( + scale_node0 != nullptr, + paddle::platform::errors::Fatal("AutogradMeta should have held " + "grad_node with type GradNodeScale*")); + PADDLE_ENFORCE( + (meta0->OutRankInfo().first == 0) && (meta0->OutRankInfo().second == 0), + paddle::platform::errors::Fatal( + "OutRankInfo in AutogradMeta should have been 0")); + + // Node 1 + AutogradMeta* meta1 = EagerUtils::autograd_meta(&out1); + GradNodeBase* grad_node1 = meta1->GradNode(); + GradNodeScale* scale_node1 = dynamic_cast(grad_node1); + PADDLE_ENFORCE( + scale_node1 != nullptr, + paddle::platform::errors::Fatal("AutogradMeta should have held " + "grad_node with type GradNodeScale*")); + PADDLE_ENFORCE( + (meta1->OutRankInfo().first == 0) && (meta1->OutRankInfo().second == 0), + paddle::platform::errors::Fatal( + "OutRankInfo in AutogradMeta should have been 0")); + + // Node 2 + AutogradMeta* meta2 = EagerUtils::autograd_meta(&out2); + GradNodeBase* grad_node2 = meta2->GradNode(); + GradNodeScale* scale_node2 = dynamic_cast(grad_node2); + PADDLE_ENFORCE( + scale_node2 != nullptr, + paddle::platform::errors::Fatal("AutogradMeta should have held " + "grad_node with type GradNodeScale*")); + PADDLE_ENFORCE( + (meta2->OutRankInfo().first == 0) && (meta2->OutRankInfo().second == 0), + paddle::platform::errors::Fatal( + "OutRankInfo in AutogradMeta should have been 0")); + + // 2. TensorWrapper: No TensorWrapper for ScaleNode + // 3. NextEdges + // Node 1 -> Node 0 + const std::vector>& node1_edges = grad_node1->GetEdges(); + PADDLE_ENFORCE(node1_edges.size() == 1, + paddle::platform::errors::Fatal( + "Node 1 should have exactly 1 edge slot")); + PADDLE_ENFORCE(node1_edges[0].size() == 1, + paddle::platform::errors::Fatal( + "Node 1 should have exactly 1 edge in slot 0")); + const Edge& node1_edge = node1_edges[0][0]; + PADDLE_ENFORCE((node1_edge.GetEdgeRankInfo().first == 0) && + (node1_edge.GetEdgeRankInfo().second == 0), + paddle::platform::errors::Fatal( + "Node1's edge should have input rank of 0")); + PADDLE_ENFORCE( + node1_edge.GetGradNode() == grad_node0, + paddle::platform::errors::Fatal("Node1's edge should point to Node 0")); + + // Node 2 -> Node 0 + const std::vector>& node2_edges = grad_node2->GetEdges(); + PADDLE_ENFORCE(node2_edges.size() == 1, + paddle::platform::errors::Fatal( + "Node 2 should have exactly 1 edge slot")); + PADDLE_ENFORCE(node1_edges[0].size() == 1, + paddle::platform::errors::Fatal( + "Node 2 should have exactly 1 edge in slot 0")); + const Edge& node2_edge = node2_edges[0][0]; + PADDLE_ENFORCE((node2_edge.GetEdgeRankInfo().first == 0) && + (node2_edge.GetEdgeRankInfo().second == 0), + paddle::platform::errors::Fatal( + "Node2's edge should have input rank of 0")); + PADDLE_ENFORCE( + node2_edge.GetGradNode() == grad_node0, + paddle::platform::errors::Fatal("Node2's edge should point to Node 0")); + } +} diff --git a/paddle/fluid/eager/tests/forward_test.cc b/paddle/fluid/eager/tests/forward_test.cc new file mode 100644 index 0000000000000..568446cfd99ba --- /dev/null +++ b/paddle/fluid/eager/tests/forward_test.cc @@ -0,0 +1,304 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "glog/logging.h" +#include "gtest/gtest.h" + +#include "paddle/fluid/eager/api/api.h" +#include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h" +#include "paddle/fluid/eager/autograd_meta.h" +#include "paddle/fluid/eager/backward.h" +#include "paddle/fluid/eager/grad_node_info.h" + +#include "paddle/pten/core/dense_tensor.h" +#include "paddle/pten/core/tensor_meta.h" + +#include "paddle/fluid/eager/tests/test_utils.h" + +// TODO(jiabin): remove nolint here!!! +using namespace egr; // NOLINT + +TEST(Forward, SingleNode) { + // Prepare Device Contexts + InitEnv(paddle::platform::CPUPlace()); + + // Prepare Inputs + std::vector target_tensors; + paddle::framework::DDim ddim = paddle::framework::make_ddim({4, 16, 16, 32}); + + // Create Target Tensor + egr::EagerTensor t = EagerUtils::CreateTensorWithValue( + ddim, paddle::platform::CPUPlace(), pten::DataType::FLOAT32, + pten::DataLayout::NCHW, 5.0 /*value*/, false /*is_leaf*/); + target_tensors.emplace_back(std::move(t)); + egr::EagerTensor& tensor = target_tensors[0]; + EagerUtils::autograd_meta(&tensor)->SetStopGradient(false); + + // Run Forward + float scale = 2.0; + float bias = 3.0; + egr::EagerTensor out = egr::scale( + tensor, scale, bias, true /*bias_after_scale*/, true /*trace_backward*/); + + // Examine Forward Output + CompareTensorWithValue(out, 13.0); + + // Examine GradNode + { + // 1. GradNode + AutogradMeta* meta = EagerUtils::autograd_meta(&out); + GradNodeBase* grad_node = meta->GradNode(); + GradNodeScale* scale_node = dynamic_cast(grad_node); + PADDLE_ENFORCE( + scale_node != nullptr, + paddle::platform::errors::Fatal("AutogradMeta should have held " + "grad_node with type GradNodeScale*")); + PADDLE_ENFORCE( + (meta->OutRankInfo().first == 0) && (meta->OutRankInfo().second == 0), + paddle::platform::errors::Fatal( + "OutRankInfo in AutogradMeta should have been 0")); + + // 2. TensorWrapper: No TensorWrapper for ScaleNode + // 3. NextEdges: No NextEdges for Single Node Case + } +} + +/* + inp + | +Node0 + | +Node1 + | + out +*/ +TEST(Forward, LinearNodes) { + InitEnv(paddle::platform::CPUPlace()); + + // Prepare Inputs + std::vector target_tensors; + paddle::framework::DDim ddim = paddle::framework::make_ddim({4, 16, 16, 32}); + + // Create Target Tensor + egr::EagerTensor t = EagerUtils::CreateTensorWithValue( + ddim, paddle::platform::CPUPlace(), pten::DataType::FLOAT32, + pten::DataLayout::NCHW, 5.0 /*value*/, false /*is_leaf*/); + target_tensors.emplace_back(std::move(t)); + egr::EagerTensor& tensor = target_tensors[0]; + EagerUtils::autograd_meta(&tensor)->SetStopGradient(false); + + // Run Forward Node 0 + float scale0 = 2.0; + float bias0 = 3.0; + egr::EagerTensor out0 = + egr::scale(tensor, scale0, bias0, true /*bias_after_scale*/, + true /*trace_backward*/); + + // Run Forward Node 1 + float scale1 = 5.0; + float bias1 = 10.0; + egr::EagerTensor out1 = egr::scale( + out0, scale1, bias1, true /*bias_after_scale*/, true /*trace_backward*/); + + // Examine Forward Output 0 + CompareTensorWithValue(out0, 13.0); + + // Examine Forward Output 1 + CompareTensorWithValue(out1, 75.0); + + // Examine GradNode + { + // 1. GradNode + // Node 0 + AutogradMeta* meta0 = EagerUtils::autograd_meta(&out0); + GradNodeBase* grad_node0 = meta0->GradNode(); + GradNodeScale* scale_node0 = dynamic_cast(grad_node0); + PADDLE_ENFORCE( + scale_node0 != nullptr, + paddle::platform::errors::Fatal("AutogradMeta should have held " + "grad_node with type GradNodeScale*")); + PADDLE_ENFORCE( + (meta0->OutRankInfo().first == 0) && (meta0->OutRankInfo().second == 0), + paddle::platform::errors::Fatal( + "OutRankInfo in AutogradMeta should have been 0")); + + // Node 1 + AutogradMeta* meta1 = EagerUtils::autograd_meta(&out1); + GradNodeBase* grad_node1 = meta1->GradNode(); + GradNodeScale* scale_node1 = dynamic_cast(grad_node1); + PADDLE_ENFORCE( + scale_node1 != nullptr, + paddle::platform::errors::Fatal("AutogradMeta should have held " + "grad_node with type GradNodeScale*")); + PADDLE_ENFORCE( + (meta1->OutRankInfo().first == 0) && (meta1->OutRankInfo().second == 0), + paddle::platform::errors::Fatal( + "OutRankInfo in AutogradMeta should have been 0")); + + // 2. TensorWrapper: No TensorWrapper for ScaleNode + // 3. NextEdges: Node 1 -> Node 0 + const std::vector>& node1_edges = grad_node1->GetEdges(); + PADDLE_ENFORCE(node1_edges.size() == 1, + paddle::platform::errors::Fatal( + "Node 1 should have exactly 1 edge slot ")); + + const auto& node1_edge = node1_edges[0]; + PADDLE_ENFORCE(node1_edge.size() == 1, + paddle::platform::errors::Fatal( + "Node 1 should have exactly 1 edge in slot 0")); + PADDLE_ENFORCE((node1_edge[0].GetEdgeRankInfo().first == 0) && + (node1_edge[0].GetEdgeRankInfo().second == 0), + paddle::platform::errors::Fatal( + "Node1's edge should have input rank of 0")); + + PADDLE_ENFORCE( + node1_edge[0].GetGradNode() == grad_node0, + paddle::platform::errors::Fatal("Node1's edge should point to Node 0")); + } +} + +/* + inp + | + Node0 + ____|____ + | | + Node1 Node2 + | | + out1 out2 +*/ +TEST(Forward, BranchedNodes) { + InitEnv(paddle::platform::CPUPlace()); + + // Prepare Inputs + std::vector target_tensors; + paddle::framework::DDim ddim = paddle::framework::make_ddim({4, 16, 16, 32}); + + // Create Target Tensor + egr::EagerTensor t = EagerUtils::CreateTensorWithValue( + ddim, paddle::platform::CPUPlace(), pten::DataType::FLOAT32, + pten::DataLayout::NCHW, 5.0 /*value*/, false /*is_leaf*/); + target_tensors.emplace_back(std::move(t)); + egr::EagerTensor& tensor = target_tensors[0]; + EagerUtils::autograd_meta(&tensor)->SetStopGradient(false); + + // Run Forward Node 0 + float scale0 = 2.0; + float bias0 = 3.0; + egr::EagerTensor out0 = + egr::scale(tensor, scale0, bias0, true /*bias_after_scale*/, + true /*trace_backward*/); + + // Run Forward Node 1 + float scale1 = 5.0; + float bias1 = 10.0; + egr::EagerTensor out1 = egr::scale( + out0, scale1, bias1, true /*bias_after_scale*/, true /*trace_backward*/); + + // Run Forward Node 2 + float scale2 = 10.0; + float bias2 = 20.0; + egr::EagerTensor out2 = egr::scale( + out0, scale2, bias2, true /*bias_after_scale*/, true /*trace_backward*/); + + // Examine Forward Output 0 + CompareTensorWithValue(out0, 13.0); + + // Examine Forward Output 1 + CompareTensorWithValue(out1, 75.0); + + // Examine Forward Output 2 + CompareTensorWithValue(out2, 150.0); + + // Examine GradNode + { + // 1. GradNode + // Node 0 + AutogradMeta* meta0 = EagerUtils::autograd_meta(&out0); + GradNodeBase* grad_node0 = meta0->GradNode(); + GradNodeScale* scale_node0 = dynamic_cast(grad_node0); + PADDLE_ENFORCE( + scale_node0 != nullptr, + paddle::platform::errors::Fatal("AutogradMeta should have held " + "grad_node with type GradNodeScale*")); + PADDLE_ENFORCE( + (meta0->OutRankInfo().first == 0) && (meta0->OutRankInfo().second == 0), + paddle::platform::errors::Fatal( + "OutRankInfo in AutogradMeta should have been 0")); + + // Node 1 + AutogradMeta* meta1 = EagerUtils::autograd_meta(&out1); + GradNodeBase* grad_node1 = meta1->GradNode(); + GradNodeScale* scale_node1 = dynamic_cast(grad_node1); + PADDLE_ENFORCE( + scale_node1 != nullptr, + paddle::platform::errors::Fatal("AutogradMeta should have held " + "grad_node with type GradNodeScale*")); + PADDLE_ENFORCE( + (meta1->OutRankInfo().first == 0) && (meta1->OutRankInfo().second == 0), + paddle::platform::errors::Fatal( + "OutRankInfo in AutogradMeta should have been 0")); + + // Node 2 + AutogradMeta* meta2 = EagerUtils::autograd_meta(&out2); + GradNodeBase* grad_node2 = meta2->GradNode(); + GradNodeScale* scale_node2 = dynamic_cast(grad_node2); + PADDLE_ENFORCE( + scale_node2 != nullptr, + paddle::platform::errors::Fatal("AutogradMeta should have held " + "grad_node with type GradNodeScale*")); + PADDLE_ENFORCE( + (meta2->OutRankInfo().first == 0) && (meta2->OutRankInfo().second == 0), + paddle::platform::errors::Fatal( + "OutRankInfo in AutogradMeta should have been 0")); + + // 2. TensorWrapper: No TensorWrapper for ScaleNode + // 3. NextEdges + // Node 1 -> Node 0 + const std::vector>& node1_edges = grad_node1->GetEdges(); + PADDLE_ENFORCE(node1_edges.size() == 1, + paddle::platform::errors::Fatal( + "Node 1 should have exactly 1 edge slot")); + PADDLE_ENFORCE(node1_edges[0].size() == 1, + paddle::platform::errors::Fatal( + "Node 1 should have exactly 1 edge in slot 0")); + const Edge& node1_edge = node1_edges[0][0]; + PADDLE_ENFORCE((node1_edge.GetEdgeRankInfo().first == 0) && + (node1_edge.GetEdgeRankInfo().second == 0), + paddle::platform::errors::Fatal( + "Node1's edge should have input rank of 0")); + PADDLE_ENFORCE( + node1_edge.GetGradNode() == grad_node0, + paddle::platform::errors::Fatal("Node1's edge should point to Node 0")); + + // Node 2 -> Node 0 + const std::vector>& node2_edges = grad_node2->GetEdges(); + PADDLE_ENFORCE(node2_edges.size() == 1, + paddle::platform::errors::Fatal( + "Node 2 should have exactly 1 edge slot")); + PADDLE_ENFORCE(node1_edges[0].size() == 1, + paddle::platform::errors::Fatal( + "Node 2 should have exactly 1 edge in slot 0")); + const Edge& node2_edge = node2_edges[0][0]; + PADDLE_ENFORCE((node2_edge.GetEdgeRankInfo().first == 0) && + (node2_edge.GetEdgeRankInfo().second == 0), + paddle::platform::errors::Fatal( + "Node2's edge should have input rank of 0")); + PADDLE_ENFORCE( + node2_edge.GetGradNode() == grad_node0, + paddle::platform::errors::Fatal("Node2's edge should point to Node 0")); + } +} From e5b43d9cfdc29fb6f2cbb418870bc8dff5230290 Mon Sep 17 00:00:00 2001 From: jim19930609 Date: Thu, 18 Nov 2021 02:41:52 +0000 Subject: [PATCH 2/9] [Second Patch PR #1]Bug Fix --- paddle/fluid/eager/CMakeLists.txt | 5 +++++ .../eager_generated/backwards/scale_node.cc | 4 ++-- .../eager_generated/forwards/function_api.cc | 12 ++++++------ .../eager_generated/forwards/function_api.h | 5 ++++- .../api/generated/eager_generated/forwards/scale.cc | 3 +-- paddle/fluid/eager/tests/forward_test.cc | 4 +--- 6 files changed, 19 insertions(+), 14 deletions(-) diff --git a/paddle/fluid/eager/CMakeLists.txt b/paddle/fluid/eager/CMakeLists.txt index 3a0f7dc95f2a5..e85e37e5fd645 100644 --- a/paddle/fluid/eager/CMakeLists.txt +++ b/paddle/fluid/eager/CMakeLists.txt @@ -3,4 +3,9 @@ add_subdirectory(api) set(eager_deps eager_api utils function_api autograd_meta grad_node_info pten pten_api) set(fluid_deps tracer layer proto_desc operator op_registry variable_helper memcpy) +cc_library(autograd_meta SRCS autograd_meta.cc DEPS pten pten_api) +cc_library(grad_node_info SRCS grad_node_info.cc DEPS function_api pten pten_api) +cc_library(utils SRCS utils.cc DEPS pten pten_api layer proto_desc operator op_registry variable_helper memcpy scale_op) +cc_library(gradient_accumulation SRCS gradient_accumulation.cc DEPS blas pten pten_api var_type_traits layer math_function) + add_subdirectory(tests) diff --git a/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.cc b/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.cc index 3994e9ddf8617..287812852a578 100644 --- a/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.cc +++ b/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.cc @@ -14,11 +14,11 @@ #include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h" #include "paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.h" - #include "paddle/fluid/eager/eager_tensor.h" -#include "paddle/fluid/platform/device_context.h" + #include "paddle/pten/api/all.h" +#include "paddle/fluid/platform/device_context.h" #include "paddle/fluid/platform/enforce.h" #include "paddle/fluid/platform/errors.h" diff --git a/paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.cc b/paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.cc index bc0969a10af5d..8039d01771292 100644 --- a/paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.cc +++ b/paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.cc @@ -30,7 +30,7 @@ static void ScaleDeviceDispatch(const pten::DenseTensor& dense_tensor, const DeviceContext& dev_ctx, float scale, float bias, bool bias_after_scale, pten::DenseTensor* dense_out) { - switch (dense_tensor.data_type()) { + switch (dense_tensor.dtype()) { case pten::DataType::FLOAT64: { pten::Scale(dev_ctx, dense_tensor /* tensor */, scale /* scale */, bias /* bias */, @@ -71,7 +71,7 @@ static void ScaleDeviceDispatch(const pten::DenseTensor& dense_tensor, static void FillConstCPUFunctor(pten::DenseTensor* tensor_dense, double value) { PADDLE_ENFORCE(tensor_dense, paddle::platform::errors::Fatal( "Receive nullptr of dense tensor")); - switch (tensor_dense->data_type()) { + switch (tensor_dense->dtype()) { case pten::DataType::INT64: { int64_t* data_ptr = tensor_dense->mutable_data(); for (int i = 0; i < tensor_dense->numel(); i++) { @@ -118,11 +118,11 @@ static void FillConstCUDAFunctor(pten::DenseTensor* tensor_dense, double value) { paddle::platform::DeviceContextPool& pool = paddle::platform::DeviceContextPool::Instance(); - auto* dev_ctx = dynamic_cast( + auto* dev_ctx = dynamic_cast( pool.Get(paddle::platform::CUDAPlace())); auto stream = dev_ctx->stream(); - switch (tensor_dense->data_type()) { + switch (tensor_dense->dtype()) { case pten::DataType::INT64: { std::vector host_data(tensor_dense->numel(), static_cast(value)); @@ -176,10 +176,10 @@ void ScaleAPI(const egr::EagerTensor& x, float scale, float bias, auto dense_tensor = std::dynamic_pointer_cast(x.impl()); // Init output tensor auto tensor_meta = pten::DenseTensorMeta( - dense_tensor->data_type(), dense_tensor->dims(), dense_tensor->layout()); + dense_tensor->dtype(), dense_tensor->dims(), dense_tensor->layout()); auto place = dense_tensor->place(); size_t bytes_size = paddle::framework::product(dense_tensor->dims()) * - SizeOf(dense_tensor->data_type()); + SizeOf(dense_tensor->dtype()); auto dense_out = std::make_shared( pten::make_intrusive( paddle::memory::Alloc(place, bytes_size), 0), diff --git a/paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.h b/paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.h index c9902376e5957..a6b916406389a 100644 --- a/paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.h +++ b/paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.h @@ -15,9 +15,12 @@ #pragma once #include "paddle/fluid/eager/eager_tensor.h" -#include "paddle/fluid/platform/enforce.h" + #include "paddle/pten/api/all.h" #include "paddle/pten/include/core.h" + +#include "paddle/fluid/platform/enforce.h" + namespace egr { // Public diff --git a/paddle/fluid/eager/api/generated/eager_generated/forwards/scale.cc b/paddle/fluid/eager/api/generated/eager_generated/forwards/scale.cc index 8bd0a10b104e6..6127ef889a5b6 100644 --- a/paddle/fluid/eager/api/generated/eager_generated/forwards/scale.cc +++ b/paddle/fluid/eager/api/generated/eager_generated/forwards/scale.cc @@ -23,12 +23,11 @@ * **/ #include "paddle/fluid/eager/api/generated/eager_generated/forwards/scale.h" - #include "paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.h" #include "paddle/fluid/eager/autograd_meta.h" - #include "paddle/fluid/eager/eager_tensor.h" #include "paddle/fluid/eager/function_api.h" + #include "paddle/pten/api/all.h" #include "paddle/pten/include/core.h" diff --git a/paddle/fluid/eager/tests/forward_test.cc b/paddle/fluid/eager/tests/forward_test.cc index 568446cfd99ba..03e34f24763b7 100644 --- a/paddle/fluid/eager/tests/forward_test.cc +++ b/paddle/fluid/eager/tests/forward_test.cc @@ -20,14 +20,12 @@ #include "paddle/fluid/eager/api/api.h" #include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h" #include "paddle/fluid/eager/autograd_meta.h" -#include "paddle/fluid/eager/backward.h" #include "paddle/fluid/eager/grad_node_info.h" +#include "paddle/fluid/eager/tests/test_utils.h" #include "paddle/pten/core/dense_tensor.h" #include "paddle/pten/core/tensor_meta.h" -#include "paddle/fluid/eager/tests/test_utils.h" - // TODO(jiabin): remove nolint here!!! using namespace egr; // NOLINT From 1c68bc2082bef1c5609538c525581983217ea4ba Mon Sep 17 00:00:00 2001 From: jim19930609 Date: Thu, 18 Nov 2021 04:49:08 +0000 Subject: [PATCH 3/9] Replaced forward_test with forward_autograd_test --- paddle/fluid/eager/tests/CMakeLists.txt | 1 - paddle/fluid/eager/tests/forward_test.cc | 302 ----------------------- 2 files changed, 303 deletions(-) delete mode 100644 paddle/fluid/eager/tests/forward_test.cc diff --git a/paddle/fluid/eager/tests/CMakeLists.txt b/paddle/fluid/eager/tests/CMakeLists.txt index f1ac3c9136bfd..68704fc32a296 100644 --- a/paddle/fluid/eager/tests/CMakeLists.txt +++ b/paddle/fluid/eager/tests/CMakeLists.txt @@ -1,2 +1 @@ -cc_test(test_egr_forward SRCS forward_test.cc DEPS ${eager_deps} ${fluid_deps}) cc_test(test_egr_forward_autograd SRCS forward_autograd_test.cc DEPS ${eager_deps} ${fluid_deps}) diff --git a/paddle/fluid/eager/tests/forward_test.cc b/paddle/fluid/eager/tests/forward_test.cc deleted file mode 100644 index 03e34f24763b7..0000000000000 --- a/paddle/fluid/eager/tests/forward_test.cc +++ /dev/null @@ -1,302 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "glog/logging.h" -#include "gtest/gtest.h" - -#include "paddle/fluid/eager/api/api.h" -#include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h" -#include "paddle/fluid/eager/autograd_meta.h" -#include "paddle/fluid/eager/grad_node_info.h" -#include "paddle/fluid/eager/tests/test_utils.h" - -#include "paddle/pten/core/dense_tensor.h" -#include "paddle/pten/core/tensor_meta.h" - -// TODO(jiabin): remove nolint here!!! -using namespace egr; // NOLINT - -TEST(Forward, SingleNode) { - // Prepare Device Contexts - InitEnv(paddle::platform::CPUPlace()); - - // Prepare Inputs - std::vector target_tensors; - paddle::framework::DDim ddim = paddle::framework::make_ddim({4, 16, 16, 32}); - - // Create Target Tensor - egr::EagerTensor t = EagerUtils::CreateTensorWithValue( - ddim, paddle::platform::CPUPlace(), pten::DataType::FLOAT32, - pten::DataLayout::NCHW, 5.0 /*value*/, false /*is_leaf*/); - target_tensors.emplace_back(std::move(t)); - egr::EagerTensor& tensor = target_tensors[0]; - EagerUtils::autograd_meta(&tensor)->SetStopGradient(false); - - // Run Forward - float scale = 2.0; - float bias = 3.0; - egr::EagerTensor out = egr::scale( - tensor, scale, bias, true /*bias_after_scale*/, true /*trace_backward*/); - - // Examine Forward Output - CompareTensorWithValue(out, 13.0); - - // Examine GradNode - { - // 1. GradNode - AutogradMeta* meta = EagerUtils::autograd_meta(&out); - GradNodeBase* grad_node = meta->GradNode(); - GradNodeScale* scale_node = dynamic_cast(grad_node); - PADDLE_ENFORCE( - scale_node != nullptr, - paddle::platform::errors::Fatal("AutogradMeta should have held " - "grad_node with type GradNodeScale*")); - PADDLE_ENFORCE( - (meta->OutRankInfo().first == 0) && (meta->OutRankInfo().second == 0), - paddle::platform::errors::Fatal( - "OutRankInfo in AutogradMeta should have been 0")); - - // 2. TensorWrapper: No TensorWrapper for ScaleNode - // 3. NextEdges: No NextEdges for Single Node Case - } -} - -/* - inp - | -Node0 - | -Node1 - | - out -*/ -TEST(Forward, LinearNodes) { - InitEnv(paddle::platform::CPUPlace()); - - // Prepare Inputs - std::vector target_tensors; - paddle::framework::DDim ddim = paddle::framework::make_ddim({4, 16, 16, 32}); - - // Create Target Tensor - egr::EagerTensor t = EagerUtils::CreateTensorWithValue( - ddim, paddle::platform::CPUPlace(), pten::DataType::FLOAT32, - pten::DataLayout::NCHW, 5.0 /*value*/, false /*is_leaf*/); - target_tensors.emplace_back(std::move(t)); - egr::EagerTensor& tensor = target_tensors[0]; - EagerUtils::autograd_meta(&tensor)->SetStopGradient(false); - - // Run Forward Node 0 - float scale0 = 2.0; - float bias0 = 3.0; - egr::EagerTensor out0 = - egr::scale(tensor, scale0, bias0, true /*bias_after_scale*/, - true /*trace_backward*/); - - // Run Forward Node 1 - float scale1 = 5.0; - float bias1 = 10.0; - egr::EagerTensor out1 = egr::scale( - out0, scale1, bias1, true /*bias_after_scale*/, true /*trace_backward*/); - - // Examine Forward Output 0 - CompareTensorWithValue(out0, 13.0); - - // Examine Forward Output 1 - CompareTensorWithValue(out1, 75.0); - - // Examine GradNode - { - // 1. GradNode - // Node 0 - AutogradMeta* meta0 = EagerUtils::autograd_meta(&out0); - GradNodeBase* grad_node0 = meta0->GradNode(); - GradNodeScale* scale_node0 = dynamic_cast(grad_node0); - PADDLE_ENFORCE( - scale_node0 != nullptr, - paddle::platform::errors::Fatal("AutogradMeta should have held " - "grad_node with type GradNodeScale*")); - PADDLE_ENFORCE( - (meta0->OutRankInfo().first == 0) && (meta0->OutRankInfo().second == 0), - paddle::platform::errors::Fatal( - "OutRankInfo in AutogradMeta should have been 0")); - - // Node 1 - AutogradMeta* meta1 = EagerUtils::autograd_meta(&out1); - GradNodeBase* grad_node1 = meta1->GradNode(); - GradNodeScale* scale_node1 = dynamic_cast(grad_node1); - PADDLE_ENFORCE( - scale_node1 != nullptr, - paddle::platform::errors::Fatal("AutogradMeta should have held " - "grad_node with type GradNodeScale*")); - PADDLE_ENFORCE( - (meta1->OutRankInfo().first == 0) && (meta1->OutRankInfo().second == 0), - paddle::platform::errors::Fatal( - "OutRankInfo in AutogradMeta should have been 0")); - - // 2. TensorWrapper: No TensorWrapper for ScaleNode - // 3. NextEdges: Node 1 -> Node 0 - const std::vector>& node1_edges = grad_node1->GetEdges(); - PADDLE_ENFORCE(node1_edges.size() == 1, - paddle::platform::errors::Fatal( - "Node 1 should have exactly 1 edge slot ")); - - const auto& node1_edge = node1_edges[0]; - PADDLE_ENFORCE(node1_edge.size() == 1, - paddle::platform::errors::Fatal( - "Node 1 should have exactly 1 edge in slot 0")); - PADDLE_ENFORCE((node1_edge[0].GetEdgeRankInfo().first == 0) && - (node1_edge[0].GetEdgeRankInfo().second == 0), - paddle::platform::errors::Fatal( - "Node1's edge should have input rank of 0")); - - PADDLE_ENFORCE( - node1_edge[0].GetGradNode() == grad_node0, - paddle::platform::errors::Fatal("Node1's edge should point to Node 0")); - } -} - -/* - inp - | - Node0 - ____|____ - | | - Node1 Node2 - | | - out1 out2 -*/ -TEST(Forward, BranchedNodes) { - InitEnv(paddle::platform::CPUPlace()); - - // Prepare Inputs - std::vector target_tensors; - paddle::framework::DDim ddim = paddle::framework::make_ddim({4, 16, 16, 32}); - - // Create Target Tensor - egr::EagerTensor t = EagerUtils::CreateTensorWithValue( - ddim, paddle::platform::CPUPlace(), pten::DataType::FLOAT32, - pten::DataLayout::NCHW, 5.0 /*value*/, false /*is_leaf*/); - target_tensors.emplace_back(std::move(t)); - egr::EagerTensor& tensor = target_tensors[0]; - EagerUtils::autograd_meta(&tensor)->SetStopGradient(false); - - // Run Forward Node 0 - float scale0 = 2.0; - float bias0 = 3.0; - egr::EagerTensor out0 = - egr::scale(tensor, scale0, bias0, true /*bias_after_scale*/, - true /*trace_backward*/); - - // Run Forward Node 1 - float scale1 = 5.0; - float bias1 = 10.0; - egr::EagerTensor out1 = egr::scale( - out0, scale1, bias1, true /*bias_after_scale*/, true /*trace_backward*/); - - // Run Forward Node 2 - float scale2 = 10.0; - float bias2 = 20.0; - egr::EagerTensor out2 = egr::scale( - out0, scale2, bias2, true /*bias_after_scale*/, true /*trace_backward*/); - - // Examine Forward Output 0 - CompareTensorWithValue(out0, 13.0); - - // Examine Forward Output 1 - CompareTensorWithValue(out1, 75.0); - - // Examine Forward Output 2 - CompareTensorWithValue(out2, 150.0); - - // Examine GradNode - { - // 1. GradNode - // Node 0 - AutogradMeta* meta0 = EagerUtils::autograd_meta(&out0); - GradNodeBase* grad_node0 = meta0->GradNode(); - GradNodeScale* scale_node0 = dynamic_cast(grad_node0); - PADDLE_ENFORCE( - scale_node0 != nullptr, - paddle::platform::errors::Fatal("AutogradMeta should have held " - "grad_node with type GradNodeScale*")); - PADDLE_ENFORCE( - (meta0->OutRankInfo().first == 0) && (meta0->OutRankInfo().second == 0), - paddle::platform::errors::Fatal( - "OutRankInfo in AutogradMeta should have been 0")); - - // Node 1 - AutogradMeta* meta1 = EagerUtils::autograd_meta(&out1); - GradNodeBase* grad_node1 = meta1->GradNode(); - GradNodeScale* scale_node1 = dynamic_cast(grad_node1); - PADDLE_ENFORCE( - scale_node1 != nullptr, - paddle::platform::errors::Fatal("AutogradMeta should have held " - "grad_node with type GradNodeScale*")); - PADDLE_ENFORCE( - (meta1->OutRankInfo().first == 0) && (meta1->OutRankInfo().second == 0), - paddle::platform::errors::Fatal( - "OutRankInfo in AutogradMeta should have been 0")); - - // Node 2 - AutogradMeta* meta2 = EagerUtils::autograd_meta(&out2); - GradNodeBase* grad_node2 = meta2->GradNode(); - GradNodeScale* scale_node2 = dynamic_cast(grad_node2); - PADDLE_ENFORCE( - scale_node2 != nullptr, - paddle::platform::errors::Fatal("AutogradMeta should have held " - "grad_node with type GradNodeScale*")); - PADDLE_ENFORCE( - (meta2->OutRankInfo().first == 0) && (meta2->OutRankInfo().second == 0), - paddle::platform::errors::Fatal( - "OutRankInfo in AutogradMeta should have been 0")); - - // 2. TensorWrapper: No TensorWrapper for ScaleNode - // 3. NextEdges - // Node 1 -> Node 0 - const std::vector>& node1_edges = grad_node1->GetEdges(); - PADDLE_ENFORCE(node1_edges.size() == 1, - paddle::platform::errors::Fatal( - "Node 1 should have exactly 1 edge slot")); - PADDLE_ENFORCE(node1_edges[0].size() == 1, - paddle::platform::errors::Fatal( - "Node 1 should have exactly 1 edge in slot 0")); - const Edge& node1_edge = node1_edges[0][0]; - PADDLE_ENFORCE((node1_edge.GetEdgeRankInfo().first == 0) && - (node1_edge.GetEdgeRankInfo().second == 0), - paddle::platform::errors::Fatal( - "Node1's edge should have input rank of 0")); - PADDLE_ENFORCE( - node1_edge.GetGradNode() == grad_node0, - paddle::platform::errors::Fatal("Node1's edge should point to Node 0")); - - // Node 2 -> Node 0 - const std::vector>& node2_edges = grad_node2->GetEdges(); - PADDLE_ENFORCE(node2_edges.size() == 1, - paddle::platform::errors::Fatal( - "Node 2 should have exactly 1 edge slot")); - PADDLE_ENFORCE(node1_edges[0].size() == 1, - paddle::platform::errors::Fatal( - "Node 2 should have exactly 1 edge in slot 0")); - const Edge& node2_edge = node2_edges[0][0]; - PADDLE_ENFORCE((node2_edge.GetEdgeRankInfo().first == 0) && - (node2_edge.GetEdgeRankInfo().second == 0), - paddle::platform::errors::Fatal( - "Node2's edge should have input rank of 0")); - PADDLE_ENFORCE( - node2_edge.GetGradNode() == grad_node0, - paddle::platform::errors::Fatal("Node2's edge should point to Node 0")); - } -} From 14a33d99478d18f7f90eeaa92796b3bcb9e774c6 Mon Sep 17 00:00:00 2001 From: jim19930609 Date: Thu, 18 Nov 2021 06:26:28 +0000 Subject: [PATCH 4/9] Bug fix v2 --- .../api/generated/eager_generated/forwards/function_api.cc | 2 +- .../fluid/eager/api/generated/eager_generated/forwards/scale.cc | 2 +- paddle/fluid/eager/tests/forward_autograd_test.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.cc b/paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.cc index 8039d01771292..8ff02cc8d6685 100644 --- a/paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.cc +++ b/paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.cc @@ -118,7 +118,7 @@ static void FillConstCUDAFunctor(pten::DenseTensor* tensor_dense, double value) { paddle::platform::DeviceContextPool& pool = paddle::platform::DeviceContextPool::Instance(); - auto* dev_ctx = dynamic_cast( + auto* dev_ctx = dynamic_cast( pool.Get(paddle::platform::CUDAPlace())); auto stream = dev_ctx->stream(); diff --git a/paddle/fluid/eager/api/generated/eager_generated/forwards/scale.cc b/paddle/fluid/eager/api/generated/eager_generated/forwards/scale.cc index 6127ef889a5b6..5f0a75c3dff83 100644 --- a/paddle/fluid/eager/api/generated/eager_generated/forwards/scale.cc +++ b/paddle/fluid/eager/api/generated/eager_generated/forwards/scale.cc @@ -23,10 +23,10 @@ * **/ #include "paddle/fluid/eager/api/generated/eager_generated/forwards/scale.h" +#include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h" #include "paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.h" #include "paddle/fluid/eager/autograd_meta.h" #include "paddle/fluid/eager/eager_tensor.h" -#include "paddle/fluid/eager/function_api.h" #include "paddle/pten/api/all.h" #include "paddle/pten/include/core.h" diff --git a/paddle/fluid/eager/tests/forward_autograd_test.cc b/paddle/fluid/eager/tests/forward_autograd_test.cc index 03e34f24763b7..f09cb1893df35 100644 --- a/paddle/fluid/eager/tests/forward_autograd_test.cc +++ b/paddle/fluid/eager/tests/forward_autograd_test.cc @@ -17,7 +17,7 @@ #include "glog/logging.h" #include "gtest/gtest.h" -#include "paddle/fluid/eager/api/api.h" +#include "paddle/fluid/eager/api/all.h" #include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h" #include "paddle/fluid/eager/autograd_meta.h" #include "paddle/fluid/eager/grad_node_info.h" From a3cd73f8160f59b15d325288b93bb92ff84e7786 Mon Sep 17 00:00:00 2001 From: jim19930609 Date: Tue, 23 Nov 2021 11:37:05 +0000 Subject: [PATCH 5/9] Removed function_api and re-arranged scale --- .../eager_generated/backwards/CMakeLists.txt | 2 +- .../eager_generated/backwards/scale_node.cc | 93 +++++- .../eager_generated/backwards/scale_node.h | 3 + .../eager_generated/forwards/CMakeLists.txt | 3 +- .../eager_generated/forwards/function_api.cc | 278 ------------------ .../eager_generated/forwards/function_api.h | 37 --- .../eager_generated/forwards/scale.cc | 2 +- 7 files changed, 98 insertions(+), 320 deletions(-) delete mode 100644 paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.cc delete mode 100644 paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.h diff --git a/paddle/fluid/eager/api/generated/eager_generated/backwards/CMakeLists.txt b/paddle/fluid/eager/api/generated/eager_generated/backwards/CMakeLists.txt index 5fa414819f343..1084f0ec573c6 100644 --- a/paddle/fluid/eager/api/generated/eager_generated/backwards/CMakeLists.txt +++ b/paddle/fluid/eager/api/generated/eager_generated/backwards/CMakeLists.txt @@ -1 +1 @@ -cc_library(scale_node SRCS scale_node.cc DEPS function_api pten pten_api grad_node_info) +cc_library(scale_node SRCS scale_node.cc DEPS global_utils pten pten_api grad_node_info) diff --git a/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.cc b/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.cc index 287812852a578..ed662d07e0db0 100644 --- a/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.cc +++ b/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.cc @@ -13,7 +13,7 @@ // limitations under the License. #include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h" -#include "paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.h" +#include "paddle/fluid/eager/api/utils/global_utils.h" #include "paddle/fluid/eager/eager_tensor.h" #include "paddle/pten/api/all.h" @@ -26,6 +26,97 @@ namespace egr { +template +static void ScaleDeviceDispatch(const pten::DenseTensor& dense_tensor, + const DeviceContext& dev_ctx, float scale, + float bias, bool bias_after_scale, + pten::DenseTensor* dense_out) { + switch (dense_tensor.dtype()) { + case pten::DataType::FLOAT64: { + pten::Scale(dev_ctx, dense_tensor /* tensor */, scale /* scale */, + bias /* bias */, + bias_after_scale /* bias_after_scale */, + dense_out /* out tensor */); + break; + } + case pten::DataType::FLOAT32: { + pten::Scale(dev_ctx, dense_tensor /* tensor */, scale /* scale */, + bias /* bias */, + bias_after_scale /* bias_after_scale */, + dense_out /* out tensor */); + break; + } + case pten::DataType::INT64: { + pten::Scale(dev_ctx, dense_tensor /* tensor */, + scale /* scale */, bias /* bias */, + bias_after_scale /* bias_after_scale */, + dense_out /* out tensor */); + break; + } + case pten::DataType::INT32: { + pten::Scale(dev_ctx, dense_tensor /* tensor */, + scale /* scale */, bias /* bias */, + bias_after_scale /* bias_after_scale */, + dense_out /* out tensor */); + break; + } + default: { + PADDLE_THROW(paddle::platform::errors::Fatal("Unsupported data type")); + break; + } + } +} + +void ScaleAPI(const egr::EagerTensor& x, float scale, float bias, + bool bias_after_scale, egr::EagerTensor* out) { + // TODO(jiabin): Support multiple tensor here, Create DenseTensor is not a + // proper way to Demo it + // Run Forward Function + auto dense_tensor = std::dynamic_pointer_cast(x.impl()); + // Init output tensor + auto tensor_meta = pten::DenseTensorMeta( + dense_tensor->dtype(), dense_tensor->dims(), dense_tensor->layout()); + auto place = dense_tensor->place(); + size_t bytes_size = paddle::framework::product(dense_tensor->dims()) * + SizeOf(dense_tensor->dtype()); + auto dense_out = std::make_shared( + pten::make_intrusive( + paddle::memory::Alloc(place, bytes_size), 0), + std::move(tensor_meta)); + // Handle Device Context + const paddle::platform::Place& expected_kernel_place = + Controller::Instance().GetExpectedPlace(); + paddle::platform::DeviceContextPool& pool = + paddle::platform::DeviceContextPool::Instance(); + + if (expected_kernel_place == paddle::platform::CPUPlace()) { + auto* dev_ctx = dynamic_cast( + pool.Get(expected_kernel_place)); + if (!dev_ctx) { + PADDLE_THROW(paddle::platform::errors::Fatal("Backend mismatch")); + } + ScaleDeviceDispatch( + *dense_tensor.get(), *dev_ctx, scale, bias, bias_after_scale, + dense_out.get()); + + } else if (expected_kernel_place == paddle::platform::CUDAPlace()) { + auto* dev_ctx = dynamic_cast( + pool.Get(expected_kernel_place)); + if (!dev_ctx) { + PADDLE_THROW(paddle::platform::errors::Fatal("Backend mismatch")); + } + ScaleDeviceDispatch( + *dense_tensor.get(), *dev_ctx, scale, bias, bias_after_scale, + dense_out.get()); + + } else { + PADDLE_THROW(paddle::platform::errors::Fatal( + "Only CPU and CUDA Backend are supported for now")); + } + + out->set_impl(dense_out); +} + void GradNodeScale::SetTensorWrappers_X( const std::vector& tensors) { // Does nothing for scale diff --git a/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h b/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h index 8881be31df79d..1618662516ce3 100644 --- a/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h +++ b/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h @@ -27,6 +27,9 @@ */ namespace egr { +void ScaleAPI(const egr::EagerTensor& x, float scale, float bias, + bool bias_after_scale, egr::EagerTensor* out); + class GradNodeScale : public GradNodeBase { public: // Constructor: configure fwd input tensors to grad node diff --git a/paddle/fluid/eager/api/generated/eager_generated/forwards/CMakeLists.txt b/paddle/fluid/eager/api/generated/eager_generated/forwards/CMakeLists.txt index 02837d20327a2..ed04e0b6f5a0c 100644 --- a/paddle/fluid/eager/api/generated/eager_generated/forwards/CMakeLists.txt +++ b/paddle/fluid/eager/api/generated/eager_generated/forwards/CMakeLists.txt @@ -1,2 +1 @@ -cc_library(function_api SRCS function_api.cc DEPS pten pten_api layer proto_desc operator op_registry variable_helper memcpy scale_op) -cc_library(eager_scale SRCS scale.cc DEPS function_api pten autograd_meta scale_node) +cc_library(eager_scale SRCS scale.cc DEPS pten_api pten autograd_meta scale_node) diff --git a/paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.cc b/paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.cc deleted file mode 100644 index 8ff02cc8d6685..0000000000000 --- a/paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.cc +++ /dev/null @@ -1,278 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.h" -#include "paddle/fluid/eager/api/utils/global_utils.h" - -#include "paddle/pten/api/all.h" -#include "paddle/pten/core/dense_tensor.h" -#include "paddle/pten/include/core.h" - -#include "paddle/fluid/memory/memcpy.h" -#include "paddle/fluid/operators/math/math_function.h" -#include "paddle/fluid/platform/device_context.h" - -namespace egr { - -template -static void ScaleDeviceDispatch(const pten::DenseTensor& dense_tensor, - const DeviceContext& dev_ctx, float scale, - float bias, bool bias_after_scale, - pten::DenseTensor* dense_out) { - switch (dense_tensor.dtype()) { - case pten::DataType::FLOAT64: { - pten::Scale(dev_ctx, dense_tensor /* tensor */, scale /* scale */, - bias /* bias */, - bias_after_scale /* bias_after_scale */, - dense_out /* out tensor */); - break; - } - case pten::DataType::FLOAT32: { - pten::Scale(dev_ctx, dense_tensor /* tensor */, scale /* scale */, - bias /* bias */, - bias_after_scale /* bias_after_scale */, - dense_out /* out tensor */); - break; - } - case pten::DataType::INT64: { - pten::Scale(dev_ctx, dense_tensor /* tensor */, - scale /* scale */, bias /* bias */, - bias_after_scale /* bias_after_scale */, - dense_out /* out tensor */); - break; - } - case pten::DataType::INT32: { - pten::Scale(dev_ctx, dense_tensor /* tensor */, - scale /* scale */, bias /* bias */, - bias_after_scale /* bias_after_scale */, - dense_out /* out tensor */); - break; - } - default: { - PADDLE_THROW(paddle::platform::errors::Fatal("Unsupported data type")); - break; - } - } -} - -// TODO(jiabin): This may have serious performance issue move code from -// gradient_accumulator.cc -static void FillConstCPUFunctor(pten::DenseTensor* tensor_dense, double value) { - PADDLE_ENFORCE(tensor_dense, paddle::platform::errors::Fatal( - "Receive nullptr of dense tensor")); - switch (tensor_dense->dtype()) { - case pten::DataType::INT64: { - int64_t* data_ptr = tensor_dense->mutable_data(); - for (int i = 0; i < tensor_dense->numel(); i++) { - data_ptr[i] = static_cast(value); - } - - break; - } - case pten::DataType::INT32: { - int32_t* data_ptr = tensor_dense->mutable_data(); - for (int i = 0; i < tensor_dense->numel(); i++) { - data_ptr[i] = static_cast(value); - } - - break; - } - case pten::DataType::FLOAT64: { - double* data_ptr = tensor_dense->mutable_data(); - for (int i = 0; i < tensor_dense->numel(); i++) { - data_ptr[i] = static_cast(value); - } - - break; - } - case pten::DataType::FLOAT32: { - float* data_ptr = tensor_dense->mutable_data(); - for (int i = 0; i < tensor_dense->numel(); i++) { - data_ptr[i] = static_cast(value); - } - break; - } - default: { - PADDLE_THROW(paddle::platform::errors::Fatal( - "Only supports tensor with fp32, fp64, int32, int64 datatypes for " - "now")); - break; - } - } -} - -// TODO(jiabin): This may have serious performance issue move code from -// gradient_accumulator.cc -static void FillConstCUDAFunctor(pten::DenseTensor* tensor_dense, - double value) { - paddle::platform::DeviceContextPool& pool = - paddle::platform::DeviceContextPool::Instance(); - auto* dev_ctx = dynamic_cast( - pool.Get(paddle::platform::CUDAPlace())); - auto stream = dev_ctx->stream(); - - switch (tensor_dense->dtype()) { - case pten::DataType::INT64: { - std::vector host_data(tensor_dense->numel(), - static_cast(value)); - int64_t* device_ptr = tensor_dense->mutable_data(); - paddle::memory::Copy(paddle::platform::CUDAPlace(), device_ptr, - paddle::platform::CPUPlace(), host_data.data(), - sizeof(int64_t) * tensor_dense->numel(), stream); - break; - } - case pten::DataType::INT32: { - std::vector host_data(tensor_dense->numel(), - static_cast(value)); - int32_t* device_ptr = tensor_dense->mutable_data(); - paddle::memory::Copy(paddle::platform::CUDAPlace(), device_ptr, - paddle::platform::CPUPlace(), host_data.data(), - sizeof(int32_t) * tensor_dense->numel(), stream); - break; - } - case pten::DataType::FLOAT64: { - std::vector host_data(tensor_dense->numel(), - static_cast(value)); - double* device_ptr = tensor_dense->mutable_data(); - paddle::memory::Copy(paddle::platform::CUDAPlace(), device_ptr, - paddle::platform::CPUPlace(), host_data.data(), - sizeof(double) * tensor_dense->numel(), stream); - break; - } - case pten::DataType::FLOAT32: { - std::vector host_data(tensor_dense->numel(), - static_cast(value)); - float* device_ptr = tensor_dense->mutable_data(); - paddle::memory::Copy(paddle::platform::CUDAPlace(), device_ptr, - paddle::platform::CPUPlace(), host_data.data(), - sizeof(float) * tensor_dense->numel(), stream); - break; - } - default: { - PADDLE_THROW(paddle::platform::errors::Fatal( - "Only supports tensor with fp32, fp64, int32, int64 datatypes for " - "now")); - break; - } - } -} - -void ScaleAPI(const egr::EagerTensor& x, float scale, float bias, - bool bias_after_scale, egr::EagerTensor* out) { - // TODO(jiabin): Support multiple tensor here, Create DenseTensor is not a - // proper way to Demo it - // Run Forward Function - auto dense_tensor = std::dynamic_pointer_cast(x.impl()); - // Init output tensor - auto tensor_meta = pten::DenseTensorMeta( - dense_tensor->dtype(), dense_tensor->dims(), dense_tensor->layout()); - auto place = dense_tensor->place(); - size_t bytes_size = paddle::framework::product(dense_tensor->dims()) * - SizeOf(dense_tensor->dtype()); - auto dense_out = std::make_shared( - pten::make_intrusive( - paddle::memory::Alloc(place, bytes_size), 0), - std::move(tensor_meta)); - // Handle Device Context - const paddle::platform::Place& expected_kernel_place = - Controller::Instance().GetExpectedPlace(); - paddle::platform::DeviceContextPool& pool = - paddle::platform::DeviceContextPool::Instance(); - - if (expected_kernel_place == paddle::platform::CPUPlace()) { - auto* dev_ctx = dynamic_cast( - pool.Get(expected_kernel_place)); - if (!dev_ctx) { - PADDLE_THROW(paddle::platform::errors::Fatal("Backend mismatch")); - } - ScaleDeviceDispatch( - *dense_tensor.get(), *dev_ctx, scale, bias, bias_after_scale, - dense_out.get()); - - } else if (expected_kernel_place == paddle::platform::CUDAPlace()) { - auto* dev_ctx = dynamic_cast( - pool.Get(expected_kernel_place)); - if (!dev_ctx) { - PADDLE_THROW(paddle::platform::errors::Fatal("Backend mismatch")); - } - ScaleDeviceDispatch( - *dense_tensor.get(), *dev_ctx, scale, bias, bias_after_scale, - dense_out.get()); - - } else { - PADDLE_THROW(paddle::platform::errors::Fatal( - "Only CPU and CUDA Backend are supported for now")); - } - - out->set_impl(dense_out); -} - -void FillConstAPI(double value, const pten::DDim& ddim, - const paddle::platform::Place& place, - const pten::DataType& dtype, const pten::DataLayout& layout, - egr::EagerTensor* target) { - // Create new tensor->impl and fill it with 1.0 - // Fill 1.0 - // TODO(jiabin): Refactor this with operators::math::set_constant - std::shared_ptr tensor_dense = nullptr; - if (!target->defined() || !target->initialized()) { - VLOG(6) << "Init undefined or uninitialized tensor in FillConstAPI"; - auto tensor_meta = pten::DenseTensorMeta(dtype, ddim, layout); - size_t bytes_size = paddle::framework::product(ddim) * SizeOf(dtype); - - tensor_dense = std::make_shared( - pten::make_intrusive( - paddle::memory::Alloc(place, bytes_size), 0), - std::move(tensor_meta)); - target->set_impl(tensor_dense); - - } else { - tensor_dense = std::dynamic_pointer_cast(target->impl()); - } - - if (!tensor_dense) { - PADDLE_THROW(paddle::platform::errors::Fatal( - "FillConstAPI Only supports InputBuffer with DenseTensor for now.")); - } - VLOG(6) << "Call FillConstKernel"; - auto t_place = tensor_dense->place(); - if (paddle::platform::is_cpu_place(t_place)) { - VLOG(8) << "Call FillConst CPU Kernel"; - FillConstCPUFunctor(tensor_dense.get(), value); - } else if (paddle::platform::is_gpu_place(t_place)) { - VLOG(8) << "Call FillConst CUDA Kernel"; - FillConstCUDAFunctor(tensor_dense.get(), value); - } else { - PADDLE_THROW(paddle::platform::errors::Fatal( - "Only CPU and CUDA Backend are supported for now")); - } -} - -void FillConstAPI(double value, const paddle::framework::DDim& ddim, - const paddle::platform::Place& place, - const paddle::framework::proto::VarType::Type& dtype, - egr::EagerTensor* target) { - auto* dst_tensor = - target->MutableVar()->GetMutable(); - auto* dev_ctx = paddle::platform::DeviceContextPool::Instance().Get(place); - dst_tensor->Resize(ddim); - // TOOD(jiabin): Ugly fix here we have fwd_data_type_ and data_type, since in - // grad mission - // we can't get data_type_ directly. We need to check if we can only use - // default data_type for now. - dst_tensor->mutable_data(place, dtype); - paddle::operators::math::set_constant(*dev_ctx, dst_tensor, value); -} - -} // namespace egr diff --git a/paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.h b/paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.h deleted file mode 100644 index a6b916406389a..0000000000000 --- a/paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.h +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include "paddle/fluid/eager/eager_tensor.h" - -#include "paddle/pten/api/all.h" -#include "paddle/pten/include/core.h" - -#include "paddle/fluid/platform/enforce.h" - -namespace egr { - -// Public -void ScaleAPI(const egr::EagerTensor& x, float scale, float bias, - bool bias_after_scale, egr::EagerTensor* out); -void FillConstAPI(double value, const pten::DDim& ddim, - const paddle::platform::Place& place, - const pten::DataType& dtype, const pten::DataLayout& layout, - egr::EagerTensor* target); -void FillConstAPI(double value, const paddle::framework::DDim& ddim, - const paddle::platform::Place& place, - const paddle::framework::proto::VarType::Type& dtype, - egr::EagerTensor* target); -} // namespace egr diff --git a/paddle/fluid/eager/api/generated/eager_generated/forwards/scale.cc b/paddle/fluid/eager/api/generated/eager_generated/forwards/scale.cc index 5f0a75c3dff83..e38118b8b3ee0 100644 --- a/paddle/fluid/eager/api/generated/eager_generated/forwards/scale.cc +++ b/paddle/fluid/eager/api/generated/eager_generated/forwards/scale.cc @@ -24,9 +24,9 @@ #include "paddle/fluid/eager/api/generated/eager_generated/forwards/scale.h" #include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h" -#include "paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.h" #include "paddle/fluid/eager/autograd_meta.h" #include "paddle/fluid/eager/eager_tensor.h" +#include "paddle/fluid/eager/utils.h" #include "paddle/pten/api/all.h" #include "paddle/pten/include/core.h" From 4f930c2fc3ca9863aeb8bc399a66d32be2fe22bd Mon Sep 17 00:00:00 2001 From: jim19930609 Date: Thu, 25 Nov 2021 08:08:56 +0000 Subject: [PATCH 6/9] Removed function_api dependencies --- paddle/fluid/eager/CMakeLists.txt | 2 +- paddle/fluid/eager/api/CMakeLists.txt | 2 +- paddle/fluid/eager/api/all.h | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/paddle/fluid/eager/CMakeLists.txt b/paddle/fluid/eager/CMakeLists.txt index 89f4d42bfc26c..7afd35f847f42 100644 --- a/paddle/fluid/eager/CMakeLists.txt +++ b/paddle/fluid/eager/CMakeLists.txt @@ -1,4 +1,4 @@ -set(eager_deps eager_api utils function_api autograd_meta grad_node_info pten pten_api) +set(eager_deps eager_api utils autograd_meta grad_node_info pten pten_api) set(fluid_deps tracer layer proto_desc operator op_registry variable_helper memcpy) add_subdirectory(api) diff --git a/paddle/fluid/eager/api/CMakeLists.txt b/paddle/fluid/eager/api/CMakeLists.txt index fee7f27b2657e..c7c58a543651d 100644 --- a/paddle/fluid/eager/api/CMakeLists.txt +++ b/paddle/fluid/eager/api/CMakeLists.txt @@ -1,4 +1,4 @@ add_subdirectory(utils) add_subdirectory(generated) -cc_library(eager_api SRCS all.cc DEPS global_utils function_api eager_scale) +cc_library(eager_api SRCS all.cc DEPS global_utils eager_scale) diff --git a/paddle/fluid/eager/api/all.h b/paddle/fluid/eager/api/all.h index 2f95c1aff6c6a..6750073c3d336 100644 --- a/paddle/fluid/eager/api/all.h +++ b/paddle/fluid/eager/api/all.h @@ -14,6 +14,5 @@ // #pragma once -#include "paddle/fluid/eager/api/generated/eager_generated/forwards/function_api.h" #include "paddle/fluid/eager/api/generated/eager_generated/forwards/scale.h" #include "paddle/fluid/eager/api/utils/global_utils.h" From 4e208a0afccb9b3f1dc022117ac845df7f6778cb Mon Sep 17 00:00:00 2001 From: jim19930609 Date: Thu, 25 Nov 2021 08:19:07 +0000 Subject: [PATCH 7/9] Resolve merge conflicts --- .../api/generated/eager_generated/forwards/scale.cc | 5 +++-- paddle/fluid/eager/api/utils/CMakeLists.txt | 1 + paddle/fluid/eager/tests/CMakeLists.txt | 4 +--- paddle/fluid/eager/tests/task_tests/CMakeLists.txt | 1 + .../tests/{ => task_tests}/forward_autograd_test.cc | 11 ++++++++--- 5 files changed, 14 insertions(+), 8 deletions(-) rename paddle/fluid/eager/tests/{ => task_tests}/forward_autograd_test.cc (97%) diff --git a/paddle/fluid/eager/api/generated/eager_generated/forwards/scale.cc b/paddle/fluid/eager/api/generated/eager_generated/forwards/scale.cc index e38118b8b3ee0..a8b3421baac02 100644 --- a/paddle/fluid/eager/api/generated/eager_generated/forwards/scale.cc +++ b/paddle/fluid/eager/api/generated/eager_generated/forwards/scale.cc @@ -55,9 +55,10 @@ egr::EagerTensor scale(const egr::EagerTensor& x, float scale, float bias, // TODO(zhanlve): which one is more efficient: // 1. construct a vector of pointers // 2. call "ComputeRequireGrad" multiple times - bool require_any_grad = ComputeRequireGrad(trace_backward, p_autograd_in); + bool require_any_grad = + EagerUtils::ComputeRequireGrad(trace_backward, p_autograd_in); if (require_any_grad) { - PassStopGradient(false /*generate_grad*/, p_autograd_out); + EagerUtils::PassStopGradient(false /*generate_grad*/, p_autograd_out); // 2.2.2 Set OutRankInfo for outputs this needs to be as same as Edges's // input_rank_ diff --git a/paddle/fluid/eager/api/utils/CMakeLists.txt b/paddle/fluid/eager/api/utils/CMakeLists.txt index 2bce8e0bc3072..64c95df963c66 100644 --- a/paddle/fluid/eager/api/utils/CMakeLists.txt +++ b/paddle/fluid/eager/api/utils/CMakeLists.txt @@ -1 +1,2 @@ cc_library(global_utils SRCS global_utils.cc DEPS place) +cc_library(tensor_utils SRCS tensor_utils.cc DEPS pten pten_api autograd_meta grad_node_info accumulation_node) diff --git a/paddle/fluid/eager/tests/CMakeLists.txt b/paddle/fluid/eager/tests/CMakeLists.txt index 5c97d67e10cfc..adff9572761e9 100644 --- a/paddle/fluid/eager/tests/CMakeLists.txt +++ b/paddle/fluid/eager/tests/CMakeLists.txt @@ -1,7 +1,5 @@ -set(eager_deps pten pten_api utils global_utils pten_tensor autograd_meta grad_node_info grad_tensor_holder gradient_accumulation accumulation_node) +set(eager_deps pten pten_api tensor_utils utils global_utils pten_tensor autograd_meta grad_node_info grad_tensor_holder gradient_accumulation accumulation_node) set(fluid_deps tracer layer proto_desc operator op_registry variable_helper memcpy) add_subdirectory(data_structure_tests) add_subdirectory(task_tests) - -cc_test(test_egr_forward_autograd SRCS forward_autograd_test.cc DEPS ${eager_deps} ${fluid_deps}) diff --git a/paddle/fluid/eager/tests/task_tests/CMakeLists.txt b/paddle/fluid/eager/tests/task_tests/CMakeLists.txt index 6ab5c70b44e82..4027ad9683082 100644 --- a/paddle/fluid/eager/tests/task_tests/CMakeLists.txt +++ b/paddle/fluid/eager/tests/task_tests/CMakeLists.txt @@ -1 +1,2 @@ cc_test(test_egr_task_eager_utils SRCS eager_utils_test.cc DEPS ${eager_deps}) +cc_test(test_egr_task_forward_autograd SRCS forward_autograd_test.cc DEPS ${eager_deps} ${fluid_deps} eager_scale scale_node) diff --git a/paddle/fluid/eager/tests/forward_autograd_test.cc b/paddle/fluid/eager/tests/task_tests/forward_autograd_test.cc similarity index 97% rename from paddle/fluid/eager/tests/forward_autograd_test.cc rename to paddle/fluid/eager/tests/task_tests/forward_autograd_test.cc index f09cb1893df35..ae9c7a3546238 100644 --- a/paddle/fluid/eager/tests/forward_autograd_test.cc +++ b/paddle/fluid/eager/tests/task_tests/forward_autograd_test.cc @@ -19,6 +19,7 @@ #include "paddle/fluid/eager/api/all.h" #include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h" +#include "paddle/fluid/eager/api/utils/tensor_utils.h" #include "paddle/fluid/eager/autograd_meta.h" #include "paddle/fluid/eager/grad_node_info.h" #include "paddle/fluid/eager/tests/test_utils.h" @@ -29,6 +30,8 @@ // TODO(jiabin): remove nolint here!!! using namespace egr; // NOLINT +namespace eager_test { + TEST(Forward, SingleNode) { // Prepare Device Contexts InitEnv(paddle::platform::CPUPlace()); @@ -38,7 +41,7 @@ TEST(Forward, SingleNode) { paddle::framework::DDim ddim = paddle::framework::make_ddim({4, 16, 16, 32}); // Create Target Tensor - egr::EagerTensor t = EagerUtils::CreateTensorWithValue( + egr::EagerTensor t = CreateTensorWithValue( ddim, paddle::platform::CPUPlace(), pten::DataType::FLOAT32, pten::DataLayout::NCHW, 5.0 /*value*/, false /*is_leaf*/); target_tensors.emplace_back(std::move(t)); @@ -91,7 +94,7 @@ TEST(Forward, LinearNodes) { paddle::framework::DDim ddim = paddle::framework::make_ddim({4, 16, 16, 32}); // Create Target Tensor - egr::EagerTensor t = EagerUtils::CreateTensorWithValue( + egr::EagerTensor t = CreateTensorWithValue( ddim, paddle::platform::CPUPlace(), pten::DataType::FLOAT32, pten::DataLayout::NCHW, 5.0 /*value*/, false /*is_leaf*/); target_tensors.emplace_back(std::move(t)); @@ -186,7 +189,7 @@ TEST(Forward, BranchedNodes) { paddle::framework::DDim ddim = paddle::framework::make_ddim({4, 16, 16, 32}); // Create Target Tensor - egr::EagerTensor t = EagerUtils::CreateTensorWithValue( + egr::EagerTensor t = CreateTensorWithValue( ddim, paddle::platform::CPUPlace(), pten::DataType::FLOAT32, pten::DataLayout::NCHW, 5.0 /*value*/, false /*is_leaf*/); target_tensors.emplace_back(std::move(t)); @@ -300,3 +303,5 @@ TEST(Forward, BranchedNodes) { paddle::platform::errors::Fatal("Node2's edge should point to Node 0")); } } + +} // namespace eager_test From 6d849597003b7f8505f5beaae57636d63604a00a Mon Sep 17 00:00:00 2001 From: jim19930609 Date: Fri, 26 Nov 2021 02:58:06 +0000 Subject: [PATCH 8/9] Fix CI issues --- paddle/fluid/eager/api/utils/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/paddle/fluid/eager/api/utils/CMakeLists.txt b/paddle/fluid/eager/api/utils/CMakeLists.txt index 39a9e0becec20..600212cae71ec 100644 --- a/paddle/fluid/eager/api/utils/CMakeLists.txt +++ b/paddle/fluid/eager/api/utils/CMakeLists.txt @@ -1,3 +1,2 @@ cc_library(tensor_utils SRCS tensor_utils.cc DEPS pten pten_api autograd_meta grad_node_info accumulation_node) cc_library(global_utils SRCS global_utils.cc DEPS place) -cc_library(tensor_utils SRCS tensor_utils.cc DEPS pten pten_api autograd_meta grad_node_info accumulation_node) From a08394c3b623f1d8f682fe8b8bd2e55e4efee95a Mon Sep 17 00:00:00 2001 From: jim19930609 Date: Fri, 26 Nov 2021 05:57:57 +0000 Subject: [PATCH 9/9] Replaced PADDLE_THROW PADDLE_ENFORCE with CHECK_EQ in test files --- .../eager_generated/backwards/scale_node.cc | 31 +++-- .../tests/task_tests/forward_autograd_test.cc | 126 +++++------------- 2 files changed, 57 insertions(+), 100 deletions(-) diff --git a/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.cc b/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.cc index 1ede557d47609..e1e8f09ddff8f 100644 --- a/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.cc +++ b/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.cc @@ -61,7 +61,9 @@ static void ScaleDeviceDispatch(const pten::DenseTensor& dense_tensor, break; } default: { - PADDLE_THROW(paddle::platform::errors::Fatal("Unsupported data type")); + PADDLE_THROW(paddle::platform::errors::Fatal( + "Detected unsupported data type." + "Only Float64, Float32, Int64, Int32 are supported for now.")); break; } } @@ -93,7 +95,10 @@ void ScaleAPI(const egr::EagerTensor& x, float scale, float bias, auto* dev_ctx = dynamic_cast( pool.Get(expected_kernel_place)); if (!dev_ctx) { - PADDLE_THROW(paddle::platform::errors::Fatal("Backend mismatch")); + PADDLE_THROW(paddle::platform::errors::Fatal( + "Cannot convert device_context to CPUDeviceContext." + "This indicates backend mismatch." + "Pleas double check your expected place")); } ScaleDeviceDispatch( *dense_tensor.get(), *dev_ctx, scale, bias, bias_after_scale, @@ -104,7 +109,10 @@ void ScaleAPI(const egr::EagerTensor& x, float scale, float bias, auto* dev_ctx = dynamic_cast( pool.Get(expected_kernel_place)); if (!dev_ctx) { - PADDLE_THROW(paddle::platform::errors::Fatal("Backend mismatch")); + PADDLE_THROW(paddle::platform::errors::Fatal( + "Cannot convert device_context to CUDADeviceContext." + "This indicates backend mismatch." + "Pleas double check your expected place")); } ScaleDeviceDispatch( *dense_tensor.get(), *dev_ctx, scale, bias, bias_after_scale, @@ -112,7 +120,10 @@ void ScaleAPI(const egr::EagerTensor& x, float scale, float bias, #endif } else { PADDLE_THROW(paddle::platform::errors::Fatal( - "Only CPU and CUDA Backend are supported for now")); + "Detected unsupported backend." + "Only CPU and CUDA Backend are supported for now." + "Please double check if your backend falls into the above two " + "categories.")); } out->set_impl(dense_out); @@ -128,11 +139,13 @@ void GradNodeScale::SetAttributes_scale(float scale) { scale_ = scale; } std::vector> GradNodeScale::operator()( const std::vector>& grads) { // 1. Check Output Size - PADDLE_ENFORCE(((grads.size() == 1) && (grads[0].size() == 1)), - paddle::platform::errors::Fatal( - "ScaleGradNode should take exactly 1 grad tensor" - "However received: %d", - grads.size())); + PADDLE_ENFORCE( + ((grads.size() == 1) && (grads[0].size() == 1)), + paddle::platform::errors::Fatal( + "ScaleGradNode takes exactly 1 grad tensor." + "However received: %d", + "This indicates an issue with Eager Dygraph Backward logic", + grads.size())); std::vector> outs; // 2. Create needed out parttern egr::EagerTensor out; diff --git a/paddle/fluid/eager/tests/task_tests/forward_autograd_test.cc b/paddle/fluid/eager/tests/task_tests/forward_autograd_test.cc index ae9c7a3546238..6e23226cde432 100644 --- a/paddle/fluid/eager/tests/task_tests/forward_autograd_test.cc +++ b/paddle/fluid/eager/tests/task_tests/forward_autograd_test.cc @@ -63,17 +63,10 @@ TEST(Forward, SingleNode) { AutogradMeta* meta = EagerUtils::autograd_meta(&out); GradNodeBase* grad_node = meta->GradNode(); GradNodeScale* scale_node = dynamic_cast(grad_node); - PADDLE_ENFORCE( - scale_node != nullptr, - paddle::platform::errors::Fatal("AutogradMeta should have held " - "grad_node with type GradNodeScale*")); - PADDLE_ENFORCE( - (meta->OutRankInfo().first == 0) && (meta->OutRankInfo().second == 0), - paddle::platform::errors::Fatal( - "OutRankInfo in AutogradMeta should have been 0")); - // 2. TensorWrapper: No TensorWrapper for ScaleNode - // 3. NextEdges: No NextEdges for Single Node Case + CHECK_NOTNULL(scale_node); + CHECK_EQ(static_cast(meta->OutRankInfo().first), 0); + CHECK_EQ(static_cast(meta->OutRankInfo().second), 0); } } @@ -127,47 +120,28 @@ TEST(Forward, LinearNodes) { AutogradMeta* meta0 = EagerUtils::autograd_meta(&out0); GradNodeBase* grad_node0 = meta0->GradNode(); GradNodeScale* scale_node0 = dynamic_cast(grad_node0); - PADDLE_ENFORCE( - scale_node0 != nullptr, - paddle::platform::errors::Fatal("AutogradMeta should have held " - "grad_node with type GradNodeScale*")); - PADDLE_ENFORCE( - (meta0->OutRankInfo().first == 0) && (meta0->OutRankInfo().second == 0), - paddle::platform::errors::Fatal( - "OutRankInfo in AutogradMeta should have been 0")); + + CHECK_NOTNULL(scale_node0); + CHECK_EQ(static_cast(meta0->OutRankInfo().first), 0); + CHECK_EQ(static_cast(meta0->OutRankInfo().second), 0); // Node 1 AutogradMeta* meta1 = EagerUtils::autograd_meta(&out1); GradNodeBase* grad_node1 = meta1->GradNode(); GradNodeScale* scale_node1 = dynamic_cast(grad_node1); - PADDLE_ENFORCE( - scale_node1 != nullptr, - paddle::platform::errors::Fatal("AutogradMeta should have held " - "grad_node with type GradNodeScale*")); - PADDLE_ENFORCE( - (meta1->OutRankInfo().first == 0) && (meta1->OutRankInfo().second == 0), - paddle::platform::errors::Fatal( - "OutRankInfo in AutogradMeta should have been 0")); + + CHECK_NOTNULL(scale_node1); + CHECK_EQ(static_cast(meta1->OutRankInfo().first), 0); + CHECK_EQ(static_cast(meta1->OutRankInfo().second), 0); // 2. TensorWrapper: No TensorWrapper for ScaleNode // 3. NextEdges: Node 1 -> Node 0 const std::vector>& node1_edges = grad_node1->GetEdges(); - PADDLE_ENFORCE(node1_edges.size() == 1, - paddle::platform::errors::Fatal( - "Node 1 should have exactly 1 edge slot ")); - const auto& node1_edge = node1_edges[0]; - PADDLE_ENFORCE(node1_edge.size() == 1, - paddle::platform::errors::Fatal( - "Node 1 should have exactly 1 edge in slot 0")); - PADDLE_ENFORCE((node1_edge[0].GetEdgeRankInfo().first == 0) && - (node1_edge[0].GetEdgeRankInfo().second == 0), - paddle::platform::errors::Fatal( - "Node1's edge should have input rank of 0")); - - PADDLE_ENFORCE( - node1_edge[0].GetGradNode() == grad_node0, - paddle::platform::errors::Fatal("Node1's edge should point to Node 0")); + + CHECK_EQ(static_cast(node1_edge[0].GetEdgeRankInfo().first), 0); + CHECK_EQ(static_cast(node1_edge[0].GetEdgeRankInfo().second), 0); + CHECK_EQ(node1_edge[0].GetGradNode(), grad_node0); } } @@ -231,76 +205,46 @@ TEST(Forward, BranchedNodes) { AutogradMeta* meta0 = EagerUtils::autograd_meta(&out0); GradNodeBase* grad_node0 = meta0->GradNode(); GradNodeScale* scale_node0 = dynamic_cast(grad_node0); - PADDLE_ENFORCE( - scale_node0 != nullptr, - paddle::platform::errors::Fatal("AutogradMeta should have held " - "grad_node with type GradNodeScale*")); - PADDLE_ENFORCE( - (meta0->OutRankInfo().first == 0) && (meta0->OutRankInfo().second == 0), - paddle::platform::errors::Fatal( - "OutRankInfo in AutogradMeta should have been 0")); + + CHECK_NOTNULL(scale_node0); + CHECK_EQ(static_cast(meta0->OutRankInfo().first), 0); + CHECK_EQ(static_cast(meta0->OutRankInfo().second), 0); // Node 1 AutogradMeta* meta1 = EagerUtils::autograd_meta(&out1); GradNodeBase* grad_node1 = meta1->GradNode(); GradNodeScale* scale_node1 = dynamic_cast(grad_node1); - PADDLE_ENFORCE( - scale_node1 != nullptr, - paddle::platform::errors::Fatal("AutogradMeta should have held " - "grad_node with type GradNodeScale*")); - PADDLE_ENFORCE( - (meta1->OutRankInfo().first == 0) && (meta1->OutRankInfo().second == 0), - paddle::platform::errors::Fatal( - "OutRankInfo in AutogradMeta should have been 0")); + + CHECK_NOTNULL(scale_node1); + CHECK_EQ(static_cast(meta1->OutRankInfo().first), 0); + CHECK_EQ(static_cast(meta1->OutRankInfo().second), 0); // Node 2 AutogradMeta* meta2 = EagerUtils::autograd_meta(&out2); GradNodeBase* grad_node2 = meta2->GradNode(); GradNodeScale* scale_node2 = dynamic_cast(grad_node2); - PADDLE_ENFORCE( - scale_node2 != nullptr, - paddle::platform::errors::Fatal("AutogradMeta should have held " - "grad_node with type GradNodeScale*")); - PADDLE_ENFORCE( - (meta2->OutRankInfo().first == 0) && (meta2->OutRankInfo().second == 0), - paddle::platform::errors::Fatal( - "OutRankInfo in AutogradMeta should have been 0")); + + CHECK_NOTNULL(scale_node2); + CHECK_EQ(static_cast(meta2->OutRankInfo().first), 0); + CHECK_EQ(static_cast(meta2->OutRankInfo().second), 0); // 2. TensorWrapper: No TensorWrapper for ScaleNode // 3. NextEdges // Node 1 -> Node 0 const std::vector>& node1_edges = grad_node1->GetEdges(); - PADDLE_ENFORCE(node1_edges.size() == 1, - paddle::platform::errors::Fatal( - "Node 1 should have exactly 1 edge slot")); - PADDLE_ENFORCE(node1_edges[0].size() == 1, - paddle::platform::errors::Fatal( - "Node 1 should have exactly 1 edge in slot 0")); const Edge& node1_edge = node1_edges[0][0]; - PADDLE_ENFORCE((node1_edge.GetEdgeRankInfo().first == 0) && - (node1_edge.GetEdgeRankInfo().second == 0), - paddle::platform::errors::Fatal( - "Node1's edge should have input rank of 0")); - PADDLE_ENFORCE( - node1_edge.GetGradNode() == grad_node0, - paddle::platform::errors::Fatal("Node1's edge should point to Node 0")); + + CHECK_EQ(static_cast(node1_edge.GetEdgeRankInfo().first), 0); + CHECK_EQ(static_cast(node1_edge.GetEdgeRankInfo().second), 0); + CHECK_EQ(node1_edge.GetGradNode(), grad_node0); // Node 2 -> Node 0 const std::vector>& node2_edges = grad_node2->GetEdges(); - PADDLE_ENFORCE(node2_edges.size() == 1, - paddle::platform::errors::Fatal( - "Node 2 should have exactly 1 edge slot")); - PADDLE_ENFORCE(node1_edges[0].size() == 1, - paddle::platform::errors::Fatal( - "Node 2 should have exactly 1 edge in slot 0")); const Edge& node2_edge = node2_edges[0][0]; - PADDLE_ENFORCE((node2_edge.GetEdgeRankInfo().first == 0) && - (node2_edge.GetEdgeRankInfo().second == 0), - paddle::platform::errors::Fatal( - "Node2's edge should have input rank of 0")); - PADDLE_ENFORCE( - node2_edge.GetGradNode() == grad_node0, - paddle::platform::errors::Fatal("Node2's edge should point to Node 0")); + + CHECK_EQ(static_cast(node2_edge.GetEdgeRankInfo().first), 0); + CHECK_EQ(static_cast(node2_edge.GetEdgeRankInfo().second), 0); + CHECK_EQ(node2_edge.GetGradNode(), grad_node0); } }