From f9593a566a7324c32d42df1bec67b6c46f479be5 Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Thu, 22 Feb 2024 17:14:01 +0800 Subject: [PATCH 001/282] [SOT][3.12] Compat for `COMPARE_OP` arg (#61932) --- .../jit/sot/opcode_translator/executor/opcode_executor.py | 7 ++++++- .../jit/sot/opcode_translator/executor/pycode_generator.py | 2 ++ test/sot/skip_files_py312 | 7 ------- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py index 8614356ce3a85..5f193cebc085d 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py +++ b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py @@ -1171,7 +1171,12 @@ def CALL_METHOD(self, instr: Instruction): push_n=1 ) # call instance, in, not in may call TensorVariable.get_py_value, which raise BreakGraphError def COMPARE_OP(self, instr: Instruction): - op = dis.cmp_op[instr.arg] + cmp_op_index = instr.arg + if sys.version_info >= (3, 12): + # Python 3.12 use lower 4 bits to store the inline cache `jump mask` + # see https://github.com/python/cpython/pull/100924 + cmp_op_index >>= 4 + op = dis.cmp_op[cmp_op_index] right, left = self.stack.pop(), self.stack.pop() self.stack.push( BuiltinVariable( diff --git a/python/paddle/jit/sot/opcode_translator/executor/pycode_generator.py b/python/paddle/jit/sot/opcode_translator/executor/pycode_generator.py index 0ec780d4b9bda..69e174818d662 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/pycode_generator.py +++ b/python/paddle/jit/sot/opcode_translator/executor/pycode_generator.py @@ -1037,6 +1037,8 @@ def gen_compare(self, cmp_op): only generator operator instruction, do nothing for operands. """ + if sys.version_info >= (3, 12): + cmp_op <<= 4 self._add_instr("COMPARE_OP", cmp_op) def _add_instr(self, *args, **kwargs): diff --git a/test/sot/skip_files_py312 b/test/sot/skip_files_py312 index ad1b19a2a4481..3cc5b8d4439e0 100644 --- a/test/sot/skip_files_py312 +++ b/test/sot/skip_files_py312 @@ -1,24 +1,19 @@ -./test_09_f_string.py ./test_10_build_unpack.py ./test_11_jumps.py ./test_12_for_loop.py ./test_14_operators.py ./test_15_slice.py ./test_17_paddle_layer.py -./test_20_string.py ./test_21_global.py ./test_analysis_inputs.py -./test_binary_operator_tracker.py ./test_break_graph.py ./test_builtin_map.py ./test_builtin_range.py ./test_builtin_zip.py -./test_dup_top.py ./test_enumerate.py ./test_guard_user_defined_fn.py ./test_inplace_api.py ./test_min_graph_size.py -./test_numpy_var_if.py ./test_output_restoration.py ./test_side_effects.py ./test_simulate_initialize.py @@ -29,6 +24,4 @@ ./test_sot_resnet50_backward.py ./test_specialization.py ./test_str_format.py -./test_tensor_dtype_in_guard.py -./test_dtype.py ./test_builtin_bool.py From 37ed407c25e8cac67a158340826d3bf1530946bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E5=91=A8=E5=91=A8?= <39978853+zhoutianzi666@users.noreply.github.com> Date: Thu, 22 Feb 2024 19:37:23 +0800 Subject: [PATCH 002/282] [paddle inference] make cutlass_conv2d compiled as a so independent of phi (#61551) * add workspcae * fix a bug in depthwise * add cmake args and add sys path * remove paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_bias_relu_few_channels.cu * remove unitest * remove somehting in paddle/phi/kernels/CMakeLists.txt * cmake release --- paddle/phi/backends/dynload/CMakeLists.txt | 3 +- paddle/phi/backends/dynload/cutlass_conv2d.cc | 57 +++++ paddle/phi/backends/dynload/cutlass_conv2d.h | 30 +++ paddle/phi/kernels/CMakeLists.txt | 32 --- .../fusion/cutlass/conv2d/CMakeLists.txt | 41 ++++ .../kernels/fusion/cutlass/conv2d/README.md | 25 ++ .../kernels/fusion/cutlass/conv2d/compile.sh | 33 +++ .../fusion/cutlass/conv2d/conv2d_bias_act.py | 3 - .../conv2d/conv2d_bias_relu_few_channels.cu | 218 ------------------ .../cutlass/conv2d/conv2d_bias_residual.py | 3 - .../fusion/cutlass/conv2d/conv2d_common.py | 18 +- .../fusion/cutlass/conv2d/conv2d_decl.h | 29 +-- .../conv2d/conv2d_depthwise_bias_act.py | 11 +- .../fusion/cutlass/conv2d/conv2d_util.cu | 31 +-- .../fusion/cutlass/conv2d/conv2d_util.h | 17 +- .../cutlass/fused_conv2d_add_act_kernel.cu | 44 +++- test/ir/inference/CMakeLists.txt | 13 -- 17 files changed, 267 insertions(+), 341 deletions(-) create mode 100644 paddle/phi/backends/dynload/cutlass_conv2d.cc create mode 100644 paddle/phi/backends/dynload/cutlass_conv2d.h create mode 100644 paddle/phi/kernels/fusion/cutlass/conv2d/CMakeLists.txt create mode 100644 paddle/phi/kernels/fusion/cutlass/conv2d/README.md create mode 100644 paddle/phi/kernels/fusion/cutlass/conv2d/compile.sh delete mode 100644 paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_bias_relu_few_channels.cu diff --git a/paddle/phi/backends/dynload/CMakeLists.txt b/paddle/phi/backends/dynload/CMakeLists.txt index 2db75d7022f0a..9fd293574e247 100644 --- a/paddle/phi/backends/dynload/CMakeLists.txt +++ b/paddle/phi/backends/dynload/CMakeLists.txt @@ -13,7 +13,8 @@ list( cusolver.cc cusparse.cc nvtx.cc - cufft.cc) + cufft.cc + cutlass_conv2d.cc) if(NOT WITH_NV_JETSON) list(APPEND CUDA_SRCS nvjpeg.cc) diff --git a/paddle/phi/backends/dynload/cutlass_conv2d.cc b/paddle/phi/backends/dynload/cutlass_conv2d.cc new file mode 100644 index 0000000000000..936a04fa3023c --- /dev/null +++ b/paddle/phi/backends/dynload/cutlass_conv2d.cc @@ -0,0 +1,57 @@ +/* Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/phi/backends/dynload/cutlass_conv2d.h" +#include +#include "paddle/phi/core/enforce.h" + +namespace phi { +namespace dynload { + +std::once_flag cutlass_dso_flag; +void* cutlass_dso_handle; + +void* GetCutlassConv2dHandle() { + std::string dso_name = "libCutlassConv2d.so"; + + std::call_once(cutlass_dso_flag, [&]() { +#if !defined(_WIN32) + int dynload_flags = RTLD_LAZY | RTLD_LOCAL; +#else + int dynload_flags = 0; +#endif // !_WIN32 + + cutlass_dso_handle = dlopen(dso_name.c_str(), dynload_flags); + + PADDLE_ENFORCE_NOT_NULL( + cutlass_dso_handle, + phi::errors::NotFound( + "libCutlassConv2d.so is needed, " + "but libCutlassConv2d.so is not found.\n" + " Suggestions:\n" + " 1. Refer paddle/phi/kernels/fusion/cutlass/conv2d/README.md, " + "and compile this library.\n" + " 2. Configure environment variables as " + "follows:\n" + " - Linux: set LD_LIBRARY_PATH by `export LD_LIBRARY_PATH=...`\n" + " - Windows: set PATH by `set PATH=XXX;%PATH%`\n" + " - Mac: set DYLD_LIBRARY_PATH by `export " + "DYLD_LIBRARY_PATH=...`\n")); + }); + + return cutlass_dso_handle; +} + +} // namespace dynload +} // namespace phi diff --git a/paddle/phi/backends/dynload/cutlass_conv2d.h b/paddle/phi/backends/dynload/cutlass_conv2d.h new file mode 100644 index 0000000000000..c342eb9d09220 --- /dev/null +++ b/paddle/phi/backends/dynload/cutlass_conv2d.h @@ -0,0 +1,30 @@ +/* Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ +#pragma once + +#if !defined(_WIN32) +#include +#endif + +#include // NOLINT + +#include "paddle/phi/backends/dynload/dynamic_loader.h" + +namespace phi { +namespace dynload { + +void* GetCutlassConv2dHandle(); + +} // namespace dynload +} // namespace phi diff --git a/paddle/phi/kernels/CMakeLists.txt b/paddle/phi/kernels/CMakeLists.txt index 85697df11bc56..80d61ebc9a9a6 100644 --- a/paddle/phi/kernels/CMakeLists.txt +++ b/paddle/phi/kernels/CMakeLists.txt @@ -54,36 +54,6 @@ if(DEFINED REDUCE_INFERENCE_LIB_SIZE) endif() if(WITH_CUTLASS) - execute_process( - COMMAND ${CMAKE_COMMAND} -E make_directory - "${CMAKE_CURRENT_SOURCE_DIR}/fusion/cutlass/conv2d/generated_tmp" - COMMAND ${PYTHON_EXECUTABLE} "conv2d_bias_act.py" - COMMAND ${PYTHON_EXECUTABLE} "conv2d_bias_residual.py" - COMMAND ${PYTHON_EXECUTABLE} "conv2d_depthwise_bias_act.py" - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/fusion/cutlass/conv2d") - set(generated_tmp_dir - ${PADDLE_SOURCE_DIR}/paddle/phi/kernels/fusion/cutlass/conv2d/generated_tmp - ) - set(generated_dir - ${PADDLE_SOURCE_DIR}/paddle/phi/kernels/fusion/cutlass/conv2d/generated) - file(GLOB con2d_generated_files ${generated_tmp_dir}/*.cu) - - if(EXISTS ${generated_dir}) - foreach(gen_file ${con2d_generated_files}) - string(REPLACE "generated_tmp" "generated" now_file ${gen_file}) - execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different - "${gen_file}" "${now_file}") - endforeach() - message("copy if different ${generated_dir}") - else() - foreach(gen_file ${con2d_generated_files}) - string(REPLACE "generated_tmp" "generated" now_file ${gen_file}) - execute_process(COMMAND ${CMAKE_COMMAND} -E copy "${gen_file}" - "${now_file}") - endforeach() - message("copy ${generated_dir}") - endif() - execute_process( COMMAND ${PYTHON_EXECUTABLE} @@ -204,8 +174,6 @@ if(WITH_CUTLASS) file( GLOB cutlass_cu RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" - "fusion/cutlass/conv2d/generated/*.cu" - "fusion/cutlass/conv2d/*.cu" "fusion/cutlass/*.cu" "fusion/cutlass/memory_efficient_attention/autogen/impl/*.cu" "fusion/cutlass/memory_efficient_attention/autogen_variable/impl/*.cu" diff --git a/paddle/phi/kernels/fusion/cutlass/conv2d/CMakeLists.txt b/paddle/phi/kernels/fusion/cutlass/conv2d/CMakeLists.txt new file mode 100644 index 0000000000000..cd82bbf1dc8b7 --- /dev/null +++ b/paddle/phi/kernels/fusion/cutlass/conv2d/CMakeLists.txt @@ -0,0 +1,41 @@ +cmake_minimum_required(VERSION 3.23) + +if(NOT DEFINED PYTHON_EXECUTABLE) + message( + FATAL_ERROR + "please set PYTHON_EXECUTABLE with -DPYTHON_EXECUTABLE=python executable path" + ) +endif() + +if(NOT DEFINED COMPUTE_CAPABILITY) + message( + FATAL_ERROR + "please set COMPUTE_CAPABILITY with -DCOMPUTE_CAPABILITY=your gpu compute capability" + ) +endif() + +include_directories("${CMAKE_CURRENT_SOURCE_DIR}/cutlass/include") +include_directories("${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../") + +execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory + "${CMAKE_CURRENT_BINARY_DIR}/generated_tmp") + +execute_process( + COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/conv2d_bias_act.py" + COMMAND ${PYTHON_EXECUTABLE} + "${CMAKE_CURRENT_SOURCE_DIR}/conv2d_bias_residual.py" + COMMAND ${PYTHON_EXECUTABLE} + "${CMAKE_CURRENT_SOURCE_DIR}/conv2d_depthwise_bias_act.py" + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") + +find_package(CUDA) + +set(CUDA_NVCC_FLAGS + -gencode arch=compute_${COMPUTE_CAPABILITY},code=sm_${COMPUTE_CAPABILITY};) +#set(CMAKE_CXX_FLAGS -fvisibility=hidden) +set(CMAKE_BUILD_TYPE "Release") +file(GLOB all_cutlass_conv2d_cu + "${CMAKE_CURRENT_BINARY_DIR}/generated_tmp/*.cu") +list(APPEND all_cutlass_conv2d_cu "${CMAKE_CURRENT_SOURCE_DIR}/conv2d_util.cu") + +cuda_add_library(CutlassConv2d SHARED ${all_cutlass_conv2d_cu}) diff --git a/paddle/phi/kernels/fusion/cutlass/conv2d/README.md b/paddle/phi/kernels/fusion/cutlass/conv2d/README.md new file mode 100644 index 0000000000000..a717b3d692b91 --- /dev/null +++ b/paddle/phi/kernels/fusion/cutlass/conv2d/README.md @@ -0,0 +1,25 @@ +# 如何编译和使用cutlass的conv2d算子 + +本文件夹下面对应的是基于cutlass开发的conv2d算子，此算子被独立编译成so，供paddle内部的phi库调用。 +这样做的好处有两个： +1. 减少paddle 发版时包的体积，避免把cutlass的代码打包进paddle inference中。 +2. 将框架和算子具体实现完全解耦开，保证paddle框架的通用性的同时，保证具体算子实现的灵活性。 + +用户可以采用如下步骤编译和使用此算子 + +step1. + +`bash compile.sh` + +注意，该脚本中有三个参数需要用户自己指定下，分别是python解释器的路径，cuda的根目录路径和用户GPU机器的计算能力。 +```shell +python_exe_path="python" +cuda_root_path="/usr/local/cuda" +gpu_cc="75" +``` +compile.sh 脚本中会下载cutlass，执行CMakeLists.txt脚本，编译生成动态库。 + + +step2. + +step1执行后，就可以看到在 build 目录生成了 `libCutlassConv2d.so` ，并将build目录添加到LD_LIBRARY_PATH中即可使用此库。 diff --git a/paddle/phi/kernels/fusion/cutlass/conv2d/compile.sh b/paddle/phi/kernels/fusion/cutlass/conv2d/compile.sh new file mode 100644 index 0000000000000..44c0fdf3a04da --- /dev/null +++ b/paddle/phi/kernels/fusion/cutlass/conv2d/compile.sh @@ -0,0 +1,33 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + +cutlass_repo_directory="cutlass" +if [ ! -d "$cutlass_repo_directory" ]; then + git clone --branch v2.11.0 https://github.com/NVIDIA/cutlass +fi + +build_directory="build" +if [ ! -d "$build_directory" ]; then + mkdir $build_directory +fi + +python_exe_path="python" +cuda_root_path="/usr/local/cuda" +gpu_cc="75" + +cd $build_directory +cmake .. -DPYTHON_EXECUTABLE=$python_exe_path -DCUDA_TOOLKIT_ROOT_DIR=$cuda_root_path -DCOMPUTE_CAPABILITY=$gpu_cc +make -j +cd - diff --git a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_bias_act.py b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_bias_act.py index 6870d191a8026..0cb925489f14a 100644 --- a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_bias_act.py +++ b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_bias_act.py @@ -12,9 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - -sys.path.append("../") import enum from conv2d_common import ( diff --git a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_bias_relu_few_channels.cu b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_bias_relu_few_channels.cu deleted file mode 100644 index fb1c3f2313c98..0000000000000 --- a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_bias_relu_few_channels.cu +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include "cutlass/conv/kernel/default_conv2d_fprop.h" -#include "cutlass/epilogue/thread/linear_combination_bias_relu.h" -#include "paddle/phi/common/memory_utils.h" -#include "paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_util.h" - -namespace phi { -namespace fusion { -namespace cutlass_internal { -template -cutlass::Status Conv2dBiasReluFewChannelsImpl(ConvAllParams params) { - using ElementAccumulator = float; - using ElementComputeEpilogue = float; - using ElementInputA = cutlass::half_t; - using ElementInputB = cutlass::half_t; - using ElementOutput = cutlass::half_t; - using LayoutInputA = cutlass::layout::TensorNHWC; - using LayoutInputB = cutlass::layout::TensorNHWC; - using LayoutOutput = cutlass::layout::TensorNHWC; - using MMAOp = cutlass::arch::OpClassTensorOp; - using SmArch = cutlass::arch::Sm75; - using ThreadblockShape = TShape; - using WarpShape = WShape; - using InstructionShape = cutlass::gemm::GemmShape<16, 8, 8>; - using SwizzleThreadBlock = - cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<4>; - constexpr int NumStages = 2; - static cutlass::conv::IteratorAlgorithm const IteratorAlgorithm = - cutlass::conv::IteratorAlgorithm::kFewChannels; - using EpilogueOp = - cutlass::epilogue::thread::LinearCombinationRelu; - - using Conv2dFpropKernel = typename cutlass::conv::kernel::DefaultConv2dFprop< - ElementInputA, - LayoutInputA, - ElementInputB, - LayoutInputB, - ElementOutput, - LayoutOutput, - ElementAccumulator, - MMAOp, - SmArch, - ThreadblockShape, - WarpShape, - InstructionShape, - EpilogueOp, - SwizzleThreadBlock, - NumStages, - cutlass::arch::OpMultiplyAdd, - IteratorAlgorithm, - cutlass::conv::StrideSupport::kStrided, - Alignment, - Alignment>::Kernel; - using ImplicitGemm = - cutlass::conv::device::ImplicitGemmConvolution; - - const half *input = params.input; - const half *weight = params.weight; - const half *bias = params.bias; - half *output = params.output; - int batch = params.batch; - int ic = params.ic; - int ih = params.ih; - int iw = params.iw; - int kh = params.kh; - int kw = params.kw; - int oc = params.oc; - int pad_h0 = params.pad_h0; - int pad_w0 = params.pad_w1; - - int stride_h = params.stride_h; - int stride_w = params.stride_w; - - int oh = params.oh; - int ow = params.ow; - int dilation_h = params.dilation_h; - int dilation_w = params.dilation_w; - - cutlass::conv::Mode mode = cutlass::conv::Mode::kCrossCorrelation; - cutlass::conv::Conv2dProblemSize problem_size({batch, ih, iw, ic}, - {oc, kh, kw, ic}, - {pad_h0, 0, pad_w0, 0}, - {stride_h, stride_w}, - {dilation_h, dilation_w}, - {batch, oh, ow, oc}, - mode, - 1); - - typename ImplicitGemm::Arguments arguments{ - problem_size, - {(cutlass::half_t *)(input), {ic, ic * iw, ic * iw * ih}}, - {(cutlass::half_t *)(weight), {ic, ic * kw, ic * kw * kh}}, - {(cutlass::half_t *)(bias), {0, 0, 0}}, - {(cutlass::half_t *)(output), {oc, oc * ow, oc * ow * oh}}, - {1.f, 1.f}}; - - ImplicitGemm implicit_gemm_op; - size_t bytes = implicit_gemm_op.get_workspace_size(arguments); - - auto ctx = params.ctx; - auto stream = ctx->stream(); - phi::Allocator::AllocationPtr tmp_gpu_ptrs_data = phi::memory_utils::Alloc( - ctx->GetPlace(), - bytes, - phi::Stream(reinterpret_cast(stream))); - void *workspace = tmp_gpu_ptrs_data->ptr(); - - cutlass::Status status = implicit_gemm_op.can_implement(arguments); - CUTLASS_CHECK(status); - status = implicit_gemm_op.initialize(arguments, workspace); - CUTLASS_CHECK(status); - status = implicit_gemm_op(stream); - CUTLASS_CHECK(status); - return status; -} - -// config 0 -template cutlass::Status Conv2dBiasReluFewChannelsImpl< - cutlass::gemm::GemmShape<64, 64, 64>, - cutlass::gemm::GemmShape<32, 32, 64>>(ConvAllParams); -// config 1 -template cutlass::Status Conv2dBiasReluFewChannelsImpl< - cutlass::gemm::GemmShape<64, 32, 64>, - cutlass::gemm::GemmShape<32, 32, 64>>(ConvAllParams); -// config 2 -template cutlass::Status Conv2dBiasReluFewChannelsImpl< - cutlass::gemm::GemmShape<128, 32, 64>, - cutlass::gemm::GemmShape<32, 32, 64>>(ConvAllParams); -// config 3 -template cutlass::Status Conv2dBiasReluFewChannelsImpl< - cutlass::gemm::GemmShape<128, 64, 64>, - cutlass::gemm::GemmShape<32, 32, 64>>(ConvAllParams); -// config 4 -template cutlass::Status Conv2dBiasReluFewChannelsImpl< - cutlass::gemm::GemmShape<64, 64, 32>, - cutlass::gemm::GemmShape<32, 32, 32>>(ConvAllParams); -// config 5 -template cutlass::Status Conv2dBiasReluFewChannelsImpl< - cutlass::gemm::GemmShape<64, 128, 32>, - cutlass::gemm::GemmShape<32, 64, 32>>(ConvAllParams); -// config 6 -template cutlass::Status Conv2dBiasReluFewChannelsImpl< - cutlass::gemm::GemmShape<64, 128, 64>, - cutlass::gemm::GemmShape<64, 64, 32>>(ConvAllParams); -// config 7 -template cutlass::Status Conv2dBiasReluFewChannelsImpl< - cutlass::gemm::GemmShape<64, 256, 32>, - cutlass::gemm::GemmShape<64, 64, 32>>(ConvAllParams); -// config 8 -template cutlass::Status Conv2dBiasReluFewChannelsImpl< - cutlass::gemm::GemmShape<128, 64, 32>, - cutlass::gemm::GemmShape<64, 32, 32>>(ConvAllParams); - -std::vector> - conv2d_bias_relu_few_channels_all_func = { - Conv2dBiasReluFewChannelsImpl, - cutlass::gemm::GemmShape<32, 32, 64>>, - Conv2dBiasReluFewChannelsImpl, - cutlass::gemm::GemmShape<32, 32, 64>>, - Conv2dBiasReluFewChannelsImpl, - cutlass::gemm::GemmShape<32, 32, 64>>, - Conv2dBiasReluFewChannelsImpl, - cutlass::gemm::GemmShape<32, 32, 64>>, - Conv2dBiasReluFewChannelsImpl, - cutlass::gemm::GemmShape<32, 32, 32>>, - Conv2dBiasReluFewChannelsImpl, - cutlass::gemm::GemmShape<32, 64, 32>>, - Conv2dBiasReluFewChannelsImpl, - cutlass::gemm::GemmShape<64, 64, 32>>, - Conv2dBiasReluFewChannelsImpl, - cutlass::gemm::GemmShape<64, 64, 32>>, - Conv2dBiasReluFewChannelsImpl, - cutlass::gemm::GemmShape<64, 32, 32>>}; -std::map, int> map_problem_conv2d_bias_relu_few_channels; - -void Conv2dBiasReluFewChannels(ConvAllParams params) { - int batch = params.batch; - int ic = params.ic; - int ih = params.ih; - int iw = params.iw; - int kh = params.kh; - int kw = params.kw; - int oc = params.oc; - int pad_h0 = params.pad_h0; - int pad_w0 = params.pad_w1; - int stride_h = params.stride_h; - int stride_w = params.stride_w; - - std::vector problem_size = { - batch, ic, ih, iw, kh, kw, oc, pad_h0, pad_w0, stride_h, stride_w}; - - if (map_problem_conv2d_bias_relu_few_channels.count(problem_size)) { - conv2d_bias_relu_few_channels_all_func - [map_problem_conv2d_bias_relu_few_channels.at(problem_size)](params); - return; - } - // -} -} // namespace cutlass_internal -} // namespace fusion -} // namespace phi diff --git a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_bias_residual.py b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_bias_residual.py index 109dac2ad65e8..55fde0722b6b3 100644 --- a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_bias_residual.py +++ b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_bias_residual.py @@ -12,9 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - -sys.path.append("../") import enum from conv2d_common import ( diff --git a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_common.py b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_common.py index 34d72a4c7443e..7c95892006c43 100644 --- a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_common.py +++ b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_common.py @@ -12,9 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import sys -sys.path.append("../") +dirname, filename = os.path.split(os.path.abspath(sys.argv[0])) +sys.path.append(dirname + "/../") from util import SubstituteTemplate # For beginners, these template parameters may be difficult to understand. @@ -90,14 +92,8 @@ ImplicitGemm implicit_gemm_op; size_t bytes = implicit_gemm_op.get_workspace_size(arguments); - auto ctx = params.ctx; - auto stream = ctx->stream(); - phi::Allocator::AllocationPtr tmp_gpu_ptrs_data = - phi::memory_utils::Alloc( - ctx->GetPlace(), - bytes, - phi::Stream(reinterpret_cast(stream))); - void *workspace = tmp_gpu_ptrs_data->ptr(); +auto stream = params.stream; +void *workspace = params.workspace; cutlass::Status status = implicit_gemm_op.can_implement(arguments); CUTLASS_CHECK(status); @@ -122,7 +118,7 @@ std::map, int> map_problem_${func_name}; std::mutex ${func_name}_mutex; -void ${func_name}(const ConvAllParams& params) { +void ${func_name}(ConvAllParams params) { int batch = params.batch; int ic = params.ic; int ih = params.ih; @@ -161,7 +157,7 @@ # this function is invoked by phi kernel CommonWrapperForPhi = """ -void ${op_name}(const ConvAllParams& params) { +void ${op_name}(ConvAllParams params) { ${dispatch_body} } """ diff --git a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_decl.h b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_decl.h index db0b9664c43ee..aaad46de5cb0d 100644 --- a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_decl.h +++ b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_decl.h @@ -13,12 +13,9 @@ // limitations under the License. #pragma once #include -#include #include #include -#include "paddle/phi/backends/gpu/gpu_context.h" - namespace phi { namespace fusion { namespace cutlass_internal { @@ -47,23 +44,27 @@ typedef struct { int oh; int ow; int groups; - const phi::GPUContext *ctx; + // const phi::GPUContext *ctx; + cudaStream_t stream; float alpha; // for leaky_relu use int sm_version = 75; + void *workspace = nullptr; } ConvAllParams; // Below functions are provided by cutlass, they are called by phi. -void Conv2dBiasAddRelu(const ConvAllParams ¶ms); -void Conv2dBiasRelu(const ConvAllParams ¶ms); -void Conv2dBiasLeakyRelu(const ConvAllParams ¶ms); -void Conv2dBiasSilu(const ConvAllParams ¶ms); -void Conv2dBias(const ConvAllParams ¶ms); -void Conv2dBiasSigmoid(const ConvAllParams ¶ms); +extern "C" void Conv2dBiasAddRelu(ConvAllParams params); +extern "C" void Conv2dBiasRelu(ConvAllParams params); +extern "C" void Conv2dBiasLeakyRelu(ConvAllParams params); +extern "C" void Conv2dBiasSilu(ConvAllParams params); +extern "C" void Conv2dBias(ConvAllParams params); +extern "C" void Conv2dBiasSigmoid(ConvAllParams params); + +extern "C" void Conv2dDepthwiseBias(ConvAllParams params); +extern "C" void Conv2dDepthwiseBiasRelu(ConvAllParams params); +extern "C" void Conv2dDepthwiseBiasSigmoid(ConvAllParams params); +extern "C" void Conv2dDepthwiseBiasSilu(ConvAllParams params); -void Conv2dDepthwiseBias(const ConvAllParams ¶ms); -void Conv2dDepthwiseBiasRelu(const ConvAllParams ¶ms); -void Conv2dDepthwiseBiasSigmoid(const ConvAllParams ¶ms); -void Conv2dDepthwiseBiasSilu(const ConvAllParams ¶ms); +extern "C" int HelloFromCutlassConv2d(int a, int b); } // namespace cutlass_internal } // namespace fusion diff --git a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_depthwise_bias_act.py b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_depthwise_bias_act.py index 0ea8e0a47130d..fb2f2be096110 100644 --- a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_depthwise_bias_act.py +++ b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_depthwise_bias_act.py @@ -12,9 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - -sys.path.append("../") import enum from conv2d_common import ( @@ -60,13 +57,7 @@ CommonCutlassConv2dDepthwiseKernelDeclare, dict_for_declare_part ) + ''' -size_t filter_size = oc * kh * kw * kc * sizeof(half); -phi::Allocator::AllocationPtr filter_gpu_ptrs_data = - phi::memory_utils::Alloc( - params.ctx->GetPlace(), - filter_size, - phi::Stream(reinterpret_cast(params.ctx->stream()))); -void *filter_workspace = filter_gpu_ptrs_data->ptr(); + void *filter_workspace = params.workspace; typename ImplicitGemm::Arguments arguments{ problem_size, diff --git a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_util.cu b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_util.cu index 62b7b439458b9..51bc71983105a 100644 --- a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_util.cu +++ b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_util.cu @@ -16,8 +16,6 @@ #include "paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_util.h" -#include "glog/logging.h" - namespace phi { namespace fusion { namespace cutlass_internal { @@ -274,35 +272,40 @@ int ProfileToGetBestConfig( } cudaEvent_t beg, end; - PADDLE_ENFORCE_GPU_SUCCESS(cudaEventCreate(&beg)); - PADDLE_ENFORCE_GPU_SUCCESS(cudaEventCreate(&end)); - PADDLE_ENFORCE_GPU_SUCCESS(cudaEventRecord(beg)); + (cudaEventCreate(&beg)); + (cudaEventCreate(&end)); + (cudaEventRecord(beg)); for (int ii = 0; ii < REPEAT; ii++) { status = func(params); } - PADDLE_ENFORCE_GPU_SUCCESS(cudaEventRecord(end)); - PADDLE_ENFORCE_GPU_SUCCESS(cudaEventSynchronize(end)); + (cudaEventRecord(end)); + (cudaEventSynchronize(end)); float elapsed_time; - PADDLE_ENFORCE_GPU_SUCCESS(cudaEventElapsedTime(&elapsed_time, beg, end)); + (cudaEventElapsedTime(&elapsed_time, beg, end)); if (elapsed_time < min_time && status == cutlass::Status::kSuccess) { min_time = elapsed_time; min_time_index = i; // debug code - VLOG(3) << OpType2String(op_type) << ": tactic " << i << " has max diff " - << conv2d_diff_gpu(params, op_type) << " compared with baseline," - << "cost_time: " << elapsed_time << "ms."; + std::cout << OpType2String(op_type) << ": tactic " << i + << " has max diff " << conv2d_diff_gpu(params, op_type) + << " compared with baseline," + << "cost_time: " << elapsed_time << "ms." << std::endl; } } if (min_time_index < 0) { - PADDLE_THROW( - phi::errors::NotFound("Can't find any cutlass config for this %s op.", - OpType2String(op_type).c_str())); + std::cout << "Can't find any cutlass config for " << OpType2String(op_type) + << std::endl; } return min_time_index; } +__attribute__((dllexport)) int HelloFromCutlassConv2d(int a, int b) { + std::cout << "welcom using Cutlass Conv2d" << std::endl; + return 1; +} + } // namespace cutlass_internal } // namespace fusion } // namespace phi diff --git a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_util.h b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_util.h index eaceb46d69d74..80865e0e1cded 100644 --- a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_util.h +++ b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_util.h @@ -17,25 +17,20 @@ #include #include "paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_decl.h" -#include "glog/logging.h" - #include "cutlass/cutlass.h" #include "cutlass/gemm/device/gemm.h" #include "cutlass/conv/device/implicit_gemm_convolution.h" -#include "paddle/phi/backends/gpu/gpu_context.h" -#include "paddle/phi/common/memory_utils.h" -#include "paddle/phi/core/enforce.h" - namespace phi { namespace fusion { namespace cutlass_internal { -#define CUTLASS_CHECK(status) \ - if (status != cutlass::Status::kSuccess) { \ - VLOG(3) \ - << "Cutlass can not deal with this problem size, skip this kernel!"; \ - return status; \ +#define CUTLASS_CHECK(status) \ + if (status != cutlass::Status::kSuccess) { \ + std::cout \ + << "Cutlass can not deal with this problem size, skip this kernel!" \ + << std::endl; \ + return status; \ } typedef enum { diff --git a/paddle/phi/kernels/fusion/cutlass/fused_conv2d_add_act_kernel.cu b/paddle/phi/kernels/fusion/cutlass/fused_conv2d_add_act_kernel.cu index ef803f0ea5f3d..dceaafd2e7172 100644 --- a/paddle/phi/kernels/fusion/cutlass/fused_conv2d_add_act_kernel.cu +++ b/paddle/phi/kernels/fusion/cutlass/fused_conv2d_add_act_kernel.cu @@ -12,14 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include "paddle/phi/backends/gpu/gpu_context.h" +#include "paddle/phi/common/memory_utils.h" #include "paddle/phi/core/dense_tensor.h" +#include "paddle/phi/core/enforce.h" #include "paddle/phi/core/kernel_registry.h" #include "paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_decl.h" +#include "paddle/phi/backends/dynload/cutlass_conv2d.h" + namespace phi { namespace fusion { namespace cutlass_internal { +typedef void (*func)(phi::fusion::cutlass_internal::ConvAllParams); + template void FusedConv2dAddActKernel(const Context& ctx, const DenseTensor& x, @@ -49,6 +57,7 @@ void FusedConv2dAddActKernel(const Context& ctx, CHECK_EQ(dilations.size() == 2UL, true); CHECK_EQ(padding_algorithm == "EXPLICIT", true); + CHECK_EQ(data_format == "NHWC", true); const int batch = in_dims[0]; const int ic = in_dims[3]; const int ih = in_dims[1]; @@ -112,27 +121,39 @@ void FusedConv2dAddActKernel(const Context& ctx, oh, ow, groups, - &ctx}; + ctx.stream()}; + + void* dlhandler = phi::dynload::GetCutlassConv2dHandle(); + func conv_func = NULL; + CHECK_EQ(dlhandler == NULL, false); // conv2d_depthwise if (groups == ic && ic == oc) { + // conv2d_depthwise need a tmp workspace. + phi::Allocator::AllocationPtr tmp_ptr = phi::memory_utils::Alloc( + ctx.GetPlace(), + oc * kh * kw * sizeof(T), + phi::Stream(reinterpret_cast(ctx.stream()))); + params.workspace = tmp_ptr->ptr(); // cutlass conv2d_depthwise not support residual if (residual) { CHECK_EQ(residual->data() == nullptr, true); } if (activation == "relu") { - Conv2dDepthwiseBiasRelu(params); + conv_func = (func)(dlsym(dlhandler, "Conv2dDepthwiseBiasRelu")); } else if (activation == "identity") { - Conv2dDepthwiseBias(params); + conv_func = (func)(dlsym(dlhandler, "Conv2dDepthwiseBias")); } else if (activation == "sigmoid") { - Conv2dDepthwiseBiasSigmoid(params); + conv_func = (func)(dlsym(dlhandler, "Conv2dDepthwiseBiasSigmoid")); } else if (activation == "swish") { - Conv2dDepthwiseBiasSilu(params); + conv_func = (func)(dlsym(dlhandler, "Conv2dDepthwiseBiasSilu")); } else { PADDLE_THROW(phi::errors::InvalidArgument( "Cutlass conv2d_depthwise does not support this activation: %s.", activation.c_str())); } + conv_func(params); + output->set_layout(DataLayout::NHWC); return; } @@ -141,26 +162,27 @@ void FusedConv2dAddActKernel(const Context& ctx, if (residual) { if (activation == "relu") { params.residual = reinterpret_cast(residual->data()); - Conv2dBiasAddRelu(params); + conv_func = (func)(dlsym(dlhandler, "Conv2dBiasAddRelu")); } else { PADDLE_THROW(phi::errors::InvalidArgument( "Cutlass now only support relu activation in a residual block")); } } else if (activation == "relu") { - Conv2dBiasRelu(params); + conv_func = (func)(dlsym(dlhandler, "Conv2dBiasRelu")); } else if (activation == "swish") { - Conv2dBiasSilu(params); + conv_func = (func)(dlsym(dlhandler, "Conv2dBiasSilu")); } else if (activation == "identity") { - Conv2dBias(params); + conv_func = (func)(dlsym(dlhandler, "Conv2dBias")); } else if (activation == "leaky_relu") { + conv_func = (func)(dlsym(dlhandler, "Conv2dBiasLeakyRelu")); params.alpha = fuse_alpha; - Conv2dBiasLeakyRelu(params); } else if (activation == "sigmoid") { - Conv2dBiasSigmoid(params); + conv_func = (func)(dlsym(dlhandler, "Conv2dBiasSigmoid")); } else { PADDLE_THROW(phi::errors::InvalidArgument( "Cutlass does not support this activation: %s.", activation.c_str())); } + conv_func(params); output->set_layout(DataLayout::NHWC); } } // namespace cutlass_internal diff --git a/test/ir/inference/CMakeLists.txt b/test/ir/inference/CMakeLists.txt index 185ca22f897f6..84abbaa986e61 100755 --- a/test/ir/inference/CMakeLists.txt +++ b/test/ir/inference/CMakeLists.txt @@ -138,19 +138,6 @@ if(WITH_XPU) endforeach() endif() -# below are cutlass unittests -file( - GLOB TEST_CUTLASS - RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" - "test_cutlass_*.py") -string(REPLACE ".py" "" TEST_CUTLASS "${TEST_CUTLASS}") -list(REMOVE_ITEM TEST_INFERENCE_IR_PASSES ${TEST_CUTLASS}) -if(WITH_CUTLASS) - foreach(target ${TEST_CUTLASS}) - py_test_modules(${target} MODULES ${target}) - endforeach() -endif() - if(WITH_MKLDNN AND TENSORRT_FOUND AND WITH_GPU) From 560225036959c49d7e6f523a546272e266be507b Mon Sep 17 00:00:00 2001 From: Lu Qi <61354321+MarioLulab@users.noreply.github.com> Date: Thu, 22 Feb 2024 19:50:42 +0800 Subject: [PATCH 003/282] Support time_major for FusedROPE (#61417) --- paddle/phi/api/yaml/fused_backward.yaml | 4 +- paddle/phi/api/yaml/fused_ops.yaml | 2 +- paddle/phi/infermeta/backward.cc | 1 + paddle/phi/infermeta/backward.h | 1 + paddle/phi/infermeta/multiary.cc | 1 + paddle/phi/infermeta/multiary.h | 1 + paddle/phi/infermeta/spmd_rules/fused_rope.cc | 53 ++-- paddle/phi/infermeta/spmd_rules/fused_rope.h | 9 +- .../fusion/gpu/fused_rope_grad_kernel.cu | 26 +- .../kernels/fusion/gpu/fused_rope_kernel.cu | 27 +- .../phi/kernels/fusion/gpu/fused_rope_utils.h | 28 +- .../fused_rotary_position_embedding.py | 11 +- .../semi_auto_parallel_for_fused_rope.py | 92 +++++++ test/cpp/auto_parallel/spmd_rule_test.cc | 8 +- .../test_fused_rotary_position_embedding.py | 241 ++++++++++++++---- 15 files changed, 412 insertions(+), 93 deletions(-) diff --git a/paddle/phi/api/yaml/fused_backward.yaml b/paddle/phi/api/yaml/fused_backward.yaml index 8a2a9786a837a..5c92b1a2a692f 100644 --- a/paddle/phi/api/yaml/fused_backward.yaml +++ b/paddle/phi/api/yaml/fused_backward.yaml @@ -40,8 +40,8 @@ support_dygraph_mode : true - backward_op : fused_rotary_position_embedding_grad - forward: fused_rotary_position_embedding (Tensor q, Tensor k, Tensor v, Tensor sin, Tensor cos, Tensor position_ids, bool use_neox_rotary_style) -> Tensor(out_q), Tensor(out_k), Tensor(out_v) - args : (Tensor sin, Tensor cos, Tensor position_ids, Tensor out_q_grad, Tensor out_k_grad,Tensor out_v_grad, bool use_neox_rotary_style) + forward: fused_rotary_position_embedding (Tensor q, Tensor k, Tensor v, Tensor sin, Tensor cos, Tensor position_ids, bool use_neox_rotary_style, bool time_major) -> Tensor(out_q), Tensor(out_k), Tensor(out_v) + args : (Tensor sin, Tensor cos, Tensor position_ids, Tensor out_q_grad, Tensor out_k_grad,Tensor out_v_grad, bool use_neox_rotary_style, bool time_major) output : Tensor(q_grad), Tensor(k_grad), Tensor(v_grad) optional : sin, cos, position_ids, out_k_grad, out_v_grad, k_grad, v_grad infer_meta : diff --git a/paddle/phi/api/yaml/fused_ops.yaml b/paddle/phi/api/yaml/fused_ops.yaml index f8dcb02cbdc72..2ca0a32be59f5 100644 --- a/paddle/phi/api/yaml/fused_ops.yaml +++ b/paddle/phi/api/yaml/fused_ops.yaml @@ -272,7 +272,7 @@ optional : cache_kv, pre_caches, rotary_pos_emb, time_step, seq_lengths, src_mask, gather_index - op : fused_rotary_position_embedding - args : (Tensor q, Tensor k, Tensor v, Tensor sin, Tensor cos, Tensor position_ids, bool use_neox_rotary_style = true) + args : (Tensor q, Tensor k, Tensor v, Tensor sin, Tensor cos, Tensor position_ids, bool use_neox_rotary_style = true, bool time_major = false) output : Tensor(out_q), Tensor(out_k), Tensor(out_v) infer_meta : func : FusedRopeInferMeta diff --git a/paddle/phi/infermeta/backward.cc b/paddle/phi/infermeta/backward.cc index 54cf403533427..4f525ef138735 100644 --- a/paddle/phi/infermeta/backward.cc +++ b/paddle/phi/infermeta/backward.cc @@ -1349,6 +1349,7 @@ void FusedRopeGradInferMeta(const MetaTensor& sin, const MetaTensor& dout_k, const MetaTensor& dout_v, bool use_neox_rotary_style, + bool time_major, MetaTensor* dq, MetaTensor* dk, MetaTensor* dv) { diff --git a/paddle/phi/infermeta/backward.h b/paddle/phi/infermeta/backward.h index 577250723c12b..bde9c57ff245a 100644 --- a/paddle/phi/infermeta/backward.h +++ b/paddle/phi/infermeta/backward.h @@ -201,6 +201,7 @@ void FusedRopeGradInferMeta(const MetaTensor& sin, const MetaTensor& dout_k, const MetaTensor& dout_v, bool use_neox_rotary_style, + bool time_major, MetaTensor* dq, MetaTensor* dk, MetaTensor* dv); diff --git a/paddle/phi/infermeta/multiary.cc b/paddle/phi/infermeta/multiary.cc index 382fe7dd6c35b..978a80674272f 100644 --- a/paddle/phi/infermeta/multiary.cc +++ b/paddle/phi/infermeta/multiary.cc @@ -4501,6 +4501,7 @@ void FusedRopeInferMeta(const MetaTensor& q, const MetaTensor& cos, const MetaTensor& position_ids, bool use_neox_rotary_style, + bool time_major, MetaTensor* out_q, MetaTensor* out_k, MetaTensor* out_v) { diff --git a/paddle/phi/infermeta/multiary.h b/paddle/phi/infermeta/multiary.h index 13747ba7595cc..0774189dd8d4f 100644 --- a/paddle/phi/infermeta/multiary.h +++ b/paddle/phi/infermeta/multiary.h @@ -887,6 +887,7 @@ void FusedRopeInferMeta(const MetaTensor& q, const MetaTensor& cos, const MetaTensor& position_ids, bool use_neox_rotary_style, + bool time_major, MetaTensor* out_q, MetaTensor* out_k, MetaTensor* out_v); diff --git a/paddle/phi/infermeta/spmd_rules/fused_rope.cc b/paddle/phi/infermeta/spmd_rules/fused_rope.cc index d744c73681c3e..138f0813be2c5 100644 --- a/paddle/phi/infermeta/spmd_rules/fused_rope.cc +++ b/paddle/phi/infermeta/spmd_rules/fused_rope.cc @@ -25,8 +25,6 @@ namespace phi { namespace distributed { using auto_parallel::str_join; -const int kBatchDimIndex = 0; -const int kSeqlenDimIndex = 1; const int kNumHeadsDimIndex = 2; const int kHeadDimIndex = 3; @@ -82,7 +80,8 @@ void check_k_or_v(const DistMetaTensor& k_or_v, void check_sin_cos(const DistMetaTensor& sin, const DistMetaTensor& cos, const DistMetaTensor& position_ids, - const std::vector& q_shape) { + const std::vector& q_shape, + bool time_major) { PADDLE_ENFORCE_EQ(sin.dims(), cos.dims(), phi::errors::InvalidArgument( @@ -99,6 +98,9 @@ void check_sin_cos(const DistMetaTensor& sin, phi::errors::InvalidArgument( "The Tensor sin/cos's ndim must be 2 or 4. but given [%d]", ndim)); + const int kBatchDimIndex = time_major ? 1 : 0; + const int kSeqlenDimIndex = time_major ? 0 : 1; + int batch_size = q_shape[kBatchDimIndex]; int seq_len = q_shape[kSeqlenDimIndex]; int head_dim = q_shape[kHeadDimIndex]; @@ -107,11 +109,11 @@ void check_sin_cos(const DistMetaTensor& sin, int head_dim_index = ndim == 2 ? 1 : 3; if (ndim == 4) { PADDLE_ENFORCE_EQ( - (shape[kBatchDimIndex] == 1 && shape[kNumHeadsDimIndex] == 1), + (shape[0] == 1 && shape[kNumHeadsDimIndex] == 1), true, phi::errors::InvalidArgument("The batch_size and num_heads of sin/cos " "must be 1, but given [%d], [%d]", - shape[kBatchDimIndex], + shape[0], shape[kNumHeadsDimIndex])); } @@ -161,6 +163,7 @@ void infer_sin_cos(const DistMetaTensor& sin, const DistMetaTensor& cos, const DistMetaTensor& position_ids, const std::vector& q_shape, + bool time_major, TensorDistAttr* sin_dist_attr_dst, TensorDistAttr* cos_dist_attr_dst) { const TensorDistAttr& sin_dist_attr_src = sin.dist_attr(); @@ -175,7 +178,7 @@ void infer_sin_cos(const DistMetaTensor& sin, // if one of sin cos is empty, they are all useless in kernel if (!IsEmpty(sin_shape) && !IsEmpty(cos_shape)) { // check sin, cos, position_ids's shape - check_sin_cos(sin, cos, position_ids, q_shape); + check_sin_cos(sin, cos, position_ids, q_shape, time_major); if (sin_shape.size() == 4) { *sin_dist_attr_dst = UnShardTensorDims(sin_dist_attr_src, {1, 3}); *cos_dist_attr_dst = UnShardTensorDims(cos_dist_attr_src, {1, 3}); @@ -192,7 +195,8 @@ SpmdInfo FusedRopeInferSpmd(const DistMetaTensor& q, const DistMetaTensor& sin, const DistMetaTensor& cos, const DistMetaTensor& position_ids, - bool use_neox_rotary_style) { + bool use_neox_rotary_style, + bool time_major) { check_q(q); std::vector>> @@ -202,7 +206,8 @@ SpmdInfo FusedRopeInferSpmd(const DistMetaTensor& q, inputs_sharding_info.emplace_back(qkv_axes, q_dist_attr_src.dims_mapping()); const TensorDistAttr& k_dist_attr_src = k.dist_attr(); - // q_shape = [bs, seq_len, num_heads, head_dim] + // q_shape equals [bs, seq_len, num_heads, head_dim] if time_major is False, + // otherwise [seq_len, bs, num_heads, head_dim] std::vector q_shape = common::vectorize(q.dims()); bool is_k_none = IsEmpty(common::vectorize(k.dims())); // except for q, all other inputs are optional. @@ -219,7 +224,7 @@ SpmdInfo FusedRopeInferSpmd(const DistMetaTensor& q, } const TensorDistAttr& position_ids_dist_attr_src = position_ids.dist_attr(); - std::string position_ids_axes = "ab"; + std::string position_ids_axes = time_major ? "ba" : "ab"; bool is_ids_none = IsEmpty(common::vectorize(position_ids.dims())); if (!is_ids_none) { inputs_sharding_info.emplace_back( @@ -232,7 +237,9 @@ SpmdInfo FusedRopeInferSpmd(const DistMetaTensor& q, GetDimsMappingForAxes(qkv_axes, axis_to_dim_map); TensorDistAttr q_dist_attr_dst = CopyTensorDistAttrForOutput(q_dist_attr_src); q_dist_attr_dst.set_dims_mapping(out_dims_mapping); - q_dist_attr_dst = UnShardTensorDims(q_dist_attr_dst, {1, 3}); + const int kSeqlenDimIndex = time_major ? 0 : 1; + q_dist_attr_dst = + UnShardTensorDims(q_dist_attr_dst, {kSeqlenDimIndex, kHeadDimIndex}); TensorDistAttr k_dist_attr_dst = CopyTensorDistAttrForOutput(k_dist_attr_src); k_dist_attr_dst.set_process_mesh(q_dist_attr_dst.process_mesh()); @@ -248,8 +255,13 @@ SpmdInfo FusedRopeInferSpmd(const DistMetaTensor& q, TensorDistAttr sin_dist_attr_dst; TensorDistAttr cos_dist_attr_dst; - infer_sin_cos( - sin, cos, position_ids, q_shape, &sin_dist_attr_dst, &cos_dist_attr_dst); + infer_sin_cos(sin, + cos, + position_ids, + q_shape, + time_major, + &sin_dist_attr_dst, + &cos_dist_attr_dst); std::vector position_ids_dims_mapping = GetDimsMappingForAxes(position_ids_axes, axis_to_dim_map); @@ -279,7 +291,8 @@ SpmdInfo FusedRopeInferSpmdReverse(const DistMetaTensor& q, const DistMetaTensor& out_q, const DistMetaTensor& out_k, const DistMetaTensor& out_v, - bool use_neox_rotary_style) { + bool use_neox_rotary_style, + bool time_major) { check_q(out_q); std::vector>> outputs_sharding_info; @@ -316,7 +329,10 @@ SpmdInfo FusedRopeInferSpmdReverse(const DistMetaTensor& q, TensorDistAttr q_dist_attr_dst = CopyTensorDistAttrForOutput(out_q_dist_attr_src); q_dist_attr_dst.set_dims_mapping(dims_mapping); - q_dist_attr_dst = UnShardTensorDims(q_dist_attr_dst, {1, 3}); + + const int kSeqlenDimIndex = time_major ? 0 : 1; + q_dist_attr_dst = + UnShardTensorDims(q_dist_attr_dst, {kSeqlenDimIndex, kHeadDimIndex}); TensorDistAttr out_q_dist_attr_dst = q_dist_attr_dst; TensorDistAttr k_dist_attr_dst = CopyTensorDistAttrForOutput(k.dist_attr()); @@ -341,10 +357,11 @@ SpmdInfo FusedRopeInferSpmdReverse(const DistMetaTensor& q, cos, position_ids, out_q_shape, + time_major, &sin_dist_attr_dst, &cos_dist_attr_dst); - std::string position_ids_axes = "ab"; + std::string position_ids_axes = time_major ? "ba" : "ab"; std::vector position_ids_dims_mapping = GetDimsMappingForAxes(position_ids_axes, axis_to_dim_map); TensorDistAttr position_ids_dist_attr_dst = @@ -372,7 +389,8 @@ SpmdInfo FusedRopeGradInferSpmd(const DistMetaTensor& sin, const DistMetaTensor& out_q_grad, const DistMetaTensor& out_k_grad, const DistMetaTensor& out_v_grad, - bool use_neox_rotary_style) { + bool use_neox_rotary_style, + bool time_major) { // NOTE(zhonghui): The forward and backward kernels of fuse rope are same, so // the spmd rules can be shared. SpmdInfo spmd_info = FusedRopeInferSpmd(out_q_grad, @@ -381,7 +399,8 @@ SpmdInfo FusedRopeGradInferSpmd(const DistMetaTensor& sin, sin, cos, position_ids, - use_neox_rotary_style); + use_neox_rotary_style, + time_major); std::vector dist_attrs; std::vector order = {3, 4, 5, 0, 1, 2}; for (int ind : order) { diff --git a/paddle/phi/infermeta/spmd_rules/fused_rope.h b/paddle/phi/infermeta/spmd_rules/fused_rope.h index f28015bc528f1..fdd9ae27500b0 100644 --- a/paddle/phi/infermeta/spmd_rules/fused_rope.h +++ b/paddle/phi/infermeta/spmd_rules/fused_rope.h @@ -29,7 +29,8 @@ SpmdInfo FusedRopeInferSpmd(const DistMetaTensor& q, const DistMetaTensor& sin, const DistMetaTensor& cos, const DistMetaTensor& position_ids, - bool use_neox_rotary_style); + bool use_neox_rotary_style, + bool time_major); SpmdInfo FusedRopeInferSpmdReverse(const DistMetaTensor& q, const DistMetaTensor& k, @@ -40,7 +41,8 @@ SpmdInfo FusedRopeInferSpmdReverse(const DistMetaTensor& q, const DistMetaTensor& out_q, const DistMetaTensor& out_k, const DistMetaTensor& out_v, - bool use_neox_rotary_style); + bool use_neox_rotary_style, + bool time_major); SpmdInfo FusedRopeGradInferSpmd(const DistMetaTensor& sin, const DistMetaTensor& cos, @@ -48,7 +50,8 @@ SpmdInfo FusedRopeGradInferSpmd(const DistMetaTensor& sin, const DistMetaTensor& out_q_grad, const DistMetaTensor& out_k_grad, const DistMetaTensor& out_v_grad, - bool use_neox_rotary_style); + bool use_neox_rotary_style, + bool time_major); } // namespace distributed } // namespace phi diff --git a/paddle/phi/kernels/fusion/gpu/fused_rope_grad_kernel.cu b/paddle/phi/kernels/fusion/gpu/fused_rope_grad_kernel.cu index 787b215d20f37..f7fd4d8589aac 100644 --- a/paddle/phi/kernels/fusion/gpu/fused_rope_grad_kernel.cu +++ b/paddle/phi/kernels/fusion/gpu/fused_rope_grad_kernel.cu @@ -32,6 +32,7 @@ void FusedRopeGradKernel(const Context& dev_ctx, const paddle::optional& dout_k, const paddle::optional& dout_v, bool use_neox_rotary_style, + bool time_major, DenseTensor* dq, DenseTensor* dk, DenseTensor* dv) { @@ -41,10 +42,10 @@ void FusedRopeGradKernel(const Context& dev_ctx, phi::Array inputs_num_heads; // small size for broadcast - auto batch_size = dout_q.dims()[0]; + auto batch_size = time_major ? dout_q.dims()[1] : dout_q.dims()[0]; + auto seq_len = time_major ? dout_q.dims()[0] : dout_q.dims()[1]; inputs_num_heads[0] = dout_q.dims()[2]; auto head_dim = dout_q.dims()[3]; - auto seq_len = dout_q.dims()[1]; PADDLE_ENFORCE_NE(head_dim % 2, 1, phi::errors::InvalidArgument( @@ -117,6 +118,9 @@ void FusedRopeGradKernel(const Context& dev_ctx, : VectorizedFusedRopeWithRotateHalfKernel; if (is_same_num_heads) { + int64_t batch_stride = + time_major ? dout_q.strides()[1] : dout_q.strides()[0]; + int64_t seq_stride = time_major ? dout_q.strides()[0] : dout_q.strides()[1]; kernel_func<<>>(ins_data, sin_cos_data, position_ids_data, @@ -126,13 +130,18 @@ void FusedRopeGradKernel(const Context& dev_ctx, seq_len, inputs_num_heads[0], head_dim, + batch_stride, + seq_stride, outs_data, num_inputs, div_c); } else { // rotary position embedding Q - + int64_t batch_stride_q = + time_major ? dout_q.strides()[1] : dout_q.strides()[0]; + int64_t seq_stride_q = + time_major ? dout_q.strides()[0] : dout_q.strides()[1]; kernel_func<<>>(ins_data, sin_cos_data, position_ids_data, @@ -142,11 +151,20 @@ void FusedRopeGradKernel(const Context& dev_ctx, seq_len, inputs_num_heads[0], head_dim, + batch_stride_q, + seq_stride_q, outs_data, 1, div_c); // rotary position embedding K,V + int64_t batch_stride_kv = time_major + ? inputs_num_heads[1] * head_dim + : seq_len * inputs_num_heads[1] * head_dim; + int64_t seq_stride_kv = time_major + ? batch_size * inputs_num_heads[1] * head_dim + : inputs_num_heads[1] * head_dim; + phi::Array input_kv{ins_data[1], ins_data[2], nullptr}; phi::Array out_kv{outs_data[1], outs_data[2], nullptr}; kernel_func<<>>(input_kv, @@ -158,6 +176,8 @@ void FusedRopeGradKernel(const Context& dev_ctx, seq_len, inputs_num_heads[1], head_dim, + batch_stride_kv, + seq_stride_kv, out_kv, num_inputs - 1, div_c); diff --git a/paddle/phi/kernels/fusion/gpu/fused_rope_kernel.cu b/paddle/phi/kernels/fusion/gpu/fused_rope_kernel.cu index 46a2a0a065652..62c09235f09d8 100644 --- a/paddle/phi/kernels/fusion/gpu/fused_rope_kernel.cu +++ b/paddle/phi/kernels/fusion/gpu/fused_rope_kernel.cu @@ -32,6 +32,7 @@ void FusedRopeKernel(const Context& dev_ctx, const paddle::optional& cos, const paddle::optional& position_ids, bool use_neox_rotary_style, + bool time_major, DenseTensor* out_q, DenseTensor* out_k, DenseTensor* out_v) { @@ -41,9 +42,10 @@ void FusedRopeKernel(const Context& dev_ctx, phi::Array inputs_num_heads; - // q.shape: [batch_size, seq_len, num_heads, head_dim] - auto batch_size = q.dims()[0]; - auto seq_len = q.dims()[1]; + // q.shape: [seq_len, batch_size, num_heads, head_dim] if time_major else + // [batch_size, seq_len, num_heads, head_dim] + auto batch_size = time_major ? q.dims()[1] : q.dims()[0]; + auto seq_len = time_major ? q.dims()[0] : q.dims()[1]; inputs_num_heads[0] = q.dims()[2]; auto head_dim = q.dims()[3]; @@ -187,6 +189,8 @@ void FusedRopeKernel(const Context& dev_ctx, : VectorizedFusedRopeWithRotateHalfKernel; if (is_same_num_heads) { + int64_t batch_stride = time_major ? q.strides()[1] : q.strides()[0]; + int64_t seq_stride = time_major ? q.strides()[0] : q.strides()[1]; kernel_func<<>>(ins_data, sin_cos_data, position_ids_data, @@ -196,10 +200,11 @@ void FusedRopeKernel(const Context& dev_ctx, seq_len, inputs_num_heads[0], head_dim, + batch_stride, + seq_stride, outs_data, num_inputs, div_c); - } else { // Multi Query Attention (MQA) or Group Query Attention (GQA) PADDLE_ENFORCE_EQ( @@ -226,6 +231,9 @@ void FusedRopeKernel(const Context& dev_ctx, inputs_num_heads[2])); } // rotary position embedding Q + int64_t batch_stride_q = time_major ? q.strides()[1] : q.strides()[0]; + int64_t seq_stride_q = time_major ? q.strides()[0] : q.strides()[1]; + kernel_func<<>>(ins_data, sin_cos_data, position_ids_data, @@ -235,6 +243,8 @@ void FusedRopeKernel(const Context& dev_ctx, seq_len, inputs_num_heads[0], head_dim, + batch_stride_q, + seq_stride_q, outs_data, 1, div_c); @@ -242,6 +252,13 @@ void FusedRopeKernel(const Context& dev_ctx, // rotary position embedding K,V phi::Array input_kv{ins_data[1], ins_data[2], nullptr}; phi::Array out_kv{outs_data[1], outs_data[2], nullptr}; + int64_t batch_stride_kv = time_major + ? inputs_num_heads[1] * head_dim + : seq_len * inputs_num_heads[1] * head_dim; + int64_t seq_stride_kv = time_major + ? batch_size * inputs_num_heads[1] * head_dim + : inputs_num_heads[1] * head_dim; + kernel_func<<>>(input_kv, sin_cos_data, position_ids_data, @@ -251,6 +268,8 @@ void FusedRopeKernel(const Context& dev_ctx, seq_len, inputs_num_heads[1], head_dim, + batch_stride_kv, + seq_stride_kv, out_kv, num_inputs - 1, div_c); diff --git a/paddle/phi/kernels/fusion/gpu/fused_rope_utils.h b/paddle/phi/kernels/fusion/gpu/fused_rope_utils.h index 44f68f42e6581..34dab8dab7d0d 100644 --- a/paddle/phi/kernels/fusion/gpu/fused_rope_utils.h +++ b/paddle/phi/kernels/fusion/gpu/fused_rope_utils.h @@ -30,6 +30,8 @@ using VectorizedFusedRopeCudaKernelFunc = int64_t seq_len, int64_t num_heads, int64_t head_dim, + int64_t batch_stride, + int64_t seq_stride, phi::Array outs_data, int num_inputs, MPType div_c); @@ -39,9 +41,12 @@ __device__ void VectorizedGetSinCos(phi::Array sin_cos_data, const int64_t* position_ids_data, bool flag_sin_cos, int64_t index, + int64_t batch_size, int64_t seq_len, int64_t num_heads, int64_t head_dim, + int64_t batch_stride, + int64_t seq_stride, MPType* out_sin, MPType* out_cos, MPType div_c) { @@ -51,17 +56,16 @@ __device__ void VectorizedGetSinCos(phi::Array sin_cos_data, if (flag_sin_cos) { #pragma unroll for (int64_t nx = 0; nx < VecSize; ++nx) { - int64_t index_wc = (index + nx) % (seq_len * num_heads * head_dim); - int64_t pos_seq_ori = index_wc / (num_heads * head_dim); + int64_t pos_seq_ori = (index + nx) / seq_stride % seq_len; int64_t pos_seq; if (position_ids_data) { - int64_t pos_bs = (index + nx) / (seq_len * num_heads * head_dim); + int64_t pos_bs = (index + nx) / batch_stride % batch_size; int64_t index_ids = pos_bs * seq_len + pos_seq_ori; pos_seq = position_ids_data[index_ids]; } else { pos_seq = pos_seq_ori; } - int64_t pos_head = index_wc % head_dim; + int64_t pos_head = (index + nx) % head_dim; int64_t index_sc = pos_seq * head_dim + pos_head; const T* sin_input = sin_cos_data[0] + index_sc; const T* cos_input = sin_cos_data[1] + index_sc; @@ -73,9 +77,9 @@ __device__ void VectorizedGetSinCos(phi::Array sin_cos_data, #pragma unroll for (int nx = 0; nx < VecSize; ++nx) { // get sin_index and cos_index - int64_t index_wc = (index + nx) % (seq_len * num_heads * head_dim); - int64_t pos_seq = index_wc / (num_heads * head_dim); - MPType idx = static_cast((index_wc % head_dim) / 2 * 2.0); + int64_t pos_seq = (index + nx) / seq_stride % seq_len; + + MPType idx = static_cast(((index + nx) % head_dim) / 2 * 2.0); MPType indicses = static_cast(1) / pow(static_cast(10000), idx * static_cast(div_c)); @@ -97,6 +101,8 @@ __global__ void VectorizedFusedRopeWithRotateEveryTwoKernel( int64_t seq_len, int64_t num_heads, int64_t head_dim, + int64_t batch_stride, + int64_t seq_stride, phi::Array outs_data, int num_inputs, MPType div_c) { @@ -119,9 +125,12 @@ __global__ void VectorizedFusedRopeWithRotateEveryTwoKernel( position_ids_data, flag_sin_cos, index, + batch_size, seq_len, num_heads, head_dim, + batch_stride, + seq_stride, sin_value, cos_value, div_c); @@ -172,6 +181,8 @@ __global__ void VectorizedFusedRopeWithRotateHalfKernel( int64_t seq_len, int64_t num_heads, int64_t head_dim, + int64_t batch_stride, + int64_t seq_stride, phi::Array outs_data, int num_inputs, MPType div_c) { @@ -194,9 +205,12 @@ __global__ void VectorizedFusedRopeWithRotateHalfKernel( position_ids_data, flag_sin_cos, index, + batch_size, seq_len, num_heads, head_dim, + batch_stride, + seq_stride, sin_value, cos_value, div_c); diff --git a/python/paddle/incubate/nn/functional/fused_rotary_position_embedding.py b/python/paddle/incubate/nn/functional/fused_rotary_position_embedding.py index 78996a34eeccd..59984b9a68e69 100644 --- a/python/paddle/incubate/nn/functional/fused_rotary_position_embedding.py +++ b/python/paddle/incubate/nn/functional/fused_rotary_position_embedding.py @@ -26,18 +26,20 @@ def fused_rotary_position_embedding( cos=None, position_ids=None, use_neox_rotary_style=True, + time_major=False, ): r""" Fused rotary position embedding. Args: - q (Tensor): The input tensor. The data type is bfloat16, float16, float32 or float64. The shape of q must be [batch_size, seq_len, num_heads, head_dim] and head_dim must be a multiple of 2. - k (Tensor, optional): The input tensor. The data type is bfloat16, float16, float32 or float64. The shape of k must be [batch_size, seq_len, num_heads, head_dim] and head_dim must be a multiple of 2. - v (Tensor, optional): The input tensor. The data type is bfloat16, float16, float32 or float64. The shape of v must be [batch_size, seq_len, num_heads, head_dim] and head_dim must be a multiple of 2. + q (Tensor): The input tensor. The data type is bfloat16, float16, float32 or float64. The shape of q must be [batch_size, seq_len, num_heads, head_dim] or [seq_len, batch_size, num_heads, head_dim] and head_dim must be a multiple of 2. + k (Tensor, optional): The input tensor. The data type is bfloat16, float16, float32 or float64. The shape of k must be [batch_size, seq_len, num_heads, head_dim] or [seq_len, batch_size, num_heads, head_dim] and head_dim must be a multiple of 2. + v (Tensor, optional): The input tensor. The data type is bfloat16, float16, float32 or float64. The shape of v must be [batch_size, seq_len, num_heads, head_dim] or [seq_len, batch_size, num_heads, head_dim] and head_dim must be a multiple of 2. sin (Tensor, optional): The input tensor. The data type is bfloat16, float16, float32 or float64. The shape of sin must be [seq_len, head_dim] or [1, seq_len, 1, head_dim] and head_dim must be a multiple of 2. cos (Tensor, optional): The input tensor. The data type is bfloat16, float16, float32 or float64. The shape of cos must be [seq_len, head_dim] or [1, seq_len, 1, head_dim] and head_dim must be a multiple of 2. position_ids (Tensor, optional): The input tensor. The data type is int64. The shape of position_ids must be [batch_size, seq_len]. use_neox_rotary_style(optional|bool): When the use_neox_rotary_style is True, every two adjacent numbers are calculated. When the use_neox_rotary_style is False, the numbers corresponding to the positions of the front half and back half segments are calculated. Default True. + time_major(optional|bool): Whether the first dimension of the q, k, v input means the time steps. If time_major is True, the shape of Tensor is [seq_len, batch_size, num_heads, head_dim], otherwise [batch_size, seq_len, num_heads, head_dime]. Defaults to False. `time_steps` means the length of input sequence. Returns: out_q/out_k/out_v Tensor representing the fused rotary position embedding, has same shape and data type as `q` . @@ -89,7 +91,7 @@ def fused_rotary_position_embedding( """ if in_dynamic_or_pir_mode(): return _C_ops.fused_rotary_position_embedding( - q, k, v, sin, cos, position_ids, use_neox_rotary_style + q, k, v, sin, cos, position_ids, use_neox_rotary_style, time_major ) helper = LayerHelper('fused_rotary_position_embedding', **locals()) @@ -120,6 +122,7 @@ def fused_rotary_position_embedding( outputs=outputs, attrs={ 'use_neox_rotary_style': use_neox_rotary_style, + 'time_major': time_major, }, ) diff --git a/test/auto_parallel/semi_auto_parallel_for_fused_rope.py b/test/auto_parallel/semi_auto_parallel_for_fused_rope.py index 23e5db193e38f..397399dd5d799 100644 --- a/test/auto_parallel/semi_auto_parallel_for_fused_rope.py +++ b/test/auto_parallel/semi_auto_parallel_for_fused_rope.py @@ -72,6 +72,31 @@ def test_only_q_input(self): out_q.backward() self.check_tensor_eq(dist_q.grad, q.grad) + def test_only_q_input_time_major(self): + paddle.seed(self._seed) + np.random.seed(self._seed) + # [seq_len, bs, num_heads, head_dim] + qkv_shape = [self._seq_len, self._bs, self._num_heads, self._head_dim] + q = paddle.randn(qkv_shape, self._dtype) + q.stop_gradient = False + + dist_q = dist.shard_tensor(q, self._mesh, dist.Shard(0)) + dist_q.stop_gradient = False + + dist_out_q, _, _ = fused_rotary_position_embedding( + q=dist_q, use_neox_rotary_style=False, time_major=True + ) + out_q, _, _ = fused_rotary_position_embedding( + q, use_neox_rotary_style=False, time_major=True + ) + self.check_tensor_eq(out_q, dist_out_q) + # NOTE: fused_rope have not supported shard on seq_len, so reshard to dist.Replicate + self.check_placements(dist_out_q, [dist.Replicate()]) + + dist_out_q.backward() + out_q.backward() + self.check_tensor_eq(dist_q.grad, q.grad) + def test_common_case(self): paddle.seed(self._seed) np.random.seed(self._seed) @@ -133,6 +158,71 @@ def test_common_case(self): self.check_tensor_eq(dist_q.grad, q.grad) self.check_tensor_eq(dist_k.grad, k.grad) + def test_common_case_time_major(self): + paddle.seed(self._seed) + np.random.seed(self._seed) + # [seq_len, bs, num_heads, head_dim] + qkv_shape = [self._seq_len, self._bs, self._num_heads, self._head_dim] + q = paddle.randn(qkv_shape, self._dtype) + q.stop_gradient = False + + dist_q = dist.shard_tensor(q, self._mesh, dist.Shard(1)) + dist_q.stop_gradient = False + + k = paddle.randn(qkv_shape, self._dtype) + k.stop_gradient = False + dist_k = dist.shard_tensor(k, self._mesh, dist.Shard(2)) + dist_k.stop_gradient = False + + sin = paddle.randn(self._sin_cos_shape, self._dtype) + sin.stop_gradient = True + dist_sin = dist.shard_tensor(sin, self._mesh, dist.Replicate()) + dist_sin.stop_gradient = True + + cos = paddle.randn(self._sin_cos_shape, self._dtype) + cos.stop_gradient = True + dist_cos = dist.shard_tensor(cos, self._mesh, dist.Replicate()) + dist_cos.stop_gradient = True + + position_ids = paddle.arange(self._seq_len, dtype="int64").expand( + (self._bs, self._seq_len) + ) + position_ids.stop_gradient = True + dist_position_ids = dist.shard_tensor( + position_ids, self._mesh, dist.Shard(0) + ) + dist_position_ids.stop_gradient = True + + dist_out_q, dist_out_k, _ = fused_rotary_position_embedding( + q=dist_q, + k=dist_k, + sin=dist_sin, + cos=dist_cos, + position_ids=dist_position_ids, + use_neox_rotary_style=False, + time_major=True, + ) + out_q, out_k, _ = fused_rotary_position_embedding( + q=q, + k=k, + sin=sin, + cos=cos, + position_ids=position_ids, + use_neox_rotary_style=False, + time_major=True, + ) + + self.check_tensor_eq(out_q, dist_out_q) + self.check_tensor_eq(out_k, dist_out_k) + + dist_out = dist_out_q + dist_out_k + out = out_q + out_k + dist_out.backward() + out.backward() + + self.check_tensor_eq(dist_q.grad, q.grad) + self.check_tensor_eq(dist_k.grad, k.grad) + def run_test_case(self): if self._backend == "gpu": paddle.set_device("gpu:" + str(dist.get_rank())) @@ -142,7 +232,9 @@ def run_test_case(self): ) self.test_only_q_input() + self.test_only_q_input_time_major() self.test_common_case() + self.test_common_case_time_major() if __name__ == '__main__': diff --git a/test/cpp/auto_parallel/spmd_rule_test.cc b/test/cpp/auto_parallel/spmd_rule_test.cc index ba9bb664c2fd4..25e99fb52575b 100644 --- a/test/cpp/auto_parallel/spmd_rule_test.cc +++ b/test/cpp/auto_parallel/spmd_rule_test.cc @@ -1213,7 +1213,7 @@ TEST(FusedRope, Ctor) { // 1.1 only q input phi::distributed::SpmdInfo forward_spmd_info = phi::distributed::FusedRopeInferSpmd( - q, none, none, none, none, none, false); + q, none, none, none, none, none, false, false); EXPECT_EQ(forward_spmd_info.first.size(), static_cast(6)); EXPECT_EQ(forward_spmd_info.second.size(), static_cast(3)); check_dim_mapping(forward_spmd_info.first[0], {0, -1, -1, -1}); @@ -1237,7 +1237,7 @@ TEST(FusedRope, Ctor) { phi::distributed::DistMetaTensor position_ids = build_input({16, 2048}, {0, 1}); forward_spmd_info = phi::distributed::FusedRopeInferSpmd( - q, k, none, sin, cos, position_ids, false); + q, k, none, sin, cos, position_ids, false, false); EXPECT_EQ(forward_spmd_info.first.size(), static_cast(6)); EXPECT_EQ(forward_spmd_info.second.size(), static_cast(3)); check_dim_mapping(forward_spmd_info.first[0], {0, -1, -1, -1}); @@ -1253,7 +1253,7 @@ TEST(FusedRope, Ctor) { check_partial_dims(forward_spmd_info.second[1], {}); // 2. test backward phi::distributed::SpmdInfo backward_spmd_info = - FusedRopeGradInferSpmd(sin, cos, position_ids, q, k, none, false); + FusedRopeGradInferSpmd(sin, cos, position_ids, q, k, none, false, false); EXPECT_EQ(backward_spmd_info.first.size(), static_cast(6)); EXPECT_EQ(backward_spmd_info.second.size(), static_cast(3)); check_dim_mapping(backward_spmd_info.first[0], {-1, -1, -1, -1}); @@ -1274,7 +1274,7 @@ TEST(FusedRope, Ctor) { phi::distributed::DistMetaTensor out_k = build_input({16, 2048, 64, 128}, {-1, 1, -1, 0}); phi::distributed::SpmdInfo reverse_spmd_info = FusedRopeInferSpmdReverse( - q, k, none, sin, cos, position_ids, out_q, out_k, none, false); + q, k, none, sin, cos, position_ids, out_q, out_k, none, false, false); EXPECT_EQ(reverse_spmd_info.first.size(), static_cast(6)); EXPECT_EQ(reverse_spmd_info.second.size(), static_cast(3)); check_dim_mapping(reverse_spmd_info.first[0], {0, -1, -1, -1}); diff --git a/test/legacy_test/test_fused_rotary_position_embedding.py b/test/legacy_test/test_fused_rotary_position_embedding.py index c63c9d8bc1843..cc0afe5202fd1 100644 --- a/test/legacy_test/test_fused_rotary_position_embedding.py +++ b/test/legacy_test/test_fused_rotary_position_embedding.py @@ -109,6 +109,7 @@ def paddle_fused_rotary_position_embedding( cos_tensor=None, position_ids=None, use_neox_rotary_style=True, + **kwargs ): # permute q, k, v from [batch_size, seq_len, num_heads, head_dim] # to [batch_size, num_heads, seq_len, head_dim] @@ -155,9 +156,9 @@ def paddle_fused_rotary_position_embedding( [ ( "qkv_input", - [2, 8, 2, 16], - [2, 8, 2, 16], - [2, 8, 2, 16], + [2, 8, 2, 16], # bs, seq_len, num_heads, head_dim + [2, 8, 2, 16], # bs, seq_len, num_heads, head_dim + [2, 8, 2, 16], # bs, seq_len, num_heads, head_dim position_ids_list, ), ("qk_input", [2, 8, 2, 16], [2, 8, 2, 16], None, position_ids_list), @@ -232,6 +233,7 @@ def get_forward_backward( with_sin_cos=True, use_neox_rotary_style=True, position_ids=None, + test_time_major=False, ): paddle.disable_static() fw = [] @@ -241,6 +243,15 @@ def get_forward_backward( seed, with_sin_cos ) + if test_time_major: + # [batch_size, seq_len, num_heads, head_dim] -> [seq_len, batch_size, num_heads, head_dim] + if tensor_q is not None: + tensor_q = paddle.transpose(tensor_q, perm=[1, 0]) + if tensor_k is not None: + tensor_k = paddle.transpose(tensor_k, perm=[1, 0]) + if tensor_v is not None: + tensor_v = paddle.transpose(tensor_v, perm=[1, 0]) + out_q, out_k, out_v = rope_function( tensor_q, tensor_k, @@ -249,6 +260,7 @@ def get_forward_backward( tensor_cos, position_ids=position_ids, use_neox_rotary_style=use_neox_rotary_style, + time_major=test_time_major, ) out_init_grad = [] @@ -262,26 +274,42 @@ def get_forward_backward( bw = list( filter(lambda x: x is not None, [tensor_q, tensor_k, tensor_v]) ) + + if test_time_major: + # transpose back + # [seq_len, batch_size, num_heads, head_dim] -> [batch_size, seq_len, num_heads, head_dim] + fw = [paddle.transpose(x, perm=[1, 0]) for x in fw] + bw = [paddle.transpose(x, perm=[1, 0]) for x in bw] + return fw, bw + def check_results(self, p_results, f_results): + for i in range(len(p_results)): + np.testing.assert_allclose( + p_results[i].numpy(), + f_results[i].numpy(), + rtol=self.rtol, + ) + def test_fused_rope(self): p_fw, p_bw = self.get_forward_backward( paddle_fused_rotary_position_embedding, seed=self.seed ) f_fw, f_bw = self.get_forward_backward( - fused_rotary_position_embedding, seed=self.seed + fused_rotary_position_embedding, + seed=self.seed, + test_time_major=False, ) - for i in range(len(p_fw)): - np.testing.assert_allclose( - p_fw[i].numpy(), - f_fw[i].numpy(), - rtol=self.rtol, - ) - np.testing.assert_allclose( - p_bw[i].numpy(), - f_bw[i].numpy(), - rtol=self.rtol, - ) + f_fw_time_major, f_bw_time_major = self.get_forward_backward( + fused_rotary_position_embedding, + seed=self.seed, + test_time_major=True, + ) + + self.check_results(p_fw, f_fw) + self.check_results(p_bw, f_bw) + self.check_results(p_fw, f_fw_time_major) + self.check_results(p_bw, f_bw_time_major) def test_fused_rope_with_sin_cos(self): p_fw, p_bw = self.get_forward_backward( @@ -293,18 +321,19 @@ def test_fused_rope_with_sin_cos(self): fused_rotary_position_embedding, seed=self.seed, with_sin_cos=True, + test_time_major=False, ) - for i in range(len(p_fw)): - np.testing.assert_allclose( - p_fw[i].numpy(), - f_fw[i].numpy(), - rtol=self.rtol, - ) - np.testing.assert_allclose( - p_bw[i].numpy(), - f_bw[i].numpy(), - rtol=self.rtol, - ) + f_fw_time_major, f_bw_time_major = self.get_forward_backward( + fused_rotary_position_embedding, + seed=self.seed, + with_sin_cos=True, + test_time_major=True, + ) + + self.check_results(p_fw, f_fw) + self.check_results(p_bw, f_bw) + self.check_results(p_fw, f_fw_time_major) + self.check_results(p_bw, f_bw_time_major) def test_fused_rope_rotate_half(self): p_fw, p_bw = self.get_forward_backward( @@ -316,18 +345,19 @@ def test_fused_rope_rotate_half(self): fused_rotary_position_embedding, seed=self.seed, use_neox_rotary_style=False, + test_time_major=False, + ) + f_fw_time_major, f_bw_time_major = self.get_forward_backward( + fused_rotary_position_embedding, + seed=self.seed, + use_neox_rotary_style=False, + test_time_major=True, ) - for i in range(len(p_fw)): - np.testing.assert_allclose( - p_fw[i].numpy(), - f_fw[i].numpy(), - rtol=self.rtol, - ) - np.testing.assert_allclose( - p_bw[i].numpy(), - f_bw[i].numpy(), - rtol=self.rtol, - ) + + self.check_results(p_fw, f_fw) + self.check_results(p_bw, f_bw) + self.check_results(p_fw, f_fw_time_major) + self.check_results(p_bw, f_bw_time_major) def test_fused_rope_position_ids(self): position_ids = paddle.to_tensor(self.position_ids_list) @@ -340,18 +370,19 @@ def test_fused_rope_position_ids(self): fused_rotary_position_embedding, seed=self.seed, position_ids=position_ids, + test_time_major=False, ) - for i in range(len(p_fw)): - np.testing.assert_allclose( - p_fw[i].numpy(), - f_fw[i].numpy(), - rtol=self.rtol, - ) - np.testing.assert_allclose( - p_bw[i].numpy(), - f_bw[i].numpy(), - rtol=self.rtol, - ) + f_fw_time_major, f_bw_time_major = self.get_forward_backward( + fused_rotary_position_embedding, + seed=self.seed, + position_ids=position_ids, + test_time_major=True, + ) + + self.check_results(p_fw, f_fw) + self.check_results(p_bw, f_bw) + self.check_results(p_fw, f_fw_time_major) + self.check_results(p_bw, f_bw_time_major) @test_with_pir_api def test_static(self): @@ -448,6 +479,120 @@ def test_static(self): ) paddle.disable_static() + @test_with_pir_api + def test_static_time_major(self): + paddle.disable_static() + tensor_q, tensor_k, tensor_v, tensor_sin, tensor_cos = self.get_inputs( + self.seed, True + ) + p_fw, p_bw = self.get_forward_backward( + paddle_fused_rotary_position_embedding, + seed=self.seed, + use_neox_rotary_style=False, + test_time_major=False, + ) + + paddle.enable_static() + + shape_q = ( + [self.shape_q[1], self.shape_q[0], self.shape_q[2], self.shape_q[3]] + if self.shape_q + else None + ) + shape_k = ( + [self.shape_k[1], self.shape_k[0], self.shape_k[2], self.shape_k[3]] + if self.shape_k + else None + ) + shape_v = ( + [self.shape_v[1], self.shape_v[0], self.shape_v[2], self.shape_v[3]] + if self.shape_v + else None + ) + + main = paddle.static.Program() + startup = paddle.static.Program() + with paddle.static.program_guard(main, startup): + q = ( + None + if shape_q is None + else paddle.static.data( + name="q", shape=shape_q, dtype=self.dtype + ) + ) + + k = ( + None + if shape_k is None + else paddle.static.data( + name="k", shape=shape_k, dtype=self.dtype + ) + ) + + v = ( + None + if shape_v is None + else paddle.static.data( + name="v", shape=shape_v, dtype=self.dtype + ) + ) + + sin = paddle.static.data( + name="sin", + shape=(1, shape_q[0], 1, shape_q[3]), + dtype=self.dtype, + ) + cos = paddle.static.data( + name="cos", + shape=(1, shape_q[0], 1, shape_q[3]), + dtype=self.dtype, + ) + + out_q, out_k, out_v = fused_rotary_position_embedding( + q, + k, + v, + sin, + cos, + position_ids=None, + use_neox_rotary_style=False, + time_major=True, + ) + + exe = paddle.static.Executor() + + feed = { + 'sin': tensor_sin.numpy(), + 'cos': tensor_cos.numpy(), + } + for var_name, input_tensor in zip( + ['q', 'k', 'v'], [tensor_q, tensor_k, tensor_v] + ): + if input_tensor is not None: + feed[var_name] = input_tensor.numpy().transpose((1, 0, 2, 3)) + + fetch_list = [] + for x, out in zip([q, k, v], [out_q, out_k, out_v]): + # The reason why fetch `out` based on `x` is that + # if input is None, the output of static function might be not NoneType + # but pir.Value with type pd_op.tensor<0xf32> in pir mode. + if x is not None: + fetch_list.append(out) + + outs = exe.run( + main, + feed=feed, + fetch_list=fetch_list, + ) + + for i in range(len(p_fw)): + np.testing.assert_allclose( + p_fw[i].numpy(), + outs[i].transpose((1, 0, 2, 3)), + rtol=self.rtol, + ) + paddle.disable_static() + if __name__ == '__main__': unittest.main() From 957b1dd229ce35f0d38fbc3d1fe5b994c85f536c Mon Sep 17 00:00:00 2001 From: Lu Qi <61354321+MarioLulab@users.noreply.github.com> Date: Thu, 22 Feb 2024 20:55:43 +0800 Subject: [PATCH 004/282] fix (#61906) --- python/paddle/distributed/fleet/base/topology.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/paddle/distributed/fleet/base/topology.py b/python/paddle/distributed/fleet/base/topology.py index f0e4ff20c2973..3b5a590ae32e2 100644 --- a/python/paddle/distributed/fleet/base/topology.py +++ b/python/paddle/distributed/fleet/base/topology.py @@ -191,12 +191,13 @@ def __init__(self, topology): assert ( self._check_valid_topo() - ), "mp_num: {}, sharding_num: {}, pp_num: {}, dp_num: {}, sep_num: {}".format( + ), "nranks: {}, mp_num: {}, sharding_num: {}, pp_num: {}, dp_num: {}, sep_num: {}".format( self.nranks, self._mp_degree, self._sharding_degree, self._pp_degree, self._dp_degree, + self._sep_degree, ) # create comm group for pipe parallel From 60902c7737a1be556f73fe03a6181650994f5db4 Mon Sep 17 00:00:00 2001 From: kevin Date: Thu, 22 Feb 2024 21:12:37 +0800 Subject: [PATCH 005/282] [Prim][PIR] add index_sample op forward prim (#61825) * add index_sample decomp * index_sample support dynamic shape * update code * update code --- .../decomp_interface_gen_op_list.py | 2 + .../pir/dialect/op_generator/op_build_gen.py | 1 + .../manual/manual_eager_prim_backend.cc | 10 +++ .../backend/manual/manual_prim_backend.h | 7 ++ .../manual/manual_static_prim_backend.cc | 18 +++++ paddle/fluid/primitive/composite/composite.h | 25 +++++++ test/legacy_test/test_index_sample_op.py | 10 ++- .../test_prim_sub_graph_dynamic_shape.py | 71 +++++++++++++++++++ 8 files changed, 142 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/pir/dialect/op_generator/decomp_interface_gen_op_list.py b/paddle/fluid/pir/dialect/op_generator/decomp_interface_gen_op_list.py index dd99c6c7212e8..b40e8b4d3dea2 100644 --- a/paddle/fluid/pir/dialect/op_generator/decomp_interface_gen_op_list.py +++ b/paddle/fluid/pir/dialect/op_generator/decomp_interface_gen_op_list.py @@ -30,6 +30,7 @@ "gelu", "hardswish", "group_norm", + "index_sample", "index_select", "instance_norm", "layer_norm", @@ -61,6 +62,7 @@ "gelu", "hardswish", "group_norm", + "index_sample", "index_select", "instance_norm", "layer_norm", diff --git a/paddle/fluid/pir/dialect/op_generator/op_build_gen.py b/paddle/fluid/pir/dialect/op_generator/op_build_gen.py index d9828f7752719..7b079605a2460 100644 --- a/paddle/fluid/pir/dialect/op_generator/op_build_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/op_build_gen.py @@ -27,6 +27,7 @@ 'InterpolateInferMeta', 'DeformableConvInferMeta', 'MatrixNMSInferMeta', + 'IndexSampleInferMeta', } _PREPARE_DATA_WITH_VECTOR_INT64_MTTABLE_ATTRIBUTE = {'FrobeniusNormOp'} diff --git a/paddle/fluid/primitive/backend/manual/manual_eager_prim_backend.cc b/paddle/fluid/primitive/backend/manual/manual_eager_prim_backend.cc index ede51d43be1a6..0a71b3f8e47d4 100644 --- a/paddle/fluid/primitive/backend/manual/manual_eager_prim_backend.cc +++ b/paddle/fluid/primitive/backend/manual/manual_eager_prim_backend.cc @@ -35,6 +35,16 @@ Tensor full(const IntArray& shape, } } +template <> +Tensor arange_with_tensor(const Tensor& start, + const Tensor& end, + const Tensor& step, + DataType dtype, + Place place) { + VLOG(4) << "Eager Prim API arange_ad_func call"; + return ::arange_ad_func(start, end, step, dtype, place); +} + } // namespace backend } // namespace primitive } // namespace paddle diff --git a/paddle/fluid/primitive/backend/manual/manual_prim_backend.h b/paddle/fluid/primitive/backend/manual/manual_prim_backend.h index fce33b08f0dff..faf22e5f9807c 100644 --- a/paddle/fluid/primitive/backend/manual/manual_prim_backend.h +++ b/paddle/fluid/primitive/backend/manual/manual_prim_backend.h @@ -41,6 +41,13 @@ Tensor reshape_with_tensor(const Tensor& x, const Tensor& shape); template Tensor expand_with_tensor(const Tensor& x, const Tensor& shape); +template +Tensor arange_with_tensor(const Tensor& start, + const Tensor& end, + const Tensor& step, + DataType dtype = DataType::FLOAT64, + Place place = CPUPlace()); + } // namespace backend } // namespace primitive } // namespace paddle diff --git a/paddle/fluid/primitive/backend/manual/manual_static_prim_backend.cc b/paddle/fluid/primitive/backend/manual/manual_static_prim_backend.cc index acaa143ba811f..a79e929a6e5cc 100644 --- a/paddle/fluid/primitive/backend/manual/manual_static_prim_backend.cc +++ b/paddle/fluid/primitive/backend/manual/manual_static_prim_backend.cc @@ -68,6 +68,24 @@ Tensor expand_with_tensor(const Tensor& x, const Tensor& shape) { return out; } +template <> +Tensor arange_with_tensor(const Tensor& start, + const Tensor& end, + const Tensor& step, + DataType dtype, + Place place) { + pir::Value start_val = + std::static_pointer_cast(start.impl())->value(); + pir::Value end_val = + std::static_pointer_cast(end.impl())->value(); + pir::Value step_val = + std::static_pointer_cast(step.impl())->value(); + auto op_res = + paddle::dialect::arange(start_val, end_val, step_val, dtype, place); + Tensor out(std::make_shared(op_res)); + return out; +} + } // namespace backend } // namespace primitive } // namespace paddle diff --git a/paddle/fluid/primitive/composite/composite.h b/paddle/fluid/primitive/composite/composite.h index 3b81af8530c09..28983fa3cfd63 100644 --- a/paddle/fluid/primitive/composite/composite.h +++ b/paddle/fluid/primitive/composite/composite.h @@ -1001,6 +1001,31 @@ Tensor embedding_decomp(const Tensor& x, return res; } +template +Tensor index_sample_decomp(const Tensor& x, const Tensor& index) { + std::vector tmp_shape{-1, 1}; + auto index_dim = get_slice(shape(index), 0); + auto start = + backend::full_with_tensor(shape(index_dim), 0, index_dim.dtype()); + auto step = + backend::full_with_tensor(shape(index_dim), 1, index_dim.dtype()); + auto arange_tmp = reshape( + backend::arange_with_tensor(start, index_dim, step, index.dtype()), + tmp_shape); + + auto index_res = reshape( + backend::expand_with_tensor(arange_tmp, shape(index)), tmp_shape); + auto index_ = reshape(index, tmp_shape); + auto concat_res = concat({index_res, index_}, 1); + auto res = backend::reshape(gather_nd(x, concat_res), shape(index)); + + if (res.dtype() != x.dtype()) { + return cast(res, x.dtype()); + } else { + return res; + } +} + } // namespace details } // namespace primitive diff --git a/test/legacy_test/test_index_sample_op.py b/test/legacy_test/test_index_sample_op.py index 674c45627b02c..854bf7179b8cb 100755 --- a/test/legacy_test/test_index_sample_op.py +++ b/test/legacy_test/test_index_sample_op.py @@ -26,7 +26,9 @@ class TestIndexSampleOp(OpTest): def setUp(self): self.op_type = "index_sample" + self.prim_op_type = "comp" self.python_api = paddle.index_sample + self.public_python_api = paddle.index_sample self.config() xnp = np.random.random(self.x_shape).astype(self.x_type) if self.x_type == np.complex64 or self.x_type == np.complex128: @@ -47,7 +49,7 @@ def setUp(self): self.outputs = {'Out': out} def test_check_output(self): - self.check_output(check_pir=True) + self.check_output(check_pir=True, check_prim_pir=True) def test_check_grad(self): self.check_grad(['X'], 'Out', check_pir=True) @@ -158,7 +160,9 @@ def config(self): class TestIndexSampleBF16Op(OpTest): def setUp(self): self.op_type = "index_sample" + self.prim_op_type = "comp" self.python_api = paddle.index_sample + self.public_python_api = paddle.index_sample self.config() xnp = np.random.random(self.x_shape).astype(self.x_type) indexnp = np.random.randint( @@ -177,7 +181,9 @@ def setUp(self): self.place = core.CUDAPlace(0) def test_check_output(self): - self.check_output_with_place(self.place, check_pir=True) + self.check_output_with_place( + self.place, check_pir=True, check_prim_pir=True + ) def test_check_grad(self): self.check_grad_with_place(self.place, ['X'], 'Out', check_pir=True) diff --git a/test/prim/pir_prim/test_prim_sub_graph_dynamic_shape.py b/test/prim/pir_prim/test_prim_sub_graph_dynamic_shape.py index 0c0698ef8a311..6be76dd54af38 100644 --- a/test/prim/pir_prim/test_prim_sub_graph_dynamic_shape.py +++ b/test/prim/pir_prim/test_prim_sub_graph_dynamic_shape.py @@ -80,6 +80,10 @@ def tile_net2(x): return y +def index_sample_net(x, index): + return paddle.index_sample(x, index) + + class TestPrimOne(unittest.TestCase): def setUp(self): np.random.seed(2023) @@ -198,5 +202,72 @@ def setUp(self): self.enable_cinn = False +class TestPrimTwo(unittest.TestCase): + def setUp(self): + np.random.seed(2023) + self.shape_x = [300, 4096] + self.shape_y = [300, 2048] + self.dtype_x = "float32" + self.dtype_y = int + self.init_x_shape = [None, 4096] + self.init_y_shape = [None, 2048] + self.x = np.random.random(self.shape_x).astype(self.dtype_x) + self.y = np.random.random(self.shape_y).astype(self.dtype_y) + self.net = index_sample_net + self.necessary_ops = "pd_op.index_sample" + self.enable_cinn = False + + def base_net(self, flag=None): + x = paddle.to_tensor(self.x) + y = paddle.to_tensor(self.y) + if flag == "prim": + core._set_prim_all_enabled(True) + fn = apply_to_static( + self.net, + use_cinn=self.enable_cinn, + input_spec=[ + InputSpec(shape=self.init_x_shape, dtype=self.dtype_x), + InputSpec(shape=self.init_y_shape, dtype=self.dtype_y), + ], + ) + fn.eval() + else: + fn = self.net + res = fn(x, y) + + if flag == "prim": + ops = [ + op.name() + for op in fn.program_cache.last()[-1][-1] + .infer_program.program.global_block() + .ops + ] + assert self.necessary_ops not in ops + core._set_prim_all_enabled(False) + return res + + def test_prim_all_dynamic(self): + res_ref = self.base_net() + res = self.base_net("prim") + for ref, actual in zip(res_ref, res): + np.testing.assert_allclose(ref, actual, rtol=1e-6) + + +class TestPrimTwoIndexSample(TestPrimTwo): + def setUp(self): + np.random.seed(2023) + self.shape_x = [300, 4096] + self.shape_y = [300, 2048] + self.dtype_x = "float32" + self.dtype_y = int + self.init_x_shape = [None, 4096] + self.init_y_shape = [300, 2048] + self.x = np.random.random(self.shape_x).astype(self.dtype_x) + self.y = np.random.random(self.shape_y).astype(self.dtype_y) + self.net = index_sample_net + self.necessary_ops = "pd_op.index_sample" + self.enable_cinn = False + + if __name__ == "__main__": unittest.main() From ba94e24d91e84af2983a381674628e2f37df7039 Mon Sep 17 00:00:00 2001 From: Kunbo Ding Date: Thu, 22 Feb 2024 21:49:23 +0800 Subject: [PATCH 006/282] fix dataloaer for toolkit (#61867) --- python/paddle/io/dataloader/dataloader_iter.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/python/paddle/io/dataloader/dataloader_iter.py b/python/paddle/io/dataloader/dataloader_iter.py index 101fbdb753881..aaa2eae2a7864 100644 --- a/python/paddle/io/dataloader/dataloader_iter.py +++ b/python/paddle/io/dataloader/dataloader_iter.py @@ -704,10 +704,11 @@ def _get_data(self): if len(failed_workers) > 0: self._exit_thread_unexpectedly() pids = ', '.join(str(w.pid) for w in failed_workers) - raise RuntimeError( - f"DataLoader {len(failed_workers)} workers exit unexpectedly, " - f"pids: {pids}" + logging.warning( + "DataLoader {} workers exit unexpectedly, " + "pids: {}".format(len(failed_workers), pids) ) + return # get(timeout) will call _poll(timeout) and may raise IOError if isinstance(e, (IOError, queue.Empty)): From d7c5cf5f1b0482bc4e8e7b93e553f72c97f824fe Mon Sep 17 00:00:00 2001 From: xiongkun Date: Fri, 23 Feb 2024 09:02:57 +0800 Subject: [PATCH 007/282] [Einsum] einsum support broadcast and ... (#61348) --- paddle/phi/infermeta/unary.cc | 6 +- paddle/phi/kernels/cpu/tile_grad_kernel.cc | 2 + paddle/phi/kernels/gpu/tile_grad_kernel.cu | 2 + paddle/phi/kernels/impl/einsum_grad_impl.h | 164 +++++------ paddle/phi/kernels/impl/einsum_impl.h | 299 +++++++-------------- python/paddle/tensor/einsum.py | 40 ++- test/legacy_test/test_einsum_op.py | 59 +++- test/xpu/test_einsum_op_xpu.py | 97 ++++--- 8 files changed, 356 insertions(+), 313 deletions(-) diff --git a/paddle/phi/infermeta/unary.cc b/paddle/phi/infermeta/unary.cc index 611b5239dccdf..5092072f5a87c 100644 --- a/paddle/phi/infermeta/unary.cc +++ b/paddle/phi/infermeta/unary.cc @@ -1152,9 +1152,8 @@ void EinsumInferMeta(const std::vector& inputs, LabelMap labeltype(LabelType::Reduction); std::vector label2perms(inputs.size(), LabelMap(-1)); std::vector all_labels; - std::vector broadcast_dims; std::vector output_dims; - std::vector> ellipsis_dims(2); + std::vector> broadcast_shapes(2); std::vector input_dims; for (auto& i : inputs) { @@ -1168,8 +1167,7 @@ void EinsumInferMeta(const std::vector& inputs, &labeltype, &all_labels, &label2perms, - &ellipsis_dims, - &broadcast_dims, + &broadcast_shapes, &output_dims, &right, &input_strs); diff --git a/paddle/phi/kernels/cpu/tile_grad_kernel.cc b/paddle/phi/kernels/cpu/tile_grad_kernel.cc index 636ade93742da..ed6bc49ed8645 100644 --- a/paddle/phi/kernels/cpu/tile_grad_kernel.cc +++ b/paddle/phi/kernels/cpu/tile_grad_kernel.cc @@ -26,4 +26,6 @@ PD_REGISTER_KERNEL(tile_grad, float, double, int, + phi::dtype::complex, + phi::dtype::complex, int64_t) {} diff --git a/paddle/phi/kernels/gpu/tile_grad_kernel.cu b/paddle/phi/kernels/gpu/tile_grad_kernel.cu index d1e356df401a8..7817d42d031bc 100644 --- a/paddle/phi/kernels/gpu/tile_grad_kernel.cu +++ b/paddle/phi/kernels/gpu/tile_grad_kernel.cu @@ -28,4 +28,6 @@ PD_REGISTER_KERNEL(tile_grad, int, int64_t, phi::dtype::float16, + phi::dtype::complex, + phi::dtype::complex, phi::dtype::bfloat16) {} diff --git a/paddle/phi/kernels/impl/einsum_grad_impl.h b/paddle/phi/kernels/impl/einsum_grad_impl.h index e9623d9f2caed..7652e5e8a9a99 100644 --- a/paddle/phi/kernels/impl/einsum_grad_impl.h +++ b/paddle/phi/kernels/impl/einsum_grad_impl.h @@ -18,6 +18,7 @@ #include "paddle/phi/core/dense_tensor.h" #include "paddle/phi/kernels/complex_kernel.h" #include "paddle/phi/kernels/impl/einsum_impl.h" +#include "paddle/phi/kernels/tile_grad_kernel.h" #include "paddle/phi/kernels/tile_kernel.h" #include "paddle/utils/string/string_helper.h" @@ -27,38 +28,30 @@ template DenseTensor PerformTileAndReduction(const Context& dev_ctx, const LabelMap& label2type, const LabelMap& label2shape, - const std::vector& broadcast_dims, - const std::vector& ellipsis_dims, + const std::vector& broadcast_shape, + const std::vector x_shape, std::string equ, // value pass DenseTensor& t) { // NOLINT auto tmp_label = equ; - ReplaceEllipsis(tmp_label); auto tmp_union = unique_labels(tmp_label); auto op_label = std::string(tmp_union.begin(), tmp_union.end()); - VLOG(5) << "Start PerformTileAndReduction" << equ; + VLOG(5) << "Start PerformTileAndReduction equation " << equ + << " with operand shape: " + << paddle::string::join_strings(common::vectorize(t.dims()), + ","); DenseTensor ret; std::vector repeat_times; std::vector resize_dims; std::vector recover_shape; for (int c : op_label) { if (label2type[c] == LabelType::Reduction) { - // '.' can't be Reduction, so we don't deal '.' here. repeat_times.push_back(label2shape[c]); resize_dims.push_back(1); recover_shape.push_back(label2shape[c]); } else { - if (c != '.') { - resize_dims.push_back(label2shape[c]); - repeat_times.push_back(1); - recover_shape.push_back(label2shape[c]); - } else { - int n_dims = broadcast_dims.size(); - resize_dims.insert( - resize_dims.end(), broadcast_dims.begin(), broadcast_dims.end()); - recover_shape.insert( - recover_shape.end(), ellipsis_dims.begin(), ellipsis_dims.end()); - while (n_dims--) repeat_times.push_back(1); - } + resize_dims.push_back(label2shape[c]); + repeat_times.push_back(1); + recover_shape.push_back(label2shape[c]); } } t.Resize(common::make_ddim(resize_dims)); @@ -72,40 +65,47 @@ DenseTensor PerformTileAndReduction(const Context& dev_ctx, << paddle::string::join_strings(repeat_times, ","); TileKernel(dev_ctx, t, repeat_times, &after_tile); } - size_t n_ellipsis_idx = op_label.find(".", 0); - if (n_ellipsis_idx != std::string::npos) { - // may be we need reduce. broadcast_dims is not equal to ellipsis dims. - std::vector to_reduce; - for (size_t i = 0; i < broadcast_dims.size() - ellipsis_dims.size(); ++i) - to_reduce.push_back(i + n_ellipsis_idx); - - int new_offset = - n_ellipsis_idx + broadcast_dims.size() - ellipsis_dims.size(); - for (size_t i = 0; i < ellipsis_dims.size(); ++i) - if (ellipsis_dims[i] == 1) to_reduce.push_back(i + new_offset); - - VLOG(5) << "PerformTileAndReduction: reduce sum axis: " - << paddle::string::join_strings(to_reduce, ","); - if (to_reduce.size() != 0) { - ret = Sum(dev_ctx, - after_tile, - phi::IntArray(to_reduce), - after_tile.dtype(), - false); // not keep dim. - } else { - ret = after_tile; - } - } else { - ret = after_tile; - } - VLOG(5) << "PerformTileAndReduction: recover shape: " + ret = after_tile; + VLOG(5) << "PermformTileAndReduction: recover shape: " << paddle::string::join_strings(recover_shape, ","); ret.Resize(common::make_ddim(recover_shape)); + // undiagonalize by einsum equation. only contain undiagonal operations. - DenseTensor out; - VLOG(5) << "Undiagonal by einsum with args: " << op_label + "->" + equ; - EinsumInferKernel(dev_ctx, {&ret}, op_label + "->" + equ, &out); - return out; + DenseTensor undiagonal_out; + if (op_label != equ) { + VLOG(5) << "Undiagonal by einsum with args: " << op_label + "->" + equ; + EinsumInferKernel( + dev_ctx, {&ret}, op_label + "->" + equ, &undiagonal_out); + } else { + undiagonal_out = ret; + } + + // call TileGradKernel to reverse broadcast operation. + VLOG(5) << "After diagonalize, we have tensor with shape: " + << paddle::string::join_strings( + common::vectorize(undiagonal_out.dims()), ','); + repeat_times.clear(); + for (size_t i = 0; i < x_shape.size(); ++i) { + VLOG(4) << "broadcast shape is " << broadcast_shape[i] << ", x_shape is " + << x_shape[i]; + repeat_times.push_back(broadcast_shape[i] / x_shape[i]); + } + bool is_all_ones = std::all_of( + repeat_times.begin(), repeat_times.end(), [](int x) { return x == 1; }); + if (is_all_ones) { + VLOG(4) << "don't need broadcast recover, we just return undiagonal_out."; + return undiagonal_out; + } + DenseTensor tmp_x; + DenseTensor broadcast_out; + tmp_x.Resize(common::make_ddim(x_shape)); + broadcast_out.Resize(common::make_ddim(x_shape)); + TileGradKernel( + dev_ctx, tmp_x, undiagonal_out, repeat_times, &broadcast_out); + VLOG(5) << "After broadcast recover, we have tensor with shape: " + << paddle::string::join_strings( + common::vectorize(broadcast_out.dims()), ','); + return broadcast_out; } template @@ -120,8 +120,7 @@ void EinsumGradKernel(const Context& dev_ctx, LabelMap labeltype(LabelType::Reduction); std::vector label2perms(x.size(), LabelMap(-1)); std::vector all_labels; // order: ABO, AO, BO, AB, Reduce - std::vector> ellipsis_dims(2); - std::vector broadcast_dims; + std::vector> broadcast_shapes(2); std::vector output_dims; std::vector input_dims; @@ -136,12 +135,13 @@ void EinsumGradKernel(const Context& dev_ctx, &labeltype, &all_labels, &label2perms, - &ellipsis_dims, - &broadcast_dims, + &broadcast_shapes, &output_dims, &right, &input_strs); + VLOG(4) << "After grad parse einsum equation."; + auto gather_labels_except_reduction = [&labeltype](std::string all) { std::string res(""); for (auto c : all) @@ -160,13 +160,17 @@ void EinsumGradKernel(const Context& dev_ctx, VLOG(5) << "new_equation is " << new_equation; EinsumInferKernel( dev_ctx, new_operands, new_equation, &before_tile); - *(x_grad[0]) = PerformTileAndReduction(dev_ctx, - labeltype, - labelshape, - broadcast_dims, - ellipsis_dims[0], - left, - before_tile); + *(x_grad[0]) = PerformTileAndReduction( + dev_ctx, + labeltype, + labelshape, + broadcast_shapes[0], + common::vectorize(x[0]->dims()), + left, + before_tile); +#ifndef PADDLE_WITH_XPU // xpu is not support conj now, we just disable it. + *(x_grad[0]) = Conj(dev_ctx, *x_grad[0]); +#endif } else { auto splits = paddle::string::split_string(equation, "->"); auto left = splits[0]; @@ -179,7 +183,11 @@ void EinsumGradKernel(const Context& dev_ctx, auto operands_for_A = std::vector(); auto operands_for_B = std::vector(); DenseTensor dA, dB; +#ifndef PADDLE_WITH_XPU // xpu is not support conj now, we just disable it. auto out_grad_conj = Conj(dev_ctx, out_grad); +#else + auto out_grad_conj = out_grad; +#endif // dA = einsum(B, dC) operands_for_A.push_back(x[1]); operands_for_A.push_back(&out_grad_conj); @@ -187,8 +195,6 @@ void EinsumGradKernel(const Context& dev_ctx, operands_for_B.push_back(&out_grad_conj); operands_for_B.push_back(x[0]); - DenseTensor before_tile; - std::vector cache(3); // set empty; TA, TB, TdC if (inner_cache.size() > 0) { // for compatibility, we can load and run v2.3 EinsumOp. @@ -215,24 +221,32 @@ void EinsumGradKernel(const Context& dev_ctx, // now. cache.clear(); if (x_grad[0]) { - *(x_grad[0]) = PerformTileAndReduction(dev_ctx, - labeltype, - labelshape, - broadcast_dims, - ellipsis_dims[0], - ops[0], - dA); + *(x_grad[0]) = PerformTileAndReduction( + dev_ctx, + labeltype, + labelshape, + broadcast_shapes[0], + common::vectorize(x[0]->dims()), + ops[0], + dA); + VLOG(4) << "After call dA"; +#ifndef PADDLE_WITH_XPU // xpu is not support conj now, we just disable it. *(x_grad[0]) = Conj(dev_ctx, *x_grad[0]); +#endif } if (x_grad[1]) { - *(x_grad[1]) = PerformTileAndReduction(dev_ctx, - labeltype, - labelshape, - broadcast_dims, - ellipsis_dims[1], - ops[1], - dB); + *(x_grad[1]) = PerformTileAndReduction( + dev_ctx, + labeltype, + labelshape, + broadcast_shapes[1], + common::vectorize(x[1]->dims()), + ops[1], + dB); +#ifndef PADDLE_WITH_XPU // xpu is not support conj now, we just disable it. *(x_grad[1]) = Conj(dev_ctx, *x_grad[1]); +#endif + VLOG(4) << "After call dA"; } } } diff --git a/paddle/phi/kernels/impl/einsum_impl.h b/paddle/phi/kernels/impl/einsum_impl.h index 6065b1e37b075..89e61eb936cbe 100644 --- a/paddle/phi/kernels/impl/einsum_impl.h +++ b/paddle/phi/kernels/impl/einsum_impl.h @@ -24,6 +24,7 @@ #include "paddle/phi/kernels/full_kernel.h" #include "paddle/phi/kernels/matmul_kernel.h" #include "paddle/phi/kernels/reduce_sum_kernel.h" +#include "paddle/phi/kernels/tile_kernel.h" #include "paddle/phi/kernels/transpose_kernel.h" #include "paddle/utils/string/string_helper.h" @@ -46,11 +47,11 @@ inline static void ValidationCheck(const std::string& equation) { size_t pos; auto trimed_equ = equation; if ((pos = trimed_equ.find("->", 0)) != std::string::npos) { - trimed_equ.replace(pos, 2, "."); + trimed_equ.replace(pos, 2, ""); } auto is_valid_char = [](char c) { if (c >= 'a' && c <= 'z') return true; - if (c == '.' || c == ',') return true; + if (c == ',') return true; return false; }; for (auto c : trimed_equ) { @@ -81,16 +82,14 @@ class LabelMap { public: explicit LabelMap(int default_value = 0) { this->default_value = default_value; - for (int i = 0; i < N; ++i) map[i] = default_value; + for (size_t i = 0; i < N; ++i) map[i] = default_value; } int& operator[](int label) { int i = label - 'a'; - if (label == '.') i = N - 1; return map[i]; } int operator[](int label) const { int i = label - 'a'; - if (label == '.') i = N - 1; return map[i]; } bool exist(char label) { return !is_default(label); } @@ -113,17 +112,6 @@ inline std::string label_to_string(const std::vector& all_labels, return str; } -inline static void ReplaceEllipsis(std::string& s) { // NOLINT - size_t pos; - if ((pos = s.find("...", 0)) != std::string::npos) { - s.replace(pos, 3, "."); - } - // remove all the space in the expression - while ((pos = s.find(" ", 0)) != std::string::npos) { - s.replace(pos, 1, ""); - } -} - template inline std::vector union_labels(const CharIterable1& a, const CharIterable2& b) { @@ -191,8 +179,6 @@ inline static void GlobalInfo(const std::vector& op_labels, if ((*label2type)[c] == LabelType::BO) (*label2type)[c] = LabelType::AO; } - (*label2type)['.'] = LabelType::Batch; - if (sorted_labels->size()) { std::set exist(all.begin(), all.end()); all.clear(); @@ -210,107 +196,61 @@ inline static void GlobalInfo(const std::vector& op_labels, LabelType::Contraction, LabelType::Reduction}); - if (counter[static_cast('.')] > 0) { - std::vector tmp; - tmp.push_back('.'); - // push '.' in the front - *sorted_labels = union_labels(tmp, *sorted_labels); - } VLOG(5) << "GlobalInfo: sorted_labels after: " << paddle::string::join_strings(*sorted_labels, ","); } -inline static void InferLabelShape(const std::vector& op_labels, - const std::vector& inputs, - LabelMap* labelshape, - std::vector>* ellipsis_dims, - std::vector* broadcast_dims) { +inline static void InferLabelShape( + const std::vector& op_labels, + const std::vector& inputs, + LabelMap* labelshape, + std::vector>* broadcast_shapes) { VLOG(5) << "Start InferLabelShape"; - int n_broadcast_dims = 0; - for (size_t i = 0; i < op_labels.size(); ++i) { - VLOG(5) << "oplabels: " << op_labels[i]; - int valid_indices = std::count_if(op_labels[i].begin(), - op_labels[i].end(), - [](char c) { return c != '.'; }); - int n_ellipsis = inputs[i].size() - valid_indices; - VLOG(5) << "valid indices and n_ellipsis: " << valid_indices << " " - << n_ellipsis; - ellipsis_dims->at(i).resize(n_ellipsis); - n_broadcast_dims = std::max(n_broadcast_dims, n_ellipsis); - } - VLOG(5) << "InferLabelShape: Broadcast ndims:" << n_broadcast_dims; - *broadcast_dims = std::vector(n_broadcast_dims, 1); - for (size_t i = 0; i < op_labels.size(); ++i) { auto& op_str = op_labels[i]; auto& op_dim = inputs[i]; int dim_ptr = 0; - for (int c : op_str) { - if (c == '.') { - for (auto& v : ellipsis_dims->at(i)) { - v = op_dim[dim_ptr]; - dim_ptr++; - } - } else if (!labelshape->exist(c) || (*labelshape)[c] == -1) { - (*labelshape)[c] = op_dim[dim_ptr]; - dim_ptr++; - } else if (op_dim[dim_ptr] != -1) { + for (auto& c : op_str) { + if (!labelshape->exist(c) || abs((*labelshape)[c]) == 1) { + (*labelshape)[c] = static_cast(op_dim[dim_ptr]); + } else if (abs(op_dim[dim_ptr]) != 1) { PADDLE_ENFORCE_EQ( (*labelshape)[c], op_dim[dim_ptr], phi::errors::InvalidArgument( "Same label have different shapes for label: `%c`", c)); - dim_ptr++; } + dim_ptr++; } } for (size_t i = 0; i < op_labels.size(); ++i) { - VLOG(5) << "InferLabelShape: Ellipsis ndims:" - << paddle::string::join_strings(ellipsis_dims->at(i), ","); - int idx = n_broadcast_dims - ellipsis_dims->at(i).size(); - for (auto v : ellipsis_dims->at(i)) { - PADDLE_ENFORCE_EQ( - v == 1 || broadcast_dims->at(idx) == 1 || - broadcast_dims->at(idx) == v, - true, - phi::errors::InvalidArgument( - "Ellipsis dims can't broadcasts. Please Check you operands.")); - broadcast_dims->at(idx) = std::max(v, broadcast_dims->at(idx)); - idx += 1; + for (auto& c : op_labels[i]) { + (*broadcast_shapes)[i].push_back((*labelshape)[c]); } } - VLOG(5) << "InferLabelShape: Broadcast dims:" - << paddle::string::join_strings(*broadcast_dims, ","); + for (size_t i = 0; i < op_labels.size(); ++i) { + VLOG(5) << "InferLabelShape: After broadcast shape is:" + << paddle::string::join_strings((*broadcast_shapes)[i], ","); + } } template inline static void InferLabelPerm(const CharIterable& op, - int n_broadcast, LabelMap* label2perm) { int cur = 0; for (int c : op) { if (!label2perm->exist( c)) // can appear repeatedly. we just record the first position. (*label2perm)[c] = cur; - if (c == '.') { - cur += n_broadcast; - } else { - cur += 1; - } + cur += 1; } } inline static void InferOutputDims(const std::string& right, - const std::vector& broadcast_dims, const LabelMap& labelshape, std::vector* output_dims) { for (int c : right) { - if (c == '.') { - output_dims->insert( - output_dims->end(), broadcast_dims.begin(), broadcast_dims.end()); - } else { - output_dims->push_back(labelshape[c]); - } + output_dims->push_back(labelshape[c]); } } // @@ -321,31 +261,26 @@ inline static void ParseEinsumEquation( LabelMap* labeltype, std::vector* all_labels, std::vector* label2perms, - std::vector>* ellipsis_dims, - std::vector* broadcast_dims, + std::vector>* broadcast_shapes, std::vector* output_dims, std::string* right, std::vector* input_strs) { VLOG(5) << "Start ParseEinsumEquation " << equation; auto results = paddle::string::split_string(equation, "->"); auto left = results[0]; - ReplaceEllipsis(left); *right = results[1]; - ReplaceEllipsis(*right); auto op_labels = paddle::string::split_string(left, ","); // split_string("i,") -> ["i", ""], we push back a "". // split_string("->") -> [], we push back a "". - if (op_labels.size() == 0) op_labels.push_back(""); - std::for_each(op_labels.begin(), op_labels.end(), ReplaceEllipsis); + if (op_labels.empty()) op_labels.emplace_back(""); GlobalInfo(op_labels, *right, labeltype, all_labels); - InferLabelShape(op_labels, inputs, labelshape, ellipsis_dims, broadcast_dims); + InferLabelShape(op_labels, inputs, labelshape, broadcast_shapes); VLOG(5) << "Einsum Infershape: right:" << *right; VLOG(5) << "Einsum Infershape: left :" << paddle::string::join_strings(op_labels, '\n'); - InferOutputDims(*right, *broadcast_dims, *labelshape, output_dims); + InferOutputDims(*right, *labelshape, output_dims); for (size_t i = 0; i < inputs.size(); ++i) { - InferLabelPerm( - op_labels[i], ellipsis_dims->at(i).size(), &((*label2perms)[i])); + InferLabelPerm(op_labels[i], &((*label2perms)[i])); (*input_strs).push_back(std::move(op_labels[i])); } } @@ -354,16 +289,11 @@ template std::vector GetLabelIndexByType(const std::vector& all_labels, const LabelMap& type, const LabelMap& perm, - const std::vector& ellipsis, LabelType filter) { std::vector res; for (T c : all_labels) { if ((filter == LabelType::ALL_TYPE || type[c] == filter) && perm[c] != -1) { - if (c == '.') { - for (size_t i = 0; i < ellipsis.size(); ++i) res.push_back(perm[c] + i); - } else { - res.push_back(perm[c]); - } + res.push_back(perm[c]); } } return res; @@ -374,17 +304,13 @@ std::vector GetShapeByType(const std::vector& all_labels, const LabelMap& type, const LabelMap& perm, const LabelMap& label2shape, - const std::vector& ellipsis, std::set filter) { std::vector res; for (T c : all_labels) { if ((filter.count(LabelType::ALL_TYPE) || filter.count(LabelType(type[c]))) && perm[c] != -1) { - if (c == '.') - res.insert(res.end(), ellipsis.begin(), ellipsis.end()); - else - res.push_back(label2shape[c]); + res.push_back(label2shape[c]); } } return res; @@ -443,29 +369,23 @@ DenseTensor Undiagonal(const Context& dev_ctx, template DenseTensor PerformUndiagonal(const Context& dev_ctx, const DenseTensor& tensor, - int n_broadcast, const std::string& equ) { // if the equ is 'iijjkij', then the tensor must be 'ijk', so we have enough // information to do un-diagonal with equ. auto res = tensor; LabelMap label2perm(-1); - InferLabelPerm(equ, n_broadcast, &label2perm); + InferLabelPerm(equ, &label2perm); // Un-Diagonal - int tot = - equ.size() + n_broadcast + (equ.find(".") != std::string::npos ? -1 : 0); + int tot = equ.size(); int cur = tot - 1; for (auto it = equ.rbegin(); it != equ.rend(); ++it) { char c = *it; - if (c == '.') { - cur -= n_broadcast; - } else { - if (cur != label2perm[c]) { - // do diagonal, followed by movedim(). - auto insert_pos = cur - tot + res.dims().size() + 1; - res = Undiagonal(dev_ctx, res, insert_pos, label2perm[c]); - } - --cur; + if (cur != label2perm[c]) { + // do diagonal, followed by movedim(). + auto insert_pos = cur - tot + res.dims().size() + 1; + res = Undiagonal(dev_ctx, res, insert_pos, label2perm[c]); } + --cur; } return res; } @@ -476,37 +396,47 @@ DenseTensor PerformDiagonalAndReduction(const Context& dev_ctx, const std::string& equ, const LabelMap& label2perm, const std::vector& all_labels, - const std::vector& ellipsis, + const std::vector& broadcast_shape, const LabelMap& label2type) { auto res = tensor; + int tot = equ.size(); + // tiling tensor for broadcast + std::vector repeat_times; + auto tensor_origin_shape = common::vectorize(tensor.dims()); + for (size_t i = 0; i < tensor_origin_shape.size(); ++i) { + VLOG(4) << "broadcast shape is " << broadcast_shape[i] + << ", tensor shape is " << tensor_origin_shape[i]; + repeat_times.push_back(broadcast_shape[i] / tensor_origin_shape[i]); + } + DenseTensor after_tile; + bool is_all_ones = std::all_of( + repeat_times.begin(), repeat_times.end(), [](int x) { return x == 1; }); + if (!is_all_ones) { + TileKernel(dev_ctx, res, repeat_times, &after_tile); + res = after_tile; + } // Diagonal - int tot = equ.size() + ellipsis.size() + - (equ.find(".") != std::string::npos ? -1 : 0); int cur = tot - 1; for (auto it = equ.rbegin(); it != equ.rend(); ++it) { char c = *it; - if (c == '.') { - cur -= ellipsis.size(); - } else { - if (cur != label2perm[c]) { - // do diagonal, followed by movedim(). - VLOG(5) << "Do diagonal with shape=" - << paddle::string::join_strings( - common::vectorize(res.dims()), ',') - << ", axis1=" << cur << ", axis2=" << label2perm[c]; - res = Diagonal(dev_ctx, res, 0, cur, label2perm[c]); - res = Transpose( - dev_ctx, res, perm_moveto(res.dims().size(), -1, label2perm[c])); - } - --cur; + if (cur != label2perm[c]) { + // do diagonal, followed by movedim(). + VLOG(5) << "Do diagonal with shape=" + << paddle::string::join_strings( + common::vectorize(res.dims()), ',') + << ", axis1=" << cur << ", axis2=" << label2perm[c]; + res = Diagonal(dev_ctx, res, 0, cur, label2perm[c]); + res = Transpose( + dev_ctx, res, perm_moveto(res.dims().size(), -1, label2perm[c])); } + --cur; } // reduction auto indices = GetLabelIndexByType( - all_labels, label2type, label2perm, ellipsis, LabelType::Reduction); + all_labels, label2type, label2perm, LabelType::Reduction); VLOG(5) << "call PerformDiagonalAndReduction: with axis: " << paddle::string::join_strings(indices, ","); - if (indices.size() == 0) return res; + if (indices.empty()) return res; return Sum( dev_ctx, res, phi::IntArray(indices), res.dtype(), true); } @@ -523,10 +453,9 @@ DenseTensor PerformTranspose(const Context& dev_ctx, const DenseTensor& tensor, const LabelMap& label2perm, const std::vector& all_labels, - const std::vector& ellipsis, const LabelMap& label2type) { auto axis = GetLabelIndexByType( - all_labels, label2type, label2perm, ellipsis, LabelType::ALL_TYPE); + all_labels, label2type, label2perm, LabelType::ALL_TYPE); VLOG(5) << "PerformTranspose: " << paddle::string::join_strings(axis, ","); if (is_no_need_transpose(axis)) { return tensor; @@ -545,35 +474,24 @@ DenseTensor PerformContraction( const std::vector& all_labels, const LabelMap& label2type, const LabelMap& label2shape, - const std::vector>& ellipsis_dims, - const std::vector& broadcast_dims, + const std::vector>& broadcast_shapes, std::vector cache, bool use_cache) { - // Get All the Batches, so perm is auto all_valid = LabelMap(1); - auto recover_dim = GetShapeByType(all_labels, - label2type, - all_valid, - label2shape, - broadcast_dims, - {LabelType::Batch}); + auto recover_dim = GetShapeByType( + all_labels, label2type, all_valid, label2shape, {LabelType::Batch}); auto preprocess = [&](const DenseTensor& t, const LabelMap& perm, - const std::vector& ellipsis, + const std::vector& broadcast, int operand_idx) -> DenseTensor { // reshape auto frees = GetShapeByType(all_labels, label2type, perm, label2shape, - ellipsis, {LabelType::AO, LabelType::BO}); - auto conts = GetShapeByType(all_labels, - label2type, - perm, - label2shape, - ellipsis, - {LabelType::Contraction}); + auto conts = GetShapeByType( + all_labels, label2type, perm, label2shape, {LabelType::Contraction}); std::vector reordered_all_labels = all_labels; if (operand_idx == 1) { reordered_all_labels = TransformLabelsOrder(all_labels, @@ -597,19 +515,15 @@ DenseTensor PerformContraction( input_strs[operand_idx], perm, all_labels, - ellipsis, + broadcast_shapes[operand_idx], label2type); trans_t = PerformTranspose( - dev_ctx, reduct_t, perm, reordered_all_labels, ellipsis, label2type); + dev_ctx, reduct_t, perm, reordered_all_labels, label2type); if (cache[operand_idx] != nullptr) cache[operand_idx]->ShareBufferWith(trans_t); } - auto mul_dims = GetShapeByType(all_labels, - label2type, - perm, - label2shape, - ellipsis, - {LabelType::Batch}); + auto mul_dims = GetShapeByType( + all_labels, label2type, perm, label2shape, {LabelType::Batch}); recover_dim.insert(recover_dim.end(), frees.begin(), frees.end()); if (operand_idx == 0) { mul_dims.push_back(std::accumulate( @@ -632,16 +546,16 @@ DenseTensor PerformContraction( DenseTensor after_contraction; if (operands.size() == 2) { auto trans_a = - preprocess(*(operands[0]), label2perm[0], ellipsis_dims[0], 0); + preprocess(*(operands[0]), label2perm[0], broadcast_shapes[0], 0); auto trans_b = - preprocess(*(operands[1]), label2perm[1], ellipsis_dims[1], 1); + preprocess(*(operands[1]), label2perm[1], broadcast_shapes[1], 1); after_contraction = Matmul(dev_ctx, trans_a, trans_b, false, false); } else if (operands.size() == 1) { after_contraction = - preprocess(*(operands[0]), label2perm[0], ellipsis_dims[0], 0); + preprocess(*(operands[0]), label2perm[0], broadcast_shapes[0], 0); } - if (recover_dim.size() == 0) recover_dim.push_back(1); + if (recover_dim.empty()) recover_dim.push_back(1); VLOG(5) << "PerformContraction: recover_dim: " << paddle::string::join_strings(recover_dim, ","); after_contraction.Resize(common::make_ddim(recover_dim)); @@ -652,31 +566,24 @@ template DenseTensor TransposeToOutput(const Context& dev_ctx, const DenseTensor& to_trans, const std::vector& right, - const std::vector& all_labels, - int n_broadcast_dims) { + const std::vector& all_labels) { std::vector axis; - int offset = 0; - if (std::find(all_labels.begin(), all_labels.end(), '.') != - all_labels.end()) { - offset = n_broadcast_dims - 1; - } for (char c : right) { - if (c == '.') { - for (int i = 0; i < n_broadcast_dims; ++i) axis.push_back(i); - } else { - auto it = std::find(all_labels.begin(), all_labels.end(), c); - PADDLE_ENFORCE_NE(it, - all_labels.end(), - phi::errors::InvalidArgument("Must in all_labels.")); - axis.push_back(it - all_labels.begin() + offset); - } + auto it = std::find(all_labels.begin(), all_labels.end(), c); + PADDLE_ENFORCE_NE(it, + all_labels.end(), + phi::errors::InvalidArgument("Must in all_labels.")); + axis.push_back(it - all_labels.begin()); } if (is_no_need_transpose(axis)) { return to_trans; } VLOG(5) << "call TransposeToOutput: with axis: " - << paddle::string::join_strings(axis, ","); - return Transpose(dev_ctx, to_trans, axis); + << paddle::string::join_strings(axis, ",") + << " to trans dims is: " << to_trans.dims(); + auto output = Transpose(dev_ctx, to_trans, axis); + VLOG(5) << "After Transpose."; + return output; } template @@ -687,15 +594,17 @@ void EinsumKernelImpl(const Context& dev_ctx, DenseTensor* out, std::vector cache, bool is_forward = true) { - VLOG(5) << "Start EinsumKernelImpl"; + VLOG(5) << "Start EinsumKernelImpl with inputs(" << inputs.size() << "): "; + for (auto& i : inputs) { + VLOG(5) << " inputs [ " << i << " ].shape=" << i->dims(); + } ValidationCheck(equation); // collect the following informations to prepare einsum. LabelMap labelshape(0); LabelMap labeltype(LabelType::Reduction); std::vector label2perms(inputs.size(), LabelMap(-1)); std::vector all_labels; // order: ABO, AO, BO, AB, Reduce - std::vector> ellipsis_dims(2); - std::vector broadcast_dims; + std::vector> broadcast_shapes(2); std::vector output_dims; std::vector input_dims; @@ -713,8 +622,7 @@ void EinsumKernelImpl(const Context& dev_ctx, &labeltype, &all_labels, &label2perms, - &ellipsis_dims, - &broadcast_dims, + &broadcast_shapes, &output_dims, &right, &input_strs); @@ -730,17 +638,12 @@ void EinsumKernelImpl(const Context& dev_ctx, all_labels, labeltype, labelshape, - ellipsis_dims, - broadcast_dims, + broadcast_shapes, cache, !is_forward); - *out = TransposeToOutput(dev_ctx, - after_contraction, - unique_labels(right), - all_labels, - broadcast_dims.size()); - *out = PerformUndiagonal( - dev_ctx, *out, broadcast_dims.size(), right); + *out = TransposeToOutput( + dev_ctx, after_contraction, unique_labels(right), all_labels); + *out = PerformUndiagonal(dev_ctx, *out, right); out->Resize(common::make_ddim(output_dims)); } diff --git a/python/paddle/tensor/einsum.py b/python/paddle/tensor/einsum.py index 3ebca4e85e3c6..01e2c2831ec85 100644 --- a/python/paddle/tensor/einsum.py +++ b/python/paddle/tensor/einsum.py @@ -698,6 +698,41 @@ def plan_einsum(operands, g_view, g_shape, g_supports, g_count, n_bcast): return plan +def replace_ellipsis(left_equation, rhs, *operands): + """ + we replace ... as unused variables to simplify the EinsumOp implementation. + """ + ellipsis_strings = None + max_ndim = 0 + new_operands = [] + unused_variables = {chr(c) for c in range(ord('a'), ord('z'))} + for equ, operand in zip(left_equation.split(','), operands): + ndims = len(operand.shape) - len(equ.replace("...", "")) + max_ndim = max(max_ndim, ndims) + for c in equ: + unused_variables.discard(c) + + for equ, operand in zip(left_equation.split(','), operands): + if '...' in equ: + start_unsqueeze_idx = equ.index('...') + to_squeeze_num = max_ndim - ( + len(operand.shape) - len(equ.replace("...", "")) + ) + operand = unsqueeze( + operand, + axis=[i + start_unsqueeze_idx for i in range(to_squeeze_num)], + ) + new_operands.append(operand) + + operands = new_operands + ellipsis_strings = ''.join(unused_variables.pop() for _ in range(max_ndim)) + + if ellipsis_strings is not None: + left_equation = left_equation.replace('...', ellipsis_strings) + rhs = rhs.replace('...', ellipsis_strings) + return left_equation, rhs, operands + + def preprocess(equation, *operands): """ check equation / raise error, default right labels generation @@ -727,7 +762,8 @@ def preprocess(equation, *operands): '...' in lhs and '...' not in rhs ), 'Invalid equation: missing ellipsis in output labels.' - return lhs, rhs, labels + lhs, rhs, operands = replace_ellipsis(lhs, rhs, *operands) + return lhs, rhs, labels, operands def parse_fake_shape(equation, operands, labels): @@ -806,7 +842,7 @@ def einsum_v2(equation, *operands): 3. V2 use opt_einsum.contract_path to optimize the multivariable einsum. """ n_op = len(operands) - lhs, rhs, labels = preprocess(equation, *operands) + lhs, rhs, labels, operands = preprocess(equation, *operands) if n_op <= 2: return gen_einsum_op(lhs + '->' + rhs, *operands) diff --git a/test/legacy_test/test_einsum_op.py b/test/legacy_test/test_einsum_op.py index 86b1cff7ea9ff..e41d1766c126e 100644 --- a/test/legacy_test/test_einsum_op.py +++ b/test/legacy_test/test_einsum_op.py @@ -120,6 +120,29 @@ def set_mandatory(self): self.equation = "ijk,kl->jl" +class TestEinsumAPI(unittest.TestCase): + def setUp(self): + paddle.disable_static() + self.set_mandatory() + + def test_api(self): + inputs = [] + for shape, ty in zip(self.shapes, self.types): + x = paddle.randn(shape).astype(ty) + x.stop_gradient = False + inputs.append(x) + output = paddle.einsum(self.equation, *inputs) + expect = np.einsum(self.equation, *[x.numpy() for x in inputs]) + np.testing.assert_allclose(output.numpy(), expect) + output = output.mean() + output.backward() + + def set_mandatory(self): + self.shapes = [(10,), (10,)] + self.types = [np.float64, np.float64] + self.equation = "...,..." + + class TestEinsumWithReduction1(TestEinsumBinary): def set_mandatory(self): self.shapes = [(10, 3, 3, 5), (10, 5, 10, 10)] @@ -142,34 +165,41 @@ def set_mandatory(self): class TestEinsumWithBroadcast1(TestEinsumBinary): + def set_mandatory(self): + self.shapes = [(5, 10, 3, 3)] + self.types = [np.float64] + self.equation = "ixyz->xyz" + + +class TestEinsumWithBroadcast1API(TestEinsumAPI): def set_mandatory(self): self.shapes = [(5, 10, 3, 3)] self.types = [np.float64] self.equation = "i...->..." -class TestEinsumWithBroadcast2(TestEinsumBinary): +class TestEinsumWithBroadcast2(TestEinsumAPI): def set_mandatory(self): self.shapes = [(10, 11), (3, 4, 5, 10)] self.types = [np.float64, np.float64] self.equation = "...ij,...i->j..." -class TestEinsumWithBroadcast3(TestEinsumBinary): +class TestEinsumWithBroadcast3(TestEinsumAPI): def set_mandatory(self): self.shapes = [(10, 3, 2, 3, 4), (12, 10)] self.types = [np.float64, np.float64] self.equation = "k...,...jk->...k" -class TestEinsumWithBroadcast4(TestEinsumBinary): +class TestEinsumWithBroadcast4(TestEinsumAPI): def set_mandatory(self): self.shapes = [(10, 3, 2, 3, 4), (12, 10)] self.types = [np.float64, np.float64] self.equation = "a...d,...cb->...abcd" -class TestEinsumWithBroadcast5(TestEinsumBinary): +class TestEinsumWithBroadcast5(TestEinsumAPI): def set_mandatory(self): self.shapes = [(3, 2, 2, 10), (10, 3, 2, 2)] self.types = [np.float64, np.float64] @@ -183,6 +213,13 @@ def set_mandatory(self): self.equation = "i,i->" +class TestEinsumWithBroadcast7(TestEinsumAPI): + def set_mandatory(self): + self.shapes = [(32, 13, 13, 12, 12), (1, 12)] + self.types = [np.float64, np.float64] + self.equation = "...ii,...i->...i" + + class TestEinsumWithDiagonal(TestEinsumBinary): def set_mandatory(self): self.shapes = [(10, 10)] @@ -198,6 +235,13 @@ def set_mandatory(self): class TestEinsumWithDiagonal3(TestEinsumBinary): + def set_mandatory(self): + self.shapes = [(5, 3, 2, 1, 4, 5)] + self.types = [np.float64] + self.equation = "axyzwa->xyzw" + + +class TestEinsumWithDiagonal3API(TestEinsumAPI): def set_mandatory(self): self.shapes = [(5, 3, 2, 1, 4, 5)] self.types = [np.float64] @@ -205,6 +249,13 @@ def set_mandatory(self): class TestEinsumWithDiagonal4(TestEinsumBinary): + def set_mandatory(self): + self.shapes = [(5, 3, 2, 1, 4, 5)] + self.types = [np.float64] + self.equation = "axyzwa->axyzw" + + +class TestEinsumWithDiagonal4API(TestEinsumAPI): def set_mandatory(self): self.shapes = [(5, 3, 2, 1, 4, 5)] self.types = [np.float64] diff --git a/test/xpu/test_einsum_op_xpu.py b/test/xpu/test_einsum_op_xpu.py index 57a82009834fa..540d75466b7a4 100644 --- a/test/xpu/test_einsum_op_xpu.py +++ b/test/xpu/test_einsum_op_xpu.py @@ -116,26 +116,6 @@ def set_mandatory(self): self.shapes = [(5, 10, 3, 3), (3, 6, 3, 10)] self.equation = "imjl,jklm->imk" - class TestEinsumWithBroadcast1(TestEinsumBinary): - def set_mandatory(self): - self.shapes = [(5, 10, 3, 3)] - self.equation = "i...->..." - - class TestEinsumWithBroadcast2(TestEinsumBinary): - def set_mandatory(self): - self.shapes = [(10, 11), (3, 4, 5, 10)] - self.equation = "...ij,...i->j..." - - class TestEinsumWithBroadcast4(TestEinsumBinary): - def set_mandatory(self): - self.shapes = [(10, 3, 2, 3, 4), (12, 10)] - self.equation = "a...d,...cb->...abcd" - - class TestEinsumWithBroadcast5(TestEinsumBinary): - def set_mandatory(self): - self.shapes = [(3, 2, 2, 10), (10, 3, 2, 2)] - self.equation = "...a,a...->..." - class TestEinsumWithBroadcast6(TestEinsumBinary): def set_mandatory(self): self.shapes = [(100), (100)] @@ -151,16 +131,6 @@ def set_mandatory(self): self.shapes = [(10, 3, 10)] self.equation = "iji->j" - class TestEinsumWithDiagonal3(TestEinsumBinary): - def set_mandatory(self): - self.shapes = [(5, 3, 2, 1, 4, 5)] - self.equation = "a...a->..." - - class TestEinsumWithDiagonal4(TestEinsumBinary): - def set_mandatory(self): - self.shapes = [(5, 3, 2, 1, 4, 5)] - self.equation = "a...a->a..." - class TestEinsumWithDiagonal5(TestEinsumBinary): def set_mandatory(self): self.shapes = [(8, 8, 8)] @@ -182,5 +152,72 @@ def set_mandatory(self): create_test_class(globals(), XPUTestEinsumOp, stype) +class TestEinsumAPI(unittest.TestCase): + def setUp(self): + paddle.disable_static() + self.set_mandatory() + + def test_api(self): + inputs = [] + for shape, ty in zip(self.shapes, self.types): + x = paddle.randn(shape).astype(ty) + x.stop_gradient = False + inputs.append(x) + output = paddle.einsum(self.equation, *inputs) + expect = np.einsum(self.equation, *[x.numpy() for x in inputs]) + np.testing.assert_allclose( + output.numpy(), expect, atol=0.0006, rtol=0.0001 + ) + output = output.mean() + output.backward() + + def set_mandatory(self): + self.shapes = [(10,), (10,)] + self.types = [np.float32, np.float32] + self.equation = "...,..." + + +class TestEinsumWithBroadcast1(TestEinsumAPI): + def set_mandatory(self): + self.shapes = [(5, 10, 3, 3)] + self.types = [np.float32] + self.equation = "i...->..." + + +class TestEinsumWithBroadcast2(TestEinsumAPI): + def set_mandatory(self): + self.shapes = [(10, 11), (3, 4, 5, 10)] + self.types = [np.float32, np.float32] + self.equation = "...ij,...i->j..." + + +class TestEinsumWithBroadcast4(TestEinsumAPI): + def set_mandatory(self): + self.shapes = [(10, 3, 2, 3, 4), (12, 10)] + self.types = [np.float32, np.float32] + self.equation = "a...d,...cb->...abcd" + + +class TestEinsumWithBroadcast5(TestEinsumAPI): + def set_mandatory(self): + self.shapes = [(3, 2, 2, 10), (10, 3, 2, 2)] + self.types = [np.float32, np.float32] + self.equation = "...a,a...->..." + + +class TestEinsumWithDiagonal3(TestEinsumAPI): + def set_mandatory(self): + self.shapes = [(5, 3, 2, 1, 4, 5)] + self.types = [np.float32] + self.equation = "a...a->..." + + +class TestEinsumWithDiagonal4(TestEinsumAPI): + def set_mandatory(self): + self.shapes = [(5, 3, 2, 1, 4, 5)] + self.types = [np.float32] + self.equation = "a...a->a..." + + if __name__ == "__main__": unittest.main() From d31684503f5c7e29400ec82b3b0d9fcb780725ca Mon Sep 17 00:00:00 2001 From: PommesPeter <54879512+PommesPeter@users.noreply.github.com> Date: Fri, 23 Feb 2024 09:15:30 +0800 Subject: [PATCH 008/282] [Cleanup][B-2] clean some paddle.base.dygraph.to_variable for test (#61904) --------- Co-authored-by: SigureMo --- .../seq2seq_dygraph_model.py | 107 ++++++++---------- .../dygraph_to_static/simnet_dygraph_model.py | 21 ++-- .../test_basic_api_transformation.py | 27 +---- test/dygraph_to_static/test_declarative.py | 15 +-- test/legacy_test/test_activation_op.py | 14 +-- test/legacy_test/test_adam_op.py | 4 +- test/legacy_test/test_adaptive_avg_pool1d.py | 2 +- test/legacy_test/test_adaptive_max_pool1d.py | 2 +- test/legacy_test/test_addmm_op.py | 6 +- test/legacy_test/test_affine_grid_function.py | 6 +- test/legacy_test/test_array_read_write_op.py | 6 +- 11 files changed, 88 insertions(+), 122 deletions(-) diff --git a/test/dygraph_to_static/seq2seq_dygraph_model.py b/test/dygraph_to_static/seq2seq_dygraph_model.py index 2359a7df50239..9be5ab3f5fe08 100644 --- a/test/dygraph_to_static/seq2seq_dygraph_model.py +++ b/test/dygraph_to_static/seq2seq_dygraph_model.py @@ -12,13 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import numpy as np from seq2seq_utils import Seq2SeqModelHyperParams as args import paddle -from paddle import base -from paddle.base import ParamAttr -from paddle.base.dygraph.base import to_variable from paddle.nn import Embedding, Layer INF = 1.0 * 1e5 @@ -112,14 +108,16 @@ def __init__( self.mode = mode self.kinf = 1e9 - param_attr = ParamAttr(initializer=uniform_initializer(self.init_scale)) - bias_attr = ParamAttr(initializer=zero_constant) + param_attr = paddle.ParamAttr( + initializer=uniform_initializer(self.init_scale) + ) + bias_attr = paddle.ParamAttr(initializer=zero_constant) forget_bias = 1.0 self.src_embeder = Embedding( self.src_vocab_size, self.hidden_size, - weight_attr=base.ParamAttr( + weight_attr=paddle.ParamAttr( initializer=uniform_initializer(init_scale) ), ) @@ -128,7 +126,7 @@ def __init__( self.tar_vocab_size, self.hidden_size, sparse=False, - weight_attr=base.ParamAttr( + weight_attr=paddle.ParamAttr( initializer=uniform_initializer(init_scale) ), ) @@ -137,7 +135,7 @@ def __init__( for i in range(num_layers): self.enc_units.append( self.add_sublayer( - "enc_units_%d" % i, + f"enc_units_{i}", BasicLSTMUnit( hidden_size=self.hidden_size, input_size=self.hidden_size, @@ -152,7 +150,7 @@ def __init__( for i in range(num_layers): self.dec_units.append( self.add_sublayer( - "dec_units_%d" % i, + f"dec_units_{i}", BasicLSTMUnit( hidden_size=self.hidden_size, input_size=self.hidden_size, @@ -211,11 +209,11 @@ def forward(self, inputs): # NOTE: modify model code about `enc_hidden` and `enc_cell` to transforme dygraph code successfully. # Because nested list can't be transformed now. - enc_hidden_0 = to_variable( - np.zeros((self.batch_size, self.hidden_size), dtype='float32') + enc_hidden_0 = paddle.zeros( + shape=[self.batch_size, self.hidden_size], dtype='float32' ) - enc_cell_0 = to_variable( - np.zeros((self.batch_size, self.hidden_size), dtype='float32') + enc_cell_0 = paddle.zeros( + shape=[self.batch_size, self.hidden_size], dtype='float32' ) zero = paddle.zeros(shape=[1], dtype="int64") enc_hidden = paddle.tensor.create_array(dtype="float32") @@ -292,8 +290,8 @@ def forward(self, inputs): dec_output = paddle.stack(dec_output) dec_output = self.fc(self._transpose_batch_time(dec_output)) - loss = paddle.nn.functional.softmax_with_cross_entropy( - logits=dec_output, label=label, soft_label=False + loss = paddle.nn.functional.cross_entropy( + input=dec_output, label=label, soft_label=False, reduction="none" ) loss = paddle.squeeze(loss, axis=[2]) max_tar_seq_len = paddle.shape(tar)[1] @@ -312,11 +310,11 @@ def beam_search(self, inputs): self.batch_size = src.shape[0] src_emb = self.src_embeder(self._transpose_batch_time(src)) - enc_hidden_0 = to_variable( - np.zeros((self.batch_size, self.hidden_size), dtype='float32') + enc_hidden_0 = paddle.zeros( + shape=[self.batch_size, self.hidden_size], dtype='float32' ) - enc_cell_0 = to_variable( - np.zeros((self.batch_size, self.hidden_size), dtype='float32') + enc_cell_0 = paddle.zeros( + shape=[self.batch_size, self.hidden_size], dtype='float32' ) zero = paddle.zeros(shape=[1], dtype="int64") enc_hidden = paddle.tensor.create_array(dtype="float32") @@ -367,23 +365,17 @@ def beam_search(self, inputs): # beam search batch_beam_shape = (self.batch_size, self.beam_size) - vocab_size_tensor = to_variable( - np.full((1), self.tar_vocab_size) - ).astype("int64") - start_token_tensor = to_variable( - np.full(batch_beam_shape, self.beam_start_token, dtype='int64') + vocab_size_tensor = paddle.full([1], self.tar_vocab_size, dtype="int64") + start_token_tensor = paddle.full( + batch_beam_shape, self.beam_start_token, dtype="int64" ) - end_token_tensor = to_variable( - np.full(batch_beam_shape, self.beam_end_token, dtype='int64') + end_token_tensor = paddle.full( + batch_beam_shape, self.beam_end_token, dtype="int64" ) step_input = self.tar_embeder(start_token_tensor) - beam_finished = to_variable( - np.full(batch_beam_shape, 0, dtype='float32') - ) - beam_state_log_probs = to_variable( - np.array( - [[0.0] + [-self.kinf] * (self.beam_size - 1)], dtype="float32" - ) + beam_finished = paddle.full(batch_beam_shape, 0, dtype="float32") + beam_state_log_probs = paddle.to_tensor( + [[0.0] + [-self.kinf] * (self.beam_size - 1)], dtype="float32" ) beam_state_log_probs = paddle.expand( beam_state_log_probs, @@ -395,8 +387,7 @@ def beam_search(self, inputs): batch_pos = paddle.expand( paddle.unsqueeze( - to_variable(np.arange(0, self.batch_size, 1, dtype="int64")), - [1], + paddle.arange(0, self.batch_size, 1, dtype="int64"), [1] ), [-1, self.beam_size], ) @@ -437,9 +428,7 @@ def beam_search(self, inputs): ) noend_array = [-self.kinf] * self.tar_vocab_size noend_array[self.beam_end_token] = 0 - noend_mask_tensor = to_variable( - np.array(noend_array, dtype='float32') - ) + noend_mask_tensor = paddle.to_tensor(noend_array, dtype="float32") step_log_probs = paddle.multiply( paddle.expand( @@ -537,14 +526,16 @@ def __init__( self.mode = mode self.kinf = 1e9 - param_attr = ParamAttr(initializer=uniform_initializer(self.init_scale)) - bias_attr = ParamAttr(initializer=zero_constant) + param_attr = paddle.ParamAttr( + initializer=uniform_initializer(self.init_scale) + ) + bias_attr = paddle.ParamAttr(initializer=zero_constant) forget_bias = 1.0 self.src_embeder = Embedding( self.src_vocab_size, self.hidden_size, - weight_attr=base.ParamAttr( + weight_attr=paddle.ParamAttr( name='source_embedding', initializer=uniform_initializer(init_scale), ), @@ -554,7 +545,7 @@ def __init__( self.tar_vocab_size, self.hidden_size, sparse=False, - weight_attr=base.ParamAttr( + weight_attr=paddle.ParamAttr( name='target_embedding', initializer=uniform_initializer(init_scale), ), @@ -564,7 +555,7 @@ def __init__( for i in range(num_layers): self.enc_units.append( self.add_sublayer( - "enc_units_%d" % i, + f"enc_units_{i}", BasicLSTMUnit( hidden_size=self.hidden_size, input_size=self.hidden_size, @@ -580,12 +571,12 @@ def __init__( if i == 0: self.dec_units.append( self.add_sublayer( - "dec_units_%d" % i, + f"dec_units_{i}", BasicLSTMUnit( hidden_size=self.hidden_size, input_size=self.hidden_size * 2, - param_attr=ParamAttr( - name="dec_units_%d" % i, + param_attr=paddle.ParamAttr( + name=f"dec_units_{i}", initializer=uniform_initializer( self.init_scale ), @@ -598,12 +589,12 @@ def __init__( else: self.dec_units.append( self.add_sublayer( - "dec_units_%d" % i, + f"dec_units_{i}", BasicLSTMUnit( hidden_size=self.hidden_size, input_size=self.hidden_size, - param_attr=ParamAttr( - name="dec_units_%d" % i, + param_attr=paddle.ParamAttr( + name=f"dec_units_{i}", initializer=uniform_initializer( self.init_scale ), @@ -726,12 +717,12 @@ def forward(self, inputs): # NOTE: modify model code about `enc_hidden` and `enc_cell` to transform dygraph code successfully. # Because nested list can't be transformed now. - enc_hidden_0 = to_variable( - np.zeros((self.batch_size, self.hidden_size), dtype='float32') + enc_hidden_0 = paddle.zeros( + shape=[self.batch_size, self.hidden_size], dtype='float32' ) enc_hidden_0.stop_gradient = True - enc_cell_0 = to_variable( - np.zeros((self.batch_size, self.hidden_size), dtype='float32') + enc_cell_0 = paddle.zeros( + shape=[self.batch_size, self.hidden_size], dtype='float32' ) enc_hidden_0.stop_gradient = True zero = paddle.zeros(shape=[1], dtype="int64") @@ -789,8 +780,8 @@ def forward(self, inputs): enc_outputs = self._transpose_batch_time(enc_outputs) # train - input_feed = to_variable( - np.zeros((self.batch_size, self.hidden_size), dtype='float32') + input_feed = paddle.zeros( + shape=[self.batch_size, self.hidden_size], dtype='float32' ) # NOTE: set stop_gradient here, otherwise grad var is null input_feed.stop_gradient = True @@ -828,8 +819,8 @@ def forward(self, inputs): dec_output = paddle.stack(dec_output) dec_output = self.fc(self._transpose_batch_time(dec_output)) - loss = paddle.nn.functional.softmax_with_cross_entropy( - logits=dec_output, label=label, soft_label=False + loss = paddle.nn.functional.cross_entropy( + input=dec_output, label=label, soft_label=False, reduction="none" ) loss = paddle.squeeze(loss, axis=[2]) max_tar_seq_len = paddle.shape(tar)[1] diff --git a/test/dygraph_to_static/simnet_dygraph_model.py b/test/dygraph_to_static/simnet_dygraph_model.py index 86d3071f616e5..519f689c77795 100644 --- a/test/dygraph_to_static/simnet_dygraph_model.py +++ b/test/dygraph_to_static/simnet_dygraph_model.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -298,15 +298,16 @@ class FC(paddle.nn.Layer): Examples: .. code-block:: python - from paddle.base.dygraph.base import to_variable - import paddle.base as base - from paddle.base.dygraph import FC - import numpy as np - data = np.random.uniform(-1, 1, [30, 10, 32]).astype('float32') - with base.dygraph.guard(): - fc = FC("fc", 64, num_flatten_dims=2) - data = to_variable(data) - conv = fc(data) + + import paddle + import paddle.base as base + from paddle.base.dygraph import FC + import numpy as np + data = np.random.uniform(-1, 1, [30, 10, 32]).astype('float32') + with base.dygraph.guard(): + fc = FC("fc", 64, num_flatten_dims=2) + data = paddle.to_tensor(data) + conv = fc(data) """ def __init__( diff --git a/test/dygraph_to_static/test_basic_api_transformation.py b/test/dygraph_to_static/test_basic_api_transformation.py index 2a2134d318267..e4dfb37af5faa 100644 --- a/test/dygraph_to_static/test_basic_api_transformation.py +++ b/test/dygraph_to_static/test_basic_api_transformation.py @@ -21,30 +21,13 @@ ) import paddle -from paddle import base, to_tensor -from paddle.base import dygraph -from paddle.base.dygraph import to_variable +from paddle import to_tensor from paddle.jit.api import to_static SEED = 2020 np.random.seed(SEED) -def dyfunc_to_variable(x): - res = base.dygraph.to_variable(x, name=None, zero_copy=None) - return res - - -def dyfunc_to_variable_2(x): - res = dygraph.to_variable(value=np.zeros(shape=(1), dtype=np.int32)) - return res - - -def dyfunc_to_variable_3(x): - res = to_variable(x, name=None, zero_copy=None) - return res - - def dyfunc_to_tensor(x): res1 = paddle.to_tensor(x, dtype=None, place=None, stop_gradient=True) res2 = paddle.tensor.to_tensor(data=res1) @@ -73,15 +56,7 @@ def setUp(self): dyfunc_bool_to_tensor, dyfunc_int_to_tensor, dyfunc_float_to_tensor, - dyfunc_to_variable, - dyfunc_to_variable_2, - dyfunc_to_variable_3, ] - self.place = ( - paddle.CUDAPlace(0) - if paddle.is_compiled_with_cuda() - else paddle.CPUPlace() - ) def get_dygraph_output(self): res = self.dygraph_func(self.input).numpy() diff --git a/test/dygraph_to_static/test_declarative.py b/test/dygraph_to_static/test_declarative.py index df3a136222f62..1ee370b1745bf 100644 --- a/test/dygraph_to_static/test_declarative.py +++ b/test/dygraph_to_static/test_declarative.py @@ -19,11 +19,9 @@ import numpy as np from dygraph_to_static_utils import ( Dy2StTestBase, - enable_to_static_guard, test_ast_only, test_legacy_and_pt_and_pir, ) -from test_basic_api_transformation import dyfunc_to_variable import paddle from paddle.framework import use_pir_api @@ -35,6 +33,11 @@ from paddle.static import InputSpec +def call_to_tensor(x): + res = paddle.to_tensor(x) + return res + + def create_simple_net(): class SimpleNet(Layer): def __init__(self): @@ -370,7 +373,7 @@ class TestDeclarativeAPI(Dy2StTestBase): @test_ast_only @test_legacy_and_pt_and_pir def test_error(self): - func = paddle.jit.to_static(dyfunc_to_variable) + func = paddle.jit.to_static(call_to_tensor) paddle.enable_static() @@ -379,12 +382,6 @@ def test_error(self): with self.assertRaises(RuntimeError): func(np.ones(5).astype("int32")) - with enable_to_static_guard(False): - with self.assertRaises(AssertionError): - # AssertionError: We Only support to_variable in imperative mode, - # please use base.dygraph.guard() as context to run it in imperative Mode - func(np.ones(5).astype("int32")) - paddle.disable_static() diff --git a/test/legacy_test/test_activation_op.py b/test/legacy_test/test_activation_op.py index 485c770dd96df..deecf7fd09a9e 100644 --- a/test/legacy_test/test_activation_op.py +++ b/test/legacy_test/test_activation_op.py @@ -370,7 +370,7 @@ def test_out_name(self): def test_dygraph(self): with base.dygraph.guard(): np_x = np.array([0.1]) - x = base.dygraph.to_variable(np_x) + x = paddle.to_tensor(np_x) z = eval("paddle.%s(x).numpy()" % self.op_type) z_expected = eval("np.%s(np_x)" % self.op_type) np.testing.assert_allclose(z, z_expected, rtol=1e-05) @@ -963,7 +963,7 @@ def test_out_name(self): def test_dygraph(self): with base.dygraph.guard(): np_x = np.array([0.1]) - x = base.dygraph.to_variable(np_x) + x = paddle.to_tensor(np_x) z = paddle.atan(x).numpy() z_expected = np.arctan(np_x) self.assertEqual(z, z_expected) @@ -1036,7 +1036,7 @@ class TestSinhAPI(unittest.TestCase): def test_dygraph(self): with base.dygraph.guard(): np_x = np.array([0.1]) - x = base.dygraph.to_variable(np_x) + x = paddle.to_tensor(np_x) z = paddle.sinh(x).numpy() z_expected = np.sinh(np_x) np.testing.assert_allclose(z, z_expected, rtol=1e-05) @@ -1075,7 +1075,7 @@ def test_backward(self): input_x = np.random.uniform(0.1, 1, test_data_shape).astype( "float32" ) - var = base.dygraph.to_variable(input_x) + var = paddle.to_tensor(input_x) var.stop_gradient = False loss = paddle.sinh(var) loss.backward() @@ -1168,7 +1168,7 @@ class TestCoshAPI(unittest.TestCase): def test_dygraph(self): with base.dygraph.guard(): np_x = np.array([0.1]) - x = base.dygraph.to_variable(np_x) + x = paddle.to_tensor(np_x) z = paddle.cosh(x).numpy() z_expected = np.cosh(np_x) np.testing.assert_allclose(z, z_expected, rtol=1e-05) @@ -1206,7 +1206,7 @@ def test_backward(self): input_x = np.random.uniform(0.1, 1, test_data_shape).astype( "float32" ) - var = base.dygraph.to_variable(input_x) + var = paddle.to_tensor(input_x) var.stop_gradient = False loss = paddle.cosh(var) loss.backward() @@ -4067,7 +4067,7 @@ def test_api(self): # dygraph with base.dygraph.guard(): np_x = np.random.uniform(0.1, 1, [11, 17]).astype("float64") - data_x = base.dygraph.to_variable(np_x) + data_x = paddle.to_tensor(np_x) z = paddle.log1p(data_x) np_z = z.numpy() z_expected = np.array(np.log1p(np_x)) diff --git a/test/legacy_test/test_adam_op.py b/test/legacy_test/test_adam_op.py index 9a4e3c15553b5..c06e249a874e0 100644 --- a/test/legacy_test/test_adam_op.py +++ b/test/legacy_test/test_adam_op.py @@ -725,7 +725,7 @@ def test_pir_adam_op(self): def test_adam_op_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") - a = base.dygraph.to_variable(value) + a = paddle.to_tensor(value) linear = paddle.nn.Linear(13, 5) adam = paddle.optimizer.Adam( @@ -773,7 +773,7 @@ def test_adam_op_with_state_dict(self): def test_adam_with_grad_clip(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") - a = base.dygraph.to_variable(value) + a = paddle.to_tensor(value) linear = paddle.nn.Linear(13, 5) clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) adam = paddle.optimizer.Adam( diff --git a/test/legacy_test/test_adaptive_avg_pool1d.py b/test/legacy_test/test_adaptive_avg_pool1d.py index bca37ba88794f..64075167363aa 100644 --- a/test/legacy_test/test_adaptive_avg_pool1d.py +++ b/test/legacy_test/test_adaptive_avg_pool1d.py @@ -87,7 +87,7 @@ def setUp(self): def check_adaptive_avg_dygraph_results(self, place): with base.dygraph.guard(place): input_np = np.random.random([2, 3, 32]).astype("float32") - input = base.dygraph.to_variable(input_np) + input = paddle.to_tensor(input_np) result = F.adaptive_avg_pool1d(input, output_size=16) result_np = avg_pool1D_forward_naive( input_np, ksize=[16], strides=[0], paddings=[0], adaptive=True diff --git a/test/legacy_test/test_adaptive_max_pool1d.py b/test/legacy_test/test_adaptive_max_pool1d.py index eb12c8d597ba9..33aaa8565bb4f 100644 --- a/test/legacy_test/test_adaptive_max_pool1d.py +++ b/test/legacy_test/test_adaptive_max_pool1d.py @@ -78,7 +78,7 @@ def setUp(self): def check_adaptive_max_dygraph_results(self, place): with base.dygraph.guard(place): input_np = np.random.random([2, 3, 32]).astype("float32") - input = base.dygraph.to_variable(input_np) + input = paddle.to_tensor(input_np) result = F.adaptive_max_pool1d(input, output_size=16) result_np = max_pool1D_forward_naive( diff --git a/test/legacy_test/test_addmm_op.py b/test/legacy_test/test_addmm_op.py index 1e339ad1ceb68..b41532cfe2fac 100644 --- a/test/legacy_test/test_addmm_op.py +++ b/test/legacy_test/test_addmm_op.py @@ -324,9 +324,9 @@ def test_api_with_dygraph(self): np_y = np.random.random((6, 30)).astype(np.float32) with base.dygraph.guard(): - input = base.dygraph.to_variable(np_input) - x = base.dygraph.to_variable(np_x) - y = base.dygraph.to_variable(np_y) + input = paddle.to_tensor(np_input) + x = paddle.to_tensor(np_x) + y = paddle.to_tensor(np_y) out = paddle.tensor.addmm(input, x, y) np.testing.assert_allclose( np_input + np.dot(np_x, np_y), out.numpy(), rtol=1e-5, atol=1e-8 diff --git a/test/legacy_test/test_affine_grid_function.py b/test/legacy_test/test_affine_grid_function.py index a3c01722b7449..e3a2455eed3e4 100644 --- a/test/legacy_test/test_affine_grid_function.py +++ b/test/legacy_test/test_affine_grid_function.py @@ -122,10 +122,12 @@ def test_static_api(self): def paddle_dygraph_layer(self): paddle.disable_static() theta_var = ( - dg.to_variable(self.theta) if not self.invalid_theta else "invalid" + paddle.to_tensor(self.theta) + if not self.invalid_theta + else "invalid" ) output_shape = ( - dg.to_variable(self.output_shape) + paddle.to_tensor(self.output_shape) if self.variable_output_shape else self.output_shape ) diff --git a/test/legacy_test/test_array_read_write_op.py b/test/legacy_test/test_array_read_write_op.py index 05452a9690e2c..499691ef9277c 100644 --- a/test/legacy_test/test_array_read_write_op.py +++ b/test/legacy_test/test_array_read_write_op.py @@ -106,9 +106,9 @@ def test_read_write(self): self.assertAlmostEqual(1.0, g_out_sum, delta=0.1) with base.dygraph.guard(place): - tensor1 = base.dygraph.to_variable(tensor) - tensor2 = base.dygraph.to_variable(tensor) - tensor3 = base.dygraph.to_variable(tensor) + tensor1 = paddle.to_tensor(tensor) + tensor2 = paddle.to_tensor(tensor) + tensor3 = paddle.to_tensor(tensor) x_dygraph = [tensor1, tensor2, tensor3] for each_x in x_dygraph: each_x.stop_gradient = False From d470588e3aac11ed3b16d98405edf60ca505e270 Mon Sep 17 00:00:00 2001 From: lanxianghit <47554610+lanxianghit@users.noreply.github.com> Date: Fri, 23 Feb 2024 09:36:22 +0800 Subject: [PATCH 009/282] [PIR][DynamicShape] Add more ops' InferSymbolicShape (#61930) * Add more ops' InferSymbolicShape * bug fix * bug fix * add log op --- paddle/cinn/hlir/dialect/operator/ir/ops.yaml | 1 + .../infer_symbolic_shape/cinn_op_infer_sym.cc | 19 +++ .../infer_symbolic_shape/cinn_op_infer_sym.h | 2 + .../infer_sym_element_wise_binary.cc | 64 ++++++++-- .../infer_sym_element_wise_binary.h | 39 +++++- .../paddle_op_infer_sym.cc | 111 +----------------- .../paddle_op_infer_sym.h | 54 --------- .../same_operands_and_result.cc | 40 +++++++ .../same_operands_and_result.h | 24 ++++ 9 files changed, 179 insertions(+), 175 deletions(-) diff --git a/paddle/cinn/hlir/dialect/operator/ir/ops.yaml b/paddle/cinn/hlir/dialect/operator/ir/ops.yaml index 9ab3e9381cc44..4faaf8ea2209f 100644 --- a/paddle/cinn/hlir/dialect/operator/ir/ops.yaml +++ b/paddle/cinn/hlir/dialect/operator/ir/ops.yaml @@ -7,6 +7,7 @@ kernel : func : expand param : [x, broadcast_axes] + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : isclose args : (Tensor x, Tensor y, float rtol=1e-5, float atol=1e-8, bool equal_nan=false) diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.cc index 443981f9ef080..ecb56292e170a 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.cc @@ -17,6 +17,25 @@ namespace cinn::dialect { +bool BroadcastOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + const std::vector &shape = + paddle::dialect::details::GetVectorAttr(op, "out_shape"); + + const std::vector &out_dims = [&] { + std::vector out_dims; + for (int64_t dim : shape) { + out_dims.emplace_back(dim); + } + return out_dims; + }(); + + symbol::ShapeOrDataDimExprs shape_data{ + symbol::TensorShapeOrDataDimExprs(out_dims)}; + shape_analysis->SetShapeOrDataForValue(op->result(0), shape_data); + return true; +} + bool SliceOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { // TODO(zhangbopd): Not implemented yet, different from the one in paddle diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.h index fad7d4893d037..896dd44d0b12b 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.h @@ -18,6 +18,8 @@ namespace cinn::dialect { // using paddle::dialect::ScaleOpInferSymbolicShape; +bool BroadcastOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool SliceOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool ConcatOpInferSymbolicShape(pir::Operation *op, diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_element_wise_binary.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_element_wise_binary.cc index 845647b4a79d0..21da5351c617d 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_element_wise_binary.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_element_wise_binary.cc @@ -90,35 +90,85 @@ bool Add_OpInferSymbolicShape(pir::Operation *op, return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } -bool MultiplyOpInferSymbolicShape( +bool BitwiseAndOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } -bool MultiplySrOpInferSymbolicShape( + +bool BitwiseAnd_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return BitwiseAndOpInferSymbolicShape(op, shape_analysis); +} + +bool DivideOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } -bool Multiply_OpInferSymbolicShape( +bool Divide_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } -bool MultiplySr_OpInferSymbolicShape( + +bool ElementwisePowOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } -bool DivideOpInferSymbolicShape( +bool GreaterThanOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } -bool Divide_OpInferSymbolicShape( + +bool GreaterThan_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return GreaterThanOpInferSymbolicShape(op, shape_analysis); +} + +bool LessThanOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } -bool ElementwisePowOpInferSymbolicShape( +bool LessThan_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return LessThanOpInferSymbolicShape(op, shape_analysis); +} + +bool LogicalAndOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } +bool LogicalAnd_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return LogicalAndOpInferSymbolicShape(op, shape_analysis); +} + +bool MultiplyOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); +} +bool MultiplySrOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); +} +bool Multiply_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); +} +bool MultiplySr_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); +} + +bool NotEqualOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); +} + +bool NotEqual_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return NotEqualOpInferSymbolicShape(op, shape_analysis); +} + } // namespace paddle::dialect diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_element_wise_binary.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_element_wise_binary.h index 21aff3276bf1f..e15d769fc8b02 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_element_wise_binary.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_element_wise_binary.h @@ -23,6 +23,38 @@ bool AddOpInferSymbolicShape(pir::Operation *op, bool Add_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool BitwiseAndOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool BitwiseAnd_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool DivideOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Divide_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool ElementwisePowOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool GreaterThanOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool GreaterThan_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool LessThanOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool LessThan_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool LogicalAndOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool LogicalAnd_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + bool MultiplyOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); @@ -35,11 +67,10 @@ bool Multiply_OpInferSymbolicShape( bool MultiplySr_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool DivideOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Divide_OpInferSymbolicShape( +bool NotEqualOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool ElementwisePowOpInferSymbolicShape( +bool NotEqual_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + } // namespace paddle::dialect diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc index f2577ce80fe67..092ecc89cb13f 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc @@ -1008,109 +1008,12 @@ bool Where_OpInferSymbolicShape( return WhereOpInferSymbolicShape(op, shape_analysis); } -bool AssignOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} - -bool Assign_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - return AssignOpInferSymbolicShape(op, shape_analysis); -} - -bool BitwiseAndOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} - -bool BitwiseAnd_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - return BitwiseAndOpInferSymbolicShape(op, shape_analysis); -} - bool FeedOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} - -bool GreaterThanOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + // This Op has NO InferMeta in yaml, just return true return true; } -bool GreaterThan_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - return GreaterThanOpInferSymbolicShape(op, shape_analysis); -} - -bool IncrementOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} - -bool Increment_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - return IncrementOpInferSymbolicShape(op, shape_analysis); -} - -bool LessThanOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} - -bool LessThan_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - return LessThanOpInferSymbolicShape(op, shape_analysis); -} - -bool LogicalAndOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} - -bool LogicalAnd_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - return LogicalAndOpInferSymbolicShape(op, shape_analysis); -} - -bool LogicalNotOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} - -bool LogicalNot_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - return LogicalNotOpInferSymbolicShape(op, shape_analysis); -} - -bool NotEqualOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} - -bool NotEqual_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - return NotEqualOpInferSymbolicShape(op, shape_analysis); -} - bool TopPSamplingOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { PADDLE_THROW(phi::errors::Unimplemented( @@ -1118,18 +1021,6 @@ bool TopPSamplingOpInferSymbolicShape( return true; } -bool LogOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} - -bool Log_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - return LogOpInferSymbolicShape(op, shape_analysis); -} - bool ExpandAsOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { PADDLE_THROW(phi::errors::Unimplemented( diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h index 6626b88226d5e..7c61075247ce0 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h @@ -100,66 +100,12 @@ bool WhereOpInferSymbolicShape(pir::Operation *op, bool Where_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool AssignOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); - -bool Assign_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - -bool BitwiseAndOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - -bool BitwiseAnd_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - bool FeedOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool GreaterThanOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - -bool GreaterThan_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - -bool IncrementOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - -bool Increment_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - -bool LessThanOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - -bool LessThan_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - -bool LogicalAndOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - -bool LogicalAnd_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - -bool LogicalNotOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - -bool LogicalNot_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - -bool NotEqualOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - -bool NotEqual_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - bool TopPSamplingOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool LogOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); - -bool Log_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); - bool ExpandAsOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.cc index c77c640222f97..571b90f7ff552 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.cc @@ -36,6 +36,16 @@ bool Abs_OpInferSymbolicShape(pir::Operation *op, return SameOperandsAndResultShape(op, shape_analysis); } +bool AssignOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} + +bool Assign_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return AssignOpInferSymbolicShape(op, shape_analysis); +} + bool CastOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); @@ -56,6 +66,36 @@ bool Exp_OpInferSymbolicShape(pir::Operation *op, return SameOperandsAndResultShape(op, shape_analysis); } +bool IncrementOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} + +bool Increment_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return IncrementOpInferSymbolicShape(op, shape_analysis); +} + +bool LogOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} + +bool Log_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return LogOpInferSymbolicShape(op, shape_analysis); +} + +bool LogicalNotOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} + +bool LogicalNot_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return LogicalNotOpInferSymbolicShape(op, shape_analysis); +} + bool FullWithTensorOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h index 19fcfac4ca5dc..706bc500048b5 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h @@ -22,6 +22,12 @@ bool AbsOpInferSymbolicShape(pir::Operation *op, bool Abs_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool AssignOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool Assign_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + bool CastOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool Cast_OpInferSymbolicShape(pir::Operation *op, @@ -35,6 +41,24 @@ bool Exp_OpInferSymbolicShape(pir::Operation *op, bool FullWithTensorOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool IncrementOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool Increment_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool LogOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool Log_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool LogicalNotOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool LogicalNot_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + bool PowOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool Pow_OpInferSymbolicShape(pir::Operation *op, From 1185aad03e212afc6ef5b0b11e23521c9ae1719e Mon Sep 17 00:00:00 2001 From: winter-wang <78149749+winter-wang@users.noreply.github.com> Date: Fri, 23 Feb 2024 09:47:57 +0800 Subject: [PATCH 010/282] [PIR] executor support keyword argument. (#61657) * [PIR] executor support keyword argument. * fix test_cinn_sub_graph unit test failed --- .../group_merge/op_with_group_merge_pass.cc | 5 +- .../eager/to_static/run_program_op_node.h | 5 ++ .../pir_adaptor/pir_adaptor_util.cc | 13 +++++ .../pir/transforms/pd_op_to_kernel_pass.cc | 33 +++++++++++++ .../pir/transforms/sub_graph_detector.cc | 9 ++-- paddle/fluid/pybind/pir.cc | 49 ++++++++----------- paddle/pir/include/core/block_argument.h | 10 ++-- paddle/pir/src/core/block.cc | 2 +- paddle/pir/src/core/block_argument.cc | 30 +++++++++++- .../jit/dy2static/pir_partial_program.py | 2 + 10 files changed, 118 insertions(+), 40 deletions(-) diff --git a/paddle/cinn/hlir/dialect/operator/transforms/group_merge/op_with_group_merge_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/group_merge/op_with_group_merge_pass.cc index 12a403740b977..1fdb03eee3e9d 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/group_merge/op_with_group_merge_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/group_merge/op_with_group_merge_pass.cc @@ -73,8 +73,9 @@ std::unordered_set GetProducerOps(pir::Operation* op) { if (!operand || !(operand.source())) { continue; } - auto* source_op = operand.source().defining_op(); - producers.insert(source_op); + if (auto* source_op = operand.source().defining_op()) { + producers.insert(source_op); + } } return producers; } diff --git a/paddle/fluid/eager/to_static/run_program_op_node.h b/paddle/fluid/eager/to_static/run_program_op_node.h index 00f6b04781cbc..17cb367e72980 100644 --- a/paddle/fluid/eager/to_static/run_program_op_node.h +++ b/paddle/fluid/eager/to_static/run_program_op_node.h @@ -183,6 +183,11 @@ static auto GetNameFromValue(const ::pir::Block *block, bool is_input) { // we use name here, later value is used directly. std::unordered_map<::pir::Value, std::string> value2name; + if (is_input) { + for (auto &kwarg : block->kwargs()) { + value2name[kwarg.second] = kwarg.first; + } + } for (auto &op : *block) { std::string name; if (is_input && op.name() == "pd_op.data") { diff --git a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc index d9005802cd24a..aa9003cb164f9 100644 --- a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc +++ b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc @@ -743,6 +743,19 @@ void BuildScope(const pir::Block& block, << GenScopeTreeDebugInfo( const_cast(value_exe_info->GetScope()->root())); + VLOG(6) << "Start handle keyword blockargument!"; + for (auto& kwarg : block.kwargs()) { + VLOG(6) << "link keyword blockargument in variable" + << value_exe_info->GetScope(); + Variable* var = value_exe_info->GetScope()->FindVar(kwarg.first); + PADDLE_ENFORCE(var, + paddle::platform::errors::InvalidArgument( + "The variable %s shoud exist", kwarg.first)); + + value_exe_info->Add(kwarg.second, kwarg.first); + } + VLOG(6) << "Finished handle keyword blockargument!"; + for (auto& op : block) { std::string op_name = op.name(); if (op.attributes().count("op_name")) { diff --git a/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc b/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc index d23819cd5b50c..9cd2c89eda866 100644 --- a/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc +++ b/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc @@ -2537,6 +2537,39 @@ void ProcessBlock( bool for_if_block) { auto inputs_by_data_op = GetInputsByDataOp(block); + for (auto& [keyword, arg] : block->kwargs()) { + auto new_arg = new_block->AddKwarg(keyword, arg.type()); + (*map_value_pair)[arg] = new_arg; + if (auto dense_tensor_type = arg.type().dyn_cast()) { + new_arg.set_type(AllocatedDenseTensorType::get( + ctx, phi::CPUPlace(), dense_tensor_type)); + } + } + if (platform::is_gpu_place(place)) { + for (auto& [keyword, arg] : block->kwargs()) { + if (auto dense_tensor_type = arg.type().dyn_cast()) { + auto dtype = dense_tensor_type.dtype(); + phi::KernelKey shadow_key{ + phi::Backend::GPU, phi::DataLayout::ANY, TransToPhiDataType(dtype)}; + std::unordered_map attr_map{ + {"op_name", pir::StrAttribute::get(ctx, "pd_op.shadow_feed")}, + {"kernel_name", pir::StrAttribute::get(ctx, "shadow_feed")}, + {"kernel_key", KernelAttribute::get(ctx, shadow_key)}}; + + auto out_type = + AllocatedDenseTensorType::get(ctx, place, dense_tensor_type); + + pir::OpInfo phi_kernel_op_info = + ctx->GetRegisteredOpInfo(PhiKernelOp::name()); + pir::Operation* shadow_op = pir::Operation::Create( + {(*map_value_pair)[arg]}, attr_map, {out_type}, phi_kernel_op_info); + + new_block->push_back(shadow_op); + (*map_value_pair)[arg] = shadow_op->result(0); + } + } + } + for (auto iter = block->begin(); iter != block->end(); ++iter) { pir::Operation* op_item = &(*iter); VLOG(6) << "op name " << op_item->name(); diff --git a/paddle/fluid/pir/transforms/sub_graph_detector.cc b/paddle/fluid/pir/transforms/sub_graph_detector.cc index 051bbb4c2d224..dcb55412feb1f 100644 --- a/paddle/fluid/pir/transforms/sub_graph_detector.cc +++ b/paddle/fluid/pir/transforms/sub_graph_detector.cc @@ -83,7 +83,7 @@ std::vector InverselyTopologicalSort(pir::Block* block) { } auto* defined_op = operand.source().defining_op(); --pending_count[defined_op]; - if (pending_count[defined_op] == 0) { + if (defined_op && pending_count[defined_op] == 0) { queue.push(defined_op); } } @@ -109,7 +109,7 @@ std::vector GetProducerOpsReverseSort( continue; } auto* source_op = operand.source().defining_op(); - if (!producers.count(source_op)) { + if (source_op && !producers.count(source_op)) { producers.insert(source_op); PADDLE_ENFORCE( op2id.count(source_op), @@ -134,8 +134,9 @@ std::unordered_set GetProducerOps(pir::Operation* op) { if (!operand || !(operand.source())) { continue; } - auto* source_op = operand.source().defining_op(); - producers.insert(source_op); + if (auto* source_op = operand.source().defining_op()) { + producers.insert(source_op); + } } return producers; } diff --git a/paddle/fluid/pybind/pir.cc b/paddle/fluid/pybind/pir.cc index fe52599c88962..99e1b624edefa 100644 --- a/paddle/fluid/pybind/pir.cc +++ b/paddle/fluid/pybind/pir.cc @@ -198,7 +198,11 @@ std::string GetValueInfo(Value v) { ss << "define_op_name=" << op_result.owner()->name(); ss << ", index=" << op_result.index(); } else if (auto arg = v.dyn_cast()) { - ss << "block_arg, index = " << arg.index(); + if (arg.is_kwarg()) { + ss << "keyword block_arg, keyword = " << arg.keyword(); + } else { + ss << "position block_arg, index = " << arg.index(); + } } if (!v.type()) { ss << ", dtype=<>"; @@ -408,6 +412,7 @@ void BindBlock(py::module *m) { }) .def("__len__", [](Block &self) { return self.size(); }) .def("args", &Block::args, return_value_policy::reference) + .def("kwargs", &Block::kwargs, return_value_policy::reference) .def( "remove_op", [](Block &self, Operation *op) { @@ -1116,7 +1121,7 @@ SplitedResult SplitForwardBackward( std::unordered_set backward_inputs; std::tie(middle_values, backward_inputs) = AnalysisMiddleVariable( program, forward_in_out_values, forward_range, backward_range); - pir::Builder backward_builder = pir::Builder(ctx, backward_program->block()); + pir::Block &backward_block = *backward_program->block(); bool has_backward = (backward_range[1] > backward_range[0]); // forward program construct. @@ -1137,28 +1142,14 @@ SplitedResult SplitForwardBackward( pir::IrMapping backward_mapper; auto &backward_value_map = backward_mapper.GetMutableMap(); int counter = 0; - auto create_data_fn = [&backward_builder, - &backward_inputs, - &backward_value_map, - &counter](const pir::Value &v) { - if (v.impl() == nullptr || !backward_inputs.count(v)) { - return; + auto create_kwarg_fn = [&backward_block, + &backward_inputs, + &backward_value_map, + &counter](const pir::Value &v) { + if (v && backward_inputs.count(v)) { + backward_value_map[v] = backward_block.AddKwarg( + "input_" + std::to_string(counter++), v.type()); } - auto value_type = v.type().dyn_cast(); - auto dtype = paddle::dialect::TransToPhiDataType(value_type.dtype()); - auto shape = common::vectorize(value_type.dims()); - auto place = phi::Place(); - - paddle::dialect::DataOp op = - backward_builder.Build( - std::string("input_") + std::to_string(counter), - shape, - dtype, - place); - counter += 1; - pir::Value target = op->results()[0].Value::impl(); - target.set_type(v.type()); - backward_value_map[v] = target; }; auto create_output_fn_forward = [&ctx, @@ -1227,21 +1218,23 @@ SplitedResult SplitForwardBackward( VLOG(4) << "Create pd.data for backward program: fo, start with input_" << counter; std::for_each( - forward_outputs.begin(), forward_outputs.end(), create_data_fn); + forward_outputs.begin(), forward_outputs.end(), create_kwarg_fn); VLOG(4) << "Create pd.data for backward program: fx, start with input_" << counter; - std::for_each(forward_inputs.begin(), forward_inputs.end(), create_data_fn); + std::for_each( + forward_inputs.begin(), forward_inputs.end(), create_kwarg_fn); VLOG(4) << "Create pd.data for backward program: fp, start with input_" << counter; - std::for_each(forward_params.begin(), forward_params.end(), create_data_fn); + std::for_each( + forward_params.begin(), forward_params.end(), create_kwarg_fn); VLOG(4) << "Create pd.data for backward program: fm, start with input_" << counter; - std::for_each(middle_values.begin(), middle_values.end(), create_data_fn); + std::for_each(middle_values.begin(), middle_values.end(), create_kwarg_fn); VLOG(4) << "Create pd.data for backward program: fo_g, start with input_" << counter; std::for_each(forward_outputs_grads.begin(), forward_outputs_grads.end(), - create_data_fn); + create_kwarg_fn); VLOG(4) << "Create pd.data for backward program end. input_" << counter; } diff --git a/paddle/pir/include/core/block_argument.h b/paddle/pir/include/core/block_argument.h index 017c0a6544f72..3ddf7847fd8a2 100644 --- a/paddle/pir/include/core/block_argument.h +++ b/paddle/pir/include/core/block_argument.h @@ -24,8 +24,8 @@ class BlockArgumentImpl; } // namespace detail /// -/// \brief BlockArgument class represents the value defined by a result of -/// operation. This class only provides interfaces, for specific implementation, +/// \brief BlockArgument class represents the value defined by a argument of +/// block. This class only provides interfaces, for specific implementation, /// see Impl class. /// class IR_API BlockArgument : public Value { @@ -33,6 +33,8 @@ class IR_API BlockArgument : public Value { BlockArgument() = default; Block *owner() const; uint32_t index() const; + const std::string &keyword() const; + bool is_kwarg() const; const AttributeMap &attributes() const; Attribute attribute(const std::string &key) const; @@ -44,6 +46,9 @@ class IR_API BlockArgument : public Value { /// create a new argument with the given type and owner. static BlockArgument Create(Type type, Block *owner, uint32_t index); + static BlockArgument Create(Type type, + Block *owner, + const std::string &keyword); /// Destroy the argument. void Destroy(); /// set the position in the block argument list. @@ -56,5 +61,4 @@ class IR_API BlockArgument : public Value { static bool classof(Value value); static BlockArgument dyn_cast_from(Value value); }; - } // namespace pir diff --git a/paddle/pir/src/core/block.cc b/paddle/pir/src/core/block.cc index 9e4c6179e5af2..258f681b303cb 100644 --- a/paddle/pir/src/core/block.cc +++ b/paddle/pir/src/core/block.cc @@ -126,7 +126,7 @@ Value Block::AddKwarg(const std::string &keyword, Type type) { IR_ENFORCE(kwargs_.find(keyword) == kwargs_.end(), "Add keyword (%s) argument which has been existed.", keyword.c_str()); - auto arg = BlockArgument::Create(type, this, 0); + auto arg = BlockArgument::Create(type, this, keyword); kwargs_[keyword] = arg; return arg; } diff --git a/paddle/pir/src/core/block_argument.cc b/paddle/pir/src/core/block_argument.cc index 19c5e2b0ef917..99a799e9f592e 100644 --- a/paddle/pir/src/core/block_argument.cc +++ b/paddle/pir/src/core/block_argument.cc @@ -50,7 +50,15 @@ class BlockArgumentImpl : public ValueImpl { private: BlockArgumentImpl(Type type, Block *owner, uint32_t index) - : ValueImpl(type, BLOCK_ARG_IDX), owner_(owner), index_(index) {} + : ValueImpl(type, BLOCK_ARG_IDX), + owner_(owner), + index_(index), + is_kwarg_(false) {} + BlockArgumentImpl(Type type, Block *owner, const std::string &keyword) + : ValueImpl(type, BLOCK_ARG_IDX), + owner_(owner), + is_kwarg_(true), + keyword_(keyword) {} ~BlockArgumentImpl(); // access construction and owner @@ -58,7 +66,9 @@ class BlockArgumentImpl : public ValueImpl { AttributeMap attributes_; Block *owner_; - uint32_t index_; + uint32_t index_ = 0xFFFFFFFF; + bool is_kwarg_; + std::string keyword_ = "uninitialized_keyword"; }; BlockArgumentImpl::~BlockArgumentImpl() { @@ -85,6 +95,16 @@ uint32_t BlockArgument::index() const { return IMPL_->index_; } +const std::string &BlockArgument::keyword() const { + CHECK_NULL_IMPL(keyword); + return IMPL_->keyword_; +} + +bool BlockArgument::is_kwarg() const { + CHECK_NULL_IMPL(is_kwarg); + return IMPL_->is_kwarg_; +} + const AttributeMap &BlockArgument::attributes() const { CHECK_NULL_IMPL(attributes_); return IMPL_->attributes_; @@ -101,6 +121,12 @@ void BlockArgument::set_attribute(const std::string &key, Attribute value) { BlockArgument BlockArgument::Create(Type type, Block *owner, uint32_t index) { return new detail::BlockArgumentImpl(type, owner, index); } + +BlockArgument BlockArgument::Create(Type type, + Block *owner, + const std::string &keyword) { + return new detail::BlockArgumentImpl(type, owner, keyword); +} /// Destroy the argument. void BlockArgument::Destroy() { if (impl_) { diff --git a/python/paddle/jit/dy2static/pir_partial_program.py b/python/paddle/jit/dy2static/pir_partial_program.py index c8068a59a0f17..2a55277fd77b4 100644 --- a/python/paddle/jit/dy2static/pir_partial_program.py +++ b/python/paddle/jit/dy2static/pir_partial_program.py @@ -134,6 +134,8 @@ def get_value_name_map(self): def _get_value_name_map_from_program(cls, program): ret = ValueDict() ret[fake_value()] = "FakeVar" + for keyword, arg in program.global_block().kwargs().items(): + ret[arg] = keyword for op in program.global_block().ops: if op.name() == "builtin.set_parameter": ret[op.operand(0).source()] = op.attrs()["parameter_name"] From 8ee6609278a91e808b7d0140bc84f327f8d2c445 Mon Sep 17 00:00:00 2001 From: diadestiny <44188454+diadestiny@users.noreply.github.com> Date: Fri, 23 Feb 2024 10:22:11 +0800 Subject: [PATCH 011/282] [SOT][3.12] Support `RETURN_CONST` opcode in Python 3.12 (#61964) --- .../jit/sot/opcode_translator/executor/opcode_executor.py | 7 +++++++ .../opcode_translator/executor/opcode_inline_executor.py | 4 ++++ test/sot/skip_files_py312 | 7 ------- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py index 5f193cebc085d..ccfae0a888f02 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py +++ b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py @@ -2114,6 +2114,13 @@ def RETURN_VALUE(self, instr: Instruction): len(self.stack) == 1 ), f"Stack must have one element, but get {len(self.stack)} elements." ret_val = self.stack.pop() + return self.compile_return(ret_val) + + def RETURN_CONST(self, instr: Instruction): + ret_const = self._co_consts[instr.arg] + return self.compile_return(ret_const) + + def compile_return(self, ret_val): compile_fn = self._graph.get_compiled_fn(ret_val) if compile_fn.graph_size() < ENV_MIN_GRAPH_SIZE.get(): self.new_code = None diff --git a/python/paddle/jit/sot/opcode_translator/executor/opcode_inline_executor.py b/python/paddle/jit/sot/opcode_translator/executor/opcode_inline_executor.py index 4baa64e884107..3832d05f04448 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/opcode_inline_executor.py +++ b/python/paddle/jit/sot/opcode_translator/executor/opcode_inline_executor.py @@ -283,6 +283,10 @@ def RETURN_VALUE(self, instr: Instruction): self.return_value = self.stack.pop() return Stop(state="Return") + def RETURN_CONST(self, instr: Instruction): + self.return_value = self._co_consts[instr.arg] + return Stop(state="Return") + def _break_graph_when_if(self, result, instr: Instruction): """ Helper method to raise a BreakGraphError when breaking the graph in a jump operation. diff --git a/test/sot/skip_files_py312 b/test/sot/skip_files_py312 index 3cc5b8d4439e0..d79956533e2d3 100644 --- a/test/sot/skip_files_py312 +++ b/test/sot/skip_files_py312 @@ -14,14 +14,7 @@ ./test_guard_user_defined_fn.py ./test_inplace_api.py ./test_min_graph_size.py -./test_output_restoration.py ./test_side_effects.py -./test_simulate_initialize.py -./test_sir_rollback.py ./test_sot_cost_model.py -./test_sot_export.py ./test_sot_resnet.py ./test_sot_resnet50_backward.py -./test_specialization.py -./test_str_format.py -./test_builtin_bool.py From e6510e8f81858e39aee6448182a0d5ef5cda47c1 Mon Sep 17 00:00:00 2001 From: Difer <707065510@qq.com> Date: Fri, 23 Feb 2024 11:00:37 +0800 Subject: [PATCH 012/282] =?UTF-8?q?=E3=80=90PIR=20Dist=20Op=20Reg=20No.13?= =?UTF-8?q?=E3=80=91=20reg=20partial=5Fsend=20(#60484)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * reg partial_send * fix cmake * fix import * fix cmake * try to remove out * add partitial_send to prim gen blacklist * fix typo --- .../pir/dialect/op_generator/ops_api_gen.py | 1 + paddle/fluid/pir/dialect/operator/ir/ops.yaml | 10 +++++ .../fluid/pir/dialect/operator/utils/utils.cc | 3 +- paddle/fluid/primitive/codegen/gen.py | 1 + paddle/phi/api/yaml/op_compat.yaml | 4 ++ paddle/phi/infermeta/unary.cc | 28 +++++++++++++ paddle/phi/infermeta/unary.h | 7 ++++ test/ir/pir/translator/CMakeLists.txt | 1 + .../test_partial_send_translator.py | 40 +++++++++++++++++++ 9 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 test/ir/pir/translator/test_partial_send_translator.py diff --git a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py index 13c656207f1b8..8328e406ae0e6 100644 --- a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py @@ -156,6 +156,7 @@ 'c_reduce_min_', 'push_sparse_v2', 'push_sparse_v2_', + 'partial_send', ] diff --git a/paddle/fluid/pir/dialect/operator/ir/ops.yaml b/paddle/fluid/pir/dialect/operator/ir/ops.yaml index d32c1d8b7a6bd..4fcd90c99fe0a 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops.yaml @@ -1654,6 +1654,16 @@ kernel: func: onednn_to_paddle_layout +- op: partial_send + args: (Tensor x, int ring_id = 0, int peer = 0, bool use_calc_stream = false, int num = 1, int id = 0) + output : + infer_meta: + func: PartialSendInferMeta + param: [x, ring_id, peer, use_calc_stream, num, id] + kernel: + func: partial_send + param: [x, ring_id, peer, use_calc_stream, num, id] + - op: sparse_momentum args: (Tensor param, Tensor grad, Tensor velocity, Tensor index, Tensor learning_rate, Tensor master_param,float mu, Scalar axis=0, bool use_nesterov=false,str regularization_method="", float regularization_coeff=0.0f, bool multi_precision=false, float rescale_grad=1.0f) output: Tensor(param_out), Tensor(velocity_out), Tensor(master_param_out) diff --git a/paddle/fluid/pir/dialect/operator/utils/utils.cc b/paddle/fluid/pir/dialect/operator/utils/utils.cc index ea8002c1c842f..b4bad427567b7 100644 --- a/paddle/fluid/pir/dialect/operator/utils/utils.cc +++ b/paddle/fluid/pir/dialect/operator/utils/utils.cc @@ -80,7 +80,8 @@ const std::unordered_set LegacyOpList = { paddle::onednn::dialect::MultiGruOp::name(), #endif CReduceMinOp::name(), - PushSparseV2Op::name()}; + PushSparseV2Op::name(), + PartialSendOp::name()}; enum class AttrType { UNDEFINED = 0, diff --git a/paddle/fluid/primitive/codegen/gen.py b/paddle/fluid/primitive/codegen/gen.py index 3c6791a344a8b..fb1579968423a 100644 --- a/paddle/fluid/primitive/codegen/gen.py +++ b/paddle/fluid/primitive/codegen/gen.py @@ -52,6 +52,7 @@ "embedding_sparse_grad", "embedding_grad", "full", + "partial_send", ] # prim op with one input and one output, with no attribute diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml index 237724dabe69f..53e0cea953b87 100755 --- a/paddle/phi/api/yaml/op_compat.yaml +++ b/paddle/phi/api/yaml/op_compat.yaml @@ -3653,6 +3653,10 @@ outputs : out : Out +- op: partial_send + inputs : + x : X + - op: read_from_array inputs: array : X diff --git a/paddle/phi/infermeta/unary.cc b/paddle/phi/infermeta/unary.cc index 5092072f5a87c..3b47085eee9b1 100644 --- a/paddle/phi/infermeta/unary.cc +++ b/paddle/phi/infermeta/unary.cc @@ -2915,6 +2915,34 @@ void Pad3dInferMeta(const MetaTensor& x, out->share_lod(x); } +void PartialSendInferMeta(const MetaTensor& x, + int ring_id, + int peer, + bool use_calc_stream, + int num, + int id) { + PADDLE_ENFORCE_GE( + peer, + 0, + phi::errors::InvalidArgument( + "The peer (%d) for partial_send op must be non-negative.", peer)); + PADDLE_ENFORCE_GE( + ring_id, + 0, + phi::errors::InvalidArgument( + "The ring_id (%d) for partial_send op must be non-negative.", + ring_id)); + PADDLE_ENFORCE_GE(num, + 1, + phi::errors::InvalidArgument( + "The num (%d) for partial_send op must >=1", num)); + PADDLE_ENFORCE_EQ( + (id >= 0 && id < num), + true, + phi::errors::InvalidArgument( + "The id (%d) for partial_send op must >=0 and Date: Fri, 23 Feb 2024 11:01:07 +0800 Subject: [PATCH 013/282] [PIR+CINN]Fix cluster node bug (#61890) * fix cluster node bug * polish cinn group cluster pass * update * polish code * polish cluster node pass --- .../transforms/cinn_group_cluster_pass.cc | 485 +++++++++++------- .../transforms/cinn_group_cluster_pass.h | 3 + paddle/cinn/hlir/framework/pir/utils.h | 10 +- test/cpp/pir/cinn/ir_op_cluster_test.cc | 2 +- 4 files changed, 305 insertions(+), 195 deletions(-) diff --git a/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_cluster_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_cluster_pass.cc index 046eb7442e8a4..b36afc9bd056f 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_cluster_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_cluster_pass.cc @@ -73,7 +73,7 @@ std::unordered_set<::pir::Value> GetListOutsideInput( std::unordered_set outside_ops; auto block_inner_output = GetInnerGeneValue(ops); - for (auto& op : ops) { + for (const auto& op : ops) { for (size_t i = 0; i < op->num_operands(); ++i) { if (!block_inner_output.count(op->operand_source(i)) && !outside_ops.count(op->operand_source(i))) { @@ -95,17 +95,33 @@ bool IsLastReshape(::pir::Operation* input_op) { return false; } +std::string BuildGroupId(const ::pir::GroupOpsVec& ops_list) { + std::string group_id; + for (const auto& op : ops_list) { + if (group_id != "") { + group_id += "_"; + } + group_id += op->name(); + } + + return group_id; +} struct GroupClusterNode { + // all the ops in each Node std::vector<::pir::Operation*> ops; + // group kind cinn::hlir::framework::OpPatternKind group_kind{ cinn::hlir::framework::kElementWise}; + // reduce_axis if kind is Reduce else empty std::vector reduce_axis; + // if kind is reduce, loop ranges equal input dim + // if kind id elementwise or broadcast, loop ranges equal output dim std::vector loop_ranges; std::unordered_map<::pir::Operation*, std::vector> alignment_schedule_info; - std::unordered_set<::pir::Value> GetOutsideInput() { + std::unordered_set<::pir::Value> GetOutsideInput() const { return GetListOutsideInput(ops); } @@ -126,7 +142,7 @@ struct GroupClusterNode { } ss << "\n"; - for (auto op : ops) { + for (const auto& op : ops) { printer.PrintOperation(op); if (alignment_schedule_info.count(op)) { for (auto& node : alignment_schedule_info.at(op)) { @@ -139,100 +155,134 @@ struct GroupClusterNode { return ss.str(); } - void GenerateOutputValue( - const std::unordered_set<::pir::Value>& outside_need_value) { - output_value.clear(); - for (auto& op : ops) { - if (op->name() == "cf.yield") { - continue; - } - - std::unordered_set<::pir::Value> inserted_val; - for (size_t i = 0; i < op->num_results(); ++i) { - if (outside_need_value.count(op->result(i))) { - if (!inserted_val.count(op->result(i))) { - output_value.push_back(op->result(i)); - - inserted_val.insert(op->result(i)); - } - } - } - } - } - void MergeNode(const GroupClusterNode& node, - const ScheduleInfoNode& sch_node) { + const ScheduleInfoNode& inner_sch_node) { std::unordered_set<::pir::Operation*> inner_ops(ops.begin(), ops.end()); - if (sch_node.type != "") { - // all the data need add sch node - for (auto op : ops) { - alignment_schedule_info[op].push_back(sch_node); + if (inner_sch_node.type != hlir::framework::pir::ScheduleAlignType::kNone) { + for (const auto& op : ops) { + this->alignment_schedule_info[op].push_back(inner_sch_node); } } - for (auto op : node.ops) { + for (const auto& op : node.ops) { if (!inner_ops.count(op)) { - ops.push_back(op); + this->ops.push_back(op); // copy align info if (node.alignment_schedule_info.count(op)) { - alignment_schedule_info[op] = node.alignment_schedule_info.at(op); + this->alignment_schedule_info[op] = + node.alignment_schedule_info.at(op); } - - // if( sch_node.type != "" ) - // { - // alignment_schedule_info[op].push_back( sch_node); - // } } } - if (group_kind < node.group_kind) { - group_kind = node.group_kind; + if (this->group_kind < node.group_kind) { + this->group_kind = node.group_kind; } if ((node.group_kind == cinn::hlir::framework::kReduction) || (node.group_kind == cinn::hlir::framework::kBroadcast)) { - loop_ranges = node.loop_ranges; + this->loop_ranges = node.loop_ranges; } if (node.group_kind == cinn::hlir::framework::kReduction) { - reduce_axis = node.reduce_axis; + this->reduce_axis = node.reduce_axis; } if ((ops.size() == 1) && (ops.front()->name() == "cinn_op.reshape")) { - loop_ranges = node.loop_ranges; + this->loop_ranges = node.loop_ranges; } } - std::vector<::pir::Value> output_value; + void MergePreNode(const GroupClusterNode& node, + const ScheduleInfoNode& pre_sch_node) { + std::unordered_set<::pir::Operation*> inner_ops(ops.begin(), ops.end()); + + for (const auto& op : node.ops) { + if (!inner_ops.count(op)) { + this->ops.push_back(op); + // copy align info + if (node.alignment_schedule_info.count(op)) { + this->alignment_schedule_info[op] = + node.alignment_schedule_info.at(op); + } + + if (pre_sch_node.type != + hlir::framework::pir::ScheduleAlignType::kNone) { + this->alignment_schedule_info[op].push_back(pre_sch_node); + } + } + } + + if (group_kind < node.group_kind) { + this->group_kind = node.group_kind; + } + } }; -::pir::Operation* ReplaceWithGroupOp(pir::PatternRewriter* rewriter, - const ::pir::GroupOpsVec& group_ops, - const GroupClusterNode& node, - ::pir::IrMapping* ir_mapping) { - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - ctx->GetOrRegisterDialect(); - ctx->GetOrRegisterDialect<::pir::ControlFlowDialect>(); +std::vector<::pir::Value> GenerateOutputValue( + const std::vector<::pir::Operation*>& ops, + const std::unordered_map<::pir::Value, size_t>& outside_need_value) { + std::vector<::pir::Value> temp_out; + for (const auto& op : ops) { + if (op->isa()) { + continue; + } + + std::unordered_set<::pir::Value> inserted_val; + for (size_t i = 0; i < op->num_results(); ++i) { + if (outside_need_value.count(op->result(i))) { + if (!inserted_val.count(op->result(i))) { + temp_out.push_back(op->result(i)); + + inserted_val.insert(op->result(i)); + } + } + } + } + std::sort(temp_out.begin(), + temp_out.end(), + [&outside_need_value](::pir::Value a, ::pir::Value b) { + return outside_need_value.at(a) < outside_need_value.at(b); + }); + + return temp_out; +} + +cinn::dialect::GroupInfo BuildGroupInfo( + const ::pir::GroupOpsVec& vec_new_op_list, + const GroupClusterNode& node, + const std::unordered_map<::pir::Operation*, std::vector>& + new_align_info) { + cinn::dialect::GroupInfo group_info({}); + group_info.group_id = BuildGroupId(vec_new_op_list); + group_info.loop_ranges = node.loop_ranges; + group_info.reduce_axis = node.reduce_axis; + group_info.op_pattern_kind = node.group_kind; + group_info.alignment_schedule_info = new_align_info; - // step 1: Ensure the insert point and create GroupOp here. - auto* last_op = group_ops.back(); + return group_info; +} - auto output_value = node.output_value; - auto alignment_schedule_info = node.alignment_schedule_info; +std::vector BuildOutType( + const std::vector<::pir::Value>& output_value) { std::vector output_types; - // std::vector outputs = ::pir::AnalysisOutputs(group_ops); - // ::pir::IrMapping ir_mapping; - for (auto& value : output_value) { + for (const auto& value : output_value) { output_types.emplace_back(value.type()); } - ::pir::CloneOptions clone_options(false, true, false); + return output_types; +} +::pir::GroupOpsVec CloneOps( + const ::pir::GroupOpsVec& group_ops, + const GroupClusterNode& node, + ::pir::IrMapping* ir_mapping, + std::unordered_map<::pir::Operation*, std::vector>* + align_info) { std::vector<::pir::Operation*> vec_new_op_list; - std::unordered_map<::pir::Operation*, std::vector> - new_align_info; + ::pir::CloneOptions clone_options(false, true, false); - std::string group_id; + auto& alignment_schedule_info = node.alignment_schedule_info; for (auto op : group_ops) { auto new_op = op->Clone(*ir_mapping, clone_options); auto& shape_analysis = @@ -244,24 +294,34 @@ ::pir::Operation* ReplaceWithGroupOp(pir::PatternRewriter* rewriter, } vec_new_op_list.push_back(new_op); - if (group_id != "") { - group_id += "_"; - } - group_id += new_op->name(); if (alignment_schedule_info.count(op)) { - new_align_info[new_op] = alignment_schedule_info.at(op); + align_info->emplace(new_op, alignment_schedule_info.at(op)); } } - cinn::dialect::GroupInfo group_info({}); - group_info.group_id = group_id; - group_info.loop_ranges = node.loop_ranges; - group_info.reduce_axis = node.reduce_axis; - group_info.op_pattern_kind = node.group_kind; - group_info.alignment_schedule_info = new_align_info; + return vec_new_op_list; +} + +::pir::Operation* ReplaceWithGroupOp( + pir::PatternRewriter* rewriter, + const ::pir::GroupOpsVec& group_ops, + const GroupClusterNode& node, + const std::vector<::pir::Value> output_value, + ::pir::IrMapping* ir_mapping) { + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + ctx->GetOrRegisterDialect(); + ctx->GetOrRegisterDialect<::pir::ControlFlowDialect>(); + std::unordered_map<::pir::Operation*, std::vector> + new_align_info; + + auto vec_new_op_list = CloneOps(group_ops, node, ir_mapping, &new_align_info); + + auto group_info = BuildGroupInfo(vec_new_op_list, node, new_align_info); // step 2: Replace the old op with GroupOp. + + auto output_types = BuildOutType(output_value); auto new_fusion_op = rewriter->Build(output_types, group_info); pir::Block* fusion_block = new_fusion_op.block(); @@ -337,7 +397,7 @@ bool CanFuse(const GroupClusterNode& first, } if (first.loop_ranges != second.loop_ranges) { - sch_node->type = "broadcast"; + sch_node->type = hlir::framework::pir::ScheduleAlignType::kBroadcast; sch_node->axis_info = first.reduce_axis; sch_node->factor_info = first.loop_ranges; } @@ -350,17 +410,22 @@ bool CanFuse(const GroupClusterNode& first, std::vector SortNodeList(std::vector* node_list_ptr, std::vector>* pre_ids_ptr) { + // sort node list by topological sort + // TODO(phlrain): One node may have two pre node, need update here auto& node_list = *node_list_ptr; auto& pre_ids = *pre_ids_ptr; - std::unordered_set<::pir::Value> all_ouput_values; - for (auto& node : node_list) { + std::unordered_map<::pir::Value, size_t> in_out_values; + for (const auto& node : node_list) { auto node_outside_input = node.GetOutsideInput(); - all_ouput_values.insert(node_outside_input.begin(), - node_outside_input.end()); + for (const auto& val : node_outside_input) { + size_t id = in_out_values.size(); + in_out_values.emplace(val, id); + } } - for (auto& node : node_list) { - node.GenerateOutputValue(all_ouput_values); + std::vector> output_values_list; + for (const auto& node : node_list) { + output_values_list.push_back(GenerateOutputValue(node.ops, in_out_values)); } std::vector> next_ids; @@ -371,7 +436,7 @@ std::vector SortNodeList(std::vector* node_list_ptr, continue; } - auto pre_out_list = node_list[i].output_value; + const auto& pre_out_list = output_values_list[i]; auto next_in_set = node_list[j].GetOutsideInput(); for (auto val : pre_out_list) { @@ -462,7 +527,7 @@ void GetClusterNodeBasicInfo(::pir::Operation* op, .dyn_cast() .dims()); - sch_node->type = "broadcast"; + sch_node->type = hlir::framework::pir::ScheduleAlignType::kBroadcast; sch_node->axis_info = cinn::dialect::ir::GetVectorAttr(op, "broadcast_axes"); sch_node->factor_info = cinn::dialect::ir::GetVectorAttr(op, "out_shape"); @@ -489,6 +554,8 @@ bool CanOpMergeNode( const std::unordered_map<::pir::Operation*, GroupClusterNode>& op_path_info, ::pir::Operation* pre_op, ::pir::Operation* cur_op) { + const auto& node1 = op_path_info.at(pre_op); + const auto& node2 = op_path_info.at(cur_op); // reduce can not fuse with any op in first stage if (cinn::hlir::framework::pir::CompatibleInfo::OpKind(*pre_op) == cinn::hlir::framework::kReduction) { @@ -528,9 +595,71 @@ bool ShouldOutputPreNode( return false; } -std::vector GroupSplit(cinn::dialect::GroupOp group_op) { - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +std::vector NodeMergeWithNode( + const std::vector& first_stage_output) { + // stage 2 merge + // for now we merge node in same pass + // only for vertial fuse + std::vector second_stage_output = first_stage_output; + while (true) { + bool fused = false; + std::vector temp_out; + + std::set fused_index; + + std::vector> pre_ids_info; + auto sort_list = SortNodeList(&second_stage_output, &pre_ids_info); + + std::reverse(sort_list.begin(), sort_list.end()); + for (auto node_index : sort_list) { + if (fused_index.count(node_index)) { + continue; + } + const auto& node = second_stage_output[node_index]; + const auto& pre_ids = pre_ids_info[node_index]; + + GroupClusterNode new_node = node; + + for (auto pre_id : pre_ids) { + // get pre id + + if (fused_index.count(pre_id)) { + continue; + } + + // can new_node merge with pre_id node + const auto& pre_node = second_stage_output[pre_id]; + + ScheduleInfoNode sch_node; + auto can_fuse = CanFuse(pre_node, new_node, &sch_node); + if (can_fuse) { + // merge pre node to new_node + new_node.MergeNode(pre_node, sch_node); + + fused_index.insert(pre_id); + fused = true; + } else { + temp_out.insert(temp_out.begin(), pre_node); + } + } + temp_out.insert(temp_out.end(), new_node); + } + + if (temp_out.size() >= second_stage_output.size()) { + break; + } + second_stage_output.swap(temp_out); + if (fused == false) { + break; + } + } + + return second_stage_output; +} + +std::vector OpMergeWithOp(cinn::dialect::GroupOp group_op) { + // op merge with op auto inner_values = GetInnerGeneValue(group_op.GetOperators()); std::unordered_map<::pir::Operation*, GroupClusterNode> op_path; @@ -540,6 +669,7 @@ std::vector GroupSplit(cinn::dialect::GroupOp group_op) { std::vector first_stage_output; std::unordered_set<::pir::Operation*> yield_output_ops; + std::unordered_set<::pir::Operation*> first_output_ops; auto yield_op = op_list.back(); for (size_t i = 0; i < yield_op->num_operands(); ++i) { if (yield_op->operand_source(i).defining_op()->result(0).use_count() == 1) { @@ -547,6 +677,7 @@ std::vector GroupSplit(cinn::dialect::GroupOp group_op) { } } + // first stage op fuse op for (auto* op : op_list) { if (op->isa<::pir::YieldOp>()) { continue; @@ -567,13 +698,16 @@ std::vector GroupSplit(cinn::dialect::GroupOp group_op) { } if (CanOpMergeNode(op_path, pre_op, op)) { - cluster_node.MergeNode(op_path.at(pre_op), sch_node); + cluster_node.MergePreNode(op_path.at(pre_op), sch_node); } // TODO(phlrain): should remove this strategy if (ShouldOutputPreNode(op_path, pre_op, op)) { // Can not merge here, should output pre_op cluster Node - first_stage_output.push_back(op_path[pre_op]); + if (!first_output_ops.count(pre_op)) { + first_stage_output.push_back(op_path[pre_op]); + first_output_ops.insert(pre_op); + } continue; } } @@ -583,70 +717,28 @@ std::vector GroupSplit(cinn::dialect::GroupOp group_op) { if (yield_output_ops.count(op) || cinn::hlir::framework::pir::CompatibleInfo::OpKind(*op) == cinn::hlir::framework::kReduction) { - first_stage_output.push_back(op_path[op]); + // TODO(phlrain): yiled output no nedd to push into first stage output, + // Update here + if (!first_output_ops.count(op)) { + first_stage_output.push_back(op_path[op]); + first_output_ops.insert(op); + } } } + return first_stage_output; +} + +std::vector GroupSplit(cinn::dialect::GroupOp group_op) { + // stage 1 + auto first_stage_output = OpMergeWithOp(group_op); + if (first_stage_output.size() <= 1) { return first_stage_output; } - // stage 2 merge - // for now we merge node in same pass - // only for vertial fuse - std::vector second_stage_output = first_stage_output; - while (true) { - bool fused = false; - std::vector temp_out; - - std::set fused_index; - - std::vector> pre_ids_info; - auto sort_list = SortNodeList(&second_stage_output, &pre_ids_info); - - std::reverse(sort_list.begin(), sort_list.end()); - for (auto node_index : sort_list) { - if (fused_index.count(node_index)) { - continue; - } - auto& node = second_stage_output[node_index]; - auto& pre_ids = pre_ids_info[node_index]; - - GroupClusterNode new_node = node; - - for (auto pre_id : pre_ids) { - // get pre id - - if (fused_index.count(pre_id)) { - continue; - } - - // can new_node merge with pre_id node - auto& pre_node = second_stage_output[pre_id]; - - ScheduleInfoNode sch_node; - auto can_fuse = CanFuse(pre_node, new_node, &sch_node); - - if (can_fuse) { - // merge pre node to new_node - new_node.MergeNode(pre_node, sch_node); - - fused_index.insert(pre_id); - fused = true; - } else { - temp_out.insert(temp_out.begin(), pre_node); - } - } - temp_out.insert(temp_out.end(), new_node); - } - if (temp_out.size() >= second_stage_output.size()) { - break; - } - second_stage_output.swap(temp_out); - if (fused == false) { - break; - } - } + // stage 2 + auto second_stage_output = NodeMergeWithNode(first_stage_output); if (second_stage_output.size() == 1) { return second_stage_output; @@ -663,6 +755,49 @@ std::vector GroupSplit(cinn::dialect::GroupOp group_op) { return sorted_out; } +std::vector<::pir::Operation*> SortByOriginalOrderAndUniq( + cinn::dialect::GroupOp group_op, + const std::vector<::pir::Operation*>& ops) { + size_t index = 0; + std::unordered_map op2order_value; + + for (auto op : group_op.GetOperators()) { + op2order_value[op] = index++; + } + + std::vector tmp_ops(ops); + std::sort(tmp_ops.begin(), + tmp_ops.end(), + [&op2order_value](pir::Operation* a, pir::Operation* b) { + return op2order_value.at(a) < op2order_value.at(b); + }); + + std::unique(tmp_ops.begin(), tmp_ops.end()); + + return tmp_ops; +} + +std::unordered_map<::pir::Value, size_t> BuildValueOrderByYieldOp( + const std::vector& node_list, + cinn::dialect::GroupOp group_op) { + std::unordered_map<::pir::Value, size_t> all_output_values; + auto yield_op = group_op.GetOperators().back(); + for (size_t i = 0; i < yield_op->num_operands(); ++i) { + size_t id = all_output_values.size(); + all_output_values.emplace(yield_op->operand_source(i), id); + } + + for (size_t i = 0; i < node_list.size(); ++i) { + auto node_outside_input = node_list[i].GetOutsideInput(); + for (const auto& val : node_outside_input) { + size_t id = all_output_values.size(); + all_output_values.emplace(val, id); + } + } + + return all_output_values; +} + } // namespace class CinnGroupClusterPattern @@ -675,68 +810,34 @@ class CinnGroupClusterPattern ::pir::IrMapping ir_mapping; auto group_outside_input = GetListOutsideInput(group_op.GetOperators()); + // insert initial input to ir mapping for (auto val : group_outside_input) { ir_mapping.Add(val, val); } auto split_res = GroupSplit(group_op); - // need sort split res - - std::unordered_set<::pir::Value> all_ouput_values; - for (auto& node : split_res) { - auto node_outside_input = node.GetOutsideInput(); - all_ouput_values.insert(node_outside_input.begin(), - node_outside_input.end()); - } - - size_t index = 0; - std::unordered_map op2id; - - for (auto op1 : group_op.GetOperators()) { - op2id[op1] = index++; - } - auto yield_op = group_op.GetOperators().back(); - for (size_t i = 0; i < yield_op->num_operands(); ++i) { - all_ouput_values.insert(yield_op->operand_source(i)); - } + auto all_output_values = BuildValueOrderByYieldOp(split_res, group_op); for (auto& node : split_res) { - node.GenerateOutputValue(all_ouput_values); - std::vector tmp_ops(node.ops.begin(), node.ops.end()); - std::sort(tmp_ops.begin(), - tmp_ops.end(), - [&op2id](pir::Operation* a, pir::Operation* b) { - return op2id.at(a) < op2id.at(b); - }); - - std::unique(tmp_ops.begin(), tmp_ops.end()); - - auto node_outside_input = node.GetOutsideInput(); - - auto insert_point = - ReplaceWithGroupOp(&rewriter, tmp_ops, node, &ir_mapping); - - for (size_t i = 0; i < node.output_value.size(); ++i) { - ir_mapping.Add(node.output_value[i], insert_point->result(i)); - } - - std::unordered_set<::pir::Value> local_outs(node.output_value.begin(), - node.output_value.end()); + auto output_values = GenerateOutputValue(node.ops, all_output_values); + auto uniq_ops = SortByOriginalOrderAndUniq(group_op, node.ops); - int local_index = 0; + auto new_group_op = ReplaceWithGroupOp( + &rewriter, uniq_ops, node, output_values, &ir_mapping); - std::unordered_map<::pir::Value, size_t> value_order; - for (size_t i = 0; i < yield_op->num_operands(); ++i) { - value_order[yield_op->operand_source(i)] = i; + // update ir mapping + for (size_t i = 0; i < output_values.size(); ++i) { + ir_mapping.Add(output_values[i], new_group_op->result(i)); } - for (size_t i = 0; i < node.output_value.size(); ++i) { - if (value_order.count(node.output_value[i])) { - // replace - rewriter.ReplaceAllUsesWith( - group_op.result(value_order.at(node.output_value[i])), - insert_point->result(i)); + for (size_t i = 0; i < output_values.size(); ++i) { + auto find_it = all_output_values.find(output_values[i]); + if ((find_it != all_output_values.end()) && + (find_it->second < group_op->num_results())) { + // id < num_results means yiled input + rewriter.ReplaceAllUsesWith(group_op.result(find_it->second), + new_group_op->result(i)); } } } diff --git a/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_cluster_pass.h b/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_cluster_pass.h index b4e6cf7d511cd..2350244fdfe38 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_cluster_pass.h +++ b/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_cluster_pass.h @@ -20,6 +20,9 @@ namespace cinn { namespace dialect { namespace ir { +// Split One GroupOp to multi small GroupOp +// Each small GroupOp can Generate one kernel by CINN backend + IR_API std::unique_ptr CreateCinnGroupClusterPass(); } // namespace ir diff --git a/paddle/cinn/hlir/framework/pir/utils.h b/paddle/cinn/hlir/framework/pir/utils.h index 56cbf2c539648..225f16f5caad2 100644 --- a/paddle/cinn/hlir/framework/pir/utils.h +++ b/paddle/cinn/hlir/framework/pir/utils.h @@ -113,8 +113,14 @@ class PrettyNamer { ::cinn::common::NameGenerator name_generator_; }; +enum class ScheduleAlignType : int { + kNone = 0, //! No need to align + kBroadcast = 1, //! Using Broadcast schedule to align +}; + struct ScheduleInfoNode { - std::string type; + // TOOD(phlrain): update align type by new loop alignment + ScheduleAlignType type{ScheduleAlignType::kNone}; std::vector axis_info; std::vector factor_info; @@ -122,7 +128,7 @@ struct ScheduleInfoNode { std::string DebugStr() { std::stringstream ss; - ss << "type " << type << "| axis info "; + ss << "type " << static_cast(type) << "| axis info "; for (auto d : axis_info) { ss << " " << d; } diff --git a/test/cpp/pir/cinn/ir_op_cluster_test.cc b/test/cpp/pir/cinn/ir_op_cluster_test.cc index dbca4fae66ebd..5fac91e0c2f48 100644 --- a/test/cpp/pir/cinn/ir_op_cluster_test.cc +++ b/test/cpp/pir/cinn/ir_op_cluster_test.cc @@ -623,5 +623,5 @@ TEST(IROpFusionPass, layer_norm2) { CHECK_EQ(pm.Run(&program), true); // TODO(phlrain): need update same as 4u - ASSERT_EQ(program.block()->size(), 11u); + ASSERT_EQ(program.block()->size(), 10u); } From 84fa05349c09700bb66b7686e4ffb63bbda6dfdc Mon Sep 17 00:00:00 2001 From: Jia Wenxuan <64853160+JiaWenxuan@users.noreply.github.com> Date: Fri, 23 Feb 2024 11:05:09 +0800 Subject: [PATCH 014/282] Substitute dim expr (#61888) * add SubstituteDimExprBasedOnConstraintPass * fixed the unittest file * fix some bugs * fix some bugs * fix some bugs * fix some bugs --- paddle/cinn/common/union_find.h | 40 ++++ ...tute_dim_expr_based_on_constraints_pass.cc | 183 ++++++++++++++++++ ...itute_dim_expr_based_on_constraints_pass.h | 28 +++ paddle/fluid/pybind/pir.cc | 3 + ...substitute_dim_expr_based_on_constraint.py | 85 ++++++++ 5 files changed, 339 insertions(+) create mode 100644 paddle/cinn/hlir/dialect/operator/transforms/group_merge/substitute_dim_expr_based_on_constraints_pass.cc create mode 100644 paddle/cinn/hlir/dialect/operator/transforms/group_merge/substitute_dim_expr_based_on_constraints_pass.h create mode 100644 test/ir/pir/cinn/symbolic/test_substitute_dim_expr_based_on_constraint.py diff --git a/paddle/cinn/common/union_find.h b/paddle/cinn/common/union_find.h index 18a2ee2bf69ae..a88f52dafe515 100644 --- a/paddle/cinn/common/union_find.h +++ b/paddle/cinn/common/union_find.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include "paddle/cinn/common/object.h" @@ -97,5 +98,44 @@ struct UnionFind { std::vector> nodes; }; +template +class UnionFindSet { + public: + T Find(const T& x) { + if (parent_.find(x) == parent_.end()) { + return x; + } + if (parent_[x] != x) { + parent_[x] = Find(parent_[x]); + } + return parent_[x]; + } + + void Union(const T& p, const T& q) { + if (parent_.find(p) == parent_.end()) { + parent_[p] = p; + } + if (parent_.find(q) == parent_.end()) { + parent_[q] = q; + } + parent_[Find(q)] = Find(p); + } + + std::vector> Clusters() const { + std::unordered_map> clusters_map; + for (auto it = parent_.begin(); it != parent_.end(); it++) { + clusters_map[it->second].emplace_back(it->first); + } + std::vector> clusters; + for (auto it = clusters_map.begin(); it != clusters_map.end(); it++) { + clusters.emplace_back(it->second); + } + return clusters; + } + + private: + std::unordered_map parent_; +}; + } // namespace common } // namespace cinn diff --git a/paddle/cinn/hlir/dialect/operator/transforms/group_merge/substitute_dim_expr_based_on_constraints_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/group_merge/substitute_dim_expr_based_on_constraints_pass.cc new file mode 100644 index 0000000000000..68372afa3e9ca --- /dev/null +++ b/paddle/cinn/hlir/dialect/operator/transforms/group_merge/substitute_dim_expr_based_on_constraints_pass.cc @@ -0,0 +1,183 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/cinn/hlir/dialect/operator/transforms/group_merge/substitute_dim_expr_based_on_constraints_pass.h" + +#include "paddle/cinn/common/dim_expr_util.h" +#include "paddle/cinn/common/union_find.h" + +namespace cinn { +namespace dialect { +namespace ir { + +namespace { + +template +void VisitEachOp(pir::ModuleOp module_op, const DoEachT& DoEach) { + for (uint32_t i = 0; i < module_op->num_regions(); i++) { + for (pir::Block& block : module_op->region(i)) { + for (pir::Operation& op : block) { + DoEach(op); + } + } + } +} + +template +void VisitEachValue(const pir::Operation& op, const DoEachT& DoEach) { + for (std::size_t i = 0; i < op.num_operands(); ++i) { + DoEach(op.operand_source(i)); + } + for (std::size_t i = 0; i < op.num_results(); ++i) { + DoEach(op.result(i)); + } +} + +symbol::TensorShapeOrDataDimExprs SubstituteTensorShapeOrData( + const symbol::TensorShapeOrDataDimExprs& shape_or_data, + const std::unordered_map& + substitution_pattern) { + auto SubstituteOneDimExpr = + [](const std::vector& original_dim_expr, + const std::unordered_map& + substitution_pattern) -> std::vector { + std::vector substituted_dim_expr{}; + for (const symbol::DimExpr& dim_expr : original_dim_expr) { + substituted_dim_expr.push_back( + cinn::common::SubstituteDimExpr(dim_expr, substitution_pattern)); + } + return substituted_dim_expr; + }; + + std::vector substituted_shape = + SubstituteOneDimExpr(shape_or_data.shape(), substitution_pattern); + if (!shape_or_data.data().has_value()) { + return symbol::ShapeOrData(substituted_shape); + } else { + std::vector substituted_data = SubstituteOneDimExpr( + shape_or_data.data().value(), substitution_pattern); + return symbol::ShapeOrData(substituted_shape, + substituted_data); + } +} + +symbol::ShapeOrDataDimExprs SubstituteShapeOrData( + const symbol::ShapeOrDataDimExprs& shape_or_data, + const std::unordered_map& + substitution_pattern) { + auto lambdas = symbol::Overloaded{ + [&](const symbol::TensorShapeOrDataDimExprs& tensor_shape_or_data) { + return symbol::ShapeOrDataDimExprs(SubstituteTensorShapeOrData( + tensor_shape_or_data, substitution_pattern)); + }, + [&](const symbol::TensorListShapeOrDataDimExprs& tensor_list) { + symbol::TensorListShapeOrDataDimExprs substituted_tensor_list; + for (symbol::TensorShapeOrDataDimExprs tensor_shape_or_data : + tensor_list) { + substituted_tensor_list.push_back(SubstituteTensorShapeOrData( + tensor_shape_or_data, substitution_pattern)); + } + return symbol::ShapeOrDataDimExprs(substituted_tensor_list); + }}; + return std::visit(lambdas, shape_or_data.variant()); +} + +std::unordered_map GetDimExprSubstitution( + pir::ShapeConstraintIRAnalysis* shape_analysis) { + const std::vector& dim_expr_constraints = + shape_analysis->CreateDimExprBuilder().constraints(); + const cinn::common::UnionFindSet& union_find_set = [&]() { + cinn::common::UnionFindSet union_find_set; + for (const auto& constraint : dim_expr_constraints) { + CHECK(std::holds_alternative>(constraint)) + << "The DimExprConstraint type is no Equal, this part is to " + "be completed."; + const auto& data = + std::get>(constraint).data; + union_find_set.Union(data->lhs, data->rhs); + } + return union_find_set; + }(); + + const std::vector>& dim_expr_clusters = + union_find_set.Clusters(); + std::unordered_map substitution_pattern; + for (const auto& dim_expr_cluster : dim_expr_clusters) { + CHECK(!dim_expr_cluster.empty()); + auto dim_expr_root = dim_expr_cluster[0]; + for (const auto& dim_expr : dim_expr_cluster) { + if (std::holds_alternative(dim_expr)) { + dim_expr_root = dim_expr; + break; + } + } + for (const auto& dim_expr : dim_expr_cluster) { + if (dim_expr != dim_expr_root) { + substitution_pattern[dim_expr] = dim_expr_root; + } + } + } + return substitution_pattern; +} + +void SubstituteDimExprBasedOnConstraints(pir::ModuleOp module_op) { + VLOG(4) << "SubstituteDimExprBasedOnConstraints start"; + pir::ShapeConstraintIRAnalysis shape_analysis = + pir::ShapeAnalysisManager::Instance().Get(module_op.program()); + const std::unordered_map& + substitution_pattern = GetDimExprSubstitution(&shape_analysis); + VisitEachOp(module_op, [&](pir::Operation& op) { + VisitEachValue(op, [&](pir::Value value) { + if (!shape_analysis.HasShapeOrDataForValue(value)) { + VLOG(4) << "Can not find ShapeOrData for value of op(" << op.name() + << ") in shape_analysis"; + } else { + const symbol::ShapeOrDataDimExprs& origin_shape_or_data = + shape_analysis.GetShapeOrDataForValue(value); + const symbol::ShapeOrDataDimExprs& substituted_shape_or_data = + SubstituteShapeOrData(origin_shape_or_data, substitution_pattern); + shape_analysis.SetShapeOrDataForValue(value, substituted_shape_or_data); + } + }); + // TODO(JiaWenxuan): substitute the attribute "sym_shape_str" of the op + }); + VLOG(4) << "SubstituteDimExprBasedOnConstraints end"; +} + +class SubstituteDimExprBasedOnConstraintsPass : public pir::Pass { + public: + SubstituteDimExprBasedOnConstraintsPass() + : pir::Pass("substitute_dim_expr_based_on_constraints_pass", 1) {} + + void Run(pir::Operation* op) override { + pir::ModuleOp module_op = op->dyn_cast(); + SubstituteDimExprBasedOnConstraints(module_op); + } + + bool CanApplyOn(pir::Operation* op) const override { + return op->isa() && op->num_regions() > 0; + } +}; + +} // namespace + +std::unique_ptr<::pir::Pass> CreateSubstituteDimExprBasedOnConstraintsPass() { + return std::make_unique(); +} + +} // namespace ir +} // namespace dialect +} // namespace cinn diff --git a/paddle/cinn/hlir/dialect/operator/transforms/group_merge/substitute_dim_expr_based_on_constraints_pass.h b/paddle/cinn/hlir/dialect/operator/transforms/group_merge/substitute_dim_expr_based_on_constraints_pass.h new file mode 100644 index 0000000000000..30c0dd7b6a7b6 --- /dev/null +++ b/paddle/cinn/hlir/dialect/operator/transforms/group_merge/substitute_dim_expr_based_on_constraints_pass.h @@ -0,0 +1,28 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/pir/include/pass/pass.h" + +namespace cinn { +namespace dialect { +namespace ir { + +// This is a helper pass for substituting DimExpr based on the +// constraints symbol::Equal. +std::unique_ptr<::pir::Pass> CreateSubstituteDimExprBasedOnConstraintsPass(); +} // namespace ir +} // namespace dialect +} // namespace cinn diff --git a/paddle/fluid/pybind/pir.cc b/paddle/fluid/pybind/pir.cc index 99e1b624edefa..3a0de137173a7 100644 --- a/paddle/fluid/pybind/pir.cc +++ b/paddle/fluid/pybind/pir.cc @@ -95,6 +95,7 @@ #include "paddle/cinn/hlir/dialect/operator/transforms/group_merge/divide_group_op_to_fusion_op_pass.h" #include "paddle/cinn/hlir/dialect/operator/transforms/group_merge/move_generate_shape_ops_to_prologue_pass.h" #include "paddle/cinn/hlir/dialect/operator/transforms/group_merge/simplify_dim_expr_pass.h" +#include "paddle/cinn/hlir/dialect/operator/transforms/group_merge/substitute_dim_expr_based_on_constraints_pass.h" #include "paddle/cinn/hlir/dialect/operator/transforms/insert_broadcast_pass.h" #include "paddle/cinn/hlir/dialect/operator/transforms/lower_cinn_fusion_op_pass.h" #include "paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.h" @@ -1582,6 +1583,8 @@ void AddCinnPass(std::shared_ptr &pass_manager, // NOLINT if (has_dynamic_shape) { pass_manager->AddPass(pir::CreateShapeOptimizationPass()); pass_manager->AddPass(cinn::dialect::ir::CreateSimplifyDimExprPass()); + pass_manager->AddPass( + cinn::dialect::ir::CreateSubstituteDimExprBasedOnConstraintsPass()); pass_manager->AddPass(cinn::dialect::ir::CreateInsertBroadcastPass()); pass_manager->AddPass(pir::CreateShapeOptimizationPass()); pass_manager->AddPass( diff --git a/test/ir/pir/cinn/symbolic/test_substitute_dim_expr_based_on_constraint.py b/test/ir/pir/cinn/symbolic/test_substitute_dim_expr_based_on_constraint.py new file mode 100644 index 0000000000000..a9119455e94fd --- /dev/null +++ b/test/ir/pir/cinn/symbolic/test_substitute_dim_expr_based_on_constraint.py @@ -0,0 +1,85 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import unittest +from os.path import dirname + +import numpy as np + +import paddle +from paddle.static import InputSpec + +sys.path.append(dirname(dirname(__file__))) +import utils + + +class TestSubstituteDimExprNet(paddle.nn.Layer): + def __init__(self): + super().__init__() + + def forward(self, x, y1, y2): + z1 = paddle.concat([y1, x], 0) + z2 = paddle.concat([y1, y2], 0) + out = z1 + z2 + return out + + +class TestSubstituteDimExprBasedOnConstraint(unittest.TestCase): + """ + Test Pir API + @to_static + CINN. + """ + + def setUp(self): + paddle.seed(2022) + self.prepare_data() + + def prepare_data(self): + self.shapex = [32, 128] + self.x = paddle.randn(self.shapex, dtype="float32") + self.x.stop_gradient = False + self.shapey = [32, 128] + self.y1 = paddle.randn(self.shapey, dtype="float32") + self.y1.stop_gradient = False + self.y2 = paddle.randn(self.shapey, dtype="float32") + self.y2.stop_gradient = False + + def check_jit_kernel_info(self, static_fn): + utils.check_jit_kernel_number(static_fn, 1) + utils.check_jit_kernel_structure(static_fn, {utils.JIT_KERNEL_NAME: 1}) + + def eval(self, use_cinn): + net = TestSubstituteDimExprNet() + input_spec = [ + InputSpec(shape=[32, 128], dtype="float32"), + InputSpec(shape=[32, None], dtype="float32"), + InputSpec(shape=[32, None], dtype="float32"), + ] + net = utils.apply_to_static(net, use_cinn, input_spec) + net.eval() + out = net(self.x, self.y1, self.y2) + if use_cinn: + self.check_jit_kernel_info(net.forward) + return out + + def test_eval(self): + dy_out = self.eval(use_cinn=False) + cinn_out = self.eval(use_cinn=True) + np.testing.assert_allclose( + cinn_out.numpy(), dy_out.numpy(), atol=1e-6, rtol=1e-6 + ) + + +if __name__ == '__main__': + unittest.main() From 7b756babae75c51f6d3b479b53c1f0e4169e11c6 Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 23 Feb 2024 12:03:49 +0800 Subject: [PATCH 015/282] Update operator.cc (#61999) --- paddle/fluid/framework/operator.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc index 22c9532880e48..c9d7af6a44cea 100644 --- a/paddle/fluid/framework/operator.cc +++ b/paddle/fluid/framework/operator.cc @@ -2936,9 +2936,9 @@ void OperatorWithKernel::ParseMultiInputDataType( proto::VarType::Type OperatorWithKernel::IndicateDataType( const ExecutionContext& ctx) const { - proto::VarType::Type dafault_data_type = + proto::VarType::Type default_data_type = static_cast(-1); - proto::VarType::Type data_type = dafault_data_type; + proto::VarType::Type data_type = default_data_type; for (auto* name : ctx.InNameList()) { if (ctx.InputSize(*name) == 1UL) { @@ -2949,7 +2949,7 @@ proto::VarType::Type OperatorWithKernel::IndicateDataType( } PADDLE_ENFORCE_NE( data_type, - dafault_data_type, + default_data_type, platform::errors::NotFound( "DataType should be indicated by input Variable at %s.", Type())); return data_type; @@ -2957,9 +2957,9 @@ proto::VarType::Type OperatorWithKernel::IndicateDataType( proto::VarType::Type OperatorWithKernel::IndicateVarDataType( const ExecutionContext& ctx, const std::string& name) const { - proto::VarType::Type dafault_data_type = + proto::VarType::Type default_data_type = static_cast(-1); - proto::VarType::Type data_type = dafault_data_type; + proto::VarType::Type data_type = default_data_type; if (ctx.InputSize(name) == 1UL) { ParseInputDataType(ctx.InputVar(name), name, &data_type); } else { @@ -2967,7 +2967,7 @@ proto::VarType::Type OperatorWithKernel::IndicateVarDataType( } PADDLE_ENFORCE_NE( data_type, - dafault_data_type, + default_data_type, platform::errors::InvalidArgument( "The Input Variable(%s) of (%s) Operator used to determine kernel " "data type is empty or not phi::DenseTensor or SelectedRows or " From 9d10d5ce2b07791a33b79585c4f593fd6814e67e Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 23 Feb 2024 12:07:40 +0800 Subject: [PATCH 016/282] Fix EmbeddingInferSpmdUnspportVocabParallel(#61928) --- paddle/phi/api/yaml/legacy_ops.yaml | 2 +- paddle/phi/infermeta/spmd_rules/embedding.cc | 8 ++++---- paddle/phi/infermeta/spmd_rules/embedding.h | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml index 39d1fda93c48b..9b1d862180903 100755 --- a/paddle/phi/api/yaml/legacy_ops.yaml +++ b/paddle/phi/api/yaml/legacy_ops.yaml @@ -384,7 +384,7 @@ infer_meta : func : EmbeddingInferMeta param : [x, weight, padding_idx] - spmd_rule: EmbeddingInferSpmdUnspportVocabParallel + spmd_rule: EmbeddingInferSpmdUnsupportVocabParallel kernel : func : embedding {dense, dense -> dense} sparse_weight_embedding {dense, selected_rows -> dense} diff --git a/paddle/phi/infermeta/spmd_rules/embedding.cc b/paddle/phi/infermeta/spmd_rules/embedding.cc index 8b8a309f66292..27284ae1a82e0 100644 --- a/paddle/phi/infermeta/spmd_rules/embedding.cc +++ b/paddle/phi/infermeta/spmd_rules/embedding.cc @@ -28,10 +28,10 @@ namespace distributed { using phi::distributed::auto_parallel::str_join; -SpmdInfo EmbeddingInferSpmdUnspportVocabParallel(const DistMetaTensor& x, - const DistMetaTensor& weight, - int padding_idx, - bool sparse) { +SpmdInfo EmbeddingInferSpmdUnsupportVocabParallel(const DistMetaTensor& x, + const DistMetaTensor& weight, + int padding_idx, + bool sparse) { DistMetaTensor w(weight.dims(), weight.dist_attr()); if (weight.dist_attr().dims_mapping()[0] >= 0) { auto w_dims_mapping = weight.dist_attr().dims_mapping(); diff --git a/paddle/phi/infermeta/spmd_rules/embedding.h b/paddle/phi/infermeta/spmd_rules/embedding.h index cc168c6fca86e..47b56048892f0 100644 --- a/paddle/phi/infermeta/spmd_rules/embedding.h +++ b/paddle/phi/infermeta/spmd_rules/embedding.h @@ -39,10 +39,10 @@ SpmdInfo EmbeddingInferSpmd(const DistMetaTensor& x, /// is used in static graph, but `embedding` used in egaer graph is not /// supported. So we need two propagation rules for `c_embedding` and /// `embedding`. -SpmdInfo EmbeddingInferSpmdUnspportVocabParallel(const DistMetaTensor& x, - const DistMetaTensor& weight, - int padding_idx, - bool sparse = false); +SpmdInfo EmbeddingInferSpmdUnsupportVocabParallel(const DistMetaTensor& x, + const DistMetaTensor& weight, + int padding_idx, + bool sparse = false); SpmdInfo EmbeddingInferSpmdReverse(const DistMetaTensor& x, const DistMetaTensor& weight, From c4dbcc81fc44b6d8169676c527e8984ec7dbcdc6 Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 23 Feb 2024 12:08:58 +0800 Subject: [PATCH 017/282] Fix grad node interface name (#61958) * Fix * ci * ci --- .../eager_manual/forwards/add_n_fwd_func.cc | 2 +- .../forwards/conv2d_fwd_function.cc | 16 +- .../forwards/multiply_fwd_func.cc | 28 ++-- .../eager_manual/forwards/reshard_fwd_func.cc | 2 +- .../forwards/sync_batch_norm_fwd_func.cc | 48 +++--- .../manual/eager_manual/nodes/conv2d_nodes.cc | 18 +- .../eager_manual/nodes/multiply_node.cc | 8 +- .../api/manual/eager_manual/nodes/nodes.h | 108 ++++++------ .../forwards/fused_attention_fwd_func.cc | 56 +++---- ...as_dropout_residual_layer_norm_fwd_func.cc | 19 ++- .../forwards/fused_feedforward_fwd_func.cc | 38 ++--- .../forwards/fused_gate_attention_fwd_func.cc | 40 ++--- .../forwards/fused_gemm_epilogue_fwd_func.cc | 4 +- .../api/manual/fluid_manual/nodes/nodes.h | 156 +++++++++--------- .../auto_code_generator/eager_generator.cc | 4 +- .../generator/eager_gen.py | 26 +-- paddle/fluid/pybind/eager_method.cc | 14 +- 17 files changed, 294 insertions(+), 293 deletions(-) diff --git a/paddle/fluid/eager/api/manual/eager_manual/forwards/add_n_fwd_func.cc b/paddle/fluid/eager/api/manual/eager_manual/forwards/add_n_fwd_func.cc index add0359ccf25d..d27ca1d242953 100644 --- a/paddle/fluid/eager/api/manual/eager_manual/forwards/add_n_fwd_func.cc +++ b/paddle/fluid/eager/api/manual/eager_manual/forwards/add_n_fwd_func.cc @@ -94,7 +94,7 @@ paddle::Tensor add_n_ad_func(const std::vector& x) { // SetAttributes if needed // Set TensorWrappers for Forward Inputs if needed - grad_node->SetTensorWrapperx(x); + grad_node->SetTensorWrapper_x(x); // SetGradOutMeta & SetEdges grad_node->SetGradOutMeta(x, 0); // SetOutRank & SetHistory & SetGradInMeta & RetainGrad diff --git a/paddle/fluid/eager/api/manual/eager_manual/forwards/conv2d_fwd_function.cc b/paddle/fluid/eager/api/manual/eager_manual/forwards/conv2d_fwd_function.cc index 33e9393a615bc..7cf3ee807b685 100644 --- a/paddle/fluid/eager/api/manual/eager_manual/forwards/conv2d_fwd_function.cc +++ b/paddle/fluid/eager/api/manual/eager_manual/forwards/conv2d_fwd_function.cc @@ -146,15 +146,15 @@ paddle::Tensor conv2d_ad_func(const paddle::Tensor& input, } // SetAttributes if needed - grad_node->SetAttributestrides(strides); - grad_node->SetAttributepaddings(paddings); - grad_node->SetAttributepadding_algorithm(padding_algorithm); - grad_node->SetAttributegroups(groups); - grad_node->SetAttributedilations(dilations); - grad_node->SetAttributedata_format(data_format); + grad_node->SetAttribute_strides(strides); + grad_node->SetAttribute_paddings(paddings); + grad_node->SetAttribute_padding_algorithm(padding_algorithm); + grad_node->SetAttribute_groups(groups); + grad_node->SetAttribute_dilations(dilations); + grad_node->SetAttribute_data_format(data_format); // Set TensorWrappers for Forward Inputs if needed - grad_node->SetTensorWrapperinput(input); - grad_node->SetTensorWrapperfilter(filter); + grad_node->SetTensorWrapper_input(input); + grad_node->SetTensorWrapper_filter(filter); // SetGradOutMeta & SetEdges grad_node->SetGradOutMeta(input, 0); grad_node->SetGradOutMeta(filter, 1); diff --git a/paddle/fluid/eager/api/manual/eager_manual/forwards/multiply_fwd_func.cc b/paddle/fluid/eager/api/manual/eager_manual/forwards/multiply_fwd_func.cc index 18e36264ebe6b..856407c58e96c 100644 --- a/paddle/fluid/eager/api/manual/eager_manual/forwards/multiply_fwd_func.cc +++ b/paddle/fluid/eager/api/manual/eager_manual/forwards/multiply_fwd_func.cc @@ -157,25 +157,25 @@ paddle::Tensor multiply_ad_func(const paddle::Tensor& x, grad_node->SetForwardTrace(egr::Controller::Instance().GetPythonStack()); } // SetAttributes if needed - grad_node->SetAttributeaxis(-1); + grad_node->SetAttribute_axis(-1); if (paddle::platform::is_gpu_place(x.place())) { if (x_autograd_meta != nullptr && x_autograd_meta->StopGradient() && y_autograd_meta != nullptr && !y_autograd_meta->StopGradient()) { - grad_node->SetTensorWrapperx(x); - grad_node->SetTensorWrapperNoNeedBuffery(y); + grad_node->SetTensorWrapper_x(x); + grad_node->SetTensorWrapperNoNeedBuffer_y(y); } else if (x_autograd_meta != nullptr && !x_autograd_meta->StopGradient() && y_autograd_meta != nullptr && y_autograd_meta->StopGradient()) { - grad_node->SetTensorWrapperNoNeedBufferx(x); - grad_node->SetTensorWrappery(y); + grad_node->SetTensorWrapperNoNeedBuffer_x(x); + grad_node->SetTensorWrapper_y(y); } else { - grad_node->SetTensorWrapperx(x); - grad_node->SetTensorWrappery(y); + grad_node->SetTensorWrapper_x(x); + grad_node->SetTensorWrapper_y(y); } } else { - grad_node->SetTensorWrapperx(x); - grad_node->SetTensorWrappery(y); + grad_node->SetTensorWrapper_x(x); + grad_node->SetTensorWrapper_y(y); } // SetGradOutMeta & SetEdges grad_node->SetGradOutMeta(x, 0); @@ -300,11 +300,11 @@ paddle::Tensor& multiply__ad_func(paddle::Tensor& x, // NOLINT grad_node->SetForwardTrace(egr::Controller::Instance().GetPythonStack()); } // SetAttributes if needed - grad_node->SetAttributeaxis(-1); + grad_node->SetAttribute_axis(-1); // Set TensorWrappers for Forward Inputs if needed auto x_clone = paddle::experimental::assign(x); - grad_node->SetTensorWrapperx(x_clone); - grad_node->SetTensorWrappery(y); + grad_node->SetTensorWrapper_x(x_clone); + grad_node->SetTensorWrapper_y(y); } // Forward API Call @@ -505,8 +505,8 @@ paddle::Tensor multiply_ad_func(const paddle::Tensor& x, // SetAttributes if needed // Set TensorWrappers for Forward Inputs if needed - grad_node->SetTensorWrapperx(x); - grad_node->SetTensorWrappery(y); + grad_node->SetTensorWrapper_x(x); + grad_node->SetTensorWrapper_y(y); // SetGradOutMeta & SetEdges grad_node->SetGradOutMeta(x, 0); grad_node->SetGradOutMeta(y, 1); diff --git a/paddle/fluid/eager/api/manual/eager_manual/forwards/reshard_fwd_func.cc b/paddle/fluid/eager/api/manual/eager_manual/forwards/reshard_fwd_func.cc index 5ee5d74094538..b227e2a06e68d 100644 --- a/paddle/fluid/eager/api/manual/eager_manual/forwards/reshard_fwd_func.cc +++ b/paddle/fluid/eager/api/manual/eager_manual/forwards/reshard_fwd_func.cc @@ -50,7 +50,7 @@ paddle::Tensor reshard_ad_function( std::shared_ptr(new ReshardGradNode(1, 1)); // NOLINT // Set TensorWrappers for Forward Inputs if needed - grad_node->SetTensorWrapperNoNeedBufferInput(input); + grad_node->SetTensorWrapperNoNeedBuffer_Input(input); } // Forward API Call diff --git a/paddle/fluid/eager/api/manual/eager_manual/forwards/sync_batch_norm_fwd_func.cc b/paddle/fluid/eager/api/manual/eager_manual/forwards/sync_batch_norm_fwd_func.cc index 654ab2bfd73db..c4e007801c66c 100644 --- a/paddle/fluid/eager/api/manual/eager_manual/forwards/sync_batch_norm_fwd_func.cc +++ b/paddle/fluid/eager/api/manual/eager_manual/forwards/sync_batch_norm_fwd_func.cc @@ -235,16 +235,16 @@ sync_batch_norm__ad_func(const paddle::Tensor& x, egr::Controller::Instance().PushBackForceSequentialNodes(grad_node.get()); // SetAttributes if needed - grad_node->SetAttributemomentum(momentum); - grad_node->SetAttributeepsilon(epsilon); - grad_node->SetAttributedata_layout(data_layout); - grad_node->SetAttributeis_test(is_test); - grad_node->SetAttributeuse_global_stats(use_global_stats); - grad_node->SetAttributetrainable_statistics(trainable_statistics); + grad_node->SetAttribute_momentum(momentum); + grad_node->SetAttribute_epsilon(epsilon); + grad_node->SetAttribute_data_layout(data_layout); + grad_node->SetAttribute_is_test(is_test); + grad_node->SetAttribute_use_global_stats(use_global_stats); + grad_node->SetAttribute_trainable_statistics(trainable_statistics); // Set TensorWrappers for Forward Inputs if needed - grad_node->SetTensorWrapperx(x); - grad_node->SetTensorWrapperscale(scale); - grad_node->SetTensorWrapperbias(bias); + grad_node->SetTensorWrapper_x(x); + grad_node->SetTensorWrapper_scale(scale); + grad_node->SetTensorWrapper_bias(bias); // SetGradOutMeta & SetEdges grad_node->SetGradOutMeta(x, 0); grad_node->SetGradOutMeta(scale, 3); @@ -293,9 +293,9 @@ sync_batch_norm__ad_func(const paddle::Tensor& x, grad_node->SetGradInMeta(saved_variance, 4); grad_node->SetGradInMeta(reserve_space, 5); // Set TensorWrappers for Forward Outputs if needed - grad_node->SetTensorWrappersaved_mean(saved_mean); - grad_node->SetTensorWrappersaved_variance(saved_variance); - grad_node->SetTensorWrapperreserve_space(reserve_space); + grad_node->SetTensorWrapper_saved_mean(saved_mean); + grad_node->SetTensorWrapper_saved_variance(saved_variance); + grad_node->SetTensorWrapper_reserve_space(reserve_space); } VLOG(4) << "Finish AD API: sync_batch_norm_"; @@ -571,16 +571,16 @@ sync_batch_norm__ad_func(const paddle::Tensor& x, new SyncBatchNormGradNode(6, 5)); egr::Controller::Instance().PushBackForceSequentialNodes(grad_node.get()); // SetAttributes if needed - grad_node->SetAttributemomentum(momentum); - grad_node->SetAttributeepsilon(epsilon); - grad_node->SetAttributedata_layout(data_layout); - grad_node->SetAttributeis_test(is_test); - grad_node->SetAttributeuse_global_stats(use_global_stats); - grad_node->SetAttributetrainable_statistics(trainable_statistics); + grad_node->SetAttribute_momentum(momentum); + grad_node->SetAttribute_epsilon(epsilon); + grad_node->SetAttribute_data_layout(data_layout); + grad_node->SetAttribute_is_test(is_test); + grad_node->SetAttribute_use_global_stats(use_global_stats); + grad_node->SetAttribute_trainable_statistics(trainable_statistics); // Set TensorWrappers for Forward Inputs if needed - grad_node->SetTensorWrapperx(x); - grad_node->SetTensorWrapperscale(scale); - grad_node->SetTensorWrapperbias(bias); + grad_node->SetTensorWrapper_x(x); + grad_node->SetTensorWrapper_scale(scale); + grad_node->SetTensorWrapper_bias(bias); // SetGradOutMeta & SetEdges grad_node->SetGradOutMeta(x, 0); grad_node->SetGradOutMeta(scale, 3); @@ -629,9 +629,9 @@ sync_batch_norm__ad_func(const paddle::Tensor& x, grad_node->SetGradInMeta(saved_variance, 4); grad_node->SetGradInMeta(reserve_space, 5); // Set TensorWrappers for Forward Outputs if needed - grad_node->SetTensorWrappersaved_mean(saved_mean); - grad_node->SetTensorWrappersaved_variance(saved_variance); - grad_node->SetTensorWrapperreserve_space(reserve_space); + grad_node->SetTensorWrapper_saved_mean(saved_mean); + grad_node->SetTensorWrapper_saved_variance(saved_variance); + grad_node->SetTensorWrapper_reserve_space(reserve_space); } VLOG(4) << "Finish AD API: sync_batch_norm_"; diff --git a/paddle/fluid/eager/api/manual/eager_manual/nodes/conv2d_nodes.cc b/paddle/fluid/eager/api/manual/eager_manual/nodes/conv2d_nodes.cc index e1bcc3bc73731..437cce80c919b 100644 --- a/paddle/fluid/eager/api/manual/eager_manual/nodes/conv2d_nodes.cc +++ b/paddle/fluid/eager/api/manual/eager_manual/nodes/conv2d_nodes.cc @@ -126,16 +126,16 @@ Conv2dGradNodeFinal::operator()( auto grad_node = std::shared_ptr( // NOLINT new Conv2dDoubleGradNodeFinal(2, 3)); // SetAttributes if needed - grad_node->SetAttributestrides(strides); - grad_node->SetAttributepaddings(paddings); - grad_node->SetAttributepadding_algorithm(padding_algorithm); - grad_node->SetAttributegroups(groups); - grad_node->SetAttributedilations(dilations); - grad_node->SetAttributedata_format(data_format); + grad_node->SetAttribute_strides(strides); + grad_node->SetAttribute_paddings(paddings); + grad_node->SetAttribute_padding_algorithm(padding_algorithm); + grad_node->SetAttribute_groups(groups); + grad_node->SetAttribute_dilations(dilations); + grad_node->SetAttribute_data_format(data_format); // Set TensorWrappers for Forward Inputs if needed - grad_node->SetTensorWrapperinput(input); - grad_node->SetTensorWrapperfilter(filter); - grad_node->SetTensorWrappergrad_out(grad_out); + grad_node->SetTensorWrapper_input(input); + grad_node->SetTensorWrapper_filter(filter); + grad_node->SetTensorWrapper_grad_out(grad_out); // SetGradOutMeta & SetEdges if (grad_filter_autograd_meta) { grad_node->SetGradOutMeta(input, 0); diff --git a/paddle/fluid/eager/api/manual/eager_manual/nodes/multiply_node.cc b/paddle/fluid/eager/api/manual/eager_manual/nodes/multiply_node.cc index b3e38e066300d..56c1f1e61a7fc 100644 --- a/paddle/fluid/eager/api/manual/eager_manual/nodes/multiply_node.cc +++ b/paddle/fluid/eager/api/manual/eager_manual/nodes/multiply_node.cc @@ -167,11 +167,11 @@ MultiplyGradNode::operator()( auto grad_node = std::shared_ptr( // NOLINT new MultiplyDoubleGradNode(2, 3)); // SetAttributes if needed - grad_node->SetAttributeaxis(axis); + grad_node->SetAttribute_axis(axis); // Set TensorWrappers for Forward Inputs if needed - grad_node->SetTensorWrapperx(x); - grad_node->SetTensorWrappery(y); - grad_node->SetTensorWrappergrad_out(grad_out); + grad_node->SetTensorWrapper_x(x); + grad_node->SetTensorWrapper_y(y); + grad_node->SetTensorWrapper_grad_out(grad_out); // SetGradOutMeta & SetEdges grad_node->SetGradOutMeta(x, 0); grad_node->SetGradOutMeta(y, 1); diff --git a/paddle/fluid/eager/api/manual/eager_manual/nodes/nodes.h b/paddle/fluid/eager/api/manual/eager_manual/nodes/nodes.h index bc6d1d9f1a1b6..12274670827f6 100644 --- a/paddle/fluid/eager/api/manual/eager_manual/nodes/nodes.h +++ b/paddle/fluid/eager/api/manual/eager_manual/nodes/nodes.h @@ -48,28 +48,28 @@ class Conv2dGradNodeFinal : public egr::GradNodeBase { } // SetTensorWrapperX, SetTensorWrapperY, ... - void SetTensorWrapperinput(const paddle::Tensor& input) { + void SetTensorWrapper_input(const paddle::Tensor& input) { input_ = egr::TensorWrapper(input, false); } - void SetTensorWrapperfilter(const paddle::Tensor& filter) { + void SetTensorWrapper_filter(const paddle::Tensor& filter) { filter_ = egr::TensorWrapper(filter, false); } // SetAttributes - void SetAttributestrides(const std::vector& strides) { + void SetAttribute_strides(const std::vector& strides) { strides_ = strides; } - void SetAttributepaddings(const std::vector& paddings) { + void SetAttribute_paddings(const std::vector& paddings) { paddings_ = paddings; } - void SetAttributepadding_algorithm(const std::string& padding_algorithm) { + void SetAttribute_padding_algorithm(const std::string& padding_algorithm) { padding_algorithm_ = padding_algorithm; } - void SetAttributegroups(const int& groups) { groups_ = groups; } - void SetAttributedilations(const std::vector& dilations) { + void SetAttribute_groups(const int& groups) { groups_ = groups; } + void SetAttribute_dilations(const std::vector& dilations) { dilations_ = dilations; } - void SetAttributedata_format(const std::string& data_format) { + void SetAttribute_data_format(const std::string& data_format) { data_format_ = data_format; } @@ -117,31 +117,31 @@ class Conv2dDoubleGradNodeFinal : public egr::GradNodeBase { } // SetTensorWrapperX, SetTensorWrapperY, ... - void SetTensorWrapperinput(const paddle::Tensor& input) { + void SetTensorWrapper_input(const paddle::Tensor& input) { input_ = egr::TensorWrapper(input, false); } - void SetTensorWrapperfilter(const paddle::Tensor& filter) { + void SetTensorWrapper_filter(const paddle::Tensor& filter) { filter_ = egr::TensorWrapper(filter, false); } - void SetTensorWrappergrad_out(const paddle::Tensor& grad_out) { + void SetTensorWrapper_grad_out(const paddle::Tensor& grad_out) { grad_out_ = egr::TensorWrapper(grad_out, false); } // SetAttributes - void SetAttributestrides(const std::vector& strides) { + void SetAttribute_strides(const std::vector& strides) { strides_ = strides; } - void SetAttributepaddings(const std::vector& paddings) { + void SetAttribute_paddings(const std::vector& paddings) { paddings_ = paddings; } - void SetAttributepadding_algorithm(const std::string& padding_algorithm) { + void SetAttribute_padding_algorithm(const std::string& padding_algorithm) { padding_algorithm_ = padding_algorithm; } - void SetAttributegroups(const int& groups) { groups_ = groups; } - void SetAttributedilations(const std::vector& dilations) { + void SetAttribute_groups(const int& groups) { groups_ = groups; } + void SetAttribute_dilations(const std::vector& dilations) { dilations_ = dilations; } - void SetAttributedata_format(const std::string& data_format) { + void SetAttribute_data_format(const std::string& data_format) { data_format_ = data_format; } @@ -190,7 +190,7 @@ class AddNGradNodeFinal : public egr::GradNodeBase { } // SetTensorWrapperX, SetTensorWrapperY, ... - void SetTensorWrapperx(const std::vector& x) { + void SetTensorWrapper_x(const std::vector& x) { for (const auto& eager_tensor : x) { x_.emplace_back(egr::TensorWrapper(eager_tensor, true)); } @@ -233,22 +233,22 @@ class MultiplyGradNode : public egr::GradNodeBase { } // SetTensorWrapperX, SetTensorWrapperY, ... - void SetTensorWrapperx(const paddle::Tensor& x) { + void SetTensorWrapper_x(const paddle::Tensor& x) { x_ = egr::TensorWrapper(x, false); } - void SetTensorWrappery(const paddle::Tensor& y) { + void SetTensorWrapper_y(const paddle::Tensor& y) { y_ = egr::TensorWrapper(y, false); } - void SetTensorWrapperNoNeedBufferx(const paddle::Tensor& x) { + void SetTensorWrapperNoNeedBuffer_x(const paddle::Tensor& x) { x_ = egr::TensorWrapper(x, true); } - void SetTensorWrapperNoNeedBuffery(const paddle::Tensor& y) { + void SetTensorWrapperNoNeedBuffer_y(const paddle::Tensor& y) { y_ = egr::TensorWrapper(y, true); } // SetAttributes - void SetAttributeaxis(const int& axis) { axis_ = axis; } + void SetAttribute_axis(const int& axis) { axis_ = axis; } private: // TensorWrappers @@ -289,18 +289,18 @@ class MultiplyDoubleGradNode : public egr::GradNodeBase { } // SetTensorWrapperX, SetTensorWrapperY, ... - void SetTensorWrapperx(const paddle::Tensor& x) { + void SetTensorWrapper_x(const paddle::Tensor& x) { x_ = egr::TensorWrapper(x, false); } - void SetTensorWrappery(const paddle::Tensor& y) { + void SetTensorWrapper_y(const paddle::Tensor& y) { y_ = egr::TensorWrapper(y, false); } - void SetTensorWrappergrad_out(const paddle::Tensor& grad_out) { + void SetTensorWrapper_grad_out(const paddle::Tensor& grad_out) { grad_out_ = egr::TensorWrapper(grad_out, false); } // SetAttributes - void SetAttributeaxis(const int& axis) { axis_ = axis; } + void SetAttribute_axis(const int& axis) { axis_ = axis; } private: // TensorWrappers @@ -345,36 +345,36 @@ class SyncBatchNormGradNode : public egr::GradNodeBase { } // SetTensorWrapperX, SetTensorWrapperY, ... - void SetTensorWrapperx(const paddle::Tensor& x) { + void SetTensorWrapper_x(const paddle::Tensor& x) { x_ = egr::TensorWrapper(x, false); } - void SetTensorWrapperscale(const paddle::Tensor& scale) { + void SetTensorWrapper_scale(const paddle::Tensor& scale) { scale_ = egr::TensorWrapper(scale, false); } - void SetTensorWrapperbias(const paddle::Tensor& bias) { + void SetTensorWrapper_bias(const paddle::Tensor& bias) { bias_ = egr::TensorWrapper(bias, false); } - void SetTensorWrappersaved_mean(const paddle::Tensor& saved_mean) { + void SetTensorWrapper_saved_mean(const paddle::Tensor& saved_mean) { saved_mean_ = egr::TensorWrapper(saved_mean, false); } - void SetTensorWrappersaved_variance(const paddle::Tensor& saved_variance) { + void SetTensorWrapper_saved_variance(const paddle::Tensor& saved_variance) { saved_variance_ = egr::TensorWrapper(saved_variance, false); } - void SetTensorWrapperreserve_space(const paddle::Tensor& reserve_space) { + void SetTensorWrapper_reserve_space(const paddle::Tensor& reserve_space) { reserve_space_ = egr::TensorWrapper(reserve_space, false); } // SetAttributes - void SetAttributemomentum(const float& momentum) { momentum_ = momentum; } - void SetAttributeepsilon(const float& epsilon) { epsilon_ = epsilon; } - void SetAttributedata_layout(const std::string& data_layout) { + void SetAttribute_momentum(const float& momentum) { momentum_ = momentum; } + void SetAttribute_epsilon(const float& epsilon) { epsilon_ = epsilon; } + void SetAttribute_data_layout(const std::string& data_layout) { data_layout_ = data_layout; } - void SetAttributeis_test(const bool& is_test) { is_test_ = is_test; } - void SetAttributeuse_global_stats(const bool& use_global_stats) { + void SetAttribute_is_test(const bool& is_test) { is_test_ = is_test; } + void SetAttribute_use_global_stats(const bool& use_global_stats) { use_global_stats_ = use_global_stats; } - void SetAttributetrainable_statistics(const bool& trainable_statistics) { + void SetAttribute_trainable_statistics(const bool& trainable_statistics) { trainable_statistics_ = trainable_statistics; } @@ -434,7 +434,7 @@ class ReshardGradNode : public egr::GradNodeBase { // SetTensorWrapperX // Only input's meta is needed. - void SetTensorWrapperNoNeedBufferInput(const paddle::Tensor& input) { + void SetTensorWrapperNoNeedBuffer_Input(const paddle::Tensor& input) { input_ = egr::TensorWrapper(input, true); } @@ -477,36 +477,36 @@ class SyncBatchNormGradNode : public egr::GradNodeBase { } // SetTensorWrapperX, SetTensorWrapperY, ... - void SetTensorWrapperx(const paddle::Tensor& x) { + void SetTensorWrapper_x(const paddle::Tensor& x) { x_ = egr::TensorWrapper(x, false); } - void SetTensorWrapperscale(const paddle::Tensor& scale) { + void SetTensorWrapper_scale(const paddle::Tensor& scale) { scale_ = egr::TensorWrapper(scale, false); } - void SetTensorWrapperbias(const paddle::Tensor& bias) { + void SetTensorWrapper_bias(const paddle::Tensor& bias) { bias_ = egr::TensorWrapper(bias, false); } - void SetTensorWrappersaved_mean(const paddle::Tensor& saved_mean) { + void SetTensorWrapper_saved_mean(const paddle::Tensor& saved_mean) { saved_mean_ = egr::TensorWrapper(saved_mean, false); } - void SetTensorWrappersaved_variance(const paddle::Tensor& saved_variance) { + void SetTensorWrapper_saved_variance(const paddle::Tensor& saved_variance) { saved_variance_ = egr::TensorWrapper(saved_variance, false); } - void SetTensorWrapperreserve_space(const paddle::Tensor& reserve_space) { + void SetTensorWrapper_reserve_space(const paddle::Tensor& reserve_space) { reserve_space_ = egr::TensorWrapper(reserve_space, false); } // SetAttributes - void SetAttributemomentum(const float& momentum) { momentum_ = momentum; } - void SetAttributeepsilon(const float& epsilon) { epsilon_ = epsilon; } - void SetAttributedata_layout(const std::string& data_layout) { + void SetAttribute_momentum(const float& momentum) { momentum_ = momentum; } + void SetAttribute_epsilon(const float& epsilon) { epsilon_ = epsilon; } + void SetAttribute_data_layout(const std::string& data_layout) { data_layout_ = data_layout; } - void SetAttributeis_test(const bool& is_test) { is_test_ = is_test; } - void SetAttributeuse_global_stats(const bool& use_global_stats) { + void SetAttribute_is_test(const bool& is_test) { is_test_ = is_test; } + void SetAttribute_use_global_stats(const bool& use_global_stats) { use_global_stats_ = use_global_stats; } - void SetAttributetrainable_statistics(const bool& trainable_statistics) { + void SetAttribute_trainable_statistics(const bool& trainable_statistics) { trainable_statistics_ = trainable_statistics; } @@ -557,10 +557,10 @@ class MultiplyGradNode : public egr::GradNodeBase { } // SetTensorWrapperX, SetTensorWrapperY, ... - void SetTensorWrapperx(const paddle::Tensor& x) { + void SetTensorWrapper_x(const paddle::Tensor& x) { x_ = egr::TensorWrapper(x, false); } - void SetTensorWrappery(const paddle::Tensor& y) { + void SetTensorWrapper_y(const paddle::Tensor& y) { y_ = egr::TensorWrapper(y, false); } diff --git a/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_attention_fwd_func.cc b/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_attention_fwd_func.cc index f3612c2830dd0..6130b79059f65 100644 --- a/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_attention_fwd_func.cc +++ b/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_attention_fwd_func.cc @@ -403,27 +403,27 @@ fused_attention_dygraph_function( grad_node->SetAttrMap(std::move(attrs)); grad_node->SetDefaultAttrMap(std::move(default_attrs)); - grad_node->SetTensorWrapperX(X); - grad_node->SetTensorWrapperQKVW(QKVW); - grad_node->SetTensorWrapperOutLinearW(OutLinearW); - grad_node->SetTensorWrapperQKVOut(QKVOut); - grad_node->SetTensorWrapperTransposeOut2(TransposeOut2); - grad_node->SetTensorWrapperQKOut(QKOut); - grad_node->SetTensorWrapperQKTVOut(QKTVOut); - grad_node->SetTensorWrapperSoftmaxOut(SoftmaxOut); - grad_node->SetTensorWrapperAttnDropoutMaskOut(AttnDropoutMaskOut); - grad_node->SetTensorWrapperAttnDropoutOut(AttnDropoutOut); - grad_node->SetTensorWrapperFMHAOut(FMHAOut); - grad_node->SetTensorWrapperOutLinearOut(OutLinearOut); - grad_node->SetTensorWrapperDropoutMaskOut(DropoutMaskOut); + grad_node->SetTensorWrapper_X(X); + grad_node->SetTensorWrapper_QKVW(QKVW); + grad_node->SetTensorWrapper_OutLinearW(OutLinearW); + grad_node->SetTensorWrapper_QKVOut(QKVOut); + grad_node->SetTensorWrapper_TransposeOut2(TransposeOut2); + grad_node->SetTensorWrapper_QKOut(QKOut); + grad_node->SetTensorWrapper_QKTVOut(QKTVOut); + grad_node->SetTensorWrapper_SoftmaxOut(SoftmaxOut); + grad_node->SetTensorWrapper_AttnDropoutMaskOut(AttnDropoutMaskOut); + grad_node->SetTensorWrapper_AttnDropoutOut(AttnDropoutOut); + grad_node->SetTensorWrapper_FMHAOut(FMHAOut); + grad_node->SetTensorWrapper_OutLinearOut(OutLinearOut); + grad_node->SetTensorWrapper_DropoutMaskOut(DropoutMaskOut); grad_node->SetGradOutMeta(X, 0); grad_node->SetGradOutMeta(QKVW, 3); grad_node->SetGradOutMeta(OutLinearW, 7); if (QKVBias.initialized()) { - grad_node->SetTensorWrapperQKVBias(QKVBias); - grad_node->SetTensorWrapperQKVBiasOut(QKVBiasOut); + grad_node->SetTensorWrapper_QKVBias(QKVBias); + grad_node->SetTensorWrapper_QKVBiasOut(QKVBiasOut); grad_node->SetGradOutMeta(QKVBias, 4); auto QKVBiasOut_accumulation_node = @@ -436,8 +436,8 @@ fused_attention_dygraph_function( } if (SrcMask.initialized()) { - grad_node->SetTensorWrapperSrcMask(SrcMask); - grad_node->SetTensorWrapperSrcMaskOut(SrcMaskOut); + grad_node->SetTensorWrapper_SrcMask(SrcMask); + grad_node->SetTensorWrapper_SrcMaskOut(SrcMaskOut); auto SrcMaskOut_accumulation_node = std::make_shared(p_autograd_SrcMaskOut); @@ -449,21 +449,21 @@ fused_attention_dygraph_function( } if (OutLinearBias.initialized()) { - grad_node->SetTensorWrapperOutLinearBias(OutLinearBias); + grad_node->SetTensorWrapper_OutLinearBias(OutLinearBias); grad_node->SetGradOutMeta(OutLinearBias, 8); } if (pre_layer_norm) { if (LnScale.initialized()) { - grad_node->SetTensorWrapperLnScale(LnScale); + grad_node->SetTensorWrapper_LnScale(LnScale); grad_node->SetGradOutMeta(LnScale, 1); } if (LnBias.initialized()) { - grad_node->SetTensorWrapperLnBias(LnBias); + grad_node->SetTensorWrapper_LnBias(LnBias); grad_node->SetGradOutMeta(LnBias, 2); } if (LnOut.initialized()) { - grad_node->SetTensorWrapperLnOut(LnOut); + grad_node->SetTensorWrapper_LnOut(LnOut); auto LnOut_accumulation_node = std::make_shared(p_autograd_LnOut); @@ -474,24 +474,24 @@ fused_attention_dygraph_function( grad_node->SetGradOutMeta(LnOut, 13); } if (LnMean.initialized()) { - grad_node->SetTensorWrapperLnMean(LnMean); + grad_node->SetTensorWrapper_LnMean(LnMean); } if (LnVariance.initialized()) { - grad_node->SetTensorWrapperLnVariance(LnVariance); + grad_node->SetTensorWrapper_LnVariance(LnVariance); } } else { if (Ln2Scale.initialized()) { - grad_node->SetTensorWrapperLn2Scale(Ln2Scale); + grad_node->SetTensorWrapper_Ln2Scale(Ln2Scale); grad_node->SetGradOutMeta(Ln2Scale, 9); } if (Ln2Bias.initialized()) { - grad_node->SetTensorWrapperLn2Bias(Ln2Bias); + grad_node->SetTensorWrapper_Ln2Bias(Ln2Bias); grad_node->SetGradOutMeta(Ln2Bias, 10); } - grad_node->SetTensorWrapperBiasDropoutResidualOut( + grad_node->SetTensorWrapper_BiasDropoutResidualOut( BiasDropoutResidualOut); - grad_node->SetTensorWrapperLn2Mean(Ln2Mean); - grad_node->SetTensorWrapperLn2Variance(Ln2Variance); + grad_node->SetTensorWrapper_Ln2Mean(Ln2Mean); + grad_node->SetTensorWrapper_Ln2Variance(Ln2Variance); auto BiasDropoutResidualOut_accumulation_node = std::make_shared( diff --git a/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_bias_dropout_residual_layer_norm_fwd_func.cc b/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_bias_dropout_residual_layer_norm_fwd_func.cc index c76073ba0b574..b67d0b40b7d0d 100644 --- a/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_bias_dropout_residual_layer_norm_fwd_func.cc +++ b/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_bias_dropout_residual_layer_norm_fwd_func.cc @@ -193,15 +193,16 @@ fused_bias_dropout_residual_layer_norm_dygraph_function( grad_node->SetDefaultAttrMap(std::move(default_attrs)); // Set Tensor Wrappers - grad_node->SetTensorWrapperBias(Bias); - grad_node->SetTensorWrapperBiasDropoutResidualOut(BiasDropoutResidualOut); - grad_node->SetTensorWrapperDropoutMaskOut(DropoutMaskOut); - grad_node->SetTensorWrapperLnBias(LnBias); - grad_node->SetTensorWrapperLnMean(LnMean); - grad_node->SetTensorWrapperLnScale(LnScale); - grad_node->SetTensorWrapperLnVariance(LnVariance); - grad_node->SetTensorWrapperResidual(Residual); - grad_node->SetTensorWrapperX(X); + grad_node->SetTensorWrapper_Bias(Bias); + grad_node->SetTensorWrapper_BiasDropoutResidualOut( + BiasDropoutResidualOut); + grad_node->SetTensorWrapper_DropoutMaskOut(DropoutMaskOut); + grad_node->SetTensorWrapper_LnBias(LnBias); + grad_node->SetTensorWrapper_LnMean(LnMean); + grad_node->SetTensorWrapper_LnScale(LnScale); + grad_node->SetTensorWrapper_LnVariance(LnVariance); + grad_node->SetTensorWrapper_Residual(Residual); + grad_node->SetTensorWrapper_X(X); grad_node->SetGradOutMeta(X, 0); grad_node->SetGradOutMeta(Residual, 1); diff --git a/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_feedforward_fwd_func.cc b/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_feedforward_fwd_func.cc index b2f5238c5be32..f3cfc39d17c7b 100644 --- a/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_feedforward_fwd_func.cc +++ b/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_feedforward_fwd_func.cc @@ -323,15 +323,15 @@ fused_feedforward_dygraph_function( grad_node->SetAttrMap(std::move(attrs)); grad_node->SetDefaultAttrMap(std::move(default_attrs)); - grad_node->SetTensorWrapperX(X); - grad_node->SetTensorWrapperLinear1Weight(Linear1Weight); - grad_node->SetTensorWrapperLinear1Bias(Linear1Bias); - grad_node->SetTensorWrapperLinear2Weight(Linear2Weight); - grad_node->SetTensorWrapperDropout1Mask(Dropout1Mask); - grad_node->SetTensorWrapperDropout2Mask(Dropout2Mask); - grad_node->SetTensorWrapperLinear1Out(Linear1Out); - grad_node->SetTensorWrapperDropout1Out(Dropout1Out); - grad_node->SetTensorWrapperDropout2Out(Dropout2Out); + grad_node->SetTensorWrapper_X(X); + grad_node->SetTensorWrapper_Linear1Weight(Linear1Weight); + grad_node->SetTensorWrapper_Linear1Bias(Linear1Bias); + grad_node->SetTensorWrapper_Linear2Weight(Linear2Weight); + grad_node->SetTensorWrapper_Dropout1Mask(Dropout1Mask); + grad_node->SetTensorWrapper_Dropout2Mask(Dropout2Mask); + grad_node->SetTensorWrapper_Linear1Out(Linear1Out); + grad_node->SetTensorWrapper_Dropout1Out(Dropout1Out); + grad_node->SetTensorWrapper_Dropout2Out(Dropout2Out); grad_node->SetGradOutMeta(X, 0); grad_node->SetGradOutMeta(Linear1Weight, 3); @@ -339,24 +339,24 @@ fused_feedforward_dygraph_function( grad_node->SetGradOutMeta(Linear2Weight, 5); if (pre_layer_norm) { - grad_node->SetTensorWrapperLn1Scale(Ln1Scale); - grad_node->SetTensorWrapperLn1Bias(Ln1Bias); - grad_node->SetTensorWrapperLn1Out(Ln1Out); - grad_node->SetTensorWrapperLn1Mean(Ln1Mean); - grad_node->SetTensorWrapperLn1Variance(Ln1Variance); + grad_node->SetTensorWrapper_Ln1Scale(Ln1Scale); + grad_node->SetTensorWrapper_Ln1Bias(Ln1Bias); + grad_node->SetTensorWrapper_Ln1Out(Ln1Out); + grad_node->SetTensorWrapper_Ln1Mean(Ln1Mean); + grad_node->SetTensorWrapper_Ln1Variance(Ln1Variance); grad_node->SetGradOutMeta(Ln1Scale, 7); grad_node->SetGradOutMeta(Ln1Bias, 8); } else { - grad_node->SetTensorWrapperLn2Scale(Ln2Scale); + grad_node->SetTensorWrapper_Ln2Scale(Ln2Scale); grad_node->SetGradOutMeta(Ln2Scale, 9); - grad_node->SetTensorWrapperLn2Bias(Ln2Bias); + grad_node->SetTensorWrapper_Ln2Bias(Ln2Bias); grad_node->SetGradOutMeta(Ln2Bias, 10); - grad_node->SetTensorWrapperLn2Mean(Ln2Mean); - grad_node->SetTensorWrapperLn2Variance(Ln2Variance); + grad_node->SetTensorWrapper_Ln2Mean(Ln2Mean); + grad_node->SetTensorWrapper_Ln2Variance(Ln2Variance); } if (Linear2Bias.initialized()) { - grad_node->SetTensorWrapperLinear2Bias(Linear2Bias); + grad_node->SetTensorWrapper_Linear2Bias(Linear2Bias); grad_node->SetGradOutMeta(Linear2Bias, 6); } diff --git a/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_gate_attention_fwd_func.cc b/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_gate_attention_fwd_func.cc index c42a099cef4b0..b9e2a52228bcb 100644 --- a/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_gate_attention_fwd_func.cc +++ b/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_gate_attention_fwd_func.cc @@ -324,28 +324,28 @@ fused_gate_attention_dygraph_function( grad_node->SetAttrMap(std::move(attrs)); grad_node->SetDefaultAttrMap(std::move(default_attrs)); - grad_node->SetTensorWrapperFMHAOut(FMHAOut); - grad_node->SetTensorWrapperQuery(Query); - grad_node->SetTensorWrapperSoftmaxOut(SoftmaxOut); - grad_node->SetTensorWrapperOutLinearBias(OutLinearBias); - grad_node->SetTensorWrapperOutLinearWeight(OutLinearWeight); + grad_node->SetTensorWrapper_FMHAOut(FMHAOut); + grad_node->SetTensorWrapper_Query(Query); + grad_node->SetTensorWrapper_SoftmaxOut(SoftmaxOut); + grad_node->SetTensorWrapper_OutLinearBias(OutLinearBias); + grad_node->SetTensorWrapper_OutLinearWeight(OutLinearWeight); grad_node->SetGradOutMeta(Query, 0); grad_node->SetGradOutMeta(OutLinearWeight, 10); grad_node->SetGradOutMeta(OutLinearBias, 11); if (merge_qkv) { - grad_node->SetTensorWrapperQKVTransposeOut(QKVTransposeOut); - grad_node->SetTensorWrapperQKVWeight(QKVWeight); + grad_node->SetTensorWrapper_QKVTransposeOut(QKVTransposeOut); + grad_node->SetTensorWrapper_QKVWeight(QKVWeight); grad_node->SetGradOutMeta(QKVWeight, 5); } else { - grad_node->SetTensorWrapperKey(Key); - grad_node->SetTensorWrapperQueryWeight(QueryWeight); - grad_node->SetTensorWrapperKeyWeight(KeyWeight); - grad_node->SetTensorWrapperValueWeight(ValueWeight); - grad_node->SetTensorWrapperQueryTransposeOut(QueryTransposeOut); - grad_node->SetTensorWrapperKeyTransposeOut(KeyTransposeOut); - grad_node->SetTensorWrapperValueTransposeOut(ValueTransposeOut); + grad_node->SetTensorWrapper_Key(Key); + grad_node->SetTensorWrapper_QueryWeight(QueryWeight); + grad_node->SetTensorWrapper_KeyWeight(KeyWeight); + grad_node->SetTensorWrapper_ValueWeight(ValueWeight); + grad_node->SetTensorWrapper_QueryTransposeOut(QueryTransposeOut); + grad_node->SetTensorWrapper_KeyTransposeOut(KeyTransposeOut); + grad_node->SetTensorWrapper_ValueTransposeOut(ValueTransposeOut); grad_node->SetGradOutMeta(Key, 1); grad_node->SetGradOutMeta(QueryWeight, 2); @@ -354,21 +354,21 @@ fused_gate_attention_dygraph_function( } if (has_gating) { - grad_node->SetTensorWrapperGateWeight(GateWeight); + grad_node->SetTensorWrapper_GateWeight(GateWeight); grad_node->SetGradOutMeta(GateWeight, 8); - grad_node->SetTensorWrapperGateBias(GateBias); + grad_node->SetTensorWrapper_GateBias(GateBias); grad_node->SetGradOutMeta(GateBias, 9); - grad_node->SetTensorWrapperGateOut(GateOut); + grad_node->SetTensorWrapper_GateOut(GateOut); } if (NonbatchedBias.initialized()) { - grad_node->SetTensorWrapperNonbatchedBias(NonbatchedBias); + grad_node->SetTensorWrapper_NonbatchedBias(NonbatchedBias); grad_node->SetGradOutMeta(NonbatchedBias, 6); } if (use_flash_attn) { - grad_node->SetTensorWrapperSoftmaxLse(SoftmaxLse); - grad_node->SetTensorWrapperSrcMask(SrcMask); + grad_node->SetTensorWrapper_SoftmaxLse(SoftmaxLse); + grad_node->SetTensorWrapper_SrcMask(SrcMask); grad_node->SetGradOutMeta(SrcMask, 7); } diff --git a/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_gemm_epilogue_fwd_func.cc b/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_gemm_epilogue_fwd_func.cc index c4ae0840c294f..15c0fdfd0d1ff 100644 --- a/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_gemm_epilogue_fwd_func.cc +++ b/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_gemm_epilogue_fwd_func.cc @@ -111,8 +111,8 @@ paddle::Tensor fused_gemm_epilogue_dygraph_function( grad_node->SetDefaultAttrMap(std::move(default_attrs)); // Set Tensor Wrappers - grad_node->SetTensorWrapperX(X); - grad_node->SetTensorWrapperY(Y); + grad_node->SetTensorWrapper_X(X); + grad_node->SetTensorWrapper_Y(Y); grad_node->SetGradOutMeta(X, 0); grad_node->SetGradOutMeta(Y, 1); diff --git a/paddle/fluid/eager/api/manual/fluid_manual/nodes/nodes.h b/paddle/fluid/eager/api/manual/fluid_manual/nodes/nodes.h index 212f9d9f1da19..e8c80e635b155 100644 --- a/paddle/fluid/eager/api/manual/fluid_manual/nodes/nodes.h +++ b/paddle/fluid/eager/api/manual/fluid_manual/nodes/nodes.h @@ -90,65 +90,65 @@ class fused_gate_attentionGradNodeCompat : public egr::GradNodeBase { } // SetX, SetY, ... - void SetTensorWrapperFMHAOut(const paddle::Tensor& FMHAOut) { + void SetTensorWrapper_FMHAOut(const paddle::Tensor& FMHAOut) { FMHAOut_ = egr::TensorWrapper(FMHAOut, false); } - void SetTensorWrapperGateBias(const paddle::Tensor& GateBias) { + void SetTensorWrapper_GateBias(const paddle::Tensor& GateBias) { GateBias_ = egr::TensorWrapper(GateBias, false); } - void SetTensorWrapperGateOut(const paddle::Tensor& GateOut) { + void SetTensorWrapper_GateOut(const paddle::Tensor& GateOut) { GateOut_ = egr::TensorWrapper(GateOut, false); } - void SetTensorWrapperGateWeight(const paddle::Tensor& GateWeight) { + void SetTensorWrapper_GateWeight(const paddle::Tensor& GateWeight) { GateWeight_ = egr::TensorWrapper(GateWeight, false); } - void SetTensorWrapperNonbatchedBias(const paddle::Tensor& NonbatchedBias) { + void SetTensorWrapper_NonbatchedBias(const paddle::Tensor& NonbatchedBias) { NonbatchedBias_ = egr::TensorWrapper(NonbatchedBias, false); } - void SetTensorWrapperSrcMask(const paddle::Tensor& SrcMask) { + void SetTensorWrapper_SrcMask(const paddle::Tensor& SrcMask) { SrcMask_ = egr::TensorWrapper(SrcMask, false); } - void SetTensorWrapperOutLinearBias(const paddle::Tensor& OutLinearBias) { + void SetTensorWrapper_OutLinearBias(const paddle::Tensor& OutLinearBias) { OutLinearBias_ = egr::TensorWrapper(OutLinearBias, false); } - void SetTensorWrapperOutLinearWeight(const paddle::Tensor& OutLinearWeight) { + void SetTensorWrapper_OutLinearWeight(const paddle::Tensor& OutLinearWeight) { OutLinearWeight_ = egr::TensorWrapper(OutLinearWeight, false); } - void SetTensorWrapperQKVTransposeOut(const paddle::Tensor& QKVTransposeOut) { + void SetTensorWrapper_QKVTransposeOut(const paddle::Tensor& QKVTransposeOut) { QKVTransposeOut_ = egr::TensorWrapper(QKVTransposeOut, false); } - void SetTensorWrapperQKVWeight(const paddle::Tensor& QKVWeight) { + void SetTensorWrapper_QKVWeight(const paddle::Tensor& QKVWeight) { QKVWeight_ = egr::TensorWrapper(QKVWeight, false); } - void SetTensorWrapperQuery(const paddle::Tensor& Query) { + void SetTensorWrapper_Query(const paddle::Tensor& Query) { Query_ = egr::TensorWrapper(Query, false); } - void SetTensorWrapperSoftmaxOut(const paddle::Tensor& SoftmaxOut) { + void SetTensorWrapper_SoftmaxOut(const paddle::Tensor& SoftmaxOut) { SoftmaxOut_ = egr::TensorWrapper(SoftmaxOut, false); } - void SetTensorWrapperSoftmaxLse(const paddle::Tensor& SoftmaxLse) { + void SetTensorWrapper_SoftmaxLse(const paddle::Tensor& SoftmaxLse) { SoftmaxLse_ = egr::TensorWrapper(SoftmaxLse, false); } - void SetTensorWrapperKey(const paddle::Tensor& Key) { + void SetTensorWrapper_Key(const paddle::Tensor& Key) { Key_ = egr::TensorWrapper(Key, false); } - void SetTensorWrapperQueryWeight(const paddle::Tensor& QueryWeight) { + void SetTensorWrapper_QueryWeight(const paddle::Tensor& QueryWeight) { QueryWeight_ = egr::TensorWrapper(QueryWeight, false); } - void SetTensorWrapperKeyWeight(const paddle::Tensor& KeyWeight) { + void SetTensorWrapper_KeyWeight(const paddle::Tensor& KeyWeight) { KeyWeight_ = egr::TensorWrapper(KeyWeight, false); } - void SetTensorWrapperValueWeight(const paddle::Tensor& ValueWeight) { + void SetTensorWrapper_ValueWeight(const paddle::Tensor& ValueWeight) { ValueWeight_ = egr::TensorWrapper(ValueWeight, false); } - void SetTensorWrapperQueryTransposeOut( + void SetTensorWrapper_QueryTransposeOut( const paddle::Tensor& QueryTransposeOut) { QueryTransposeOut_ = egr::TensorWrapper(QueryTransposeOut, false); } - void SetTensorWrapperKeyTransposeOut(const paddle::Tensor& KeyTransposeOut) { + void SetTensorWrapper_KeyTransposeOut(const paddle::Tensor& KeyTransposeOut) { KeyTransposeOut_ = egr::TensorWrapper(KeyTransposeOut, false); } - void SetTensorWrapperValueTransposeOut( + void SetTensorWrapper_ValueTransposeOut( const paddle::Tensor& ValueTransposeOut) { ValueTransposeOut_ = egr::TensorWrapper(ValueTransposeOut, false); } @@ -240,63 +240,63 @@ class fused_feedforwardGradNodeCompat : public egr::GradNodeBase { } // SetX, SetY, ... - void SetTensorWrapperDropout1Mask(const paddle::Tensor& Dropout1Mask) { + void SetTensorWrapper_Dropout1Mask(const paddle::Tensor& Dropout1Mask) { Dropout1Mask_ = egr::TensorWrapper(Dropout1Mask, false); } - void SetTensorWrapperDropout1Out(const paddle::Tensor& Dropout1Out) { + void SetTensorWrapper_Dropout1Out(const paddle::Tensor& Dropout1Out) { Dropout1Out_ = egr::TensorWrapper(Dropout1Out, false); } - void SetTensorWrapperDropout2Mask(const paddle::Tensor& Dropout2Mask) { + void SetTensorWrapper_Dropout2Mask(const paddle::Tensor& Dropout2Mask) { Dropout2Mask_ = egr::TensorWrapper(Dropout2Mask, false); } - void SetTensorWrapperDropout2Out(const paddle::Tensor& Dropout2Out) { + void SetTensorWrapper_Dropout2Out(const paddle::Tensor& Dropout2Out) { auto pre_layer_norm = GetAttrWithDefault( attr_map_, default_attr_map_, "pre_layer_norm"); Dropout2Out_ = egr::TensorWrapper(Dropout2Out, pre_layer_norm); } - void SetTensorWrapperLinear1Bias(const paddle::Tensor& Linear1Bias) { + void SetTensorWrapper_Linear1Bias(const paddle::Tensor& Linear1Bias) { Linear1Bias_ = egr::TensorWrapper(Linear1Bias, false); } - void SetTensorWrapperLinear1Out(const paddle::Tensor& Linear1Out) { + void SetTensorWrapper_Linear1Out(const paddle::Tensor& Linear1Out) { Linear1Out_ = egr::TensorWrapper(Linear1Out, false); } - void SetTensorWrapperLinear1Weight(const paddle::Tensor& Linear1Weight) { + void SetTensorWrapper_Linear1Weight(const paddle::Tensor& Linear1Weight) { Linear1Weight_ = egr::TensorWrapper(Linear1Weight, false); } - void SetTensorWrapperLinear2Bias(const paddle::Tensor& Linear2Bias) { + void SetTensorWrapper_Linear2Bias(const paddle::Tensor& Linear2Bias) { Linear2Bias_ = egr::TensorWrapper(Linear2Bias, false); } - void SetTensorWrapperLinear2Weight(const paddle::Tensor& Linear2Weight) { + void SetTensorWrapper_Linear2Weight(const paddle::Tensor& Linear2Weight) { Linear2Weight_ = egr::TensorWrapper(Linear2Weight, false); } - void SetTensorWrapperLn2Bias(const paddle::Tensor& Ln2Bias) { + void SetTensorWrapper_Ln2Bias(const paddle::Tensor& Ln2Bias) { Ln2Bias_ = egr::TensorWrapper(Ln2Bias, false); } - void SetTensorWrapperLn2Mean(const paddle::Tensor& Ln2Mean) { + void SetTensorWrapper_Ln2Mean(const paddle::Tensor& Ln2Mean) { Ln2Mean_ = egr::TensorWrapper(Ln2Mean, false); } - void SetTensorWrapperLn2Scale(const paddle::Tensor& Ln2Scale) { + void SetTensorWrapper_Ln2Scale(const paddle::Tensor& Ln2Scale) { Ln2Scale_ = egr::TensorWrapper(Ln2Scale, false); } - void SetTensorWrapperLn2Variance(const paddle::Tensor& Ln2Variance) { + void SetTensorWrapper_Ln2Variance(const paddle::Tensor& Ln2Variance) { Ln2Variance_ = egr::TensorWrapper(Ln2Variance, false); } - void SetTensorWrapperX(const paddle::Tensor& X) { + void SetTensorWrapper_X(const paddle::Tensor& X) { X_ = egr::TensorWrapper(X, false); } - void SetTensorWrapperLn1Scale(const paddle::Tensor& Ln1Scale) { + void SetTensorWrapper_Ln1Scale(const paddle::Tensor& Ln1Scale) { Ln1Scale_ = egr::TensorWrapper(Ln1Scale, false); } - void SetTensorWrapperLn1Bias(const paddle::Tensor& Ln1Bias) { + void SetTensorWrapper_Ln1Bias(const paddle::Tensor& Ln1Bias) { Ln1Bias_ = egr::TensorWrapper(Ln1Bias, false); } - void SetTensorWrapperLn1Out(const paddle::Tensor& Ln1Out) { + void SetTensorWrapper_Ln1Out(const paddle::Tensor& Ln1Out) { Ln1Out_ = egr::TensorWrapper(Ln1Out, false); } - void SetTensorWrapperLn1Mean(const paddle::Tensor& Ln1Mean) { + void SetTensorWrapper_Ln1Mean(const paddle::Tensor& Ln1Mean) { Ln1Mean_ = egr::TensorWrapper(Ln1Mean, false); } - void SetTensorWrapperLn1Variance(const paddle::Tensor& Ln1Variance) { + void SetTensorWrapper_Ln1Variance(const paddle::Tensor& Ln1Variance) { Ln1Variance_ = egr::TensorWrapper(Ln1Variance, false); } // SetAttrMap @@ -393,90 +393,90 @@ class fused_attentionGradNodeCompat : public egr::GradNodeBase { } // SetX, SetY, ... - void SetTensorWrapperAttnDropoutMaskOut( + void SetTensorWrapper_AttnDropoutMaskOut( const paddle::Tensor& AttnDropoutMaskOut) { AttnDropoutMaskOut_ = egr::TensorWrapper(AttnDropoutMaskOut, false); } - void SetTensorWrapperAttnDropoutOut(const paddle::Tensor& AttnDropoutOut) { + void SetTensorWrapper_AttnDropoutOut(const paddle::Tensor& AttnDropoutOut) { AttnDropoutOut_ = egr::TensorWrapper(AttnDropoutOut, false); } - void SetTensorWrapperBiasDropoutResidualOut( + void SetTensorWrapper_BiasDropoutResidualOut( const paddle::Tensor& BiasDropoutResidualOut) { BiasDropoutResidualOut_ = egr::TensorWrapper(BiasDropoutResidualOut, false); } - void SetTensorWrapperDropoutMaskOut(const paddle::Tensor& DropoutMaskOut) { + void SetTensorWrapper_DropoutMaskOut(const paddle::Tensor& DropoutMaskOut) { DropoutMaskOut_ = egr::TensorWrapper(DropoutMaskOut, false); } - void SetTensorWrapperFMHAOut(const paddle::Tensor& FMHAOut) { + void SetTensorWrapper_FMHAOut(const paddle::Tensor& FMHAOut) { FMHAOut_ = egr::TensorWrapper(FMHAOut, false); } - void SetTensorWrapperLn2Bias(const paddle::Tensor& Ln2Bias) { + void SetTensorWrapper_Ln2Bias(const paddle::Tensor& Ln2Bias) { Ln2Bias_ = egr::TensorWrapper(Ln2Bias, false); } - void SetTensorWrapperLn2Mean(const paddle::Tensor& Ln2Mean) { + void SetTensorWrapper_Ln2Mean(const paddle::Tensor& Ln2Mean) { Ln2Mean_ = egr::TensorWrapper(Ln2Mean, false); } - void SetTensorWrapperLn2Scale(const paddle::Tensor& Ln2Scale) { + void SetTensorWrapper_Ln2Scale(const paddle::Tensor& Ln2Scale) { Ln2Scale_ = egr::TensorWrapper(Ln2Scale, false); } - void SetTensorWrapperLn2Variance(const paddle::Tensor& Ln2Variance) { + void SetTensorWrapper_Ln2Variance(const paddle::Tensor& Ln2Variance) { Ln2Variance_ = egr::TensorWrapper(Ln2Variance, false); } - void SetTensorWrapperOutLinearBias(const paddle::Tensor& OutLinearBias) { + void SetTensorWrapper_OutLinearBias(const paddle::Tensor& OutLinearBias) { OutLinearBias_ = egr::TensorWrapper(OutLinearBias, false); } - void SetTensorWrapperOutLinearOut(const paddle::Tensor& OutLinearOut) { + void SetTensorWrapper_OutLinearOut(const paddle::Tensor& OutLinearOut) { OutLinearOut_ = egr::TensorWrapper(OutLinearOut, true); } - void SetTensorWrapperOutLinearW(const paddle::Tensor& OutLinearW) { + void SetTensorWrapper_OutLinearW(const paddle::Tensor& OutLinearW) { OutLinearW_ = egr::TensorWrapper(OutLinearW, false); } - void SetTensorWrapperQKOut(const paddle::Tensor& QKOut) { + void SetTensorWrapper_QKOut(const paddle::Tensor& QKOut) { QKOut_ = egr::TensorWrapper(QKOut, true); } - void SetTensorWrapperQKTVOut(const paddle::Tensor& QKTVOut) { + void SetTensorWrapper_QKTVOut(const paddle::Tensor& QKTVOut) { QKTVOut_ = egr::TensorWrapper(QKTVOut, true); } - void SetTensorWrapperQKVBias(const paddle::Tensor& QKVBias) { + void SetTensorWrapper_QKVBias(const paddle::Tensor& QKVBias) { QKVBias_ = egr::TensorWrapper(QKVBias, false); } - void SetTensorWrapperQKVBiasOut(const paddle::Tensor& QKVBiasOut) { + void SetTensorWrapper_QKVBiasOut(const paddle::Tensor& QKVBiasOut) { QKVBiasOut_ = egr::TensorWrapper(QKVBiasOut, true); } - void SetTensorWrapperQKVOut(const paddle::Tensor& QKVOut) { + void SetTensorWrapper_QKVOut(const paddle::Tensor& QKVOut) { QKVOut_ = egr::TensorWrapper(QKVOut, true); } - void SetTensorWrapperQKVW(const paddle::Tensor& QKVW) { + void SetTensorWrapper_QKVW(const paddle::Tensor& QKVW) { QKVW_ = egr::TensorWrapper(QKVW, false); } - void SetTensorWrapperSoftmaxOut(const paddle::Tensor& SoftmaxOut) { + void SetTensorWrapper_SoftmaxOut(const paddle::Tensor& SoftmaxOut) { SoftmaxOut_ = egr::TensorWrapper(SoftmaxOut, false); } - void SetTensorWrapperSrcMask(const paddle::Tensor& SrcMask) { + void SetTensorWrapper_SrcMask(const paddle::Tensor& SrcMask) { SrcMask_ = egr::TensorWrapper(SrcMask, true); } - void SetTensorWrapperSrcMaskOut(const paddle::Tensor& SrcMaskOut) { + void SetTensorWrapper_SrcMaskOut(const paddle::Tensor& SrcMaskOut) { SrcMaskOut_ = egr::TensorWrapper(SrcMaskOut, false); } - void SetTensorWrapperTransposeOut2(const paddle::Tensor& TransposeOut2) { + void SetTensorWrapper_TransposeOut2(const paddle::Tensor& TransposeOut2) { TransposeOut2_ = egr::TensorWrapper(TransposeOut2, false); } - void SetTensorWrapperX(const paddle::Tensor& X) { + void SetTensorWrapper_X(const paddle::Tensor& X) { X_ = egr::TensorWrapper(X, false); } - void SetTensorWrapperLnScale(const paddle::Tensor& LnScale) { + void SetTensorWrapper_LnScale(const paddle::Tensor& LnScale) { LnScale_ = egr::TensorWrapper(LnScale, false); } - void SetTensorWrapperLnBias(const paddle::Tensor& LnBias) { + void SetTensorWrapper_LnBias(const paddle::Tensor& LnBias) { LnBias_ = egr::TensorWrapper(LnBias, false); } - void SetTensorWrapperLnOut(const paddle::Tensor& LnOut) { + void SetTensorWrapper_LnOut(const paddle::Tensor& LnOut) { LnOut_ = egr::TensorWrapper(LnOut, false); } - void SetTensorWrapperLnMean(const paddle::Tensor& LnMean) { + void SetTensorWrapper_LnMean(const paddle::Tensor& LnMean) { LnMean_ = egr::TensorWrapper(LnMean, false); } - void SetTensorWrapperLnVariance(const paddle::Tensor& LnVariance) { + void SetTensorWrapper_LnVariance(const paddle::Tensor& LnVariance) { LnVariance_ = egr::TensorWrapper(LnVariance, false); } @@ -563,10 +563,10 @@ class fused_gemm_epilogueGradNodeCompat : public egr::GradNodeBase { } // SetX, SetY, ... - void SetTensorWrapperX(const paddle::Tensor& X) { + void SetTensorWrapper_X(const paddle::Tensor& X) { X_ = egr::TensorWrapper(X, false); } - void SetTensorWrapperY(const paddle::Tensor& Y) { + void SetTensorWrapper_Y(const paddle::Tensor& Y) { Y_ = egr::TensorWrapper(Y, false); } @@ -640,32 +640,32 @@ class fused_bias_dropout_residual_layer_normGradNodeCompat } // SetX, SetY, ... - void SetTensorWrapperBias(const paddle::Tensor& Bias) { + void SetTensorWrapper_Bias(const paddle::Tensor& Bias) { Bias_ = egr::TensorWrapper(Bias, false); } - void SetTensorWrapperBiasDropoutResidualOut( + void SetTensorWrapper_BiasDropoutResidualOut( const paddle::Tensor& BiasDropoutResidualOut) { BiasDropoutResidualOut_ = egr::TensorWrapper(BiasDropoutResidualOut, false); } - void SetTensorWrapperDropoutMaskOut(const paddle::Tensor& DropoutMaskOut) { + void SetTensorWrapper_DropoutMaskOut(const paddle::Tensor& DropoutMaskOut) { DropoutMaskOut_ = egr::TensorWrapper(DropoutMaskOut, false); } - void SetTensorWrapperLnBias(const paddle::Tensor& LnBias) { + void SetTensorWrapper_LnBias(const paddle::Tensor& LnBias) { LnBias_ = egr::TensorWrapper(LnBias, false); } - void SetTensorWrapperLnMean(const paddle::Tensor& LnMean) { + void SetTensorWrapper_LnMean(const paddle::Tensor& LnMean) { LnMean_ = egr::TensorWrapper(LnMean, false); } - void SetTensorWrapperLnScale(const paddle::Tensor& LnScale) { + void SetTensorWrapper_LnScale(const paddle::Tensor& LnScale) { LnScale_ = egr::TensorWrapper(LnScale, false); } - void SetTensorWrapperLnVariance(const paddle::Tensor& LnVariance) { + void SetTensorWrapper_LnVariance(const paddle::Tensor& LnVariance) { LnVariance_ = egr::TensorWrapper(LnVariance, false); } - void SetTensorWrapperResidual(const paddle::Tensor& Residual) { + void SetTensorWrapper_Residual(const paddle::Tensor& Residual) { Residual_ = egr::TensorWrapper(Residual, false); } - void SetTensorWrapperX(const paddle::Tensor& X) { + void SetTensorWrapper_X(const paddle::Tensor& X) { X_ = egr::TensorWrapper(X, false); } diff --git a/paddle/fluid/eager/auto_code_generator/eager_generator.cc b/paddle/fluid/eager/auto_code_generator/eager_generator.cc index b9e04b3e318ac..66b4d05f68bf0 100644 --- a/paddle/fluid/eager/auto_code_generator/eager_generator.cc +++ b/paddle/fluid/eager/auto_code_generator/eager_generator.cc @@ -1204,7 +1204,7 @@ static std::string GenerateGradNodeCreationContent( for (auto& kv : grad_ins_fwd_slotname_map) { const std::string& tensor_wrapper_name = kv.second; const char* SET_TENSOR_WRAPPER_TEMPLATE = - " grad_node->SetTensorWrapper%s(%s);\n"; + " grad_node->SetTensorWrapper_%s(%s);\n"; // Replace output directly with input in inplace op. if (!forward_inplace_map.empty() && forward_inplace_map.count(tensor_wrapper_name)) { @@ -2941,7 +2941,7 @@ static std::string GenerateGradNodeHeaderContents( CLEAR_TENSOR_WRAPPER_TEMPLATE, struct_tensor_wrapper_name); } const char* SET_TENSOR_WRAPPER_TEMPLATE = - " void SetTensorWrapper%s(%s) {\n %s\n }\n"; + " void SetTensorWrapper_%s(%s) {\n %s\n }\n"; set_tensor_wrappers_str += paddle::string::Sprintf(SET_TENSOR_WRAPPER_TEMPLATE, tensor_wrapper_name, diff --git a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py index e1ad1a0dc81b2..dad46949d70ea 100644 --- a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py +++ b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py @@ -135,12 +135,12 @@ def ParseArguments(): ###################### # Code Gen Templates # ###################### -SET_PLAIN_TENSOR_WRAPPER_TEMPLATE = """ void SetTensorWrapper{}(const paddle::Tensor& {}) {{ +SET_PLAIN_TENSOR_WRAPPER_TEMPLATE = """ void SetTensorWrapper_{}(const paddle::Tensor& {}) {{ {} = egr::TensorWrapper({}, {}); }} """ -SET_VECTOR_TENSOR_WRAPPER_TEMPLATE = """ void SetTensorWrapper{}(const std::vector& {}) {{ +SET_VECTOR_TENSOR_WRAPPER_TEMPLATE = """ void SetTensorWrapper_{}(const std::vector& {}) {{ for(const auto& eager_tensor : {}) {{ {}.emplace_back(egr::TensorWrapper(eager_tensor, {})); }}; @@ -161,7 +161,7 @@ def ParseArguments(): }} """ -SET_ATTR_METHOD_TEMPLATE = """ void SetAttribute{}({} {}) {{ +SET_ATTR_METHOD_TEMPLATE = """ void SetAttribute_{}({} {}) {{ {} = {}; }} """ @@ -1062,10 +1062,10 @@ def GenerateNodeCreationCodes(self, for_backward=False, is_inplaced=False): for name, _, default_val_attr, _ in backward_attrs_list: if name in forward_attrs_name_set: set_attributes = ( - f"{indent}grad_node->SetAttribute{name}({name});" + f"{indent}grad_node->SetAttribute_{name}({name});" ) else: - set_attributes = f"{indent}grad_node->SetAttribute{name}({default_val_attr});" + set_attributes = f"{indent}grad_node->SetAttribute_{name}({default_val_attr});" set_attributes_list.append(set_attributes) set_attributes_str = "\n".join(set_attributes_list) @@ -1089,7 +1089,7 @@ def GenerateNodeCreationCodes(self, for_backward=False, is_inplaced=False): if is_inplace_input: set_tensor_wrappers = """{indent}if({name}) { auto {name}_clone = paddle::experimental::assign({name}); - grad_node->SetTensorWrapper{name}(*{name}_clone);}""".format_map( + grad_node->SetTensorWrapper_{name}(*{name}_clone);}""".format_map( {"indent": indent, "name": name} ) else: @@ -1100,16 +1100,16 @@ def GenerateNodeCreationCodes(self, for_backward=False, is_inplaced=False): or (name in self.optional_inputs) ): if for_backward is False: - set_tensor_wrappers = f"{indent}if({name}) grad_node->SetTensorWrapper{name}(*{name});" + set_tensor_wrappers = f"{indent}if({name}) grad_node->SetTensorWrapper_{name}(*{name});" else: - set_tensor_wrappers = f"{indent}if({name}_optional) grad_node->SetTensorWrapper{name}(*{name}_optional);" + set_tensor_wrappers = f"{indent}if({name}_optional) grad_node->SetTensorWrapper_{name}(*{name}_optional);" else: need_pre_contiguous_set.add(name) - set_tensor_wrappers = f"{indent}if({name}) grad_node->SetTensorWrapper{name}(*{name}_tmp);" + set_tensor_wrappers = f"{indent}if({name}) grad_node->SetTensorWrapper_{name}(*{name}_tmp);" else: if is_inplace_input: - set_tensor_wrappers = f"{indent}auto {name}_clone = paddle::experimental::assign({name});\n{indent}grad_node->SetTensorWrapper{name}({name}_clone);" + set_tensor_wrappers = f"{indent}auto {name}_clone = paddle::experimental::assign({name});\n{indent}grad_node->SetTensorWrapper_{name}({name}_clone);" else: if ( (forward_api_name in strided_op_list) @@ -1117,10 +1117,10 @@ def GenerateNodeCreationCodes(self, for_backward=False, is_inplaced=False): or IsVectorTensorType(atype) or (name in self.optional_inputs) ): - set_tensor_wrappers = f"{indent}grad_node->SetTensorWrapper{name}({name});" + set_tensor_wrappers = f"{indent}grad_node->SetTensorWrapper_{name}({name});" else: need_pre_contiguous_set.add(name) - set_tensor_wrappers = f"{indent}grad_node->SetTensorWrapper{name}({name}_tmp);" + set_tensor_wrappers = f"{indent}grad_node->SetTensorWrapper_{name}({name}_tmp);" set_input_tensor_wrappers_list.append(set_tensor_wrappers) else: # Forwad's output as backward's input if num_fwd_outputs > 1: @@ -1130,7 +1130,7 @@ def GenerateNodeCreationCodes(self, for_backward=False, is_inplaced=False): ), AssertMessage(name, forward_outputs_position_map.keys()) set_tensor_wrappers = ( - f"{indent}grad_node->SetTensorWrapper{name}({name});" + f"{indent}grad_node->SetTensorWrapper_{name}({name});" ) set_output_tensor_wrappers_list.append(set_tensor_wrappers) set_input_tensor_wrappers_str = "\n".join( diff --git a/paddle/fluid/pybind/eager_method.cc b/paddle/fluid/pybind/eager_method.cc index 6611d108adcf5..2094fef07a873 100644 --- a/paddle/fluid/pybind/eager_method.cc +++ b/paddle/fluid/pybind/eager_method.cc @@ -1783,13 +1783,13 @@ static PyObject* tensor__setitem_dygraph(TensorObject* self, grad_node = std::shared_ptr( new SetValueWithTensorGradNode(1, 2)); // NOLINT - grad_node->SetAttributestarts(slice_starts); - grad_node->SetAttributeends(slice_ends); - grad_node->SetAttributesteps(slice_strides); - grad_node->SetAttributeaxes(slice_axes); - grad_node->SetAttributedecrease_axes(decrease_axis); - grad_node->SetAttributenone_axes(none_axes); - grad_node->SetTensorWrappervalues(values_tmp); + grad_node->SetAttribute_starts(slice_starts); + grad_node->SetAttribute_ends(slice_ends); + grad_node->SetAttribute_steps(slice_strides); + grad_node->SetAttribute_axes(slice_axes); + grad_node->SetAttribute_decrease_axes(decrease_axis); + grad_node->SetAttribute_none_axes(none_axes); + grad_node->SetTensorWrapper_values(values_tmp); paddle::memory::LogDeviceMemoryStats( egr::Controller::Instance().GetExpectedPlace(), From 52498e2a302cd6f09c126219c04c4879182c26e2 Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 23 Feb 2024 12:10:12 +0800 Subject: [PATCH 018/282] Update test_auto_parallel_partitioner.py, test=document_fix (#61982) --- test/legacy_test/test_auto_parallel_partitioner.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/legacy_test/test_auto_parallel_partitioner.py b/test/legacy_test/test_auto_parallel_partitioner.py index 1a0d70c232b36..da402d93ce58e 100644 --- a/test/legacy_test/test_auto_parallel_partitioner.py +++ b/test/legacy_test/test_auto_parallel_partitioner.py @@ -480,7 +480,7 @@ def test_mlp_mp(self): self.assertTrue( distributed_attr_check_for_program(dist_main_prog, dist_context) ) - # check distribured attr for dist op + # check distributed attr for dist op serial_op_idx = [1, 4] dist_op_idx = [[1, 2], [4, 5]] self.assertTrue( @@ -573,7 +573,7 @@ def test_mlp_dp_mp(self): self.assertTrue( distributed_attr_check_for_program(dist_main_prog, dist_context) ) - # check distribured attr for dist op + # check distributed attr for dist op serial_op_idx = [1, 4] dist_op_idx = [[1, 2], [4, 5]] self.assertTrue( @@ -869,7 +869,7 @@ def test_attn_mp(self): self.assertTrue( distributed_attr_check_for_program(dist_main_prog, dist_context) ) - # check distribured attr for dist op + # check distributed attr for dist op serial_op_idx = [0, 4, 6, 18] dist_op_idx = [[0, 1], [4, 5], [6, 7], [18, 19]] @@ -976,7 +976,7 @@ def test_attn_dp_mp(self): self.assertTrue( distributed_attr_check_for_program(dist_main_prog, dist_context) ) - # check distribured attr for dist op + # check distributed attr for dist op serial_op_idx = [0, 4, 6, 18] dist_op_idx = [[0, 1], [4, 5], [6, 7], [18, 19]] @@ -1364,7 +1364,7 @@ def test_decoder_dp_mp(self): self.assertTrue( distributed_attr_check_for_program(dist_main_prog, dist_context) ) - # check distribured attr + # check distributed attr serial_op_idx = [0, 5, 9, 11, 24, 29, 32] dist_op_idx = [ [2, 3], From 508c717ed4e54adc05fb3eeacacfccc45ebd741d Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 23 Feb 2024 12:11:03 +0800 Subject: [PATCH 019/282] Fix some typos (dtyte, paramerter, etc.) (#61996) --- test/legacy_test/test_adadelta_op.py | 2 +- test/legacy_test/test_adagrad_op.py | 4 +- test/legacy_test/test_adam_op.py | 2 +- test/legacy_test/test_adamax_op.py | 2 +- test/legacy_test/test_adamw_op.py | 4 +- test/legacy_test/test_add_n_op.py | 10 ++-- test/legacy_test/test_arange.py | 2 +- test/legacy_test/test_backward.py | 6 +-- test/legacy_test/test_bicubic_interp_v2_op.py | 8 ++-- test/legacy_test/test_checkpoint_saver.py | 2 +- test/legacy_test/test_collective_api_base.py | 6 +-- test/legacy_test/test_conv2d_api.py | 2 +- test/legacy_test/test_conv3d_layer.py | 8 ++-- test/legacy_test/test_conv3d_op.py | 2 +- .../test_conv3d_transpose_layer.py | 8 ++-- .../test_fused_attention_op_api.py | 4 +- ...bias_dropout_residual_layer_norm_op_api.py | 4 +- test/legacy_test/test_log_softmax.py | 2 +- .../test_paddle_save_load_binary.py | 2 +- test/legacy_test/test_scatter_nd_op.py | 8 ++-- test/legacy_test/test_set_value_op.py | 28 +++++------ test/legacy_test/test_sgd_op.py | 4 +- test/legacy_test/test_sgd_op_bf16.py | 2 +- test/legacy_test/test_signal.py | 2 +- test/legacy_test/test_slice_scatter.py | 8 ++-- test/legacy_test/test_softmax_op.py | 6 +-- .../test_softmax_with_cross_entropy_op.py | 2 +- test/legacy_test/test_sparse_attention_op.py | 6 +-- test/legacy_test/test_split_op.py | 24 ++++++---- test/legacy_test/test_static_pylayer.py | 2 +- test/legacy_test/test_static_save_load.py | 46 +++++++++---------- .../legacy_test/test_static_save_load_bf16.py | 4 +- .../test_static_save_load_large.py | 2 +- ...tatic_shape_inferrence_for_shape_tensor.py | 2 +- test/legacy_test/test_sum_op.py | 12 ++--- test/legacy_test/test_svd_op.py | 2 +- test/legacy_test/test_sync_batch_norm_op.py | 2 +- test/legacy_test/test_tensor.py | 2 +- test/legacy_test/test_tensor_register_hook.py | 2 +- test/legacy_test/test_tensor_uva.py | 2 +- test/legacy_test/test_traced_layer_err_msg.py | 2 +- test/legacy_test/test_trans_layout_op.py | 8 ++-- test/legacy_test/test_transformer_api.py | 4 +- test/legacy_test/test_tril_triu_op.py | 2 +- .../test_truncated_gaussian_random_op.py | 2 +- test/legacy_test/test_vision_models.py | 2 +- test/legacy_test/test_viterbi_decode_op.py | 6 +-- test/legacy_test/test_warpctc_op.py | 2 +- test/legacy_test/test_where_op.py | 4 +- test/legacy_test/test_while_loop_op.py | 2 +- test/legacy_test/test_while_op.py | 2 +- test/legacy_test/test_zeros_like_op.py | 2 +- test/mkldnn/test_elementwise_add_mkldnn_op.py | 8 ++-- test/mkldnn/test_fused_vit_attention.py | 4 +- test/mkldnn/test_fusion_gru_bf16_mkldnn_op.py | 2 +- 55 files changed, 153 insertions(+), 147 deletions(-) diff --git a/test/legacy_test/test_adadelta_op.py b/test/legacy_test/test_adadelta_op.py index 9202d6e2aa80f..c7ae043fbac6f 100644 --- a/test/legacy_test/test_adadelta_op.py +++ b/test/legacy_test/test_adadelta_op.py @@ -236,7 +236,7 @@ def test_adadelta_dygraph(self): adam.clear_gradients() -class TestAdadeltaOpMultiPrecison(unittest.TestCase): +class TestAdadeltaOpMultiPrecision(unittest.TestCase): def _test_adadelta_op_dygraph_place_amp(self, place, use_amp=False): import paddle diff --git a/test/legacy_test/test_adagrad_op.py b/test/legacy_test/test_adagrad_op.py index b1aab3c903248..4d356e89d4784 100644 --- a/test/legacy_test/test_adagrad_op.py +++ b/test/legacy_test/test_adagrad_op.py @@ -130,7 +130,7 @@ def check_with_place(self, place): param_array = np.full((height, row_numel), 5.0).astype("float32") param.set(param_array, place) - # create and initialize LeraningRate Variable + # create and initialize LearningRate Variable lr = scope.var('LearningRate').get_tensor() lr_array = np.full((1), 2.0).astype("float32") lr.set(lr_array, place) @@ -211,7 +211,7 @@ def test_sparse_adagrad(self): self.check_with_place(place) -class TestAdagradOpMultiPrecison(unittest.TestCase): +class TestAdagradOpMultiPrecision(unittest.TestCase): def _test_adagrad_op_dygraph_place_amp(self, place, use_amp=False): import paddle diff --git a/test/legacy_test/test_adam_op.py b/test/legacy_test/test_adam_op.py index c06e249a874e0..0693d4f664356 100644 --- a/test/legacy_test/test_adam_op.py +++ b/test/legacy_test/test_adam_op.py @@ -758,7 +758,7 @@ def test_adam_op_with_state_dict(self): state_dict = adam.state_dict() adam.set_state_dict(state_dict) - # leanrning_rate is Tensor + # learning_rate is Tensor with self.assertRaises(TypeError): learning_rate = np.array([0.01]).astype("float32") learning_rate = paddle.to_tensor(learning_rate) diff --git a/test/legacy_test/test_adamax_op.py b/test/legacy_test/test_adamax_op.py index 4087e75398266..48549d6275a9f 100644 --- a/test/legacy_test/test_adamax_op.py +++ b/test/legacy_test/test_adamax_op.py @@ -239,7 +239,7 @@ def test_adamax_op_invalid_input(self): ) -class TestAdamaxOpMultiPrecison(unittest.TestCase): +class TestAdamaxOpMultiPrecision(unittest.TestCase): def _test_adamax_op_dygraph_place_amp(self, place, use_amp=False): import paddle diff --git a/test/legacy_test/test_adamw_op.py b/test/legacy_test/test_adamw_op.py index 752e8076d3b14..1c901e8d4baf5 100644 --- a/test/legacy_test/test_adamw_op.py +++ b/test/legacy_test/test_adamw_op.py @@ -404,7 +404,7 @@ def test_adamw_op_dygraph_bypassing_step(self): adam.clear_gradients() -class TestAdamWOpMultiPrecisonWithMainGrad(unittest.TestCase): +class TestAdamWOpMultiPrecisionWithMainGrad(unittest.TestCase): def _test_adamw_op_dygraph_place_amp_with_maingrad( self, place, shape, use_main_grad ): @@ -543,7 +543,7 @@ def test_main(self): ) -class TestAdamWOpMultiPrecison(unittest.TestCase): +class TestAdamWOpMultiPrecision(unittest.TestCase): def _test_adamw_op_dygraph_place_amp(self, place, use_amp=False): paddle.disable_static() paddle.seed(10) diff --git a/test/legacy_test/test_add_n_op.py b/test/legacy_test/test_add_n_op.py index e543b4f05c74b..60981b9cd02f0 100644 --- a/test/legacy_test/test_add_n_op.py +++ b/test/legacy_test/test_add_n_op.py @@ -70,15 +70,15 @@ def test_add_n_api(self): if not paddle.is_compiled_with_cuda(): return dtypes = ['float32', 'complex64', 'complex128'] - for dtyte in dtypes: - if dtyte == 'complex64' or dtyte == 'complex128': + for dtype in dtypes: + if dtype == 'complex64' or dtype == 'complex128': self.x_np = ( np.random.random([self.l, 16, 256]) + 1j * np.random.random([self.l, 16, 256]) - ).astype(dtyte) + ).astype(dtype) - y_np_32, x_g_np_32 = self.check_main(self.x_np, dtyte) - y_np_gt = np.sum(self.x_np, axis=0).astype(dtyte) + y_np_32, x_g_np_32 = self.check_main(self.x_np, dtype) + y_np_gt = np.sum(self.x_np, axis=0).astype(dtype) np.testing.assert_allclose(y_np_32, y_np_gt, rtol=1e-06) diff --git a/test/legacy_test/test_arange.py b/test/legacy_test/test_arange.py index bb4bb0f430b68..fbcc6eb421da5 100644 --- a/test/legacy_test/test_arange.py +++ b/test/legacy_test/test_arange.py @@ -59,7 +59,7 @@ def init_config(self): self.case = (0, 5, 1) -class TestFloa16ArangeOp(TestArangeOp): +class TestFloat16ArangeOp(TestArangeOp): def init_config(self): self.dtype = np.float16 self.python_api = paddle.arange diff --git a/test/legacy_test/test_backward.py b/test/legacy_test/test_backward.py index 2ae9ede04987a..04aeadc038213 100644 --- a/test/legacy_test/test_backward.py +++ b/test/legacy_test/test_backward.py @@ -378,12 +378,12 @@ def callback(block, context): class TestGradientsWithOptimizer(unittest.TestCase): - def _check_grad_op_name(self, forward_list, optimiezed_list): + def _check_grad_op_name(self, forward_list, optimized_list): backward_list = [op + "_grad" for op in reversed(forward_list)] - idx = optimiezed_list.index(backward_list[0], len(backward_list)) + idx = optimized_list.index(backward_list[0], len(backward_list)) self.assertListEqual( - backward_list, optimiezed_list[idx : idx + len(backward_list)] + backward_list, optimized_list[idx : idx + len(backward_list)] ) def test_gradient_with_optimizer(self): diff --git a/test/legacy_test/test_bicubic_interp_v2_op.py b/test/legacy_test/test_bicubic_interp_v2_op.py index 86b998c635648..d2d0092f1e9f6 100644 --- a/test/legacy_test/test_bicubic_interp_v2_op.py +++ b/test/legacy_test/test_bicubic_interp_v2_op.py @@ -674,7 +674,7 @@ def test_case(self): class TestBicubicOpError(unittest.TestCase): def test_imperative_errors(self): - # the input of interpoalte must be Variable. + # the input of interpolate must be Variable. x1 = base.create_lod_tensor( np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], base.CPUPlace() ) @@ -687,7 +687,7 @@ def test_mode_type(): ) out = interpolate( - x, size=[12, 12], mode='UNKONWN', align_corners=False + x, size=[12, 12], mode='UNKNOWN', align_corners=False ) def test_input_shape(): @@ -696,7 +696,7 @@ def test_input_shape(): x, size=[12, 12], mode='BICUBIC', align_corners=False ) - def test_align_corcers(): + def test_align_corners(): x = paddle.static.data( name="x", shape=[2, 3, 6, 6], dtype="float32" ) @@ -887,7 +887,7 @@ def test_input_shape_1(): self.assertRaises(ValueError, test_mode_type) self.assertRaises(ValueError, test_input_shape) - self.assertRaises(TypeError, test_align_corcers) + self.assertRaises(TypeError, test_align_corners) self.assertRaises(ValueError, test_attr_data_format) self.assertRaises(TypeError, test_actual_shape) self.assertRaises(ValueError, test_scale_value) diff --git a/test/legacy_test/test_checkpoint_saver.py b/test/legacy_test/test_checkpoint_saver.py index 643ea78816579..0390d4f8c60f0 100644 --- a/test/legacy_test/test_checkpoint_saver.py +++ b/test/legacy_test/test_checkpoint_saver.py @@ -18,7 +18,7 @@ from paddle.distributed.fleet.utils.fs import HDFSClient -class CheckpointerSaverTest(unittest.TestCase): +class CheckpointSaverTest(unittest.TestCase): def test(self): fs = HDFSClient("/usr/local/hadoop-2.7.7", None) dir_path = "./checkpointsaver_test" diff --git a/test/legacy_test/test_collective_api_base.py b/test/legacy_test/test_collective_api_base.py index 8f6a382297a1f..f71b524344aec 100644 --- a/test/legacy_test/test_collective_api_base.py +++ b/test/legacy_test/test_collective_api_base.py @@ -611,19 +611,19 @@ def convertbf16(origin): result1 = [] result2 = [] - def is_empyt_list(x): + def is_empty_list(x): if isinstance(x, list) and len(x) == 0: return True return False for i in range(tot_expert): for arr in output1[i]: - if is_empyt_list(arr): + if is_empty_list(arr): continue result1.append(arr) for i in range(tot_expert): for arr in output2[i]: - if is_empyt_list(arr): + if is_empty_list(arr): continue result2.append(arr) diff --git a/test/legacy_test/test_conv2d_api.py b/test/legacy_test/test_conv2d_api.py index 9d2398a5782ca..433dafbcd7fed 100644 --- a/test/legacy_test/test_conv2d_api.py +++ b/test/legacy_test/test_conv2d_api.py @@ -201,7 +201,7 @@ def run_5(): self.assertRaises(ValueError, run_5) - # ValueError: channel dimmention + # ValueError: channel dimension x = paddle.static.data( name="x", shape=[2, 5, 5, -1], diff --git a/test/legacy_test/test_conv3d_layer.py b/test/legacy_test/test_conv3d_layer.py index d514f56c2631a..55c4c569568aa 100644 --- a/test/legacy_test/test_conv3d_layer.py +++ b/test/legacy_test/test_conv3d_layer.py @@ -27,7 +27,7 @@ def __init__( self, methodName='runTest', batch_size=4, - spartial_shape=(8, 8, 8), + spatial_shape=(8, 8, 8), num_channels=6, num_filters=8, filter_size=3, @@ -43,7 +43,7 @@ def __init__( self.batch_size = batch_size self.num_channels = num_channels self.num_filters = num_filters - self.spartial_shape = spartial_shape + self.spatial_shape = spatial_shape self.filter_size = filter_size self.padding = padding @@ -58,13 +58,13 @@ def setUp(self): self.channel_last = self.data_format == "NDHWC" if self.channel_last: input_shape = ( - (self.batch_size,) + self.spartial_shape + (self.num_channels,) + (self.batch_size,) + self.spatial_shape + (self.num_channels,) ) else: input_shape = ( self.batch_size, self.num_channels, - ) + self.spartial_shape + ) + self.spatial_shape self.input = np.random.randn(*input_shape).astype(self.dtype) if isinstance(self.filter_size, int): diff --git a/test/legacy_test/test_conv3d_op.py b/test/legacy_test/test_conv3d_op.py index cfa39de922075..cd0d6449020ca 100644 --- a/test/legacy_test/test_conv3d_op.py +++ b/test/legacy_test/test_conv3d_op.py @@ -1212,7 +1212,7 @@ def run_5(): self.assertRaises(ValueError, run_5) - # ValueError: channel dimmention + # ValueError: channel dimension x = paddle.static.data( name="x", shape=[2, 5, 5, 5, -1], diff --git a/test/legacy_test/test_conv3d_transpose_layer.py b/test/legacy_test/test_conv3d_transpose_layer.py index 7624253ba6031..910513f6b4176 100644 --- a/test/legacy_test/test_conv3d_transpose_layer.py +++ b/test/legacy_test/test_conv3d_transpose_layer.py @@ -27,7 +27,7 @@ def __init__( self, methodName='runTest', batch_size=2, - spartial_shape=(8, 8, 8), + spatial_shape=(8, 8, 8), num_channels=6, num_filters=8, filter_size=3, @@ -44,7 +44,7 @@ def __init__( self.batch_size = batch_size self.num_channels = num_channels self.num_filters = num_filters - self.spartial_shape = spartial_shape + self.spatial_shape = spatial_shape self.filter_size = filter_size self.output_size = output_size @@ -60,13 +60,13 @@ def setUp(self): self.channel_last = self.data_format == "NDHWC" if self.channel_last: input_shape = ( - (self.batch_size,) + self.spartial_shape + (self.num_channels,) + (self.batch_size,) + self.spatial_shape + (self.num_channels,) ) else: input_shape = ( self.batch_size, self.num_channels, - ) + self.spartial_shape + ) + self.spatial_shape self.input = np.random.randn(*input_shape).astype(self.dtype) if isinstance(self.filter_size, int): diff --git a/test/legacy_test/test_fused_attention_op_api.py b/test/legacy_test/test_fused_attention_op_api.py index 1570c0b0dd733..e3ea9491e3782 100644 --- a/test/legacy_test/test_fused_attention_op_api.py +++ b/test/legacy_test/test_fused_attention_op_api.py @@ -53,9 +53,9 @@ def layer_norm(x, has_scale, has_bias, weight, bias, epsilon=1e-05): batch_size, src_len, d_model = x.shape x = x.reshape((batch_size * src_len, d_model)) mu = np.mean(x, axis=1, keepdims=True) - sigma_squar = np.sum(np.square(x - mu), axis=1) / d_model + sigma_square = np.sum(np.square(x - mu), axis=1) / d_model x1_up = x - mu - x1_down_1 = sigma_squar + epsilon + x1_down_1 = sigma_square + epsilon x1_down = np.sqrt(x1_down_1) x1_down = x1_down.reshape((x1_down.shape[0], 1)) x1 = x1_up / x1_down diff --git a/test/legacy_test/test_fused_bias_dropout_residual_layer_norm_op_api.py b/test/legacy_test/test_fused_bias_dropout_residual_layer_norm_op_api.py index dae2f2ba61c88..9efa1cd354cb3 100644 --- a/test/legacy_test/test_fused_bias_dropout_residual_layer_norm_op_api.py +++ b/test/legacy_test/test_fused_bias_dropout_residual_layer_norm_op_api.py @@ -27,9 +27,9 @@ def layer_norm(x, has_scale, has_bias, weight, bias, epsilon=1e-05): batch_size, src_len, d_model = x.shape x = x.reshape((batch_size * src_len, d_model)) mu = np.mean(x, axis=1, keepdims=True) - sigma_squar = np.sum(np.square(x - mu), axis=1) / d_model + sigma_square = np.sum(np.square(x - mu), axis=1) / d_model x1_up = x - mu - x1_down_1 = sigma_squar + epsilon + x1_down_1 = sigma_square + epsilon x1_down = np.sqrt(x1_down_1) x1_down = x1_down.reshape((x1_down.shape[0], 1)) x1 = x1_up / x1_down diff --git a/test/legacy_test/test_log_softmax.py b/test/legacy_test/test_log_softmax.py index 64be97bb1aba1..494ba127032e4 100644 --- a/test/legacy_test/test_log_softmax.py +++ b/test/legacy_test/test_log_softmax.py @@ -194,7 +194,7 @@ def check_api(self, axis=-1): out = exe.run(feed={'x': self.x}, fetch_list=[y]) np.testing.assert_allclose(out[0], ref_out, rtol=1e-05) - # test dygrapg api + # test dygraph api paddle.disable_static() x = paddle.to_tensor(self.x) y = logsoftmax(x) diff --git a/test/legacy_test/test_paddle_save_load_binary.py b/test/legacy_test/test_paddle_save_load_binary.py index df7304cf1d19e..22b62e082cc94 100644 --- a/test/legacy_test/test_paddle_save_load_binary.py +++ b/test/legacy_test/test_paddle_save_load_binary.py @@ -94,7 +94,7 @@ def test_replace_save_load_vars(self): t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been update + # make sure all the parameter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t # test for replace_save_vars/io.load_vars diff --git a/test/legacy_test/test_scatter_nd_op.py b/test/legacy_test/test_scatter_nd_op.py index 4936ccdb8989f..531b865a7c50b 100644 --- a/test/legacy_test/test_scatter_nd_op.py +++ b/test/legacy_test/test_scatter_nd_op.py @@ -29,16 +29,16 @@ def numpy_scatter_nd(ref, index, updates, fun): index_shape = index.shape end_size = index_shape[-1] - remain_numl = 1 + remain_numel = 1 for i in range(len(index_shape) - 1): - remain_numl *= index_shape[i] + remain_numel *= index_shape[i] slice_size = 1 for i in range(end_size, len(ref_shape)): slice_size *= ref_shape[i] - flat_index = index.reshape([remain_numl] + list(index_shape[-1:])) - flat_updates = updates.reshape((remain_numl, slice_size)) + flat_index = index.reshape([remain_numel] + list(index_shape[-1:])) + flat_updates = updates.reshape((remain_numel, slice_size)) flat_output = ref.reshape(list(ref_shape[:end_size]) + [slice_size]) for i_up, i_out in enumerate(flat_index): diff --git a/test/legacy_test/test_set_value_op.py b/test/legacy_test/test_set_value_op.py index c289185e58d21..4113805c663b4 100644 --- a/test/legacy_test/test_set_value_op.py +++ b/test/legacy_test/test_set_value_op.py @@ -265,7 +265,7 @@ def _get_answer(self): # 1.2.3 step < 0 -class TestSetValueItemSliceNegetiveStep(TestSetValueApi): +class TestSetValueItemSliceNegativeStep(TestSetValueApi): def set_shape(self): self.shape = [5, 2] @@ -283,7 +283,7 @@ def _get_answer(self): self.data[5:2:-1] = self.value -class TestSetValueItemSliceNegetiveStep2(TestSetValueApi): +class TestSetValueItemSliceNegativeStep2(TestSetValueApi): def set_shape(self): self.shape = [5] @@ -301,7 +301,7 @@ def _get_answer(self): self.data[1::-1] = self.value -class TestSetValueItemSliceNegetiveStep3(TestSetValueApi): +class TestSetValueItemSliceNegativeStep3(TestSetValueApi): def set_shape(self): self.shape = [3] @@ -319,7 +319,7 @@ def _get_answer(self): self.data[::-1] = self.value -class TestSetValueItemSliceNegetiveStep4(TestSetValueApi): +class TestSetValueItemSliceNegativeStep4(TestSetValueApi): def set_shape(self): self.shape = [3, 4, 5] @@ -1504,14 +1504,14 @@ def set_value(t, value): np.testing.assert_array_equal( inps.grad.numpy(), input_grad, - err_msg='The gradient of value should be \n{},\n but reveived {}'.format( + err_msg='The gradient of value should be \n{},\n but received {}'.format( input_grad, inps.grad.numpy() ), ) np.testing.assert_array_equal( value.grad.numpy(), value_grad, - err_msg='The gradient of input should be \n{},\n but reveived {}'.format( + err_msg='The gradient of input should be \n{},\n but received {}'.format( value_grad, value.grad.numpy() ), ) @@ -1538,14 +1538,14 @@ def set_value(t, value): np.testing.assert_array_equal( inps2.grad.numpy(), input_grad2, - err_msg='The gradient of value should be \n{},\n but reveived {}'.format( + err_msg='The gradient of value should be \n{},\n but received {}'.format( input_grad, inps2.grad.numpy() ), ) np.testing.assert_array_equal( value2.grad.numpy(), value_grad2, - err_msg='The gradient of input should be \n{},\n but reveived {}'.format( + err_msg='The gradient of input should be \n{},\n but received {}'.format( value_grad, value2.grad.numpy() ), ) @@ -1592,14 +1592,14 @@ def set_value3(t, value): np.testing.assert_array_equal( inps.grad.numpy(), input_grad, - err_msg='The gradient of value should be \n{},\n but reveived {}'.format( + err_msg='The gradient of value should be \n{},\n but received {}'.format( input_grad, inps.grad.numpy() ), ) np.testing.assert_array_equal( value.grad.numpy(), value_grad, - err_msg='The gradient of input should be \n{},\n but reveived {}'.format( + err_msg='The gradient of input should be \n{},\n but received {}'.format( value_grad, value.grad.numpy() ), ) @@ -1640,14 +1640,14 @@ def set_value4(t, value): np.testing.assert_array_equal( inps.grad.numpy(), input_grad, - err_msg='The gradient of value should be \n{},\n but reveived {}'.format( + err_msg='The gradient of value should be \n{},\n but received {}'.format( input_grad, inps.grad.numpy() ), ) np.testing.assert_array_equal( value.grad.numpy(), value_grad, - err_msg='The gradient of input should be \n{},\n but reveived {}'.format( + err_msg='The gradient of input should be \n{},\n but received {}'.format( value_grad, value.grad.numpy() ), ) @@ -1692,14 +1692,14 @@ def set_value5(t, value): np.testing.assert_array_equal( inps.grad.numpy(), input_grad, - err_msg='The gradient of value should be \n{},\n but reveived {}'.format( + err_msg='The gradient of value should be \n{},\n but received {}'.format( input_grad, inps.grad.numpy() ), ) np.testing.assert_array_equal( value.grad.numpy(), value_grad, - err_msg='The gradient of input should be \n{},\n but reveived {}'.format( + err_msg='The gradient of input should be \n{},\n but received {}'.format( value_grad, value.grad.numpy() ), ) diff --git a/test/legacy_test/test_sgd_op.py b/test/legacy_test/test_sgd_op.py index d71b297185892..ba7dbb99d1b87 100644 --- a/test/legacy_test/test_sgd_op.py +++ b/test/legacy_test/test_sgd_op.py @@ -85,7 +85,7 @@ def check_with_place(self, place): param_array = np.full((height, self.row_numel), 5.0).astype("float32") param.set(param_array, place) - # create and initialize LeraningRate Variable + # create and initialize LearningRate Variable lr = scope.var('LearningRate').get_tensor() lr_array = np.full((1), 2.0).astype("float32") lr.set(lr_array, place) @@ -170,7 +170,7 @@ def check_with_place(self, place): w_before_optimize = np.array(w_tensor) - # create and initialize LeraningRate Variable + # create and initialize LearningRate Variable lr_value = 0.1 lr = scope.var('LearningRate').get_tensor() lr_array = np.full((1), lr_value).astype("float32") diff --git a/test/legacy_test/test_sgd_op_bf16.py b/test/legacy_test/test_sgd_op_bf16.py index 3baf0a490cbf5..3af7d9c6bc93a 100644 --- a/test/legacy_test/test_sgd_op_bf16.py +++ b/test/legacy_test/test_sgd_op_bf16.py @@ -286,7 +286,7 @@ def _reference(self, data, emb_weight, bf16=False): out_dtype = np.uint16 if bf16 else np.float32 lookup_table_grad = np.zeros(self.w_shape, dtype=out_dtype) - # indexes may dupplicate + # indexes may duplicate if bf16: for i, idx in enumerate(data): idxv = idx[0] diff --git a/test/legacy_test/test_signal.py b/test/legacy_test/test_signal.py index 1d86f15f51095..8d70702a26ad6 100644 --- a/test/legacy_test/test_signal.py +++ b/test/legacy_test/test_signal.py @@ -574,7 +574,7 @@ def decorate(cls): def setUpModule(): global rtol global atol - # All test case will use float64 for compare percision, refs: + # All test case will use float64 for compare precision, refs: # https://github.com/PaddlePaddle/Paddle/wiki/Upgrade-OP-Precision-to-Float64 rtol = { 'float32': 1e-06, diff --git a/test/legacy_test/test_slice_scatter.py b/test/legacy_test/test_slice_scatter.py index 075b5a5741886..bb46ddffb9b94 100644 --- a/test/legacy_test/test_slice_scatter.py +++ b/test/legacy_test/test_slice_scatter.py @@ -264,7 +264,7 @@ def init_dtype(self): self.dtype = 'float32' -class TestSliceScatterApiBroadcase2D(TestSliceScatterApi): +class TestSliceScatterApiBroadcast2D(TestSliceScatterApi): def init_shape(self): self.x_shape = [8, 9] self.value_shape = [8, 1] @@ -274,12 +274,12 @@ def init_shape(self): self.strides = [2] -class TestSliceScatterApiBroadcase2DFloat32(TestSliceScatterApiBroadcase2D): +class TestSliceScatterApiBroadcast2DFloat32(TestSliceScatterApiBroadcast2D): def init_dtype(self): self.dtype = 'float32' -class TestSliceScatterApiBroadcase3D(TestSliceScatterApi): +class TestSliceScatterApiBroadcast3D(TestSliceScatterApi): def init_shape(self): self.x_shape = [8, 9, 6] self.value_shape = [1, 9, 1] @@ -289,7 +289,7 @@ def init_shape(self): self.strides = [3, 2] -class TestSliceScatterApiBroadcase3DFloat32(TestSliceScatterApiBroadcase3D): +class TestSliceScatterApiBroadcast3DFloat32(TestSliceScatterApiBroadcast3D): def init_dtype(self): self.dtype = 'float32' diff --git a/test/legacy_test/test_softmax_op.py b/test/legacy_test/test_softmax_op.py index 6b9952e569ae5..1876424cf4d4b 100644 --- a/test/legacy_test/test_softmax_op.py +++ b/test/legacy_test/test_softmax_op.py @@ -59,7 +59,7 @@ def setUp(self): self.public_python_api = F.softmax self.use_cudnn = False self.use_mkldnn = False - # explicilty use float32 for ROCm, as MIOpen does not yet support float64 + # explicitly use float32 for ROCm, as MIOpen does not yet support float64 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 self.init_kernel_type() self.shape = self.get_x_shape() @@ -142,7 +142,7 @@ def setUp(self): self.public_python_api = F.softmax self.use_cudnn = False self.use_mkldnn = False - # explicilty use float32 for ROCm, as MIOpen does not yet support float64 + # explicitly use float32 for ROCm, as MIOpen does not yet support float64 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 self.init_kernel_type() @@ -190,7 +190,7 @@ def setUp(self): self.prim_op_type = "comp" self.use_cudnn = True self.use_mkldnn = False - # explicilty use float32 for ROCm, as MIOpen does not yet support float64 + # explicitly use float32 for ROCm, as MIOpen does not yet support float64 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 np.random.seed(0) diff --git a/test/legacy_test/test_softmax_with_cross_entropy_op.py b/test/legacy_test/test_softmax_with_cross_entropy_op.py index 62f475cd922a7..8bafae13efc70 100644 --- a/test/legacy_test/test_softmax_with_cross_entropy_op.py +++ b/test/legacy_test/test_softmax_with_cross_entropy_op.py @@ -99,7 +99,7 @@ def initParams(self): self.python_out_sig = ["Loss", "Softmax"] self.numeric_stable_mode = False self.soft_label = False - # explicilty use float32 for ROCm, as MIOpen does not yet support float64 + # explicitly use float32 for ROCm, as MIOpen does not yet support float64 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 self.axis = -1 self.ignore_index = -1 diff --git a/test/legacy_test/test_sparse_attention_op.py b/test/legacy_test/test_sparse_attention_op.py index 48946522864aa..24272c5e416b0 100644 --- a/test/legacy_test/test_sparse_attention_op.py +++ b/test/legacy_test/test_sparse_attention_op.py @@ -438,7 +438,7 @@ def test_dygraph(self): paddle_key = paddle.to_tensor(key, place=self.place) paddle_value = paddle.to_tensor(value, place=self.place) paddle_offset = paddle.to_tensor(offset, place=self.place) - paddle_colunmns = paddle.to_tensor(columns, place=self.place) + paddle_columns = paddle.to_tensor(columns, place=self.place) paddle_kp_mask = paddle.to_tensor(key_padding_mask, place=self.place) paddle_attn_mask = paddle.to_tensor(attn_mask, place=self.place) @@ -448,7 +448,7 @@ def test_dygraph(self): paddle_key, paddle_value, paddle_offset, - paddle_colunmns, + paddle_columns, key_padding_mask=paddle_kp_mask, attn_mask=paddle_attn_mask, ) @@ -469,7 +469,7 @@ def test_dygraph(self): paddle_key, paddle_value, paddle_offset, - paddle_colunmns, + paddle_columns, ) numpy_result, __, __ = ref_batch_sparse_attention( diff --git a/test/legacy_test/test_split_op.py b/test/legacy_test/test_split_op.py index 9311a5f2d9957..0f780b2ddfff6 100644 --- a/test/legacy_test/test_split_op.py +++ b/test/legacy_test/test_split_op.py @@ -567,9 +567,11 @@ def test_out1(self): eager_x2_out = x2.numpy() loss = x0.sum() loss.backward() - manul_grad = np.zeros_like(input_1) - manul_grad[:, :2, :] = 1 - np.testing.assert_allclose(input.gradient(), manul_grad, rtol=1e-05) + manual_grad = np.zeros_like(input_1) + manual_grad[:, :2, :] = 1 + np.testing.assert_allclose( + input.gradient(), manual_grad, rtol=1e-05 + ) np.testing.assert_allclose(ex_x0, eager_x0_out, rtol=1e-05) np.testing.assert_allclose(ex_x1, eager_x1_out, rtol=1e-05) np.testing.assert_allclose(ex_x2, eager_x2_out, rtol=1e-05) @@ -597,9 +599,11 @@ def test_out2(self): eager_x2_out = x2.numpy() loss = x0.sum() loss.backward() - manul_grad = np.zeros_like(input_1) - manul_grad[:, :2, :] = 1 - np.testing.assert_allclose(input.gradient(), manul_grad, rtol=1e-05) + manual_grad = np.zeros_like(input_1) + manual_grad[:, :2, :] = 1 + np.testing.assert_allclose( + input.gradient(), manual_grad, rtol=1e-05 + ) np.testing.assert_allclose(ex_x0, eager_x0_out, rtol=1e-05) np.testing.assert_allclose(ex_x1, eager_x1_out, rtol=1e-05) np.testing.assert_allclose(ex_x2, eager_x2_out, rtol=1e-05) @@ -630,9 +634,11 @@ def test_out1(self): eager_x2_out = x2.numpy() loss = x0.sum() loss.backward() - manul_grad = np.zeros_like(input_1) - manul_grad[:, :2, :] = 1 - np.testing.assert_allclose(input.gradient(), manul_grad, rtol=1e-05) + manual_grad = np.zeros_like(input_1) + manual_grad[:, :2, :] = 1 + np.testing.assert_allclose( + input.gradient(), manual_grad, rtol=1e-05 + ) np.testing.assert_allclose(ex_x0, eager_x0_out, rtol=1e-05) np.testing.assert_allclose(ex_x1, eager_x1_out, rtol=1e-05) np.testing.assert_allclose(ex_x2, eager_x2_out, rtol=1e-05) diff --git a/test/legacy_test/test_static_pylayer.py b/test/legacy_test/test_static_pylayer.py index fd5075a4904aa..34a5afe577a67 100644 --- a/test/legacy_test/test_static_pylayer.py +++ b/test/legacy_test/test_static_pylayer.py @@ -133,7 +133,7 @@ def backward_fn(dy): self.assertEqual(x_grad.shape, ()) @test_with_pir_api - def test_return_var_typle(self): + def test_return_var_type(self): def forward_fn(a, b): return 3 * a, -2 * b diff --git a/test/legacy_test/test_static_save_load.py b/test/legacy_test/test_static_save_load.py index ca1adaed4ef0a..f662ee3f95e69 100644 --- a/test/legacy_test/test_static_save_load.py +++ b/test/legacy_test/test_static_save_load.py @@ -340,7 +340,7 @@ def test_ptb_rnn_cpu_float32(self): t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been update + # make sure all the parameter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t @@ -357,7 +357,7 @@ def test_ptb_rnn_cpu_float32(self): new_t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been set to zero + # make sure all the parameter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) paddle.static.load( @@ -480,7 +480,7 @@ def test_ptb_rnn_cpu_float32(self): t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been update + # make sure all the parameter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t @@ -497,7 +497,7 @@ def test_ptb_rnn_cpu_float32(self): new_t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been set to zero + # make sure all the parameter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) paddle.static.load( @@ -613,7 +613,7 @@ def test_ptb_rnn_cpu_float32(self): t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been update + # make sure all the parameter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t @@ -630,7 +630,7 @@ def test_ptb_rnn_cpu_float32(self): new_t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been set to zero + # make sure all the parameter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) paddle.static.load( @@ -751,7 +751,7 @@ def test_ptb_rnn_cpu_float32(self): t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been update + # make sure all the parameter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t @@ -768,7 +768,7 @@ def test_ptb_rnn_cpu_float32(self): new_t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been set to zero + # make sure all the parameter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) # base.load(test_program, "./test_1", None ) @@ -807,7 +807,7 @@ def test_ptb_rnn_cpu_float32(self): new_t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been set to zero + # make sure all the parameter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) paddle.static.set_program_state(test_program, program_state_1) @@ -829,7 +829,7 @@ def test_ptb_rnn_cpu_float32(self): new_t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been set to zero + # make sure all the parameter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) paddle.static.set_program_state(test_program, program_state_2) @@ -851,7 +851,7 @@ def test_ptb_rnn_cpu_float32(self): new_t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been set to zero + # make sure all the parameter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) paddle.static.set_program_state(test_program, program_state_3) @@ -954,7 +954,7 @@ def set_var(var, ndarray): t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been update + # make sure all the parameter or optimizer var have been update base_map[var.name] = t for var in program.list_vars(): @@ -1073,7 +1073,7 @@ def test_load_from_old_interface(self): t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been update + # make sure all the parameter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t @@ -1091,7 +1091,7 @@ def test_load_from_old_interface(self): new_t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been set to zero + # make sure all the parameter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) paddle.static.load( @@ -1214,7 +1214,7 @@ def test_load_from_old_interface_var_list(self): t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been update + # make sure all the parameter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t @@ -1237,7 +1237,7 @@ def test_load_from_old_interface_var_list(self): new_t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been set to zero + # make sure all the parameter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) paddle.static.load( @@ -1355,7 +1355,7 @@ def test_load_from_old_interface(self): t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been update + # make sure all the parameter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t save_dir = os.path.join(temp_dir.name, "test_path") @@ -1373,7 +1373,7 @@ def test_load_from_old_interface(self): new_t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been set to zero + # make sure all the parameter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) file_model_path = os.path.join(save_dir, "model_single") @@ -1556,7 +1556,7 @@ def test_ptb_rnn_cpu_float32(self): t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been update + # make sure all the parameter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t save_dir = os.path.join(self.temp_dir.name, "test_program_1") @@ -1571,7 +1571,7 @@ def test_ptb_rnn_cpu_float32(self): new_t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been set to zero + # make sure all the parameter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) # case 1: load basic @@ -1731,7 +1731,7 @@ def test_ptb_rnn_cpu_float32(self): t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been update + # make sure all the parameter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t @@ -1749,7 +1749,7 @@ def test_ptb_rnn_cpu_float32(self): new_t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been set to zero + # make sure all the parameter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) # base.load(test_program, "./test_1", None ) @@ -1816,7 +1816,7 @@ def test_pickle_protocol(self): t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been update + # make sure all the parameter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t diff --git a/test/legacy_test/test_static_save_load_bf16.py b/test/legacy_test/test_static_save_load_bf16.py index 1ca1bec7d15e7..bc91f34b3f60c 100644 --- a/test/legacy_test/test_static_save_load_bf16.py +++ b/test/legacy_test/test_static_save_load_bf16.py @@ -127,7 +127,7 @@ def test_ptb_rnn_cpu_bfloat16(self): t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been update + # make sure all the parameter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t save_dir = os.path.join(self.temp_dir.name, "test_1") @@ -142,7 +142,7 @@ def test_ptb_rnn_cpu_bfloat16(self): new_t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been set to zero + # make sure all the parameter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) paddle.static.load( diff --git a/test/legacy_test/test_static_save_load_large.py b/test/legacy_test/test_static_save_load_large.py index 2011dd45cfaf8..d0ef10ae3a9ab 100644 --- a/test/legacy_test/test_static_save_load_large.py +++ b/test/legacy_test/test_static_save_load_large.py @@ -49,7 +49,7 @@ def test_large_parameters_static_save(self): t = np.array( base.global_scope().find_var(var.name).get_tensor() ) - # make sure all the paramerter or optimizer var have been update + # make sure all the parameter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t temp_dir = tempfile.TemporaryDirectory() diff --git a/test/legacy_test/test_static_shape_inferrence_for_shape_tensor.py b/test/legacy_test/test_static_shape_inferrence_for_shape_tensor.py index a38fd88881937..eaa910637bbd2 100644 --- a/test/legacy_test/test_static_shape_inferrence_for_shape_tensor.py +++ b/test/legacy_test/test_static_shape_inferrence_for_shape_tensor.py @@ -17,7 +17,7 @@ import paddle -class StaticShapeInferrenceTest(unittest.TestCase): +class StaticShapeInferenceTest(unittest.TestCase): def test_static_graph(self): paddle.enable_static() data = paddle.static.data(name="x", shape=[-1, 2], dtype='float32') diff --git a/test/legacy_test/test_sum_op.py b/test/legacy_test/test_sum_op.py index cd2f9a4f6cdec..a2570a566c348 100644 --- a/test/legacy_test/test_sum_op.py +++ b/test/legacy_test/test_sum_op.py @@ -86,16 +86,16 @@ def setUp(self): self.init_kernel_type() def check_with_place(self, place, inplace): - self.check_input_and_optput( + self.check_input_and_output( core.Scope(), place, inplace, True, True, True ) - self.check_input_and_optput( + self.check_input_and_output( core.Scope(), place, inplace, False, True, True ) - self.check_input_and_optput( + self.check_input_and_output( core.Scope(), place, inplace, False, False, True ) - self.check_input_and_optput( + self.check_input_and_output( core.Scope(), place, inplace, False, False, False ) @@ -108,7 +108,7 @@ def _get_array(self, rows, row_numel): array[i] *= rows[i] return array - def check_input_and_optput( + def check_input_and_output( self, scope, place, @@ -198,7 +198,7 @@ def _get_array(self, rows, row_numel): else: return np.ndarray((0, row_numel), dtype=self.dtype) - def check_input_and_optput( + def check_input_and_output( self, scope, place, diff --git a/test/legacy_test/test_svd_op.py b/test/legacy_test/test_svd_op.py index a74404e408524..6b41fa7cead8d 100644 --- a/test/legacy_test/test_svd_op.py +++ b/test/legacy_test/test_svd_op.py @@ -90,7 +90,7 @@ def test_check_grad(self): class TestSvdCheckGrad2(TestSvdOp): - # NOTE(xiongkun03): because we want to construct some full rank matrics, + # NOTE(xiongkun03): because we want to construct some full rank matrices, # so we can't specifize matrices which numel() > 100 no_need_check_grad = True diff --git a/test/legacy_test/test_sync_batch_norm_op.py b/test/legacy_test/test_sync_batch_norm_op.py index 17daa24996b4f..3b7ea63cb5963 100644 --- a/test/legacy_test/test_sync_batch_norm_op.py +++ b/test/legacy_test/test_sync_batch_norm_op.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -test for sync bachnorm op. +test for sync batchnorm op. for both FP64 and FP16 input. """ diff --git a/test/legacy_test/test_tensor.py b/test/legacy_test/test_tensor.py index 8c007d4675390..9207d3a181789 100644 --- a/test/legacy_test/test_tensor.py +++ b/test/legacy_test/test_tensor.py @@ -339,7 +339,7 @@ def test_print_tensor(self): print(tensor) self.assertTrue(isinstance(str(tensor), str)) - def test_tensor_poiter(self): + def test_tensor_pointer(self): place = core.CPUPlace() scope = core.Scope() var = scope.var("test_tensor") diff --git a/test/legacy_test/test_tensor_register_hook.py b/test/legacy_test/test_tensor_register_hook.py index 29c614713dff3..c7826c983adcd 100644 --- a/test/legacy_test/test_tensor_register_hook.py +++ b/test/legacy_test/test_tensor_register_hook.py @@ -589,7 +589,7 @@ def test_register_backward_hook_for_var_without_gradient(self): x._register_backward_hook(global_void_hook) -class TestRegsiterBackwardFinalHook(unittest.TestCase): +class TestRegisterBackwardFinalHook(unittest.TestCase): def setUp(self): self.devices = ["cpu"] if paddle.is_compiled_with_cuda(): diff --git a/test/legacy_test/test_tensor_uva.py b/test/legacy_test/test_tensor_uva.py index 34d7e59609e0b..e7b6d03fe8bd9 100644 --- a/test/legacy_test/test_tensor_uva.py +++ b/test/legacy_test/test_tensor_uva.py @@ -53,7 +53,7 @@ def test_uva_tensor_creation(self): np.testing.assert_allclose(tensor.numpy(), data, rtol=1e-05) np.testing.assert_allclose(tensor2.numpy(), data, rtol=1e-05) - def test_uva_tensor_corectness(self): + def test_uva_tensor_correctness(self): if paddle.base.core.is_compiled_with_cuda(): a = np.arange(0, 100, dtype="int32") a = a.reshape([10, 10]) diff --git a/test/legacy_test/test_traced_layer_err_msg.py b/test/legacy_test/test_traced_layer_err_msg.py index 16880dafdcbf7..4927fdea82a54 100644 --- a/test/legacy_test/test_traced_layer_err_msg.py +++ b/test/legacy_test/test_traced_layer_err_msg.py @@ -241,7 +241,7 @@ def test_linear_net_with_none(self): class TestTracedLayerSaveInferenceModel(unittest.TestCase): - """test save_inference_model will automaticlly create non-exist dir""" + """test save_inference_model will automatically create non-exist dir""" def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() diff --git a/test/legacy_test/test_trans_layout_op.py b/test/legacy_test/test_trans_layout_op.py index da59301aacfc3..b936abc95df95 100644 --- a/test/legacy_test/test_trans_layout_op.py +++ b/test/legacy_test/test_trans_layout_op.py @@ -45,17 +45,17 @@ def test_check_output(self): class LayoutAutoTune(unittest.TestCase): def test_config(self): paddle.base.core.enable_layout_autotune() - if self.use_autoune(): + if self.use_autotune(): self.assertEqual(paddle.base.core.use_layout_autotune(), True) paddle.base.core.disable_layout_autotune() self.assertEqual(paddle.base.core.use_layout_autotune(), False) - self.use_autoune() + self.use_autotune() def setUp(self): paddle.disable_static() - self.use_autoune() + self.use_autotune() - def use_autoune(self): + def use_autotune(self): if paddle.is_compiled_with_cuda(): paddle.incubate.autotune.set_config( config={"layout": {"enable": True}} diff --git a/test/legacy_test/test_transformer_api.py b/test/legacy_test/test_transformer_api.py index 5945ac4a7d5d2..fd55abde22093 100644 --- a/test/legacy_test/test_transformer_api.py +++ b/test/legacy_test/test_transformer_api.py @@ -252,9 +252,9 @@ def layer_norm(x, normalized_shape, norm, epsilon=1e-05, act=None): batch_size, src_len, d_model = x.shape x = x.reshape((batch_size * src_len, d_model)) mu = np.mean(x, axis=1, keepdims=True) - sigma_squar = np.sum(np.square(x - mu), axis=1) / d_model + sigma_square = np.sum(np.square(x - mu), axis=1) / d_model x1_up = x - mu - x1_down_1 = sigma_squar + epsilon + x1_down_1 = sigma_square + epsilon x1_down = np.sqrt(x1_down_1) x1_down = x1_down.reshape((x1_down.shape[0], 1)) x1 = x1_up / x1_down diff --git a/test/legacy_test/test_tril_triu_op.py b/test/legacy_test/test_tril_triu_op.py index 36a2ddb0383a7..391b7f83f9dc3 100644 --- a/test/legacy_test/test_tril_triu_op.py +++ b/test/legacy_test/test_tril_triu_op.py @@ -116,7 +116,7 @@ def test_check_grad_normal(self): def case_generator(op_type, Xshape, diagonal, expected, dtype): """ Generate testcases with the params shape of X, diagonal and op_type. - If arg`expercted` is 'success', it will register an Optest case and expect to pass. + If arg `expected` is 'success', it will register an OpTest case and expect to pass. Otherwise, it will register an API case and check the expect failure. """ cls_name = ( diff --git a/test/legacy_test/test_truncated_gaussian_random_op.py b/test/legacy_test/test_truncated_gaussian_random_op.py index eb8b502b082d4..0572d0da6face 100644 --- a/test/legacy_test/test_truncated_gaussian_random_op.py +++ b/test/legacy_test/test_truncated_gaussian_random_op.py @@ -22,7 +22,7 @@ from paddle.base.executor import Executor -class TestTrunctedGaussianRandomOp(unittest.TestCase): +class TestTruncatedGaussianRandomOp(unittest.TestCase): def setUp(self): self.op_type = "truncated_gaussian_random" self.inputs = {} diff --git a/test/legacy_test/test_vision_models.py b/test/legacy_test/test_vision_models.py index b53f110030549..150ae03c58fe2 100644 --- a/test/legacy_test/test_vision_models.py +++ b/test/legacy_test/test_vision_models.py @@ -20,7 +20,7 @@ from paddle.vision import models -class TestVisonModels(unittest.TestCase): +class TestVisionModels(unittest.TestCase): def models_infer(self, arch, pretrained=False, batch_norm=False): x = np.array(np.random.random((2, 3, 224, 224)), dtype=np.float32) if batch_norm: diff --git a/test/legacy_test/test_viterbi_decode_op.py b/test/legacy_test/test_viterbi_decode_op.py index 91f79565a2caa..fd5ff4b61f789 100644 --- a/test/legacy_test/test_viterbi_decode_op.py +++ b/test/legacy_test/test_viterbi_decode_op.py @@ -31,7 +31,7 @@ def __call__(self, inputs, length): bs, seq_len, n_label = inputs.shape inputs_t = np.transpose(inputs, (1, 0, 2)) trans_exp = np.expand_dims(self.transitions, axis=0) - historys = [] + histories = [] left_length = np.array(length) max_seq_len = np.amax(left_length) left_length = np.expand_dims(left_length, 1) @@ -49,7 +49,7 @@ def __call__(self, inputs, length): alpha_exp = np.expand_dims(alpha, 2) alpha_trn_sum = alpha_exp + trans_exp max_res = np.amax(alpha_trn_sum, 1), np.argmax(alpha_trn_sum, 1) - historys = historys + [max_res[1]] if i >= 1 else [] + histories = histories + [max_res[1]] if i >= 1 else [] alpha_nxt = max_res[0] + logit mask = left_length > 0 alpha = mask * alpha_nxt + (1 - mask) * alpha @@ -61,7 +61,7 @@ def __call__(self, inputs, length): last_ids_update = last_ids * (left_length >= 0) batch_path = [last_ids_update] batch_offset = np.arange(bs) * n_label - for hist in reversed(historys): + for hist in reversed(histories): left_length = left_length + 1 gather_idx = batch_offset + last_ids last_ids_update = np.take(hist, gather_idx) * (left_length > 0) diff --git a/test/legacy_test/test_warpctc_op.py b/test/legacy_test/test_warpctc_op.py index e747f381af608..9355eeec21ad5 100644 --- a/test/legacy_test/test_warpctc_op.py +++ b/test/legacy_test/test_warpctc_op.py @@ -110,7 +110,7 @@ def forward_a_sequence(self, softmax_a_sequence, labels_a_sequence): required_times = labels_a_sequence.shape[0] old_label = -1 for i in range(labels_a_sequence.shape[0]): - # two contingous labels with the same value + # two contiguous labels with the same value if labels_a_sequence[i, 0] == old_label: required_times = required_times + 1 old_label = labels_a_sequence[i, 0] diff --git a/test/legacy_test/test_where_op.py b/test/legacy_test/test_where_op.py index 16c6020d54650..d88b1b3b3a5a7 100644 --- a/test/legacy_test/test_where_op.py +++ b/test/legacy_test/test_where_op.py @@ -319,7 +319,7 @@ def __test_where_with_broadcast_static(self, cond_shape, x_shape, y_shape): np.testing.assert_array_equal(out[0], expect) def __test_where_with_type_promotion( - self, x_dtype, y_dtype, expeced_dtype=None + self, x_dtype, y_dtype, expected_dtype=None ): paddle.enable_static() main_program = paddle.static.Program() @@ -367,7 +367,7 @@ def __test_where_with_type_promotion( ) expect = np.where(cond_data, x_data_convert, y_data_convert) np.testing.assert_array_equal(out[0], expect) - self.assertEqual(out[0].dtype.__str__(), expeced_dtype) + self.assertEqual(out[0].dtype.__str__(), expected_dtype) else: expect = np.where(cond_data, x_data, y_data) np.testing.assert_array_equal(out[0], expect) diff --git a/test/legacy_test/test_while_loop_op.py b/test/legacy_test/test_while_loop_op.py index ec63375043e05..3b8123d48df4c 100644 --- a/test/legacy_test/test_while_loop_op.py +++ b/test/legacy_test/test_while_loop_op.py @@ -684,7 +684,7 @@ def type_error_cond_returns_not_variable(): self.assertRaises(TypeError, type_error_cond_returns_not_variable) - # The type of `cond` returns in Op(while_loop) must be a bollean variable + # The type of `cond` returns in Op(while_loop) must be a boolean variable def type_error_cond_returns_not_boolean(): out = paddle.static.nn.while_loop( cond_returns_not_bool_tensor, body, [data_1d] diff --git a/test/legacy_test/test_while_op.py b/test/legacy_test/test_while_op.py index d763576f9ff3a..69dc4e1b8c070 100644 --- a/test/legacy_test/test_while_op.py +++ b/test/legacy_test/test_while_op.py @@ -235,7 +235,7 @@ def body(i, s, x): x = paddle.static.data(shape=[-1], name='x', dtype='float32') func(x) - # NOTE(winter-wang): The while_op in pir mode doesn't need following constrait, so hre only check when in non-pir mode. + # NOTE(winter-wang): The while_op in pir mode doesn't need following constraint, so here only check when in non-pir mode. if not in_pir_mode(): for op in main_program.block(0).ops: if op.type == "while": diff --git a/test/legacy_test/test_zeros_like_op.py b/test/legacy_test/test_zeros_like_op.py index 538556cd4f1fc..4768a2506f249 100644 --- a/test/legacy_test/test_zeros_like_op.py +++ b/test/legacy_test/test_zeros_like_op.py @@ -59,7 +59,7 @@ def test_api(self): self.assertEqual((outs[i] == np.zeros(shape, dtype)).all(), True) -class TestZerosLikeImpeartive(unittest.TestCase): +class TestZerosLikeImperative(unittest.TestCase): def test_out(self): shape = [3, 4] place = ( diff --git a/test/mkldnn/test_elementwise_add_mkldnn_op.py b/test/mkldnn/test_elementwise_add_mkldnn_op.py index 9d62f1cf55e97..8b9dded0129bd 100644 --- a/test/mkldnn/test_elementwise_add_mkldnn_op.py +++ b/test/mkldnn/test_elementwise_add_mkldnn_op.py @@ -102,21 +102,21 @@ def test_check_grad_ignore_x(self): pass -class TestOneDNNlementwiseAddOpZeroDim(TestOneDNNElementwiseAddOp): +class TestOneDNNElementwiseAddOpZeroDim(TestOneDNNElementwiseAddOp): def init_input_output(self): self.x = np.random.random((100,)).astype(self.dtype) self.y = np.array(3.0).astype(self.dtype) self.out = np.add(self.x, self.y) -class TestOneDNNlementwiseAddOpZeroDim2(TestOneDNNElementwiseAddOp): +class TestOneDNNElementwiseAddOpZeroDim2(TestOneDNNElementwiseAddOp): def init_input_output(self): self.x = np.array(3.0).astype(self.dtype) self.y = np.random.random((100,)).astype(self.dtype) self.out = np.add(self.x, self.y) -class TestOneDNNlementwiseAddOpZeroDim3(TestOneDNNElementwiseAddOp): +class TestOneDNNElementwiseAddOpZeroDim3(TestOneDNNElementwiseAddOp): def init_input_output(self): self.x = np.array(3.0).astype(self.dtype) self.y = np.array(3.0).astype(self.dtype) @@ -127,7 +127,7 @@ def init_input_output(self): @skip_check_grad_ci( - reason="oneDNN's int8 elementwise_ops don't implemend grad kernel." + reason="oneDNN's int8 elementwise_ops don't implement grad kernel." ) class TestInt8(TestElementwiseAddOp): def init_kernel_type(self): diff --git a/test/mkldnn/test_fused_vit_attention.py b/test/mkldnn/test_fused_vit_attention.py index 8c4876e4281b2..b980f8bff912a 100644 --- a/test/mkldnn/test_fused_vit_attention.py +++ b/test/mkldnn/test_fused_vit_attention.py @@ -23,7 +23,7 @@ np.random.seed(0) -def test_fuse_resenet_unit(): +def test_fuse_resnet_unit(): tests = [[1, 4096, 768, 12], [10, 4097, 756, 12], [10, 4097, 756, 12]] for test in tests: batch_size = test[0] @@ -74,4 +74,4 @@ def test_fuse_resenet_unit(): if __name__ == '__main__': - test_fuse_resenet_unit() + test_fuse_resnet_unit() diff --git a/test/mkldnn/test_fusion_gru_bf16_mkldnn_op.py b/test/mkldnn/test_fusion_gru_bf16_mkldnn_op.py index ae44798dce4eb..fd9925df082e0 100644 --- a/test/mkldnn/test_fusion_gru_bf16_mkldnn_op.py +++ b/test/mkldnn/test_fusion_gru_bf16_mkldnn_op.py @@ -59,7 +59,7 @@ def setUp(self): N = len(self.lod[0]) # fp32 X input for reference implementation and - # corressponding bf16 data as input to GRU oneDNN bf16 kernel + # corresponding bf16 data as input to GRU oneDNN bf16 kernel x_fp32 = np.random.rand(T, self.M).astype('float32') x_bf16 = convert_float_to_uint16(x_fp32) From 7c9fcfa248c43c5caeb23ef3d752870b220bb90e Mon Sep 17 00:00:00 2001 From: zyfncg Date: Fri, 23 Feb 2024 14:05:44 +0800 Subject: [PATCH 020/282] [CINN Unittest] Add unittest for complex symbol shape (#61976) * add unittest for complex symbol shape * refine test --- .../symbolic/test_complex_symbol_subgraph.py | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 test/ir/pir/cinn/symbolic/test_complex_symbol_subgraph.py diff --git a/test/ir/pir/cinn/symbolic/test_complex_symbol_subgraph.py b/test/ir/pir/cinn/symbolic/test_complex_symbol_subgraph.py new file mode 100644 index 0000000000000..04e0600b82654 --- /dev/null +++ b/test/ir/pir/cinn/symbolic/test_complex_symbol_subgraph.py @@ -0,0 +1,83 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import unittest +from os.path import dirname + +import numpy as np + +import paddle +from paddle import nn +from paddle.static import InputSpec + +sys.path.append(dirname(dirname(__file__))) + +import utils + + +class ComplexSymbolSubgraph(nn.Layer): + def __init__(self): + super().__init__() + self.hidden_size = 768 + self.intermediate_size = 1008 + self.linear = nn.Linear( + self.hidden_size, self.intermediate_size, bias_attr=False + ) + + def forward(self, a, b): + c = paddle.concat([a, a, b], 1) + d = self.linear(c) + return paddle.exp(d) - d + + +class TestComplexSymbolSubgraph(unittest.TestCase): + def setUp(self): + paddle.seed(2024) + self.prepare_data() + + def prepare_data(self): + self.shape = [1, 2048, 768] + self.hidden_states = paddle.randn(self.shape, dtype="float32") + self.hidden_states.stop_gradient = False + + def check_jit_kernel_info(self, static_fn): + utils.check_jit_kernel_number(static_fn, 2) + utils.check_jit_kernel_structure(static_fn, {utils.JIT_KERNEL_NAME: 2}) + + def eval(self, use_cinn): + paddle.seed(2024) + net = ComplexSymbolSubgraph() + input_spec = [ + InputSpec(shape=[1, None, 768], dtype='float32'), + InputSpec(shape=[1, None, 768], dtype='float32'), + ] + net = utils.apply_to_static(net, use_cinn, input_spec) + net.eval() + out = net(self.hidden_states, self.hidden_states) + if use_cinn: + self.check_jit_kernel_info(net.forward) + return out + + def test_eval(self): + dy_out = self.eval(use_cinn=False) + if utils.unittest_use_cinn(): + cinn_out = self.eval(use_cinn=True) + np.testing.assert_allclose( + cinn_out.numpy(), dy_out.numpy(), atol=1e-6, rtol=1e-6 + ) + + +if __name__ == '__main__': + unittest.main() From 2de02d19f7254e050a82555ed5af9f249b5fa36b Mon Sep 17 00:00:00 2001 From: xiaoguoguo626807 <100397923+xiaoguoguo626807@users.noreply.github.com> Date: Fri, 23 Feb 2024 14:06:16 +0800 Subject: [PATCH 021/282] modify paddledetection output_grad is none should pass call_vjp (#61927) --- python/paddle/autograd/backward_utils.py | 8 ++++++++ python/paddle/autograd/ir_backward.py | 5 ++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/python/paddle/autograd/backward_utils.py b/python/paddle/autograd/backward_utils.py index e901a1fc5a7a5..e3e6326ba61cc 100644 --- a/python/paddle/autograd/backward_utils.py +++ b/python/paddle/autograd/backward_utils.py @@ -439,6 +439,14 @@ def all_stop_gradient_true(block): return True +def all_output_grad_none(list_of_list): + for list_ in list_of_list: + for value in list_: + if value is not None: + return False + return True + + def parent_total_ops(block): ''' when block is sub_block, forward op should include its parent block ops diff --git a/python/paddle/autograd/ir_backward.py b/python/paddle/autograd/ir_backward.py index aa5d1d990786e..9c751f82238fa 100644 --- a/python/paddle/autograd/ir_backward.py +++ b/python/paddle/autograd/ir_backward.py @@ -22,6 +22,7 @@ ValueDict, ValueSet, _as_list, + all_output_grad_none, all_stop_gradient_true, argument_to_value, check_type, @@ -630,7 +631,9 @@ def append_yield( # all(zero_flag) support this op has no contribution for grad # should be delete (prune sub_graph) if ( - len(output_grads) == 0 or all(zero_flag) + len(output_grads) == 0 + or all(zero_flag) + or all_output_grad_none(output_grads) ) and op.name() not in [ "pd_op.while", "pd_op.if", From c2286f135c3397286c5d8053ef556424a9b3a1d8 Mon Sep 17 00:00:00 2001 From: risemeup1 <62429225+risemeup1@users.noreply.github.com> Date: Fri, 23 Feb 2024 14:22:35 +0800 Subject: [PATCH 022/282] optimize proxy of ci (#61988) * delete --force-install * optimize proxy of ci --- paddle/scripts/paddle_build.sh | 3 ++- tools/auto_parallel/ci_auto_parallel.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 320e969ef73bb..71ee30a115ef7 100644 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -3348,7 +3348,8 @@ function distribute_test() { echo "Dowloading ...." cd ${work_dir} - git clone --depth=1 https://github.com/PaddlePaddle/PaddleNLP.git -b stable/paddle-ci + wget https://paddlenlp.bj.bcebos.com/wheels/PaddleNLP_stable_paddle.tar.gz --no-proxy + tar -zvxf PaddleNLP_stable_paddle.tar.gz cd PaddleNLP sed -i '/lac/d' scripts/regression/requirements_ci.txt diff --git a/tools/auto_parallel/ci_auto_parallel.sh b/tools/auto_parallel/ci_auto_parallel.sh index e536fe9df9fc5..21468833321ef 100644 --- a/tools/auto_parallel/ci_auto_parallel.sh +++ b/tools/auto_parallel/ci_auto_parallel.sh @@ -24,7 +24,7 @@ export case_list=() install_paddle(){ echo -e "\033[31m ---- Install paddlepaddle-gpu \033" if [ -n "$paddle" ];then - python -m pip install --user ${paddle} --force-reinstall --no-dependencies; + python -m pip install --user ${paddle} --no-dependencies; fi python -c "import paddle; print('paddle version:',paddle.__version__,'\npaddle commit:',paddle.version.commit)"; } From b99ca0f01c64a4b242da05a63db2a8eb7aa45a7c Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 23 Feb 2024 14:25:14 +0800 Subject: [PATCH 023/282] Fix sincethe, diffrent, etc. (#61953) --- paddle/cinn/backends/codegen_cuda_dev.cc | 2 +- paddle/cinn/frontend/interpreter.cc | 2 +- .../transforms/cinn_group_cluster_pass.cc | 6 +++--- paddle/cinn/hlir/framework/graph.h | 4 ++-- paddle/cinn/hlir/framework/instruction_test.cc | 2 +- paddle/cinn/hlir/framework/memory.h | 2 +- paddle/cinn/hlir/framework/node.h | 4 ++-- paddle/cinn/hlir/framework/op_lowering_impl.cc | 2 +- paddle/cinn/hlir/framework/op_lowering_impl.h | 4 ++-- .../hlir/framework/op_lowering_impl_base.h | 2 +- paddle/cinn/hlir/framework/op_lowering_test.cc | 2 +- paddle/cinn/hlir/framework/op_lowering_util.cc | 4 ++-- paddle/cinn/hlir/framework/pir_compiler.h | 2 +- .../default_horizontal_fuse_pass.cc | 2 +- .../default_input_fuse_pass.cc | 2 +- paddle/cinn/hlir/pass/op_fusion_pass.cc | 2 +- paddle/cinn/hlir/pe/load_x86_params.cc | 2 +- paddle/cinn/hlir/pe/reduction.cc | 4 ++-- paddle/cinn/hlir/pe/schedule.cc | 14 +++++++------- paddle/cinn/hlir/pe/schedule.h | 2 +- paddle/cinn/hlir/pe/transform.h | 6 +++--- paddle/cinn/ir/buffer.h | 2 +- .../group_schedule/st_shape_group_scheduler.cc | 2 +- .../tactic/compute_inline_tactic.cc | 2 +- .../ir/schedule/impl/loop_transformation.cc | 2 +- paddle/cinn/ir/schedule/ir_schedule_util.h | 4 ++-- paddle/cinn/ir/schedule/schedule_desc.h | 6 +++--- paddle/cinn/ir/tensor.cc | 4 ++-- paddle/cinn/ir/utils/ir_nodes_collector.h | 2 +- paddle/cinn/lang/README.md | 2 +- paddle/cinn/lang/lower_impl.cc | 2 +- paddle/cinn/optim/buffer_assign.cc | 2 +- paddle/cinn/optim/compute_inline_expand.cc | 2 +- paddle/cinn/optim/resize_buffer.cc | 2 +- paddle/cinn/optim/update_buffer_axis_pass.cc | 2 +- paddle/cinn/optim/vectorize_loops.cc | 2 +- paddle/cinn/poly/poly_scheduler.cc | 2 +- paddle/cinn/poly/stage.h | 8 ++++---- paddle/cinn/pybind/common.cc | 2 +- paddle/cinn/runtime/custom_function_test.cc | 4 ++-- paddle/cinn/runtime/tiny_runtime.cc | 2 +- paddle/cinn/utils/event.h | 2 +- paddle/cinn/utils/multi_threading.h | 2 +- paddle/cinn/utils/random_engine.h | 2 +- paddle/cinn/utils/registry.h | 2 +- paddle/cinn/utils/string.h | 2 +- .../ps/service/coordinator_client.cc | 2 +- .../fluid/distributed/ps/service/ps_client.h | 2 +- .../distributed/ps/service/ps_graph_client.cc | 2 +- .../distributed/ps/service/ps_local_client.h | 2 +- .../ps/service/ps_service/graph_py_service.cc | 4 ++-- .../fluid/distributed/ps/table/depends/dense.h | 2 +- .../fluid/distributed/ps/table/tensor_table.h | 2 +- .../fluid/distributed/test/graph_node_test.cc | 2 +- paddle/fluid/eager/general_grad.h | 4 ++-- .../eager/to_static/run_program_op_node.h | 2 +- .../details/fetch_barrier_op_handle.h | 2 +- .../framework/ir/transfer_layout_elim_pass.cc | 2 +- .../new_executor/interpreter/data_transfer.cc | 10 +++++----- .../framework/new_executor/new_executor_defs.h | 4 ++-- paddle/fluid/imperative/layer.cc | 2 +- paddle/fluid/imperative/layout_transformer.h | 2 +- paddle/fluid/imperative/parallel_context.h | 2 +- paddle/fluid/imperative/partial_grad_engine.cc | 2 +- .../inference/api/onnxruntime_predictor.h | 2 +- paddle/fluid/inference/api/paddle_api.h | 4 ++-- paddle/fluid/inference/api/paddle_tensor.h | 2 +- paddle/fluid/inference/capi_exp/pd_config.h | 18 +++++++++--------- paddle/fluid/inference/capi_exp/pd_predictor.h | 2 +- .../ir_adaptor/translator/op_compat_info.h | 2 +- .../ir_adaptor/translator/op_translator.cc | 4 ++-- .../ir_adaptor/translator/program_translator.h | 6 +++--- .../pir/transforms/pd_op_to_kernel_pass.cc | 8 ++++---- .../transforms/transform_general_functions.h | 2 +- paddle/fluid/pybind/eager_method.cc | 8 ++++---- paddle/fluid/pybind/pir.cc | 4 ++-- .../phi/api/yaml/generator/backward_api_gen.py | 2 +- .../phi/api/yaml/generator/dist_bw_api_gen.py | 2 +- .../core/distributed/auto_parallel/dist_attr.h | 2 +- paddle/phi/infermeta/spmd_rules/concat.cc | 4 ++-- .../spmd_rules/cross_entropy_with_softmax.cc | 12 ++++++------ paddle/phi/kernels/cpu/rnn_grad_kernel.cc | 2 +- paddle/phi/kernels/gpu/mode_grad_kernel.cu | 2 +- paddle/phi/kernels/gpu/mode_kernel.cu | 2 +- paddle/phi/kernels/gpu/top_k_grad_kernel.cu | 2 +- paddle/phi/kernels/gpu/top_k_kernel.cu | 4 ++-- 86 files changed, 142 insertions(+), 142 deletions(-) diff --git a/paddle/cinn/backends/codegen_cuda_dev.cc b/paddle/cinn/backends/codegen_cuda_dev.cc index 83b5da6c0b138..eb70ebe8fff8e 100644 --- a/paddle/cinn/backends/codegen_cuda_dev.cc +++ b/paddle/cinn/backends/codegen_cuda_dev.cc @@ -436,7 +436,7 @@ void CodeGenCUDA_Dev::Visit(const ir::Let *op) { str_ += " "; IrPrinter::Visit(op->symbol); vectorized_tensor_names_.insert(utils::GetStreamCnt(op->symbol)); - // skip "=0" in "half8 temp = 0;" sincethe operator= of half8 may not + // skip "=0" in "half8 temp = 0;" since the operator= of half8 may not // overloaded. if (op->body.As() && op->body.As()->value == 0) { return; diff --git a/paddle/cinn/frontend/interpreter.cc b/paddle/cinn/frontend/interpreter.cc index 2a5685572a045..12964fb8e79ad 100644 --- a/paddle/cinn/frontend/interpreter.cc +++ b/paddle/cinn/frontend/interpreter.cc @@ -108,7 +108,7 @@ void Interpreter::Impl::Build(const Target& target, const std::string& model_name) { CHECK(!var_map_.empty()); VLOG(3) << "Program:\n" << *program_; - // applay frontend pass + // apply frontend pass std::unordered_set fetch_var_ids; for (auto& name : fetch_names_) { CHECK(var_map_.count(name)) << "var_map finds no fetch var " << name; diff --git a/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_cluster_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_cluster_pass.cc index b36afc9bd056f..9f9856004646f 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_cluster_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_cluster_pass.cc @@ -563,7 +563,7 @@ bool CanOpMergeNode( } // TODO(phlrain): need update here - // diffrent loop range can merge, like [128, 128, 1], with [128, 128] + // different loop range can merge, like [128, 128, 1], with [128, 128] if ((cinn::hlir::framework::pir::CompatibleInfo::OpKind(*cur_op) != cinn::hlir::framework::kBroadcast) && (op_path_info.at(cur_op).loop_ranges != @@ -584,7 +584,7 @@ bool ShouldOutputPreNode( } // TODO(phlrain): need update here - // diffrent loop range can merge, like [128, 128, 1], with [128, 128] + // different loop range can merge, like [128, 128, 1], with [128, 128] if ((cinn::hlir::framework::pir::CompatibleInfo::OpKind(*cur_op) != cinn::hlir::framework::kBroadcast) && (op_path_info.at(cur_op).loop_ranges != @@ -599,7 +599,7 @@ std::vector NodeMergeWithNode( const std::vector& first_stage_output) { // stage 2 merge // for now we merge node in same pass - // only for vertial fuse + // only for vertical fuse std::vector second_stage_output = first_stage_output; while (true) { bool fused = false; diff --git a/paddle/cinn/hlir/framework/graph.h b/paddle/cinn/hlir/framework/graph.h index d6ef914f0846e..9ce024059439c 100644 --- a/paddle/cinn/hlir/framework/graph.h +++ b/paddle/cinn/hlir/framework/graph.h @@ -189,7 +189,7 @@ class Graph : public cinn::common::Graph { SharedGroupHasher, SharedGroupComparator> producer_groups_; - // output grous + // output groups std::unordered_set, SharedGroupHasher, SharedGroupComparator> @@ -271,7 +271,7 @@ class Graph : public cinn::common::Graph { const std::unordered_set& fetch_var_ids = {}); /** - * \brief Genereate the python test code for group test + * \brief Generate the python test code for group test */ std::string GenerateGroupPythonCode( const std::vector& group, diff --git a/paddle/cinn/hlir/framework/instruction_test.cc b/paddle/cinn/hlir/framework/instruction_test.cc index 2e2b412cf4fdf..f665c628b5a0a 100644 --- a/paddle/cinn/hlir/framework/instruction_test.cc +++ b/paddle/cinn/hlir/framework/instruction_test.cc @@ -104,7 +104,7 @@ TEST(Instruction, RunWithRawPodArgs) { const auto& shape = Shape({M, N}); std::map name2podargs; - // case 1: create cinn_pod_value_t arguments dicrectly + // case 1: create cinn_pod_value_t arguments directly std::vector args_buffer( 3); // store {"x", "y", "z"} buffer objects auto* default_memory_mng = MemoryManager::Global().RetrieveSafely( diff --git a/paddle/cinn/hlir/framework/memory.h b/paddle/cinn/hlir/framework/memory.h index 3b8c59887d7fe..889e32e7fca0b 100644 --- a/paddle/cinn/hlir/framework/memory.h +++ b/paddle/cinn/hlir/framework/memory.h @@ -37,7 +37,7 @@ class MemoryInterface { }; /** - * MemoryManager holds a map of MemoryInterface for each articture. + * MemoryManager holds a map of MemoryInterface for each architecture. */ class MemoryManager final { public: diff --git a/paddle/cinn/hlir/framework/node.h b/paddle/cinn/hlir/framework/node.h index 764492df45f38..7f08257bd2d9c 100644 --- a/paddle/cinn/hlir/framework/node.h +++ b/paddle/cinn/hlir/framework/node.h @@ -77,7 +77,7 @@ class Node : public cinn::common::GraphNode { std::tuple LinkTo( NodeData *other); - // This node determines another node, which means the other node depeneds on + // This node determines another node, which means the other node depends on // this node. void Controls(NodeData *other); @@ -161,7 +161,7 @@ class NodeData : public cinn::common::GraphNode { std::tuple LinkTo( Node *other); - // This node determines another node, which means the other node depeneds on + // This node determines another node, which means the other node depends on // this node. void Controls(Node *other); diff --git a/paddle/cinn/hlir/framework/op_lowering_impl.cc b/paddle/cinn/hlir/framework/op_lowering_impl.cc index cef5968639511..a9bb46c8a4f26 100644 --- a/paddle/cinn/hlir/framework/op_lowering_impl.cc +++ b/paddle/cinn/hlir/framework/op_lowering_impl.cc @@ -547,7 +547,7 @@ ir::Expr OpLowererImpl::DoGroupSchedule( << ir_sch.GetModule().GetExprs().at(0); continue; } - // find master to computeat. + // find master to compute at. auto master = GetMasterToComputeAt(node, nodes_in_order, nodes_inline, diff --git a/paddle/cinn/hlir/framework/op_lowering_impl.h b/paddle/cinn/hlir/framework/op_lowering_impl.h index 5e57c607c93e1..80c79b3c64b8d 100644 --- a/paddle/cinn/hlir/framework/op_lowering_impl.h +++ b/paddle/cinn/hlir/framework/op_lowering_impl.h @@ -29,7 +29,7 @@ // Fusion Op lowering, there are four kinds of lowering function: // Elementwise/Broadcast/Injective,Reduce,OutEWiseFusable,NonFusible. -// Elementwise/Broadcast/Injective Ops is with same shcedule. +// Elementwise/Broadcast/Injective Ops is with same schedule. // Reduce,OutEWiseFusable,NonFusible are using different schedule. namespace cinn { @@ -183,7 +183,7 @@ class OpLowererImpl : public OpLowererImplBase { const absl::flat_hash_map& type_dict_; const absl::flat_hash_map& shape_dict_; - // fucntion name prefix + // function name prefix const std::string func_name_prefix = "fn_"; }; diff --git a/paddle/cinn/hlir/framework/op_lowering_impl_base.h b/paddle/cinn/hlir/framework/op_lowering_impl_base.h index b67deedbbb7c5..edd5c6e8e627e 100644 --- a/paddle/cinn/hlir/framework/op_lowering_impl_base.h +++ b/paddle/cinn/hlir/framework/op_lowering_impl_base.h @@ -20,7 +20,7 @@ // Fusion Op lowering, there are four kinds of lowering function: // Elementwise/Broadcast/Injective,Reduce,OutEWiseFusable,NonFusible. -// Elementwise/Broadcast/Injective Ops is with same shcedule. +// Elementwise/Broadcast/Injective Ops is with same schedule. // Reduce,OutEWiseFusable,NonFusible are using different schedule. namespace cinn { diff --git a/paddle/cinn/hlir/framework/op_lowering_test.cc b/paddle/cinn/hlir/framework/op_lowering_test.cc index 07fcc7a48e016..be33fa25125d2 100644 --- a/paddle/cinn/hlir/framework/op_lowering_test.cc +++ b/paddle/cinn/hlir/framework/op_lowering_test.cc @@ -208,7 +208,7 @@ TEST(OP_LOWERING, Reduce_Fuse_Broadcast_Softmax) { { // softmax auto A = net_builder.CreateInput(Float(32), {h, w}, "A"); - // redece max + // reduce max auto B = net_builder.ReduceMax(A, {1}); // broadcast auto C = net_builder.BroadcastTo(B, {h, w}, {0}); diff --git a/paddle/cinn/hlir/framework/op_lowering_util.cc b/paddle/cinn/hlir/framework/op_lowering_util.cc index a7b988a735cdb..2366fd584aa0b 100644 --- a/paddle/cinn/hlir/framework/op_lowering_util.cc +++ b/paddle/cinn/hlir/framework/op_lowering_util.cc @@ -622,7 +622,7 @@ void LoopAssignReduceWithoutLast(ir::IRSchedule& ir_sch, // NOLINT // the loop size at axis is 1, need remove axes_shift_num[j] = -1; } else if (axes[j] > idx) { - // the axies value need left shift + // the axes value need left shift axes_shift_num[j]++; } } @@ -902,7 +902,7 @@ Node* GetMasterToComputeAt( done_schedule.insert(tmp); } } - // remove all consuemr reducer node of node from done_schedule. + // remove all consumer reducer node of node from done_schedule. std::unordered_set visited; std::queue candidates; candidates.push(node); diff --git a/paddle/cinn/hlir/framework/pir_compiler.h b/paddle/cinn/hlir/framework/pir_compiler.h index bcb1e835e3cb1..5edf5e25bf46b 100644 --- a/paddle/cinn/hlir/framework/pir_compiler.h +++ b/paddle/cinn/hlir/framework/pir_compiler.h @@ -28,7 +28,7 @@ namespace hlir { namespace framework { // TODO(Aurelius84): Need abstract this logic to implement Proxy for -// the co-existance with GraphCompiler. +// the co-existence with GraphCompiler. class PirCompiler final { public: PirCompiler(const ::pir::Program& prog, diff --git a/paddle/cinn/hlir/pass/general_fusion_merge_pass/default_horizontal_fuse_pass.cc b/paddle/cinn/hlir/pass/general_fusion_merge_pass/default_horizontal_fuse_pass.cc index 36ac222e099c4..e953caf20ab7a 100644 --- a/paddle/cinn/hlir/pass/general_fusion_merge_pass/default_horizontal_fuse_pass.cc +++ b/paddle/cinn/hlir/pass/general_fusion_merge_pass/default_horizontal_fuse_pass.cc @@ -71,7 +71,7 @@ class DefaultHorizontalFusePass final : public HorizontalFusePass { break; } - // if can't fuse to othors Groups, new Groups. + // if can't fuse to other Groups, new Groups. if (!fusionable) { fusionable_consumers.push_back({candidate}); } diff --git a/paddle/cinn/hlir/pass/general_fusion_merge_pass/default_input_fuse_pass.cc b/paddle/cinn/hlir/pass/general_fusion_merge_pass/default_input_fuse_pass.cc index a5ba335f049f1..7dc68d65599f9 100644 --- a/paddle/cinn/hlir/pass/general_fusion_merge_pass/default_input_fuse_pass.cc +++ b/paddle/cinn/hlir/pass/general_fusion_merge_pass/default_input_fuse_pass.cc @@ -72,7 +72,7 @@ class DefaultInputFusePass final : public InputFusePass { break; } - // if can't fuse to othors Groups, new Groups. + // if can't fuse to other Groups, new Groups. if (!fusionable) { fusionable_consumers.push_back({candidate}); } diff --git a/paddle/cinn/hlir/pass/op_fusion_pass.cc b/paddle/cinn/hlir/pass/op_fusion_pass.cc index 242b72f77e77f..1f9922899b69f 100644 --- a/paddle/cinn/hlir/pass/op_fusion_pass.cc +++ b/paddle/cinn/hlir/pass/op_fusion_pass.cc @@ -361,7 +361,7 @@ class OpFusionPassHelper : public FusionHelperBase { struct FusionRelation { // producer -> consumer std::unordered_set op_kind = {}; - // producer -> fusion sonsumer + // producer -> fusion consumer std::unordered_map fusion_op_kind = {}; }; diff --git a/paddle/cinn/hlir/pe/load_x86_params.cc b/paddle/cinn/hlir/pe/load_x86_params.cc index aa0fd02218f90..36278f5a0b276 100644 --- a/paddle/cinn/hlir/pe/load_x86_params.cc +++ b/paddle/cinn/hlir/pe/load_x86_params.cc @@ -156,7 +156,7 @@ void LoadX86DefaultParams( InputX86Param(model_data, "X86ScheduleConv input 1 256 56 56 weight 512 256 1 1 stride 2 " "2 padding 0 0 dilation 1 1", - // Todo: tempory fix, enhance alterlayout and test performance + // Todo: temporary fix, enhance alterlayout and test performance {{"ic_bn", {1, 256}}, {"oc_bn", {16, 32}}, {"ow_bn", {7, 4}}, diff --git a/paddle/cinn/hlir/pe/reduction.cc b/paddle/cinn/hlir/pe/reduction.cc index e4850e96dabcd..7e33a1475e48b 100644 --- a/paddle/cinn/hlir/pe/reduction.cc +++ b/paddle/cinn/hlir/pe/reduction.cc @@ -287,7 +287,7 @@ std::vector WarpReduce(const ir::Tensor& A, reduce_width = reduce_width * A->shape[idx].as_int32(); } - // comput tmp output shape. + // compute tmp output shape. std::vector tmp_shape(A->shape.begin(), A->shape.begin() + shape_size_without_reduce_dim); tmp_shape.push_back(Expr(32)); @@ -390,7 +390,7 @@ std::vector BlockReduceInternal(const ir::Tensor& A, auto tmp_out = Compute( tmp_shape, [=](const std::vector& indexs) -> Expr { - // comput index map from output to input. + // compute index map from output to input. auto last_index = indexs.back(); std::vector input_indexs(indexs.begin(), indexs.begin() + indexs.size() - 1); diff --git a/paddle/cinn/hlir/pe/schedule.cc b/paddle/cinn/hlir/pe/schedule.cc index c75f9aefccf29..3c3067ce436ab 100644 --- a/paddle/cinn/hlir/pe/schedule.cc +++ b/paddle/cinn/hlir/pe/schedule.cc @@ -220,7 +220,7 @@ void MatmulScheduleCPU(poly::StageMap stages, int packed_last_dim = packedB->shape[packedB_dims - 1].as_int32(); int packedB_split_factor = GetBetterSplitFactor(packed_last_dim, basic_split_factor); - // tempory solution for indivisible case + // temporary solution for indivisible case if (packedB_split_factor >= 8 && packed_last_dim % packedB_split_factor == 0) { stages[packedB]->Vectorize(packedB_dims - 1, packedB_split_factor); @@ -243,7 +243,7 @@ void MatmulScheduleCPU(poly::StageMap stages, std::vector all_axes_inner; bool is_m_splited = false; bool is_n_splited = false; - // tempory solution for isl for1 wrong elimination + // temporary solution for isl for1 wrong elimination if (bm >= 4 && M != bm) { auto axes = stages[output]->Split(i_axis, bm); all_axes_outer.push_back(std::get<0>(axes)); @@ -305,7 +305,7 @@ void MatmulScheduleCPU(poly::StageMap stages, std::swap(all_axes[out_axis_dims - 1], all_axes[out_axis_dims - 2]); } stages[output]->Reorder(all_axes); - // vectorize output's last dimemsion + // vectorize output's last dimension auto out_domain = stages[output]->transformed_domain(); auto range = poly::isl_set_get_axis_range(out_domain.get(), out_axis_dims - 1); @@ -315,7 +315,7 @@ void MatmulScheduleCPU(poly::StageMap stages, int out_last_dim = max.get_num_si() + 1; int output_split_factor = GetBetterSplitFactor(out_last_dim, basic_split_factor); - // tempory solution for indivisible case + // temporary solution for indivisible case if (output_split_factor >= 8 && packed_last_dim % output_split_factor == 0) { stages[output]->Vectorize(out_axis_dims - 1, output_split_factor); } @@ -945,7 +945,7 @@ void Conv2d_NCHWc_1X1_Schedule_CPU(poly::StageMap stages, // oh_inner, ow, oc_inner, ic, kh, kw] stages[CC]->ComputeAt2(stages[packed_out], 0); VLOG(3) << "cache write shape: " << utils::Join(CC->shape, ", "); - // tempory solution because reorder may be wrong before ComputeAt + // temporary solution because reorder may be wrong before ComputeAt // reorder: [batch_oc_outer_oh_outer_fused, oh_inner, ow_outer, ow_inner, // oc_inner] -> [batch_oc_outer_oh_outer_fused, ow_outer, oh_inner, ow_inner, // oc_inner] @@ -1082,7 +1082,7 @@ void Conv2d_NCHWc_1X1_Schedule_CPU_Nofuse(poly::StageMap stages, << stages[packed_out]->transformed_domain(); VLOG(3) << "stages[CC]->transformed_domain()" << stages[CC]->transformed_domain(); - // tempory solution because reordering before computeAt may be wrong + // temporary solution because reordering before computeAt may be wrong // reorder: [batch, oc_outer, oh_outer, oh_inner, ow_outer, ow_inner, // oc_inner] -> [batch, oc_outer, oh_outer, ow_outer, oh_inner, ow_inner, // oc_inner] @@ -2700,7 +2700,7 @@ void CudaScheduleInjectiveWithVectorize(poly::Stage *stage, // the first bind position from tail int bind_idx = stage->n_out_dims() - 1; // it will add a new dim by split before vectorize, but the new dim will - // be eleminated when vectorizng, so the bind_idx does't need to increase + // be eliminated when vectorizing, so the bind_idx does't need to increase if (vector_width > 1) { stage->Split(bind_idx, vector_width); } diff --git a/paddle/cinn/hlir/pe/schedule.h b/paddle/cinn/hlir/pe/schedule.h index 8e863c50e5b6c..7aef85c77518e 100644 --- a/paddle/cinn/hlir/pe/schedule.h +++ b/paddle/cinn/hlir/pe/schedule.h @@ -182,7 +182,7 @@ void CudaScheduleMul(poly::StageMap stages, const std::vector &output_shape, const cinn::common::Target &target); -// reduce shedules. +// reduce schedules. void CudaReduceSchedule(poly::StageMap stages, ir::Tensor output, int last_dimension_num, diff --git a/paddle/cinn/hlir/pe/transform.h b/paddle/cinn/hlir/pe/transform.h index 8f46ae400694f..ad3ca5a0f9caa 100644 --- a/paddle/cinn/hlir/pe/transform.h +++ b/paddle/cinn/hlir/pe/transform.h @@ -154,7 +154,7 @@ ir::Tensor Reverse(const ir::Tensor& input, /** * @brief Perform meta op Transpose * @param input The input tensor - * @param axis tranpsoe axis + * @param axis transpose axis * @param output_name the name of the output tensor */ ir::Tensor Transpose( @@ -197,8 +197,8 @@ ir::Tensor SliceSymbolic(const ir::Tensor& A, * @param input The input tensor * @param assign The assign tensor * @param axis select axis - * @param starts select reigon starts - * @param strides select reigon strides + * @param starts select region starts + * @param strides select region strides * @param output_name the name of the output tensor */ ir::Tensor SliceAssign( diff --git a/paddle/cinn/ir/buffer.h b/paddle/cinn/ir/buffer.h index b5e162ae52bc6..5b173b6792c19 100755 --- a/paddle/cinn/ir/buffer.h +++ b/paddle/cinn/ir/buffer.h @@ -83,7 +83,7 @@ class _Buffer_ : public ExprNode<_Buffer_> { int offset_factor{0}; //! The place the buffer locates. Target target{UnkTarget()}; - //! Aignment requirement of data pointer in bytes. + //! Alignment requirement of data pointer in bytes. mutable int data_alignment{0}; //! The memory type of the buffer. MemoryType memory_type{MemoryType::Heap}; diff --git a/paddle/cinn/ir/group_schedule/st_shape_group_scheduler.cc b/paddle/cinn/ir/group_schedule/st_shape_group_scheduler.cc index 86f114def4146..7c999205f646f 100644 --- a/paddle/cinn/ir/group_schedule/st_shape_group_scheduler.cc +++ b/paddle/cinn/ir/group_schedule/st_shape_group_scheduler.cc @@ -373,7 +373,7 @@ void StaticShapeGroupScheduler::DoLoopAlignment() { source_loops = {source_loop}; } - // 3. Rerorder loops to match the target loops + // 3. Reorder loops to match the target loops if (total_source_extent == total_master_loop_extents) { ir_sch_->Reorder(node->id(), recover_loop_order); } diff --git a/paddle/cinn/ir/group_schedule/tactic/compute_inline_tactic.cc b/paddle/cinn/ir/group_schedule/tactic/compute_inline_tactic.cc index e58929eb0845b..8da8f44d32695 100644 --- a/paddle/cinn/ir/group_schedule/tactic/compute_inline_tactic.cc +++ b/paddle/cinn/ir/group_schedule/tactic/compute_inline_tactic.cc @@ -32,7 +32,7 @@ void ComputeInlineTactic::Init(ScheduleContext* context) { void ComputeInlineTactic::Apply(ir::IRSchedule* sch, const std::string& block_id) { - // TODO(LiuYang): Compute of ops will be rewrited so that we + // TODO(LiuYang): Compute of ops will be rewritten so that we // don't use it in dynamic group_schedule rules temporarily. // if (IsProhibitScheduleExternCallBlock(node->Block())) { // return; diff --git a/paddle/cinn/ir/schedule/impl/loop_transformation.cc b/paddle/cinn/ir/schedule/impl/loop_transformation.cc index e222489b53daf..b320f6ace3f69 100644 --- a/paddle/cinn/ir/schedule/impl/loop_transformation.cc +++ b/paddle/cinn/ir/schedule/impl/loop_transformation.cc @@ -166,7 +166,7 @@ std::vector DyScheduleImpl::Split(const Expr& loop, CINN_IR_SCHEDULE_END(this->err_msg_level_); } -// TODO(@LiuYang): now -1 can't exsit in factors, +// TODO(@LiuYang): now -1 can't exist in factors. std::vector DyScheduleImpl::Split(const Expr& loop, const std::vector& factors) { CINN_IR_SCHEDULE_BEGIN(); diff --git a/paddle/cinn/ir/schedule/ir_schedule_util.h b/paddle/cinn/ir/schedule/ir_schedule_util.h index ecf0521555d59..c71f12cab42c7 100644 --- a/paddle/cinn/ir/schedule/ir_schedule_util.h +++ b/paddle/cinn/ir/schedule/ir_schedule_util.h @@ -128,7 +128,7 @@ void ReplaceExpr(Expr* source, * and change -1 to positive integer. * @param factors The original factors. * @param total_extent The extent of the loop to be splitted. - * @return return The valiated factors. + * @return return The validated factors. */ std::vector ValidateFactors(const std::vector& factors, int total_extent, @@ -312,7 +312,7 @@ IterRange RangeUnion(const IterRange& range1, const IterRange& range2); * block * \param is_store_provided Whether Store nodes of the block provide the * tensor, true means it is in compute_at case, otherwise false means in - * reverse_compuate_at case + * reverse_compute_at case * \return Each index's range and can_keep_loop flag of block's tensor. * Indicating the buffer region being required. */ diff --git a/paddle/cinn/ir/schedule/schedule_desc.h b/paddle/cinn/ir/schedule/schedule_desc.h index 4458bcb4ed117..db7dc551e7ddd 100644 --- a/paddle/cinn/ir/schedule/schedule_desc.h +++ b/paddle/cinn/ir/schedule/schedule_desc.h @@ -31,10 +31,10 @@ namespace ir { // records all transform/getting operations executed by a corresponding // ir::IRSchedule. A ScheduleDesc can be serialized to JSON format and saved to // file. For deserializing, it can be re-applied to a new IRSchedule that is -// initialzied by a semantics-equal original ir::ModuleExpr, and then achieves +// initialized by a semantics-equal original ir::ModuleExpr, and then achieves // the same result. -class IRSchedule; // forward declartion to avoid cross-reference +class IRSchedule; // forward declaration to avoid cross-reference class ScheduleDesc { public: // each operation executed through IRSchedule is recorded as a step @@ -77,7 +77,7 @@ class ScheduleDesc { void Pop(); /** - * \brief Replay this description to a new IRSchedule that is initialzied by a + * \brief Replay this description to a new IRSchedule that is initialized by a * semantics-equal original ModuleExpr. * @param schedule The original IRSchedule to be replayed the description on. * @param without_post_schedule Determine whether to delete the post diff --git a/paddle/cinn/ir/tensor.cc b/paddle/cinn/ir/tensor.cc index fe746f1b8daa3..7b3f15c6ed0be 100644 --- a/paddle/cinn/ir/tensor.cc +++ b/paddle/cinn/ir/tensor.cc @@ -336,7 +336,7 @@ ir::Tensor _Tensor_::InitReduction(poly::StageMap stages, const Target &target) const { CHECK(contains_reduce_axis()) << "InitReduction only works on a reduce tensor"; - // return if already rexists. + // return if already exists. std::string init_reduce_tensor_name = GenReduceInitTensorNameOf(name); if (stages->Lookup(init_reduce_tensor_name)) return stages[this]->LookupCtrlDepend(init_reduce_tensor_name); @@ -471,7 +471,7 @@ void _Tensor_::Bind(lang::Buffer &buffer) { if (this->buffer == buffer.buffer()) return; this->buffer->Unbind(this); } - // Extract the tensors thouse has binded to this buffer. + // Extract the tensors those has binded to this buffer. buffer_depended_tensor_names_ = buffer.buffer()->binded_tensor_names(); buffer.buffer()->BindTo(this); diff --git a/paddle/cinn/ir/utils/ir_nodes_collector.h b/paddle/cinn/ir/utils/ir_nodes_collector.h index 7bfb1b3b4e6b3..28f77b3c7021c 100644 --- a/paddle/cinn/ir/utils/ir_nodes_collector.h +++ b/paddle/cinn/ir/utils/ir_nodes_collector.h @@ -80,7 +80,7 @@ std::map CollectTensorMap( std::vector CollectUndefinedVars(const Expr* e); /** - * Collect the Tensor Nodes which will be Writed by Store or Call Nodes + * Collect the Tensor Nodes which will be written by Store or Call Nodes */ std::set CollectTensorNeedsWrite(const Expr* e); } // namespace ir_utils diff --git a/paddle/cinn/lang/README.md b/paddle/cinn/lang/README.md index 078759b6f4d25..2e6c81ad655ac 100644 --- a/paddle/cinn/lang/README.md +++ b/paddle/cinn/lang/README.md @@ -1,6 +1,6 @@ # Design of CINN/DSL This module is a simple DSL defined in CINN project. -The DSL module aims to represent the overall computation in a hardware indenpendent way. +The DSL module aims to represent the overall computation in a hardware independent way. ## Concepts ### Object diff --git a/paddle/cinn/lang/lower_impl.cc b/paddle/cinn/lang/lower_impl.cc index 41904b7742d82..1b085c03e2240 100644 --- a/paddle/cinn/lang/lower_impl.cc +++ b/paddle/cinn/lang/lower_impl.cc @@ -108,7 +108,7 @@ Expr LowerGroup(const poly::ScheduleGroup& group, // poly::IslAstNodeToCinnExpr(ast, &e); poly::IslAstNodeToCinnExpr(ast, gen.domain(), &e); // now we get a workable expression, but the statement are something like - // `B(((16 * po0) + po1), po2)`, we need to transform this to some realworld + // `B(((16 * po0) + po1), po2)`, we need to transform this to some real world // statement in CINN. VLOG(1) << "ast to expr: \n" << e << std::endl; diff --git a/paddle/cinn/optim/buffer_assign.cc b/paddle/cinn/optim/buffer_assign.cc index 256624617cc43..47cf714e1d684 100644 --- a/paddle/cinn/optim/buffer_assign.cc +++ b/paddle/cinn/optim/buffer_assign.cc @@ -90,7 +90,7 @@ std::map InitialAssignBuffer( for (auto& item : buffer_updated_tensor) { auto* cur_n = uf_map[item.first]; for (auto& other : stages[item.second]->meta.tensors_to_share_buffer_with) { - // we might intialize the buffer in args. + // we might initialize the buffer in args. auto* other_n = uf_map[other]; if (!other_n) continue; diff --git a/paddle/cinn/optim/compute_inline_expand.cc b/paddle/cinn/optim/compute_inline_expand.cc index 576d438280e34..f6b7c6f24e2b8 100644 --- a/paddle/cinn/optim/compute_inline_expand.cc +++ b/paddle/cinn/optim/compute_inline_expand.cc @@ -187,7 +187,7 @@ struct SSANode : public cinn::common::GraphNode { static constexpr char *__type_info__ = "optim::SSANode"; }; -// TODO(Superjomn) the graph here is not a SSA now, it is flattern for the +// TODO(Superjomn) the graph here is not a SSA now, it is flatten for the // ir::CollectIRNodes method collects all the tensors recursively, so it can not // reserve the level information, fix it. struct SSABuilder : public ir::IRMutator<> { diff --git a/paddle/cinn/optim/resize_buffer.cc b/paddle/cinn/optim/resize_buffer.cc index c725d9d0c3c01..e73929a97aa57 100644 --- a/paddle/cinn/optim/resize_buffer.cc +++ b/paddle/cinn/optim/resize_buffer.cc @@ -153,7 +153,7 @@ class AnalyzeLoopVarRange : public ir::IRMutator<> { // We only use the maximal of var, maximal of Mod operation, // which may not be the maximal of index - // mathmetically, but it works for current CINN. + // mathematically, but it works for current CINN. // // We may add better computation of MaxIndexRange if we need for (int i = 0; i < vars.size(); ++i) { diff --git a/paddle/cinn/optim/update_buffer_axis_pass.cc b/paddle/cinn/optim/update_buffer_axis_pass.cc index d86f2e61a0019..b43b7fc834914 100644 --- a/paddle/cinn/optim/update_buffer_axis_pass.cc +++ b/paddle/cinn/optim/update_buffer_axis_pass.cc @@ -219,7 +219,7 @@ class ReplaceSameAxisToZero : public ir::IRMutator<> { for (auto p : buffer_name_access_same_index_expr_.at(buffer_name)) { int r = p.first; // After optimization, some load indice may be removed, so we need this - // conditioin + // condition if (indices->size() > r) { ir::ir_utils::IrReplace( &(indices->at(r)), indices->at(r), ir::Expr(0)); diff --git a/paddle/cinn/optim/vectorize_loops.cc b/paddle/cinn/optim/vectorize_loops.cc index 25530c74b2358..67e309c73a6a0 100644 --- a/paddle/cinn/optim/vectorize_loops.cc +++ b/paddle/cinn/optim/vectorize_loops.cc @@ -80,7 +80,7 @@ class TensorVectorizeTeller : public ir::IRMutator { const int factor_; const absl::flat_hash_map *var_intervals_; - // save (tensor name) -> (bool flag) to indentify whether tensors can be + // save (tensor name) -> (bool flag) to identify whether tensors can be // vectorized or not std::unordered_map tensor2flag_; diff --git a/paddle/cinn/poly/poly_scheduler.cc b/paddle/cinn/poly/poly_scheduler.cc index 8d0b02ca69b49..539be8221d8df 100644 --- a/paddle/cinn/poly/poly_scheduler.cc +++ b/paddle/cinn/poly/poly_scheduler.cc @@ -124,7 +124,7 @@ std::vector PartitionGraphByIterationDomain(cinn::common::Graph* graph) { } // NOTE DEBUG - // check there are same count of nodes both in the orginal graph and the + // check there are same count of nodes both in the original graph and the // groups. // @{ int num_node_in_groups = 0; diff --git a/paddle/cinn/poly/stage.h b/paddle/cinn/poly/stage.h index ac36e5fd98e09..f9d2204312e81 100644 --- a/paddle/cinn/poly/stage.h +++ b/paddle/cinn/poly/stage.h @@ -59,7 +59,7 @@ struct StageForloopInfo { ir::DeviceAPI device; }; -//! Store the infomations about some other tensor `compute_at` this tensor. +//! Store the informations about some other tensor `compute_at` this tensor. struct ComputeAtInfo { ComputeAtInfo(const std::string& consumer_tensor_name, const std::string& producer_tensor_name, @@ -277,7 +277,7 @@ class Stage : public Object { * \brief Mark the stage compute at the level of some other stage. Usually * used when there is no access relation between two tensors. * - * The difference bewteen ComputeAt2 and ComputeAt is that ComputeAt2 can be + * The difference between ComputeAt2 and ComputeAt is that ComputeAt2 can be * used when there is no access relation between two tensors. * * @param other the target stage to compute at. @@ -373,7 +373,7 @@ class Stage : public Object { const isl::map& transform() const { return transform_; } isl::set transformed_domain() const; - // Dealing with the `ComputateAt` transform. + // Dealing with the `ComputeAt` transform. std::vector compute_ats() const; //! Get the level-th dimensional name. @@ -470,7 +470,7 @@ class Stage : public Object { */ void InitTransform(); - //! Lock the \p level-th axis and disallow the futher schedules on this axis. + //! Lock the \p level-th axis and disallow the further schedules on this axis. void LockAxis(uint32_t level); //! Unlock the \p level-th axis. void UnlockAxis(uint32_t level); diff --git a/paddle/cinn/pybind/common.cc b/paddle/cinn/pybind/common.cc index 80ff3abba928d..7d777af91204a 100644 --- a/paddle/cinn/pybind/common.cc +++ b/paddle/cinn/pybind/common.cc @@ -217,7 +217,7 @@ void BindShared(py::module *m) { .def("val", &cinn::common::RefCount::val); } -// TODO(wanghaipeng03) using true_type or false_type as tag disptcher losses +// TODO(wanghaipeng03) using true_type or false_type as tag dispatcher losses // semantic context template inline auto __binary_op_fn_dispatch(T1 x, T2 y, F fn, std::true_type) { diff --git a/paddle/cinn/runtime/custom_function_test.cc b/paddle/cinn/runtime/custom_function_test.cc index 546599f252cc2..b2dc09b1862f0 100644 --- a/paddle/cinn/runtime/custom_function_test.cc +++ b/paddle/cinn/runtime/custom_function_test.cc @@ -132,7 +132,7 @@ TEST(CinnAssertTrue, test_true) { CinnBufferAllocHelper x(cinn_x86_device, cinn_bool_t(), {1}); - // set inpute value true + // set input value true bool input_h = true; auto* input = x.mutable_data(target); @@ -170,7 +170,7 @@ TEST(CinnAssertTrue, test_false_only_warning) { CinnBufferAllocHelper x(cinn_x86_device, cinn_bool_t(), {1}); - // set inpute value false + // set input value false bool input_h = false; auto* input = x.mutable_data(target); diff --git a/paddle/cinn/runtime/tiny_runtime.cc b/paddle/cinn/runtime/tiny_runtime.cc index fc2a4693328c2..2f940007aed40 100644 --- a/paddle/cinn/runtime/tiny_runtime.cc +++ b/paddle/cinn/runtime/tiny_runtime.cc @@ -25,7 +25,7 @@ extern "C" { int max_num_workers = std::thread::hardware_concurrency(); -// move to standlone file +// move to standalone file struct param_context_t { int major_v; int minor_v; diff --git a/paddle/cinn/utils/event.h b/paddle/cinn/utils/event.h index 5d7b8113a1d8b..0ceccaa029d76 100644 --- a/paddle/cinn/utils/event.h +++ b/paddle/cinn/utils/event.h @@ -76,7 +76,7 @@ class Summary { struct Item { HostEvent info; Ratio sub_ratio{0.0}; // percentage of EventType - Ratio total_ratio{0.0}; // precentage of total process + Ratio total_ratio{0.0}; // percentage of total process explicit Item(const HostEvent& e) : info(e) {} bool operator<(const Item& other) const { diff --git a/paddle/cinn/utils/multi_threading.h b/paddle/cinn/utils/multi_threading.h index aff17e32bb762..6cf0afccbe311 100644 --- a/paddle/cinn/utils/multi_threading.h +++ b/paddle/cinn/utils/multi_threading.h @@ -43,7 +43,7 @@ class SequenceDispatcher : public JobDispatcher { int Next() const override; private: - // the maxmimum index of extent + // the maximum index of extent int end_; // the traversal step to the next one int step_; diff --git a/paddle/cinn/utils/random_engine.h b/paddle/cinn/utils/random_engine.h index 05162b288c781..49e8e6ecfd2a2 100644 --- a/paddle/cinn/utils/random_engine.h +++ b/paddle/cinn/utils/random_engine.h @@ -34,7 +34,7 @@ namespace utils { class LinearRandomEngine { public: using StateType = int64_t; - // the type name "resule_type" is needed by std::xxx_distribution + // the type name "result_type" is needed by std::xxx_distribution using result_type = uint32_t; // The minimum possible value of random state diff --git a/paddle/cinn/utils/registry.h b/paddle/cinn/utils/registry.h index 8f6aa8b5b0c7f..3958b19d84378 100644 --- a/paddle/cinn/utils/registry.h +++ b/paddle/cinn/utils/registry.h @@ -145,7 +145,7 @@ class Registry { * \endcode * * @tparam EntryType The type of subclass that inheritate the base. - * @tparam FunctionType The function type this registry is registerd. + * @tparam FunctionType The function type this registry is registered. */ template class FunctionRegEntryBase { diff --git a/paddle/cinn/utils/string.h b/paddle/cinn/utils/string.h index 900e1a6a2ed57..b891d3abb980d 100644 --- a/paddle/cinn/utils/string.h +++ b/paddle/cinn/utils/string.h @@ -31,7 +31,7 @@ std::string GetStreamCnt(const T& x); * Construct a formatted string with arguments. * @param fmt_str The format. * @param ... The parameters of the format. - * @return The formated string. + * @return The formatted string. */ std::string StringFormat(const std::string& fmt_str, ...); diff --git a/paddle/fluid/distributed/ps/service/coordinator_client.cc b/paddle/fluid/distributed/ps/service/coordinator_client.cc index c9c2ba49c9bf3..691b427d2bfde 100644 --- a/paddle/fluid/distributed/ps/service/coordinator_client.cc +++ b/paddle/fluid/distributed/ps/service/coordinator_client.cc @@ -44,7 +44,7 @@ void CoordinatorService::FLService( uint32_t from_client_id = request->client_id(); VLOG(0) << "fl-ps > recv from client id: " << from_client_id << ", msg_type: " << msg_type; - // TODO(ziyoujiyi): find is not thread safe, beacuse of RB_Tree traversal + // TODO(ziyoujiyi): find is not thread safe, because of RB_Tree traversal auto itr = _service_handle_map.find(msg_type); if (itr == _service_handle_map.end()) { LOG(ERROR) << "fl-ps > unknown flClient2Coordinator msg type: " << msg_type; diff --git a/paddle/fluid/distributed/ps/service/ps_client.h b/paddle/fluid/distributed/ps/service/ps_client.h index 63a9967793976..ecebe9bcd3ac1 100644 --- a/paddle/fluid/distributed/ps/service/ps_client.h +++ b/paddle/fluid/distributed/ps/service/ps_client.h @@ -197,7 +197,7 @@ class PSClient { // server优雅退出 virtual std::future StopServer() = 0; - // server profilera + // server profiler virtual std::future StartProfiler() = 0; virtual std::future StopProfiler() = 0; diff --git a/paddle/fluid/distributed/ps/service/ps_graph_client.cc b/paddle/fluid/distributed/ps/service/ps_graph_client.cc index 4bf084e109e22..e4ce866c8ebc5 100644 --- a/paddle/fluid/distributed/ps/service/ps_graph_client.cc +++ b/paddle/fluid/distributed/ps/service/ps_graph_client.cc @@ -73,7 +73,7 @@ void PsGraphClient::FinalizeWorker() { } simple::global_rpc_server().finalize(); } -// add maco +// add macro #define DIM_PASS_ID(dim_id, pass_id) \ uint32_t((uint32_t(dim_id) << 16) | pass_id) #define GET_PASS_ID(id) (id & 0xffff) diff --git a/paddle/fluid/distributed/ps/service/ps_local_client.h b/paddle/fluid/distributed/ps/service/ps_local_client.h index 61e9bf7a688f0..d05dab680039b 100644 --- a/paddle/fluid/distributed/ps/service/ps_local_client.h +++ b/paddle/fluid/distributed/ps/service/ps_local_client.h @@ -99,7 +99,7 @@ class PsLocalClient : public PSClient { size_t num); virtual ::std::future Flush(); - // server profilera + // server profiler virtual std::future StartProfiler() { std::promise prom; std::future fut = prom.get_future(); diff --git a/paddle/fluid/distributed/ps/service/ps_service/graph_py_service.cc b/paddle/fluid/distributed/ps/service/ps_service/graph_py_service.cc index f041143a80836..ff4035a39d30f 100644 --- a/paddle/fluid/distributed/ps/service/ps_service/graph_py_service.cc +++ b/paddle/fluid/distributed/ps/service/ps_service/graph_py_service.cc @@ -52,7 +52,7 @@ void GraphPyService::add_table_feat_conf(std::string table_name, VLOG(0) << "table_name " << table_name << " mapping id " << idx; VLOG(0) << " feat name " << feat_name << " feat id" << feat_idx; if (static_cast(feat_idx) < table_feat_conf_feat_name[idx].size()) { - // overide + // override table_feat_conf_feat_name[idx][feat_idx] = feat_name; table_feat_conf_feat_dtype[idx][feat_idx] = feat_dtype; table_feat_conf_feat_shape[idx][feat_idx] = feat_shape; @@ -285,7 +285,7 @@ void GraphPyClient::load_edge_file(std::string name, status.wait(); } // if (this->table_id_map.count(name)) { - // VLOG(0) << "loadding data with type " << name << " from " << filepath; + // VLOG(0) << "loading data with type " << name << " from " << filepath; // uint32_t table_id = this->table_id_map[name]; // auto status = // get_ps_client()->Load(table_id, std::string(filepath), params); diff --git a/paddle/fluid/distributed/ps/table/depends/dense.h b/paddle/fluid/distributed/ps/table/depends/dense.h index 14f99d8b41779..146b9de91ac87 100644 --- a/paddle/fluid/distributed/ps/table/depends/dense.h +++ b/paddle/fluid/distributed/ps/table/depends/dense.h @@ -29,7 +29,7 @@ namespace paddle { namespace distributed { // dense optimizer -// TODO(tangwei12) integrate with sparse optimzer later. +// TODO(tangwei12) integrate with sparse optimizer later. class DenseOptimizer { public: DenseOptimizer() {} diff --git a/paddle/fluid/distributed/ps/table/tensor_table.h b/paddle/fluid/distributed/ps/table/tensor_table.h index fa58ddfdd705d..5dd27d7298e5f 100644 --- a/paddle/fluid/distributed/ps/table/tensor_table.h +++ b/paddle/fluid/distributed/ps/table/tensor_table.h @@ -185,7 +185,7 @@ class GlobalStepTable : public DenseTensorTable { } if (main_program_id_ != -1) { - // Run main porgram, if program is used for learning decay + // Run main program, if program is used for learning decay auto main_program_desc = sub_program_->at(main_program_id_); auto main_ctx = executor_->Prepare(main_program_desc, 0); exec_context_ = std::move(main_ctx); diff --git a/paddle/fluid/distributed/test/graph_node_test.cc b/paddle/fluid/distributed/test/graph_node_test.cc index bcf822dc0156f..26207a9ad8c9e 100644 --- a/paddle/fluid/distributed/test/graph_node_test.cc +++ b/paddle/fluid/distributed/test/graph_node_test.cc @@ -103,7 +103,7 @@ void testFeatureNodeSerializeFloat64() { ASSERT_LE(eps * eps, 1e-5); } -// void testSingleSampleNeighboor( +// void testSingleSampleNeighbour( // std::shared_ptr& worker_ptr_) { // std::vector> vs; // std::vector> vs1; diff --git a/paddle/fluid/eager/general_grad.h b/paddle/fluid/eager/general_grad.h index 724b6938c28e2..443455619cae6 100644 --- a/paddle/fluid/eager/general_grad.h +++ b/paddle/fluid/eager/general_grad.h @@ -166,10 +166,10 @@ class GeneralGrad { } // TODO(jiabin): May we need some check here. } - // Get Graph Info Betweent input target GradNode and outputs, + // Get Graph Info Between input target GradNode and outputs, // record depending_nodes_ void GetGraphInfoBetweenTargets(const std::deque& init_queue) { - VLOG(6) << "Runing In GetGraphInfoBetweenTargets"; + VLOG(6) << "Running In GetGraphInfoBetweenTargets"; // Copy nodes std::deque queue = init_queue; diff --git a/paddle/fluid/eager/to_static/run_program_op_node.h b/paddle/fluid/eager/to_static/run_program_op_node.h index 17cb367e72980..fdebfbb1e3771 100644 --- a/paddle/fluid/eager/to_static/run_program_op_node.h +++ b/paddle/fluid/eager/to_static/run_program_op_node.h @@ -1518,7 +1518,7 @@ class PirGradNodeRunProgram : public egr::GradNodeBase { x.size(), x_grad_values.size())); - // TODO(dev): Need an elegant way to determine inforamtion of grad_tensor, + // TODO(dev): Need an elegant way to determine information of grad_tensor, // such as: name, tensor type(DenseTensor or SelectedRows). for (size_t i = 0; i < x.size(); i++) { if (x[i].is_dense_tensor()) { diff --git a/paddle/fluid/framework/details/fetch_barrier_op_handle.h b/paddle/fluid/framework/details/fetch_barrier_op_handle.h index 1364c742890cc..b48b379339500 100644 --- a/paddle/fluid/framework/details/fetch_barrier_op_handle.h +++ b/paddle/fluid/framework/details/fetch_barrier_op_handle.h @@ -39,7 +39,7 @@ namespace paddle { namespace framework { namespace details { -// **NOTE**: fetch_barrier op is special it outputs all recved variables on +// **NOTE**: fetch_barrier op is special it outputs all received variables on // all places if there are multiple places, must init with // multiple dev_ctxes_ !!!! diff --git a/paddle/fluid/framework/ir/transfer_layout_elim_pass.cc b/paddle/fluid/framework/ir/transfer_layout_elim_pass.cc index 9ca4d4f482f08..3a9a2c81889ee 100644 --- a/paddle/fluid/framework/ir/transfer_layout_elim_pass.cc +++ b/paddle/fluid/framework/ir/transfer_layout_elim_pass.cc @@ -156,7 +156,7 @@ bool TransferLayoutElimPass::AllInputIsTransferlayout( for (auto var : op_node->inputs) { // If this input is a 1D persistable tensor，we allow transfer_layout not - // appear before this var, but temporarily diasble this if. + // appear before this var, but temporarily disable this if. if (var->Var()->Persistable() && false) { auto var_dims = scope->FindVar(var->Name())->GetMutable()->dims(); diff --git a/paddle/fluid/framework/new_executor/interpreter/data_transfer.cc b/paddle/fluid/framework/new_executor/interpreter/data_transfer.cc index 929b20cb27752..3bc5893a162b3 100644 --- a/paddle/fluid/framework/new_executor/interpreter/data_transfer.cc +++ b/paddle/fluid/framework/new_executor/interpreter/data_transfer.cc @@ -60,7 +60,7 @@ bool DataTransferHelper::apply(const phi::KernelKey& kernel_type_for_var, is_transferred = true; } - // 2. dype transform + // 2. dtype transform if (need_dtype_transform(kernel_type_for_var, expected_kernel_key)) { auto op = TransferDtype( *src_var_name, @@ -168,7 +168,7 @@ void DataTransferHelper::RunAndConstructOpFuncNode( // their implementations are device-related. // For example, consider changing the layout of a gpu tensor // while the gpu kernel of transfer_layout op does not exist. - // To use the cpu kernel, you must insert memcpy_d2h/mepcpy_h2d op + // To use the cpu kernel, you must insert memcpy_d2h/memcpy_h2d op // in addition. But such operation should not be done here. // Maybe in future we will support this. } @@ -357,7 +357,7 @@ std::shared_ptr TransferDtype(const std::string& var_name, AttributeMap attr_map; attr_map["in_dtype"] = static_cast(in_dtype); attr_map["out_dtype"] = static_cast(out_dtype); - // NOTE(Aurelius84): In whice case use_mkldnn = true? + // NOTE(Aurelius84): In which case use_mkldnn = true? attr_map["use_mkldnn"] = false; // 3. Create transfer_dtype_op @@ -532,7 +532,7 @@ void ApplyDataTransform(const OpKernelType& expected_kernel_key, // for some situation like InferShape(). // In this situation We cannot skip Var analysis, as // MKL-DNN shape of Var may differ from kNHWC Var - // In such situation corressponding resized Var + // In such situation corresponding resized Var // has to be created and registered if ((tensor_in->layout() == DataLayout::ONEDNN) && (var->IsType() == true) && @@ -736,7 +736,7 @@ void ApplyDataTransform(const OpKernelType& expected_kernel_key, } if (transfered) { - // NOTE(zhiqiu): UPDATE the corresponding OeratorBase to make it consistent + // NOTE(zhiqiu): UPDATE the corresponding OperatorBase to make it consistent // with instruction. op_base->Inputs() = new_ins; op_base->Outputs() = new_outs; diff --git a/paddle/fluid/framework/new_executor/new_executor_defs.h b/paddle/fluid/framework/new_executor/new_executor_defs.h index 49b2ed3b73f96..df82aedfcec5f 100644 --- a/paddle/fluid/framework/new_executor/new_executor_defs.h +++ b/paddle/fluid/framework/new_executor/new_executor_defs.h @@ -348,7 +348,7 @@ static constexpr char kFetchVarName[] = "fetch"; // static_ref_ is the numer of last live ops calculated to statically after // `build` the Instructions. dynamic_ref_ is the runtime version ref which will -// be decreased by one dynamiclly after the execution of an op (in last ops +// be decreased by one dynamically after the execution of an op (in last ops // list). var_ is the related variable // The dynamic_ref_ is initialized to static_ref_ first, and is decreased to 1 @@ -379,7 +379,7 @@ class VarRefInfo { // static_dep_ is the numer of dependencies (ops that must run before it) of // each op which is calculated to statically. static_dep_ is the runtime -// version dep which will be decreased by one dynamiclly after the execution of +// version dep which will be decreased by one dynamically after the execution of // one dependency op. // The dynamic_dep_ is initialized to static_dep_ first, and is decreased to 1 diff --git a/paddle/fluid/imperative/layer.cc b/paddle/fluid/imperative/layer.cc index fcb20b2a1109a..5192e8c773888 100644 --- a/paddle/fluid/imperative/layer.cc +++ b/paddle/fluid/imperative/layer.cc @@ -251,7 +251,7 @@ void VarBase::ClearGradient(bool set_to_zero) { #endif } } - // TODO(zhouwei): It's better to free memory of grad by grad_t->claer. + // TODO(zhouwei): It's better to free memory of grad by grad_t->clear. // But will have some bug on mac CPU of yolov3 model, why? // After fix this bug, function SetIsEmpty() isn't need grad_var_->SharedVar()->SetIsEmpty(true); diff --git a/paddle/fluid/imperative/layout_transformer.h b/paddle/fluid/imperative/layout_transformer.h index 87748ca5d102e..349d2f5b5eb36 100644 --- a/paddle/fluid/imperative/layout_transformer.h +++ b/paddle/fluid/imperative/layout_transformer.h @@ -337,7 +337,7 @@ class TransposeOpTransformer auto desired_layout = LayoutAutoTune::Instance().GetDesiredLayout(); if (var_layout == desired_layout && desired_layout == DataLayout::NHWC) { auto axis = PADDLE_GET_CONST(std::vector, (*attrs)["axis"]); - // NHWC->NCHW, permutaion will be set as follows. + // NHWC->NCHW, permutation will be set as follows. std::vector perm = {0, 3, 1, 2}; // fuse the transpose Ops by transforming axis. std::vector fusion_axis = { diff --git a/paddle/fluid/imperative/parallel_context.h b/paddle/fluid/imperative/parallel_context.h index e0fd05562a413..2d15ed51e58a1 100644 --- a/paddle/fluid/imperative/parallel_context.h +++ b/paddle/fluid/imperative/parallel_context.h @@ -65,7 +65,7 @@ class ParallelContext { // if CPU, should do nothing. virtual void WaitComm(int ring_id) = 0; - // synchorize compute stream + // synchronize compute stream virtual void SynchronizeCompute() = 0; inline int GetNRings() const { return strategy_.nrings_; } diff --git a/paddle/fluid/imperative/partial_grad_engine.cc b/paddle/fluid/imperative/partial_grad_engine.cc index 3fd37d5ec3674..0a5d44a1e1e57 100644 --- a/paddle/fluid/imperative/partial_grad_engine.cc +++ b/paddle/fluid/imperative/partial_grad_engine.cc @@ -325,7 +325,7 @@ static void FillConstantLike(const VariableWrapper &ref_var, auto *dst_tensor = dst_var->MutableVar()->GetMutable(); auto *dev_ctx = platform::DeviceContextPool::Instance().Get(place); dst_tensor->Resize(ref_tensor.dims()); - // TOOD(jiabin): Ugly fix here we have fwd_data_type_ and data_type, since in + // TODO(jiabin): Ugly fix here we have fwd_data_type_ and data_type, since in // grad mission // we can't get data_type_ directly. We need to check if we can only use // default data_type for now. diff --git a/paddle/fluid/inference/api/onnxruntime_predictor.h b/paddle/fluid/inference/api/onnxruntime_predictor.h index b52a40d29ff26..33c37042aac43 100644 --- a/paddle/fluid/inference/api/onnxruntime_predictor.h +++ b/paddle/fluid/inference/api/onnxruntime_predictor.h @@ -155,7 +155,7 @@ class ONNXRuntimePredictor : public PaddlePredictor { /// /// \brief Get the Output Tensor object /// - /// \param[in] name otuput name + /// \param[in] name output name /// \return output tensor /// std::unique_ptr GetOutputTensor( diff --git a/paddle/fluid/inference/api/paddle_api.h b/paddle/fluid/inference/api/paddle_api.h index 2828fd65a6ee7..8c66b66363603 100644 --- a/paddle/fluid/inference/api/paddle_api.h +++ b/paddle/fluid/inference/api/paddle_api.h @@ -167,7 +167,7 @@ struct PD_INFER_DECL PaddleTensor { /// to device, /// eliminating additional CPU copy. ZeroCopyTensor is only used in the /// AnalysisPredictor. -/// It is obtained through PaddlePredictor::GetinputTensor() +/// It is obtained through PaddlePredictor::GetInputTensor() /// and PaddlePredictor::GetOutputTensor() interface. class PD_INFER_DECL ZeroCopyTensor : public paddle_infer::Tensor { @@ -467,7 +467,7 @@ PD_INFER_DECL std::shared_ptr MakeCipher( } // namespace paddle -// forward declation +// forward declaration using cudaStream_t = struct CUstream_st*; using hipStream_t = struct ihipStream_t*; diff --git a/paddle/fluid/inference/api/paddle_tensor.h b/paddle/fluid/inference/api/paddle_tensor.h index 22cd023c1fee2..0817f0a1b9919 100644 --- a/paddle/fluid/inference/api/paddle_tensor.h +++ b/paddle/fluid/inference/api/paddle_tensor.h @@ -78,7 +78,7 @@ enum class DataLayout { kUNK = -1, kAny, kNHWC, kNCHW }; /// to device, /// eliminating additional CPU copy. Tensor is only used in the /// AnalysisPredictor. -/// It is obtained through PaddlePredictor::GetinputTensor() +/// It is obtained through PaddlePredictor::GetInputTensor() /// and PaddlePredictor::GetOutputTensor() interface. class PD_INFER_DECL Tensor { public: diff --git a/paddle/fluid/inference/capi_exp/pd_config.h b/paddle/fluid/inference/capi_exp/pd_config.h index 92d5d0e3a6308..427e9b95ac499 100644 --- a/paddle/fluid/inference/capi_exp/pd_config.h +++ b/paddle/fluid/inference/capi_exp/pd_config.h @@ -46,7 +46,7 @@ PADDLE_CAPI_EXPORT extern __pd_give PD_Config* PD_ConfigCreate(); /// PADDLE_CAPI_EXPORT extern void PD_ConfigDestroy(__pd_take PD_Config* pd_config); /// -/// \brief Set the combined model with two specific pathes for program and +/// \brief Set the combined model with two specific paths for program and /// parameters. /// /// \param[in] pd_config config @@ -169,7 +169,7 @@ PADDLE_CAPI_EXPORT extern void PD_ConfigEnableONNXRuntime( PADDLE_CAPI_EXPORT extern void PD_ConfigDisableONNXRuntime( __pd_keep PD_Config* pd_config); /// -/// \brief A boolean state telling whether the ONNXRutnime is turned on. +/// \brief A boolean state telling whether the ONNXRuntime is turned on. /// /// \return Whether the ONNXRuntime is turned on. /// @@ -238,11 +238,11 @@ PADDLE_CAPI_EXPORT extern int32_t PD_ConfigGpuDeviceId( PADDLE_CAPI_EXPORT extern int32_t PD_ConfigXpuDeviceId( __pd_keep PD_Config* pd_config); /// -/// \brief Turn on custome device. +/// \brief Turn on custom device. /// /// \param[in] pd_config config /// \param[in] device_type device type -/// \param[in] device_id device_id the custome device card to use. +/// \param[in] device_id device_id the custom device card to use. /// PADDLE_CAPI_EXPORT extern void PD_ConfigEnableCustomDevice( __pd_keep PD_Config* pd_config, char* device_type, int32_t device_id); @@ -306,13 +306,13 @@ PADDLE_CAPI_EXPORT extern PD_Bool PD_ConfigCudnnEnabled( /// If turned off, the AnalysisConfig will act just like a NativeConfig. /// /// \param[in] pd_config config -/// \param[in] x Whether the ir graph optimization is actived. +/// \param[in] x Whether the ir graph optimization is activated. /// PADDLE_CAPI_EXPORT extern void PD_ConfigSwitchIrOptim( __pd_keep PD_Config* pd_config, PD_Bool x); /// /// \brief A boolean state telling whether the ir graph optimization is -/// actived. +/// activated. /// /// \param[in] pd_config config /// \return Whether to use ir graph optimization. @@ -321,7 +321,7 @@ PADDLE_CAPI_EXPORT extern PD_Bool PD_ConfigIrOptim( __pd_keep PD_Config* pd_config); /// /// \brief Turn on the TensorRT engine. -/// The TensorRT engine will accelerate some subgraphes in the original Fluid +/// The TensorRT engine will accelerate some subgraphs in the original Fluid /// computation graph. In some models such as resnet50, GoogleNet and so on, /// it gains significant performance acceleration. /// @@ -330,7 +330,7 @@ PADDLE_CAPI_EXPORT extern PD_Bool PD_ConfigIrOptim( /// workspace. /// \param[in] max_batch_size The maximum batch size of this prediction task, /// better set as small as possible for less performance loss. -/// \param[in] min_subgrpah_size The minimum TensorRT subgraph size needed, if a +/// \param[in] min_subgraph_size The minimum TensorRT subgraph size needed, if a /// subgraph is smaller than this, it will not be transferred to TensorRT /// engine. /// \param[in] precision The precision used in TensorRT. @@ -490,7 +490,7 @@ PADDLE_CAPI_EXPORT extern PD_Bool PD_ConfigTensorRtDlaEnabled( /// \brief Turn on the usage of Lite sub-graph engine. /// /// \param[in] pd_config config -/// \param[in] precision Precion used in Lite sub-graph engine. +/// \param[in] precision Precision used in Lite sub-graph engine. /// \param[in] zero_copy whether use zero copy. /// \param[in] passes_filter_num The number of passes used in Lite sub-graph /// engine. diff --git a/paddle/fluid/inference/capi_exp/pd_predictor.h b/paddle/fluid/inference/capi_exp/pd_predictor.h index a35defb910070..f17d78f9155af 100644 --- a/paddle/fluid/inference/capi_exp/pd_predictor.h +++ b/paddle/fluid/inference/capi_exp/pd_predictor.h @@ -40,7 +40,7 @@ extern "C" { /// \brief Create a new Predictor /// /// \param[in] Config config -/// \return new predicor. +/// \return new predictor. /// PADDLE_CAPI_EXPORT extern __pd_give PD_Predictor* PD_PredictorCreate( __pd_take PD_Config* pd_config); diff --git a/paddle/fluid/ir_adaptor/translator/op_compat_info.h b/paddle/fluid/ir_adaptor/translator/op_compat_info.h index fa13415ffdfd6..bcd56ac07da92 100644 --- a/paddle/fluid/ir_adaptor/translator/op_compat_info.h +++ b/paddle/fluid/ir_adaptor/translator/op_compat_info.h @@ -126,7 +126,7 @@ class OpNameNormalizer { return ret.value(); } } else if (is_grad_op && !is_grad_arg) { - // backwward op using forward args: like trace_grad using forward input + // backward op using forward args: like trace_grad using forward input size_t type_pos = op_type.find(kPhiGradSuffix); if (auto ret = GetDirectMapping(op_type.substr(0, type_pos), arg_name)) { VLOG(10) << "[" << op_type << "] found " << ret.value(); diff --git a/paddle/fluid/ir_adaptor/translator/op_translator.cc b/paddle/fluid/ir_adaptor/translator/op_translator.cc index 9295e5c643e5f..3acb0f4fc0718 100644 --- a/paddle/fluid/ir_adaptor/translator/op_translator.cc +++ b/paddle/fluid/ir_adaptor/translator/op_translator.cc @@ -556,7 +556,7 @@ std::vector OpTranscriber::GenerateOperationInput( auto defining_info = (*param_map)[legacy_input_vars[0]]; op_inputs.push_back(defining_info.value); - // if src type is Vector , need an additional `CombineOp` to + // if src type is Vector , need an additional `CombineOp` to // assemble them. } else { auto* combine_op = InsertCombineOperationForTarget( @@ -654,7 +654,7 @@ OpTranscriber::GenerateOperationOutput(pir::IrContext* ctx, arg_to_idx[var_name] = {cur_output_idx, 0}; op_output_types.push_back(translated_var_type); - // if src type is Vector + // if src type is Vector } else { VLOG(10) << "[output translating]" << "[" << op_desc.Type() << "]" << info.name << " :" diff --git a/paddle/fluid/ir_adaptor/translator/program_translator.h b/paddle/fluid/ir_adaptor/translator/program_translator.h index f2c4096113a8e..c335c36670c6d 100644 --- a/paddle/fluid/ir_adaptor/translator/program_translator.h +++ b/paddle/fluid/ir_adaptor/translator/program_translator.h @@ -74,7 +74,7 @@ class TranslationContext { Container container_; TranslationContext* parent_ = nullptr; std::vector> - sons_; // used to seperate different block + sons_; // used to separate different block }; class ProgramTranslator { @@ -100,11 +100,11 @@ class ProgramTranslator { std::unordered_map parameter_name_mappings_; std::unordered_set parameter_visited_; - /// In the legacy program desc, there are two special named varibales: + /// In the legacy program desc, there are two special named variables: /// 1. "feed", the input variable of feed op /// 2. "fetch", the output variable of fetch op /// However, new feed has no input and new fetch has no output - /// So we don't handle these two vairables when + /// So we don't handle these two variables when /// `Get/SetParameterFromSingleBlock` static const std::unordered_set no_cast_var_names; diff --git a/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc b/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc index 9cd2c89eda866..f82ec0cbcdf1d 100644 --- a/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc +++ b/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc @@ -251,7 +251,7 @@ static phi::Backend DeriveBackend(const std::string& op, const OpYamlInfoParser* op_info_parser, phi::Backend kernel_backend, size_t input_index) { - // NOTE: Parameters are initilizered on executor place defined + // NOTE: Parameters are initialized on executor place defined if ((op.compare(pir::SetParameterOp::name()) == 0 || op.compare(pir::ShadowOutputOp::name()) == 0) && place.GetType() == phi::AllocationType::GPU) { @@ -958,7 +958,7 @@ phi::KernelKey GetKernelKey( phi::DataType kernel_dtype = phi::DataType::UNDEFINED; if (op_info_parser != nullptr) { - // only suppurt non vector input for now + // only support non vector input for now int tensor_input_number = static_cast(op_info_parser->InputTensorNumber()); VLOG(8) << "Begin to infer kernel key from op_info_parser(defined by yaml " @@ -1018,7 +1018,7 @@ phi::KernelKey GetKernelKey( // Because we can't make sure the place when build data op // and the output place of data op is undefined. It means we // don't know how to select the kernel in the next of op that - // uses data op outout as inputs. So, we need set kernel backend + // uses data op output as inputs. So, we need set kernel backend // manually. auto op_res = input_tmp.dyn_cast(); if (!op_res) { @@ -2089,7 +2089,7 @@ std::vector BuildInputs( new_in, out_type, in_place, out_place, kernel_key, block); } } else if (new_in_type.isa()) { - // [ todo need update here, support combine data transfomer] + // [ todo need update here, support combine data transformer] // deal with pre combine op auto pre_define_op = cur_in.defining_op(); if (pre_define_op->isa<::pir::CombineOp>()) { diff --git a/paddle/fluid/pir/transforms/transform_general_functions.h b/paddle/fluid/pir/transforms/transform_general_functions.h index cec04f3712990..8b9ffdd8cf477 100644 --- a/paddle/fluid/pir/transforms/transform_general_functions.h +++ b/paddle/fluid/pir/transforms/transform_general_functions.h @@ -24,7 +24,7 @@ namespace pir { /** - * @brief Get the name of pararmeter from a value. + * @brief Get the name of parameter from a value. * * @note The value must be a output of a ParameterOp or a ConstantTensorOp. * diff --git a/paddle/fluid/pybind/eager_method.cc b/paddle/fluid/pybind/eager_method.cc index 2094fef07a873..9dc8897a10a41 100644 --- a/paddle/fluid/pybind/eager_method.cc +++ b/paddle/fluid/pybind/eager_method.cc @@ -2208,7 +2208,7 @@ PyDoc_STRVAR(tensor_method_indices__doc__, Returns the indices of non zero elements in input SparseCooTensor. Returns: - DenseTesnor + DenseTensor Examples: @@ -2252,7 +2252,7 @@ PyDoc_STRVAR(tensor_method_values__doc__, Returns the values of non zero elements in input SparseCooTensor. Returns: - DenseTesnor + DenseTensor Examples: @@ -2305,7 +2305,7 @@ PyDoc_STRVAR(tensor_method_crows__doc__, Returns the compressed row index of non zero elements in input SparseCsrTensor. Returns: - DenseTesnor + DenseTensor Examples: @@ -2349,7 +2349,7 @@ PyDoc_STRVAR(tensor_method_cols__doc__, Returns the column index of non zero elements in input SparseCsrTensor. Returns: - DenseTesnor + DenseTensor Examples: diff --git a/paddle/fluid/pybind/pir.cc b/paddle/fluid/pybind/pir.cc index 3a0de137173a7..723ff501450c0 100644 --- a/paddle/fluid/pybind/pir.cc +++ b/paddle/fluid/pybind/pir.cc @@ -1138,7 +1138,7 @@ SplitedResult SplitForwardBackward( }); auto &forward_value_map = forward_mapper.GetMutableMap(); - // backward program construc. + // backward program construct. // Step1. insert data op for inputs_values and middle_values pir::IrMapping backward_mapper; auto &backward_value_map = backward_mapper.GetMutableMap(); @@ -1160,7 +1160,7 @@ SplitedResult SplitForwardBackward( if (v.impl() == nullptr) { return; } - // NOTE(Aurelius84): we should skip insert ShadowOutputOp repeatly by + // NOTE(Aurelius84): we should skip insert ShadowOutputOp repeatedly by // calling SplitForwardBackward multi-times. std::string shadow_output_name = std::string("output_") + std::to_string(counter); diff --git a/paddle/phi/api/yaml/generator/backward_api_gen.py b/paddle/phi/api/yaml/generator/backward_api_gen.py index e2c58a30dd9a7..2201d74093b90 100644 --- a/paddle/phi/api/yaml/generator/backward_api_gen.py +++ b/paddle/phi/api/yaml/generator/backward_api_gen.py @@ -345,7 +345,7 @@ def generate_backward_api( source_include(include_header_file, include_fw_header_file) ) source_file.write(namespace[0]) - # not all fused ops supoort dygraph + # not all fused ops support dygraph if is_fused_backward_yaml is True: new_bw_apis = [ bw_api diff --git a/paddle/phi/api/yaml/generator/dist_bw_api_gen.py b/paddle/phi/api/yaml/generator/dist_bw_api_gen.py index 70c12aa9f8d42..0d2d4d16a2b63 100644 --- a/paddle/phi/api/yaml/generator/dist_bw_api_gen.py +++ b/paddle/phi/api/yaml/generator/dist_bw_api_gen.py @@ -493,7 +493,7 @@ def generate_backward_api( source_include(include_header_file, include_fw_header_file) ) source_file.write(namespace[0]) - # not all fused ops supoort dygraph + # not all fused ops support dygraph if is_fused_backward_yaml is True: new_bw_apis = [ bw_api diff --git a/paddle/phi/core/distributed/auto_parallel/dist_attr.h b/paddle/phi/core/distributed/auto_parallel/dist_attr.h index e4016b9f65cdc..a3e05c9fcdacb 100644 --- a/paddle/phi/core/distributed/auto_parallel/dist_attr.h +++ b/paddle/phi/core/distributed/auto_parallel/dist_attr.h @@ -206,7 +206,7 @@ class TEST_API TensorDistAttr { std::map annotated_; int64_t chunk_id_{0}; // partial map would be small (less than mesh.size) - // iterate operation (copy and comparision) would more frequency than random + // iterate operation (copy and comparison) would more frequency than random // element access. paddle::flat_hash_map partial_status_; }; diff --git a/paddle/phi/infermeta/spmd_rules/concat.cc b/paddle/phi/infermeta/spmd_rules/concat.cc index 666e5a8bdea3c..4e3c2ead16983 100644 --- a/paddle/phi/infermeta/spmd_rules/concat.cc +++ b/paddle/phi/infermeta/spmd_rules/concat.cc @@ -74,7 +74,7 @@ SpmdInfo ConcatInferSpmd(const std::vector& x, int axis) { }); auto non_empty_index = non_empty_iter - tensor_shapes.begin(); int64_t ndim = static_cast(tensor_shapes[non_empty_index].size()); - // normlize dim + // normalize dim auto dim = axis < 0 ? ndim + axis : axis; std::vector input_attrs; std::transform( @@ -138,7 +138,7 @@ SpmdInfo ConcatGradInferSpmdDynamic(const std::vector& x, auto non_empty_index = non_empty_iter - tensor_shapes.begin(); int64_t ndim = static_cast(tensor_shapes[non_empty_index].size()); auto dim = axis.to(); - // normlize dim + // normalize dim dim = dim < 0 ? ndim + dim : dim; std::vector input_attrs; std::transform( diff --git a/paddle/phi/infermeta/spmd_rules/cross_entropy_with_softmax.cc b/paddle/phi/infermeta/spmd_rules/cross_entropy_with_softmax.cc index 95ce5a6ecf7ff..21dc00ac1fc18 100644 --- a/paddle/phi/infermeta/spmd_rules/cross_entropy_with_softmax.cc +++ b/paddle/phi/infermeta/spmd_rules/cross_entropy_with_softmax.cc @@ -142,7 +142,7 @@ SpmdInfo CrossEntropyWithSoftmaxInferSpmdBase(const DistMetaTensor& x, &softmax_out_axes_dst, support_shard_softmax_dim); - // Step2: Sharding Propogation + // Step2: Sharding Propagation // Step2.1: merge input shardings std::unordered_map axis_to_dim_map = ShardingMergeForTensors({{x_axes_src, x_dims_mapping_src}, @@ -189,8 +189,8 @@ SpmdInfo CrossEntropyWithSoftmaxInferSpmdBase(const DistMetaTensor& x, // todo if softmax_normalize axis is sharded, notify downstream phi api to // select c_softmax_with_entropy_kernel. - // according to the phi api implemetation, the softmax_out tensor will alway - // be genereated not matter the value of use_softmax. + // according to the phi api implementation, the softmax_out tensor will alway + // be generated not matter the value of use_softmax. return {{x_dist_attr_dst, label_dist_attr_dst}, {softmax_out_dist_attr_dst, loss_dist_attr_dst}}; } @@ -290,7 +290,7 @@ SpmdInfo CrossEntropyWithSoftmaxInferSpmdReverse( &softmax_out_axes_dst, true); - // Step2: Sharding Propogation + // Step2: Sharding Propagation // Step2.1 merge output dims mappings std::unordered_map axis_to_dim_map = ShardingMergeForTensors({{loss_axes, loss_dims_mapping_src}, @@ -363,8 +363,8 @@ SpmdInfo CrossEntropyWithSoftmaxInferSpmdReverse( << str_join(x_dims_mapping) << "]\nLabel dims_mapping: [" << str_join(label_dims_mapping) << "]\n\n"; - // according to the phi api implemetation, the softmax_out tensor will alway - // be genereated not matter the value of use_softmax. + // according to the phi api implementation, the softmax_out tensor will alway + // be generated not matter the value of use_softmax. return {{x_dist_attr, label_dist_attr}, {s_out_dist_attr_dst, loss_dist_attr_dst}}; } diff --git a/paddle/phi/kernels/cpu/rnn_grad_kernel.cc b/paddle/phi/kernels/cpu/rnn_grad_kernel.cc index 3e0e4c7a3d7a5..a48d05b8d783e 100644 --- a/paddle/phi/kernels/cpu/rnn_grad_kernel.cc +++ b/paddle/phi/kernels/cpu/rnn_grad_kernel.cc @@ -1215,7 +1215,7 @@ void RnnGradFunc(const CPUContext& dev_ctx, gate_num_tmp); } - // calcluate the dropout gradient for the layer_x_grad_holder + // calculate the dropout gradient for the layer_x_grad_holder // dropout_state save in the forward process if (i > 0) { if ((!is_test) && (dropout_prob != 0)) { diff --git a/paddle/phi/kernels/gpu/mode_grad_kernel.cu b/paddle/phi/kernels/gpu/mode_grad_kernel.cu index 20eb2c29882f7..1d79f7756ae3d 100644 --- a/paddle/phi/kernels/gpu/mode_grad_kernel.cu +++ b/paddle/phi/kernels/gpu/mode_grad_kernel.cu @@ -73,7 +73,7 @@ void ModeGradKernel(const Context& dev_ctx, int pre, n, post; funcs::GetDims(in_dims, axis, &pre, &n, &post); - // calcluate the block and grid num + // calculate the block and grid num int block_size = funcs::ComputeBlockSize(post); int max_threads = dev_ctx.GetMaxPhysicalThreadCount(); const int max_blocks = std::max(((max_threads - 1) / block_size + 1), 1); diff --git a/paddle/phi/kernels/gpu/mode_kernel.cu b/paddle/phi/kernels/gpu/mode_kernel.cu index 653bd241e72b7..16ed4b9349019 100644 --- a/paddle/phi/kernels/gpu/mode_kernel.cu +++ b/paddle/phi/kernels/gpu/mode_kernel.cu @@ -38,7 +38,7 @@ void ModeKernel(const Context& dev_ctx, errors::InvalidArgument( "The dims of Input(X) should be greater than 0.")); } - // calcluate the real axis + // calculate the real axis if (axis < 0) axis += in_dims.size(); auto out_dims = out->dims(); diff --git a/paddle/phi/kernels/gpu/top_k_grad_kernel.cu b/paddle/phi/kernels/gpu/top_k_grad_kernel.cu index 6c2e880e9a9ef..e34bc5f9f6e5a 100644 --- a/paddle/phi/kernels/gpu/top_k_grad_kernel.cu +++ b/paddle/phi/kernels/gpu/top_k_grad_kernel.cu @@ -55,7 +55,7 @@ void TopkGradKernel(const Context& dev_ctx, int pre, n, post; phi::funcs::GetDims(in_dims, axis, &pre, &n, &post); - // calcluate the block and grid num + // calculate the block and grid num auto ComputeBlockSize = [](int col) { if (col > 512) return 1024; diff --git a/paddle/phi/kernels/gpu/top_k_kernel.cu b/paddle/phi/kernels/gpu/top_k_kernel.cu index 354f104e48681..1d93ef1a2790f 100644 --- a/paddle/phi/kernels/gpu/top_k_kernel.cu +++ b/paddle/phi/kernels/gpu/top_k_kernel.cu @@ -73,7 +73,7 @@ void TopkKernel(const Context& dev_ctx, phi::funcs::set_constant(dev_ctx, indices, static_cast(0)); return; } - // calcluate the real axis + // calculate the real axis if (axis < 0) axis += in_dims.size(); int k = k_scalar.to(); @@ -255,7 +255,7 @@ void TopkKernel(const Context& dev_ctx, int ndims = trans.size(); funcs::TransCompute( ndims, dev_ctx, *input, &trans_input, trans); - // third step, calcluate the topk + // third step, calculate the topk // allocate the tmp cuda memory for the tmp result DenseTensor trans_ind; DenseTensor trans_out; From 9222271cc1a7261d943b07b8e1952245f48a49d4 Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 23 Feb 2024 14:25:40 +0800 Subject: [PATCH 024/282] Update multi_threading_test.cc (#61998) --- paddle/cinn/utils/multi_threading_test.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/paddle/cinn/utils/multi_threading_test.cc b/paddle/cinn/utils/multi_threading_test.cc index abd429a4b1677..bd081fea2b56c 100644 --- a/paddle/cinn/utils/multi_threading_test.cc +++ b/paddle/cinn/utils/multi_threading_test.cc @@ -34,19 +34,19 @@ TEST(JobDispatcher, SequenceDispatcher) { TEST(parallel_run, Basic) { std::vector results(100, -1); - auto woker_fn = [&results](int index) { + auto worker_fn = [&results](int index) { CHECK_LT(index, results.size()) << "index invalid"; results[index] = index; }; // check process every index in the extent of [0, 100) with step 1 - parallel_run(woker_fn, SequenceDispatcher(0, 100), 2); + parallel_run(worker_fn, SequenceDispatcher(0, 100), 2); for (int i = 0; i < 100; ++i) { ASSERT_EQ(results[i], i); } // check only indexes in the extent of [0, 100) with step 3 are processed results.assign(100, -1); - parallel_run(woker_fn, SequenceDispatcher(0, 100, 3), 3); + parallel_run(worker_fn, SequenceDispatcher(0, 100, 3), 3); for (int i = 0; i < 100; ++i) { if (i % 3 == 0) { ASSERT_EQ(results[i], i); From 5d79cdada70f2d6c3cddaed77b50fd89bcd29f05 Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 23 Feb 2024 14:28:34 +0800 Subject: [PATCH 025/282] Add WITH_GPU check to copy FLASHATTN_LIBRARIES (#61961) * Fix * ci * ci --- python/setup.py.in | 7 ++++--- setup.py | 11 ++++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/python/setup.py.in b/python/setup.py.in index f3d80419ebeac..520a9f7f7a56c 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -685,9 +685,10 @@ if not sys.platform.startswith("linux"): package_data['paddle.libs']+=[os.path.basename('${GNU_RT_LIB_2}')] shutil.copy('${GNU_RT_LIB_2}', libs_path) -if len('${FLASHATTN_LIBRARIES}') > 1: - package_data['paddle.libs']+=[os.path.basename('${FLASHATTN_LIBRARIES}')] - shutil.copy('${FLASHATTN_LIBRARIES}', libs_path) +if '${WITH_GPU}' == 'ON': + if len('${FLASHATTN_LIBRARIES}') > 1: + package_data['paddle.libs']+=[os.path.basename('${FLASHATTN_LIBRARIES}')] + shutil.copy('${FLASHATTN_LIBRARIES}', libs_path) if '${WITH_MKL}' == 'ON': shutil.copy('${MKLML_SHARED_LIB}', libs_path) diff --git a/setup.py b/setup.py index 18dc54b6fd140..350c62bdf6301 100644 --- a/setup.py +++ b/setup.py @@ -1064,11 +1064,12 @@ def get_package_data_and_package_dir(): shutil.copy(env_dict.get("OPENBLAS_LIB") + '.0', libs_path) package_data['paddle.libs'] += ['libopenblas.so.0'] - if len(env_dict.get("FLASHATTN_LIBRARIES", "")) > 1: - package_data['paddle.libs'] += [ - os.path.basename(env_dict.get("FLASHATTN_LIBRARIES")) - ] - shutil.copy(env_dict.get("FLASHATTN_LIBRARIES"), libs_path) + if env_dict.get("WITH_GPU") == 'ON': + if len(env_dict.get("FLASHATTN_LIBRARIES", "")) > 1: + package_data['paddle.libs'] += [ + os.path.basename(env_dict.get("FLASHATTN_LIBRARIES")) + ] + shutil.copy(env_dict.get("FLASHATTN_LIBRARIES"), libs_path) if env_dict.get("WITH_LITE") == 'ON': shutil.copy(env_dict.get("LITE_SHARED_LIB"), libs_path) package_data['paddle.libs'] += [ From 3f00547fb9bbe9779e268e52e65f85b0374b5e97 Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 23 Feb 2024 14:29:40 +0800 Subject: [PATCH 026/282] Fix some typos (dst_strategys, etc.) (#62003) --- .../auto_parallel/static/auto_align_tool.py | 12 ++++++------ .../auto_parallel/static/completion.py | 2 +- .../auto_parallel/static/converter.py | 6 +++--- .../auto_parallel/static/cost/base_cost.py | 2 +- .../auto_parallel/static/cost_model.py | 10 +++++----- .../distributed/auto_parallel/static/helper.py | 4 ++-- .../auto_parallel/static/parallelizer_v2.py | 2 +- .../fleet/base/distributed_strategy.py | 2 +- python/paddle/distributed/fleet/fleet.py | 4 ++-- python/paddle/distributed/fleet/launch.py | 2 +- .../paddle/distributed/fleet/launch_utils.py | 14 +++++++------- .../hybrid_parallel_gradscaler.py | 2 +- .../fleet/meta_optimizers/sharding/utils.py | 2 +- .../sharding/weight_decay_helper.py | 2 +- .../meta_optimizers/sharding_optimizer.py | 18 +++++++++--------- .../fleet/meta_parallel/pipeline_parallel.py | 4 ++-- .../sharding/group_sharded_utils.py | 2 +- .../distributed/fleet/recompute/recompute.py | 16 ++++++++-------- .../fleet/recompute/recompute_hybrid.py | 4 ++-- .../fleet/runtime/parameter_server_runtime.py | 14 +++++++------- .../distributed/fleet/runtime/the_one_ps.py | 8 ++++---- python/paddle/distributed/fleet/scaler.py | 2 +- python/paddle/distributed/fleet/utils/fs.py | 8 ++++---- .../fleet/utils/hybrid_parallel_inference.py | 4 ++-- .../fleet/utils/hybrid_parallel_util.py | 4 ++-- .../fleet/utils/mix_precision_utils.py | 2 +- .../fleet/utils/tensor_fusion_helper.py | 2 +- .../fleet/utils/tensor_parallel_utils.py | 8 ++++---- .../distributed/launch/context/__init__.py | 4 ++-- .../launch/controllers/controller.py | 2 +- python/paddle/distributed/models/moe/utils.py | 2 +- ...auto_parallel_data_parallel_optimization.py | 2 +- .../passes/auto_parallel_recompute.py | 16 ++++++++-------- .../passes/auto_parallel_sharding.py | 4 ++-- .../distributed/passes/ps_trainer_pass.py | 4 ++-- .../ps/utils/collective_transpiler.py | 4 ++-- 36 files changed, 100 insertions(+), 100 deletions(-) diff --git a/python/paddle/distributed/auto_parallel/static/auto_align_tool.py b/python/paddle/distributed/auto_parallel/static/auto_align_tool.py index d7d98f75d80f1..b1ced07b8b24e 100644 --- a/python/paddle/distributed/auto_parallel/static/auto_align_tool.py +++ b/python/paddle/distributed/auto_parallel/static/auto_align_tool.py @@ -352,13 +352,13 @@ def convert_src_tensor_2_dst_tensor(vars_list, src_attr_map, dst_attr_map): if src_attr_map is None or len(src_attr_map) == 0: return vars_list[0] - dst_strategys = {} - src_strategys = {} + dst_strategies = {} + src_strategies = {} tensors_dict = {} convert_tensor_dict = None for var_name in src_attr_map.keys(): - assert var_name not in dst_strategys + assert var_name not in dst_strategies dist_vars = [] for vars in vars_list: if var_name in vars.keys(): @@ -367,13 +367,13 @@ def convert_src_tensor_2_dst_tensor(vars_list, src_attr_map, dst_attr_map): continue if var_name in dst_attr_map and var_name in src_attr_map: - dst_strategys[var_name] = copy.deepcopy(dst_attr_map[var_name]) - src_strategys[var_name] = copy.deepcopy(src_attr_map[var_name]) + dst_strategies[var_name] = copy.deepcopy(dst_attr_map[var_name]) + src_strategies[var_name] = copy.deepcopy(src_attr_map[var_name]) tensors_dict[var_name] = dist_vars if src_attr_map == dst_attr_map: return tensors_dict - converter = Converter(tensors_dict, src_strategys, dst_strategys) + converter = Converter(tensors_dict, src_strategies, dst_strategies) convert_tensor_dict = converter.convert() return convert_tensor_dict diff --git a/python/paddle/distributed/auto_parallel/static/completion.py b/python/paddle/distributed/auto_parallel/static/completion.py index a671582a3293f..900b90a0f6496 100644 --- a/python/paddle/distributed/auto_parallel/static/completion.py +++ b/python/paddle/distributed/auto_parallel/static/completion.py @@ -202,7 +202,7 @@ def _update_op_dims_mapping_and_distoperatorimpl( updated = dist_op_container.update_dims_mapping(dist_op) changed = updated or changed - # TODO(ljz) remove the below code once we introduce general reshard to replace specifc distopimpls + # TODO(ljz) remove the below code once we introduce general reshard to replace specific distopimpls reverted = dist_op_container.mapping_to_dist_operator_impl( dist_op, original_op_dist_attr ) diff --git a/python/paddle/distributed/auto_parallel/static/converter.py b/python/paddle/distributed/auto_parallel/static/converter.py index c7cd4e32d6e42..241a83aaf4f5d 100644 --- a/python/paddle/distributed/auto_parallel/static/converter.py +++ b/python/paddle/distributed/auto_parallel/static/converter.py @@ -105,9 +105,9 @@ def convert(self, strict=True): >>> import numpy as np >>> from paddle.distributed.auto_parallel.static.converter import Converter >>> complete_tensors = np.arange(4).reshape([2, 2]) - >>> partitial_tensors = np.split(complete_tensors, 2, axis=0) + >>> partial_tensors = np.split(complete_tensors, 2, axis=0) >>> name = "tmp_0" - >>> tensors_dict = {name: partitial_tensors} + >>> tensors_dict = {name: partial_tensors} >>> strategy_1 = { ... name: { ... "process_shape": [2], @@ -345,7 +345,7 @@ def slice_with_dist_attr(tensor, dist_attr): @staticmethod def merge(partition_tensor_list, tensor, partition_index, complete_shape): """ - Merge partitial tensors to a complete. + Merge partial tensors to a complete. Returns: None diff --git a/python/paddle/distributed/auto_parallel/static/cost/base_cost.py b/python/paddle/distributed/auto_parallel/static/cost/base_cost.py index 957e5dba46bf0..495cff26844d7 100644 --- a/python/paddle/distributed/auto_parallel/static/cost/base_cost.py +++ b/python/paddle/distributed/auto_parallel/static/cost/base_cost.py @@ -784,7 +784,7 @@ def comm_count(self): shape = None if self.op is not None: vars = self.op.block.vars - # NOTE: The tensor communicated input_name is "X" in default. Otherwise, this function should be overrided + # NOTE: The tensor communicated input_name is "X" in default. Otherwise, this function should be overridden try: var_name = self.op.input("X")[0] except: diff --git a/python/paddle/distributed/auto_parallel/static/cost_model.py b/python/paddle/distributed/auto_parallel/static/cost_model.py index 55690e4f3de8f..ad0f353815772 100644 --- a/python/paddle/distributed/auto_parallel/static/cost_model.py +++ b/python/paddle/distributed/auto_parallel/static/cost_model.py @@ -98,18 +98,18 @@ def init_comm_cost(self, cluster=None): # should get from `cluster` BANDWIDTH = 32 * 1024 / 1000 # MB/ms, V100 PCIe num_ranks = len(self.ranks) - comm_volumn = np.prod(self.input_shape) * 4 + comm_volume = np.prod(self.input_shape) * 4 if 'allreduce' in self.comm_type: - self._cost = comm_volumn / ( + self._cost = comm_volume / ( BANDWIDTH * num_ranks / (2 * (num_ranks - 1)) ) elif 'gather' in self.comm_type: - self._cost = comm_volumn / (BANDWIDTH * num_ranks / (num_ranks - 1)) + self._cost = comm_volume / (BANDWIDTH * num_ranks / (num_ranks - 1)) elif 'broadcast' in self.comm_type: - self._cost = comm_volumn / BANDWIDTH + self._cost = comm_volume / BANDWIDTH elif 'send' in self.comm_type or 'recv' in self.comm_type: - self._cost = comm_volumn / BANDWIDTH + self._cost = comm_volume / BANDWIDTH else: self._cost = 0 diff --git a/python/paddle/distributed/auto_parallel/static/helper.py b/python/paddle/distributed/auto_parallel/static/helper.py index c730a68e6ae49..e7bd7553d5094 100644 --- a/python/paddle/distributed/auto_parallel/static/helper.py +++ b/python/paddle/distributed/auto_parallel/static/helper.py @@ -211,8 +211,8 @@ class ProgramHelper: def __init__(self, layer, loss_func, metrics, inputs_spec, labels_spec): # original model config information - # TODO(Aurelius84): Implenet append_backward and optimizer in ProxyLayer - # after distribute engine satisify basic condition. + # TODO(Aurelius84): Implement append_backward and optimizer in ProxyLayer + # after distribute engine satisfy basic condition. self.proxy_layer = ProxyLayer(layer, loss_func, metrics) self.inputs_spec = inputs_spec self.labels_spec = labels_spec diff --git a/python/paddle/distributed/auto_parallel/static/parallelizer_v2.py b/python/paddle/distributed/auto_parallel/static/parallelizer_v2.py index fb924288988d1..27a13fd1d9107 100644 --- a/python/paddle/distributed/auto_parallel/static/parallelizer_v2.py +++ b/python/paddle/distributed/auto_parallel/static/parallelizer_v2.py @@ -231,7 +231,7 @@ def _generate_backward( # NOTE(zhaoyinglia): # Guarantee the order of params_grads is same between dynamic mode and static mode # by making parameter_list equal to model.parameters(), - # because the order affact the result of ClipGradByGLobalNorm. + # because the order affect the result of ClipGradByGLobalNorm. # If parameter_list is not None, the order of params_grads is same with parameter_list. # If parameter_list is None, params_grads will be as prog.global_block().all_parameters(). with program_guard(main_program, startup_program): diff --git a/python/paddle/distributed/fleet/base/distributed_strategy.py b/python/paddle/distributed/fleet/base/distributed_strategy.py index 2c3c4728d4f2e..62b79302f32dd 100755 --- a/python/paddle/distributed/fleet/base/distributed_strategy.py +++ b/python/paddle/distributed/fleet/base/distributed_strategy.py @@ -1498,7 +1498,7 @@ def sharding_configs(self): This configuration will affect the communication speed in sharding training, and should be an empirical value decided by your model size and network topology. Only enable when sharding_segment_strategy = segment_broadcast_MB. Default is 32.0 . - segment_anchors(list): list of anchors used to segment the program, which allows a finner control of program segmentation. + segment_anchors(list): list of anchors used to segment the program, which allows a finer control of program segmentation. this strategy is experimental by now. Only enable when sharding_segment_strategy = segment_anchors. sharding_degree(int, optional): specific the number of gpus within each sharding parallelism group; and sharding will be turn off if sharding_degree=1. Default is 8. diff --git a/python/paddle/distributed/fleet/fleet.py b/python/paddle/distributed/fleet/fleet.py index 81547d24878d5..c9ea552815a83 100755 --- a/python/paddle/distributed/fleet/fleet.py +++ b/python/paddle/distributed/fleet/fleet.py @@ -1194,7 +1194,7 @@ def save_persistables(self, executor, dirname, main_program=None, mode=0): dirname(str, optional): The saving directory path. When you need to save the parameter to the memory, set it to None. - main_program(Program, optional): The program whose persistbale tensors will + main_program(Program, optional): The program whose persistable tensors will be saved. Default: None. @@ -1419,7 +1419,7 @@ def amp_init( ... init_loss_scaling=128.0, ... use_dynamic_loss_scaling=True, ... use_pure_fp16=True) - ... # If you don't use the default_startup_program(), you sholud pass + ... # If you don't use the default_startup_program(), you should pass ... # your defined `startup_program` into `minimize`. ... optimizer.minimize(loss) ... exe.run(paddle.static.default_startup_program()) diff --git a/python/paddle/distributed/fleet/launch.py b/python/paddle/distributed/fleet/launch.py index dcb5e55f0c25a..146d8a627e5c5 100755 --- a/python/paddle/distributed/fleet/launch.py +++ b/python/paddle/distributed/fleet/launch.py @@ -115,7 +115,7 @@ def _parse_args(): "--backend", type=str, default=os.environ.get('PADDLE_DISTRI_BACKEND', 'auto'), - help="Specifize the backend, can be gloo|nccl|bkcl|auto|heter. " + help="Specify the backend, can be gloo|nccl|bkcl|auto|heter. " "Default value is auto which prefers nccl or bkcl.", ) base_group.add_argument( diff --git a/python/paddle/distributed/fleet/launch_utils.py b/python/paddle/distributed/fleet/launch_utils.py index 0b87df4a9c3af..c0a01d43fd688 100755 --- a/python/paddle/distributed/fleet/launch_utils.py +++ b/python/paddle/distributed/fleet/launch_utils.py @@ -339,12 +339,12 @@ def terminate_local_procs(procs): p.log_fn.close() logger.debug(f"terminate process id:{p.proc.pid}") - # wait all process terminiated + # wait all process terminated time.sleep(3) for step in range(0, 50): alive = False for p in procs: - if p.proc.poll() is None: # not termniate + if p.proc.poll() is None: # not terminate os.kill(p.proc.pid, signal.SIGKILL) alive = True @@ -414,7 +414,7 @@ def __free_port(): step += 1 if step > 400: print( - "can't find avilable port and use the specified static port now!" + "can't find available port and use the specified static port now!" ) return None @@ -705,7 +705,7 @@ def get_gpus(gpus): for x in gpus.split(',') ] logger.info( - f"Change selected_gpus into reletive values. --ips:{gpus} " + f"Change selected_gpus into relative values. --ips:{gpus} " f"will change into relative_ips:{res_gpus} according to your " f"CUDA_VISIBLE_DEVICES:{cuda_visible_devices_list}" ) @@ -736,7 +736,7 @@ def get_xpus(xpus): for x in xpus.split(',') ] logger.info( - f"Change selected_xpus into reletive values. --ips:{xpus} " + f"Change selected_xpus into relative values. --ips:{xpus} " f"will change into relative_ips:{res_xpus} according to your " f"XPU_VISIBLE_DEVICES:{xpu_visible_devices_list}" ) @@ -859,9 +859,9 @@ def get_custom_endpoints(origin_endpoints, offset=0): # assert paddle_pserver_endpoints != None # # # hard code for paddlecloud custom-framework -# avilable_ports = os.getenv("TRAINER_PORTS", "").split(",") +# available_ports = os.getenv("TRAINER_PORTS", "").split(",") # assert len( -# avilable_ports +# available_ports # ) >= 2, "set paddle_ports_num >= 2 in config.ini for paddlecloud job submit" # # # hard code for paddlecloud custom-framework diff --git a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_gradscaler.py b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_gradscaler.py index 4924d523ded05..36833fd7b5a97 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_gradscaler.py +++ b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_gradscaler.py @@ -73,7 +73,7 @@ def _unscale(self, optimizer): if not self._use_dp_mode: self._found_inf = paddle.cast(self._found_inf, dtype="int32") # TODO(shenliang03) Since the minimize call in the optimizer is - # after the gradscaler, check_finite needs to synchronize global + # after the grad scaler, check_finite needs to synchronize global # information. In the future, we should use check_group paddle.distributed.all_reduce( self._found_inf, op=paddle.distributed.ReduceOp.MAX, group=None diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding/utils.py b/python/paddle/distributed/fleet/meta_optimizers/sharding/utils.py index 05f2a4f2a28d6..852e7ced16e4a 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding/utils.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding/utils.py @@ -103,7 +103,7 @@ def check_allreduce_sum(block, shard, sharding_ring_id, dp_ring_id=-1): - 1: sync_calc - 2: reduce_sum_sharding (allreduce --> reduce) - 3: sync_comm - - 4: allreuce_sum_dp (dp_grads) + - 4: allreduce_sum_dp (dp_grads) - 5: sync_comm (dp_grads) - 6: op that use Var (dp_grads & sum) diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding/weight_decay_helper.py b/python/paddle/distributed/fleet/meta_optimizers/sharding/weight_decay_helper.py index 2ff259be18b79..1c10efb340618 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding/weight_decay_helper.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding/weight_decay_helper.py @@ -32,7 +32,7 @@ def prune_weight_decay(self, block, shard): continue if OP_ROLE_VAR_KEY not in op.attr_names: raise ValueError( - "The Weight Dacay op should hold op_role_var attribute" + "The Weight Decay op should hold op_role_var attribute" f"but the {op.type} op does not hold op_role_var" ) op_role_var = op.all_attrs()[OP_ROLE_VAR_KEY] diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py index 5d2f561ca974d..298e84ace66f1 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py @@ -217,7 +217,7 @@ def _get_hybrid_dp_mode(self): # pipeline: communication across nodes, and therefore should insert in update segment, # conduct just once per global step. dp_mode = None - # dp here is the pure dp as the outest parallelism + # dp here is the pure dp as the outermost parallelism if self.hybrid_dp: if self.pp_degree > 1: dp_mode = "pp_hybrid_dp" @@ -598,8 +598,8 @@ def _adapt_amp_clip_without_sharding(self): rings = [self.mp_ring_id, self.pp_ring_id] FP16Utils.sync_amp_check_nan_inf(main_block, rings) - gradientclip_helper = GradientClipHelper(None) - gradientclip_helper.sync_global_norm( + gradient_clip_helper = GradientClipHelper(None) + gradient_clip_helper.sync_global_norm( main_block, [self.mp_ring_id, self.pp_ring_id], self.mp_rank ) @@ -996,8 +996,8 @@ def _prune_main_program(self, block, shard, rings): 4. prune optimizer op + param + gradient """ - weightdecay_helper = WeightDecayHelper() - weightdecay_helper.prune_weight_decay(block, shard) + weight_decay_helper = WeightDecayHelper() + weight_decay_helper.prune_weight_decay(block, shard) # FIXME(wangxi): mp should prune duplicated param_grads # NOTE (JZ-LIANG) the sync of FoundInfinite should among one entire Model Parallelism @@ -1006,8 +1006,8 @@ def _prune_main_program(self, block, shard, rings): FP16Utils.prune_fp16(block, shard, self._reduced_grads_to_param, rings) # clipbyglobalnorm should only use the Model parallelism group (mp-sharding-pp) - gradientclip_helper = GradientClipHelper(None) - gradientclip_helper.prune_gradient_clip(block, shard, rings) + gradient_clip_helper = GradientClipHelper(None) + gradient_clip_helper.prune_gradient_clip(block, shard, rings) # build prog deps reduced_grads = [] @@ -1645,7 +1645,7 @@ def _build_groups(self): # global group # use for gen_nccl_comm_sync, amp check nan inf, clip by global norm - # NOTE (JZ-LIANG) when use global ring for calc global norm and dp_degree > 1, the allreduce result should be devided by dp_degree + # NOTE (JZ-LIANG) when use global ring for calc global norm and dp_degree > 1, the allreduce result should be divided by dp_degree self.global_ring_id = 3 logger.info(f"global word size: {self.global_word_size}") @@ -1727,7 +1727,7 @@ def recreate_not_persist_param_as_var(program): def _initialization_broadcast(self): """ - this funtion is to ensure the initialization between dp group to be + this function is to ensure the initialization between dp group to be identical when hybrid-dp is used, and the initialization of not distributed param between mp group to be identical. """ diff --git a/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py b/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py index 21e5dbfbffefc..384d89b4d9c12 100644 --- a/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py +++ b/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py @@ -170,7 +170,7 @@ def __init__(self, layers, hcg, strategy): 'accumulate_steps' ] # If sent tensor are not the same from different hosts, - # they shouldn't been sent partially and then concated as a whole tensor. + # they shouldn't been sent partially and then concatenated as a whole tensor. self._enable_partial_send_recv = self._strategy.pipeline_configs[ 'enable_partial_send_recv' ] @@ -640,7 +640,7 @@ def _prepare_training(self, data, optimizer, lr_scheduler): def _wrap_data(self, data): """ - for backward compatibilty, wrap data to Fake FakeMicroDataset if it is of type list or tuple + for backward compatibility, wrap data to Fake FakeMicroDataset if it is of type list or tuple """ if (not isinstance(data, tuple)) and (not isinstance(data, list)): return data diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py index 2a691c2c4d4fc..046143c79842f 100644 --- a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py +++ b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py @@ -342,6 +342,6 @@ def cvt_to_device(x, dev_id, blocking=True): place = paddle.XPUPlace(dev_id) else: raise OSError( - "Only supported compiled paddle with gpu/rocm and xpu , but current verison is compiled with cpu." + "Only supported compiled paddle with gpu/rocm and xpu, but current version is compiled with cpu." ) return x._copy_to(place, blocking) diff --git a/python/paddle/distributed/fleet/recompute/recompute.py b/python/paddle/distributed/fleet/recompute/recompute.py index 8cfa7fbec353d..b59f304d69a42 100644 --- a/python/paddle/distributed/fleet/recompute/recompute.py +++ b/python/paddle/distributed/fleet/recompute/recompute.py @@ -93,7 +93,7 @@ def check_recompute_necessary(inputs): @contextlib.contextmanager -def swith_rng_state_tracker(rng_state, tracker): +def switch_rng_state_tracker(rng_state, tracker): orig_rng_state = paddle.get_rng_state() orig_rng_tracker = get_rng_state_tracker().get_states_tracker() paddle.set_rng_state(rng_state) @@ -155,8 +155,8 @@ def forward(ctx, run_function, preserve_rng_state, *args, **kwargs): ctx.inputs.append(arg) ctx.save_for_backward(*tensor_inputs) - # NOTE recompute with restore RNG only support one senario where one process for one cuda gpu. - # one process with multiple gpu and mix-gpu-cpu senarios are not support + # NOTE recompute with restore RNG only support one scenario where one process for one cuda gpu. + # one process with multiple gpu and mix-gpu-cpu scenarios are not support if ctx.preserve_rng_state: ctx.fw_rng_state = paddle.get_rng_state() ctx.fwd_rng_state_tracker = ( @@ -208,7 +208,7 @@ def backward(ctx, *args): # NOTE support AMP # need restore auto_cast state as well as w/b list if ctx.preserve_rng_state: - with swith_rng_state_tracker( + with switch_rng_state_tracker( ctx.fw_rng_state, ctx.fwd_rng_state_tracker ): with paddle.amp.auto_cast( @@ -273,7 +273,7 @@ def backward(ctx, *args): # all tensors in the tuple doesn't need grad, only return a None for the whole tuple grads.append(None) else: - # all tensors in the tuple nees grad, should return a tuple of grads + # all tensors in the tuple need grad, should return a tuple of grads grads.append(tuple(i._grad_ivar() for i in inp)) if in_dynamic_mode(): @@ -303,7 +303,7 @@ def _recompute_without_reentrant( fw_cuda_rng_state = paddle.get_rng_state(cur_device) else: raise RuntimeError( - "Recompute with RNG perserve is not support current device: {}.".format( + "Recompute with RNG preserve is not support current device: {}.".format( cur_device ) ) @@ -358,10 +358,10 @@ def inner_pack(inner_x): return def inner_unpack(inner_x): - raise Exception("An unexcepted backward called on a tensor!") + raise Exception("An unexpected backward called on a tensor!") if preserve_rng_state: - with swith_rng_state_tracker( + with switch_rng_state_tracker( fw_cuda_rng_state, fwd_cuda_rng_state_tracker ): with paddle.set_grad_enabled(True): diff --git a/python/paddle/distributed/fleet/recompute/recompute_hybrid.py b/python/paddle/distributed/fleet/recompute/recompute_hybrid.py index 789f0cac73d94..29e7c73459854 100644 --- a/python/paddle/distributed/fleet/recompute/recompute_hybrid.py +++ b/python/paddle/distributed/fleet/recompute/recompute_hybrid.py @@ -22,7 +22,7 @@ from .recompute import ( check_recompute_necessary, detach_variable, - swith_rng_state_tracker, + switch_rng_state_tracker, ) __all__ = [] @@ -198,7 +198,7 @@ def backward(ctx, *args): tracer._has_grad = True # need restore auto_cast state as well as w/b list - with swith_rng_state_tracker( + with switch_rng_state_tracker( ctx.fwd_rng_state, ctx.fwd_rng_state_tracker ): if ctx.is_fw_autocast: diff --git a/python/paddle/distributed/fleet/runtime/parameter_server_runtime.py b/python/paddle/distributed/fleet/runtime/parameter_server_runtime.py index 865571cfeca6f..f69470397e1d9 100644 --- a/python/paddle/distributed/fleet/runtime/parameter_server_runtime.py +++ b/python/paddle/distributed/fleet/runtime/parameter_server_runtime.py @@ -43,7 +43,7 @@ def _set_basic_info(self, context): self.origin_main_program = context["origin_main_program"] self.origin_startup_program = context["origin_startup_program"] self.async_strategy = self._get_distributed_strategy() - self.compiled_strategy = self.build_compiled_startegy() + self.compiled_strategy = self.build_compiled_strategy() def _get_distributed_strategy(self): strategy = None @@ -69,7 +69,7 @@ def _get_distributed_strategy(self): return strategy - def build_compiled_startegy(self): + def build_compiled_strategy(self): from paddle.incubate.distributed.fleet.parameter_server.ir.public import ( CompileTimeStrategy, ) @@ -203,7 +203,7 @@ def get_sparse_attrs(): if len(dist_varnames) != 0: raise ValueError( - "GeoStrategy can not support large scale embeding now, please use paddle.static.nn.embedding" + "GeoStrategy can not support large scale embedding now, please use paddle.static.nn.embedding" ) init_attrs = [] @@ -354,11 +354,11 @@ def _init_server(self, *args, **kwargs): sparse_related_optimize_varnames = list( set(sparse_related_optimize_varnames) ) - distribtued_varnames = self.compiled_strategy.get_sparse_varname_on_ps( + distributed_varnames = self.compiled_strategy.get_sparse_varname_on_ps( True ) distributed_related_optimize_varnames = [] - for var_name in distribtued_varnames: + for var_name in distributed_varnames: distributed_related_optimize_varnames += ( self.compiled_strategy.get_optimize_varname_on_ps(var_name) ) @@ -370,7 +370,7 @@ def _init_server(self, *args, **kwargs): filter( ParameterServerRuntime.__exclude_vars( sparse_varnames - + distribtued_varnames + + distributed_varnames + sparse_related_optimize_varnames + distributed_related_optimize_varnames ), @@ -402,7 +402,7 @@ def _init_server(self, *args, **kwargs): # load large scale self._load_distributed_params( dirname=model_dirname, - varnames=distribtued_varnames + varnames=distributed_varnames + distributed_related_optimize_varnames, ) diff --git a/python/paddle/distributed/fleet/runtime/the_one_ps.py b/python/paddle/distributed/fleet/runtime/the_one_ps.py index a14c337a4fad1..94d403765b1a0 100644 --- a/python/paddle/distributed/fleet/runtime/the_one_ps.py +++ b/python/paddle/distributed/fleet/runtime/the_one_ps.py @@ -684,7 +684,7 @@ def _set_basic_info(self, context): self.origin_main_program = context["origin_main_program"] self.origin_startup_program = context["origin_startup_program"] self.async_strategy = self._get_distributed_strategy() - self.compiled_strategy = self.build_compiled_startegy() + self.compiled_strategy = self.build_compiled_strategy() def _get_distributed_strategy(self): strategy = None @@ -712,7 +712,7 @@ def _get_distributed_strategy(self): strategy.use_ps_gpu = True return strategy - def build_compiled_startegy(self): + def build_compiled_strategy(self): from paddle.incubate.distributed.fleet.parameter_server.ir.public import ( CompileTimeStrategy, ) @@ -1125,8 +1125,8 @@ def _get_tables(): if len(tensor_table_dict) > 0: tables = _add_tensor_table(tables) else: - empty_porgram = Program() - self._server_sub_program.append(empty_porgram.desc) + empty_program = Program() + self._server_sub_program.append(empty_program.desc) barrier_table = _build_barrier_table(len(tables)) tables.append(barrier_table) diff --git a/python/paddle/distributed/fleet/scaler.py b/python/paddle/distributed/fleet/scaler.py index 40e182e7f2e40..977b336eb31bb 100755 --- a/python/paddle/distributed/fleet/scaler.py +++ b/python/paddle/distributed/fleet/scaler.py @@ -139,7 +139,7 @@ def unscale_method(self, optimizer): self._found_inf = self._found_inf.cast("int32") # TODO(shenliang03) Since dp allreduce in the optimizer is - # after the gradscaler, check_finite needs to synchronize global + # after the grad scaler, check_finite needs to synchronize global # information. In the future, we should use check_group to speed. paddle.distributed.all_reduce( self._found_inf, op=paddle.distributed.ReduceOp.MAX, group=None diff --git a/python/paddle/distributed/fleet/utils/fs.py b/python/paddle/distributed/fleet/utils/fs.py index aa7ec2e544efe..5c2ec7fece24d 100644 --- a/python/paddle/distributed/fleet/utils/fs.py +++ b/python/paddle/distributed/fleet/utils/fs.py @@ -994,7 +994,7 @@ def mv(self, fs_src_path, fs_dst_path, overwrite=False, test_exists=True): fs_src_path(str): Name of the file or directory, that's needed to be moved. fs_dst_path(str): Name of the file or directory to which to move to. overwrite(bool): Whether to re-write `fs_dst_path` if that exists. Default is False. - test_exists(bool): Check the existence of `fs_src_path` and `fs_dst_path` . When `test_exists` is set true, if `fs_src_path` doesn't exist or `fs_dst_path` exists, program will throw an Excetption. + test_exists(bool): Check the existence of `fs_src_path` and `fs_dst_path` . When `test_exists` is set true, if `fs_src_path` doesn't exist or `fs_dst_path` exists, program will throw an Exception. Examples: @@ -1177,7 +1177,7 @@ def _split_files(self, files, trainer_id, trainers): trainer_id(int): trainer mpi rank id trainers(int): all trainers num Returns: - fileist(list): file list of current trainer + filelist(list): file list of current trainer """ remainder = len(files) % trainers blocksize = len(files) // trainers @@ -1200,7 +1200,7 @@ def list_files_info(self, path_list): Args: path_list(list): file list Returns: - fileist(list): file list with file path and size + filelist(list): file list with file path and size """ if len(path_list) <= 0: return [] @@ -1650,7 +1650,7 @@ def _split_files(self, files, trainer_id, trainers): trainer_id(int): trainer mpi rank id trainers(int): all trainers num Returns: - fileist(list): file list of current trainer + filelist(list): file list of current trainer """ remainder = len(files) % trainers blocksize = len(files) // trainers diff --git a/python/paddle/distributed/fleet/utils/hybrid_parallel_inference.py b/python/paddle/distributed/fleet/utils/hybrid_parallel_inference.py index d8142b7081f2b..38e6eeca008d6 100644 --- a/python/paddle/distributed/fleet/utils/hybrid_parallel_inference.py +++ b/python/paddle/distributed/fleet/utils/hybrid_parallel_inference.py @@ -63,7 +63,7 @@ class HybridParallelInferenceHelper: ... with paddle.base.device_guard(f'{device}:all'): ... # read data from global lod_tensor_array ... element_in_arr = paddle.tensor.array_read(array=arr, i=step_idx) - ... # write placehold data to global lod_tensor_array, + ... # write placeholder data to global lod_tensor_array, ... # it need for send_v2 of lod_tensor_array ... paddle.increment(x=step_idx, value=1.0) ... paddle.tensor.array_write(element_in_arr, i=step_idx, array=arr) @@ -455,7 +455,7 @@ def _find_prev_op(self, index, var_name): def _add_op_device_attr(self, block): """ - Add op_device attrribute for ops in block that have + Add op_device attribute for ops in block that have not that attribute set. Args: diff --git a/python/paddle/distributed/fleet/utils/hybrid_parallel_util.py b/python/paddle/distributed/fleet/utils/hybrid_parallel_util.py index fc0f897b1454c..27aa4c9f54074 100644 --- a/python/paddle/distributed/fleet/utils/hybrid_parallel_util.py +++ b/python/paddle/distributed/fleet/utils/hybrid_parallel_util.py @@ -263,7 +263,7 @@ def fused_allreduce_gradients(parameter_list, hcg): def broadcast_sharding_parameters(model, hcg): - # TODO TO save memory, use un-fused broadcast to avoid potentional OOM + # TODO TO save memory, use un-fused broadcast to avoid potential OOM logger.debug("sharding start init parameters sync") sharding_parallel_group = hcg.get_sharding_parallel_group() src_rank = hcg.get_sharding_parallel_group_src_rank() @@ -273,7 +273,7 @@ def broadcast_sharding_parameters(model, hcg): def broadcast_sep_parameters(model, hcg): - # TODO TO save memory, use un-fused broadcast to avoid potentional OOM + # TODO TO save memory, use un-fused broadcast to avoid potential OOM logger.debug("sep start init parameters sync") sep_group = hcg.get_sep_parallel_group() src_rank = hcg.get_sep_parallel_group_src_rank() diff --git a/python/paddle/distributed/fleet/utils/mix_precision_utils.py b/python/paddle/distributed/fleet/utils/mix_precision_utils.py index 7b4ff7a0410e5..bbc632029a59b 100644 --- a/python/paddle/distributed/fleet/utils/mix_precision_utils.py +++ b/python/paddle/distributed/fleet/utils/mix_precision_utils.py @@ -47,7 +47,7 @@ def __init__(self, layers, dtype="float16"): param._register_grad_hook(self._update_main_grad_hook(param)) def _update_main_grad_hook(self, param): - """Create the update_main_grad hook for backprop.""" + """Create the update_main_grad hook for back-prop.""" # Hook used for back-prop and grad-merge. @paddle.autograd.no_grad() diff --git a/python/paddle/distributed/fleet/utils/tensor_fusion_helper.py b/python/paddle/distributed/fleet/utils/tensor_fusion_helper.py index 959f9eb49f40f..dff62c1a22db1 100644 --- a/python/paddle/distributed/fleet/utils/tensor_fusion_helper.py +++ b/python/paddle/distributed/fleet/utils/tensor_fusion_helper.py @@ -783,7 +783,7 @@ def fused_parameters( :param fuse_param: fuse param or not :param scale_after_comm: if enable comm overlap, specify the location of grad scale :param group_params: the format of the input parameters is param group - :param apply_decay_param_fun: the funtion to filter decay param + :param apply_decay_param_fun: the function to filter decay param :return: param storage if fused, comm buffers if comm overlap, param groups if use group params """ if act is None: diff --git a/python/paddle/distributed/fleet/utils/tensor_parallel_utils.py b/python/paddle/distributed/fleet/utils/tensor_parallel_utils.py index 9ca0a7fdfc89f..88cb6ff27b1aa 100644 --- a/python/paddle/distributed/fleet/utils/tensor_parallel_utils.py +++ b/python/paddle/distributed/fleet/utils/tensor_parallel_utils.py @@ -44,7 +44,7 @@ def tensor_parallel_sync_filter_fn( param, pos_emb=True, layer_norm=True, bias=True ): """ - Layer fliter function for tensor parallelism transformer. + Layer filter function for tensor parallelism transformer. In tensor parallelism of transformer like model, there is 4 kind of param that are supposed to be the same in all tensor parallel peers: @@ -111,7 +111,7 @@ def copy_parameters(block_, params): ) assert ( param.is_distributed is False - ), f"Try to sync Distribted Parameter: {param}" + ), f"Try to sync Distributed Parameter: {param}" new_p.is_distributed = False block_.vars[new_p.name] = new_p @@ -291,7 +291,7 @@ def add_extra_synchronization( sync_mode(string): select from "broadcast": parameter is sync by broadcasted from 'src_rank' to all other ranks. - "average": paramter is sync by average amonge all ranks + "average": parameter is sync by average among all ranks src_rank(int): the src used in broadcast sync_mode. @@ -324,7 +324,7 @@ def add_extra_synchronization( if params_filter_fn(param): params_to_sync.append(param) logger.info( - "The following param are goning to be synchronization everytime the optimizer update phase of the program is runned: " + "The following param are going to be synchronization everytime the optimizer update phase of the program is runned: " ) logger.info([p.name for p in params_to_sync]) diff --git a/python/paddle/distributed/launch/context/__init__.py b/python/paddle/distributed/launch/context/__init__.py index 0c326c91f5cc6..3bee69f5d7deb 100644 --- a/python/paddle/distributed/launch/context/__init__.py +++ b/python/paddle/distributed/launch/context/__init__.py @@ -91,7 +91,7 @@ def get_logger(self, level=logging.INFO): logger.addHandler(ch) return logger - def continous_log(self) -> bool: + def continuous_log(self) -> bool: if self.args.log_level.upper() in ['DEBUG', 'ERROR']: return True else: @@ -102,6 +102,6 @@ def set_env_in_args(self): attr, attr_type = v if k in self.envs: print( - f"LAUNCH WARNNING args {attr} will be overridden by env: {k} value: {self.envs[k]}" + f"LAUNCH WARNING args {attr} will be overridden by env: {k} value: {self.envs[k]}" ) setattr(self.args, attr, attr_type(self.envs[k])) diff --git a/python/paddle/distributed/launch/controllers/controller.py b/python/paddle/distributed/launch/controllers/controller.py index 4553ea1bb776b..e6eae1a94e3f6 100644 --- a/python/paddle/distributed/launch/controllers/controller.py +++ b/python/paddle/distributed/launch/controllers/controller.py @@ -95,7 +95,7 @@ def watch(self) -> bool: while not self.ctx.status.is_done(): status = self.pod.watch(timeout=2) - # if self.ctx.continous_log(): + # if self.ctx.continuous_log(): # default to print log self.pod.logs() diff --git a/python/paddle/distributed/models/moe/utils.py b/python/paddle/distributed/models/moe/utils.py index 5a2009b2fd0f2..4ebda8bc64c25 100644 --- a/python/paddle/distributed/models/moe/utils.py +++ b/python/paddle/distributed/models/moe/utils.py @@ -59,7 +59,7 @@ def _number_count(numbers, upper_range): def _assign_pos(x, cum_count): """ Assign pos decides which tokens should be fetched belong to - specially expert orderingly. + specially expert orderly. Args: x (Tensor): Tensor. Every element in the list must be a Tensor whose data type diff --git a/python/paddle/distributed/passes/auto_parallel_data_parallel_optimization.py b/python/paddle/distributed/passes/auto_parallel_data_parallel_optimization.py index f2b2c140cd6cf..c820a3d882274 100644 --- a/python/paddle/distributed/passes/auto_parallel_data_parallel_optimization.py +++ b/python/paddle/distributed/passes/auto_parallel_data_parallel_optimization.py @@ -63,7 +63,7 @@ class DataParallelOptimizationPass(PassBase): def __init__(self): super().__init__() - # NOTE not use depence on loss and param_grads + # NOTE not use dependence on loss and param_grads self.set_attr("dist_context", None) self.set_attr("global_rank", -1) self.set_attr("use_sharding", False) diff --git a/python/paddle/distributed/passes/auto_parallel_recompute.py b/python/paddle/distributed/passes/auto_parallel_recompute.py index 9fe72c8aabd75..822bdbd6801b2 100644 --- a/python/paddle/distributed/passes/auto_parallel_recompute.py +++ b/python/paddle/distributed/passes/auto_parallel_recompute.py @@ -293,7 +293,7 @@ def _check_self(self): def _check_conflict(self, other_pass): return True - def get_ops_per_device(self, ops, all_ops_process_meshs, sr=0): + def get_ops_per_device(self, ops, all_ops_process_meshes, sr=0): """ Get ops and op_names of each process mesh excluding ops within the first "sr" chunks """ @@ -302,7 +302,7 @@ def reset_recompute_op(op): if is_recompute_op(op) or is_recompute_exclude_op(op): op._set_attr("op_namescope", "") - all_process_meshes_count = len(all_ops_process_meshs) + all_process_meshes_count = len(all_ops_process_meshes) ops_of_stages = [[] for _ in range(all_process_meshes_count)] op_names_of_stages = [[] for _ in range(all_process_meshes_count)] pushed_ops_count = 0 @@ -321,7 +321,7 @@ def reset_recompute_op(op): if chunk_id // all_process_meshes_count < sr: continue - for id, process_mesh in enumerate(all_ops_process_meshs): + for id, process_mesh in enumerate(all_ops_process_meshes): if op.dist_attr.process_mesh == process_mesh: pushed_ops_count += 1 ops_of_stages[id].append(op) @@ -346,15 +346,15 @@ def _apply_single_impl(self, main_program, startup_program, context): op_path = _find_op_path(main_program, loss, no_grad_set) # 1. mark exclude ops for refined-recompute according to ops-patterns(mainly linear and flash_attn) - # 1.1 get all process_meshs in op_path - all_ops_process_meshs = [] + # 1.1 get all process_meshes in op_path + all_ops_process_meshes = [] for op in op_path: - if op.dist_attr.process_mesh not in all_ops_process_meshs: - all_ops_process_meshs.append(op.dist_attr.process_mesh) + if op.dist_attr.process_mesh not in all_ops_process_meshes: + all_ops_process_meshes.append(op.dist_attr.process_mesh) # 1.2 get ops_devices and op_names_devices ops_devices, op_names_devices = self.get_ops_per_device( - op_path, all_ops_process_meshs, self._sr + op_path, all_ops_process_meshes, self._sr ) all_ops_len = len(op_path) all_exclude_ops_ids = [[] for _ in op_names_devices] diff --git a/python/paddle/distributed/passes/auto_parallel_sharding.py b/python/paddle/distributed/passes/auto_parallel_sharding.py index adddb37d26b43..617425158dd89 100644 --- a/python/paddle/distributed/passes/auto_parallel_sharding.py +++ b/python/paddle/distributed/passes/auto_parallel_sharding.py @@ -1187,7 +1187,7 @@ def _overlap_grad_comm( 2.2 insert after communication dependencies only when need 3. there is not need to add explicit dependencies for non-coalesce gradient communication - P.S. this overlap pass is ONLY adapted for standalone executor (graph based) and stream awared allocator. + P.S. this overlap pass is ONLY adapted for standalone executor (graph based) and stream award allocator. """ if not self.enable_overlap: @@ -1309,7 +1309,7 @@ def _overlap_grad_comm( # hierarchical grad comm if self.enable_hierarchical_comm: # NOTE so far we only support Isomorphic cluster with 8 ranks per node - # TODO unifiy here create communicators + # TODO unify here create communicators # create communicators nranks_per_node = 8 assert self.sharding_world_size % nranks_per_node == 0 diff --git a/python/paddle/distributed/passes/ps_trainer_pass.py b/python/paddle/distributed/passes/ps_trainer_pass.py index 113f5275d8e7b..eb3e0368c49a8 100755 --- a/python/paddle/distributed/passes/ps_trainer_pass.py +++ b/python/paddle/distributed/passes/ps_trainer_pass.py @@ -890,8 +890,8 @@ def _create_heter_program( # joint_var.0_1 -> slice -> reshape -> origin_var # origin_var -> origin_program # reshape -> concat -> joint_var.1_2 - # d) copy send op from origin program for var@grad which loacted in current heter block - # e) re-check every op in current blcok if its device is not current heter devie + # d) copy send op from origin program for var@grad which located in current heter block + # e) re-check every op in current block if its device is not current heter device # 2. Create send op for step counter in last heter-block # 3. Create Listen&Serv OP and Send&Recv OP for distributed training # 4. update CompileTimeStrategy for heter_program diff --git a/python/paddle/distributed/ps/utils/collective_transpiler.py b/python/paddle/distributed/ps/utils/collective_transpiler.py index 7f398842fd701..8d0ff9a53e551 100644 --- a/python/paddle/distributed/ps/utils/collective_transpiler.py +++ b/python/paddle/distributed/ps/utils/collective_transpiler.py @@ -357,7 +357,7 @@ def _insert_allreduce_ops(self): ) offset += 1 - # As we search ops reversedly, we should insert c_allreduce_sum + # As we search ops reversely, we should insert c_allreduce_sum # op in the same way to keep the ring_id alternate ring_id = (ring_id + 1) % self.nrings block._insert_op( @@ -631,7 +631,7 @@ def _insert_allgather_ops(self): ) offset += 1 - # As we search ops reversedly, we should insert c_allgather + # As we search ops reversely, we should insert c_allgather # op in the same way to keep the ring_id alternate ring_id = (ring_id + 1) % self.nrings block._insert_op( From 34871d28c9fe2727c5cb9d5a26cf2cb690b2f920 Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 23 Feb 2024 14:49:48 +0800 Subject: [PATCH 027/282] Fix typo (Arraow -> Arrow) (#61921) --- paddle/pir/src/core/parser/lexer.cc | 6 +++--- paddle/pir/src/core/parser/lexer.h | 2 +- paddle/pir/src/core/parser/token.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/paddle/pir/src/core/parser/lexer.cc b/paddle/pir/src/core/parser/lexer.cc index 54c39e8006ba0..7914063d148c0 100644 --- a/paddle/pir/src/core/parser/lexer.cc +++ b/paddle/pir/src/core/parser/lexer.cc @@ -18,7 +18,7 @@ Token Lexer::ConsumeToken() { SkipWhitespace(); if (auto token = LexIdentifier()) { return *token; - } else if (auto token = LexNumberOrArraow()) { + } else if (auto token = LexNumberOrArrow()) { return *token; } else if (auto token = LexEndTagOrNullVal()) { return *token; @@ -84,7 +84,7 @@ std::unique_ptr Lexer::LexIdentifier() { return token; } -std::unique_ptr Lexer::LexNumberOrArraow() { +std::unique_ptr Lexer::LexNumberOrArrow() { if (!isdigit(is.peek()) && is.peek() != '-') { return nullptr; } @@ -94,7 +94,7 @@ std::unique_ptr Lexer::LexNumberOrArraow() { if (token_digit[0] == '-' && is.peek() == '>') { GetChar(); - std::unique_ptr arrow_token(new Token{"->", ARRAOW}); + std::unique_ptr arrow_token(new Token{"->", ARROW}); return arrow_token; } while (isdigit(is.peek())) { diff --git a/paddle/pir/src/core/parser/lexer.h b/paddle/pir/src/core/parser/lexer.h index 30365172b686f..6606b2291d9a6 100644 --- a/paddle/pir/src/core/parser/lexer.h +++ b/paddle/pir/src/core/parser/lexer.h @@ -30,7 +30,7 @@ class Lexer { Token ConsumeToken(); Token PeekToken(); std::unique_ptr LexIdentifier(); - std::unique_ptr LexNumberOrArraow(); + std::unique_ptr LexNumberOrArrow(); std::unique_ptr LexEndTagOrNullVal(); std::unique_ptr LexValueId(); std::unique_ptr LexEOF(); diff --git a/paddle/pir/src/core/parser/token.h b/paddle/pir/src/core/parser/token.h index 6fe9e7bd79a3d..fd50ff1a7c580 100644 --- a/paddle/pir/src/core/parser/token.h +++ b/paddle/pir/src/core/parser/token.h @@ -23,7 +23,7 @@ enum Token_type { ENDTAG = 3, VALUEID = 4, STRING = 5, - ARRAOW = 6, + ARROW = 6, NULL_ = 7, }; From fa211197faeebeb11b11376ef786aeced0262c65 Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 23 Feb 2024 14:51:00 +0800 Subject: [PATCH 028/282] Update inference_lib.cmake (#61881) --- cmake/inference_lib.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/inference_lib.cmake b/cmake/inference_lib.cmake index 9f1268ce36c41..f4a8286985094 100755 --- a/cmake/inference_lib.cmake +++ b/cmake/inference_lib.cmake @@ -66,7 +66,7 @@ function(copy TARGET) endforeach() endfunction() -function(copy_part_of_thrid_party TARGET DST) +function(copy_part_of_third_party TARGET DST) if(${CBLAS_PROVIDER} STREQUAL MKLML) set(dst_dir "${DST}/third_party/install/mklml") if(WIN32) @@ -233,7 +233,7 @@ copy( SRCS ${CMAKE_CURRENT_BINARY_DIR}/CMakeCache.txt DSTS ${PADDLE_INFERENCE_INSTALL_DIR}) -copy_part_of_thrid_party(inference_lib_dist ${PADDLE_INFERENCE_INSTALL_DIR}) +copy_part_of_third_party(inference_lib_dist ${PADDLE_INFERENCE_INSTALL_DIR}) set(src_dir "${PADDLE_SOURCE_DIR}/paddle/fluid") @@ -365,7 +365,7 @@ add_custom_command( set(PADDLE_INFERENCE_C_INSTALL_DIR "${CMAKE_BINARY_DIR}/paddle_inference_c_install_dir" CACHE STRING "A path setting CAPI paddle inference shared") -copy_part_of_thrid_party(inference_lib_dist ${PADDLE_INFERENCE_C_INSTALL_DIR}) +copy_part_of_third_party(inference_lib_dist ${PADDLE_INFERENCE_C_INSTALL_DIR}) set(src_dir "${PADDLE_SOURCE_DIR}/paddle/fluid") if(WIN32) From 3431e994064fb75c606148308892459fbeba4d1d Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 23 Feb 2024 14:56:30 +0800 Subject: [PATCH 029/282] Update linalg.py (#61746) --- python/paddle/tensor/linalg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/paddle/tensor/linalg.py b/python/paddle/tensor/linalg.py index 0e3c4922be6ba..5ff36cdb754d5 100644 --- a/python/paddle/tensor/linalg.py +++ b/python/paddle/tensor/linalg.py @@ -1514,7 +1514,7 @@ def svd_norm(input, porder, axis=[-1]): type='elementwise_div', inputs={'X': max_out, 'Y': min_out}, outputs={'Out': out}, - attrs={'aixs': axis}, + attrs={'axis': -1}, ) return out if porder == -2: @@ -1522,7 +1522,7 @@ def svd_norm(input, porder, axis=[-1]): type='elementwise_div', inputs={'X': min_out, 'Y': max_out}, outputs={'Out': out}, - attrs={'aixs': axis}, + attrs={'axis': -1}, ) return out From 7568f4824d63567ae8b5cfc0736c4fd507790cd0 Mon Sep 17 00:00:00 2001 From: yinwei Date: Fri, 23 Feb 2024 15:02:02 +0800 Subject: [PATCH 030/282] memory_efficient_attention support q,k,v stop_gradient (#60594) --- paddle/phi/infermeta/backward.cc | 48 +++---- .../memory_efficient_attention_grad_kernel.cu | 35 +++++- .../test_memory_efficient_attention.py | 118 ++++++++++++++++++ 3 files changed, 176 insertions(+), 25 deletions(-) diff --git a/paddle/phi/infermeta/backward.cc b/paddle/phi/infermeta/backward.cc index 4f525ef138735..845a8e6835729 100644 --- a/paddle/phi/infermeta/backward.cc +++ b/paddle/phi/infermeta/backward.cc @@ -746,27 +746,33 @@ void MemoryEfficientAttentionGradInferMeta(const MetaTensor& query, const int64_t value_num_head = value.dims()[2]; const int64_t value_head_size = value.dims()[3]; - std::vector query_grad_dims( - {query_batch_size, query_seq_length, query_num_head, query_head_size}); - std::vector key_grad_dims( - {key_batch_size, key_seq_length, key_num_head, key_head_size}); - std::vector value_grad_dims( - {value_batch_size, value_seq_length, value_num_head, value_head_size}); - - query_grad->set_dims(common::make_ddim(query_grad_dims)); - query_grad->share_lod(query); - query_grad->set_dtype(query.dtype()); - query_grad->set_layout(query.layout()); - - key_grad->set_dims(common::make_ddim(key_grad_dims)); - key_grad->share_lod(key); - key_grad->set_dtype(key.dtype()); - key_grad->set_layout(key.layout()); - - value_grad->set_dims(common::make_ddim(value_grad_dims)); - value_grad->share_lod(value); - value_grad->set_dtype(value.dtype()); - value_grad->set_layout(value.layout()); + if (query_grad) { + std::vector query_grad_dims; + query_grad_dims = { + query_batch_size, query_seq_length, query_num_head, query_head_size}; + query_grad->set_dims(common::make_ddim(query_grad_dims)); + query_grad->share_lod(query); + query_grad->set_dtype(query.dtype()); + query_grad->set_layout(query.layout()); + } + if (key_grad) { + std::vector key_grad_dims; + key_grad_dims = { + key_batch_size, key_seq_length, key_num_head, key_head_size}; + key_grad->set_dims(common::make_ddim(key_grad_dims)); + key_grad->share_lod(key); + key_grad->set_dtype(key.dtype()); + key_grad->set_layout(key.layout()); + } + if (value_grad) { + std::vector value_grad_dims; + value_grad_dims = { + value_batch_size, value_seq_length, value_num_head, value_head_size}; + value_grad->set_dims(common::make_ddim(value_grad_dims)); + value_grad->share_lod(value); + value_grad->set_dtype(value.dtype()); + value_grad->set_layout(value.layout()); + } if (bias && bias_grad) { const int64_t bias_batch_size = bias.dims()[0]; diff --git a/paddle/phi/kernels/fusion/cutlass/memory_efficient_attention_grad_kernel.cu b/paddle/phi/kernels/fusion/cutlass/memory_efficient_attention_grad_kernel.cu index c72a1b69e7ef8..62625936e192a 100644 --- a/paddle/phi/kernels/fusion/cutlass/memory_efficient_attention_grad_kernel.cu +++ b/paddle/phi/kernels/fusion/cutlass/memory_efficient_attention_grad_kernel.cu @@ -58,8 +58,14 @@ void MemoryEfficientAttentionGradKernel( DenseTensor* bias_grad) { bool kernel_launched = false; + DenseTensor dq_tmp; + DenseTensor dk_tmp; + DenseTensor dv_tmp; + bool has_query_grad = (query_grad != nullptr); + bool has_key_grad = (key_grad != nullptr); + bool has_value_grad = (value_grad != nullptr); + auto launchKernel = [&](auto k_, auto kernel_fn) { - // ndim PADDLE_ENFORCE_EQ( query.dims().size(), output_grad.dims().size(), @@ -289,7 +295,6 @@ void MemoryEfficientAttentionGradKernel( int compute_capacity = ctx.GetComputeCapability(); const auto max_shmem = getMaximumSharedMemoryPerBlockKb(compute_capacity) * 1024; - using KernelType = decltype(k_); using scalar_t = typename KernelType::scalar_t; if (kernel_launched) { @@ -404,9 +409,28 @@ void MemoryEfficientAttentionGradKernel( VLOG(3) << "logsumexp_ptr" << p.logsumexp_ptr; p.output_ptr = phi::SafeGetTensorPtr(output); p.grad_output_ptr = phi::SafeGetTensorPtr(output_grad); + + if (!has_query_grad) { + dq_tmp.clear(); + dq_tmp = EmptyLike(ctx, query); + query_grad = &dq_tmp; + } p.grad_query_ptr = phi::SafeAllocTensor(ctx, query_grad); + + if (!has_key_grad) { + dk_tmp.clear(); + dk_tmp = EmptyLike(ctx, key); + key_grad = &dk_tmp; + } p.grad_key_ptr = phi::SafeAllocTensor(ctx, key_grad); + + if (!has_value_grad) { + dv_tmp.clear(); + dv_tmp = EmptyLike(ctx, value); + value_grad = &dv_tmp; + } p.grad_value_ptr = phi::SafeAllocTensor(ctx, value_grad); + p.delta_ptr = phi::SafeGetTensorPtr(delta); PD_MEA_CHECK_OVERFLOW(p.head_dim, q_dims[3]); PD_MEA_CHECK_OVERFLOW(p.head_dim_value, v_dims[3]); @@ -444,11 +468,14 @@ void MemoryEfficientAttentionGradKernel( PD_MEA_CHECK_OVERFLOW(p.o_strideB, DimStride(output.dims(), 0)); PD_MEA_CHECK_OVERFLOW(p.gQ_strideH, DimStride(query_grad->dims(), 2)); - PD_MEA_CHECK_OVERFLOW(p.gK_strideH, DimStride(key_grad->dims(), 2)); - PD_MEA_CHECK_OVERFLOW(p.gV_strideH, DimStride(value_grad->dims(), 2)); PD_MEA_CHECK_OVERFLOW(p.gQ_strideB, DimStride(query_grad->dims(), 0)); + + PD_MEA_CHECK_OVERFLOW(p.gK_strideH, DimStride(key_grad->dims(), 2)); PD_MEA_CHECK_OVERFLOW(p.gK_strideB, DimStride(key_grad->dims(), 0)); + + PD_MEA_CHECK_OVERFLOW(p.gV_strideH, DimStride(value_grad->dims(), 2)); PD_MEA_CHECK_OVERFLOW(p.gV_strideB, DimStride(value_grad->dims(), 0)); + p.gQKV_strideM_multiplier = 1; PADDLE_ENFORCE_EQ(q_dims[2] * q_dims[3], DimStride(query_grad->dims(), 1), diff --git a/test/legacy_test/test_memory_efficient_attention.py b/test/legacy_test/test_memory_efficient_attention.py index 24e1d5115f44f..6298a3100a930 100644 --- a/test/legacy_test/test_memory_efficient_attention.py +++ b/test/legacy_test/test_memory_efficient_attention.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import os import random import re @@ -378,5 +379,122 @@ def setUp(self): self.seed = 2023 +@unittest.skipIf( + not core.is_compiled_with_cuda() or get_cuda_version() < 11030, + "core is not compiled with CUDA and cuda version need larger than or equal to 11.3", +) +class TestMemEffAttentionAPIWithStopGradient(unittest.TestCase): + def setUp(self): + self.name = "MemEffAttnQKV_FFF" + self.place = paddle.CUDAPlace(0) + self.shape = (1, 128, 8, 16) + self.dtype = 'float32' + self.dropout = 0.0 + self.training = True + self.attention_bias = None + self.scale = 1.0 / np.sqrt(self.shape[-1]) + self.seed = 2023 + self.q_grad_stop_gradient = True + self.k_grad_stop_gradient = False + self.v_grad_stop_gradient = False + + def test_all(self): + logging.info( + f"Test All case shape {self.shape} dtype {self.dtype} name {self.name}" + ) + + paddle.disable_static() + + query = np.random.random(self.shape) + q = paddle.to_tensor( + query, + place=self.place, + dtype=self.dtype, + stop_gradient=self.q_grad_stop_gradient, + ) + q_ = paddle.to_tensor( + query, + place=self.place, + dtype=self.dtype, + stop_gradient=self.q_grad_stop_gradient, + ) + key = np.random.random(self.shape) + k = paddle.to_tensor( + key, + place=self.place, + dtype=self.dtype, + stop_gradient=self.k_grad_stop_gradient, + ) + k_ = paddle.to_tensor( + key, + place=self.place, + dtype=self.dtype, + stop_gradient=self.k_grad_stop_gradient, + ) + value = np.random.random(self.shape) + v = paddle.to_tensor( + value, + place=self.place, + dtype=self.dtype, + stop_gradient=self.v_grad_stop_gradient, + ) + v_ = paddle.to_tensor( + value, + place=self.place, + dtype=self.dtype, + stop_gradient=self.v_grad_stop_gradient, + ) + + out_ = attention_naive( + q_, k_, v_, self.attention_bias, self.dropout, self.scale, self.seed + ) + + paddle.seed(self.seed) + out = memory_efficient_attention( + q, + k, + v, + self.attention_bias, + self.dropout, + self.scale, + self.training, + ) + + np.testing.assert_allclose(out.numpy(), out_, rtol=5e-03, atol=1e-03) + + out.backward() + out_.backward() + + if q.stop_gradient is not True: + np.testing.assert_allclose( + q.grad.numpy(), q_.grad.numpy(), rtol=5e-03, atol=1e-03 + ) + + if k.stop_gradient is not True: + np.testing.assert_allclose( + k.grad.numpy(), k.grad.numpy(), rtol=5e-03, atol=1e-03 + ) + if v.stop_gradient is not True: + np.testing.assert_allclose( + v.grad.numpy(), v_.grad.numpy(), rtol=5e-03, atol=1e-03 + ) + + +class TestQKVFTT(TestMemEffAttentionAPIWithStopGradient): + def setUp(self): + self.name = "MemEffAttnQKV_TTT" + self.place = paddle.CUDAPlace(0) + self.shape = (1, 128, 8, 16) + self.dtype = 'float32' + self.dropout = 0.0 + self.training = True + self.attention_bias = None + self.scale = 1.0 / np.sqrt(self.shape[-1]) + self.seed = 2023 + self.q_grad_stop_gradient = False + self.k_grad_stop_gradient = True + self.v_grad_stop_gradient = True + + if __name__ == '__main__': unittest.main() From 901c76bd03794c2d79279779a06ba60ce77b796d Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Fri, 23 Feb 2024 15:25:16 +0800 Subject: [PATCH 031/282] [PIR+CINN]Fix Convert0DTo1D Pass bug in CombineOp (#61977) --- .../group_merge/convert_0d_to_1d_pass.cc | 35 +++++++++++++++++++ .../pir/cinn/sub_graphs/test_sub_graph_23.py | 8 ++--- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/paddle/cinn/hlir/dialect/operator/transforms/group_merge/convert_0d_to_1d_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/group_merge/convert_0d_to_1d_pass.cc index f60878a9e1d99..325421d92abe6 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/group_merge/convert_0d_to_1d_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/group_merge/convert_0d_to_1d_pass.cc @@ -58,6 +58,40 @@ class FullOpPattern : public pir::OpRewritePattern { } }; +class CombineOpPattern : public pir::OpRewritePattern { + public: + using pir::OpRewritePattern::OpRewritePattern; + + bool Match(pir::CombineOp op) const override { + auto out_type = op.result(0).type().dyn_cast(); + for (auto type : out_type.data()) { + if (HasZeroDim(type)) return true; + } + return false; + } + + void Rewrite(pir::CombineOp op, + pir::PatternRewriter &rewriter) const override { + pir::Builder builder(rewriter.ir_context()); + + const std::vector inputs_type = [&]() { + std::vector types; + for (auto value : op->operands_source()) { + types.push_back(value.type()); + } + return types; + }(); + op.result(0).set_type(builder.vec_type(inputs_type)); + } + + private: + bool HasZeroDim(pir::Type type) const { + if (!type) return false; + const auto dense_tensor_type = type.dyn_cast(); + return dense_tensor_type && (dense_tensor_type.dims().size() == 0U); + } +}; + class Convert0DTo1DPass : public pir::PatternRewritePass { public: Convert0DTo1DPass() : pir::PatternRewritePass("convert_0D_to_1D", 1) {} @@ -65,6 +99,7 @@ class Convert0DTo1DPass : public pir::PatternRewritePass { pir::RewritePatternSet InitializePatterns(pir::IrContext *context) override { pir::RewritePatternSet ps(context); ps.Add(context); + ps.Add(context); return ps; } diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_23.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_23.py index 0d140fda01484..5f04f7b0f9bd2 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_23.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_23.py @@ -31,8 +31,7 @@ def forward( var_0, # (shape: [11, 24, 56, 56], dtype: paddle.float32, stop_gradient: False) var_1, # (shape: [11, 24, 56, 56], dtype: paddle.float32, stop_gradient: False) ): - var_2 = paddle.tensor.attribute.shape(var_0) - var_3 = var_2[0] + var_3 = var_0.shape[0] var_4 = paddle.tensor.random.rand(shape=[var_3, 1, 1, 1]) var_5 = 0.975 + var_4 var_6 = paddle.tensor.ops.floor(var_5) @@ -65,16 +64,15 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): outs = net(*self.inputs) return outs - # NOTE prim + cinn lead to error def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) cinn_out = self.train( - self.net, to_static=True, with_prim=True, with_cinn=False + self.net, to_static=True, with_prim=True, with_cinn=True ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) ): - np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) + np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-6) if __name__ == '__main__': From 4c173f64ab76c212e1c810e11881f51655c00a59 Mon Sep 17 00:00:00 2001 From: cyber-pioneer <116002591+cyber-pioneer@users.noreply.github.com> Date: Fri, 23 Feb 2024 15:30:17 +0800 Subject: [PATCH 032/282] [Prim][PIR] add llama if graph dy shape case (#61986) * add llama if case * fix code --- test/ir/pir/cinn/symbolic/CMakeLists.txt | 12 ++ test/ir/pir/cinn/symbolic/test_llama_if_dy.py | 112 ++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 test/ir/pir/cinn/symbolic/test_llama_if_dy.py diff --git a/test/ir/pir/cinn/symbolic/CMakeLists.txt b/test/ir/pir/cinn/symbolic/CMakeLists.txt index 08e1d9d33d456..8d9463d870fda 100644 --- a/test/ir/pir/cinn/symbolic/CMakeLists.txt +++ b/test/ir/pir/cinn/symbolic/CMakeLists.txt @@ -10,6 +10,7 @@ if(WITH_GPU) test_cinn_reduce_symbolic_demo.py test_if_st.py test_if_dy.py + test_llama_if_dy.py test_sub_graph_for_backend.py test_sub_graph_for_frontend.py test_check_infer_symbolic.py @@ -57,6 +58,17 @@ if(WITH_GPU) WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_tests_properties(test_if_dy PROPERTIES LABELS "RUN_TYPE=CINN") + add_test( + NAME test_llama_if_dy + COMMAND + ${CMAKE_COMMAND} -E env + PYTHONPATH=${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/python/:$ENV{PYTHONPATH} + FLAGS_prim_all=true FLAGS_cinn_bucket_compile=false + FLAGS_pir_apply_shape_optimization_pass=0 FLAGS_enable_pir_api=1 + ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_llama_if_dy.py + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + set_tests_properties(test_llama_if_dy PROPERTIES LABELS "RUN_TYPE=CINN") + add_test( NAME test_cinn_reduce_symbolic_demo COMMAND diff --git a/test/ir/pir/cinn/symbolic/test_llama_if_dy.py b/test/ir/pir/cinn/symbolic/test_llama_if_dy.py new file mode 100644 index 0000000000000..7d2c338797260 --- /dev/null +++ b/test/ir/pir/cinn/symbolic/test_llama_if_dy.py @@ -0,0 +1,112 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import sys +import unittest +from os.path import dirname + +import numpy as np + +import paddle +from paddle import nn +from paddle.static import InputSpec + +sys.path.append(dirname(dirname(__file__))) + + +def apply_to_static(net, use_cinn, input_spec=None): + build_strategy = paddle.static.BuildStrategy() + build_strategy.build_cinn_pass = use_cinn + return paddle.jit.to_static( + net, + input_spec=input_spec, + build_strategy=build_strategy, + full_graph=True, + ) + + +class PrepareDecoderAttentionMask(nn.Layer): + def __init__(self): + super().__init__() + + # [batch_size, src_length] -> [batch_size, 1, tgt_length, src_length] + def _expand_2d_mask(self, mask, target_length): + batch_size, src_length = mask.shape[0], mask.shape[-1] + + mask = mask[:, None, None, :].astype("bool") + mask.stop_gradient = True + expanded_mask = mask.expand([batch_size, 1, target_length, src_length]) + + return expanded_mask + + def _make_causal_mask(self, input_ids_shape): + batch_size, seq_len = input_ids_shape + + mask = paddle.tril(paddle.ones((seq_len, seq_len), dtype="bool")) + + # [bs, 1, seq_len, seq_len] + return mask[None, None, :, :].expand([batch_size, 1, seq_len, seq_len]) + + def forward(self, input_ids, attention_mask): + input_shape = paddle.shape(input_ids) + + expanded_attn_mask = self._expand_2d_mask( + attention_mask, target_length=input_shape[-1] + ) + combined_attention_mask = self._make_causal_mask(input_shape) + if input_shape[-1] > 1: + expanded_attn_mask = expanded_attn_mask & combined_attention_mask + expanded_attn_mask = paddle.where( + expanded_attn_mask, 0.0, paddle.finfo("float32").min + ).astype("float32") + return expanded_attn_mask + + +class TestPrepareDecoderAttentionMask(unittest.TestCase): + def setUp(self): + paddle.seed(2024) + self.prepare_data() + + def prepare_data(self): + self.input_ids = paddle.randint( + low=0, high=2048, shape=[1, 2048], dtype="int64" + ) + self.input_ids.stop_gradient = False + + self.attention_mask = paddle.ones([1, 2048], dtype="bool") + self.attention_mask.stop_gradient = False + + def eval(self, use_cinn=False, mode="static"): + net = PrepareDecoderAttentionMask() + input_spec = [ + InputSpec(shape=[None, None], dtype="int64"), + InputSpec(shape=[None, None], dtype="bool"), + ] + if mode == "static": + net = apply_to_static(net, use_cinn, input_spec) + net.eval() + out = net(self.input_ids, self.attention_mask) + return out + + def test_eval(self): + eager_outs = self.eval(mode="eager") + dy_outs = self.eval(use_cinn=False) + + for cinn_out, dy_out in zip(eager_outs, dy_outs): + np.testing.assert_allclose( + cinn_out.numpy(), dy_out.numpy(), atol=1e-8 + ) + + +if __name__ == '__main__': + unittest.main() From 18df00746ecbb65b60cec6cdd1cf50b6875b6e37 Mon Sep 17 00:00:00 2001 From: YUNSHEN XIE <1084314248@qq.com> Date: Fri, 23 Feb 2024 15:31:51 +0800 Subject: [PATCH 033/282] Modify the judgment conditions of the PADDLE_CUDA_INSTALL_REQUIREMENTS option (#61973) * fix * update --- python/setup.py.in | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/setup.py.in b/python/setup.py.in index 520a9f7f7a56c..329f092d44801 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -399,7 +399,7 @@ def get_paddle_extra_install_requirements(): paddle_cuda_install_requirements = os.getenv( "PADDLE_CUDA_INSTALL_REQUIREMENTS", None ) - if paddle_cuda_install_requirements is not None: + if paddle_cuda_install_requirements == "ON": PADDLE_CUDA_INSTALL_REQUIREMENTS = { "V11": ( "nvidia-cuda-runtime-cu11==11.8.89; platform_system == 'Linux' and platform_machine == 'x86_64' | " diff --git a/setup.py b/setup.py index 350c62bdf6301..f19c22f909d07 100644 --- a/setup.py +++ b/setup.py @@ -927,7 +927,7 @@ def get_paddle_extra_install_requirements(): paddle_cuda_install_requirements = os.getenv( "PADDLE_CUDA_INSTALL_REQUIREMENTS", None ) - if paddle_cuda_install_requirements is not None: + if paddle_cuda_install_requirements == "ON": PADDLE_CUDA_INSTALL_REQUIREMENTS = { "V11": ( "nvidia-cuda-runtime-cu11==11.8.89; platform_system == 'Linux' and platform_machine == 'x86_64' | " From 139db934339a67e57c58ca221cfcafdbcfc2b006 Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 23 Feb 2024 15:32:17 +0800 Subject: [PATCH 034/282] Fix some typos(dynamicly, etc) (#61955) --- python/paddle/amp/grad_scaler.py | 36 +++++++-------- python/paddle/autograd/autograd.py | 6 +-- python/paddle/autograd/backward_utils.py | 2 +- python/paddle/autograd/ir_backward.py | 2 +- python/paddle/base/compiler.py | 6 +-- python/paddle/base/core.py | 2 +- python/paddle/base/device_worker.py | 2 +- python/paddle/base/dygraph/base.py | 4 +- python/paddle/base/dygraph/tracer.py | 2 +- python/paddle/base/framework.py | 12 ++--- python/paddle/base/layers/math_op_patch.py | 8 ++-- python/paddle/base/unique_name.py | 2 +- python/paddle/base/variable_index.py | 12 ++--- python/paddle/framework/io.py | 12 ++--- python/paddle/hapi/static_flops.py | 4 +- python/paddle/incubate/asp/asp.py | 30 ++++++------ python/paddle/incubate/autograd/functional.py | 10 ++-- .../distributed/utils/io/save_for_auto.py | 8 ++-- .../transformers/call_transformer.py | 2 +- .../transformers/ifelse_transformer.py | 4 +- .../jit/dy2static/transformers/utils.py | 2 +- python/paddle/jit/sot/symbolic/interpreter.py | 2 +- python/paddle/nn/functional/vision.py | 2 +- python/paddle/nn/initializer/Bilinear.py | 2 +- python/paddle/nn/initializer/assign.py | 2 +- python/paddle/nn/initializer/kaiming.py | 6 +-- python/paddle/nn/initializer/normal.py | 2 +- python/paddle/nn/initializer/xavier.py | 2 +- python/paddle/nn/layer/activation.py | 10 ++-- python/paddle/nn/layer/loss.py | 46 +++++++++---------- python/paddle/nn/layer/norm.py | 14 +++--- python/paddle/nn/layer/pooling.py | 6 +-- python/paddle/profiler/profiler.py | 2 +- .../quantization/imperative/fuse_utils.py | 2 +- .../quantization/imperative/ptq_registry.py | 2 +- python/paddle/tensor/manipulation.py | 2 +- 36 files changed, 135 insertions(+), 135 deletions(-) diff --git a/python/paddle/amp/grad_scaler.py b/python/paddle/amp/grad_scaler.py index 98053bd4d6398..3ba6f28fd4467 100644 --- a/python/paddle/amp/grad_scaler.py +++ b/python/paddle/amp/grad_scaler.py @@ -62,7 +62,7 @@ class AmpScaler: steps with finite gradients. Default is 1000. decr_every_n_nan_or_inf(int, optional): Decreases loss scaling every n accumulated steps with nan or inf gradients. Default is 2. - use_dynamic_loss_scaling(bool, optional): Whether to use dynamic loss scaling. If False, fixed loss_scaling is used. If True, the loss scaling is updated dynamicly. Default is True. + use_dynamic_loss_scaling(bool, optional): Whether to use dynamic loss scaling. If False, fixed loss_scaling is used. If True, the loss scaling is updated dynamically. Default is True. Returns: An AmpScaler object. @@ -258,7 +258,7 @@ def minimize(self, optimizer, *args, **kwargs): self._cache_founf_inf = False if self._use_dynamic_loss_scaling: - # uopdate the scale + # update the scale self._update() self._optimizer_states = defaultdict(_refresh_optimizer_state) @@ -412,7 +412,7 @@ def is_use_dynamic_loss_scaling(self): Whether to use dynamic loss scaling. Returns: - bool: if fixed loss_scaling is used return False, if the loss scaling is updated dynamicly return true. + bool: if fixed loss_scaling is used return False, if the loss scaling is updated dynamically return true. """ return self._use_dynamic_loss_scaling @@ -420,7 +420,7 @@ def get_init_loss_scaling(self): """ Return the initial loss scaling factor. - Reurns: + Returns: float: the initial loss scaling factor. """ return self._init_loss_scaling @@ -441,7 +441,7 @@ def get_incr_ratio(self): """ Return the multiplier to use when increasing the loss scaling. - Reurns: + Returns: float: the multiplier to use when increasing the loss scaling. """ return self._incr_ratio @@ -460,7 +460,7 @@ def get_decr_ratio(self): """ Get the less-than-one-multiplier to use when decreasing the loss scaling. - Reurns: + Returns: float: the less-than-one-multiplier to use when decreasing the loss scaling. """ return self._decr_ratio @@ -479,7 +479,7 @@ def get_incr_every_n_steps(self): """ Return the num `n`, `n` represent increases loss scaling every `n` consecutive steps with finite gradients. - Reurns: + Returns: int: the num `n`, `n` represent increases loss scaling every `n` consecutive steps with finite gradients. """ return self._incr_every_n_steps @@ -497,7 +497,7 @@ def get_decr_every_n_nan_or_inf(self): """ Return the num `n`, `n` represent decreases loss scaling every `n` accumulated steps with nan or inf gradients. - Reurns: + Returns: int: the num `n`, `n` represent decreases loss scaling every `n` accumulated steps with nan or inf gradients. """ return self._decr_every_n_nan_or_inf @@ -515,7 +515,7 @@ def state_dict(self): """ Returns the state of the scaler as a `dict`, If this instance is not enabled, returns an empty dict. - Reurns: + Returns: A dict of scaler includes: scale (tensor): The loss scaling factor. incr_ratio(float): The multiplier to use when increasing the loss scaling. @@ -524,7 +524,7 @@ def state_dict(self): decr_every_n_nan_or_inf(int): Decreases loss scaling every n accumulated steps with nan or inf gradients. incr_count(int): The number of recent consecutive unskipped steps. decr_count(int): The number of recent consecutive skipped steps. - use_dynamic_loss_scaling(bool): Whether to use dynamic loss scaling. If False, fixed loss_scaling is used. If True, the loss scaling is updated dynamicly. Default is True. + use_dynamic_loss_scaling(bool): Whether to use dynamic loss scaling. If False, fixed loss_scaling is used. If True, the loss scaling is updated dynamically. Default is True. """ return ( { @@ -597,7 +597,7 @@ class GradScaler(AmpScaler): steps with finite gradients. Default is 2000. decr_every_n_nan_or_inf(int, optional): Decreases loss scaling every n accumulated steps with nan or inf gradients. Default is 1. - use_dynamic_loss_scaling(bool, optional): Whether to use dynamic loss scaling. If False, fixed loss_scaling is used. If True, the loss scaling is updated dynamicly. Default is True. + use_dynamic_loss_scaling(bool, optional): Whether to use dynamic loss scaling. If False, fixed loss_scaling is used. If True, the loss scaling is updated dynamically. Default is True. Returns: An GradScaler object. @@ -869,7 +869,7 @@ def is_use_dynamic_loss_scaling(self): Whether to use dynamic loss scaling. Returns: - bool: if fixed loss_scaling is used return False, if the loss scaling is updated dynamicly return true. + bool: if fixed loss_scaling is used return False, if the loss scaling is updated dynamically return true. Examples: .. code-block:: python @@ -895,7 +895,7 @@ def get_init_loss_scaling(self): """ Return the initial loss scaling factor. - Reurns: + Returns: float: the initial loss scaling factor. Examples: @@ -952,7 +952,7 @@ def get_incr_ratio(self): """ Return the multiplier to use when increasing the loss scaling. - Reurns: + Returns: float: the multiplier to use when increasing the loss scaling. Examples: @@ -1009,7 +1009,7 @@ def get_decr_ratio(self): """ Get the less-than-one-multiplier to use when decreasing the loss scaling. - Reurns: + Returns: float: the less-than-one-multiplier to use when decreasing the loss scaling. Examples: @@ -1066,7 +1066,7 @@ def get_incr_every_n_steps(self): """ Return the num `n`, `n` represent increases loss scaling every `n` consecutive steps with finite gradients. - Reurns: + Returns: int: the num `n`, `n` represent increases loss scaling every `n` consecutive steps with finite gradients. Examples: @@ -1123,7 +1123,7 @@ def get_decr_every_n_nan_or_inf(self): """ Return the num `n`, `n` represent decreases loss scaling every `n` accumulated steps with nan or inf gradients. - Reurns: + Returns: int: the num `n`, `n` represent decreases loss scaling every `n` accumulated steps with nan or inf gradients. Examples: @@ -1189,7 +1189,7 @@ def state_dict(self): decr_every_n_nan_or_inf(int): Decreases loss scaling every n accumulated steps with nan or inf gradients. incr_count(int): The number of recent consecutive unskipped steps. decr_count(int): The number of recent consecutive skipped steps. - use_dynamic_loss_scaling(bool): Whether to use dynamic loss scaling. If False, fixed loss_scaling is used. If True, the loss scaling is updated dynamicly. Default is True. + use_dynamic_loss_scaling(bool): Whether to use dynamic loss scaling. If False, fixed loss_scaling is used. If True, the loss scaling is updated dynamically. Default is True. Examples: diff --git a/python/paddle/autograd/autograd.py b/python/paddle/autograd/autograd.py index 93e0a845908b1..1a1d5a8d66611 100644 --- a/python/paddle/autograd/autograd.py +++ b/python/paddle/autograd/autograd.py @@ -48,7 +48,7 @@ class Jacobian: Notes: - Eclipsis index is not supported currently. + Ellipsis index is not supported currently. Args: @@ -495,7 +495,7 @@ def jacobian( Returns: - Union[Tuple[Tuple[Jacobian, ...], ...], Tuple[Jacobian, ...], Jacobian]: Jacobian(s) of ys deriveted from xs. + Union[Tuple[Tuple[Jacobian, ...], ...], Tuple[Jacobian, ...], Jacobian]: Jacobian(s) of ys derived from xs. Examples: @@ -579,7 +579,7 @@ def hessian( Returns: - Union[Tuple[Tuple[Hessian, ...], ...], Tuple[Hessian, ...], Hessian]: Hessian(s) of ys deriveted from xs. + Union[Tuple[Tuple[Hessian, ...], ...], Tuple[Hessian, ...], Hessian]: Hessian(s) of ys derived from xs. Examples: diff --git a/python/paddle/autograd/backward_utils.py b/python/paddle/autograd/backward_utils.py index e3e6326ba61cc..f0d90d08426d3 100644 --- a/python/paddle/autograd/backward_utils.py +++ b/python/paddle/autograd/backward_utils.py @@ -188,7 +188,7 @@ def __init__(self, block): self.opgrad_to_op = collections.defaultdict(list) # only for controlflow # inside_value is sub block value, which will yield to parent block, - # parant block value is outside_value + # parent block value is outside_value self.inside_value_to_outside_value_map = ValueDict() def turn_map(self) -> None: diff --git a/python/paddle/autograd/ir_backward.py b/python/paddle/autograd/ir_backward.py index 9c751f82238fa..042a541eb69f5 100644 --- a/python/paddle/autograd/ir_backward.py +++ b/python/paddle/autograd/ir_backward.py @@ -555,7 +555,7 @@ def append_yield( # there are four patterns: # [builtin.combine , op1] (op1's one input is vectorType, outputs are not vectorType) # [op2 , builtin.split] (op2's inputs are not vectorType, one output is vectorType) - # [builtin.combine , op3 , buitin.split] (op3's one input and one output are vectorType) + # [builtin.combine , op3 , builtin.split] (op3's one input and one output are vectorType) # [op4] (op4's inputs and outputs are not vectorType) # -----------------only for control flow-----------------# diff --git a/python/paddle/base/compiler.py b/python/paddle/base/compiler.py index 79aae31a1b147..7b8646eb00b70 100644 --- a/python/paddle/base/compiler.py +++ b/python/paddle/base/compiler.py @@ -495,7 +495,7 @@ def func_compile(): @staticmethod def patch_program_cache(ipu_strategy): - """Monkey patch ProgramCache discriptor to support dynamic2static in IPU. + """Monkey patch ProgramCache descriptor to support dynamic2static in IPU. Args: ipu_strategy: The ipu_strategy used in dynamic graph. @@ -528,7 +528,7 @@ def patch_getter(self, item): ) if self._caches and not ipu_strategy.need_compile: logging_utils.warn( - "dynamic2static on IPU doesn't support mutiple caches. Please make sure" + "dynamic2static on IPU doesn't support multiple caches. Please make sure" "dynamic inputs is not used." ) concrete_program, _ = self._build_once(item) @@ -751,7 +751,7 @@ def set_graph_config( num_ipus (int, optional): Number of IPU devices. Default 1, which means only use 1 IPU. is_training (bool, optional): True is training graph, False is inference graph. Default True, which means is training mode. batch_size (int, optional): The batch-size in the graph. Used to make the graph batch-size fixed, - if the batch-size in the graph is dynamic. Default 1, which means the batch-size would be set 1, if the batch-size is dynamice. + if the batch-size in the graph is dynamic. Default 1, which means the batch-size would be set 1, if the batch-size is dynamic. enable_manual_shard (bool, optional): Enable graph sharding or not. Only if num_ipus > 1, enable_manual_shard is able to be set True. Default False, which means disabled. diff --git a/python/paddle/base/core.py b/python/paddle/base/core.py index fcb1f5605e823..765c63fd2d6d0 100644 --- a/python/paddle/base/core.py +++ b/python/paddle/base/core.py @@ -249,7 +249,7 @@ def to_list(s): # NOTE(zhiqiu): An error may occurs when import paddle in linux platform with glibc < 2.22, # the error message of which is "dlopen: cannot load any more object with static TLS". # This happens when: -# (1) the number of dynamic shared librarys (DSO) loaded > 14, +# (1) the number of dynamic shared libraries (DSO) loaded > 14, # (2) after that, load a dynamic shared library (DSO) with static TLS. # For paddle, the problem is that 'libgomp' is a DSO with static TLS, and it is loaded after 14 DSOs. # So, here is a tricky way to solve the problem by pre load 'libgomp' before 'libpaddle.so'. diff --git a/python/paddle/base/device_worker.py b/python/paddle/base/device_worker.py index c20677f6acd5e..26b351befda2f 100644 --- a/python/paddle/base/device_worker.py +++ b/python/paddle/base/device_worker.py @@ -629,7 +629,7 @@ def _gen_worker_desc(self, trainer_desc): schedule_mode_str = pipeline_opt["schedule_mode"] # F-then-B scheduler which runs Forward phase for all microbatches, # then runs Backward phase for all microbatches. - # 1F1B scheduler, which runs forward phase and backward phase altertively + # 1F1B scheduler, which runs forward phase and backward phase alternatively # after startup phase. assert schedule_mode_str in ["F-then-B", "1F1B"], ( "The schedule mode " "for pipeline must be one of F-then-B or 1F1B" diff --git a/python/paddle/base/dygraph/base.py b/python/paddle/base/dygraph/base.py index 1e20398440bec..4f233cfe4d671 100644 --- a/python/paddle/base/dygraph/base.py +++ b/python/paddle/base/dygraph/base.py @@ -925,11 +925,11 @@ def to_variable(value, name=None, zero_copy=None, dtype=None): framework._current_expected_place(), framework.core.CPUPlace ): # TODO(zhiqiu): we found two problems when enable zero_copy on CPUPlace. - # (1): eigen requires 16-bytes alignments, but the data of numpy array may not statisfy. + # (1): eigen requires 16-bytes alignments, but the data of numpy array may not satisfy. # Details: https://eigen.tuxfamily.org/dox/group__TopicUnalignedArrayAssert.html # (2): when used in flask framework, it may result in hang. # Details: https://github.com/PaddlePaddle/Paddle/issues/26635 - # So, we temporally diable the zero_copy strategy. + # So, we temporally disable the zero_copy strategy. if zero_copy is True: warnings.warn( "Currently, zero_copy is not supported, and it will be discarded." diff --git a/python/paddle/base/dygraph/tracer.py b/python/paddle/base/dygraph/tracer.py index 4df9517073c66..966004e5035f4 100644 --- a/python/paddle/base/dygraph/tracer.py +++ b/python/paddle/base/dygraph/tracer.py @@ -142,7 +142,7 @@ def eager_legacy_trace_op( assert out_name in outputs.keys() num_outs = len(outputs[out_name]) arg_to_append = num_outs - # NOTE(dev): For MasterParam/MasterParamOut in optimzer op + # NOTE(dev): For MasterParam/MasterParamOut in optimizer op elif "Var" in arg_name[-3:]: out_name = arg_name[:-3] print(out_name) diff --git a/python/paddle/base/framework.py b/python/paddle/base/framework.py index 8fe13c16bdd0b..84077b768b995 100644 --- a/python/paddle/base/framework.py +++ b/python/paddle/base/framework.py @@ -4430,7 +4430,7 @@ def create_parameter(self, *args, **kwargs): else: param = Parameter(global_block, *args, **kwargs) # NOTE(Aurelius84): we deliver stop_gradient in append_op, so we - # need recorde it state and reset it back after calling this API + # need record it state and reset it back after calling this API stop_gradient = param.stop_gradient if 'initializer' in kwargs: @@ -6588,7 +6588,7 @@ def _prune_with_input(self, feeded_var_names, targets): "Variable or Operator, but received %s." % type(t) ) - # NOTEZ(zhiqiu): For variable to be fed in fetch_list, there two cases: + # NOTE(zhiqiu): For variable to be fed in fetch_list, there two cases: # (1) the variable is leaf, it has no op that generates it; # (2) the variable is not leaf, and we need to prune the op that generates it. # In both cases, wo can just skip target_op of that it. @@ -6810,7 +6810,7 @@ def parse_from_string(binary_str): Args: - binary_str_type (str): the binary prootbuf string. + binary_str_type (str): the binary protobuf string. Returns: Program: A deserialized Program. @@ -7198,7 +7198,7 @@ def all_parameters(self): Get all :ref:`api_guide_parameter_en` from this Program. A list object is returned. Returns: - list[ :ref:`api_guide_parameter_en` ]: The list contians all parameters in this program. + list[ :ref:`api_guide_parameter_en` ]: The list contains all parameters in this program. Examples: .. code-block:: python @@ -7250,7 +7250,7 @@ def state_dict(self, mode='all', scope=None): obtained through 'paddle.static.global_scope()'. Otherwise, value will be set to scope. Default: None - Retruns: + Returns: dict: a dict contains the parameters and persistable buffers. Examples: @@ -7274,7 +7274,7 @@ def state_dict(self, mode='all', scope=None): >>> paddle.save(prog.state_dict(), path) """ # The 'framework' is a low-level module, and 'executor' - # can not be imported at the begainning of this file. + # can not be imported at the beginning of this file. # Therefore, the above two modules are dynamically imported. from .executor import global_scope diff --git a/python/paddle/base/layers/math_op_patch.py b/python/paddle/base/layers/math_op_patch.py index e4b9ed5198a9e..00d0faaedd0dd 100644 --- a/python/paddle/base/layers/math_op_patch.py +++ b/python/paddle/base/layers/math_op_patch.py @@ -258,7 +258,7 @@ def place(self): """ Variable don't have 'place' interface in static graph mode But this interface can greatly facilitate dy2static. - So we give a warnning here and return None. + So we give a warning here and return None. """ warnings.warn( "Variable do not have 'place' interface for static graph mode, try not to use it. None will be returned." @@ -269,7 +269,7 @@ def contiguous(self): """ Variable don't have 'contiguous' interface in static graph mode But this interface can greatly facilitate dy2static. - So we give a warnning here and return None. + So we give a warning here and return None. """ warnings.warn( "Variable do not have 'contiguous' interface for static graph mode, try not to use it. self will be returned." @@ -281,7 +281,7 @@ def is_contiguous(self): """ Variable don't have 'is_contiguous' interface in static graph mode But this interface can greatly facilitate dy2static. - So we give a warnning here and return None. + So we give a warning here and return None. """ warnings.warn( "Variable do not have 'is_contiguous' interface for static graph mode, try not to use it. True will be returned." @@ -360,7 +360,7 @@ def append(self, var): """ if not isinstance(var, Variable): if in_to_static_mode(): - """in dy2static mode, x may be tensorable values such as int, float, np.array""" + """In dy2static mode, x may be tensor values such as int, float, np.array""" from paddle.tensor.creation import to_tensor var = to_tensor(var) diff --git a/python/paddle/base/unique_name.py b/python/paddle/base/unique_name.py index 95acd00cc60ea..9541d411078aa 100644 --- a/python/paddle/base/unique_name.py +++ b/python/paddle/base/unique_name.py @@ -211,7 +211,7 @@ def guard(new_generator=None): Args: new_generator(str|bytes, optional): New name of global namespace. Note that str - in Python2 was spilted into str and bytes in Python3, so here are two + in Python2 was splitted into str and bytes in Python3, so here are two types. Default is None. If not None, new_generator will be added into the prefix of unique name generated by :code:`generate()`. diff --git a/python/paddle/base/variable_index.py b/python/paddle/base/variable_index.py index dd202bfb93d13..6ccfe1c6164d2 100644 --- a/python/paddle/base/variable_index.py +++ b/python/paddle/base/variable_index.py @@ -173,12 +173,12 @@ def deal_advanced_index( Transpose origin Tensor and advanced indices to the front. Returns: - transed_tensor (Tensor): transposed tensor, corresbonding with advanced indices - transed_index (List): advanced indices transed to the front + transed_tensor (Tensor): transposed tensor, corresponding with advanced indices + transed_index (List): advanced indices transposed to the front trans_back_dim (List): order of axes to transpose back to original order. Only used in __setitem__. pos_of_new_dim (int): axis of new dim in the result. Only used in __getitem__. rank_of_new_dim (int): rank of new dim in the result. Only used in __getitem__. - transed_value_tensor (Tensor): value tensor transed to the front. Only used in __setitem__. + transed_value_tensor (Tensor): value tensor transposed to the front. Only used in __setitem__. """ transed_dim = [] transed_index = [] @@ -771,7 +771,7 @@ def get_tensor_with_basic_indexing( else: stride = attrs['strides'] if use_strided_slice: - # TODO(zoooo0820): suppport strided_slice_array until PIR API is ready + # TODO(zoooo0820): support strided_slice_array until PIR API is ready out = paddle._C_ops.strided_slice(x, axes, st, end, stride) if len(decrease_axes) > 0: @@ -883,7 +883,7 @@ def _getitem_static(x, indices): _, ) = deal_advanced_index(out, advanced_index, False, None) - # TODO(zooooo0820): Replacing gather_nd to another advanded OP for handling of mixed indexes more efficiently + # TODO(zooooo0820): Replacing gather_nd to another advanced OP for handling of mixed indexes more efficiently if len(adjusted_advanced_index) == 1 and adjusted_advanced_index[ 0 ].dtype in (paddle.bool, paddle.base.libpaddle.BOOL): @@ -919,7 +919,7 @@ def _getitem_static(x, indices): def parse_bool_and_broadcast_indices(indices): # deal with multiple Tensors and translating bool tensor to int tensor. - # In static mode, bool-tensor cannot be broadcasted since its corressponding int tensor's shape cannot be infered. + # In static mode, bool-tensor cannot be broadcasted since its corresponding int tensor's shape cannot be infered. for i, indice in enumerate(indices): if ( indice.dtype == paddle.bool diff --git a/python/paddle/framework/io.py b/python/paddle/framework/io.py index 8c794b4ff2ef0..c0015f6704a88 100644 --- a/python/paddle/framework/io.py +++ b/python/paddle/framework/io.py @@ -68,7 +68,7 @@ def async_save(obj, path, protocol=4, sync_other_task=False, **configs): Note: currently only support dygraph mode. Note: - any argument passed through configs will be overrided by default setting. + any argument passed through configs will be overridden by default setting. Args: obj(Object) : The object to be saved. path(str|BytesIO) : The path/buffer of the object to be saved. @@ -76,7 +76,7 @@ def async_save(obj, path, protocol=4, sync_other_task=False, **configs): protocol(int, optional): The protocol version of pickle module must be greater than 1 and less than 5. Default: 4 sync_other_task(bool) : Determine whether to wait other async save task to be finished before this one be put in queue. - **configs(dict, optional): compatible argument to paddle.save, but will be overrided by default setting. + **configs(dict, optional): compatible argument to paddle.save, but will be overridden by default setting. Examples: .. code-block:: python :name: code-example-1 @@ -98,7 +98,7 @@ def async_save(obj, path, protocol=4, sync_other_task=False, **configs): ) if len(configs) > 0: warnings.warn( - "configs are not supported in async mode, will be overided by default settings." + "configs are not supported in async mode, will be overridden by default settings." ) # TODO: make this part async @@ -596,13 +596,13 @@ def tuple_to_tensor(obj): def ndarray_to_tensor(obj): return _ndarray_to_tensor(obj, return_numpy=return_numpy) - # tuple(name, ndarry) was converted from varbase of paddle2.1, - # and all tuple(name, ndarry) are converted to tensor. + # tuple(name, ndarray) was converted from varbase of paddle2.1, + # and all tuple(name, ndarray) are converted to tensor. if _contain_x(obj, _transformed_from_varbase): return _parse_every_object( obj, _transformed_from_varbase, tuple_to_tensor ) - # If there is no tuple(name, ndary), it is considered to be saved by paddle2.0 + # If there is no tuple(name, ndarray), it is considered to be saved by paddle2.0 # or converted from LoDTensor, and all ndarrays are converted to tensor. else: return _parse_every_object( diff --git a/python/paddle/hapi/static_flops.py b/python/paddle/hapi/static_flops.py index 6d9209ae76a4a..a627dbb68ea4a 100644 --- a/python/paddle/hapi/static_flops.py +++ b/python/paddle/hapi/static_flops.py @@ -55,7 +55,7 @@ def type(self): def inputs(self, name): """ - Get all the varibales by the input name. + Get all the variables by the input name. """ if name in self._op.input_names: return [ @@ -66,7 +66,7 @@ def inputs(self, name): def outputs(self, name): """ - Get all the varibales by the output name. + Get all the variables by the output name. """ return [self._graph.var(var_name) for var_name in self._op.output(name)] diff --git a/python/paddle/incubate/asp/asp.py b/python/paddle/incubate/asp/asp.py index a1de1937c70cd..fbe1eac9b9d26 100644 --- a/python/paddle/incubate/asp/asp.py +++ b/python/paddle/incubate/asp/asp.py @@ -106,7 +106,7 @@ def set_excluded_layers(param_names, main_program=None): ... prob = my_layer(input_data) ... loss = paddle.mean(paddle.nn.functional.square_error_cost(prob, label)) ... - ... # Setup exluded layers out from ASP workflow. + ... # Setup excluded layers out from ASP workflow. ... # Please note, excluded_layers must be set before calling optimizer.minimize(). ... paddle.incubate.asp.set_excluded_layers([my_layer.linear1.full_name()], main_program) ... @@ -126,7 +126,7 @@ def set_excluded_layers(param_names, main_program=None): def reset_excluded_layers(main_program=None): r""" - Reset exculded layers setting corresponding to :attr:`main_program`. If :attr:`main_program` + Reset excluded layers setting corresponding to :attr:`main_program`. If :attr:`main_program` is None, then all configurations of excluded_layers would be cleaned. Args: @@ -196,7 +196,7 @@ def reset_excluded_layers(main_program=None): ... prob = my_layer(input_data) ... loss = paddle.mean(paddle.nn.functional.square_error_cost(prob, label)) ... - ... # Setup exluded layers out from ASP workflow. + ... # Setup excluded layers out from ASP workflow. ... # Please note, excluded_layers must be set before calling optimizer.minimize(). ... paddle.incubate.asp.set_excluded_layers([my_layer.linear1.full_name()], main_program) ... # Reset excluded_layers, all supported layers would be included into Automatic SParsity's workflow. @@ -216,7 +216,7 @@ def reset_excluded_layers(main_program=None): def decorate(optimizer): r""" Wrap the given optimizer as a OptimizerWithSparsityGuarantee, - If runnig with dynamic graph mode. ASP would creates mask variables for supported parameters. + If running with dynamic graph mode. ASP would creates mask variables for supported parameters. Else if in static graph mode, ASP would creates mask variables and inserts necessary ops when calling minimize() @@ -310,15 +310,15 @@ def prune_model(model, n=2, m=4, mask_algo='mask_1d', with_mask=True): *Note*: (Static graph mode) If calling this function with :attr:`with_mask`, it should call `OptimizerWithSparsityGuarantee.minimize` and initialization (`exe.run(startup_program`)) before (For successfully obtain mask Variable). Typically set `with_mask` as true for training (have called `OptimizerWithSparsityGuarantee.minimize`) and false for - inference only. To obtain OptimizerWithSparsityGuarantee, please see `paddle.incubate.asp.decoreate()`. + inference only. To obtain OptimizerWithSparsityGuarantee, please see `paddle.incubate.asp.decorate()`. Args: model (Program|nn.Layer): Program with model definition and its parameters, or a object of `paddle.nn.Layer`. n (int, optional): n of `n:m` sparse pattern. Default is 2. m (int, optional): m of `n:m` sparse pattern. Default is 4. - mask_algo (string, optional): The function name to generate spase mask. Default is `mask_1d`. - The vaild inputs should be one of 'mask_1d', 'mask_2d_greedy' and 'mask_2d_best'. - with_mask (bool, optional): To prune mask Variables related to parameters or not. True is purning also, False is not. Default is True. + mask_algo (string, optional): The function name to generate sparse mask. Default is `mask_1d`. + The valid inputs should be one of 'mask_1d', 'mask_2d_greedy' and 'mask_2d_best'. + with_mask (bool, optional): To prune mask Variables related to parameters or not. True is pruning also, False is not. Default is True. Returns: dictionary: A dictionary with key: `parameter name` (string) and value: its corresponding mask Variable. Examples: @@ -476,9 +476,9 @@ def prune_model(model, n=2, m=4, mask_algo='mask_1d', with_mask=True): class ProgramASPInfo: r""" - ProgramASPInfo is a container to keep ASP relevant information of Pragrom. It contains three inner-variables: - 1. __mask_vars (Dictionary): Key is parameter's name and vaule is its corresponding sparse mask Variable object, which is created by `ASPHelper.create_mask_variables`. - 2. __masks (Dictionary): Key is parameter's name and vaule is its corressponding sparse mask Numpy array, which is created by `ASPHelper.prune_model`. + ProgramASPInfo is a container to keep ASP relevant information of Program. It contains three inner-variables: + 1. __mask_vars (Dictionary): Key is parameter's name and value is its corresponding sparse mask Variable object, which is created by `ASPHelper.create_mask_variables`. + 2. __masks (Dictionary): Key is parameter's name and value is its corresponding sparse mask Numpy array, which is created by `ASPHelper.prune_model`. 3. __excluded_layers (List): It stores name of layers which should not involve into ASP workflow. """ @@ -552,7 +552,7 @@ def decorate(optimizer): if paddle.in_dynamic_mode(): # main_prog and startup_prog would be used with paddle.static.program_guard # to create ASP masks. Moreover, main_prog is a key to map paddle.static.Program - # to its own ASP informantion, like ASP mask variables. For dynamic graph, we use + # to its own ASP information, like ASP mask variables. For dynamic graph, we use # default_main_program as the key. main_prog = paddle.static.default_main_program() startup_prog = paddle.static.default_startup_program() @@ -809,7 +809,7 @@ def _minimize( 3. Insert masking ops in the end of parameters update. *Note*: Please use `ASP.decorate` instead when applying distributed training with `Fleet`. - (Due to there is a invisiable graphs optimization in `Fleet.minimize()` which make training graph + (Due to there is a invisible graphs optimization in `Fleet.minimize()` which make training graph cannot be modified anymore.) Args: @@ -849,7 +849,7 @@ def _step(cls, optimizer): 2. Mask parameters with sparse masks. *Note*: Please use `ASP.decorate` instead when applying distributed training with `Fleet`. - (Due to there is a invisiable graphs optimization in `Fleet.minimize()` which make training graph + (Due to there is a invisible graphs optimization in `Fleet.minimize()` which make training graph cannot be modified anymore.) Args: @@ -963,7 +963,7 @@ def step(self): 2. Mask parameters with sparse masks. *Note*: Please use `ASP.decorate` instead when applying distributed training with `Fleet`. - (Due to there is a invisiable graphs optimization in `Fleet.minimize()` which make training graph + (Due to there is a invisible graphs optimization in `Fleet.minimize()` which make training graph cannot be modified anymore.) Args: diff --git a/python/paddle/incubate/autograd/functional.py b/python/paddle/incubate/autograd/functional.py index 5f4f5c6a76f17..8bec01b1c39ae 100644 --- a/python/paddle/incubate/autograd/functional.py +++ b/python/paddle/incubate/autograd/functional.py @@ -577,9 +577,9 @@ def _grad(ys, xs, v=None): inputs. """ if framework.in_dygraph_mode(): - # paddle.grad returns a list though the inputs is a signle Tensor. The + # paddle.grad returns a list though the inputs is a single Tensor. The # follow code snippet fixes the problem by return the first element of - # xs_grad when the xs is a signle Tensor. + # xs_grad when the xs is a single Tensor. xs_grad = paddle.grad(ys, xs, v, create_graph=True, allow_unused=True) if ( isinstance(xs, paddle.base.framework.Variable) @@ -595,12 +595,12 @@ def _grad(ys, xs, v=None): def _separate(xs): """ ``_separate`` separates ``xs`` from the computation graph through ``clone`` - or ``deteach`` . + or ``detach`` . - Interally, ``paddle.grad(xs, ys)`` is stateful API implemented based on + Internally, ``paddle.grad(xs, ys)`` is stateful API implemented based on computional graph, which will reduce gradients along all path from ys to xs. - However, funcional autograd API such as ``vjp``, ``jvp`` is stateless, and + However, functional autograd API such as ``vjp``, ``jvp`` is stateless, and only compute gradients with a given ``func`` . For example, given a ``func`` :math:`y0=f(x0)`, supposing forward path is: diff --git a/python/paddle/incubate/distributed/utils/io/save_for_auto.py b/python/paddle/incubate/distributed/utils/io/save_for_auto.py index 00a1c7feb6c69..4e3658d8ddd10 100644 --- a/python/paddle/incubate/distributed/utils/io/save_for_auto.py +++ b/python/paddle/incubate/distributed/utils/io/save_for_auto.py @@ -40,12 +40,12 @@ def save_for_auto_inference(path_prefix, dist_model, cvt2cpu=False): MoE not supported till MoE is supported in auto parallel mode. Args: - path_prefix: path prefix to save. If `path_preifx` ends with path sepreator, + path_prefix: path prefix to save. If `path_prefix` ends with path separator, the path is processed as a directory and parameters will be saved in it, - automatically named saved_parameters. Otherwisw, the parameters will be saved with name - path_preifx_dist{global_rank}.pdparams and path_preifx_dist{global_rank}.pdattrs. + automatically named saved_parameters. Otherwise, the parameters will be saved with name + path_prefix_dist{global_rank}.pdparams and path_prefix_dist{global_rank}.pdattrs. dist_model: model in distributed model. - cvt2cpu: wheather to move parameters to CPU when using sharding stage 3. + cvt2cpu: whether to move parameters to CPU when using sharding stage 3. The var is invalid if not using sharding stage 3. Returns: diff --git a/python/paddle/jit/dy2static/transformers/call_transformer.py b/python/paddle/jit/dy2static/transformers/call_transformer.py index eaa181d48ce02..9e2c73f4cb7fc 100644 --- a/python/paddle/jit/dy2static/transformers/call_transformer.py +++ b/python/paddle/jit/dy2static/transformers/call_transformer.py @@ -69,7 +69,7 @@ def visit_Call(self, node): func_str = ast_to_source_code(node.func).strip() - # NOTE(liym27): Don't convert `pad.set_trace` even if the convertion doesn't work finally, because + # NOTE(liym27): Don't convert `pad.set_trace` even if the conversion doesn't work finally, because # it is clearer to see where it is called from. if PDB_SET in func_str: return node diff --git a/python/paddle/jit/dy2static/transformers/ifelse_transformer.py b/python/paddle/jit/dy2static/transformers/ifelse_transformer.py index 7645c6617769c..fc8ab6cd0bb1a 100644 --- a/python/paddle/jit/dy2static/transformers/ifelse_transformer.py +++ b/python/paddle/jit/dy2static/transformers/ifelse_transformer.py @@ -231,7 +231,7 @@ def visit_Assign(self, node): def visit_FunctionDef(self, node): # NOTE: We skip to visit names of get_args and set_args, because they contains # nonlocal statement such as 'nonlocal x, self' where 'self' should not be - # parsed as returned value in contron flow. + # parsed as returned value in control flow. if ( GET_ARGS_FUNC_PREFIX in node.name or SET_ARGS_FUNC_PREFIX in node.name @@ -343,7 +343,7 @@ def transform_if_else(node, root): nonlocal_names = _valid_nonlocal_names(return_name_ids, nonlocal_names) # TODO(dev): Need a better way to deal this. - # LoopTransformer will create some special vars, which is not visiable by users. so we can sure it's safe to remove them. + # LoopTransformer will create some special vars, which is not visible by users. so we can sure it's safe to remove them. filter_names = [ ARGS_NAME, FOR_ITER_INDEX_PREFIX, diff --git a/python/paddle/jit/dy2static/transformers/utils.py b/python/paddle/jit/dy2static/transformers/utils.py index e74d95e1af9e0..37e5a400e31c4 100644 --- a/python/paddle/jit/dy2static/transformers/utils.py +++ b/python/paddle/jit/dy2static/transformers/utils.py @@ -499,7 +499,7 @@ def pre_func(): def post_func(): """NOTE: why we need merge w_vars and push_pop_vars here ? - because we do ifelse_transformer after loop_transformer. Loops will changed into functioons. but we know this function will be called in if. so we add w_vars to father function scope. + because we do ifelse_transformer after loop_transformer. Loops will changed into functions. but we know this function will be called in if. so we add w_vars to father function scope. """ control_flow_function_def = [ WHILE_BODY_PREFIX, diff --git a/python/paddle/jit/sot/symbolic/interpreter.py b/python/paddle/jit/sot/symbolic/interpreter.py index 3179a4c518f82..6b60a2bbbb5fe 100644 --- a/python/paddle/jit/sot/symbolic/interpreter.py +++ b/python/paddle/jit/sot/symbolic/interpreter.py @@ -187,7 +187,7 @@ def wrapper(args): def prepare_state(SIR, inputs): state = {} - # update free vars if exsits + # update free vars if exists if SIRRuntimeCache().has_key(SIR.name): free_var_seeker = SIRRuntimeCache().get_free_vars(SIR.name) if free_var_seeker: diff --git a/python/paddle/nn/functional/vision.py b/python/paddle/nn/functional/vision.py index 7a76f35b9589c..2e5c988ab0c8e 100644 --- a/python/paddle/nn/functional/vision.py +++ b/python/paddle/nn/functional/vision.py @@ -148,7 +148,7 @@ def grid_sample( indexing the 5th dimension (in width dimension) of input data x, y is indexing the 4th dimension (in height dimension) and z is indexing the 3rd dimension (in depth dimension) finally results is the bilinear - interpolation or nearest value of 8 nearest cornerpoints. The output + interpolation or nearest value of 8 nearest corner points. The output tensor shape will be [N, C, D, H, W]. diff --git a/python/paddle/nn/initializer/Bilinear.py b/python/paddle/nn/initializer/Bilinear.py index 1da82cbeee970..05ac3641caff2 100644 --- a/python/paddle/nn/initializer/Bilinear.py +++ b/python/paddle/nn/initializer/Bilinear.py @@ -120,7 +120,7 @@ def forward(self, var, block=None): weight[i] = (1 - abs(x / f - c)) * (1 - abs(y / f - c)) weight = np.reshape(weight, shape) - # to be compatible of fp16 initalizers + # to be compatible of fp16 initializers if var.dtype in [ core.VarDesc.VarType.FP16, core.VarDesc.VarType.BF16, diff --git a/python/paddle/nn/initializer/assign.py b/python/paddle/nn/initializer/assign.py index 3988f9f14859d..0d04cbbb78398 100644 --- a/python/paddle/nn/initializer/assign.py +++ b/python/paddle/nn/initializer/assign.py @@ -66,7 +66,7 @@ def forward(self, var, block=None): ) assert isinstance(block, (framework.Block, paddle.pir.Block)) - # to be compatible of fp16 initalizers + # to be compatible of fp16 initializers if var.dtype in [core.VarDesc.VarType.FP16, core.VarDesc.VarType.BF16]: out_dtype = core.VarDesc.VarType.FP32 np_value = self._value.astype("float32") diff --git a/python/paddle/nn/initializer/kaiming.py b/python/paddle/nn/initializer/kaiming.py index 39329acaf7da1..efb1fc486d059 100644 --- a/python/paddle/nn/initializer/kaiming.py +++ b/python/paddle/nn/initializer/kaiming.py @@ -52,7 +52,7 @@ class MSRAInitializer(Initializer): Args: uniform (bool, optional): whether to use uniform or normal distribution. Default is True. - fan_in (float32|None, optional): fan_in (in_features) of trainable Tensor, If None, it will be infered automaticly. If you don't want to use in_features of the Tensor, you can set the value of 'fan_in' smartly by yourself. Default is None. + fan_in (float32|None, optional): fan_in (in_features) of trainable Tensor, If None, it will be infered automatically. If you don't want to use in_features of the Tensor, you can set the value of 'fan_in' smartly by yourself. Default is None. seed (int32, optional): random seed. Default is 0. negative_slope (float, optional): negative_slope (only used with leaky_relu). Default is 0.0. nonlinearity(str, optional): the non-linear function. Default is relu. @@ -107,7 +107,7 @@ def forward(self, var, block=None): if self._seed == 0: self._seed = block.program.random_seed - # to be compatible of fp16 initalizers + # to be compatible of fp16 initializers if var.dtype == core.VarDesc.VarType.FP16 or ( var.dtype == core.VarDesc.VarType.BF16 and not self._uniform ): @@ -252,7 +252,7 @@ class KaimingNormal(MSRAInitializer): \frac{gain}{\sqrt{{fan\_in}}} Args: - fan_in (float32|None, optional): fan_in (in_features) of trainable Tensor, If None, it will be infered automaticly. If you don't want to use in_features of the Tensor, you can set the value of 'fan_in' smartly by yourself. Default is None. + fan_in (float32|None, optional): fan_in (in_features) of trainable Tensor, If None, it will be infered automatically. If you don't want to use in_features of the Tensor, you can set the value of 'fan_in' smartly by yourself. Default is None. negative_slope (float, optional): negative_slope (only used with leaky_relu). Default is 0.0. nonlinearity(str, optional): the non-linear function. Default is relu. diff --git a/python/paddle/nn/initializer/normal.py b/python/paddle/nn/initializer/normal.py index 4ca0a0902246c..77ecd855b0556 100644 --- a/python/paddle/nn/initializer/normal.py +++ b/python/paddle/nn/initializer/normal.py @@ -200,7 +200,7 @@ def forward(self, var, block=None): if self._seed == 0: self._seed = block.program.random_seed - # to be compatible of fp16 initalizers + # to be compatible of fp16 initializers if var.dtype in [core.VarDesc.VarType.FP16, core.VarDesc.VarType.BF16]: out_dtype = core.VarDesc.VarType.FP32 out_var = block.create_var( diff --git a/python/paddle/nn/initializer/xavier.py b/python/paddle/nn/initializer/xavier.py index e455ca455cd00..fd47805c22133 100644 --- a/python/paddle/nn/initializer/xavier.py +++ b/python/paddle/nn/initializer/xavier.py @@ -110,7 +110,7 @@ def forward(self, var, block=None): if (isinstance(var, framework.EagerParamBase) and var.is_dist()) else var.shape ) - # to be compatible of fp16 initalizers + # to be compatible of fp16 initializers if var.dtype == core.VarDesc.VarType.FP16 or ( var.dtype == core.VarDesc.VarType.BF16 and not self._uniform ): diff --git a/python/paddle/nn/layer/activation.py b/python/paddle/nn/layer/activation.py index 60e3a95a20d18..59a9436dadb51 100644 --- a/python/paddle/nn/layer/activation.py +++ b/python/paddle/nn/layer/activation.py @@ -134,8 +134,8 @@ class GLU(Layer): For more information, please refer to :ref:`api_guide_Name`. Shape: - - input: Tensor which the size of the given aixs is even. - - output: Tensor which the size of the given aixs is halved. + - input: Tensor which the size of the given axis is even. + - output: Tensor which the size of the given axis is halved. Examples: .. code-block:: python @@ -799,7 +799,7 @@ def extra_repr(self): class Sigmoid(Layer): r""" - this interface is used to construct a callable object of the ``Sigmoid`` class. This layer calcluate the `sigmoid` of input x. + this interface is used to construct a callable object of the ``Sigmoid`` class. This layer calculate the `sigmoid` of input x. .. math:: @@ -842,8 +842,8 @@ def extra_repr(self): class Hardsigmoid(Layer): r""" - ``Hardsigmoid`` Activiation Layers, Construct a callable object of - the ``Hardsigmoid`` class. This layer calcluate the `hardsigmoid` of input x. + ``Hardsigmoid`` Activation Layers, Construct a callable object of + the ``Hardsigmoid`` class. This layer calculate the `hardsigmoid` of input x. A 3-part piecewise linear approximation of sigmoid(https://arxiv.org/abs/1603.00391), which is much faster than sigmoid. diff --git a/python/paddle/nn/layer/loss.py b/python/paddle/nn/layer/loss.py index 310e06a35a557..1fd2501698c2f 100644 --- a/python/paddle/nn/layer/loss.py +++ b/python/paddle/nn/layer/loss.py @@ -68,7 +68,7 @@ class BCEWithLogitsLoss(Layer): batch element. If given, it has to be a 1D Tensor whose size is `[N, ]`, The data type is float32, float64. Default is ``'None'``. reduction (str, optional): Indicate how to average the loss by batch_size, - the candicates are ``'none'`` | ``'mean'`` | ``'sum'``. + the candidates are ``'none'`` | ``'mean'`` | ``'sum'``. If :attr:`reduction` is ``'none'``, the unreduced loss is returned; If :attr:`reduction` is ``'mean'``, the reduced mean loss is returned; If :attr:`reduction` is ``'sum'``, the summed loss is returned. @@ -253,7 +253,7 @@ class CrossEntropyLoss(Layer): value needs to be ignored. Only valid when soft_label = False. Default is ``-100`` . reduction (str, optional): Indicate how to average the loss by batch_size, - the candicates are ``'none'`` | ``'mean'`` | ``'sum'``. + the candidates are ``'none'`` | ``'mean'`` | ``'sum'``. If :attr:`reduction` is ``'mean'``, the reduced mean loss is returned; If :attr:`size_average` is ``'sum'``, the reduced sum loss is returned. If :attr:`reduction` is ``'none'``, the unreduced loss is returned. @@ -366,15 +366,15 @@ class CrossEntropyLoss(Layer): >>> reduction='mean' >>> weight = None >>> logits = paddle.uniform(shape, dtype='float64', min=0.1, max=1.0) - >>> interger_labels = paddle.randint(low=0, high=C, shape=[N], dtype='int64') - >>> one_hot_labels = paddle.nn.functional.one_hot(interger_labels, C).astype('float32') + >>> integer_labels = paddle.randint(low=0, high=C, shape=[N], dtype='int64') + >>> one_hot_labels = paddle.nn.functional.one_hot(integer_labels, C).astype('float32') >>> cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( ... weight=weight, reduction=reduction, label_smoothing=label_smoothing) >>> # integer labels - >>> interger_label_dy_ret = cross_entropy_loss(logits, interger_labels) - >>> print(interger_label_dy_ret) + >>> integer_label_dy_ret = cross_entropy_loss(logits, integer_labels) + >>> print(integer_label_dy_ret) Tensor(shape=[], dtype=float64, place=Place(cpu), stop_gradient=True, 1.10520368) @@ -669,7 +669,7 @@ class L1Loss(Layer): Parameters: reduction (str, optional): Indicate the reduction to apply to the loss, - the candicates are ``'none'`` | ``'mean'`` | ``'sum'``. + the candidates are ``'none'`` | ``'mean'`` | ``'sum'``. If `reduction` is ``'none'``, the unreduced loss is returned; If `reduction` is ``'mean'``, the reduced mean loss is returned. If `reduction` is ``'sum'``, the reduced sum loss is returned. @@ -765,7 +765,7 @@ class BCELoss(Layer): batch element. If given, has to be a Tensor of size nbatch and the data type is float32, float64. Default is ``'None'``. reduction (str, optional): Indicate how to average the loss by batch_size, - the candicates are ``'none'`` | ``'mean'`` | ``'sum'``. + the candidates are ``'none'`` | ``'mean'`` | ``'sum'``. If :attr:`reduction` is ``'none'``, the unreduced loss is returned; If :attr:`reduction` is ``'mean'``, the reduced mean loss is returned; If :attr:`reduction` is ``'sum'``, the summed loss is returned. @@ -864,10 +864,10 @@ class NLLLoss(Layer): ignore_index (int, optional): Specifies a target value that is ignored and does not contribute to the input gradient. reduction (str, optional): Indicate how to average the loss, - the candicates are ``'none'`` | ``'mean'`` | ``'sum'``. Default is ``'mean'``. + the candidates are ``'none'`` | ``'mean'`` | ``'sum'``. Default is ``'mean'``. If `reduction` is ``'mean'``, the reduced mean loss is returned; if `reduction` is ``'sum'``, the reduced sum loss is returned; - if `reduction` is ``'none'``, no reduction will be apllied. + if `reduction` is ``'none'``, no reduction will be applied. Default is ``'mean'``. name (str, optional): For details, please refer to :ref:`api_guide_Name`. Generally, no setting is required. Default is ``'None'``. @@ -959,10 +959,10 @@ class PoissonNLLLoss(Layer): A small value to avoid evaluation of :math:`\log(0)` when ``log_input`` = ``False``. ``epsilon > 0``. Default: 1e-8. reduction (str, optional): - Indicate how to reduce the loss, the candicates are ``'none'`` | ``'mean'`` | ``'sum'``. + Indicate how to reduce the loss, the candidates are ``'none'`` | ``'mean'`` | ``'sum'``. If `reduction` is ``'mean'``, the reduced mean loss is returned; if `reduction` is ``'sum'``, the reduced sum loss is returned; - if `reduction` is ``'none'``, no reduction will be apllied. + if `reduction` is ``'none'``, no reduction will be applied. Default is ``'mean'``. name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. @@ -997,7 +997,7 @@ def __init__( ): if epsilon <= 0: raise ValueError( - "The value of `epsilon` in PoissonNLLLoss should be positve, but received %f, which is not allowed" + "The value of `epsilon` in PoissonNLLLoss should be positive, but received %f, which is not allowed" % epsilon ) if reduction not in ['sum', 'mean', 'none']: @@ -1048,11 +1048,11 @@ class KLDivLoss(Layer): Parameters: reduction (str, optional): Indicate how to average the loss, - the candicates are ``'none'`` | ``'batchmean'`` | ``'mean'`` | ``'sum'``. + the candidates are ``'none'`` | ``'batchmean'`` | ``'mean'`` | ``'sum'``. If `reduction` is ``'mean'``, the reduced mean loss is returned; If `reduction` is ``'batchmean'``, the sum loss divided by batch size is returned; if `reduction` is ``'sum'``, the reduced sum loss is returned; - if `reduction` is ``'none'``, no reduction will be apllied. + if `reduction` is ``'none'``, no reduction will be applied. Default is ``'mean'``. Shape: @@ -1132,7 +1132,7 @@ class MarginRankingLoss(Layer): Parameters: margin (float, optional): The margin value to add, default value is 0; - reduction (str, optional): Indicate the reduction to apply to the loss, the candicates are ``'none'``, ``'mean'``, ``'sum'``.If :attr:`reduction` is ``'none'``, the unreduced loss is returned; If :attr:`reduction` is ``'mean'``, the reduced mean loss is returned. If :attr:`reduction` is ``'sum'``, the reduced sum loss is returned. Default is ``'mean'``. + reduction (str, optional): Indicate the reduction to apply to the loss, the candidates are ``'none'``, ``'mean'``, ``'sum'``.If :attr:`reduction` is ``'none'``, the unreduced loss is returned; If :attr:`reduction` is ``'mean'``, the reduced mean loss is returned. If :attr:`reduction` is ``'sum'``, the reduced sum loss is returned. Default is ``'mean'``. name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Shape: @@ -1188,11 +1188,11 @@ class CTCLoss(Layer): An operator integrating the open source Warp-CTC library (https://github.com/baidu-research/warp-ctc) to compute Connectionist Temporal Classification (CTC) loss. It can be aliased as softmax with CTC, since a native softmax activation - is interated to the Warp-CTC library to normalize values for each row of the input tensor. + is integrated to the Warp-CTC library to normalize values for each row of the input tensor. Parameters: blank (int, optional): The blank label index of Connectionist Temporal Classification (CTC) loss, which is in the half-opened interval [0, num_classes + 1). The data type must be int32. Default is 0. - reduction (string, optional): Indicate how to average the loss, the candicates are ``'none'`` | ``'mean'`` | ``'sum'``. If :attr:`reduction` is ``'mean'``, the output loss will be divided by the label_lengths, and then return the mean of quotient; If :attr:`reduction` is ``'sum'``, return the sum of loss; If :attr:`reduction` is ``'none'``, no reduction will be applied. Default is ``'mean'``. + reduction (string, optional): Indicate how to average the loss, the candidates are ``'none'`` | ``'mean'`` | ``'sum'``. If :attr:`reduction` is ``'mean'``, the output loss will be divided by the label_lengths, and then return the mean of quotient; If :attr:`reduction` is ``'sum'``, return the sum of loss; If :attr:`reduction` is ``'none'``, no reduction will be applied. Default is ``'mean'``. Shape: - log_probs (Tensor): The unscaled probability sequence with padding, which is a 3-D Tensor. The tensor shape is [max_logit_length, batch_size, num_classes + 1], where max_logit_length is the longest length of input logit sequence. The data type should be float32 or float64. @@ -1364,7 +1364,7 @@ class SmoothL1Loss(Layer): Parameters: reduction (str, optional): Indicate how to average the loss by batch_size, - the candicates are ``'none'`` | ``'mean'`` | ``'sum'``. + the candidates are ``'none'`` | ``'mean'`` | ``'sum'``. If :attr:`reduction` is ``'mean'``, the reduced mean loss is returned; If :attr:`reduction` is ``'sum'``, the reduced sum loss is returned. If :attr:`reduction` is ``'none'``, the unreduced loss is returned. @@ -1437,7 +1437,7 @@ class MultiLabelSoftMarginLoss(Layer): If given, has to be a Tensor of size C and the data type is float32, float64. Default is ``'None'`` . reduction (str, optional): Indicate how to average the loss by batch_size, - the candicates are ``'none'`` | ``'mean'`` | ``'sum'``. + the candidates are ``'none'`` | ``'mean'`` | ``'sum'``. If :attr:`reduction` is ``'none'``, the unreduced loss is returned; If :attr:`reduction` is ``'mean'``, the reduced mean loss is returned; If :attr:`reduction` is ``'sum'``, the summed loss is returned. @@ -1531,7 +1531,7 @@ class HingeEmbeddingLoss(Layer): hinge_embedding_loss. When label is -1, Input smaller than margin are minimized with hinge_embedding_loss. Default = 1.0 reduction (str, optional): Indicate how to average the loss by batch_size, - the candicates are ``'none'`` | ``'mean'`` | ``'sum'``. + the candidates are ``'none'`` | ``'mean'`` | ``'sum'``. If :attr:`reduction` is ``'none'``, the unreduced loss is returned; If :attr:`reduction` is ``'mean'``, the reduced mean loss is returned; If :attr:`reduction` is ``'sum'``, the summed loss is returned. @@ -1733,7 +1733,7 @@ class TripletMarginWithDistanceLoss(Layer): and negative samples) if swap distance smaller than negative distance. Default: ``False``. reduction (str, Optional):Indicate how to average the loss by batch_size. - the candicates are ``'none'`` | ``'mean'`` | ``'sum'``. + the candidates are ``'none'`` | ``'mean'`` | ``'sum'``. If :attr:`reduction` is ``'none'``, the unreduced loss is returned; If :attr:`reduction` is ``'mean'``, the reduced mean loss is returned; If :attr:`reduction` is ``'sum'``, the summed loss is returned. @@ -1845,7 +1845,7 @@ class TripletMarginLoss(Layer): Default: ``False``. reduction (str, Optional):Indicate how to average the loss by batch_size. - the candicates are ``'none'`` | ``'mean'`` | ``'sum'``. + the candidates are ``'none'`` | ``'mean'`` | ``'sum'``. If :attr:`reduction` is ``'none'``, the unreduced loss is returned; If :attr:`reduction` is ``'mean'``, the reduced mean loss is returned; If :attr:`reduction` is ``'sum'``, the summed loss is returned. diff --git a/python/paddle/nn/layer/norm.py b/python/paddle/nn/layer/norm.py index 5fc885e981c5b..ff64b4dfd3de8 100644 --- a/python/paddle/nn/layer/norm.py +++ b/python/paddle/nn/layer/norm.py @@ -57,7 +57,7 @@ class _InstanceNormBase(Layer): """ This class is based class for InstanceNorm1D, 2d, 3d. - See InstaceNorm1D, InstanceNorm2D or InstanceNorm3D for more details. + See InstanceNorm1D, InstanceNorm2D or InstanceNorm3D for more details. """ def __init__( @@ -779,7 +779,7 @@ def __init__( ) self._variance.stop_gradient = True - # TODO(qili93): temporary for ascned npu performance to be removed along with npu_identity op + # TODO(qili93): temporary for ascend npu performance to be removed along with npu_identity op if ( _global_flags()['FLAGS_npu_storage_format'] and 'npu' in get_all_custom_device_type() @@ -1018,7 +1018,7 @@ def __init__( ) self._variance.stop_gradient = True - # TODO(qili93): temporary for ascned npu performance to be removed along with npu_identity op + # TODO(qili93): temporary for ascend npu performance to be removed along with npu_identity op if ( _global_flags()['FLAGS_npu_storage_format'] and 'npu' in get_all_custom_device_type() @@ -1157,7 +1157,7 @@ def forward(self, input): class BatchNorm1D(_BatchNormBase): r""" - Applies Batch Normalization over a 2D or 3D input (a mini-batch of 1D inputswith additional channel dimension) as described in the paper Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift . + Applies Batch Normalization over a 2D or 3D input (a mini-batch of 1D inputs with additional channel dimension) as described in the paper Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift . When use_global_stats = False, the :math:`\mu_{\beta}` and :math:`\sigma_{\beta}^{2}` are the statistics of one mini-batch. @@ -1274,7 +1274,7 @@ def _check_input_dim(self, input): class BatchNorm2D(_BatchNormBase): r""" - Applies Batch Normalization over a 4D input (a mini-batch of 2D inputswith additional channel dimension) as described in the paper Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift . + Applies Batch Normalization over a 4D input (a mini-batch of 2D inputs with additional channel dimension) as described in the paper Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift . When use_global_stats = False, the :math:`\mu_{\beta}` and :math:`\sigma_{\beta}^{2}` are the statistics of one mini-batch. @@ -1365,7 +1365,7 @@ def _check_input_dim(self, input): class BatchNorm3D(_BatchNormBase): r""" - Applies Batch Normalization over a 5D input (a mini-batch of 3D inputswith additional channel dimension) as described in the paper Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift . + Applies Batch Normalization over a 5D input (a mini-batch of 3D inputs with additional channel dimension) as described in the paper Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift . When use_global_stats = False, the :math:`\mu_{\beta}` and :math:`\sigma_{\beta}^{2}` are the statistics of one mini-batch. @@ -1539,7 +1539,7 @@ class SyncBatchNorm(_BatchNormBase): epsilon(float, optional): The small value added to the variance to prevent division by zero. Default: 1e-5. momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9. weight_attr(ParamAttr|bool, optional): The parameter attribute for Parameter `scale` - of this layer. If it is set to None or one attribute of ParamAttr, this layerr + of this layer. If it is set to None or one attribute of ParamAttr, this layer will create ParamAttr as param_attr. If the Initializer of the param_attr is not set, the parameter is initialized with ones. If it is set to False, this layer will not have trainable scale parameter. Default: None. diff --git a/python/paddle/nn/layer/pooling.py b/python/paddle/nn/layer/pooling.py index 38fee16aad1b3..23eaf467d916d 100755 --- a/python/paddle/nn/layer/pooling.py +++ b/python/paddle/nn/layer/pooling.py @@ -1143,7 +1143,7 @@ def extra_repr(self): class MaxUnPool1D(Layer): r""" - This API implements max unpooling 1d opereation. + This API implements max unpooling 1d operation. `max_unpool1d` accepts the output of `max_pool1d` as input, including the indices of the maximum value and calculate the partial inverse. @@ -1231,7 +1231,7 @@ def extra_repr(self): class MaxUnPool2D(Layer): r""" - This API implements max unpooling 2d opereation. + This API implements max unpooling 2d operation. 'max_unpool2d' accepts the output of 'max_unpool2d' as input Including the indices of the maximum value and calculating the partial inverse @@ -1323,7 +1323,7 @@ def extra_repr(self): class MaxUnPool3D(Layer): r""" - This API implements max unpooling 3d opereation. + This API implements max unpooling 3d operation. `max_unpool3d` accepts the output of `max_pool3d` as input, including the indices of the maximum value and calculate the partial inverse. diff --git a/python/paddle/profiler/profiler.py b/python/paddle/profiler/profiler.py index 7e35344206c1a..61c44b9ea19b5 100644 --- a/python/paddle/profiler/profiler.py +++ b/python/paddle/profiler/profiler.py @@ -855,7 +855,7 @@ def summary( views=None, ): r""" - Print the Summary table. Currently support overview, model, distributed, operator, memory manipulation and userdefined summary. + Print the Summary table. Currently support overview, model, distributed, operator, memory manipulation and user-defined summary. Args: sorted_by( :ref:`SortedKeys ` , optional): how to rank the op table items, default value is SortedKeys.CPUTotal. diff --git a/python/paddle/quantization/imperative/fuse_utils.py b/python/paddle/quantization/imperative/fuse_utils.py index 0f8ad443d43ca..f31a70297893e 100644 --- a/python/paddle/quantization/imperative/fuse_utils.py +++ b/python/paddle/quantization/imperative/fuse_utils.py @@ -92,7 +92,7 @@ def _fuse_layers(model, layers_list): def _fuse_func(layer_list): - '''choose the fuser method and fuse layers''' + '''choose the fuse method and fuse layers''' types = tuple(type(m) for m in layer_list) fusion_method = types_to_fusion_method.get(types, None) new_layers = [None] * len(layer_list) diff --git a/python/paddle/quantization/imperative/ptq_registry.py b/python/paddle/quantization/imperative/ptq_registry.py index 52e4c487e342f..a865bc5d912f8 100644 --- a/python/paddle/quantization/imperative/ptq_registry.py +++ b/python/paddle/quantization/imperative/ptq_registry.py @@ -17,7 +17,7 @@ class LayerInfo: """ - Store the argnames of the inputs and outputs. + Store the arg names of the inputs and outputs. """ def __init__(self, layer, input_names, weight_names, output_names): diff --git a/python/paddle/tensor/manipulation.py b/python/paddle/tensor/manipulation.py index e6c8cef4f905c..2d2d9375f4a09 100644 --- a/python/paddle/tensor/manipulation.py +++ b/python/paddle/tensor/manipulation.py @@ -6758,7 +6758,7 @@ def slice_scatter(x, value, axes, starts, ends, strides, name=None): axes (list|tuple) : the dimensions to insert the value. starts (list|tuple) : the start indices of where to insert. ends (list|tuple) : the stop indices of where to insert. - strids (list|tuple) : the steps for each insert. + strides (list|tuple) : the steps for each insert. name (str, optional): Name for the operation (optional, default is None). Returns: From a845436ef3e753489bf39164c3c49203c4ea1a89 Mon Sep 17 00:00:00 2001 From: zyfncg Date: Fri, 23 Feb 2024 15:34:38 +0800 Subject: [PATCH 035/282] fix build of generate_shape_op (#61993) --- paddle/cinn/hlir/dialect/operator/ir/manual_op.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/paddle/cinn/hlir/dialect/operator/ir/manual_op.cc b/paddle/cinn/hlir/dialect/operator/ir/manual_op.cc index 42fdf23664a8d..54299cc2ff7ff 100644 --- a/paddle/cinn/hlir/dialect/operator/ir/manual_op.cc +++ b/paddle/cinn/hlir/dialect/operator/ir/manual_op.cc @@ -267,13 +267,12 @@ void GenerateShapeOp::Build( const std::vector& inputs, const std::vector& output_dim_exprs, const GenerateShapeOp::SymbolBindings& symbol_bindings) { - CHECK(!inputs.empty()) << ". output_dim_exprs: " << [&] { - std::stringstream ss; + if (inputs.empty()) { + VLOG(3) << "GenerateShapeOp inputs is empty"; for (const auto& attr : output_dim_exprs) { - ss << attr; + CHECK(attr.isa()); } - return ss.str(); - }(); + } argument.AddInputs(inputs); argument.AddAttribute("output_dim_exprs", builder.array_attr(output_dim_exprs)); From 7aca52c1e5ec2e0b5806bdf0d67969b38c701ddb Mon Sep 17 00:00:00 2001 From: risemeup1 <62429225+risemeup1@users.noreply.github.com> Date: Fri, 23 Feb 2024 16:16:09 +0800 Subject: [PATCH 036/282] fix,test=document_fix (#62018) --- cmake/third_party.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/third_party.cmake b/cmake/third_party.cmake index 6c715e8cf7510..17c428660b223 100755 --- a/cmake/third_party.cmake +++ b/cmake/third_party.cmake @@ -46,7 +46,7 @@ if(NOT WITH_SETUP_INSTALL) endif() execute_process( - COMMAND git submodule update --init --recursive --force + COMMAND git submodule update --init --recursive WORKING_DIRECTORY ${PADDLE_SOURCE_DIR} RESULT_VARIABLE result_var) if(NOT result_var EQUAL 0) From 934afd220ead2818c60a77dd4c2742e821548e82 Mon Sep 17 00:00:00 2001 From: Bo Zhang <105368690+zhangbopd@users.noreply.github.com> Date: Fri, 23 Feb 2024 16:24:40 +0800 Subject: [PATCH 037/282] [PIR][DynamicShape] Add InferSymbolicShape interface for WhileOp (#61939) * add while infer * yield * bug fix --- paddle/fluid/pir/dialect/CMakeLists.txt | 3 +- .../infer_symbolic_shape/cinn_op_infer_sym.cc | 117 ++++++++++-------- .../infer_symbolic_shape/cinn_op_infer_sym.h | 11 +- .../infer_symbolic_shape/infer_sym_utils.cc | 21 ++++ .../infer_symbolic_shape/infer_sym_utils.h | 11 ++ .../paddle_op_infer_sym.cc | 97 ++++++++------- .../same_operands_and_result.h | 4 + .../dialect/operator/ir/control_flow_op.cc | 112 +++++++++++++++++ .../pir/dialect/operator/ir/control_flow_op.h | 9 +- .../pir/transforms/shape_optimization_pass.cc | 51 ++++---- .../pir/transforms/shape_optimization_pass.h | 3 + paddle/pir/CMakeLists.txt | 7 +- .../include/dialect/control_flow/ir/cf_op.h | 7 +- .../pir/src/dialect/control_flow/ir/cf_op.cc | 7 ++ .../shape_dialect/shape_optimization_test.cc | 23 ++-- test/ir/pir/cinn/symbolic/test_if_else_dy.py | 92 ++++++++++++++ test/ir/pir/cinn/symbolic/test_while_dy.py | 10 +- 17 files changed, 443 insertions(+), 142 deletions(-) create mode 100644 test/ir/pir/cinn/symbolic/test_if_else_dy.py diff --git a/paddle/fluid/pir/dialect/CMakeLists.txt b/paddle/fluid/pir/dialect/CMakeLists.txt index ff3e03d3963dd..535f8cdc7c818 100644 --- a/paddle/fluid/pir/dialect/CMakeLists.txt +++ b/paddle/fluid/pir/dialect/CMakeLists.txt @@ -246,7 +246,8 @@ set(op_dialect_srcs ${pir_op_source_file} ${pir_bwd_op_source_file} ${pir_update_op_source_file} - ${api_source_file}) + ${api_source_file} + ${PADDLE_SOURCE_DIR}/paddle/fluid/pir/transforms/shape_optimization_pass.cc) if(WITH_MKLDNN) set(op_dialect_srcs ${op_dialect_srcs} ${onednn_op_source_file} diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.cc index ecb56292e170a..0e8240434e070 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.cc @@ -36,74 +36,32 @@ bool BroadcastOpInferSymbolicShape( return true; } -bool SliceOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - // TODO(zhangbopd): Not implemented yet, different from the one in paddle - // dialect. And Currently only support start/end/axis with single value. - pir::AttributeMap attributes = op->attributes(); - - auto GetAttrInt64Value = [&](const std::string &name) -> int64_t { - std::vector attr = - attributes[name].dyn_cast().AsVector(); - PADDLE_ENFORCE_GT( - attr.size(), - 0, - phi::errors::PreconditionNotMet( - "Only Support [%s] op len(%s) == 1 , but received %d.", - op->name(), - name, - attr.size())); - return attr[0].dyn_cast().data(); - }; - - const int64_t start = GetAttrInt64Value("starts"); - const int64_t end = GetAttrInt64Value("ends"); - const int64_t axis = GetAttrInt64Value("axes"); - - const pir::Value operand_source = op->operand_source(0); - const auto &operand_shape_or_data = - shape_analysis->GetShapeOrDataForValue(operand_source); - - const auto GetOutDimExprs = [&]() -> symbol::TensorShapeOrDataDimExprs { - std::vector out_sym_shape = operand_shape_or_data.shape(); - if (end == std::numeric_limits::max()) { - out_sym_shape[axis] = out_sym_shape[axis] - start; - } else { - out_sym_shape[axis] = end - start; - } - symbol::TensorShapeOrDataDimExprs shape_dim_expr(out_sym_shape); - if (operand_shape_or_data.data().has_value()) { - std::vector out_data; - for (int64_t i = start; i < end; i++) { - out_data.push_back(operand_shape_or_data.data().value()[i]); - } - shape_dim_expr.SetData(out_data); - } - return shape_dim_expr; - }; - symbol::ShapeOrDataDimExprs shape_data{GetOutDimExprs()}; - - shape_analysis->SetShapeOrDataForValue(op->result(0), shape_data); - return true; -} - bool ConcatOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { const auto input_values = op->operands_source(); const auto input_size = input_values.size(); - const int axis = - op->attributes().at("axis").dyn_cast().data(); + int axis = op->attributes().at("axis").dyn_cast().data(); - // TODO(zhangbopd): Need support GetShapeOrDataForValue().data() case. const auto &GetOutDimExprs = [&]() -> std::vector { std::vector out_dims = shape_analysis->GetShapeOrDataForValue(input_values[0]).shape(); + + size_t rank = out_dims.size(); + axis = axis >= 0 ? axis : std::max(int64_t(0), int64_t(axis + rank)); + for (size_t i = 1; i < input_size; ++i) { const auto &operand_shape_or_data = shape_analysis->GetShapeOrDataForValue(input_values[i]); out_dims[axis] = out_dims[axis] + operand_shape_or_data.shape()[axis]; } + + for (size_t i = 1; i < rank; ++i) { + if (i == static_cast(axis)) continue; + paddle::dialect::details::BuildCstrEqForTensorListAlongAxis( + shape_analysis, input_values, i); + } + return out_dims; }; @@ -164,4 +122,55 @@ bool ReshapeOpInferSymbolicShape( return true; } +bool SliceOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + // TODO(zhangbopd): Not implemented yet, different from the one in paddle + // dialect. And Currently only support start/end/axis with single value. + pir::AttributeMap attributes = op->attributes(); + + auto GetAttrInt64Value = [&](const std::string &name) -> int64_t { + std::vector attr = + attributes[name].dyn_cast().AsVector(); + PADDLE_ENFORCE_GT( + attr.size(), + 0, + phi::errors::PreconditionNotMet( + "Only Support [%s] op len(%s) == 1 , but received %d.", + op->name(), + name, + attr.size())); + return attr[0].dyn_cast().data(); + }; + + const int64_t start = GetAttrInt64Value("starts"); + const int64_t end = GetAttrInt64Value("ends"); + const int64_t axis = GetAttrInt64Value("axes"); + + const pir::Value operand_source = op->operand_source(0); + const auto &operand_shape_or_data = + shape_analysis->GetShapeOrDataForValue(operand_source); + + const auto GetOutDimExprs = [&]() -> symbol::TensorShapeOrDataDimExprs { + std::vector out_sym_shape = operand_shape_or_data.shape(); + if (end == std::numeric_limits::max()) { + out_sym_shape[axis] = out_sym_shape[axis] - start; + } else { + out_sym_shape[axis] = end - start; + } + symbol::TensorShapeOrDataDimExprs shape_dim_expr(out_sym_shape); + if (operand_shape_or_data.data().has_value()) { + std::vector out_data; + for (int64_t i = start; i < end; i++) { + out_data.push_back(operand_shape_or_data.data().value()[i]); + } + shape_dim_expr.SetData(out_data); + } + return shape_dim_expr; + }; + symbol::ShapeOrDataDimExprs shape_data{GetOutDimExprs()}; + + shape_analysis->SetShapeOrDataForValue(op->result(0), shape_data); + return true; +} + } // namespace cinn::dialect diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.h index 896dd44d0b12b..b98f8e02d66e9 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.h @@ -16,23 +16,32 @@ #include "paddle/pir/include/dialect/shape/utils/shape_analysis.h" namespace cinn::dialect { -// using paddle::dialect::ScaleOpInferSymbolicShape; bool BroadcastOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + bool SliceOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + bool ConcatOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + bool ReduceMaxOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + bool ReduceMinOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + bool ReduceProdOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + bool ReduceSumOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + bool ReshapeOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool SliceOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); + } // namespace cinn::dialect diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.cc index 6337d125b5610..4e5f5df08732a 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.cc @@ -78,4 +78,25 @@ bool ReduceInferDim(pir::Operation *op, return true; } +void BuildCstrEqForTensorListAlongAxis( + pir::ShapeConstraintIRAnalysis *shape_analysis, + const symbol::TensorListShapeOrDataDimExprs &shape_data_list, + int axis) { + for (size_t i = 1; i < shape_data_list.size(); ++i) { + shape_analysis->CreateDimExprBuilder().CstrEq( + shape_data_list[0].shape()[axis], shape_data_list[i].shape()[axis]); + } +} + +void BuildCstrEqForTensorListAlongAxis( + pir::ShapeConstraintIRAnalysis *shape_analysis, + const std::vector &values, + int axis) { + for (size_t i = 1; i < values.size(); ++i) { + shape_analysis->CreateDimExprBuilder().CstrEq( + shape_analysis->GetShapeOrDataForValue(values[0]).shape()[axis], + shape_analysis->GetShapeOrDataForValue(values[i]).shape()[axis]); + } +} + } // namespace paddle::dialect::details diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.h index 513636344ea2b..8a14e40e6337a 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.h @@ -65,4 +65,15 @@ bool ReduceInferDim(pir::Operation *op, const std::vector &axis, bool keep_dim, bool reduce_all); + +void BuildCstrEqForTensorListAlongAxis( + pir::ShapeConstraintIRAnalysis *shape_analysis, + const symbol::TensorListShapeOrDataDimExprs &shape_data_list, + int axis); + +void BuildCstrEqForTensorListAlongAxis( + pir::ShapeConstraintIRAnalysis *shape_analysis, + const std::vector &values, + int axis); + } // namespace paddle::dialect::details diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc index 092ecc89cb13f..0b1dff55f4c41 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc @@ -80,16 +80,6 @@ bool ShapeSrOpInferSymbolicShape( return ShapeOpInferSymbolicShape(op, shape_analysis); } -void BuildCstrEqForTensorListAlongAxis( - pir::ShapeConstraintIRAnalysis *shape_analysis, - const symbol::TensorListShapeOrDataDimExprs &shape_data_list, - int axis) { - for (size_t i = 1; i < shape_data_list.size(); ++i) { - shape_analysis->CreateDimExprBuilder().CstrEq( - shape_data_list[0].shape()[axis], shape_data_list[i].shape()[axis]); - } -} - bool StackOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { pir::Value operand_source = op->operand_source(0); @@ -119,7 +109,8 @@ bool StackOpInferSymbolicShape(pir::Operation *op, } else { for (int i = 0; i < rank; ++i) { if (i == axis) continue; - BuildCstrEqForTensorListAlongAxis(shape_analysis, shape_data_list, i); + details::BuildCstrEqForTensorListAlongAxis( + shape_analysis, shape_data_list, i); } shape_dim_exprs.insert(shape_dim_exprs.begin() + axis, static_cast(shape_data_list.size())); @@ -194,31 +185,42 @@ bool ReshapeOpInferSymbolicShape( const symbol::ShapeOrDataDimExprs &operand_shape_or_data = shape_analysis->GetShapeOrDataForValue(operand_source_shape); - const std::vector out_dims = [&] { - std::vector out_dims; - out_dims = operand_shape_or_data.data().value(); + const auto &GetProduct = [&](const auto &dim_exprs, const auto &Filter) { + symbol::DimExpr product{1}; + for (const auto &dim_expr : dim_exprs) { + if (Filter(dim_expr)) { + product = product * dim_expr; + } + } + return product; + }; - symbol::DimExpr product = symbol::DimExpr(1); - symbol::DimExpr numel = symbol::DimExpr(1); + const auto &IsNotMinusOne = [&](const symbol::DimExpr &dim_expr) { + if (dim_expr.isa()) { + return dim_expr.dyn_cast() != static_cast(-1); + } + return true; + }; + const std::vector out_dims = [&] { const auto &original_shape = shape_analysis->GetShapeOrDataForValue(op->operand_source(0)).shape(); - for (auto &dim_expr : original_shape) { - numel = numel * dim_expr; - } - for (size_t i = 0; i < out_dims.size(); i++) { - if (out_dims[i].isa()) { - if (out_dims[i].dyn_cast() != static_cast(-1)) { - product = product * out_dims[i]; - } else if (i == out_dims.size() - 1) { - out_dims[i] = numel / product; - } else { - // doing nothing - } - } else { - product = product * out_dims[i]; - } + const auto &numel = + GetProduct(original_shape, [](const auto &) { return true; }); + + const auto &product_exclude_minus_one = + GetProduct(operand_shape_or_data.data().value(), IsNotMinusOne); + + const auto &input_dims = operand_shape_or_data.data().value(); + + std::vector out_dims; + out_dims.reserve(input_dims.size()); + for (const auto &dim_expr : input_dims) { + const auto &out_dim_expr = IsNotMinusOne(dim_expr) + ? dim_expr + : (numel / product_exclude_minus_one); + out_dims.emplace_back(out_dim_expr); } return out_dims; @@ -352,15 +354,20 @@ bool SliceOpInferSymbolicShape(pir::Operation *op, int64_t axis = axes[i]; auto end = IsMaxInt(dim_expr_ends[i]) ? out_shape[axis] : dim_expr_ends[i]; - if ((starts[i] >= 0 && ends[i] >= 0) || - (starts[i] <= 0 && ends[i] <= 0)) { // both negtive or positive. + + bool both_negative_or_positive = + (starts[i] >= 0 && ends[i] >= 0) || (starts[i] <= 0 && ends[i] <= 0); + bool start_negative_end_positive = starts[i] <= 0 && ends[i] >= 0; + bool start_positive_end_negative = starts[i] >= 0 && ends[i] <= 0; + + if (both_negative_or_positive) { out_shape[axis] = end - dim_expr_starts[i]; - } else if (starts[i] <= 0 && - ends[i] >= 0) { // negtive start, positive end + } else if (start_negative_end_positive) { out_shape[axis] = end - dim_expr_starts[i] - out_shape[axis]; - } else if (starts[i] >= 0 && - ends[i] <= 0) { // positive start, negtive end + } else if (start_positive_end_negative) { out_shape[axis] = out_shape[axis] - dim_expr_starts[i] + end; + } else { + LOG(FATAL) << "Dead code"; } } @@ -429,14 +436,14 @@ bool ConcatOpInferSymbolicShape( const std::vector &out_dims = [&] { std::vector out_dims = shape_data_list[0].shape(); - for (size_t i = 1; i < shape_data_list.size(); ++i) { - for (size_t j = 0; j < rank; ++j) { - if (j != static_cast(axis)) { - // This func have bug - BuildCstrEqForTensorListAlongAxis(shape_analysis, shape_data_list, i); - continue; - } - out_dims[axis] = out_dims[axis] + shape_data_list[i].shape()[axis]; + for (size_t i = 0; i < rank; ++i) { + if (i != static_cast(axis)) { + details::BuildCstrEqForTensorListAlongAxis( + shape_analysis, shape_data_list, i); + continue; + } + for (size_t j = 1; j < shape_data_list.size(); ++j) { + out_dims[axis] = out_dims[axis] + shape_data_list[j].shape()[axis]; } } return out_dims; diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h index 706bc500048b5..b72111b1173d5 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h @@ -95,3 +95,7 @@ bool Tril_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); } // namespace paddle::dialect + +namespace cinn::dialect { +using paddle::dialect::ScaleOpInferSymbolicShape; +} diff --git a/paddle/fluid/pir/dialect/operator/ir/control_flow_op.cc b/paddle/fluid/pir/dialect/operator/ir/control_flow_op.cc index f1fcb7c7b75bb..7f3929d0b9967 100644 --- a/paddle/fluid/pir/dialect/operator/ir/control_flow_op.cc +++ b/paddle/fluid/pir/dialect/operator/ir/control_flow_op.cc @@ -24,6 +24,7 @@ paddle::dialect::IfOp, paddle::dialect::WhileOp, paddle::dialect::HasElementsOp, #include "paddle/fluid/pir/dialect/operator/ir/op_type.h" #include "paddle/fluid/pir/dialect/operator/ir/pd_op.h" #include "paddle/fluid/pir/dialect/operator/utils/utils.h" +#include "paddle/fluid/pir/transforms/shape_optimization_pass.h" #include "paddle/phi/core/enforce.h" #include "paddle/pir/include/core/builder.h" #include "paddle/pir/include/core/builtin_attribute.h" @@ -167,6 +168,7 @@ void IfOp::Print(pir::IrPrinter &printer) { printer.PrintOpResult(op); os << " = pd_op.if"; printer.PrintOpOperands(op); + printer.PrintAttributeMap(op); os << " -> "; printer.PrintOpReturnType(op); os << "{\n"; @@ -306,6 +308,75 @@ std::vector> IfOp::Vjp( return res; } +bool IfOp::InferSymbolicShape(pir::ShapeConstraintIRAnalysis *shape_analysis) { + // infer true block + pir::InferSymExprForBlock(true_block(), shape_analysis); + + // infer false block + pir::InferSymExprForBlock(false_block(), shape_analysis); + + auto GetSymExprForBlockResult = + [shape_analysis](const pir::Operation &op, + uint32_t idx) -> const std::vector & { + const auto &shape_or_data = + shape_analysis->GetShapeOrDataForValue(op.operand_source(idx)); + if (shape_or_data.data().has_value()) { + return shape_or_data.data().value(); + } else { + return shape_or_data.shape(); + } + }; + + // TODO(lanxianghit): for llama, `if` op's result num always > 0, but + // result_num == 0 should be supported in future + if (num_results() > 0) { + for (uint32_t rst_idx = 0; rst_idx < num_results(); rst_idx++) { + const auto &true_dims = + GetSymExprForBlockResult(true_block().back(), rst_idx); + const auto &false_dims = + GetSymExprForBlockResult(false_block().back(), rst_idx); + + // merge shape for true and false block, new symbol will be assigned when + // the dims is not equal in true and false block, even if the dims are all + // constant, since we don't know which will be returned in compile time + // examples: + // true_block false_block return + // [1, 128] [1, 256] [1, S0] + // [1, S0] [1, S1] [1, S2] + // [1, S0] [S1, S2] [S1, S3] + // [1, S0] [1, S0] [1, S0] + + std::vector out_dims = true_dims; + if (false_dims.size() != 0) { + // now only support results of true and false block have same rank. + PADDLE_ENFORCE_EQ(true_dims.size(), + false_dims.size(), + phi::errors::PreconditionNotMet( + "The true and false block should have same rank, " + "but got true_rank(%d) and false_rank(%d)", + true_dims.size(), + false_dims.size())); + for (size_t i = 0; i < true_dims.size(); i++) { + if (true_dims[i] != false_dims[i]) { + out_dims[i] = symbol::DimExpr{shape_analysis->GetNextSymName()}; + } + } + } + + shape_analysis->SetShapeOrDataForValue( + result(rst_idx), + symbol::ShapeOrDataDimExprs{ + symbol::TensorShapeOrDataDimExprs(out_dims)}); + } + + return true; + } else { + PADDLE_THROW( + phi::errors::Unimplemented("IfOp::InferSymbolicShape: now only " + "support num_results() == 1.")); + } +} + void PyLayerOp::Build(pir::Builder &builder, // NOLINT pir::OperationArgument &argument, // NOLINT pir::Value combined_inputs, @@ -649,6 +720,47 @@ std::vector> WhileOp::Vjp( } return res; } + +bool WhileOp::InferSymbolicShape( + pir::ShapeConstraintIRAnalysis *shape_analysis) { + VLOG(3) << "############ WhileOp::InferSymbolicShape start..."; + pir::Program *body_program = body().parent_program(); + VLOG(3) << "##### WhileOp::InferSymbolicShape: sub_program id = " + << body_program->module_op().operation()->id(); + + for (auto &value : block_args()) { + std::vector sym_dims; + const std::vector &dims = + common::vectorize(value.type().dyn_cast().dims()); + + for (auto dim : dims) { + symbol::DimExpr dim_expr; + if (dim == pir::ShapedTypeInterface::kDynamic) { + symbol::DimExpr symbolic_dim_expr(shape_analysis->GetNextSymName()); + dim_expr = symbolic_dim_expr; + } else { + symbol::DimExpr numeric_dim_expr(dim); + dim_expr = numeric_dim_expr; + } + sym_dims.push_back(dim_expr); + } + symbol::ShapeOrDataDimExprs shape_data{ + symbol::TensorShapeOrDataDimExprs(sym_dims)}; + shape_analysis->SetShapeOrDataForValue(value, shape_data); + } + + pir::InferSymExprForBlock(body(), shape_analysis); + + const auto &last_op = body().back(); + for (size_t i = 1; i < last_op.operands_source().size(); ++i) { + shape_analysis->SetShapeOrDataForValue( + result(i - 1), + shape_analysis->GetShapeOrDataForValue(last_op.operand_source(i))); + } + + return true; +} + std::vector> TuplePushOpVjpInterfaceModel::Vjp( pir::Operation *op, const std::vector> &inputs, diff --git a/paddle/fluid/pir/dialect/operator/ir/control_flow_op.h b/paddle/fluid/pir/dialect/operator/ir/control_flow_op.h index d59fd41f77b40..f8a6bbb9f3b0f 100644 --- a/paddle/fluid/pir/dialect/operator/ir/control_flow_op.h +++ b/paddle/fluid/pir/dialect/operator/ir/control_flow_op.h @@ -15,6 +15,7 @@ #pragma once #include +#include "paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_symbolic_shape.h" #include "paddle/fluid/pir/dialect/operator/interface/op_yaml_info.h" #include "paddle/fluid/pir/dialect/operator/interface/vjp.h" #include "paddle/pir/include/core/block.h" @@ -23,7 +24,7 @@ namespace paddle { namespace dialect { -class IfOp : public pir::Op { +class IfOp : public pir::Op { public: using Op::Op; static const char *name() { return "pd_op.if"; } @@ -55,6 +56,8 @@ class IfOp : public pir::Op { const std::vector> &outputs, const std::vector> &out_grads, const std::vector> &stop_gradients); + + bool InferSymbolicShape(pir::ShapeConstraintIRAnalysis *shape_analysis); }; class PyLayerOp : public pir::Op { @@ -94,7 +97,8 @@ class PyLayerOp : public pir::Op { /// cond, outputs = body(outputs) /// } /// -class WhileOp : public pir::Op { +class WhileOp + : public pir::Op { public: using Op::Op; static const char *name() { return "pd_op.while"; } @@ -118,6 +122,7 @@ class WhileOp : public pir::Op { const std::vector> &outputs, const std::vector> &out_grads, const std::vector> &stop_gradients); + bool InferSymbolicShape(pir::ShapeConstraintIRAnalysis *shape_analysis); }; struct TuplePushOpVjpInterfaceModel : public VjpInterface::Concept { diff --git a/paddle/fluid/pir/transforms/shape_optimization_pass.cc b/paddle/fluid/pir/transforms/shape_optimization_pass.cc index f2cadd7f1b74d..69377af0d30b5 100644 --- a/paddle/fluid/pir/transforms/shape_optimization_pass.cc +++ b/paddle/fluid/pir/transforms/shape_optimization_pass.cc @@ -85,29 +85,7 @@ void InferSymExprForAllValues(ModuleOp module_op) { shape_analysis.Init(); for (uint32_t i = 0; i < module_op->num_regions(); i++) { for (auto& block : module_op->region(i)) { - for (auto& op : block) { - auto infer_symbolic_shape_interface = - op.dyn_cast(); - if (infer_symbolic_shape_interface) { - VLOG(vlog_level) << op.name() << " has InferSymbolicShapeInterface."; - PADDLE_ENFORCE(infer_symbolic_shape_interface.InferSymbolicShape( - &shape_analysis), - "InferSymbolicShape for %s failed.", - op.name()); - if (op.num_results() > 0) { - // TODO(lanxianghit): deal with the ops which have more than 1 - // ACTUAL results - pir::shape::SetShapeAttrForOp( - &op, shape_analysis.GetShapeOrDataForValue(op.result(0))); - } - } else { - VLOG(vlog_level) << op.name() + - " DOES NOT have InferSymbolicShapeInterface!"; - PADDLE_THROW(phi::errors::Unimplemented( - op.name() + " DOES NOT have InferSymbolicShapeInterface!")); - } - DebugPrintOpInfo(&op, &shape_analysis); - } + InferSymExprForBlock(block, &shape_analysis); } } } @@ -143,6 +121,33 @@ class ShapeOptimizationPass : public pir::Pass { } // namespace +void InferSymExprForBlock(const Block& block, + ShapeConstraintIRAnalysis* shape_analysis) { + for (auto& op : block) { + auto infer_symbolic_shape_interface = + op.dyn_cast(); + if (infer_symbolic_shape_interface) { + VLOG(vlog_level) << op.name() << " has InferSymbolicShapeInterface."; + PADDLE_ENFORCE( + infer_symbolic_shape_interface.InferSymbolicShape(shape_analysis), + "InferSymbolicShape for %s failed.", + op.name()); + if (op.num_results() > 0) { + // TODO(lanxianghit): deal with the ops which have more than 1 + // ACTUAL results + pir::shape::SetShapeAttrForOp( + &op, shape_analysis->GetShapeOrDataForValue(op.result(0))); + } + } else { + VLOG(vlog_level) << op.name() + + " DOES NOT have InferSymbolicShapeInterface!"; + PADDLE_THROW(phi::errors::Unimplemented( + op.name() + " DOES NOT have InferSymbolicShapeInterface!")); + } + DebugPrintOpInfo(&op, shape_analysis); + } +} + std::unique_ptr CreateShapeOptimizationPass() { return std::make_unique(); } diff --git a/paddle/fluid/pir/transforms/shape_optimization_pass.h b/paddle/fluid/pir/transforms/shape_optimization_pass.h index 64658504bbe97..a23de56f35d6e 100644 --- a/paddle/fluid/pir/transforms/shape_optimization_pass.h +++ b/paddle/fluid/pir/transforms/shape_optimization_pass.h @@ -24,4 +24,7 @@ class Pass; IR_API std::unique_ptr CreateShapeOptimizationPass(); +void InferSymExprForBlock(const Block &block, + ShapeConstraintIRAnalysis *shape_analysis); + } // namespace pir diff --git a/paddle/pir/CMakeLists.txt b/paddle/pir/CMakeLists.txt index c50f7cb3d8eb1..0f9adcb74fc2e 100644 --- a/paddle/pir/CMakeLists.txt +++ b/paddle/pir/CMakeLists.txt @@ -1,7 +1,12 @@ add_definitions(-DIR_LIBRARY) set_property(GLOBAL PROPERTY IR_TARGETS "") -file(GLOB_RECURSE PIR_CPP_SOURCES "*.cc") +file( + GLOB_RECURSE + PIR_CPP_SOURCES + "*.cc" + ${PADDLE_SOURCE_DIR}/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_symbolic_shape.cc +) if(WIN32) if(WITH_SHARED_IR) diff --git a/paddle/pir/include/dialect/control_flow/ir/cf_op.h b/paddle/pir/include/dialect/control_flow/ir/cf_op.h index 0883b8e1727a8..ed3e51df121c4 100644 --- a/paddle/pir/include/dialect/control_flow/ir/cf_op.h +++ b/paddle/pir/include/dialect/control_flow/ir/cf_op.h @@ -14,13 +14,16 @@ #pragma once #include +#include "paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_symbolic_shape.h" #include "paddle/pir/include/core/builder.h" #include "paddle/pir/include/core/op_base.h" #include "paddle/pir/include/core/op_trait.h" #include "paddle/pir/include/dialect/control_flow/ir/cf_interface.h" namespace pir { -class IR_API YieldOp : public Op { +class IR_API YieldOp : public Op { public: using Op::Op; static const char *name() { return "cf.yield"; } @@ -31,6 +34,8 @@ class IR_API YieldOp : public Op { OperationArgument &argument, // NOLINT const std::vector &Value); void VerifySig() {} + + bool InferSymbolicShape(pir::ShapeConstraintIRAnalysis *shape_analysis); }; /// diff --git a/paddle/pir/src/dialect/control_flow/ir/cf_op.cc b/paddle/pir/src/dialect/control_flow/ir/cf_op.cc index bc98389c50a4e..c203fbafb5a02 100644 --- a/paddle/pir/src/dialect/control_flow/ir/cf_op.cc +++ b/paddle/pir/src/dialect/control_flow/ir/cf_op.cc @@ -25,6 +25,13 @@ void YieldOp::Build(Builder &builder, argument.AddInputs(inputs); } +bool YieldOp::InferSymbolicShape( + pir::ShapeConstraintIRAnalysis *shape_analysis) { + VLOG(3) << "############ YieldOp::InferSymbolicShape start..."; + // YieldOp has no output, just return true + return true; +} + void TuplePushOp::Build(Builder &builder, // NOLINT OperationArgument &argument, // NOLINT Value inlet, diff --git a/test/cpp/pir/shape_dialect/shape_optimization_test.cc b/test/cpp/pir/shape_dialect/shape_optimization_test.cc index 7e1e0ff1509dd..b48f84db4d1b8 100644 --- a/test/cpp/pir/shape_dialect/shape_optimization_test.cc +++ b/test/cpp/pir/shape_dialect/shape_optimization_test.cc @@ -116,17 +116,20 @@ TEST(shape_optimization, shape_optimization_pass) { symbol::ShapeOrDataDimExprs subtract_res = shape_analysis.GetShapeOrDataForValue(subtract_op.result(0)); - // TODO(zhangbopd): after shape infer is completed, we can check the results - // EXPECT_EQ(cast_res.shape()[0], 1); - // EXPECT_EQ(cast_res.shape()[1], 64); - // EXPECT_EQ(symbol::ToString(cast_res.shape()[2]) == "Mul(S0, 32)"); - // EXPECT_EQ(cast_res.shape()[3] == 2); - // EXPECT_EQ(symbol::ToString(relu_res.shape()[2]), "Add(S2, -4)"); - // EXPECT_EQ(symbol::ToString(relu_res.shape()[2]), "Add(S3, -4)"); - // EXPECT_EQ(symbol::ToString(relu_res.shape()[2]), "Add(S4, -4)"); - // EXPECT_EQ(symbol::ToString(relu_res.shape()[2]), "Add(S5, -4)"); + EXPECT_EQ(cast_res.shape()[0], 1); + EXPECT_EQ(cast_res.shape()[1], 64); + EXPECT_EQ(symbol::ToString(cast_res.shape()[2]), + "Mul(Mul(Mul(Mul(1, S1), 128), 32), 1 / (128))"); + EXPECT_EQ(cast_res.shape()[3], 2); + + EXPECT_EQ(symbol::ToString(relu_res.shape()[0]), "Add(Add(S2, -2), -2)"); + EXPECT_EQ(symbol::ToString(relu_res.shape()[1]), "Add(Add(S3, -2), -2)"); + EXPECT_EQ(symbol::ToString(relu_res.shape()[2]), "Add(Add(S4, -2), -2)"); + EXPECT_EQ(symbol::ToString(relu_res.shape()[3]), "Add(Add(S5, -2), -2)"); EXPECT_EQ(subtract_res.shape()[0], 1); EXPECT_EQ(subtract_res.shape()[1], 64); - EXPECT_EQ(symbol::ToString(subtract_res.shape()[2]), "Broadcast(S0, -1)"); + EXPECT_EQ(symbol::ToString(subtract_res.shape()[2]), + "Broadcast(S0, Mul(Mul(Mul(Mul(1, S1), 128), 32), 1 / (128)))"); + EXPECT_EQ(subtract_res.shape()[3], 2); } diff --git a/test/ir/pir/cinn/symbolic/test_if_else_dy.py b/test/ir/pir/cinn/symbolic/test_if_else_dy.py new file mode 100644 index 0000000000000..c8b2276027898 --- /dev/null +++ b/test/ir/pir/cinn/symbolic/test_if_else_dy.py @@ -0,0 +1,92 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import unittest +from os.path import dirname + +import numpy as np + +import paddle +from paddle import nn +from paddle.static import InputSpec + +sys.path.append(dirname(dirname(__file__))) + +import utils + + +class IfSubgraph(nn.Layer): + def __init__(self): + super().__init__() + + def exp_sub(self, x): + y = paddle.exp(x) + return y - x + + def forward(self, x, y): + if x.shape[-1] > 1: + x = self.exp_sub(x) + else: + y = paddle.abs(y) + x = paddle.nn.functional.relu(x) + y = paddle.logical_not(y) + return x, y + + +class TestIfSubgraph(unittest.TestCase): + def setUp(self): + paddle.seed(2024) + self.prepare_data() + + def prepare_data(self): + self.shape = [1, 2048] + self.x = paddle.randn(self.shape, dtype="float32") + self.x.stop_gradient = False + + self.y_shape = [2, 256] + self.y = paddle.randn(self.y_shape, dtype="float32") + self.y.stop_gradient = False + + def check_jit_kernel_info(self, static_fn): + utils.check_jit_kernel_number(static_fn, 1) + utils.check_jit_kernel_structure(static_fn, {utils.JIT_KERNEL_NAME: 1}) + + def eval(self, use_cinn): + net = IfSubgraph() + input_spec = [ + InputSpec(shape=[None, None], dtype="float32"), + InputSpec(shape=[None, None], dtype="float32"), + ] + net = utils.apply_to_static(net, use_cinn, input_spec) + net.eval() + out = net(self.x, self.y) + if use_cinn: + self.check_jit_kernel_info(net.forward) + return out + + def test_eval(self): + dy_out_x, dy_out_y = self.eval(use_cinn=False) + if utils.unittest_use_cinn(): + cinn_out_x, cinn_out_y = self.eval(use_cinn=True) + np.testing.assert_allclose( + cinn_out_x.numpy(), dy_out_x.numpy(), atol=1e-6, rtol=1e-6 + ) + np.testing.assert_allclose( + cinn_out_y.numpy(), dy_out_y.numpy(), atol=1e-6, rtol=1e-6 + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/ir/pir/cinn/symbolic/test_while_dy.py b/test/ir/pir/cinn/symbolic/test_while_dy.py index a8ba57ed39494..627d03ab838c5 100644 --- a/test/ir/pir/cinn/symbolic/test_while_dy.py +++ b/test/ir/pir/cinn/symbolic/test_while_dy.py @@ -32,11 +32,13 @@ def __init__(self): super().__init__() def forward(self, x): - loop_count = 0 - while loop_count < 1: - y = paddle.exp(x) - x = y - x + loop_count = paddle.full([1], 0) + while x.sum() > paddle.full([1], 0) and loop_count < paddle.full( + [1], 1 + ): + x = paddle.exp(x) - x loop_count += 1 + x = paddle.exp(x) return x From c81ab0593908d6ab385a5bc3753e8dc004ab028a Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 23 Feb 2024 17:27:12 +0800 Subject: [PATCH 038/282] Update group_sharded_optimizer_stage2.py (#62002) --- .../meta_parallel/sharding/group_sharded_optimizer_stage2.py | 4 ++-- .../fleet/meta_parallel/sharding/group_sharded_stage3.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_optimizer_stage2.py b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_optimizer_stage2.py index 1bb2a712071b5..2e1086c654cd0 100644 --- a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_optimizer_stage2.py +++ b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_optimizer_stage2.py @@ -75,7 +75,7 @@ def __init__( group=None, offload=False, device="gpu", - pertrain_sync_models=True, + pretrain_sync_models=True, dp_group=None, **kw ): @@ -178,7 +178,7 @@ def __init__( ), "Not support! when using offload with sharding stage2, please use pure sharding stage2, exclude data parallel." # Synchronous all ranks models - if pertrain_sync_models: + if pretrain_sync_models: self._sync_params_and_buffers() self.param_storages = {} # {dtype: {rank: InternalStorage}} diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage3.py b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage3.py index 67559a3c7e6ad..628aa9da082f8 100644 --- a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage3.py +++ b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage3.py @@ -106,7 +106,7 @@ def __init__( sync_buffers=False, device="gpu", segment_size=2**20, - pertrain_sync_models=True, + pretrain_sync_models=True, offload=False, sync_comm=False, dp_group=None, @@ -213,7 +213,7 @@ def __init__( item["grad_clip"] = self._optim._grad_clip # Synchronous all ranks models - if pertrain_sync_models: + if pretrain_sync_models: self._sync_params_and_buffers() self._segment_rank_params(self._layer) From 47b3ff375f9918f410bdd4a236cb1e1a28fb76c1 Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 23 Feb 2024 17:27:33 +0800 Subject: [PATCH 039/282] Update common.py (#62001) --- .../distributed/auto_parallel/static/operators/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/paddle/distributed/auto_parallel/static/operators/common.py b/python/paddle/distributed/auto_parallel/static/operators/common.py index e66a337e90ec9..9f95b049cce3c 100644 --- a/python/paddle/distributed/auto_parallel/static/operators/common.py +++ b/python/paddle/distributed/auto_parallel/static/operators/common.py @@ -75,8 +75,8 @@ class SyncMode: the synchronization mode for communication or auxiliary operator """ - AmpFlagSync = "auto_parallel/amp_flag_synchorization" - GlobalNormSync = "auto_parallel/global_norm_synchorization" + AmpFlagSync = "auto_parallel/amp_flag_synchronization" + GlobalNormSync = "auto_parallel/global_norm_synchronization" def is_elementwise_op(op_type): From 646edcca85a96277fd464715a6b9f294c657f827 Mon Sep 17 00:00:00 2001 From: LiYuRio <63526175+LiYuRio@users.noreply.github.com> Date: Fri, 23 Feb 2024 17:36:32 +0800 Subject: [PATCH 040/282] support sharding stage 1, 3 (#61926) --- python/paddle/distributed/__init__.py | 4 + .../paddle/distributed/auto_parallel/api.py | 164 ++++++++++++++++++ test/auto_parallel/CMakeLists.txt | 4 + .../hybrid_strategy/CMakeLists.txt | 8 + .../semi_auto_parallel_sharding_stage_1.py | 84 +++++++++ .../semi_auto_parallel_sharding_stage_3.py | 84 +++++++++ ..._auto_parallel_hybrid_sharding_strategy.py | 56 ++++++ .../hybrid_strategy/testslist.csv | 1 + .../semi_auto_parallel_sharding_stage_1.py | 75 ++++++++ .../semi_auto_parallel_sharding_stage_3.py | 75 ++++++++ ...st_semi_auto_parallel_sharding_strategy.py | 56 ++++++ 11 files changed, 611 insertions(+) create mode 100644 test/auto_parallel/hybrid_strategy/semi_auto_parallel_sharding_stage_1.py create mode 100644 test/auto_parallel/hybrid_strategy/semi_auto_parallel_sharding_stage_3.py create mode 100644 test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_hybrid_sharding_strategy.py create mode 100644 test/auto_parallel/semi_auto_parallel_sharding_stage_1.py create mode 100644 test/auto_parallel/semi_auto_parallel_sharding_stage_3.py create mode 100644 test/auto_parallel/test_semi_auto_parallel_sharding_strategy.py diff --git a/python/paddle/distributed/__init__.py b/python/paddle/distributed/__init__.py index b882b3dad144b..feae03521c84b 100644 --- a/python/paddle/distributed/__init__.py +++ b/python/paddle/distributed/__init__.py @@ -86,6 +86,8 @@ shard_layer, shard_optimizer, shard_scaler, + ShardingStage1, + ShardingStage3, to_static, Strategy, DistModel, @@ -171,6 +173,8 @@ "load_state_dict", "shard_optimizer", "shard_scaler", + "ShardingStage1", + "ShardingStage3", "to_static", "Strategy", "DistModel", diff --git a/python/paddle/distributed/auto_parallel/api.py b/python/paddle/distributed/auto_parallel/api.py index 7eadcd63f2054..73a69c91b74a4 100644 --- a/python/paddle/distributed/auto_parallel/api.py +++ b/python/paddle/distributed/auto_parallel/api.py @@ -551,6 +551,35 @@ def replicate_layer_params_and_buffers( ) +def get_placement_with_sharding(param): + shard_axis = -1 + for placement in param.placements: + if isinstance(placement, dist.Shard): + # the parameter can't be shard twice on different mesh now + # assert here in case + assert ( + shard_axis == -1 + ), "The parameter can't be shard twich even in different mesh now." + shard_axis = placement.get_dim() + + placement_with_sharding = None + for dim in range(param.ndim): + if dim != shard_axis: + placement_with_sharding = dist.Shard(dim) + + new_placements = param.placements + for mesh_axis, placement in enumerate(param.placements): + # we need to keep the placement replicate if the it is out of tensor's dim + if ( + isinstance(placement, dist.Replicate) + and placement_with_sharding is not None + ): + new_placements[mesh_axis] = placement_with_sharding + break + + return new_placements + + class _ShardOptimizer: def __init__(self, optimizer, shard_fn=None): assert ( @@ -576,6 +605,13 @@ def __init__(self, optimizer, shard_fn=None): self._inner_opt = optimizer self._shard_fn = shard_fn + # Invoke shard_fn if it is not None to shard parameters + if self._shard_fn is not None and isinstance( + self._shard_fn, ShardingStage3 + ): + for param in self._inner_opt._parameter_list: + self._shard_fn._shard_parameter(param) + def _shard_accumulator(self, param): # create the accumulators self._inner_opt._create_accumulators(self.target_block, [param]) @@ -733,6 +769,134 @@ def __getattr__(self, item): return getattr(self._inner_opt, item) +class ShardingStage1: + """ + A builtin shard_fn for shard_optimizer interface, users can pass it to shard_optimizer to implement sharding optimization with stage 1. + + Examples: + .. code-block:: python + + >>> import paddle + >>> import paddle.distributed as dist + + >>> mesh = dist.ProcessMesh([0, 1], dim_names=["x"]) + + >>> class MLP(paddle.nn.Layer): + ... def __init__(self): + ... super().__init__() + ... self.fc1 = paddle.nn.Linear(8, 8) + ... self.fc2 = paddle.nn.Linear(8, 8) + ... + ... def forward(self, input): + ... return self.fc2(self.fc1(input)) + + >>> # doctest: +REQUIRES(env:DISTRIBUTED) + >>> layer = MLP() + >>> batch = paddle.rand(shape=[8, 8]) + >>> opt = paddle.optimizer.AdamW(parameters=layer.parameters()) + >>> opt = dist.shard_optimizer(opt, dist.ShardingStage1()) + >>> for _ in range(5): + >>> loss = layer(batch) + >>> loss.backward() + >>> opt.step() + >>> opt.clear_grad() + >>> # This case need to be executed in multi-card environment + >>> # python -m paddle.distributed.launch --gpus=0,1 {test_case}.py + """ + + def __call__(self, key, param, accumulator): + if param.is_dist(): + # Only deal with momentum in optimizer, beta should be replicated cross param's mesh + if 'beta' not in key: + placements = get_placement_with_sharding(param) + else: + placements = [ + dist.Replicate() + for _ in range(len(param.process_mesh.shape)) + ] + return shard_tensor( + accumulator, + mesh=param.process_mesh, + placements=placements, + ) + return accumulator + + +class ShardingStage3: + """ + A builtin shard_fn for shard_optimizer interface, users can pass it to shard_optimizer to implement sharding optimization with stage 3. + + Args: + mesh(paddle.distributed.ProcessMesh): The `ProcessMesh` object describes the Cartesian topology of the used processes. + + Examples: + .. code-block:: python + + >>> import paddle + >>> import paddle.distributed as dist + + >>> mesh = dist.ProcessMesh([0, 1], dim_names=["x"]) + + >>> class MLP(paddle.nn.Layer): + ... def __init__(self): + ... super().__init__() + ... self.fc1 = paddle.nn.Linear(8, 8) + ... self.fc2 = paddle.nn.Linear(8, 8) + ... + ... def forward(self, input): + ... return self.fc2(self.fc1(input)) + + >>> # doctest: +REQUIRES(env:DISTRIBUTED) + >>> layer = MLP() + >>> batch = paddle.rand(shape=[8, 8]) + >>> opt = paddle.optimizer.AdamW(parameters=layer.parameters()) + >>> opt = dist.shard_optimizer(opt, dist.ShardingStage3(mesh)) + >>> for _ in range(5): + >>> loss = layer(batch) + >>> loss.backward() + >>> opt.step() + >>> opt.clear_grad() + >>> # This case need to be executed in multi-card environment + >>> # python -m paddle.distributed.launch --gpus=0,1 {test_case}.py + """ + + def __init__(self, mesh): + self._mesh = mesh + + def _shard_parameter(self, param): + # TODO(liyurui): remove this trick dense to dist convert after adding + # dense_tensor.to_dist method. + if param.is_dense(): + zero_dense = paddle.zeros(param.shape) + placements = [] + for _ in range(len(self._mesh.shape)): + placements.append(dist.Replicate()) + zero_dist = dist.shard_tensor(zero_dense, self._mesh, placements) + res = param + zero_dist + + new_placements = get_placement_with_sharding(param) + shard_param = dist.reshard(param, param.process_mesh, new_placements) + # change the holder of param to new shard_param + param.get_tensor()._share_data_with(shard_param.get_tensor()) + + def __call__(self, key, param, accumulator): + if param.is_dist(): + # Only deal with momentum in optimizer, beta should be replicated cross param's mesh + if 'beta' not in key: + placements = param.placements + else: + placements = [ + dist.Replicate() + for _ in range(len(param.process_mesh.shape)) + ] + return shard_tensor( + accumulator, + mesh=param.process_mesh, + placements=placements, + ) + return accumulator + + def shard_optimizer(optimizer, shard_fn=None): """ diff --git a/test/auto_parallel/CMakeLists.txt b/test/auto_parallel/CMakeLists.txt index 1bc0398fa500f..a72e7831e1a13 100644 --- a/test/auto_parallel/CMakeLists.txt +++ b/test/auto_parallel/CMakeLists.txt @@ -166,6 +166,10 @@ if(WITH_DISTRIBUTE AND WITH_GPU) test_semi_auto_parallel_single_strategy) set_tests_properties(test_semi_auto_parallel_single_strategy PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 400) + py_test_modules(test_semi_auto_parallel_sharding_strategy MODULES + test_semi_auto_parallel_sharding_strategy) + set_tests_properties(test_semi_auto_parallel_sharding_strategy + PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 200) py_test_modules(test_semi_auto_parallel_lazy_init MODULES test_semi_auto_parallel_lazy_init) set_tests_properties(test_semi_auto_parallel_lazy_init diff --git a/test/auto_parallel/hybrid_strategy/CMakeLists.txt b/test/auto_parallel/hybrid_strategy/CMakeLists.txt index 3fbd643528480..2d205031a433e 100644 --- a/test/auto_parallel/hybrid_strategy/CMakeLists.txt +++ b/test/auto_parallel/hybrid_strategy/CMakeLists.txt @@ -50,3 +50,11 @@ if((WITH_GPU) AND (LINUX)) set_tests_properties(test_semi_auto_parallel_llama_model_amp PROPERTIES TIMEOUT "180" LABELS "RUN_TYPE=HYBRID") endif() +if((WITH_GPU) AND (LINUX)) + py_test_modules( + test_semi_auto_parallel_hybrid_sharding_strategy MODULES + test_semi_auto_parallel_hybrid_sharding_strategy ENVS + "http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python") + set_tests_properties(test_semi_auto_parallel_hybrid_sharding_strategy + PROPERTIES TIMEOUT "120" LABELS "RUN_TYPE=HYBRID") +endif() diff --git a/test/auto_parallel/hybrid_strategy/semi_auto_parallel_sharding_stage_1.py b/test/auto_parallel/hybrid_strategy/semi_auto_parallel_sharding_stage_1.py new file mode 100644 index 0000000000000..10b53fa0f443c --- /dev/null +++ b/test/auto_parallel/hybrid_strategy/semi_auto_parallel_sharding_stage_1.py @@ -0,0 +1,84 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import numpy as np + +import paddle +import paddle.distributed as dist + + +class TestSemiAutoParallelShardingStage1: + def __init__(self): + self._backend = os.getenv("backend") + self._seed = eval(os.getenv("seed")) + self._mesh = dist.ProcessMesh([[0, 1], [2, 3]], dim_names=["x", "y"]) + + def check_tensor_eq(self, a, b, rtol=1e-05, atol=0, verbose=True): + np.testing.assert_allclose(a, b, rtol=rtol, atol=atol, verbose=verbose) + + def shard_layer_fn(self, layer_name, layer, process_mesh): + layer.weight = dist.shard_tensor( + layer.weight, process_mesh, [dist.Shard(1)] + ) + layer.bias = dist.shard_tensor( + layer.bias, process_mesh, [dist.Shard(0)] + ) + + def get_single_card_rst(self): + paddle.seed(self._seed) + linear = paddle.nn.Linear(10, 10) + batch = paddle.rand(shape=[10, 10]) + opt = paddle.optimizer.AdamW(parameters=linear.parameters()) + for _ in range(5): + loss = linear(batch) + loss.backward() + opt.step() + opt.clear_grad() + self.weight = linear.weight.numpy() + self.bias = linear.bias.numpy() + + def test_sharding_stage_1_with_mp(self): + paddle.seed(self._seed) + linear = paddle.nn.Linear(10, 10) + linear = dist.shard_layer(linear, self._mesh, self.shard_layer_fn) + batch = paddle.rand(shape=[10, 10]) + # shard the input by sharding degree + batch = dist.shard_tensor(batch, self._mesh, [dist.Shard(0)]) + # shard optimizer with stage 1 fn + opt = paddle.optimizer.AdamW(parameters=linear.parameters()) + opt = dist.shard_optimizer(opt, dist.ShardingStage1()) + for _ in range(5): + loss = linear(batch) + loss.backward() + opt.step() + opt.clear_grad() + self.check_tensor_eq(self.weight, linear.weight.numpy()) + self.check_tensor_eq(self.bias, linear.bias.numpy()) + + def run_test_case(self): + if self._backend == "cpu": + paddle.set_device("cpu") + elif self._backend == "gpu": + paddle.set_device("gpu:" + str(dist.get_rank())) + else: + raise ValueError("Only support cpu or gpu backend.") + + self.get_single_card_rst() + self.test_sharding_stage_1_with_mp() + + +if __name__ == '__main__': + TestSemiAutoParallelShardingStage1().run_test_case() diff --git a/test/auto_parallel/hybrid_strategy/semi_auto_parallel_sharding_stage_3.py b/test/auto_parallel/hybrid_strategy/semi_auto_parallel_sharding_stage_3.py new file mode 100644 index 0000000000000..143e1963c5041 --- /dev/null +++ b/test/auto_parallel/hybrid_strategy/semi_auto_parallel_sharding_stage_3.py @@ -0,0 +1,84 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import numpy as np + +import paddle +import paddle.distributed as dist + + +class TestSemiAutoParallelShardingStage3: + def __init__(self): + self._backend = os.getenv("backend") + self._seed = eval(os.getenv("seed")) + self._mesh = dist.ProcessMesh([[0, 1], [2, 3]], dim_names=["x", "y"]) + + def check_tensor_eq(self, a, b, rtol=1e-05, atol=0, verbose=True): + np.testing.assert_allclose(a, b, rtol=rtol, atol=atol, verbose=verbose) + + def shard_layer_fn(self, layer_name, layer, process_mesh): + layer.weight = dist.shard_tensor( + layer.weight, process_mesh, [dist.Shard(1)] + ) + layer.bias = dist.shard_tensor( + layer.bias, process_mesh, [dist.Shard(0)] + ) + + def get_single_card_rst(self): + paddle.seed(self._seed) + linear = paddle.nn.Linear(10, 10) + batch = paddle.rand(shape=[10, 10]) + opt = paddle.optimizer.AdamW(parameters=linear.parameters()) + for _ in range(5): + loss = linear(batch) + loss.backward() + opt.step() + opt.clear_grad() + self.weight = linear.weight.numpy() + self.bias = linear.bias.numpy() + + def test_sharding_stage_3_with_mp(self): + paddle.seed(self._seed) + linear = paddle.nn.Linear(10, 10) + linear = dist.shard_layer(linear, self._mesh, self.shard_layer_fn) + batch = paddle.rand(shape=[10, 10]) + # shard the input by sharding degree + batch = dist.shard_tensor(batch, self._mesh, [dist.Shard(0)]) + # shard optimizer with stage 1 fn + opt = paddle.optimizer.AdamW(parameters=linear.parameters()) + opt = dist.shard_optimizer(opt, dist.ShardingStage3(self._mesh)) + for _ in range(5): + loss = linear(batch) + loss.backward() + opt.step() + opt.clear_grad() + self.check_tensor_eq(self.weight, linear.weight.numpy()) + self.check_tensor_eq(self.bias, linear.bias.numpy()) + + def run_test_case(self): + if self._backend == "cpu": + paddle.set_device("cpu") + elif self._backend == "gpu": + paddle.set_device("gpu:" + str(dist.get_rank())) + else: + raise ValueError("Only support cpu or gpu backend.") + + self.get_single_card_rst() + self.test_sharding_stage_3_with_mp() + + +if __name__ == '__main__': + TestSemiAutoParallelShardingStage3().run_test_case() diff --git a/test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_hybrid_sharding_strategy.py b/test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_hybrid_sharding_strategy.py new file mode 100644 index 0000000000000..e358c18ba2a21 --- /dev/null +++ b/test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_hybrid_sharding_strategy.py @@ -0,0 +1,56 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import collective.test_communication_api_base as test_base + + +class TestSemiAutoParallelInShardingStrategy( + test_base.CommunicationTestDistBase +): + def setUp(self): + super().setUp( + num_of_devices=4, + timeout=120, + ) + self._default_envs = { + "dtype": "float32", + "seed": "2023", + } + self._changeable_envs = {"backend": ["gpu"]} + + def test_sharding_stage_1_strategy(self): + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) + for envs in envs_list: + self.run_test_case( + "semi_auto_parallel_sharding_stage_1.py", + user_defined_envs=envs, + ) + + def test_sharding_stage_3_strategy(self): + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) + for envs in envs_list: + self.run_test_case( + "semi_auto_parallel_sharding_stage_3.py", + user_defined_envs=envs, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/auto_parallel/hybrid_strategy/testslist.csv b/test/auto_parallel/hybrid_strategy/testslist.csv index 29ae9dd9dce18..7b64e2d93ea6b 100644 --- a/test/auto_parallel/hybrid_strategy/testslist.csv +++ b/test/auto_parallel/hybrid_strategy/testslist.csv @@ -5,3 +5,4 @@ test_save_load_state_dict,LINUX,GPU,120,HYBRID,test_runner.py,,,http_proxy=;http test_semi_auto_parallel_c_cross_entropy,LINUX,GPU,120,HYBRID,test_runner.py,,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_cross_mesh_reshard,LINUX,GPU,120,HYBRID,test_runner.py,,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_semi_auto_parallel_llama_model_amp,LINUX,GPU,180,HYBRID,test_runner.py,,,http_proxy=;https_proxy=;PYTHONPATH=../.., +test_semi_auto_parallel_hybrid_sharding_strategy,LINUX,GPU,120,HYBRID,test_runner.py,,,http_proxy=;https_proxy=;PYTHONPATH=../.., diff --git a/test/auto_parallel/semi_auto_parallel_sharding_stage_1.py b/test/auto_parallel/semi_auto_parallel_sharding_stage_1.py new file mode 100644 index 0000000000000..ffe1d5725f1d1 --- /dev/null +++ b/test/auto_parallel/semi_auto_parallel_sharding_stage_1.py @@ -0,0 +1,75 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import numpy as np + +import paddle +import paddle.distributed as dist + + +class TestSemiAutoParallelShardingStage1: + def __init__(self): + self._backend = os.getenv("backend") + self._seed = eval(os.getenv("seed")) + self._mesh = dist.ProcessMesh([0, 1], dim_names=["x"]) + + def check_tensor_eq(self, a, b, rtol=1e-05, atol=0, verbose=True): + np.testing.assert_allclose(a, b, rtol=rtol, atol=atol, verbose=verbose) + + def get_single_card_rst(self): + paddle.seed(self._seed) + linear = paddle.nn.Linear(10, 10) + batch = paddle.rand(shape=[10, 10]) + opt = paddle.optimizer.AdamW(parameters=linear.parameters()) + for _ in range(5): + loss = linear(batch) + loss.backward() + opt.step() + opt.clear_grad() + self.weight = linear.weight.numpy() + self.bias = linear.bias.numpy() + + def test_pure_sharding_stage_1(self): + paddle.seed(self._seed) + linear = paddle.nn.Linear(10, 10) + batch = paddle.rand(shape=[10, 10]) + # shard the input by sharding degree + batch = dist.shard_tensor(batch, self._mesh, [dist.Shard(0)]) + # shard optimizer with stage 1 fn + opt = paddle.optimizer.AdamW(parameters=linear.parameters()) + opt = dist.shard_optimizer(opt, dist.ShardingStage1()) + for _ in range(5): + loss = linear(batch) + loss.backward() + opt.step() + opt.clear_grad() + self.check_tensor_eq(self.weight, linear.weight.numpy()) + self.check_tensor_eq(self.bias, linear.bias.numpy()) + + def run_test_case(self): + if self._backend == "cpu": + paddle.set_device("cpu") + elif self._backend == "gpu": + paddle.set_device("gpu:" + str(dist.get_rank())) + else: + raise ValueError("Only support cpu or gpu backend.") + + self.get_single_card_rst() + self.test_pure_sharding_stage_1() + + +if __name__ == '__main__': + TestSemiAutoParallelShardingStage1().run_test_case() diff --git a/test/auto_parallel/semi_auto_parallel_sharding_stage_3.py b/test/auto_parallel/semi_auto_parallel_sharding_stage_3.py new file mode 100644 index 0000000000000..f391ca9ef54f2 --- /dev/null +++ b/test/auto_parallel/semi_auto_parallel_sharding_stage_3.py @@ -0,0 +1,75 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import numpy as np + +import paddle +import paddle.distributed as dist + + +class TestSemiAutoParallelShardingStage3: + def __init__(self): + self._backend = os.getenv("backend") + self._seed = eval(os.getenv("seed")) + self._mesh = dist.ProcessMesh([0, 1], dim_names=["x"]) + + def check_tensor_eq(self, a, b, rtol=1e-05, atol=0, verbose=True): + np.testing.assert_allclose(a, b, rtol=rtol, atol=atol, verbose=verbose) + + def get_single_card_rst(self): + paddle.seed(self._seed) + linear = paddle.nn.Linear(10, 10) + batch = paddle.rand(shape=[10, 10]) + opt = paddle.optimizer.AdamW(parameters=linear.parameters()) + for _ in range(5): + loss = linear(batch) + loss.backward() + opt.step() + opt.clear_grad() + self.weight = linear.weight.numpy() + self.bias = linear.bias.numpy() + + def test_pure_sharding_stage_3(self): + paddle.seed(self._seed) + linear = paddle.nn.Linear(10, 10) + batch = paddle.rand(shape=[10, 10]) + # shard the input by sharding degree + batch = dist.shard_tensor(batch, self._mesh, [dist.Shard(0)]) + # shard optimizer with stage 1 fn + opt = paddle.optimizer.AdamW(parameters=linear.parameters()) + opt = dist.shard_optimizer(opt, dist.ShardingStage3(self._mesh)) + for _ in range(5): + loss = linear(batch) + loss.backward() + opt.step() + opt.clear_grad() + self.check_tensor_eq(self.weight, linear.weight.numpy()) + self.check_tensor_eq(self.bias, linear.bias.numpy()) + + def run_test_case(self): + if self._backend == "cpu": + paddle.set_device("cpu") + elif self._backend == "gpu": + paddle.set_device("gpu:" + str(dist.get_rank())) + else: + raise ValueError("Only support cpu or gpu backend.") + + self.get_single_card_rst() + self.test_pure_sharding_stage_3() + + +if __name__ == '__main__': + TestSemiAutoParallelShardingStage3().run_test_case() diff --git a/test/auto_parallel/test_semi_auto_parallel_sharding_strategy.py b/test/auto_parallel/test_semi_auto_parallel_sharding_strategy.py new file mode 100644 index 0000000000000..489cba334c1b0 --- /dev/null +++ b/test/auto_parallel/test_semi_auto_parallel_sharding_strategy.py @@ -0,0 +1,56 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import collective.test_communication_api_base as test_base + + +class TestSemiAutoParallelInShardingStrategy( + test_base.CommunicationTestDistBase +): + def setUp(self): + super().setUp( + num_of_devices=2, + timeout=120, + ) + self._default_envs = { + "dtype": "float32", + "seed": "2023", + } + self._changeable_envs = {"backend": ["gpu"]} + + def test_sharding_stage_1_strategy(self): + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) + for envs in envs_list: + self.run_test_case( + "semi_auto_parallel_sharding_stage_1.py", + user_defined_envs=envs, + ) + + def test_sharding_stage_3_strategy(self): + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) + for envs in envs_list: + self.run_test_case( + "semi_auto_parallel_sharding_stage_3.py", + user_defined_envs=envs, + ) + + +if __name__ == "__main__": + unittest.main() From 23fdbd170a0fb6944c6a0f404fb8610b20c563ba Mon Sep 17 00:00:00 2001 From: houj04 <35131887+houj04@users.noreply.github.com> Date: Fri, 23 Feb 2024 17:39:59 +0800 Subject: [PATCH 041/282] [XPU] AdamW support multi_precision (#61694) * [XPU] AdamW support multi_precision * [XPU] use xdnn api adamw_v2 * update for KL2 --- cmake/external/xpu.cmake | 2 +- paddle/phi/backends/xpu/xpu2_op_list.cc | 8 +- paddle/phi/backends/xpu/xpu3_op_list.cc | 3 +- paddle/phi/kernels/xpu/adamw_kernel.cc | 383 ++++++++++++++++-- .../kernels/xpu/reduce_mean_grad_kernel.cc | 8 +- python/paddle/optimizer/adamw.py | 4 +- test/xpu/test_adamw_op_xpu.py | 198 ++++++++- test/xpu/test_flash_attention_op_xpu.py | 2 +- 8 files changed, 566 insertions(+), 42 deletions(-) diff --git a/cmake/external/xpu.cmake b/cmake/external/xpu.cmake index 73210ac9fbc56..bd2471e0f7e1d 100644 --- a/cmake/external/xpu.cmake +++ b/cmake/external/xpu.cmake @@ -29,7 +29,7 @@ if(NOT DEFINED XPU_BASE_DATE) set(XPU_BASE_DATE "20240104") endif() if(NOT DEFINED XPU_XHPC_BASE_DATE) - set(XPU_XHPC_BASE_DATE "20240218") + set(XPU_XHPC_BASE_DATE "20240222") endif() set(XPU_XCCL_BASE_VERSION "1.1.8.1") if(NOT DEFINED XPU_XFT_BASE_VERSION) diff --git a/paddle/phi/backends/xpu/xpu2_op_list.cc b/paddle/phi/backends/xpu/xpu2_op_list.cc index 491b47442725a..55aae9f24c1a6 100644 --- a/paddle/phi/backends/xpu/xpu2_op_list.cc +++ b/paddle/phi/backends/xpu/xpu2_op_list.cc @@ -36,7 +36,10 @@ XPUOpMap& get_kl2_ops() { XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, {"accuracy", XPUKernelSet({phi::DataType::FLOAT32})}, {"adadelta", XPUKernelSet({phi::DataType::FLOAT32})}, - {"adamw", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, + {"adamw", + XPUKernelSet({phi::DataType::FLOAT32, + phi::DataType::FLOAT16, + phi::DataType::BFLOAT16})}, {"adam", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, {"adam_dense_param_sparse_grad", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, @@ -723,7 +726,8 @@ XPUOpMap& get_kl2_ops() { phi::DataType::INT32, phi::DataType::INT64, phi::DataType::FLOAT16})}, - {"reduce_mean_grad", XPUKernelSet({phi::DataType::FLOAT32})}, + {"reduce_mean_grad", + XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, {"reduce_mean", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16, diff --git a/paddle/phi/backends/xpu/xpu3_op_list.cc b/paddle/phi/backends/xpu/xpu3_op_list.cc index 1d3d6001bca9c..39e79ba0c4934 100644 --- a/paddle/phi/backends/xpu/xpu3_op_list.cc +++ b/paddle/phi/backends/xpu/xpu3_op_list.cc @@ -721,7 +721,8 @@ XPUOpMap& get_kl3_ops() { phi::DataType::BFLOAT16, phi::DataType::INT32, phi::DataType::INT64})}, - {"reduce_mean_grad", XPUKernelSet({phi::DataType::FLOAT32})}, + {"reduce_mean_grad", + XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, {"reduce_mean", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16, diff --git a/paddle/phi/kernels/xpu/adamw_kernel.cc b/paddle/phi/kernels/xpu/adamw_kernel.cc index 4df7ab633ab4e..ca39a9932a609 100644 --- a/paddle/phi/kernels/xpu/adamw_kernel.cc +++ b/paddle/phi/kernels/xpu/adamw_kernel.cc @@ -24,6 +24,8 @@ #include "paddle/phi/core/kernel_registry.h" #include "paddle/phi/core/tensor_utils.h" +#include "paddle/phi/common/amp_type_traits.h" + namespace phi { template @@ -44,6 +46,234 @@ float GetAbsMax(const Context& dev_ctx, return *std::max_element(buffer_cpu.begin(), buffer_cpu.end()); } +template +void AdamwDenseKernelKL3(const Context& dev_ctx, + const DenseTensor& param, + const DenseTensor& grad, + const DenseTensor& learning_rate, + const DenseTensor& moment1, + const DenseTensor& moment2, + const DenseTensor& beta1_pow, + const DenseTensor& beta2_pow, + const paddle::optional& master_param, + const paddle::optional& skip_update, + const Scalar& beta1, + const Scalar& beta2, + const Scalar& epsilon, + float lr_ratio, + float coeff, + bool with_decay, + bool lazy_mode, + int64_t min_row_size_to_use_multithread, + bool multi_precision, + bool use_global_beta_pow, + DenseTensor* param_out, + DenseTensor* moment1_out, + DenseTensor* moment2_out, + DenseTensor* beta1_pow_out, + DenseTensor* beta2_pow_out, + DenseTensor* master_param_outs) { + // TODO(houj04): + // 当KL3稳定以后，并且不需要支持KL1和KL2的时候，拿这里的AdamwDenseKernelKL3替换掉AdamwDenseKernel + using MPDType = typename phi::dtype::MPTypeTrait::Type; + using XPUType = typename XPUTypeTrait::Type; + + const auto grad_type = grad.dtype(); + + VLOG(4) << "use_global_beta_pow:" << use_global_beta_pow; + + MPDType coeff_ = static_cast(coeff); + MPDType lr_ratio_ = static_cast(lr_ratio); + + bool skip_update_ = false; + if (skip_update.is_initialized()) { + PADDLE_ENFORCE_EQ( + skip_update->numel(), + 1, + errors::InvalidArgument("Input(SkipUpdate) size must be 1, but get %d", + skip_update->numel())); + std::vector skip_update_vec; + phi::TensorToVector(*skip_update, dev_ctx, &skip_update_vec); + skip_update_ = skip_update_vec[0]; + } + + // skip_update=true, just copy input to output + if (skip_update_) { + VLOG(4) << "Adamw skip update"; + phi::Copy(dev_ctx, param, dev_ctx.GetPlace(), false, param_out); + phi::Copy(dev_ctx, moment1, dev_ctx.GetPlace(), false, moment1_out); + phi::Copy(dev_ctx, moment2, dev_ctx.GetPlace(), false, moment2_out); + if (!use_global_beta_pow) { + phi::Copy(dev_ctx, beta1_pow, beta1_pow.place(), false, beta1_pow_out); + phi::Copy(dev_ctx, beta2_pow, beta2_pow.place(), false, beta2_pow_out); + } + return; + } + + // if with_decay = false, coeff = 0 + if (!with_decay) { + coeff_ = static_cast(0.0); + } + + MPDType beta1_ = beta1.to(); + MPDType beta2_ = beta2.to(); + MPDType epsilon_ = epsilon.to(); + VLOG(3) << "beta1_pow.numel() : " << beta1_pow.numel() + << "beta2_pow.numel() : " << beta2_pow.numel(); + VLOG(3) << "param.numel(): " << param.numel(); + PADDLE_ENFORCE_EQ( + beta1_pow_out->numel(), + 1, + errors::InvalidArgument("beta1 pow output size should be 1, but received " + "value is:%d.", + beta1_pow_out->numel())); + + PADDLE_ENFORCE_EQ( + beta2_pow_out->numel(), + 1, + errors::InvalidArgument("beta2 pow output size should be 1, but received " + "value is:%d.", + beta2_pow_out->numel())); + + const MPDType* master_in_data = + multi_precision ? master_param->data() : nullptr; + MPDType* master_out_data = + multi_precision ? dev_ctx.template Alloc(master_param_outs) + : nullptr; + // template DLL_EXPORT int + // adamw_v2(Context* ctx, MT beta1, MT beta2, MT epsilon, MT coeff, MT + // lr_ratio, const MT* beta1_pow, MT* beta1_pow_out, const MT* beta2_pow, MT* + // beta2_pow_out, const MT* moment1, MT* moment1_out, const MT* moment2, MT* + // moment2_out, const MT* lr, const TG* grad, const T* param, T* param_out, + // const MT* master_param, MT* master_param_out, int64_t n); + + if (beta1_pow.place() == CPUPlace() && beta2_pow.place() == CPUPlace()) { + DenseTensor xpu_beta1_pow; + DenseTensor xpu_beta2_pow; + phi::Copy(dev_ctx, beta1_pow, dev_ctx.GetPlace(), false, &xpu_beta1_pow); + phi::Copy(dev_ctx, beta2_pow, dev_ctx.GetPlace(), false, &xpu_beta2_pow); + dev_ctx.Wait(); + const MPDType* beta1_pow_ptr = xpu_beta1_pow.data(); + const MPDType* beta2_pow_ptr = xpu_beta2_pow.data(); + + if (grad_type == phi::DataType::FLOAT32) { + int r = xpu::adamw_v2( + dev_ctx.x_context(), + beta1_, + beta2_, + epsilon_, + coeff_, + lr_ratio_, + beta1_pow_ptr, + nullptr, + beta2_pow_ptr, + nullptr, + moment1.data(), + dev_ctx.template Alloc(moment1_out), + moment2.data(), + dev_ctx.template Alloc(moment2_out), + learning_rate.data(), + grad.data(), + reinterpret_cast(param.data()), + reinterpret_cast(dev_ctx.template Alloc(param_out)), + master_in_data, + master_out_data, + param.numel()); + PADDLE_ENFORCE_XDNN_SUCCESS(r, "adamw"); + } else { + int r = xpu::adamw_v2( + dev_ctx.x_context(), + beta1_, + beta2_, + epsilon_, + coeff_, + lr_ratio_, + beta1_pow_ptr, + nullptr, + beta2_pow_ptr, + nullptr, + moment1.data(), + dev_ctx.template Alloc(moment1_out), + moment2.data(), + dev_ctx.template Alloc(moment2_out), + learning_rate.data(), + reinterpret_cast(grad.data()), + reinterpret_cast(param.data()), + reinterpret_cast(dev_ctx.template Alloc(param_out)), + master_in_data, + master_out_data, + param.numel()); + PADDLE_ENFORCE_XDNN_SUCCESS(r, "adamw"); + } + if (!use_global_beta_pow) { + // Cpu update + dev_ctx.template HostAlloc(beta1_pow_out)[0] = + beta1_ * beta1_pow.data()[0]; + dev_ctx.template HostAlloc(beta2_pow_out)[0] = + beta2_ * beta2_pow.data()[0]; + } + } else { + MPDType* beta1_pow_out_ptr = nullptr; + MPDType* beta2_pow_out_ptr = nullptr; + + if (!use_global_beta_pow) { + beta1_pow_out_ptr = dev_ctx.template Alloc(beta1_pow_out); + beta2_pow_out_ptr = dev_ctx.template Alloc(beta2_pow_out); + } + + if (grad_type == phi::DataType::FLOAT32) { + int r = xpu::adamw_v2( + dev_ctx.x_context(), + beta1_, + beta2_, + epsilon_, + coeff_, + lr_ratio_, + beta1_pow.data(), + beta1_pow_out_ptr, + beta2_pow.data(), + beta2_pow_out_ptr, + moment1.data(), + dev_ctx.template Alloc(moment1_out), + moment2.data(), + dev_ctx.template Alloc(moment2_out), + learning_rate.data(), + grad.data(), + reinterpret_cast(param.data()), + reinterpret_cast(dev_ctx.template Alloc(param_out)), + master_in_data, + master_out_data, + param.numel()); + PADDLE_ENFORCE_XDNN_SUCCESS(r, "adamw"); + } else { + int r = xpu::adamw_v2( + dev_ctx.x_context(), + beta1_, + beta2_, + epsilon_, + coeff_, + lr_ratio_, + beta1_pow.data(), + beta1_pow_out_ptr, + beta2_pow.data(), + beta2_pow_out_ptr, + moment1.data(), + dev_ctx.template Alloc(moment1_out), + moment2.data(), + dev_ctx.template Alloc(moment2_out), + learning_rate.data(), + reinterpret_cast(grad.data()), + reinterpret_cast(param.data()), + reinterpret_cast(dev_ctx.template Alloc(param_out)), + master_in_data, + master_out_data, + param.numel()); + PADDLE_ENFORCE_XDNN_SUCCESS(r, "adamw"); + } + } + return; +} + template void AdamwDenseKernel(const Context& dev_ctx, const DenseTensor& param, @@ -71,6 +301,38 @@ void AdamwDenseKernel(const Context& dev_ctx, DenseTensor* beta1_pow_out, DenseTensor* beta2_pow_out, DenseTensor* master_param_outs) { + auto dev_version = + phi::backends::xpu::get_xpu_version(dev_ctx.GetPlace().GetDeviceId()); + if (dev_version == phi::backends::xpu::XPUVersion::XPU3) { + AdamwDenseKernelKL3(dev_ctx, + param, + grad, + learning_rate, + moment1, + moment2, + beta1_pow, + beta2_pow, + master_param, + skip_update, + beta1, + beta2, + epsilon, + lr_ratio, + coeff, + with_decay, + lazy_mode, + min_row_size_to_use_multithread, + multi_precision, + use_global_beta_pow, + param_out, + moment1_out, + moment2_out, + beta1_pow_out, + beta2_pow_out, + master_param_outs); + return; + } + // check moment_dtype auto moment1_dtype = moment1.dtype(); auto moment2_dtype = moment2.dtype(); @@ -228,30 +490,85 @@ void AdamwDenseKernel(const Context& dev_ctx, 0.0f); PADDLE_ENFORCE_XDNN_SUCCESS(r, "scale"); - // int adamw(Context* ctx, const T* g, const float* mom1, const float* mom2, - // const T* param, const float* beta1_pow, const float* beta2_pow, const - // float* lr, float* moment1_out, float* moment2_out, T* param_out, float - // beta1, float beta2, float epsilon, float coeff, int64_t n); - r = xpu::adamw( - dev_ctx.x_context(), - reinterpret_cast(grad.template data()), - moment_in_fp16 ? moment1_input_for_xdnn : moment1.template data(), - moment_in_fp16 ? moment2_input_for_xdnn : moment2.template data(), - reinterpret_cast(param.template data()), - beta1_pow_ptr, - beta2_pow_ptr, - new_lr, - moment_in_fp16 ? moment1_output_for_xdnn - : dev_ctx.template Alloc(moment1_out), - moment_in_fp16 ? moment2_output_for_xdnn - : dev_ctx.template Alloc(moment2_out), - reinterpret_cast(dev_ctx.template Alloc(param_out)), - beta1_, - beta2_, - epsilon_, - coeff, - param.numel()); - PADDLE_ENFORCE_XDNN_SUCCESS(r, "adamw"); + if (multi_precision) { + const float* master_param_in_data = master_param->data(); + float* master_param_out_data = + dev_ctx.template Alloc(master_param_outs); + // convert grad to float if necessary + float* grad_fp32 = nullptr; + const auto grad_type = grad.dtype(); + if (grad_type != phi::DataType::FLOAT32) { + grad_fp32 = RAII_GUARD.alloc_l3_or_gm(grad.numel()); + PADDLE_ENFORCE_XDNN_NOT_NULL(grad_fp32); + // int cast(Context* ctx, const TX* x, TY* y, int64_t len); + int r = xpu::cast( + dev_ctx.x_context(), + reinterpret_cast(grad.template data()), + grad_fp32, + grad.numel()); + PADDLE_ENFORCE_XDNN_SUCCESS(r, "cast"); + } + // int adamw(Context* ctx, const T* g, const float* mom1, const float* mom2, + // const T* param, const float* beta1_pow, const float* beta2_pow, const + // float* lr, float* moment1_out, float* moment2_out, T* param_out, float + // beta1, float beta2, float epsilon, float coeff, int64_t n); + r = xpu::adamw( + dev_ctx.x_context(), + (grad_type == phi::DataType::FLOAT32) ? grad.data() : grad_fp32, + moment_in_fp16 ? moment1_input_for_xdnn + : moment1.template data(), + moment_in_fp16 ? moment2_input_for_xdnn + : moment2.template data(), + master_param_in_data, + beta1_pow_ptr, + beta2_pow_ptr, + new_lr, + moment_in_fp16 ? moment1_output_for_xdnn + : dev_ctx.template Alloc(moment1_out), + moment_in_fp16 ? moment2_output_for_xdnn + : dev_ctx.template Alloc(moment2_out), + master_param_out_data, + beta1_, + beta2_, + epsilon_, + coeff, + param.numel()); + PADDLE_ENFORCE_XDNN_SUCCESS(r, "adamw"); + // convert master_param_out(fp32) to param_out(T) + r = xpu::cast( + dev_ctx.x_context(), + master_param_out_data, + reinterpret_cast(dev_ctx.template Alloc(param_out)), + param_out->numel()); + PADDLE_ENFORCE_XDNN_SUCCESS(r, "cast"); + } else { + // int adamw(Context* ctx, const T* g, const float* mom1, const float* mom2, + // const T* param, const float* beta1_pow, const float* beta2_pow, const + // float* lr, float* moment1_out, float* moment2_out, T* param_out, float + // beta1, float beta2, float epsilon, float coeff, int64_t n); + r = xpu::adamw( + dev_ctx.x_context(), + reinterpret_cast(grad.template data()), + moment_in_fp16 ? moment1_input_for_xdnn + : moment1.template data(), + moment_in_fp16 ? moment2_input_for_xdnn + : moment2.template data(), + reinterpret_cast(param.template data()), + beta1_pow_ptr, + beta2_pow_ptr, + new_lr, + moment_in_fp16 ? moment1_output_for_xdnn + : dev_ctx.template Alloc(moment1_out), + moment_in_fp16 ? moment2_output_for_xdnn + : dev_ctx.template Alloc(moment2_out), + reinterpret_cast(dev_ctx.template Alloc(param_out)), + beta1_, + beta2_, + epsilon_, + coeff, + param.numel()); + PADDLE_ENFORCE_XDNN_SUCCESS(r, "adamw"); + } if (moment_in_fp16) { int r = 0; @@ -369,11 +686,15 @@ PD_REGISTER_KERNEL(adamw, kernel->InputAt(5).SetBackend(phi::Backend::ALL_BACKEND); kernel->InputAt(6).SetBackend(phi::Backend::ALL_BACKEND); kernel->InputAt(8).SetBackend(phi::Backend::ALL_BACKEND); - // Skip beta1_pow, beta2_pow, skip_update data transform - kernel->OutputAt(3) - .SetBackend(phi::Backend::UNDEFINED) - .SetDataType(phi::DataType::FLOAT32); - kernel->OutputAt(4) - .SetBackend(phi::Backend::UNDEFINED) - .SetDataType(phi::DataType::FLOAT32); + + if (kernel_key.dtype() == phi::DataType::FLOAT16 || + kernel_key.dtype() == phi::DataType::BFLOAT16) { + kernel->OutputAt(1).SetDataType(phi::DataType::FLOAT32); + kernel->OutputAt(2).SetDataType(phi::DataType::FLOAT32); + kernel->OutputAt(3).SetDataType(phi::DataType::FLOAT32); + kernel->OutputAt(4).SetDataType(phi::DataType::FLOAT32); + kernel->OutputAt(5).SetDataType(phi::DataType::FLOAT32); + } + kernel->OutputAt(3).SetBackend(phi::Backend::UNDEFINED); + kernel->OutputAt(4).SetBackend(phi::Backend::UNDEFINED); } diff --git a/paddle/phi/kernels/xpu/reduce_mean_grad_kernel.cc b/paddle/phi/kernels/xpu/reduce_mean_grad_kernel.cc index c5b0950552629..37ace904b2b80 100644 --- a/paddle/phi/kernels/xpu/reduce_mean_grad_kernel.cc +++ b/paddle/phi/kernels/xpu/reduce_mean_grad_kernel.cc @@ -84,5 +84,9 @@ void ReduceMeanGradKernel(const Context& dev_ctx, } // namespace phi -PD_REGISTER_KERNEL( - mean_grad, XPU, ALL_LAYOUT, phi::ReduceMeanGradKernel, float) {} +PD_REGISTER_KERNEL(mean_grad, + XPU, + ALL_LAYOUT, + phi::ReduceMeanGradKernel, + float, + phi::dtype::float16) {} diff --git a/python/paddle/optimizer/adamw.py b/python/paddle/optimizer/adamw.py index b14f8603be89e..f3a23ce846bf1 100644 --- a/python/paddle/optimizer/adamw.py +++ b/python/paddle/optimizer/adamw.py @@ -177,9 +177,9 @@ def __init__( assert epsilon is not None if not isinstance(beta1, Value) and not 0 <= beta1 < 1: raise ValueError("Invalid value of beta1, expect beta1 in [0,1).") - if not isinstance(beta1, Value) and not 0 <= beta2 < 1: + if not isinstance(beta2, Value) and not 0 <= beta2 < 1: raise ValueError("Invalid value of beta2, expect beta2 in [0,1).") - if not isinstance(beta1, Value) and not 0 <= epsilon: + if not isinstance(epsilon, Value) and not 0 <= epsilon: raise ValueError("Invalid value of epsilon, expect epsilon >= 0.") if not isinstance(weight_decay, float) and not isinstance( weight_decay, (framework.Variable, Value) diff --git a/test/xpu/test_adamw_op_xpu.py b/test/xpu/test_adamw_op_xpu.py index b9120779c40f6..f8e0b7cd545bf 100644 --- a/test/xpu/test_adamw_op_xpu.py +++ b/test/xpu/test_adamw_op_xpu.py @@ -59,8 +59,8 @@ def adamw_step(inputs, attributes): moment1_out = beta1 * moment1 + (1 - beta1) * grad moment2_out = beta2 * moment2 + (1 - beta2) * np.square(grad) - lr_t = lr * np.sqrt(1 - beta2_pow) / (1 - beta1_pow) - param_out = param - lr_t * (moment1_out / (np.sqrt(moment2_out) + epsilon)) + denom = (np.sqrt(moment2_out) / np.sqrt(1.0 - beta2_pow)) + epsilon + param_out = param + ((moment1_out / denom) * (-(lr / (1.0 - beta1_pow)))) return param_out, moment1_out, moment2_out @@ -650,6 +650,200 @@ def get_numpy_output(param, grad, moment1, moment2, lr_ratio, t): paddle.disable_static() +class TestAdamWOpMultiPrecisonWithMainGrad(unittest.TestCase): + def _test_adamw_op_dygraph_place_amp_with_maingrad( + self, place, shape, use_main_grad + ): + paddle.disable_static() + paddle.seed(10) + paddle.set_device(place) + + found_inf = None + + _weight_decay = 0.1 + with_decay = True + _lazy_mode = False + find_master = True + + _epsilon = 1e-8 + + _beta1 = 0.9 + _beta2 = 0.99 + lr_ratio_ = 1.0 + + lr_rate = 1e-8 + + param = paddle.randn(shape).astype(paddle.bfloat16) + master_weight = param.astype(paddle.float32) + grad = paddle.randn(shape).astype(paddle.bfloat16) + main_grad = grad.astype(paddle.float32) + moment1 = paddle.randn(shape).astype(paddle.float32) + moment2 = paddle.randn(shape).astype(paddle.float32).abs() + lr = paddle.zeros([1]).astype(paddle.float32) + lr[0] = lr_rate + beta1_pow_acc = paddle.ones([1]).astype(paddle.float32) + beta1_pow_acc[0] = _beta1**10 + beta2_pow_acc = paddle.ones([1]).astype(paddle.float32) + beta2_pow_acc[0] = _beta2**10 + + ref_param = param.astype(paddle.float32) + ref_beta1_pow_acc = beta1_pow_acc.astype(paddle.float32) + ref_beta2_pow_acc = beta2_pow_acc.astype(paddle.float32) + ref_moment_1 = moment1.astype(paddle.float32) + ref_moment_2 = moment2.astype(paddle.float32) + + # reference code + _, _, _, _, _, _ = paddle._C_ops.adamw_( + ref_param, + main_grad, + lr, + ref_moment_1, + ref_moment_2, + ref_beta1_pow_acc, + ref_beta2_pow_acc, + master_weight, + found_inf, + _beta1, + _beta2, + _epsilon, + lr_ratio_, + _weight_decay, + with_decay, + _lazy_mode, + 1000, + False, + False, + ) + + if use_main_grad: + _, _, _, _, _, _ = paddle._C_ops.adamw_( + param, + main_grad, + lr, + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + found_inf, + _beta1, + _beta2, + _epsilon, + lr_ratio_, + _weight_decay, + with_decay, + _lazy_mode, + 1000, + find_master, + False, + ) + np.testing.assert_allclose( + param.astype("float32").numpy(), ref_param.numpy(), rtol=1e-2 + ) + np.testing.assert_allclose( + master_weight.numpy(), ref_param.numpy(), rtol=1e-6 + ) + else: + _, _, _, _, _, _ = paddle._C_ops.adamw_( + param, + grad, + lr, + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + found_inf, + _beta1, + _beta2, + _epsilon, + lr_ratio_, + _weight_decay, + with_decay, + _lazy_mode, + 1000, + find_master, + False, + ) + np.testing.assert_allclose( + param.astype("float32").numpy(), ref_param.numpy(), rtol=1e-2 + ) + np.testing.assert_allclose( + master_weight.numpy(), ref_param.numpy(), rtol=1e-6 + ) + + def _get_places(self): + places = [] + if paddle.is_compiled_with_xpu(): + places.append('xpu') + return places + + def test_main(self): + for _ in range(1): + shape = paddle.randint(1, 1024, [2]) + for place in self._get_places(): + use_main_grad_list = [True, False] + for use_main_grad in use_main_grad_list: + self._test_adamw_op_dygraph_place_amp_with_maingrad( + place, shape, use_main_grad + ) + + +class TestAdamWOpMultiPrecison(unittest.TestCase): + def _test_adamw_op_dygraph_place_amp(self, place, use_amp=False): + paddle.disable_static() + paddle.seed(10) + paddle.set_device(place) + + input = paddle.randn((5, 5)) + + model = paddle.nn.Linear(5, 5) + + optimizer = paddle.optimizer.AdamW( + parameters=[ + { + 'params': model.parameters(), + 'weight_decay': 0.001, + 'beta1': 0.1, + 'beta2': 0.99, + } + ], + multi_precision=use_amp, + ) + + for idx in range(2): + if place == 'xpu' and use_amp: + model = paddle.amp.decorate(models=model, level='O2') + scaler = paddle.amp.GradScaler(init_loss_scaling=1024) + + if place == 'xpu' and use_amp: + with paddle.amp.auto_cast(level='O2'): + output = model(input) + loss = paddle.mean(output) + scaled = scaler.scale(loss) + scaled.backward() + scaler.step(optimizer) + optimizer.clear_grad() + else: + output = model(input) + loss = paddle.mean(output) + loss.backward() + optimizer.step() + optimizer.clear_grad() + + def _get_places(self): + places = ['cpu'] + if paddle.is_compiled_with_xpu(): + places.append('xpu') + return places + + def test_main(self): + for place in self._get_places(): + use_amp_list = [True, False] + for use_amp in use_amp_list: + self._test_adamw_op_dygraph_place_amp(place, use_amp) + + support_types = get_xpu_op_support_types('adamw') for stype in support_types: create_test_class(globals(), XPUTestAdamwOp1, stype) diff --git a/test/xpu/test_flash_attention_op_xpu.py b/test/xpu/test_flash_attention_op_xpu.py index 8aadadfc40ecc..372a2ee91f1dd 100644 --- a/test/xpu/test_flash_attention_op_xpu.py +++ b/test/xpu/test_flash_attention_op_xpu.py @@ -79,7 +79,7 @@ def setUp(self): def test_all(self): self.run_case(dtype="float32", tolerance=5e-4, tolerance_dv=5e-4) self.run_case(dtype="float16", tolerance=5e-4, tolerance_dv=1e-3) - self.run_case(dtype="bfloat16", tolerance=5e-3, tolerance_dv=1e-2) + self.run_case(dtype="bfloat16", tolerance=6e-3, tolerance_dv=1e-2) def run_case(self, dtype, tolerance, tolerance_dv): # TODO(houj04) remove debug codes after correctness check From bd3639b80740dacbe6c754952e1677c4714b1acd Mon Sep 17 00:00:00 2001 From: skywalker2012 <108259496+skywalker2012@users.noreply.github.com> Date: Fri, 23 Feb 2024 18:16:19 +0800 Subject: [PATCH 042/282] [XPU] add support for local path compilation (#61970) * [XPU] add support for local path compilation * [XPU] pre-commit change * [XPU] remove os dependency --- tools/xpu/check_xpu_dependence.sh | 13 +++ tools/xpu/pack_paddle_depence.sh | 135 ++++++++++++++++++++++-------- 2 files changed, 111 insertions(+), 37 deletions(-) diff --git a/tools/xpu/check_xpu_dependence.sh b/tools/xpu/check_xpu_dependence.sh index 3091205f12ea3..2dffe5fea6824 100644 --- a/tools/xpu/check_xpu_dependence.sh +++ b/tools/xpu/check_xpu_dependence.sh @@ -23,6 +23,7 @@ fi xpu_base_url=$1 xccl_base_url=$2 +BOS_PATTERN="https://baidu-kunlun-product.su.bcebos.com" echo "xpu_base_url: $xpu_base_url" echo "xccl_base_url: $xccl_base_url" @@ -73,6 +74,18 @@ function check_files() { rm -rf ./$local_dir } +# check xpu_base_url type +if [[ $xpu_base_url != *"$BOS_PATTERN"* ]]; then + echo "The xpu_base_url does not contain bos url, assume it is local path" + if [[ ! -d $xpu_base_url ]]; then + echo "The xpu_base_url does not exist, please check it" + exit 1 + fi + exit 0 +else + echo "The URL is a bos url, will follow default download & compile logic" +fi + # XRE xre_tar_file_names=("xre-kylin_aarch64" "xre-bdcentos_x86_64" "xre-ubuntu_x86_64" "xre-centos7_x86_64") xre_inner_file_names=("include/xpu/runtime.h" "so/libxpurt.so") diff --git a/tools/xpu/pack_paddle_depence.sh b/tools/xpu/pack_paddle_depence.sh index 2df7c98cbf48a..25fe0bd1c51b9 100644 --- a/tools/xpu/pack_paddle_depence.sh +++ b/tools/xpu/pack_paddle_depence.sh @@ -17,12 +17,6 @@ set -e set -x -if [[ $# -eq 8 ]]; then - echo "Compiling Paddle with XHPC" - XHPC_URL=$7 - XHPC_DIR_NAME=$8 -fi - XRE_URL=$1 XRE_DIR_NAME=$2 @@ -32,41 +26,108 @@ XDNN_DIR_NAME=$4 XCCL_URL=$5 XCCL_DIR_NAME=$6 -wget --no-check-certificate ${XRE_URL} -q -O xre.tar.gz -tar xvf xre.tar.gz - -wget --no-check-certificate ${XDNN_URL} -q -O xdnn.tar.gz -tar xvf xdnn.tar.gz +if [[ $# -eq 8 ]]; then + echo "Compiling Paddle with XHPC" + XHPC_URL=$7 + XHPC_DIR_NAME=$8 +elif [[ $# -eq 7 ]]; then + XHPC_DIR_NAME=$7 +fi -wget --no-check-certificate ${XCCL_URL} -q -O xccl.tar.gz -tar xvf xccl.tar.gz +BOS_PATTERN="https://baidu-kunlun-product.su.bcebos.com" +mkdir -p xpu/include/xhpc/xblas +mkdir -p xpu/include/xhpc/xfa mkdir -p xpu/include/xpu mkdir -p xpu/lib -if ! [ -z ${XHPC_URL} ]; then - echo "Compiling Paddle with XHPC" - echo "XHPC_URL: ${XHPC_URL}" - wget --no-check-certificate ${XHPC_URL} -q -O xhpc.tar.gz - tar xvf xhpc.tar.gz - - mkdir -p xpu/include/xhpc/xblas - mkdir -p xpu/include/xhpc/xfa - - cp -r ${XHPC_DIR_NAME}/xblas/include/* xpu/include/xhpc/xblas - cp -r ${XHPC_DIR_NAME}/xblas/so/* xpu/lib/ - - cp -r ${XHPC_DIR_NAME}/xdnn/include/* xpu/include/ - cp -r ${XHPC_DIR_NAME}/xdnn/so/* xpu/lib - - cp -r ${XHPC_DIR_NAME}/xfa/include/* xpu/include/xhpc/xfa - cp -r ${XHPC_DIR_NAME}/xfa/so/* xpu/lib/ +function download_from_bos() { + wget --no-check-certificate ${XRE_URL} -q -O xre.tar.gz + tar xvf xre.tar.gz + + wget --no-check-certificate ${XDNN_URL} -q -O xdnn.tar.gz + tar xvf xdnn.tar.gz + + wget --no-check-certificate ${XCCL_URL} -q -O xccl.tar.gz + tar xvf xccl.tar.gz +} + +function xhpc_prepare() { + if ! [ -z ${XHPC_URL} ]; then + echo "XHPC_URL: ${XHPC_URL}" + wget --no-check-certificate ${XHPC_URL} -q -O xhpc.tar.gz + tar xvf xhpc.tar.gz + + cp -r ${XHPC_DIR_NAME}/xblas/include/* xpu/include/xhpc/xblas + cp -r ${XHPC_DIR_NAME}/xblas/so/* xpu/lib/ + + cp -r ${XHPC_DIR_NAME}/xdnn/include/* xpu/include/ + cp -r ${XHPC_DIR_NAME}/xdnn/so/* xpu/lib + + cp -r ${XHPC_DIR_NAME}/xfa/include/* xpu/include/xhpc/xfa + cp -r ${XHPC_DIR_NAME}/xfa/so/* xpu/lib/ + else + cp -r ${XDNN_DIR_NAME}/include/xpu/* xpu/include/xpu/ + cp -r ${XDNN_DIR_NAME}/so/* xpu/lib/ + fi +} + +function local_prepare() { + # xre prepare + if [[ ! -d ${LOCAL_PATH}/${XRE_DIR_NAME} ]]; then + XRE_TAR_NAME=${XRE_DIR_NAME}.tar.gz + tar -zxf ${LOCAL_PATH}/${XRE_TAR_NAME} -C ${LOCAL_PATH} + fi + + # xccl prepare + if [[ ! -d ${LOCAL_PATH}/${XCCL_DIR_NAME} ]]; then + XCCL_TAR_NAME=${XCCL_DIR_NAME}.tar.gz + tar -zxf ${LOCAL_PATH}/${XCCL_TAR_NAME} -C ${LOCAL_PATH} + fi + + # xhpc prepare + if [[ ! -d ${LOCAL_PATH}/${XHPC_DIR_NAME} ]]; then + XHPC_TAR_NAME=${XHPC_DIR_NAME}.tar.gz + tar -zxf ${LOCAL_PATH}/${XHPC_TAR_NAME} -C ${LOCAL_PATH} + fi +} + +function local_assemble() { + # xre assemble + cp -r ${LOCAL_PATH}/$XRE_DIR_NAME/include/xpu/* xpu/include/xpu/ + cp -r ${LOCAL_PATH}/$XRE_DIR_NAME/so/libxpurt* xpu/lib/ + + # xccl assemble + cp -r ${LOCAL_PATH}/$XCCL_DIR_NAME/include/* xpu/include/xpu/ + cp -r ${LOCAL_PATH}/$XCCL_DIR_NAME/so/* xpu/lib/ + + # xhpc assemble + cp -r ${LOCAL_PATH}/${XHPC_DIR_NAME}/xblas/include/* xpu/include/xhpc/xblas + cp -r ${LOCAL_PATH}/${XHPC_DIR_NAME}/xblas/so/* xpu/lib/ + + cp -r ${LOCAL_PATH}/${XHPC_DIR_NAME}/xdnn/include/* xpu/include/ + cp -r ${LOCAL_PATH}/${XHPC_DIR_NAME}/xdnn/so/* xpu/lib + + cp -r ${LOCAL_PATH}/${XHPC_DIR_NAME}/xfa/include/* xpu/include/xhpc/xfa + cp -r ${LOCAL_PATH}/${XHPC_DIR_NAME}/xfa/so/* xpu/lib/ +} + +if [[ $XRE_URL != *"$BOS_PATTERN"* ]]; then + # below is local way + build_from="local" + LOCAL_PATH=$(dirname "$XRE_URL") + echo "LOCAL_PATH: ${LOCAL_PATH}" + + local_prepare + local_assemble else - cp -r $XDNN_DIR_NAME/include/xpu/* xpu/include/xpu/ - cp -r $XDNN_DIR_NAME/so/* xpu/lib/ + # below is default way + build_from="bos" + download_from_bos + xhpc_prepare + + cp -r $XRE_DIR_NAME/include/xpu/* xpu/include/xpu/ + cp -r $XRE_DIR_NAME/so/libxpurt* xpu/lib/ + cp -r $XCCL_DIR_NAME/include/* xpu/include/xpu/ + cp -r $XCCL_DIR_NAME/so/* xpu/lib/ fi - -cp -r $XRE_DIR_NAME/include/xpu/* xpu/include/xpu/ -cp -r $XRE_DIR_NAME/so/libxpurt* xpu/lib/ -cp -r $XCCL_DIR_NAME/include/* xpu/include/xpu/ -cp -r $XCCL_DIR_NAME/so/* xpu/lib/ From 8de4febee77c74f5be3a549780ba64738fd1f902 Mon Sep 17 00:00:00 2001 From: risemeup1 <62429225+risemeup1@users.noreply.github.com> Date: Fri, 23 Feb 2024 18:45:19 +0800 Subject: [PATCH 043/282] fix (#61934) --- python/setup_cinn.py.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/setup_cinn.py.in b/python/setup_cinn.py.in index f2fcd3029d231..cbdef191c4cd3 100644 --- a/python/setup_cinn.py.in +++ b/python/setup_cinn.py.in @@ -185,7 +185,7 @@ if platform.system() == 'Linux' and platform.machine() == 'x86_64': paddle_cuda_install_requirements = os.getenv( "PADDLE_CUDA_INSTALL_REQUIREMENTS", None ) - if paddle_cuda_install_requirements is not None: + if paddle_cuda_install_requirements == "ON": PADDLE_CUDA_INSTALL_REQUIREMENTS = { "V11": ( "nvidia-cuda-nvrtc-cu11==11.8.89; platform_system == 'Linux' and platform_machine == 'x86_64' | " From a68d9ccbd68492e0070c890ecb2a3eaee5ba36b9 Mon Sep 17 00:00:00 2001 From: WangZhen <23097963+0x45f@users.noreply.github.com> Date: Fri, 23 Feb 2024 19:51:13 +0800 Subject: [PATCH 044/282] [PIR]Fix infershape error when infershape function has MetaConfig (#62010) --- .../pir/dialect/op_generator/op_build_gen.py | 110 ++++++++++++++++-- paddle/phi/infermeta/unary.cc | 1 + 2 files changed, 101 insertions(+), 10 deletions(-) diff --git a/paddle/fluid/pir/dialect/op_generator/op_build_gen.py b/paddle/fluid/pir/dialect/op_generator/op_build_gen.py index 7b079605a2460..3365421990f1b 100644 --- a/paddle/fluid/pir/dialect/op_generator/op_build_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/op_build_gen.py @@ -14,20 +14,110 @@ # generator build function _INFERMETA_NEED_META_CONFIG = { - 'SplitInferMeta', - 'SumInferMeta', - 'SplitWithNumInferMeta', + # binary.h + 'AllValueCompareInferMeta', + 'KLDivInferMeta', + 'ArrayWriteInferMeta', + 'ArrayReadInferMeta', + 'BCELossInferMeta', + 'BinomialInferMeta', + 'ConvInferMeta', + 'Conv3DInferMeta', + 'ConvTransposeInferMeta', + 'Conv2dTransposeInferMeta', + 'CrossEntropyWithSoftmaxInferMeta', + 'CSoftmaxWithCrossEntropyInferMeta', + 'DepthwiseConvInferMeta', + 'DistributeFpnProposalsInferMeta', + 'ElementwiseRawInferMeta', + 'GridSampleBaseInferMeta', + 'HuberLossInferMeta', + 'IndexSampleInferMeta', + 'LogLossInferMeta', + 'MarginCrossEntropyInferMeta', + 'MatrixNMSInferMeta', + 'PReluInferMeta', + 'SegmentPoolInferMeta', + 'YoloBoxInferMeta', + 'ValueCompareInferMeta', + 'UnpoolInferMeta', + 'Unpool3dInferMeta', + # fusion.h + 'FusedAttentionInferMeta', + 'BNActXPUInferMeta', + 'FusedFCElementwiseLayerNormInferMeta', + 'FusedConv2dAddActInferMeta', + # multiary.h + 'AddNTensorArrayInferMeta', + 'AddNInferMeta', + 'AucInferMeta', + 'BatchNormInferMeta', + 'BatchNormInferInferMeta', + 'BilinearInferMeta', + 'CoalesceTensorInferMeta', + 'CheckMemoryContinueInferMeta', 'ConcatInferMeta', + 'DeformableConvInferMeta', + 'FusedBiasActInferMeta', + 'InterpolateInferMeta', + 'NceInferMeta', + 'SigmoidCrossEntropyWithLogitsInferMeta', + 'StackInferMeta', + 'FusedConvInferMeta', + # nullary.h + 'EyeInferMeta', + # ternary.h + 'AccuracyInferMeta', + 'BoxCoderInferMeta', + 'InstanceNormInferMeta', + 'LayerNormInferMeta', + 'MatchMatrixTensorInferMeta', + 'MultiClassNMSInferMeta', + 'NllLossRawInferMeta', + 'RoiAlignInferMeta', + 'SpectralNormInferMeta', + 'ViterbiDecodeInferMeta', + 'TdmSamplerInferMeta', + # unary.h + 'GetSplitAxisValue', + 'ArgMinMaxInferMeta', + 'ArrayToTensorInferMeta', + 'CropInferMeta', + 'EigvalsInferMeta', + 'FractionalMaxPoolInferMeta', + 'MaxPoolWithIndexInferMeta', + 'MaxPoolV2InferMeta', + 'MultinomialInferMeta', + 'OverlapAddInferMeta', + 'PadInferMeta', + 'Pad3dInferMeta', + 'PoolInferMeta', + 'Pool2DInferMeta', + 'ReduceIntArrayAxisInferMetaBase', 'ReduceIntArrayAxisInferMeta', + 'ReshapeInferMeta', 'ReshapeWithXShapeInferMeta', + 'ReverseInferMeta', + 'ReverseArrayInferMeta', + 'ShardIndexInferMeta', + 'SliceArrayInferMeta', + 'SliceArrayDenseInferMeta', 'SliceRawInferMeta', - 'StackInferMeta', - 'Conv2dTransposeInferMeta', - 'FusedConv2dAddActInferMeta', - 'InterpolateInferMeta', - 'DeformableConvInferMeta', - 'MatrixNMSInferMeta', - 'IndexSampleInferMeta', + 'SplitInferMeta', + 'SplitWithNumInferMeta', + 'SqueezeInferMeta', + 'SqueezeWithXShapeInferMeta', + 'StridedSliceRawInferMeta', + 'StridedSliceInferMeta', + 'SumInferMeta', + 'SumRawInferMeta', + 'TemporalShiftInferMeta', + 'TileInferMeta', + 'TopKInferMeta', + 'UnfoldInferMeta', + 'UnsqueezeInferMeta', + 'UnsqueezeWithXShapeInferMeta', + 'ArrayPopInferMeta', } _PREPARE_DATA_WITH_VECTOR_INT64_MTTABLE_ATTRIBUTE = {'FrobeniusNormOp'} diff --git a/paddle/phi/infermeta/unary.cc b/paddle/phi/infermeta/unary.cc index 3b47085eee9b1..42eaa2670a0b5 100644 --- a/paddle/phi/infermeta/unary.cc +++ b/paddle/phi/infermeta/unary.cc @@ -3595,6 +3595,7 @@ void ReshapeInferMeta(const MetaTensor& x, if (!config.is_runtime && shape.FromTensor()) { out->set_dims(common::make_ddim(shape_data)); out->share_lod(x); + out->set_dtype(x.dtype()); return; } InferMetaFromVecValue(x, shape_data, out); From c3074d6061a4f5017363ed121ec0436d53a5fe83 Mon Sep 17 00:00:00 2001 From: diadestiny <44188454+diadestiny@users.noreply.github.com> Date: Fri, 23 Feb 2024 21:22:52 +0800 Subject: [PATCH 045/282] [SOT][3.12] Support `END_FOR` opcode by skiping `END_FOR` in `FOR_ITER` in Python 3.12 (#62008) --- .../jit/sot/opcode_translator/executor/opcode_executor.py | 2 ++ test/sot/skip_files_py312 | 4 ---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py index ccfae0a888f02..e9a985e5b728c 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py +++ b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py @@ -2100,6 +2100,8 @@ def FOR_ITER(self, instr): self._inline_call_for_loop(iterator, instr) self._lasti = self.indexof(instr.jump_to) + next_instr = self._instructions[self._lasti] + self._lasti += int(next_instr.opname == 'END_FOR') except BreakGraphError as e: log(3, f"[BreakGraph] FOR_ITER sim for loop failed for: {e}\n") if backup_iter_idx: diff --git a/test/sot/skip_files_py312 b/test/sot/skip_files_py312 index d79956533e2d3..815f3a9e68b49 100644 --- a/test/sot/skip_files_py312 +++ b/test/sot/skip_files_py312 @@ -3,14 +3,10 @@ ./test_12_for_loop.py ./test_14_operators.py ./test_15_slice.py -./test_17_paddle_layer.py ./test_21_global.py ./test_analysis_inputs.py ./test_break_graph.py -./test_builtin_map.py -./test_builtin_range.py ./test_builtin_zip.py -./test_enumerate.py ./test_guard_user_defined_fn.py ./test_inplace_api.py ./test_min_graph_size.py From 4f42c2b6cb26a7f2d6a0f7502f870578b48ddf44 Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Fri, 23 Feb 2024 22:06:57 +0800 Subject: [PATCH 046/282] [Dy2St] Remove internal API `to_variable` (#61952) --- python/paddle/base/dygraph/__init__.py | 1 - python/paddle/base/dygraph/base.py | 121 ------------------ python/paddle/framework/__init__.py | 2 +- ...ransformer.py => name_load_transformer.py} | 71 ---------- .../jit/dy2static/transformers/transform.py | 7 +- test/dygraph_to_static/test_se_resnet.py | 5 +- 6 files changed, 8 insertions(+), 199 deletions(-) rename python/paddle/jit/dy2static/transformers/{basic_api_transformer.py => name_load_transformer.py} (64%) diff --git a/python/paddle/base/dygraph/__init__.py b/python/paddle/base/dygraph/__init__.py index fc77a5367c3fc..28a94ba061a0a 100644 --- a/python/paddle/base/dygraph/__init__.py +++ b/python/paddle/base/dygraph/__init__.py @@ -24,7 +24,6 @@ guard, no_grad, no_grad_, - to_variable, ) from .tracer import Tracer # noqa: F401 diff --git a/python/paddle/base/dygraph/base.py b/python/paddle/base/dygraph/base.py index 4f233cfe4d671..27b4e4ae675cb 100644 --- a/python/paddle/base/dygraph/base.py +++ b/python/paddle/base/dygraph/base.py @@ -16,14 +16,12 @@ import warnings import decorator -import numpy as np import paddle from paddle.base import core, framework from paddle.base.framework import global_var from paddle.base.multiprocess_utils import CleanupFuncRegistrar -from ..data_feeder import convert_dtype from ..framework import _get_paddle_place from ..wrapped_decorator import signature_safe_contextmanager, wrap_decorator from .tracer import Tracer @@ -837,122 +835,3 @@ def check_in_out(in_out_list, name): allow_unused, no_grad_vars, ) - - -@framework.dygraph_only -def to_variable(value, name=None, zero_copy=None, dtype=None): - r""" - :api_attr: imperative - - The API will create a ``Variable`` object from - tuple, list, numpy\.ndarray or Variable object. - - Parameters: - value(tuple|list|ndarray|Variable|Tensor): Initial data. - Can be a list, tuple, NumPy ndarray, Variable, Tensor. - The shape can be multi-dimensional. The data type is one of - numpy\.{float16, float32, float64, int16, int32, int64, - uint8, uint16, complex64, complex128}. - name(str, optional): The default value is None. Normally there is no - need for user to set this property. For more information, please - refer to :ref:`api_guide_Name` . - zero_copy(bool, optional): Whether to share memory with the input numpy - array. This parameter only works with CPUPlace and will be set to - True when it is None. Default: None. (Note: zero_copy is discarded temporally for some reason.) - dtype(str, optional): The desired data type of returned ``Variable`` . - Can be 'bool' , 'float16' , 'float32' , 'float64' , 'int8' , 'int16' , - 'int32' , 'int64' , 'uint8' . Default: None. - - Returns: - Variable : If ``value`` is a tuple/list/numpy\.ndarray object, - return ``Tensor`` created from the corresponding numpy\.ndarray object, which has - same data type and shape with ``value``. - - - Examples: - - .. code-block:: python - - >>> import numpy as np - >>> import paddle.base as base - - >>> with base.dygraph.guard(base.CPUPlace()): - ... x = np.ones([2, 2], np.float32) - ... y = base.dygraph.to_variable(x, zero_copy=False) - ... x[0][0] = -1 - ... print(y[0][0].numpy()) - ... y = base.dygraph.to_variable(x) - ... x[0][0] = 0 - ... print(y[0][0].numpy()) - ... c = np.array([2+1j, 2]) - ... z = base.dygraph.to_variable(c) - ... print(z.numpy()) - ... print(z.dtype) - ... - ... y = base.dygraph.to_variable([[0.1, 1.2], [2.2, 3.1], [4.9, 5.2]]) - ... print(y.shape) - ... - ... y = base.dygraph.to_variable(((0.1, 1.2), (2.2, 3.1), (4.9, 5.2)), dtype='int32') - ... print(y.shape) - 1 - -1 - [2.+1.j, 2.+0.j] - paddle.complex128 - [3, 2] - [3, 2] - """ - support_type = ( - list, - tuple, - np.ndarray, - core.eager.Tensor, - framework.Variable, - core.Tensor, - core.LoDTensor, - ) - if not isinstance(value, support_type): - raise TypeError( - "The type of 'value' in base.dygraph.to_variable must be {}, but received {}.".format( - support_type, type(value) - ) - ) - if isinstance(value, (core.eager.Tensor, framework.Variable)): - return value - elif isinstance(value, (core.Tensor, core.LoDTensor)): - return core.eager.Tensor(value) - else: - if isinstance( - framework._current_expected_place(), framework.core.CPUPlace - ): - # TODO(zhiqiu): we found two problems when enable zero_copy on CPUPlace. - # (1): eigen requires 16-bytes alignments, but the data of numpy array may not satisfy. - # Details: https://eigen.tuxfamily.org/dox/group__TopicUnalignedArrayAssert.html - # (2): when used in flask framework, it may result in hang. - # Details: https://github.com/PaddlePaddle/Paddle/issues/26635 - # So, we temporally disable the zero_copy strategy. - if zero_copy is True: - warnings.warn( - "Currently, zero_copy is not supported, and it will be discarded." - ) - zero_copy = False - else: - assert ( - not zero_copy - ), "zero_copy mode can only be used with CPUPlace" - - if not isinstance(value, np.ndarray): - value = np.array(value) - - if dtype is not None: - dtype = convert_dtype(dtype) - if value.dtype != dtype: - value = value.astype(dtype) - - return core.eager.Tensor( - value, - framework._current_expected_place(), - False, - zero_copy, - name if name else None, - True, - ) diff --git a/python/paddle/framework/__init__.py b/python/paddle/framework/__init__.py index 4a6019705b0da..445ad10cd1f91 100755 --- a/python/paddle/framework/__init__.py +++ b/python/paddle/framework/__init__.py @@ -23,7 +23,7 @@ IPUPlace, XPUPlace, ) -from ..base.dygraph import base, to_variable # noqa: F401 +from ..base.dygraph import base # noqa: F401 from ..base.dygraph.base import ( # noqa: F401 disable_dygraph as enable_static, enable_dygraph as disable_static, diff --git a/python/paddle/jit/dy2static/transformers/basic_api_transformer.py b/python/paddle/jit/dy2static/transformers/name_load_transformer.py similarity index 64% rename from python/paddle/jit/dy2static/transformers/basic_api_transformer.py rename to python/paddle/jit/dy2static/transformers/name_load_transformer.py index f3a2bdc1ab995..8e24cec812870 100644 --- a/python/paddle/jit/dy2static/transformers/basic_api_transformer.py +++ b/python/paddle/jit/dy2static/transformers/name_load_transformer.py @@ -21,46 +21,6 @@ __all__ = [] -class BasicApiTransformer(BaseTransformer): - """ - Class to transform basic API from dygraph to static graph. - """ - - def __init__(self, root): - self.root = root - - def transform(self): - to_tensor_transformer = ToTensorTransformer(self.root) - to_tensor_transformer.transform() - attribute_transformer = AttributeJstTransformer(self.root) - attribute_transformer.transform() - self.visit(self.root) - return self.root - - -class ToTensorTransformer(BaseTransformer): - """ - Class to transform paddle.to_tensor and paddle.to_variable to paddle.assign - """ - - def __init__(self, node): - assert isinstance( - node, gast.AST - ), "Input non-gast.AST node for the initialization of ToTensorTransformer." - self.root = node - - def transform(self): - self.visit(self.root) - return self.root - - def visit_Call(self, node): - assert isinstance(node, gast.Call) - if is_to_variable(node): - node = to_assign_node(node) - self.generic_visit(node) - return node - - class NameloadJstTransformer(BaseTransformer): """ change name and attribute load to __jst.Ld(name) pattern. @@ -168,34 +128,3 @@ def visit_Attribute(self, node): ) self.generic_visit(node) return node - - -def is_to_variable(node): - assert isinstance(node, gast.Call) - api_name = ast_to_source_code(node.func).strip() - - return api_name.split(".")[-1] == "to_variable" - - -def to_assign_node(node): - # Transform dygraph api `base.dygraph.to_variable` alias `paddle.to_tensor` to static api `paddle.assign`. - # NOTE: - # 1. Api `to_variable` supports data type {float16, float32, float64, int16, int32, int64, uint8, uint16}, - # but api `assign` only supports {float32, float64, int32, int64, bool}; - # 2. If the input of api `assign` is numpy.ndarray, its size cannot be greater than 1024 * 1024. - - assert isinstance(node, gast.Call) - assign_api = gast.parse('paddle.assign').body[0].value - node.func = assign_api - - if node.args: - node.args = [node.args[0]] - node.keywords = [] - else: - for idx, kw in enumerate(node.keywords): - if kw.arg == 'value' or kw.arg == 'data': - node.keywords[idx].arg = 'x' - node.keywords = [node.keywords[idx]] - node.args = [] - break - return node diff --git a/python/paddle/jit/dy2static/transformers/transform.py b/python/paddle/jit/dy2static/transformers/transform.py index b07f416a1af6e..9ae5edb3fb68e 100644 --- a/python/paddle/jit/dy2static/transformers/transform.py +++ b/python/paddle/jit/dy2static/transformers/transform.py @@ -23,7 +23,6 @@ from ..utils import ast_to_source_code from .assert_transformer import AssertTransformer from .base import BaseTransformer -from .basic_api_transformer import BasicApiTransformer, NameloadJstTransformer from .break_continue_transformer import ( BreakContinueTransformer, BreakTransformOptimizer, @@ -36,6 +35,10 @@ from .ifelse_transformer import IfElseTransformer from .logical_transformer import LogicalTransformer from .loop_transformer import LoopTransformer +from .name_load_transformer import ( + AttributeJstTransformer, + NameloadJstTransformer, +) from .return_transformer import ReturnTransformer from .tensor_shape_transformer import TensorShapeTransformer from .tensorhook_transformer import RegisterHookTransformer @@ -91,7 +94,7 @@ def transfer_from_node_type(self, node): transformers = [ RegisterHookTransformer, EarlyReturnTransformer, - BasicApiTransformer, # Basic Api + AttributeJstTransformer, # Tensor.size -> Tensor.size(), it's unnecessary in PIR mode TensorShapeTransformer, # Tensor.shape -> paddle.shape(Tensor) BreakContinueTransformer, # break/continue in loops ReturnTransformer, # return in functions diff --git a/test/dygraph_to_static/test_se_resnet.py b/test/dygraph_to_static/test_se_resnet.py index a9d11b2959994..113dde8dde3d3 100644 --- a/test/dygraph_to_static/test_se_resnet.py +++ b/test/dygraph_to_static/test_se_resnet.py @@ -29,7 +29,6 @@ import paddle from paddle import base -from paddle.base.dygraph.base import to_variable from paddle.jit.api import to_static from paddle.jit.translated_layer import INFER_MODEL_SUFFIX, INFER_PARAMS_SUFFIX from paddle.nn import BatchNorm, Linear @@ -403,8 +402,8 @@ def train(self, train_reader, to_static): .reshape(BATCH_SIZE, 1) ) - img = to_variable(dy_x_data) - label = to_variable(y_data) + img = paddle.to_tensor(dy_x_data) + label = paddle.to_tensor(y_data) label.stop_gradient = True pred, avg_loss, acc_top1, acc_top5 = se_resnext(img, label) From dc6071aa709b2f5b339e3f39a17f9d5b72c8d8d6 Mon Sep 17 00:00:00 2001 From: winter-wang <78149749+winter-wang@users.noreply.github.com> Date: Sat, 24 Feb 2024 00:01:42 +0800 Subject: [PATCH 047/282] [PIR] fix keyword argument bug in executor. (#62006) --- paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc b/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc index f82ec0cbcdf1d..469ab96a3c0cb 100644 --- a/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc +++ b/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc @@ -2541,8 +2541,8 @@ void ProcessBlock( auto new_arg = new_block->AddKwarg(keyword, arg.type()); (*map_value_pair)[arg] = new_arg; if (auto dense_tensor_type = arg.type().dyn_cast()) { - new_arg.set_type(AllocatedDenseTensorType::get( - ctx, phi::CPUPlace(), dense_tensor_type)); + new_arg.set_type( + AllocatedDenseTensorType::get(ctx, phi::Place(), dense_tensor_type)); } } if (platform::is_gpu_place(place)) { From 9cc5bafa0dc8e032ce4adc5c6ee4f4547fd8883e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=98=A5=E4=B9=94?= <83450930+Liyulingyue@users.noreply.github.com> Date: Sat, 24 Feb 2024 18:51:41 +0800 Subject: [PATCH 048/282] Add some empty cmakelist.txt to avoid upcoming conflicts (#62033) * add cmakelist * Update CMakeLists.txt * Update CMakeLists.txt --- test/cpp/fluid/CMakeLists.txt | 1 + test/cpp/fluid/platform/CMakeLists.txt | 2 ++ test/cpp/fluid/platform/device/CMakeLists.txt | 1 + test/cpp/fluid/platform/device/custom/CMakeLists.txt | 1 + test/cpp/fluid/platform/profiler/CMakeLists.txt | 1 + 5 files changed, 6 insertions(+) create mode 100644 test/cpp/fluid/platform/CMakeLists.txt create mode 100644 test/cpp/fluid/platform/device/CMakeLists.txt create mode 100644 test/cpp/fluid/platform/device/custom/CMakeLists.txt create mode 100644 test/cpp/fluid/platform/profiler/CMakeLists.txt diff --git a/test/cpp/fluid/CMakeLists.txt b/test/cpp/fluid/CMakeLists.txt index 6e006b16ad6ef..f49eefb4354d0 100644 --- a/test/cpp/fluid/CMakeLists.txt +++ b/test/cpp/fluid/CMakeLists.txt @@ -1,6 +1,7 @@ add_subdirectory(memory) add_subdirectory(benchmark) add_subdirectory(framework) +add_subdirectory(platform) if(WITH_CINN) add_subdirectory(cinn) diff --git a/test/cpp/fluid/platform/CMakeLists.txt b/test/cpp/fluid/platform/CMakeLists.txt new file mode 100644 index 0000000000000..d57ed923b4a63 --- /dev/null +++ b/test/cpp/fluid/platform/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(device) +add_subdirectory(profiler) diff --git a/test/cpp/fluid/platform/device/CMakeLists.txt b/test/cpp/fluid/platform/device/CMakeLists.txt new file mode 100644 index 0000000000000..33311abc3d279 --- /dev/null +++ b/test/cpp/fluid/platform/device/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(custom) diff --git a/test/cpp/fluid/platform/device/custom/CMakeLists.txt b/test/cpp/fluid/platform/device/custom/CMakeLists.txt new file mode 100644 index 0000000000000..0a95e9a292a4d --- /dev/null +++ b/test/cpp/fluid/platform/device/custom/CMakeLists.txt @@ -0,0 +1 @@ +# Note(Liyulingyue): create an empty cmake file to avoid conflict diff --git a/test/cpp/fluid/platform/profiler/CMakeLists.txt b/test/cpp/fluid/platform/profiler/CMakeLists.txt new file mode 100644 index 0000000000000..0a95e9a292a4d --- /dev/null +++ b/test/cpp/fluid/platform/profiler/CMakeLists.txt @@ -0,0 +1 @@ +# Note(Liyulingyue): create an empty cmake file to avoid conflict From 7efc5235b34fdbd2bd74d8e3294c43c54a45c22e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=98=A5=E4=B9=94?= <83450930+Liyulingyue@users.noreply.github.com> Date: Sun, 25 Feb 2024 17:58:06 +0800 Subject: [PATCH 049/282] =?UTF-8?q?=E3=80=90paddle=5Ftest=20No.36=E3=80=91?= =?UTF-8?q?replace=20cc=5Ftest=20with=20paddle=5Ftest=20(#62036)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * mv floder * cc2paddle --- paddle/fluid/platform/device/custom/CMakeLists.txt | 4 ---- test/cpp/fluid/platform/device/custom/CMakeLists.txt | 4 +++- .../cpp}/fluid/platform/device/custom/custom_device_test.cc | 0 3 files changed, 3 insertions(+), 5 deletions(-) rename {paddle => test/cpp}/fluid/platform/device/custom/custom_device_test.cc (100%) diff --git a/paddle/fluid/platform/device/custom/CMakeLists.txt b/paddle/fluid/platform/device/custom/CMakeLists.txt index c01bead7b03e6..023efe02e08bf 100644 --- a/paddle/fluid/platform/device/custom/CMakeLists.txt +++ b/paddle/fluid/platform/device/custom/CMakeLists.txt @@ -3,8 +3,4 @@ if(WITH_CUSTOM_DEVICE) custom_device_resource_pool SRCS custom_device_resource_pool.cc DEPS phi common glog enforce monitor) - cc_test( - custom_device_test - SRCS custom_device_test.cc - DEPS phi common gradient_accumulator) endif() diff --git a/test/cpp/fluid/platform/device/custom/CMakeLists.txt b/test/cpp/fluid/platform/device/custom/CMakeLists.txt index 0a95e9a292a4d..87f4a0ec50251 100644 --- a/test/cpp/fluid/platform/device/custom/CMakeLists.txt +++ b/test/cpp/fluid/platform/device/custom/CMakeLists.txt @@ -1 +1,3 @@ -# Note(Liyulingyue): create an empty cmake file to avoid conflict +if(WITH_CUSTOM_DEVICE) + paddle_test(custom_device_test SRCS custom_device_test.cc) +endif() diff --git a/paddle/fluid/platform/device/custom/custom_device_test.cc b/test/cpp/fluid/platform/device/custom/custom_device_test.cc similarity index 100% rename from paddle/fluid/platform/device/custom/custom_device_test.cc rename to test/cpp/fluid/platform/device/custom/custom_device_test.cc From e213188465e9d3b89ed29fedf98dbe7b846c9576 Mon Sep 17 00:00:00 2001 From: ShenLiang <1422485404@qq.com> Date: Sun, 25 Feb 2024 23:18:08 +0800 Subject: [PATCH 050/282] add overlap p2p (#61935) --- .../framework/distributed_strategy.proto | 1 + .../fleet/meta_parallel/pipeline_parallel.py | 359 ++++++++++++----- .../pp_utils/p2p_communication.py | 360 +++++++++++++----- 3 files changed, 516 insertions(+), 204 deletions(-) diff --git a/paddle/fluid/framework/distributed_strategy.proto b/paddle/fluid/framework/distributed_strategy.proto index 2042a313c41e6..58460fcf9064b 100755 --- a/paddle/fluid/framework/distributed_strategy.proto +++ b/paddle/fluid/framework/distributed_strategy.proto @@ -82,6 +82,7 @@ message PpConfig { optional bool sharding_comm_overlap = 4 [ default = false ]; optional bool profiling = 5 [ default = false ]; optional bool release_gradients = 6 [ default = false ]; + optional bool overlap_p2p_comm = 7 [default = false]; } message DygraphShardingConfig { diff --git a/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py b/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py index 384d89b4d9c12..e5233c87a199b 100644 --- a/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py +++ b/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py @@ -216,6 +216,12 @@ def __init__(self, layers, hcg, strategy): "sharding_configs" ].split_param + self._overlap_p2p_comm = self._strategy.hybrid_configs[ + "pp_configs" + ].overlap_p2p_comm + + self._batch_p2p_comm = not self._overlap_p2p_comm + logger.info( f"dp_comm_overlap {self._dp_comm_overlap}; \ sharding_comm_overlap {self._sharding_comm_overlap}; \ @@ -1229,12 +1235,21 @@ def _process_bwd_buffer(step_id, tensor): if not static_scheduler: self.input_tensors[0].append( self._p2p_helper.recv_forward( - self.is_pipeline_first_stage(), sync_recv=False + self.is_pipeline_first_stage(), + sync_recv=False, + batch_p2p_comm=self._batch_p2p_comm, ) ) + fwd_wait_handles = None + bwd_wait_handles = None + # run startup steps for micro_step in range(startup_steps): + if fwd_wait_handles is not None: + for req in fwd_wait_handles: + req.wait() + if static_scheduler: virtual_pp_rank = self._get_virtual_pp_rank( micro_step, forward=True @@ -1270,39 +1285,77 @@ def _process_bwd_buffer(step_id, tensor): if self.is_pipeline_last_stage(ignore_virtual=True): output_tensor = _process_fwd_buffer(micro_step, output_tensor) - # prepare for the first steady step - if ( - micro_step == (startup_steps - 1) - and (not forward_only) - and steady_steps - ): - input_tensor_grad = None - recv_next = True - if self.is_pipeline_last_stage(ignore_virtual=True): - recv_next = False + if not self._overlap_p2p_comm: + # prepare for the first steady step + if ( + micro_step == (startup_steps - 1) + and (not forward_only) + and steady_steps + ): + input_tensor_grad = None + recv_next = True + if self.is_pipeline_last_stage(ignore_virtual=True): + recv_next = False - # the last startup step needs on four direction comm to set up for steady 1f1b + # the last startup step needs on four direction comm to set up for steady 1f1b + ( + input_tensor, + output_tensor_grad, + ) = self._p2p_helper.send_forward_backward_recv_forward_backward( + output_tensor, + input_tensor_grad, + recv_prev=recv_prev, + recv_next=recv_next, + batch_p2p_comm=self._batch_p2p_comm, + ) + # output_tensor_grad is not none if recv_next + # append output_tensor_grad no matter none or not + self.output_tensor_grads[self.num_model_chunks - 1].append( + output_tensor_grad + ) + else: + input_tensor = self._p2p_helper.send_forward_recv_forward( + output_tensor, + recv_prev=recv_prev, + batch_p2p_comm=self._batch_p2p_comm, + ) + # append input_tensor no matter none or not + self.input_tensors[next_virtual_pp_rank].append(input_tensor) + else: ( input_tensor, - output_tensor_grad, - ) = self._p2p_helper.send_forward_backward_recv_forward_backward( + fwd_wait_handles, + ) = self._p2p_helper.send_forward_recv_forward( output_tensor, - input_tensor_grad, recv_prev=recv_prev, - recv_next=recv_next, + batch_p2p_comm=self._batch_p2p_comm, + overlap_p2p_comm=True, ) - # output_tensor_grad is not none if recv_next - # append output_tensor_grad no matter none or not - self.output_tensor_grads[self.num_model_chunks - 1].append( - output_tensor_grad - ) - else: - input_tensor = self._p2p_helper.send_forward_recv_forward( - output_tensor, recv_prev=recv_prev - ) - # append input_tensor no matter none or not - self.input_tensors[next_virtual_pp_rank].append(input_tensor) + if ( + micro_step == (startup_steps - 1) + and (not forward_only) + and steady_steps + ): + input_tensor_grad = None + recv_next = True + if self.is_pipeline_last_stage(ignore_virtual=True): + recv_next = False + ( + output_tensor_grad, + bwd_wait_handles, + ) = self._p2p_helper.send_backward_recv_backward( + input_tensor_grad, + recv_next=recv_next, + batch_p2p_comm=self._batch_p2p_comm, + overlap_p2p_comm=True, + ) + self.output_tensor_grads[self.num_model_chunks - 1].append( + output_tensor_grad + ) + + # append input_tensor no matter none or not + self.input_tensors[next_virtual_pp_rank].append(input_tensor) self._release_output(output_tensor) # run 1f1b steady steps @@ -1339,85 +1392,186 @@ def _process_bwd_buffer(step_id, tensor): continue # forward forward_micro_step_id = micro_step + startup_steps - self._record_stamp("F", forward_micro_step_id, '"B"', forward=True) - output_tensor = self._forward_step_helper( - micro_dataset, forward_micro_step_id - ) - self._record_stamp("F", forward_micro_step_id, '"E"', forward=True) - # backward - backward_micro_step_id = micro_step - self._record_stamp( - "B", backward_micro_step_id, '"B"', forward=False - ) - input_tensor_grad = self._backward_step_helper( - backward_micro_step_id - ) - self._record_stamp( - "B", backward_micro_step_id, '"E"', forward=False - ) + if self._overlap_p2p_comm: + if fwd_wait_handles is not None: + for req in fwd_wait_handles: + req.wait() - # four directions comm - # send output tensor to downstream - # send input tensor grad to upstream - # recv input tensor from upstream - # recv output tensor grad from downstream + self._release_output(output_tensor) + output_tensor = self._forward_step_helper( + micro_dataset, forward_micro_step_id + ) - # last stage doesn't send rst to downstream - forward_virtual_pp_rank = self._get_virtual_pp_rank( - forward_micro_step_id, forward=True - ) - self.set_virtual_pipeline_rank(forward_virtual_pp_rank) - if self.is_pipeline_last_stage(ignore_virtual=True): - output_tensor = _process_fwd_buffer( - forward_micro_step_id, output_tensor + forward_virtual_pp_rank = self._get_virtual_pp_rank( + forward_micro_step_id, forward=True ) + self.set_virtual_pipeline_rank(forward_virtual_pp_rank) + if self.is_pipeline_last_stage(ignore_virtual=True): + output_tensor = _process_fwd_buffer( + forward_micro_step_id, output_tensor + ) - # first stage doesn't send grad to upstream - backward_virtual_pp_rank = self._get_virtual_pp_rank( - backward_micro_step_id, forward=False - ) - self.set_virtual_pipeline_rank(backward_virtual_pp_rank) - if self.is_pipeline_first_stage(ignore_virtual=True): - input_tensor_grad = _process_bwd_buffer( - backward_micro_step_id, input_tensor_grad + # determine whether to recv input tensor from upstream + recv_prev = True + if self.is_pipeline_first_stage(ignore_virtual=True): + next_forward_virtual_pp_rank = self._get_virtual_pp_rank( + forward_micro_step_id + 1, forward=True + ) + if next_forward_virtual_pp_rank == 0: + # next chunk is the first chunk, not need to pre recv an input tensor + recv_prev = False + else: + next_forward_virtual_pp_rank = self._get_virtual_pp_rank( + forward_micro_step_id + 1, forward=True + ) + + # last iteration doesn't need recv from upstream + if micro_step == (steady_steps - 1): + recv_prev = False + + # Send activation tensor to the next stage and receive activation tensor from the + # previous stage + ( + input_tensor, + fwd_wait_handles, + ) = self._p2p_helper.send_forward_recv_forward( + output_tensor, + recv_prev=recv_prev, + batch_p2p_comm=self._batch_p2p_comm, + overlap_p2p_comm=True, ) - # determine whether to recv input tensor from upstream - recv_prev = True - next_forward_virtual_pp_rank = self._get_virtual_pp_rank( - forward_micro_step_id + 1, forward=True - ) - if self.is_pipeline_first_stage(ignore_virtual=True) and ( - next_forward_virtual_pp_rank == 0 - ): - # first pp stage and first virtual stage - recv_prev = False + if bwd_wait_handles is not None: + for req in bwd_wait_handles: + req.wait() - # last iteration doesn't need recv from upstream - if micro_step == (steady_steps - 1): - recv_prev = False + # backward pass + backward_micro_step_id = micro_step + input_tensor_grad = self._backward_step_helper( + backward_micro_step_id + ) - # determine whether to recv grad from downstream - recv_next = True - next_backward_virtual_pp_rank = self._get_virtual_pp_rank( - backward_micro_step_id + 1, forward=False - ) - if self.is_pipeline_last_stage(ignore_virtual=True) and ( - next_backward_virtual_pp_rank == (self.num_model_chunks - 1) - ): - # last pp stage and last virtual stage - recv_next = False - - ( - input_tensor, - output_tensor_grad, - ) = self._p2p_helper.send_forward_backward_recv_forward_backward( - output_tensor, - input_tensor_grad, - recv_prev=recv_prev, - recv_next=recv_next, - ) + # first stage doesn't send grad to upstream + backward_virtual_pp_rank = self._get_virtual_pp_rank( + backward_micro_step_id, forward=False + ) + self.set_virtual_pipeline_rank(backward_virtual_pp_rank) + if self.is_pipeline_first_stage(ignore_virtual=True): + input_tensor_grad = _process_bwd_buffer( + backward_micro_step_id, input_tensor_grad + ) + + recv_next = True + if self.is_pipeline_last_stage(ignore_virtual=True): + next_backward_virtual_pp_rank = self._get_virtual_pp_rank( + backward_micro_step_id + 1, + forward=False, + ) + if next_backward_virtual_pp_rank == ( + self.num_model_chunks - 1 + ): + # next chunk is the last chunk, not need to pre recv an output tensor grad + recv_next = False + else: + next_backward_virtual_pp_rank = self._get_virtual_pp_rank( + backward_micro_step_id + 1, forward=False + ) + + ( + output_tensor_grad, + bwd_wait_handles, + ) = self._p2p_helper.send_backward_recv_backward( + input_tensor_grad, + recv_next=recv_next, + batch_p2p_comm=self._batch_p2p_comm, + overlap_p2p_comm=True, + ) + else: + self._record_stamp( + "F", forward_micro_step_id, '"B"', forward=True + ) + output_tensor = self._forward_step_helper( + micro_dataset, forward_micro_step_id + ) + self._record_stamp( + "F", forward_micro_step_id, '"E"', forward=True + ) + + # backward + backward_micro_step_id = micro_step + self._record_stamp( + "B", backward_micro_step_id, '"B"', forward=False + ) + input_tensor_grad = self._backward_step_helper( + backward_micro_step_id + ) + self._record_stamp( + "B", backward_micro_step_id, '"E"', forward=False + ) + + # four directions comm + # send output tensor to downstream + # send input tensor grad to upstream + # recv input tensor from upstream + # recv output tensor grad from downstream + + # last stage doesn't send rst to downstream + forward_virtual_pp_rank = self._get_virtual_pp_rank( + forward_micro_step_id, forward=True + ) + self.set_virtual_pipeline_rank(forward_virtual_pp_rank) + if self.is_pipeline_last_stage(ignore_virtual=True): + output_tensor = _process_fwd_buffer( + forward_micro_step_id, output_tensor + ) + + # first stage doesn't send grad to upstream + backward_virtual_pp_rank = self._get_virtual_pp_rank( + backward_micro_step_id, forward=False + ) + self.set_virtual_pipeline_rank(backward_virtual_pp_rank) + if self.is_pipeline_first_stage(ignore_virtual=True): + input_tensor_grad = _process_bwd_buffer( + backward_micro_step_id, input_tensor_grad + ) + + # determine whether to recv input tensor from upstream + recv_prev = True + next_forward_virtual_pp_rank = self._get_virtual_pp_rank( + forward_micro_step_id + 1, forward=True + ) + if self.is_pipeline_first_stage(ignore_virtual=True) and ( + next_forward_virtual_pp_rank == 0 + ): + # first pp stage and first virtual stage + recv_prev = False + + # last iteration doesn't need recv from upstream + if micro_step == (steady_steps - 1): + recv_prev = False + + # determine whether to recv grad from downstream + recv_next = True + next_backward_virtual_pp_rank = self._get_virtual_pp_rank( + backward_micro_step_id + 1, forward=False + ) + if self.is_pipeline_last_stage(ignore_virtual=True) and ( + next_backward_virtual_pp_rank == (self.num_model_chunks - 1) + ): + # last pp stage and last virtual stage + recv_next = False + + ( + input_tensor, + output_tensor_grad, + ) = self._p2p_helper.send_forward_backward_recv_forward_backward( + output_tensor, + input_tensor_grad, + recv_prev=recv_prev, + recv_next=recv_next, + batch_p2p_comm=self._batch_p2p_comm, + ) # append input_tensor no matter none or not self.input_tensors[next_forward_virtual_pp_rank].append( input_tensor @@ -1434,10 +1588,15 @@ def _process_bwd_buffer(step_id, tensor): # remaining backward steps if not forward_only: + if self._overlap_p2p_comm and bwd_wait_handles is not None: + for wait_handles in bwd_wait_handles: + wait_handles.wait() + # no steady steps, which only occurs when accumulate_step == num_stage if not steady_steps: output_tensor_grad = p2p.recv_backward( - self.is_pipeline_last_stage() + self.is_pipeline_last_stage(), + batch_p2p_comm=self._batch_p2p_comm, ) self.output_tensor_grads[self.num_model_chunks - 1].append( output_tensor_grad @@ -1482,7 +1641,9 @@ def _process_bwd_buffer(step_id, tensor): # append output_tensor_grad no matter none or not self.output_tensor_grads[next_backward_virtual_pp_rank].append( self._p2p_helper.send_backward_recv_backward( - input_tensor_grad, recv_next=recv_next + input_tensor_grad, + recv_next=recv_next, + batch_p2p_comm=self._batch_p2p_comm, ) ) diff --git a/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py b/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py index 4566f89290fc0..6d470d541f66b 100644 --- a/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py +++ b/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py @@ -292,91 +292,33 @@ def batch_send_recv_on_calc_stream(p2p_op_list): op(tensor, comm_group, peer, nranks, rank_id) -def _process_p2p_tuple_or_tensor( +def _batch_p2p_tuple_or_tensor( tensors, p2p_func, pp_rank, pp_group, mp_degree=1, mp_rank=0 ): - ops = [] - if isinstance(tensors, tuple): - for tensor in tensors: - op = P2PonCalcStream( - p2p_func, tensor, pp_rank, pp_group, mp_degree, mp_rank - ) - ops.append(op) - else: - op = P2PonCalcStream( - p2p_func, tensors, pp_rank, pp_group, mp_degree, mp_rank - ) - ops.append(op) + if not isinstance(tensors, tuple): + tensors = (tensors,) + ops = [ + P2PonCalcStream(p2p_func, tensor, pp_rank, pp_group, mp_degree, mp_rank) + for tensor in tensors + ] return ops -def _p2p_helper( - tensor_send_next, - tensor_send_prev, - recv_prev, - recv_next, - sync_recv=True, - send_recv_meta=None, +def _batched_p2p_ops( + tensor_send_prev, tensor_recv_prev, tensor_send_next, tensor_recv_next, hcg ): - global _hcg - - tensor_recv_prev = None - tensor_recv_next = None - - # send / recv message - assert send_recv_meta is not None, "send_recv_meta should not be None" - recv_shape_msg = send_recv_meta.recv_shape_message - recv_dtype_msg = send_recv_meta.recv_dtype_message - recv_stop_gradient = send_recv_meta.recv_stop_gradient - - send_shape_msg = send_recv_meta.send_shape_message - send_dtype_msg = send_recv_meta.send_dtype_message - - # model parallel message - mp_group = _hcg.get_model_parallel_group() - mp_degree = _hcg.get_model_parallel_world_size() - mp_rank = _hcg.get_model_parallel_rank() - - if recv_prev: - if isinstance(recv_shape_msg, tuple): - tensor_recv_prev = [] - for idx, shape in enumerate(recv_shape_msg): - tmp = paddle.empty( - shape=shape, dtype=number_2_dtype(recv_dtype_msg[idx]) - ) - tmp.stop_gradient = recv_stop_gradient[idx] - tensor_recv_prev.append(tmp) - tensor_recv_prev = tuple(tensor_recv_prev) - else: - tensor_recv_prev = paddle.empty( - shape=recv_shape_msg, dtype=number_2_dtype(recv_dtype_msg) - ) - tensor_recv_prev.stop_gradient = recv_stop_gradient - - if recv_next: - if isinstance(send_shape_msg, tuple): - tensor_recv_next = [] - for idx, shape in enumerate(send_shape_msg): - tensor_recv_next.append( - paddle.empty( - shape=shape, dtype=number_2_dtype(send_dtype_msg[idx]) - ) - ) - tensor_recv_next = tuple(tensor_recv_next) - else: - tensor_recv_next = paddle.empty( - shape=send_shape_msg, dtype=number_2_dtype(send_dtype_msg) - ) - ops = [] - pipe_group = _hcg.get_pipe_parallel_group() + pipe_group = hcg.get_pipe_parallel_group() + mp_degree = hcg.get_model_parallel_world_size() + mp_rank = hcg.get_model_parallel_rank() + mp_group = hcg.get_model_parallel_group() # start to p2p communicate if not _sync_send: if tensor_send_prev is not None: - src_rank = _hcg._get_p2p_prev_rank() + src_rank = hcg._get_p2p_prev_rank() ops.extend( - _process_p2p_tuple_or_tensor( + _batch_p2p_tuple_or_tensor( tensor_send_prev, _send_on_calc_stream, src_rank, @@ -386,9 +328,9 @@ def _p2p_helper( ) ) if tensor_recv_prev is not None: - dst_rank = _hcg._get_p2p_prev_rank() + dst_rank = hcg._get_p2p_prev_rank() ops.extend( - _process_p2p_tuple_or_tensor( + _batch_p2p_tuple_or_tensor( tensor_recv_prev, _recv_on_calc_stream, dst_rank, @@ -398,9 +340,9 @@ def _p2p_helper( ) ) if tensor_send_next is not None: - src_rank = _hcg._get_p2p_next_rank() + src_rank = hcg._get_p2p_next_rank() ops.extend( - _process_p2p_tuple_or_tensor( + _batch_p2p_tuple_or_tensor( tensor_send_next, _send_on_calc_stream, src_rank, @@ -410,9 +352,9 @@ def _p2p_helper( ) ) if tensor_recv_next is not None: - dst_rank = _hcg._get_p2p_next_rank() + dst_rank = hcg._get_p2p_next_rank() ops.extend( - _process_p2p_tuple_or_tensor( + _batch_p2p_tuple_or_tensor( tensor_recv_next, _recv_on_calc_stream, dst_rank, @@ -427,9 +369,9 @@ def _p2p_helper( # When using this order, the environment variable # 'PADDLE_P2P_SYNC_SEND' should be set True if tensor_recv_prev is not None: - dst_rank = _hcg._get_p2p_prev_rank() + dst_rank = hcg._get_p2p_prev_rank() ops.extend( - _process_p2p_tuple_or_tensor( + _batch_p2p_tuple_or_tensor( tensor_recv_prev, _recv_on_calc_stream, dst_rank, @@ -439,9 +381,9 @@ def _p2p_helper( ) ) if tensor_send_next is not None: - src_rank = _hcg._get_p2p_next_rank() + src_rank = hcg._get_p2p_next_rank() ops.extend( - _process_p2p_tuple_or_tensor( + _batch_p2p_tuple_or_tensor( tensor_send_next, _send_on_calc_stream, src_rank, @@ -451,9 +393,9 @@ def _p2p_helper( ) ) if tensor_recv_next is not None: - dst_rank = _hcg._get_p2p_next_rank() + dst_rank = hcg._get_p2p_next_rank() ops.extend( - _process_p2p_tuple_or_tensor( + _batch_p2p_tuple_or_tensor( tensor_recv_next, _recv_on_calc_stream, dst_rank, @@ -463,9 +405,9 @@ def _p2p_helper( ) ) if tensor_send_prev is not None: - src_rank = _hcg._get_p2p_prev_rank() + src_rank = hcg._get_p2p_prev_rank() ops.extend( - _process_p2p_tuple_or_tensor( + _batch_p2p_tuple_or_tensor( tensor_send_prev, _send_on_calc_stream, src_rank, @@ -477,7 +419,6 @@ def _p2p_helper( if len(ops) > 0: batch_send_recv_on_calc_stream(ops) - if distutils.util.strtobool( os.getenv('FLAGS_p2p_device_synchronize', '0') ): @@ -506,7 +447,176 @@ def _p2p_helper( use_calc_stream=True, ) - return tensor_recv_prev, tensor_recv_next + +def _p2p_ops_tuple_or_tensor(tensors, p2p_func, pp_rank, pp_group): + if not isinstance(tensors, tuple): + tensors = (tensors,) + reqs = [] + for tensor in tensors: + reqs.append(p2p_func(tensor, pp_rank, pp_group)) + return reqs + + +def _p2p_ops( + tensor_send_prev, tensor_recv_prev, tensor_send_next, tensor_recv_next, hcg +): + reqs = [] + group = hcg.get_pipe_parallel_group() + if hcg.get_stage_id() % 2 == 0: + if tensor_send_next is not None: + reqs.extend( + _p2p_ops_tuple_or_tensor( + tensor_send_next, + paddle.distributed.isend, + hcg._get_p2p_next_rank(), + group, + ) + ) + if tensor_recv_prev is not None: + reqs.extend( + _p2p_ops_tuple_or_tensor( + tensor_recv_prev, + paddle.distributed.irecv, + hcg._get_p2p_prev_rank(), + group, + ) + ) + + if tensor_send_prev is not None: + reqs.extend( + _p2p_ops_tuple_or_tensor( + tensor_send_prev, + paddle.distributed.isend, + _hcg._get_p2p_prev_rank(), + group, + ) + ) + + if tensor_recv_next is not None: + reqs.extend( + _p2p_ops_tuple_or_tensor( + tensor_recv_next, + paddle.distributed.irecv, + hcg._get_p2p_next_rank(), + group, + ) + ) + else: + if tensor_recv_prev is not None: + reqs.extend( + _p2p_ops_tuple_or_tensor( + tensor_recv_prev, + paddle.distributed.irecv, + hcg._get_p2p_prev_rank(), + group, + ) + ) + if tensor_send_next is not None: + reqs.extend( + _p2p_ops_tuple_or_tensor( + tensor_send_next, + paddle.distributed.isend, + hcg._get_p2p_next_rank(), + group, + ) + ) + if tensor_recv_next is not None: + reqs.extend( + _p2p_ops_tuple_or_tensor( + tensor_recv_next, + paddle.distributed.irecv, + hcg._get_p2p_next_rank(), + group, + ) + ) + if tensor_send_prev is not None: + reqs.extend( + _p2p_ops_tuple_or_tensor( + tensor_send_prev, + paddle.distributed.isend, + hcg._get_p2p_prev_rank(), + group, + ) + ) + return reqs + + +def _p2p_helper( + tensor_send_next, + tensor_send_prev, + recv_prev, + recv_next, + sync_recv=True, + send_recv_meta=None, + batch_p2p_comm=True, + wait_on_reqs=True, +): + global _hcg + + tensor_recv_prev = None + tensor_recv_next = None + + # send / recv message + assert send_recv_meta is not None, "send_recv_meta should not be None" + recv_shape_msg = send_recv_meta.recv_shape_message + recv_dtype_msg = send_recv_meta.recv_dtype_message + recv_stop_gradient = send_recv_meta.recv_stop_gradient + + send_shape_msg = send_recv_meta.send_shape_message + send_dtype_msg = send_recv_meta.send_dtype_message + + # model parallel message + mp_group = _hcg.get_model_parallel_group() + mp_degree = _hcg.get_model_parallel_world_size() + mp_rank = _hcg.get_model_parallel_rank() + + if recv_prev: + if isinstance(recv_shape_msg, tuple): + tensor_recv_prev = [] + for idx, shape in enumerate(recv_shape_msg): + tmp = paddle.empty( + shape=shape, dtype=number_2_dtype(recv_dtype_msg[idx]) + ) + tmp.stop_gradient = recv_stop_gradient[idx] + tensor_recv_prev.append(tmp) + tensor_recv_prev = tuple(tensor_recv_prev) + else: + tensor_recv_prev = paddle.empty( + shape=recv_shape_msg, dtype=number_2_dtype(recv_dtype_msg) + ) + tensor_recv_prev.stop_gradient = recv_stop_gradient + + if recv_next: + if isinstance(send_shape_msg, tuple): + tensor_recv_next = [] + for idx, shape in enumerate(send_shape_msg): + tensor_recv_next.append( + paddle.empty( + shape=shape, dtype=number_2_dtype(send_dtype_msg[idx]) + ) + ) + tensor_recv_next = tuple(tensor_recv_next) + else: + tensor_recv_next = paddle.empty( + shape=send_shape_msg, dtype=number_2_dtype(send_dtype_msg) + ) + + p2p_func = _batched_p2p_ops if batch_p2p_comm else _p2p_ops + reqs = p2p_func( + tensor_send_prev, + tensor_recv_prev, + tensor_send_next, + tensor_recv_next, + _hcg, + ) + + # NOTE(shenliang03): batch_p2p_comm no need wait because of using calculate stream + if wait_on_reqs and not batch_p2p_comm and len(reqs) > 0: + for req in reqs: + req.wait() + reqs = None + + return tensor_recv_prev, tensor_recv_next, reqs class P2pHelper: @@ -527,7 +637,7 @@ def _recv_meta(self): self._send_recv_meta.recv_meta(_hcg.get_pipe_parallel_group()) self._send_recv_meta.has_recv_meta = self._use_cache - def recv_forward(self, pp_first_stage, sync_recv=True): + def recv_forward(self, pp_first_stage, sync_recv=True, batch_p2p_comm=True): global _timers if _timers is not None: _timers("recv_forward").start() @@ -536,38 +646,40 @@ def recv_forward(self, pp_first_stage, sync_recv=True): else: self._recv_meta() - input_tensor, _ = _p2p_helper( + input_tensor, _, _ = _p2p_helper( tensor_send_next=None, tensor_send_prev=None, recv_prev=True, recv_next=False, sync_recv=sync_recv, send_recv_meta=self._send_recv_meta, + batch_p2p_comm=batch_p2p_comm, ) if _timers is not None: _timers("recv_forward").stop() return input_tensor - def recv_backward(self, pp_last_stage, sync_recv=True): + def recv_backward(self, pp_last_stage, sync_recv=True, batch_p2p_comm=True): global _timers if _timers is not None: _timers("recv_backward").start() if pp_last_stage: output_tensor_grad = None else: - _, output_tensor_grad = _p2p_helper( + _, output_tensor_grad, _ = _p2p_helper( tensor_send_next=None, tensor_send_prev=None, recv_prev=False, recv_next=True, sync_recv=sync_recv, send_recv_meta=self._send_recv_meta, + batch_p2p_comm=batch_p2p_comm, ) if _timers is not None: _timers("recv_backward").stop() return output_tensor_grad - def send_forward(self, output_tensor, pp_last_stage): + def send_forward(self, output_tensor, pp_last_stage, batch_p2p_comm=True): global _timers if _timers is not None: _timers("send_forward").start() @@ -580,11 +692,14 @@ def send_forward(self, output_tensor, pp_last_stage): recv_prev=False, recv_next=False, send_recv_meta=self._send_recv_meta, + batch_p2p_comm=batch_p2p_comm, ) if _timers is not None: _timers("send_forward").stop() - def send_backward(self, input_tensor_grad, pp_first_stage): + def send_backward( + self, input_tensor_grad, pp_first_stage, batch_p2p_comm=True + ): global _timers if _timers is not None: _timers("send_backward").start() @@ -595,48 +710,60 @@ def send_backward(self, input_tensor_grad, pp_first_stage): recv_prev=False, recv_next=False, send_recv_meta=self._send_recv_meta, + batch_p2p_comm=batch_p2p_comm, ) if _timers is not None: _timers("send_backward").stop() - def send_forward_recv_backward(self, output_tensor, pp_last_stage): + def send_forward_recv_backward( + self, output_tensor, pp_last_stage, batch_p2p_comm=True + ): global _timers if _timers is not None: _timers("send_forward_recv_backward").start() if pp_last_stage: output_tensor_grad = None else: - _, output_tensor_grad = _p2p_helper( + _, output_tensor_grad, _ = _p2p_helper( tensor_send_next=output_tensor, tensor_send_prev=None, recv_prev=False, recv_next=True, send_recv_meta=self._send_recv_meta, + batch_p2p_comm=batch_p2p_comm, ) if _timers is not None: _timers("send_forward_recv_backward").stop() return output_tensor_grad - def send_backward_recv_forward(self, input_tensor_grad, pp_first_stage): + def send_backward_recv_forward( + self, input_tensor_grad, pp_first_stage, batch_p2p_comm=True + ): global _timers if _timers is not None: _timers("send_backward_recv_forward").start() if pp_first_stage: input_tensor = None else: - input_tensor, _ = _p2p_helper( + input_tensor, _, _ = _p2p_helper( tensor_send_next=None, tensor_send_prev=input_tensor_grad, recv_prev=True, recv_next=False, send_recv_meta=self._send_recv_meta, + batch_p2p_comm=batch_p2p_comm, ) if _timers is not None: _timers("send_backward_recv_forward").stop() return input_tensor def send_forward_backward_recv_forward_backward( - self, output_tensor, input_tensor_grad, recv_prev, recv_next + self, + output_tensor, + input_tensor_grad, + recv_prev, + recv_next, + batch_p2p_comm=True, ): # always have to send dtype info to downstream global _timers @@ -648,19 +775,26 @@ def send_forward_backward_recv_forward_backward( if recv_prev: self._recv_meta() - input_tensor, output_tensor_grad = _p2p_helper( + input_tensor, output_tensor_grad, _ = _p2p_helper( tensor_send_next=output_tensor, tensor_send_prev=input_tensor_grad, recv_prev=recv_prev, recv_next=recv_next, sync_recv=False, send_recv_meta=self._send_recv_meta, + batch_p2p_comm=batch_p2p_comm, ) if _timers is not None: _timers("send_forward_backward_recv_forward_backward").stop() return input_tensor, output_tensor_grad - def send_forward_recv_forward(self, output_tensor, recv_prev): + def send_forward_recv_forward( + self, + output_tensor, + recv_prev, + batch_p2p_comm=True, + overlap_p2p_comm=False, + ): # always have to send dtype info to downstream global _timers if _timers is not None: @@ -672,32 +806,48 @@ def send_forward_recv_forward(self, output_tensor, recv_prev): if recv_prev: self._recv_meta() - input_tensor, _ = _p2p_helper( + input_tensor, _, wait_handles = _p2p_helper( tensor_send_next=output_tensor, tensor_send_prev=None, recv_prev=recv_prev, recv_next=False, sync_recv=False, send_recv_meta=self._send_recv_meta, + batch_p2p_comm=batch_p2p_comm, + wait_on_reqs=(not overlap_p2p_comm), ) if _timers is not None: _timers("send_forward_recv_forward").stop() + + if overlap_p2p_comm: + return input_tensor, wait_handles return input_tensor - def send_backward_recv_backward(self, input_tensor_grad, recv_next): + def send_backward_recv_backward( + self, + input_tensor_grad, + recv_next, + batch_p2p_comm=True, + overlap_p2p_comm=False, + ): global _timers if _timers is not None: _timers("send_backward_recv_backward").start() - _, output_tensor_grad = _p2p_helper( + _, output_tensor_grad, wait_handles = _p2p_helper( tensor_send_next=None, tensor_send_prev=input_tensor_grad, recv_prev=False, recv_next=recv_next, sync_recv=False, send_recv_meta=self._send_recv_meta, + batch_p2p_comm=batch_p2p_comm, + wait_on_reqs=(not overlap_p2p_comm), ) if _timers is not None: _timers("send_backward_recv_backward").stop() + + if overlap_p2p_comm: + return output_tensor_grad, wait_handles return output_tensor_grad def __repr__(self): From 89902e0db4ee704f29815d40bc6d151c87abfa71 Mon Sep 17 00:00:00 2001 From: lanxianghit <47554610+lanxianghit@users.noreply.github.com> Date: Mon, 26 Feb 2024 10:10:53 +0800 Subject: [PATCH 051/282] [PIR][DynamicShape] Add InferSymbolicShape for if op (#61937) * Add InferSymbolicShape for if op --- paddle/fluid/pir/transforms/shape_optimization_pass.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/pir/transforms/shape_optimization_pass.cc b/paddle/fluid/pir/transforms/shape_optimization_pass.cc index 69377af0d30b5..80d56f75ae12b 100644 --- a/paddle/fluid/pir/transforms/shape_optimization_pass.cc +++ b/paddle/fluid/pir/transforms/shape_optimization_pass.cc @@ -128,8 +128,9 @@ void InferSymExprForBlock(const Block& block, op.dyn_cast(); if (infer_symbolic_shape_interface) { VLOG(vlog_level) << op.name() << " has InferSymbolicShapeInterface."; - PADDLE_ENFORCE( + PADDLE_ENFORCE_EQ( infer_symbolic_shape_interface.InferSymbolicShape(shape_analysis), + true, "InferSymbolicShape for %s failed.", op.name()); if (op.num_results() > 0) { From 60405397acf3e9b7de3d94556f06db7f97d7f19d Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 26 Feb 2024 10:23:46 +0800 Subject: [PATCH 052/282] Fix GetErrorSumaryString (#61997) --- paddle/cinn/utils/error.h | 8 ++++---- paddle/common/enforce.h | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/paddle/cinn/utils/error.h b/paddle/cinn/utils/error.h index b0cd09f0c2b0e..7b5af324d7081 100644 --- a/paddle/cinn/utils/error.h +++ b/paddle/cinn/utils/error.h @@ -55,9 +55,9 @@ inline std::string demangle(std::string name) { inline std::string demangle(std::string name) { return name; } #endif -static std::string GetErrorSumaryString(const std::string& what, - const char* file, - int line) { +static std::string GetErrorSummaryString(const std::string& what, + const char* file, + int line) { std::ostringstream sout; sout << "\n----------------------\nError Message " "Summary:\n----------------------\n"; @@ -99,7 +99,7 @@ static std::string GetCurrentTraceBackString() { static std::string GetTraceBackString(const std::string& what, const char* file, int line) { - return GetCurrentTraceBackString() + GetErrorSumaryString(what, file, line); + return GetCurrentTraceBackString() + GetErrorSummaryString(what, file, line); } struct EnforceNotMet : public std::exception { diff --git a/paddle/common/enforce.h b/paddle/common/enforce.h index e7584ee98be23..856cf28d0221a 100644 --- a/paddle/common/enforce.h +++ b/paddle/common/enforce.h @@ -118,9 +118,9 @@ TEST_API int GetCallStackLevel(); TEST_API std::string SimplifyErrorTypeFormat(const std::string& str); TEST_API std::string GetCurrentTraceBackString(bool for_signal = false); template -static std::string GetErrorSumaryString(StrType&& what, - const char* file, - int line) { +static std::string GetErrorSummaryString(StrType&& what, + const char* file, + int line) { std::ostringstream sout; if (GetCallStackLevel() > 1) { sout << "\n----------------------\nError Message " @@ -139,9 +139,9 @@ static std::string GetTraceBackString(StrType&& what, if (GetCallStackLevel() > 1) { // FLAGS_call_stack_level>1 means showing c++ call stack return ::common::enforce::GetCurrentTraceBackString() + - GetErrorSumaryString(what, file, line); + GetErrorSummaryString(what, file, line); } else { - return GetErrorSumaryString(what, file, line); + return GetErrorSummaryString(what, file, line); } } From 4daec8acf8bbb052bb7ece2aa8f4051dcbaac723 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=98=A5=E4=B9=94?= <83450930+Liyulingyue@users.noreply.github.com> Date: Mon, 26 Feb 2024 10:27:30 +0800 Subject: [PATCH 053/282] =?UTF-8?q?=E3=80=90paddle=5Ftest=20No.40=E3=80=91?= =?UTF-8?q?replace=20of=20cc=5Ftest=20with=20paddle=5Ftest=20(#61945)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update CMakeLists.txt * mv cc file * add TEST_API * delete use_op_itself * Update test_reference_count_pass_last_lived_ops.cc * Update CMakeLists.txt --- .../ir/memory_optimize_pass/CMakeLists.txt | 11 - ...est_reference_count_pass_last_lived_ops.cc | 7 - paddle/fluid/framework/parallel_executor.h | 20 +- test/cpp/fluid/framework/CMakeLists.txt | 2 + test/cpp/fluid/framework/ir/CMakeLists.txt | 1 + .../ir/memory_optimize_pass/CMakeLists.txt | 14 ++ .../share_varinfo_into_cinn_pass_test.cc | 154 ++++++++++++ ...est_reference_count_pass_last_lived_ops.cc | 228 ++++++++++++++++++ 8 files changed, 409 insertions(+), 28 deletions(-) create mode 100644 test/cpp/fluid/framework/ir/CMakeLists.txt create mode 100644 test/cpp/fluid/framework/ir/memory_optimize_pass/CMakeLists.txt create mode 100644 test/cpp/fluid/framework/ir/memory_optimize_pass/share_varinfo_into_cinn_pass_test.cc create mode 100644 test/cpp/fluid/framework/ir/memory_optimize_pass/test_reference_count_pass_last_lived_ops.cc diff --git a/paddle/fluid/framework/ir/memory_optimize_pass/CMakeLists.txt b/paddle/fluid/framework/ir/memory_optimize_pass/CMakeLists.txt index 85923aafc23a7..222fef33c5ea6 100644 --- a/paddle/fluid/framework/ir/memory_optimize_pass/CMakeLists.txt +++ b/paddle/fluid/framework/ir/memory_optimize_pass/CMakeLists.txt @@ -41,11 +41,6 @@ if(WITH_CINN) share_varinfo_into_cinn_pass SRCS share_varinfo_into_cinn_pass.cc DEPS pass enforce common graph_helper) - cc_test( - share_varinfo_into_cinn_pass_test - SRCS share_varinfo_into_cinn_pass_test.cc - DEPS share_varinfo_into_cinn_pass parallel_executor elementwise_add_op - mul_op cinn_launch_op) list(APPEND EAGER_DELETETION_PASS_DEPS share_varinfo_into_cinn_pass) endif() @@ -72,9 +67,3 @@ cc_library( inplace_addto_op_pass SRCS inplace_addto_op_pass.cc DEPS memory_reuse_pass) - -cc_test( - test_reference_count_pass_last_lived_ops - SRCS test_reference_count_pass_last_lived_ops.cc - DEPS parallel_executor elementwise_mul_op elementwise_add_op generated_op phi - common) diff --git a/paddle/fluid/framework/ir/memory_optimize_pass/test_reference_count_pass_last_lived_ops.cc b/paddle/fluid/framework/ir/memory_optimize_pass/test_reference_count_pass_last_lived_ops.cc index fc2173f36316d..eeec6fd8788d4 100644 --- a/paddle/fluid/framework/ir/memory_optimize_pass/test_reference_count_pass_last_lived_ops.cc +++ b/paddle/fluid/framework/ir/memory_optimize_pass/test_reference_count_pass_last_lived_ops.cc @@ -23,13 +23,6 @@ #include "paddle/fluid/framework/program_desc.h" #include "paddle/phi/core/kernel_registry.h" -USE_OP_ITSELF(scale); -USE_OP_ITSELF(elementwise_mul); -USE_OP_ITSELF(elementwise_add); -USE_OP_ITSELF(elementwise_add_grad); - -PD_DECLARE_KERNEL(scale, CPU, ALL_LAYOUT); - COMMON_DECLARE_double(eager_delete_tensor_gb); namespace paddle { diff --git a/paddle/fluid/framework/parallel_executor.h b/paddle/fluid/framework/parallel_executor.h index 32514089763c6..29df757d17c8a 100644 --- a/paddle/fluid/framework/parallel_executor.h +++ b/paddle/fluid/framework/parallel_executor.h @@ -52,14 +52,14 @@ class ParallelExecutor { DISABLE_COPY_AND_ASSIGN(ParallelExecutor); public: - explicit ParallelExecutor(const std::vector &places, - const std::vector &bcast_vars, - const std::string &loss_var_name, - Scope *scope, - const std::vector &local_scopes, - const ExecutionStrategy &exec_strategy, - const BuildStrategy &build_strategy, - ir::Graph *graph); + TEST_API explicit ParallelExecutor(const std::vector &places, + const std::vector &bcast_vars, + const std::string &loss_var_name, + Scope *scope, + const std::vector &local_scopes, + const ExecutionStrategy &exec_strategy, + const BuildStrategy &build_strategy, + ir::Graph *graph); // NOTE(Aurelius84): Construct a PE running on single device for @to_static explicit ParallelExecutor(const platform::Place &place, @@ -68,7 +68,7 @@ class ParallelExecutor { const BuildStrategy &build_strategy, ir::Graph *graph); - ~ParallelExecutor(); + TEST_API ~ParallelExecutor(); size_t DeviceCount() const; @@ -98,7 +98,7 @@ class ParallelExecutor { void ResetOpHandleScopeMapOfGraphs( const std::unordered_map &scope_map); - const ir::Graph &Graph() const; + TEST_API const ir::Graph &Graph() const; void PrepareVariables(Scope *scope); void SkipMemoryReuse(size_t scope_idx, diff --git a/test/cpp/fluid/framework/CMakeLists.txt b/test/cpp/fluid/framework/CMakeLists.txt index 5e0e7404f6999..8e1686b242993 100644 --- a/test/cpp/fluid/framework/CMakeLists.txt +++ b/test/cpp/fluid/framework/CMakeLists.txt @@ -346,3 +346,5 @@ cc_test( workqueue_test SRCS new_executor/workqueue_test.cc DEPS standalone_executor) + +add_subdirectory(ir) diff --git a/test/cpp/fluid/framework/ir/CMakeLists.txt b/test/cpp/fluid/framework/ir/CMakeLists.txt new file mode 100644 index 0000000000000..81a68ccb22f83 --- /dev/null +++ b/test/cpp/fluid/framework/ir/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(memory_optimize_pass) diff --git a/test/cpp/fluid/framework/ir/memory_optimize_pass/CMakeLists.txt b/test/cpp/fluid/framework/ir/memory_optimize_pass/CMakeLists.txt new file mode 100644 index 0000000000000..841ebd7c0fcc0 --- /dev/null +++ b/test/cpp/fluid/framework/ir/memory_optimize_pass/CMakeLists.txt @@ -0,0 +1,14 @@ +if(WITH_CINN) + paddle_test(share_varinfo_into_cinn_pass_test SRCS + share_varinfo_into_cinn_pass_test.cc) + list(APPEND EAGER_DELETETION_PASS_DEPS share_varinfo_into_cinn_pass) +endif() + +paddle_test(test_reference_count_pass_last_lived_ops SRCS + test_reference_count_pass_last_lived_ops.cc DEPS common) + +if(WITH_ONNXRUNTIME AND WIN32) + # Copy onnxruntime for some c++ test in Windows, since the test will + # be build only in CI, so suppose the generator in Windows is Ninja. + copy_onnx(test_reference_count_pass_last_lived_ops) +endif() diff --git a/test/cpp/fluid/framework/ir/memory_optimize_pass/share_varinfo_into_cinn_pass_test.cc b/test/cpp/fluid/framework/ir/memory_optimize_pass/share_varinfo_into_cinn_pass_test.cc new file mode 100644 index 0000000000000..1f78e293a21a3 --- /dev/null +++ b/test/cpp/fluid/framework/ir/memory_optimize_pass/share_varinfo_into_cinn_pass_test.cc @@ -0,0 +1,154 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "gtest/gtest.h" +#include "paddle/fluid/framework/details/computation_op_handle.h" +#include "paddle/fluid/framework/details/eager_deletion_op_handle.h" +#include "paddle/fluid/framework/ir/graph.h" +#include "paddle/fluid/framework/ir/memory_optimize_pass/memory_optimization_var_info.h" +#include "paddle/fluid/framework/ir/pass.h" +#include "paddle/fluid/framework/paddle2cinn/build_cinn_pass.h" +#include "paddle/fluid/framework/paddle2cinn/cinn_compiler.h" +#include "paddle/fluid/framework/parallel_executor.h" +#include "paddle/fluid/framework/program_desc.h" + +USE_OP_ITSELF(mul); +USE_OP_ITSELF(elementwise_add); + +USE_OP_ITSELF(cinn_launch); +PD_DECLARE_KERNEL(cinn_launch, CPU, ALL_LAYOUT); +#ifdef PADDLE_WITH_CUDA +PD_DECLARE_KERNEL(cinn_launch, GPU, ALL_LAYOUT); +#endif + +namespace paddle::framework { + +using Name2VarInfoMap = + std::unordered_map>; + +static ProgramDesc BuildProgramInsideCinnLaunchOp() { + ProgramDesc program; + auto* block = program.MutableBlock(0); + block->Var("var1"); + block->Var("var2"); + block->Var("var3"); + block->Var("var4"); + block->Var("var5"); + + auto add_op = + std::unique_ptr(new OpDesc("elementwise_add", + {{"X", {"var1"}}, {"Y", {"var2"}}}, + {{"Out", {"var3"}}}, + {})); + block->AppendAllocatedOp(std::move(add_op)); + auto mul_op = std::unique_ptr(new OpDesc( + "mul", {{"X", {"var3"}}, {"Y", {"var4"}}}, {{"Out", {"var5"}}}, {})); + block->AppendAllocatedOp(std::move(mul_op)); + return program; +} + +static ProgramDesc BuildProgramWithCinnLaunchOp(int64_t compilation_key) { + // create a cinn_launch op + ProgramDesc program; + auto* block = program.MutableBlock(0); + block->Var("var1"); + block->Var("var2"); + block->Var("var4"); + block->Var("var5"); + + auto cinn_launch_op = std::unique_ptr( + new OpDesc("cinn_launch", + {{"X", {"var1", "var2", "var4"}}}, + {{"Out", {"var5"}}}, + {{"compilation_key", compilation_key}})); + block->AppendAllocatedOp(std::move(cinn_launch_op)); + return program; +} + +struct TestPassContext { + explicit TestPassContext(const ProgramDesc& program) { + graph = std::make_unique(program); + details::BuildStrategy build_strategy; + details::ExecutionStrategy exec_strategy; + exec_strategy.use_device_ = paddle::platform::kCUDA; + executor.reset(new ParallelExecutor(platform::CUDAPlace(0), + &scope, + exec_strategy, + build_strategy, + graph.get())); + } + + Scope scope; + std::unique_ptr graph; + std::unique_ptr executor; +}; + +TEST(ShareMemInfoToSubGraphPassTest, test_main_graph_share_varinfo) { + // add a subgraph to CinnCompiler + auto subgraph = std::make_unique(BuildProgramInsideCinnLaunchOp()); + subgraph->GetOrInit( + paddle2cinn::kMemOptVarInfoFromMainGraph); + auto compilation_key = + paddle2cinn::CinnCompiler::GetInstance()->AddGraph(std::move(subgraph)); + + // build test data and apply pass + auto context = std::make_unique( + BuildProgramWithCinnLaunchOp(compilation_key)); + + // check result + const ir::Graph& result_subgraph = + paddle2cinn::CinnCompiler::GetInstance()->FindGraph(compilation_key); + const auto& dst_varinfo_map = result_subgraph.Get( + paddle2cinn::kMemOptVarInfoFromMainGraph); + ASSERT_EQ(dst_varinfo_map.size(), 4); + EXPECT_EQ(dst_varinfo_map.count("var1"), 1); + EXPECT_EQ(dst_varinfo_map.count("var5"), 1); + EXPECT_EQ(dst_varinfo_map.at("var1").use_count(), 2); + EXPECT_EQ(dst_varinfo_map.at("var5").use_count(), 2); +} + +TEST(ShareMemInfoToSubGraphPassTest, test_subgraph_take_varinfo) { + // build test data and apply pass + auto context = + std::make_unique(BuildProgramInsideCinnLaunchOp()); + auto& varinfo_map_shared = context->graph->GetOrInit( + paddle2cinn::kMemOptVarInfoFromMainGraph); + varinfo_map_shared = { + {"var1", std::make_shared("var1", 1)}, + {"var2", std::make_shared("var2", 2)}, + }; + + ir::MemOptVarInfoMapList varinfo_maps(1); + auto& dst_varinfo_map = varinfo_maps.front(); + dst_varinfo_map = {{"var1", std::make_shared("var1", 1)}, + {"var2", std::make_shared("var2", 1)}, + {"var3", std::make_shared("var3", 1)}, + {"var4", std::make_shared("var4", 1)}, + {"var5", std::make_shared("var5", 1)}}; + auto share_pass = + ir::PassRegistry::Instance().Get("share_varinfo_into_cinn_pass"); + share_pass->SetNotOwned(ir::kMemOptVarInfoMapList, &varinfo_maps); + share_pass->Apply(context->graph.get()); + + // check result + ASSERT_NE(dst_varinfo_map.at("var1")->ParentHolder(), nullptr); + ASSERT_NE(dst_varinfo_map.at("var2")->ParentHolder(), nullptr); + ASSERT_EQ(dst_varinfo_map.at("var3")->ParentHolder(), nullptr); + ASSERT_EQ(dst_varinfo_map.at("var4")->ParentHolder(), nullptr); + ASSERT_EQ(dst_varinfo_map.at("var5")->ParentHolder(), nullptr); +} + +} // namespace paddle::framework diff --git a/test/cpp/fluid/framework/ir/memory_optimize_pass/test_reference_count_pass_last_lived_ops.cc b/test/cpp/fluid/framework/ir/memory_optimize_pass/test_reference_count_pass_last_lived_ops.cc new file mode 100644 index 0000000000000..eeec6fd8788d4 --- /dev/null +++ b/test/cpp/fluid/framework/ir/memory_optimize_pass/test_reference_count_pass_last_lived_ops.cc @@ -0,0 +1,228 @@ +// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gtest/gtest.h" +#include "paddle/common/flags.h" +#include "paddle/fluid/framework/details/multi_devices_helper.h" +#include "paddle/fluid/framework/ir/graph.h" +#include "paddle/fluid/framework/ir/graph_helper.h" +#include "paddle/fluid/framework/ir/memory_optimize_pass/memory_optimization_var_info.h" +#include "paddle/fluid/framework/ir/memory_optimize_pass/reference_count_pass_helper.h" +#include "paddle/fluid/framework/parallel_executor.h" +#include "paddle/fluid/framework/program_desc.h" +#include "paddle/phi/core/kernel_registry.h" + +COMMON_DECLARE_double(eager_delete_tensor_gb); + +namespace paddle { +namespace framework { +namespace p = paddle::platform; + +static std::vector CreatePlaces(size_t num, bool use_cuda) { + std::vector result; + result.reserve(num); + for (size_t i = 0; i < num; ++i) { + if (use_cuda) { + result.emplace_back(platform::CUDAPlace(static_cast(i))); + } else { + result.emplace_back(platform::CPUPlace()); + } + } + return result; +} + +static void NewVar(BlockDesc *block, + const std::string &name, + const std::vector &shape) { + auto *var_desc = block->Var(name); + var_desc->SetShape(shape); +} + +static void AppendOp(BlockDesc *block, + const std::string &type, + VariableNameMap inputs, + VariableNameMap outputs, + AttributeMap attrs) { + auto &op_info = OpInfoMap::Instance().Get(type); + if (op_info.Checker()) { + op_info.Checker()->Check(&attrs); + } + + auto *op = block->AppendOp(); + op->SetType(type); + for (auto &pair : inputs) { + op->SetInput(pair.first, pair.second); + } + + for (auto &pair : outputs) { + op->SetOutput(pair.first, pair.second); + for (auto &var_name : pair.second) { + if (!block->FindVarRecursive(var_name)) { + NewVar(block, var_name, {}); + } + } + } + + op->SetAttrMap(attrs); + op->InferVarType(block); + op->InferShape(*block); +} + +class ReferenceCountPassTestHelper { + public: + ReferenceCountPassTestHelper(const ProgramDesc &program, bool use_cuda) + : graph_(program) { + details::BuildStrategy build_strategy; + build_strategy.enable_inplace_ = false; + build_strategy.memory_optimize_ = false; + FLAGS_eager_delete_tensor_gb = -1; + + details::ExecutionStrategy exec_strategy; + exec_strategy.use_device_ = use_cuda ? p::kCUDA : p::kCPU; + + executor_ = std::make_unique(CreatePlaces(1, use_cuda), + std::vector(), + "", + &scope_, + std::vector(), + exec_strategy, + build_strategy, + &graph_); + + auto ref_cnt_pass = + ir::PassRegistry::Instance().Get("reference_count_pass"); + ref_cnt_pass->SetNotOwned(ir::kMemOptVarInfoMapList, &mem_opt_var_infos_); + ref_cnt_pass->SetNotOwned(ir::kLastLiveOpsOfVars, &last_live_ops_of_vars_); + ref_cnt_pass->Apply(&const_cast(executor_->Graph())); + } + + bool IsLastLivedOps(const std::string &name, + std::vector ops) const { + std::sort(ops.begin(), ops.end()); + return LastLivedOpTypes(name) == ops; + } + + std::vector LastLivedOps(const std::string &name) const { + auto &ops = last_live_ops_of_vars_[0].at(name).ops(); + std::vector ret; + ret.reserve(ops.size()); + for (auto *op : ops) { + ret.emplace_back(op->GetOp()); + } + return ret; + } + + private: + std::vector LastLivedOpTypes(const std::string &name) const { + auto iter = last_live_ops_of_vars_[0].find(name); + std::vector ret; + if (iter != last_live_ops_of_vars_[0].end()) { + for (auto *op : iter->second.ops()) { + ret.emplace_back(op->GetOp()->Type()); + } + } + std::sort(ret.begin(), ret.end()); + return ret; + } + + private: + ir::Graph graph_; + Scope scope_; + std::unique_ptr executor_; + + ir::MemOptVarInfoMapList mem_opt_var_infos_; + std::vector last_live_ops_of_vars_; +}; + +TEST(test_reference_count_pass, test_no_need_buffer_var_shrink) { + ProgramDesc program; + auto *block = program.MutableBlock(0); + std::vector shape{{3, 4, 5}}; + + /** + * The network is: + * + * x0 = fluid.layer.data(...) + * x1 = scale(x0, scale=1) + * x2 = scale(x1, scale=2) + * x3 = elementwise_mul(x1, x2) + * scale(x3, out=x1, scale=3) # produce a new version of x1 + * x4, x5 = elementwise_add_grad(dout=x3, x=x2, y=x1) + * x6 = elementwise_mul(x4, x5) + * x7 = elementwise_add(x5, x5) + */ + std::string x0 = "x0"; + std::string x1 = "x1"; + std::string x2 = "x2"; + std::string x3 = "x3"; + std::string x4 = "x4"; + std::string x5 = "x5"; + std::string x6 = "x6"; + std::string x7 = "x7"; + + NewVar(block, x0, shape); + AppendOp(block, "scale", {{"X", {x0}}}, {{"Out", {x1}}}, {{"scale", 1.0f}}); + AppendOp(block, "scale", {{"X", {x1}}}, {{"Out", {x2}}}, {{"scale", 2.0f}}); + AppendOp(block, + "elementwise_mul", + {{"X", {x1}}, {"Y", {x2}}}, + {{"Out", {x3}}}, + {}); + AppendOp(block, "scale", {{"X", {x3}}}, {{"Out", {x1}}}, {{"scale", 3.0f}}); + AppendOp(block, + "elementwise_add_grad", + {{GradVarName("Out"), {x3}}, {"X", {x2}}, {"Y", {x1}}}, + {{GradVarName("X"), {x4}}, {GradVarName("Y"), {x5}}}, + {}); + AppendOp(block, + "elementwise_mul", + {{"X", {x4}}, {"Y", {x5}}}, + {{"Out", {x6}}}, + {}); + AppendOp(block, + "elementwise_add", + {{"X", {x5}}, {"Y", {x5}}}, + {{"Out", {x7}}}, + {}); + + std::vector use_cuda_list{false}; +#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) + use_cuda_list.push_back(true); +#endif + for (auto use_cuda : use_cuda_list) { + ReferenceCountPassTestHelper helper(program, use_cuda); + ASSERT_TRUE(helper.IsLastLivedOps(x0, {"scale"})); + ASSERT_EQ(PADDLE_GET_CONST(float, + helper.LastLivedOps(x0)[0]->Attrs().at("scale")), + 1.0f); + + ASSERT_TRUE(helper.IsLastLivedOps(x1, {"scale"})); + ASSERT_EQ(PADDLE_GET_CONST(float, + helper.LastLivedOps(x1)[0]->Attrs().at("scale")), + 3.0f); + + ASSERT_TRUE(helper.IsLastLivedOps(x2, {"elementwise_mul"})); + ASSERT_TRUE(helper.IsLastLivedOps(x3, {"elementwise_add_grad"})); + + ASSERT_TRUE(helper.IsLastLivedOps(x4, {"elementwise_mul"})); + ASSERT_TRUE( + helper.IsLastLivedOps(x5, {"elementwise_mul", "elementwise_add"})); + + ASSERT_TRUE(helper.IsLastLivedOps(x6, {"elementwise_mul"})); + ASSERT_TRUE(helper.IsLastLivedOps(x7, {"elementwise_add"})); + } +} + +} // namespace framework +} // namespace paddle From a6e926856b2ac84b8635ccbde66ae6c14d5a9cc7 Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 26 Feb 2024 10:29:13 +0800 Subject: [PATCH 054/282] Fix some typos (pendding_ops, altertively, etc.) (#61765) --- python/paddle/base/backward.py | 10 ++++----- python/paddle/base/executor.py | 6 ++--- .../incubate/checkpoint/auto_checkpoint.py | 2 +- python/paddle/nn/layer/layers.py | 22 +++++++++---------- test/dygraph_to_static/test_legacy_error.py | 2 +- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/python/paddle/base/backward.py b/python/paddle/base/backward.py index 75e5899afdece..9f39d9c3ea03f 100755 --- a/python/paddle/base/backward.py +++ b/python/paddle/base/backward.py @@ -830,7 +830,7 @@ class Var: def __init__(self, var_name): self.var_name = var_name self.gen_op = None - self.pendding_ops = [] + self.pending_ops = [] def set_gen_op(self, gen_op): assert isinstance(gen_op, Op) @@ -839,7 +839,7 @@ def set_gen_op(self, gen_op): def add_pending_op(self, op): assert isinstance(op, Op) - self.pendding_ops.append(op) + self.pending_ops.append(op) class Op: def __init__(self, op_desc): @@ -916,8 +916,8 @@ def _create_op_node(op_desc): op_node = candidate_ops.pop(0) if _all_in_set_(op_node.inputs, ready_vars): for out_var in op_node.outputs: - candidate_ops.extend(out_var.pendding_ops) - op_list.extend(out_var.pendding_ops) + candidate_ops.extend(out_var.pending_ops) + op_list.extend(out_var.pending_ops) ready_vars.update(op_node.outputs) else: remove_ops = False @@ -1571,7 +1571,7 @@ def find_op_index(block_desc, cur_op_desc): # NOTE: In primitive mode, the intermediate variable generated by # decompositing raw grad op are not satisfied the rule of 'XX@GRAD', # which will cause it be pruned according to current pruning logic. - # For simplicity, we treate all prmitive operators as one raw + # For simplicity, we treat all primitive operators as one raw # operator, and keep the pruning logic consistent with currently # logic. The drawback of this solution is may lead to some primitive # operators are not pruned, which is needed to fixed. diff --git a/python/paddle/base/executor.py b/python/paddle/base/executor.py index 6fe3f71d481ca..3162d27e05059 100755 --- a/python/paddle/base/executor.py +++ b/python/paddle/base/executor.py @@ -739,7 +739,7 @@ def _as_lodtensor(data, place, dtype=None): data = np.array(data) if data.dtype == np.object_: raise TypeError( - "\n\tFaild to convert input data to a regular ndarray :\n\t* Usually " + "\n\tFailed to convert input data to a regular ndarray :\n\t* Usually " "this means the input data contains nested lists with different lengths. " "Please consider using 'base.create_lod_tensor' to convert it to a LoD-Tensor." ) @@ -1675,7 +1675,7 @@ def run( needed to generate :code:`fetch_list` will be pruned. The default is False, which means the program will not pruned and all the operators and variables will be executed during running. Note that if the tuple returned from :code:`Optimizer.minimize()` is passed to :code:`fetch_list`, - :code:`use_prune` will be overrided to True, and the program will be pruned. + :code:`use_prune` will be overridden to True, and the program will be pruned. Returns: @@ -1880,7 +1880,7 @@ def _run_impl( if scope is None: scope = global_scope() - # use_prune can be overrided by putting optimize_ops in fetch_list + # use_prune can be overridden by putting optimize_ops in fetch_list _origin_fetch_list = fetch_list _origin_program = program fetch_list, optimize_ops = self._split_optimize_ops_in_fetch_list( diff --git a/python/paddle/base/incubate/checkpoint/auto_checkpoint.py b/python/paddle/base/incubate/checkpoint/auto_checkpoint.py index 9bf737fb055dc..742289acd27f1 100644 --- a/python/paddle/base/incubate/checkpoint/auto_checkpoint.py +++ b/python/paddle/base/incubate/checkpoint/auto_checkpoint.py @@ -293,7 +293,7 @@ def __init__( self._save_checkpoint_inter = self._checker.save_checkpoint_inter assert ( self._save_checkpoint_inter >= 0 - ), f"checkpointer:{self._save_checkpoint_inter} must >=0" + ), f"checkpoint inter:{self._save_checkpoint_inter} must >=0" self._last_checkpoint_time = time.time() self._load_cp_nos = None diff --git a/python/paddle/nn/layer/layers.py b/python/paddle/nn/layer/layers.py index 36810ba974d24..a4f20abb97c7f 100644 --- a/python/paddle/nn/layer/layers.py +++ b/python/paddle/nn/layer/layers.py @@ -302,7 +302,7 @@ def is_instance(self, param, cls): ) -class LayerOpsRecoder: +class LayerOpsRecorder: """ Record generated operators information in nn.Layer. """ @@ -405,7 +405,7 @@ def __init__(self, name_scope=None, dtype="float32"): self._loaddict_holder = collections.OrderedDict() # Record generated op_descs in this layer - self._op_recorder = LayerOpsRecoder(ops=[], hooks=[]) + self._op_recorder = LayerOpsRecorder(ops=[], hooks=[]) self._customized_attrs = {} self._forward_pre_hooks = collections.OrderedDict() @@ -636,7 +636,7 @@ def register_forward_post_hook(self, hook): >>> # the forward_post_hook change the output of the layer: output = output * 2 >>> def forward_post_hook(layer, input, output): - ... # user can use layer, input and output for information statistis tasks + ... # user can use layer, input and output for information statistics tasks ... ... # change the output ... return output * 2 @@ -690,7 +690,7 @@ def register_forward_pre_hook(self, hook): >>> # the forward_pre_hook change the input of the layer: input = input * 2 >>> def forward_pre_hook(layer, input): - ... # user can use layer and input for information statistis tasks + ... # user can use layer and input for information statistics tasks ... ... # change the input ... input_return = (input[0] * 2) @@ -998,7 +998,7 @@ def astype(self, dtype=None): return self else: raise ValueError( - "dtype value error, must be 'bfloat16', 'float16', 'float32', 'float64', 'int8', 'int16', 'int32', 'int64', 'uint8', 'complex64', 'complex128', 'bool', or paddle.dtype, numpy.dtype, but recieve " + "dtype value error, must be 'bfloat16', 'float16', 'float32', 'float64', 'int8', 'int16', 'int32', 'int64', 'uint8', 'complex64', 'complex128', 'bool', or paddle.dtype, numpy.dtype, but receive " + str(dtype) ) @@ -1951,7 +1951,7 @@ def to_static_state_dict( include_sublayers(bool, optional) : If true, also include the parameters and persistable buffers from sublayers. Default: True. use_hook(bool, optional) : If true, the operations contained in _state_dict_hooks will be appended to the destination. Default: True. - Retruns: + Returns: dict, a dict contains all the parameters and persistable buffers. Examples: @@ -1988,7 +1988,7 @@ def state_dict( include_sublayers(bool, optional) : If true, also include the parameters and persistable buffers from sublayers. Default: True. use_hook(bool, optional) : If true, the operations contained in _state_dict_hooks will be appended to the destination. Default: True. - Retruns: + Returns: dict: a dict contains all the parameters and persistable buffers. Examples: @@ -2049,7 +2049,7 @@ def _check_match(key, param): if len(state) != len(param): missing_keys.append(key) raise ValueError( - f"{key} receieves the length of {len(state)}, " + f"{key} receives the length of {len(state)}, " f"but the expected shape is {len(param)}" ) else: @@ -2126,7 +2126,7 @@ def _set_var(var, ndarray): _set_var(param, state) except ValueError as e: raise ValueError( - "This error might happens in dy2static, while calling 'set_state_dict' dynamicly in 'forward', which is not supported. If you only need call 'set_state_dict' once, move it to '__init__'." + "This error might happens in dy2static, while calling 'set_state_dict' dynamically in 'forward', which is not supported. If you only need call 'set_state_dict' once, move it to '__init__'." ) return missing_keys, unexpected_keys @@ -2230,7 +2230,7 @@ def _transform(self, t, device, dtype, blocking): if t.place.is_gpu_place(): # for gpu, minimum memory allocation unit is 256 bytes. size_dtype = core.size_of_dtype(dtype) - # Note(zhangbo): Paddle GPU minimum memory allocation unit is 256 bytes, waiting_alloc_memory will comput ‘t’ occupied memory space. + # Note(zhangbo): Paddle GPU minimum memory allocation unit is 256 bytes, waiting_alloc_memory will compute ‘t’ occupied memory space. # Coefficient 1.2 is used to avoid OOM that may occur in this critical state when the memory is just enough. waiting_alloc_memory = ( ((np.prod(t.shape) * size_dtype) / 256 + 1) * 256 * 1.2 @@ -2345,7 +2345,7 @@ def transform(t, device, dtype, blocking): def _startup_program(self): """ - Return starup program containing initialization operations of all parameters. + Return startup program containing initialization operations of all parameters. NOTE(dev): This is a very low level API and only for inner developer. """ diff --git a/test/dygraph_to_static/test_legacy_error.py b/test/dygraph_to_static/test_legacy_error.py index c1225d3b83f03..faa1d34adaddd 100644 --- a/test/dygraph_to_static/test_legacy_error.py +++ b/test/dygraph_to_static/test_legacy_error.py @@ -453,7 +453,7 @@ def test_set_state_dict_err(self): error_message = str(new_exception) self.assertIn( - "This error might happens in dy2static, while calling 'set_state_dict' dynamicly in 'forward', which is not supported. If you only need call 'set_state_dict' once, move it to '__init__'.", + "This error might happens in dy2static, while calling 'set_state_dict' dynamically in 'forward', which is not supported. If you only need call 'set_state_dict' once, move it to '__init__'.", error_message, ) From 735fd08419c64b6afb7a82ce1c58a30a66fcd1c3 Mon Sep 17 00:00:00 2001 From: zhengzhonghui Date: Mon, 26 Feb 2024 10:29:57 +0800 Subject: [PATCH 055/282] [AutoParallel] Add Global mesh and sub mesh reshard function (#61796) * add global and sub mesh reshard function * add unittest * use broadcast kernel * rm unused header * revert broadcast * polish code --- .../auto_parallel/reshard/CMakeLists.txt | 1 + .../global_and_sub_mesh_reshard_function.cc | 137 ++++++++++++++++++ .../global_and_sub_mesh_reshard_function.h | 49 +++++++ .../reshard/nd_mesh_reshard_function.cc | 7 +- .../reshard/reshard_function_registry.cc | 3 + .../auto_parallel/reshard/reshard_utils.cc | 37 +++++ .../auto_parallel/reshard/reshard_utils.h | 3 + .../reshard/s_to_s_reshard_function.cc | 6 + paddle/phi/kernels/cpu/broadcast_kernel.cc | 1 + paddle/phi/kernels/gpu/broadcast_kernel.cu | 2 + .../hybrid_strategy/CMakeLists.txt | 7 + ...mi_auto_parallel_2d_global_mesh_reshard.py | 65 +++++++++ ...mi_auto_parallel_3d_global_mesh_reshard.py | 72 +++++++++ .../test_global_mesh_reshard.py | 73 ++++++++++ .../hybrid_strategy/testslist.csv | 1 + 15 files changed, 462 insertions(+), 2 deletions(-) create mode 100644 paddle/phi/core/distributed/auto_parallel/reshard/global_and_sub_mesh_reshard_function.cc create mode 100644 paddle/phi/core/distributed/auto_parallel/reshard/global_and_sub_mesh_reshard_function.h create mode 100644 test/auto_parallel/hybrid_strategy/semi_auto_parallel_2d_global_mesh_reshard.py create mode 100644 test/auto_parallel/hybrid_strategy/semi_auto_parallel_3d_global_mesh_reshard.py create mode 100644 test/auto_parallel/hybrid_strategy/test_global_mesh_reshard.py diff --git a/paddle/phi/core/distributed/auto_parallel/reshard/CMakeLists.txt b/paddle/phi/core/distributed/auto_parallel/reshard/CMakeLists.txt index 133a8c01de9f4..9699c7b25eadf 100644 --- a/paddle/phi/core/distributed/auto_parallel/reshard/CMakeLists.txt +++ b/paddle/phi/core/distributed/auto_parallel/reshard/CMakeLists.txt @@ -14,4 +14,5 @@ collect_srcs( r_to_x_reshard_function.cc nd_mesh_reshard_function.cc same_status_reshard_function.cc + global_and_sub_mesh_reshard_function.cc reshard_function_registry.cc) diff --git a/paddle/phi/core/distributed/auto_parallel/reshard/global_and_sub_mesh_reshard_function.cc b/paddle/phi/core/distributed/auto_parallel/reshard/global_and_sub_mesh_reshard_function.cc new file mode 100644 index 0000000000000..8cb78b9c7719b --- /dev/null +++ b/paddle/phi/core/distributed/auto_parallel/reshard/global_and_sub_mesh_reshard_function.cc @@ -0,0 +1,137 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/core/distributed/auto_parallel/reshard/global_and_sub_mesh_reshard_function.h" + +#include "glog/logging.h" +#include "paddle/phi/core/distributed/auto_parallel/dist_attr.h" +#include "paddle/phi/core/distributed/auto_parallel/dist_tensor.h" +#include "paddle/phi/core/distributed/auto_parallel/reshard/reshard_utils.h" +#include "paddle/phi/core/distributed/store/store_utils.h" +#include "paddle/phi/kernels/p_recv_kernel.h" +#include "paddle/phi/kernels/p_send_kernel.h" + +namespace phi { +namespace distributed { + +bool GlobalToSubMeshReshardFunction::IsSuitable( + const DistTensor& in, const TensorDistAttr& out_dist_attr) { + const TensorDistAttr& in_dist_attr = in.dist_attr(); + // 1. first dimension(pp) must be replicated + RESHARD_SHORTCUT_IF_FALSE(in_dist_attr.is_replicated(0)); + // 2. out mesh is the value of a certain dimension of global mesh + // e.g. global_mesh = [[1, 2], [3, 4]], out_mesh = [1, 2] or [3, 4] + // global_mesh = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]] + // out_mesh = [[1, 2], [3, 4]] or [[5, 6], [7, 8]] + + const ProcessMesh& in_process_mesh = in_dist_attr.process_mesh(); + const ProcessMesh& out_process_mesh = out_dist_attr.process_mesh(); + + RESHARD_SHORTCUT_IF_FALSE(in_process_mesh.ndim() == + out_process_mesh.ndim() + 1); + + return IsSubMesh(in_process_mesh, out_process_mesh); +} + +void GlobalToSubMeshReshardFunction::Eval(phi::DeviceContext* dev_ctx, + const DistTensor& in, + const TensorDistAttr& out_dist_attr, + DistTensor* out) { + VLOG(3) << "Call GlobalToSubMeshReshardFunction Eval"; + const DenseTensor& in_dense_value = in.value(); + const ProcessMesh& out_process_mesh = out_dist_attr.process_mesh(); + if (IsCurRankInMesh(out_process_mesh)) { + SetValue(out, in_dense_value); + } else { + *(out->unsafe_mutable_value()) = + phi::DenseTensor(std::make_shared( + nullptr, 0, phi::distributed::GetDefaultPlace()), + in.value().meta()); + } + SetDistProps(out, in.dims(), out_dist_attr); +} + +bool SubMeshToGlobalReshardFunction::IsSuitable( + const DistTensor& in, const TensorDistAttr& out_dist_attr) { + RESHARD_SHORTCUT_IF_FALSE(out_dist_attr.is_replicated(0)); + + const TensorDistAttr& in_dist_attr = in.dist_attr(); + const ProcessMesh& in_process_mesh = in_dist_attr.process_mesh(); + const ProcessMesh& out_process_mesh = out_dist_attr.process_mesh(); + + RESHARD_SHORTCUT_IF_FALSE(in_process_mesh.ndim() == + out_process_mesh.ndim() - 1); + + return IsSubMesh(out_process_mesh, in_process_mesh); +} + +void SubMeshToGlobalReshardFunction::Eval(phi::DeviceContext* dev_ctx, + const DistTensor& in, + const TensorDistAttr& out_dist_attr, + DistTensor* out) { + VLOG(3) << "Call SubMeshToGlobalReshardFunction Eval"; + const TensorDistAttr& in_dist_attr = in.dist_attr(); + const ProcessMesh& in_process_mesh = in_dist_attr.process_mesh(); + const ProcessMesh& out_process_mesh = out_dist_attr.process_mesh(); + + std::vector sub_process_meshes = GetSubMeshes(out_process_mesh); + const std::vector& in_process_ids = in_process_mesh.process_ids(); + const std::vector& out_process_ids = out_process_mesh.process_ids(); + std::unordered_map> send2recv_map; + std::unordered_map recv2send_map; + + for (const ProcessMesh& sub_mesh : sub_process_meshes) { + if (sub_mesh == in_process_mesh) { + continue; + } + const std::vector& sub_process_ids = sub_mesh.process_ids(); + for (size_t i = 0; i < sub_process_ids.size(); ++i) { + int64_t send_id = in_process_ids[i]; + send2recv_map[send_id].push_back(sub_process_ids[i]); + recv2send_map[sub_process_ids[i]] = send_id; + } + } + + std::vector all_process_ids = + GetUnionProcessIds(in_process_ids, out_process_ids); + int64_t cur_global_rank = GetCurGlobalRank(); + DataType dtype = in.dtype(); + if (IsCurRankInMesh(in_process_mesh)) { + const DenseTensor& in_dense_value = in.value(); + std::vector& recv_vec = send2recv_map[cur_global_rank]; + for (int64_t recv_id : recv_vec) { + RESHARD_FUNCTOR_WITH_COMM(dev_ctx, + PSendKernel, + dtype, + all_process_ids, + in_dense_value, + recv_id, + true /*dynamic_shape*/); + } + SetValue(out, in_dense_value); + } else { + int64_t send_id = recv2send_map[cur_global_rank]; + RESHARD_FUNCTOR_WITH_COMM(dev_ctx, + PRecv, + dtype, + all_process_ids, + send_id, + true /*dynamic_shape*/, + GetMutableTensor(out)); + } + SetDistProps(out, in.dims(), out_dist_attr); +} + +} // namespace distributed +} // namespace phi diff --git a/paddle/phi/core/distributed/auto_parallel/reshard/global_and_sub_mesh_reshard_function.h b/paddle/phi/core/distributed/auto_parallel/reshard/global_and_sub_mesh_reshard_function.h new file mode 100644 index 0000000000000..e93a454520ff3 --- /dev/null +++ b/paddle/phi/core/distributed/auto_parallel/reshard/global_and_sub_mesh_reshard_function.h @@ -0,0 +1,49 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/phi/core/distributed/auto_parallel/reshard/reshard_function.h" + +namespace phi { +namespace distributed { + +class GlobalToSubMeshReshardFunction final : public ReshardFunction { + public: + bool IsSuitable(const DistTensor& in, + const TensorDistAttr& out_dist_attr) override; + + void Eval(DeviceContext* dev_ctx, + const DistTensor& in, + const TensorDistAttr& out_dist_attr, + DistTensor* out) override; + + std::string Name() override { return "GlobalToSubMeshReshardFunction"; } +}; + +class SubMeshToGlobalReshardFunction final : public ReshardFunction { + public: + bool IsSuitable(const DistTensor& in, + const TensorDistAttr& out_dist_attr) override; + + void Eval(DeviceContext* dev_ctx, + const DistTensor& in, + const TensorDistAttr& out_dist_attr, + DistTensor* out) override; + + std::string Name() override { return "SubMeshToGlobalReshardFunction"; } +}; + +} // namespace distributed +} // namespace phi diff --git a/paddle/phi/core/distributed/auto_parallel/reshard/nd_mesh_reshard_function.cc b/paddle/phi/core/distributed/auto_parallel/reshard/nd_mesh_reshard_function.cc index 82ddfc6354934..b7a6679590e63 100644 --- a/paddle/phi/core/distributed/auto_parallel/reshard/nd_mesh_reshard_function.cc +++ b/paddle/phi/core/distributed/auto_parallel/reshard/nd_mesh_reshard_function.cc @@ -273,8 +273,11 @@ void SameNdMeshReshardFunction::Eval(phi::DeviceContext* dev_ctx, bool CrossNdMeshReshardFunction::IsSuitable( const DistTensor& in, const TensorDistAttr& out_dist_attr) { - RESHARD_SHORTCUT_IF_FALSE(in.dist_attr().process_mesh() != - out_dist_attr.process_mesh()); + const ProcessMesh& in_process_mesh = in.dist_attr().process_mesh(); + const ProcessMesh& out_process_mesh = out_dist_attr.process_mesh(); + RESHARD_SHORTCUT_IF_FALSE(in_process_mesh != out_process_mesh); + RESHARD_SHORTCUT_IF_FALSE(in_process_mesh.shape() == + out_process_mesh.shape()); RESHARD_SHORTCUT_IF_FALSE(out_dist_attr.process_mesh().ndim() > 1); // check the input and output dims_mapping is not equal diff --git a/paddle/phi/core/distributed/auto_parallel/reshard/reshard_function_registry.cc b/paddle/phi/core/distributed/auto_parallel/reshard/reshard_function_registry.cc index 3e7ad115999a2..eb2e8527e87c6 100644 --- a/paddle/phi/core/distributed/auto_parallel/reshard/reshard_function_registry.cc +++ b/paddle/phi/core/distributed/auto_parallel/reshard/reshard_function_registry.cc @@ -17,6 +17,7 @@ #include "glog/logging.h" #include "paddle/phi/core/distributed/auto_parallel/dist_tensor.h" +#include "paddle/phi/core/distributed/auto_parallel/reshard/global_and_sub_mesh_reshard_function.h" #include "paddle/phi/core/distributed/auto_parallel/reshard/nd_mesh_reshard_function.h" #include "paddle/phi/core/distributed/auto_parallel/reshard/p_to_r_reshard_function.h" #include "paddle/phi/core/distributed/auto_parallel/reshard/p_to_s_reshard_function.h" @@ -75,6 +76,8 @@ REGISTER_RESHARD_FUNC(RToXExpandReshardFunction); REGISTER_RESHARD_FUNC(SameStatusReshardFunction); REGISTER_RESHARD_FUNC(SameNdMeshReshardFunction); REGISTER_RESHARD_FUNC(CrossNdMeshReshardFunction); +REGISTER_RESHARD_FUNC(GlobalToSubMeshReshardFunction); +REGISTER_RESHARD_FUNC(SubMeshToGlobalReshardFunction); } // namespace distributed } // namespace phi diff --git a/paddle/phi/core/distributed/auto_parallel/reshard/reshard_utils.cc b/paddle/phi/core/distributed/auto_parallel/reshard/reshard_utils.cc index c7dd423d62e52..a2a769ef3a2d4 100644 --- a/paddle/phi/core/distributed/auto_parallel/reshard/reshard_utils.cc +++ b/paddle/phi/core/distributed/auto_parallel/reshard/reshard_utils.cc @@ -221,5 +221,42 @@ phi::DDim InferShapeForReshardFromReplicate( return out_dim; } +// 1. Get all the sub meshes of global_mesh +// e.g. global_mesh = [[1, 2], [3, 4]], out_mesh = [1, 2] and [3, 4] +// global_mesh = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]] +// out_mesh = [[1, 2], [3, 4]] and [[5, 6], [7, 8]] +std::vector GetSubMeshes(const ProcessMesh& process_mesh) { + const std::vector& shape = process_mesh.shape(); + const std::vector& process_ids = process_mesh.process_ids(); + const std::vector& dim_names = process_mesh.dim_names(); + int64_t total_process_num = process_ids.size(); + int64_t sub_process_num = total_process_num / shape[0]; + std::vector sub_process_mesh_shape(shape.begin() + 1, shape.end()); + std::vector sub_process_mesh_dim_names(dim_names.begin() + 1, + dim_names.end()); + + std::vector sub_process_meshes; + for (int i = 0; i < shape[0]; ++i) { + int64_t start_position = i * sub_process_num; + int64_t end_position = start_position + sub_process_num; + std::vector sub_process_ids(process_ids.begin() + start_position, + process_ids.begin() + end_position); + + sub_process_meshes.emplace_back(ProcessMesh( + sub_process_mesh_shape, sub_process_ids, sub_process_mesh_dim_names)); + } + return sub_process_meshes; +} + +bool IsSubMesh(const ProcessMesh& global_mesh, const ProcessMesh& sub_mesh) { + std::vector sub_process_meshes = GetSubMeshes(global_mesh); + for (const ProcessMesh& mesh : sub_process_meshes) { + if (mesh == sub_mesh) { + return true; + } + } + return false; +} + } // namespace distributed } // namespace phi diff --git a/paddle/phi/core/distributed/auto_parallel/reshard/reshard_utils.h b/paddle/phi/core/distributed/auto_parallel/reshard/reshard_utils.h index 5a63bef285825..8828222c4ceda 100644 --- a/paddle/phi/core/distributed/auto_parallel/reshard/reshard_utils.h +++ b/paddle/phi/core/distributed/auto_parallel/reshard/reshard_utils.h @@ -168,5 +168,8 @@ phi::DDim InferShapeForReshardFromReplicate( } \ } while (0) +std::vector GetSubMeshes(const ProcessMesh& process_mesh); +bool IsSubMesh(const ProcessMesh& global_mesh, const ProcessMesh& sub_mesh); + } // namespace distributed } // namespace phi diff --git a/paddle/phi/core/distributed/auto_parallel/reshard/s_to_s_reshard_function.cc b/paddle/phi/core/distributed/auto_parallel/reshard/s_to_s_reshard_function.cc index 4ff30a13cd033..57b5e8209fce6 100644 --- a/paddle/phi/core/distributed/auto_parallel/reshard/s_to_s_reshard_function.cc +++ b/paddle/phi/core/distributed/auto_parallel/reshard/s_to_s_reshard_function.cc @@ -32,6 +32,9 @@ bool SToSReshardFunction::IsSuitable(const DistTensor& in, const TensorDistAttr& out_dist_attr) { const auto& in_dist_attr = in.dist_attr(); + RESHARD_SHORTCUT_IF_FALSE(in_dist_attr.dims_mapping() != + out_dist_attr.dims_mapping()); + RESHARD_SHORTCUT_IF_FALSE(in_dist_attr.is_shard()); RESHARD_SHORTCUT_IF_FALSE(out_dist_attr.is_shard()); @@ -141,6 +144,9 @@ bool SToSReshardFunctionCrossMesh::IsSuitable( const DistTensor& in, const TensorDistAttr& out_dist_attr) { const auto& in_dist_attr = in.dist_attr(); + RESHARD_SHORTCUT_IF_FALSE(in_dist_attr.dims_mapping() != + out_dist_attr.dims_mapping()); + RESHARD_SHORTCUT_IF_FALSE(in_dist_attr.is_shard()); RESHARD_SHORTCUT_IF_FALSE(out_dist_attr.is_shard()); diff --git a/paddle/phi/kernels/cpu/broadcast_kernel.cc b/paddle/phi/kernels/cpu/broadcast_kernel.cc index 0deb8d8bbc562..02b984112d83c 100644 --- a/paddle/phi/kernels/cpu/broadcast_kernel.cc +++ b/paddle/phi/kernels/cpu/broadcast_kernel.cc @@ -61,6 +61,7 @@ PD_REGISTER_KERNEL(broadcast, bool, int8_t, uint8_t, + int16_t, int64_t, phi::dtype::float16, phi::dtype::complex, diff --git a/paddle/phi/kernels/gpu/broadcast_kernel.cu b/paddle/phi/kernels/gpu/broadcast_kernel.cu index e4986f752b1ae..9af8bd4d6d510 100644 --- a/paddle/phi/kernels/gpu/broadcast_kernel.cu +++ b/paddle/phi/kernels/gpu/broadcast_kernel.cu @@ -65,6 +65,7 @@ PD_REGISTER_KERNEL(broadcast, bool, int8_t, uint8_t, + int16_t, int64_t, phi::dtype::float16, phi::dtype::complex, @@ -80,6 +81,7 @@ PD_REGISTER_KERNEL(broadcast, bool, int8_t, uint8_t, + int16_t, int64_t, phi::dtype::float16, phi::dtype::complex, diff --git a/test/auto_parallel/hybrid_strategy/CMakeLists.txt b/test/auto_parallel/hybrid_strategy/CMakeLists.txt index 2d205031a433e..9d19c4e08b64d 100644 --- a/test/auto_parallel/hybrid_strategy/CMakeLists.txt +++ b/test/auto_parallel/hybrid_strategy/CMakeLists.txt @@ -58,3 +58,10 @@ if((WITH_GPU) AND (LINUX)) set_tests_properties(test_semi_auto_parallel_hybrid_sharding_strategy PROPERTIES TIMEOUT "120" LABELS "RUN_TYPE=HYBRID") endif() +if((WITH_GPU) AND (LINUX)) + py_test_modules( + test_global_mesh_reshard MODULES test_global_mesh_reshard ENVS + "http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python") + set_tests_properties(test_global_mesh_reshard PROPERTIES TIMEOUT "120" LABELS + "RUN_TYPE=HYBRID") +endif() diff --git a/test/auto_parallel/hybrid_strategy/semi_auto_parallel_2d_global_mesh_reshard.py b/test/auto_parallel/hybrid_strategy/semi_auto_parallel_2d_global_mesh_reshard.py new file mode 100644 index 0000000000000..93c26a767ccd5 --- /dev/null +++ b/test/auto_parallel/hybrid_strategy/semi_auto_parallel_2d_global_mesh_reshard.py @@ -0,0 +1,65 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import numpy as np + +import paddle +import paddle.distributed as dist + + +class TestSemiAutoParallel2DGlobalMeshReshard: + def __init__(self): + self._backend = os.getenv("backend") + self._seed = eval(os.getenv("seed")) + self._global_mesh = dist.ProcessMesh( + [[0, 1], [2, 3]], dim_names=["pp", "dp"] + ) + self._mesh0 = dist.ProcessMesh([0, 1], dim_names=["dp"]) + self._mesh1 = dist.ProcessMesh([2, 3], dim_names=["dp"]) + paddle.set_device(self._backend) + + def test_basic(self): + input = paddle.ones(shape=[2, 3], dtype='float32') + input = dist.shard_tensor( + input, self._global_mesh, [dist.Replicate(), dist.Shard(0)] + ) + input.stop_gradient = False + global_input = input + 1.0 # global_input: 2.0 + + # forward on pp0 + input_pp0 = dist.reshard(global_input, self._mesh0, [dist.Shard(0)]) + output = input_pp0 + 1.0 # output_pp0: 3.0 + + # forward on pp1 + output = dist.reshard(output, self._mesh1, [dist.Shard(0)]) + input_pp1 = dist.reshard(global_input, self._mesh1, [dist.Shard(0)]) + output = input_pp1 + output # output_pp1: 5.0 + loss = paddle.sum(output) # 30.0 + np.testing.assert_allclose(loss.numpy(), 30.0, rtol=1e-06, verbose=True) + loss.backward() + np.testing.assert_allclose( + input.grad.numpy(), + np.full(shape=(2, 3), fill_value=2.0, dtype=np.float32), + rtol=1e-06, + verbose=True, + ) + + def run_test_case(self): + self.test_basic() + + +if __name__ == '__main__': + TestSemiAutoParallel2DGlobalMeshReshard().run_test_case() diff --git a/test/auto_parallel/hybrid_strategy/semi_auto_parallel_3d_global_mesh_reshard.py b/test/auto_parallel/hybrid_strategy/semi_auto_parallel_3d_global_mesh_reshard.py new file mode 100644 index 0000000000000..bdc256a8a6493 --- /dev/null +++ b/test/auto_parallel/hybrid_strategy/semi_auto_parallel_3d_global_mesh_reshard.py @@ -0,0 +1,72 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import numpy as np + +import paddle +import paddle.distributed as dist + + +class TestSemiAutoParallel3DGlobalMeshReshard: + def __init__(self): + self._backend = os.getenv("backend") + self._seed = eval(os.getenv("seed")) + self._global_mesh = dist.ProcessMesh( + [[[0, 1], [2, 3]], [[4, 5], [6, 7]]], dim_names=['pp', 'dp', 'mp'] + ) + self._mesh0 = dist.ProcessMesh([[0, 1], [2, 3]], dim_names=['dp', 'mp']) + self._mesh1 = dist.ProcessMesh([[4, 5], [6, 7]], dim_names=['dp', 'mp']) + paddle.set_device(self._backend) + + def test_basic(self): + global_input = dist.shard_tensor( + paddle.ones(shape=[6, 8], dtype='float32'), + self._global_mesh, + [dist.Replicate(), dist.Replicate(), dist.Replicate()], + ) # 1.0 + global_input.stop_gradient = False + # forward on mesh0 + input_mesh0 = dist.reshard( + global_input, self._mesh0, [dist.Replicate(), dist.Replicate()] + ) + output = input_mesh0 + 1.0 # 2.0 + + # forward on mesh1 + output = dist.reshard( + output, self._mesh1, [dist.Replicate(), dist.Replicate()] + ) + input_mesh1 = dist.reshard( + global_input, self._mesh1, [dist.Replicate(), dist.Replicate()] + ) + output = output + input_mesh1 # 3.0 + loss = paddle.sum(output) # 144.0 + np.testing.assert_allclose( + loss.numpy(), 144.0, rtol=1e-06, verbose=True + ) + loss.backward() + np.testing.assert_allclose( + global_input.grad.numpy(), + np.full(shape=(6, 8), fill_value=2.0, dtype=np.float32), + rtol=1e-06, + verbose=True, + ) + + def run_test_case(self): + self.test_basic() + + +if __name__ == '__main__': + TestSemiAutoParallel3DGlobalMeshReshard().run_test_case() diff --git a/test/auto_parallel/hybrid_strategy/test_global_mesh_reshard.py b/test/auto_parallel/hybrid_strategy/test_global_mesh_reshard.py new file mode 100644 index 0000000000000..48bea0b88efcd --- /dev/null +++ b/test/auto_parallel/hybrid_strategy/test_global_mesh_reshard.py @@ -0,0 +1,73 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import collective.test_communication_api_base as test_base + + +class TestSemiAutoParallel2DGlobalMeshReshard( + test_base.CommunicationTestDistBase +): + def setUp(self): + super().setUp( + num_of_devices=4, + timeout=120, + nnode=1, + ) + self._default_envs = { + "dtype": "float32", + "seed": "2023", + } + self._changeable_envs = {"backend": ["gpu"]} + + def test_2d_global_mesh_reshard(self): + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) + for envs in envs_list: + self.run_test_case( + "semi_auto_parallel_2d_global_mesh_reshard.py", + user_defined_envs=envs, + ) + + +class TestSemiAutoParallel3DGlobalMeshReshard( + test_base.CommunicationTestDistBase +): + def setUp(self): + super().setUp( + num_of_devices=8, + timeout=120, + nnode=1, + ) + self._default_envs = { + "dtype": "float32", + "seed": "2023", + } + self._changeable_envs = {"backend": ["gpu"]} + + def test_3d_global_mesh_reshard(self): + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) + for envs in envs_list: + self.run_test_case( + "semi_auto_parallel_3d_global_mesh_reshard.py", + user_defined_envs=envs, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/auto_parallel/hybrid_strategy/testslist.csv b/test/auto_parallel/hybrid_strategy/testslist.csv index 7b64e2d93ea6b..9c1105615890b 100644 --- a/test/auto_parallel/hybrid_strategy/testslist.csv +++ b/test/auto_parallel/hybrid_strategy/testslist.csv @@ -6,3 +6,4 @@ test_semi_auto_parallel_c_cross_entropy,LINUX,GPU,120,HYBRID,test_runner.py,,,ht test_cross_mesh_reshard,LINUX,GPU,120,HYBRID,test_runner.py,,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_semi_auto_parallel_llama_model_amp,LINUX,GPU,180,HYBRID,test_runner.py,,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_semi_auto_parallel_hybrid_sharding_strategy,LINUX,GPU,120,HYBRID,test_runner.py,,,http_proxy=;https_proxy=;PYTHONPATH=../.., +test_global_mesh_reshard,LINUX,GPU,120,HYBRID,test_runner.py,,,http_proxy=;https_proxy=;PYTHONPATH=../.., From 6e8cd366529491305b153df21d235faadd81aec0 Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 26 Feb 2024 10:33:15 +0800 Subject: [PATCH 056/282] Fix some typos(infomation, unnecesary, etc) (#61956) * Fix * Fix --- test/cinn/op_mappers/op_mapper_test.py | 8 ++++---- test/cinn/op_mappers/test_norm_op.py | 2 +- test/cinn/test_paddle_model_convertor.py | 4 ++-- .../framework/no_need_buffer_vars_inference_test.cc | 2 +- .../paddle2cinn/cinn_graph_symbolization_test.cc | 4 ++-- test/cpp/fluid/framework/program_utils_test.cc | 4 ++-- .../pattern_rewrite/drr_same_type_binding_test.cc | 2 +- test/ir/inference/test_trt_convert_conv2d.py | 12 +++++------- .../inference/test_trt_convert_conv2d_transpose.py | 3 +-- .../inference/test_trt_convert_conv3d_transpose.py | 2 +- .../inference/test_trt_convert_depthwise_conv2d.py | 2 +- .../test_trt_convert_depthwise_conv2d_transpose.py | 3 +-- .../test_trt_convert_elementwiseadd_transpose.py | 2 +- test/ir/inference/test_trt_convert_expand_as_v2.py | 2 +- test/ir/inference/test_trt_convert_expand_v2.py | 4 ++-- test/ir/inference/test_trt_ops_fp32_mix_precision.py | 2 +- .../ir/inference/test_xpu_convert_mixed_precision.py | 2 +- .../cinn/symbolic/test_cinn_broadcast_symbolic.py | 2 +- .../cinn/symbolic/test_cinn_reduce_symbolic_demo.py | 2 +- test/ir/pir/test_ir_pybind.py | 2 +- 20 files changed, 31 insertions(+), 35 deletions(-) diff --git a/test/cinn/op_mappers/op_mapper_test.py b/test/cinn/op_mappers/op_mapper_test.py index d77a1b4dc7bf0..f3a5ef5d1847b 100644 --- a/test/cinn/op_mappers/op_mapper_test.py +++ b/test/cinn/op_mappers/op_mapper_test.py @@ -127,7 +127,7 @@ def __set_paddle_op(self): self.skip_outputs = self.skip_check_outputs() # dict of inplace var self.inplace_outputs = self.set_inplace_outputs() - # collect some important infomation + # collect some important information self.input_arg_map = self.__get_arguments_map(self.inputs) self.fetch_targets = [] self.skip_check_list = [] @@ -169,7 +169,7 @@ def __check_valid(self): msg=f"The shape of input {var.name} in feed_data is error", ) self.assertEqual( - self.paddleddtype2nptype(var.dtype), + self.paddledtype2nptype(var.dtype), str(self.feed_data[name].dtype), msg=f"The dtype of input {var.name} in feed_data is error", ) @@ -312,7 +312,7 @@ def build_cinn_program(self, target): for var_name, var in self.input_arg_map.items(): convertor.create_input( - dtype=self.paddleddtype2nptype(var.dtype), + dtype=self.paddledtype2nptype(var.dtype), shape=var.shape, name=var_name, ) @@ -411,7 +411,7 @@ def get_program_vars(program) -> dict: return vars @staticmethod - def paddleddtype2nptype(dtype): + def paddledtype2nptype(dtype): switch_map = { paddle.float16: "float16", paddle.float32: "float32", diff --git a/test/cinn/op_mappers/test_norm_op.py b/test/cinn/op_mappers/test_norm_op.py index a163925f2bc63..aaed4637efe6a 100644 --- a/test/cinn/op_mappers/test_norm_op.py +++ b/test/cinn/op_mappers/test_norm_op.py @@ -59,7 +59,7 @@ def set_op_attrs(self): return {"axis": -1, "epsilon": 1e-10, "is_test": True} def skip_check_outputs(self): - # in test mode, 'Norm' is unnecesary + # in test mode, 'Norm' is unnecessary return {"Norm"} diff --git a/test/cinn/test_paddle_model_convertor.py b/test/cinn/test_paddle_model_convertor.py index dc878127013f5..0b2f3b15b36b6 100644 --- a/test/cinn/test_paddle_model_convertor.py +++ b/test/cinn/test_paddle_model_convertor.py @@ -132,7 +132,7 @@ def init_case(self): msg="Repeat feed name: " + self.feed_names[i], ) - dtype = self.paddleddtype2nptype(self.feed_dtypes[i]) + dtype = self.paddledtype2nptype(self.feed_dtypes[i]) # random int type data should not limited to [0, 1] high = 1 if ("int" not in dtype) else self.feed_shapes[i][0] @@ -204,7 +204,7 @@ def build_cinn_program(self, target): convertor = PaddleModelConvertor(target) for i in range(len(self.feed_names)): convertor.create_input( - dtype=self.paddleddtype2nptype(self.feed_dtypes[i]), + dtype=self.paddledtype2nptype(self.feed_dtypes[i]), shape=self.feed_data[self.feed_names[i]].shape, name=self.feed_names[i], ) diff --git a/test/cpp/fluid/framework/no_need_buffer_vars_inference_test.cc b/test/cpp/fluid/framework/no_need_buffer_vars_inference_test.cc index d31a9680c16ea..5d200324e435e 100644 --- a/test/cpp/fluid/framework/no_need_buffer_vars_inference_test.cc +++ b/test/cpp/fluid/framework/no_need_buffer_vars_inference_test.cc @@ -51,7 +51,7 @@ TEST(test_no_need_buffer_vars_inference, test_dygraph) { DECLARE_NO_NEED_BUFFER_VARS_INFERER(TestNoNeedBufferVarsInferer, "X1", "X2"); -TEST(test_no_need_buffer_vars_inference, test_nullptr_comparation) { +TEST(test_no_need_buffer_vars_inference, test_nullptr_comparison) { InferNoNeedBufferVarsFN infer_fn; ASSERT_FALSE(static_cast(infer_fn)); ASSERT_TRUE(!infer_fn); diff --git a/test/cpp/fluid/framework/paddle2cinn/cinn_graph_symbolization_test.cc b/test/cpp/fluid/framework/paddle2cinn/cinn_graph_symbolization_test.cc index b936c0dfd5975..335f8a53da895 100644 --- a/test/cpp/fluid/framework/paddle2cinn/cinn_graph_symbolization_test.cc +++ b/test/cpp/fluid/framework/paddle2cinn/cinn_graph_symbolization_test.cc @@ -265,7 +265,7 @@ TEST_F(CinnGraphSymbolizationTest, scope) { ASSERT_EQ(cinn_tensor->type(), ::cinn::common::F32()); } -TEST_F(CinnGraphSymbolizationTest, sortgraph) { +TEST_F(CinnGraphSymbolizationTest, sort_graph) { auto cinn_op_descs = test_->TransformAllGraphOpToCinn(); ASSERT_FALSE(cinn_op_descs.empty()); std::vector sort_names; @@ -276,7 +276,7 @@ TEST_F(CinnGraphSymbolizationTest, sortgraph) { std::vector({"feed", "feed", "mul", "add", "relu"})); } -TEST_F(CinnGraphSymbolizationTest, runop) { +TEST_F(CinnGraphSymbolizationTest, run_op) { auto cinn_op_descs = test_->TransformAllGraphOpToCinn(); auto feed_map = test_->GetFeedInfoMapFromInput(); diff --git a/test/cpp/fluid/framework/program_utils_test.cc b/test/cpp/fluid/framework/program_utils_test.cc index 051aa89e4b5f3..624c5697e537b 100644 --- a/test/cpp/fluid/framework/program_utils_test.cc +++ b/test/cpp/fluid/framework/program_utils_test.cc @@ -203,8 +203,8 @@ TEST(ProgramDesc, GetInputsOutputsInBlock) { ASSERT_EQ(5UL, inner_inputs.size()); ASSERT_EQ(2UL, inner_outputs.size()); - // varible "Less_than_2_Out" is the input of cond_op, it also is the output of - // less_than_op. + // variable "Less_than_2_Out" is the input of cond_op, it also is the output + // of less_than_op. std::set inner_inputs_{"Less_than_2_Out", "Less_than_2_X", "Less_than_2_Y", diff --git a/test/cpp/pir/pattern_rewrite/drr_same_type_binding_test.cc b/test/cpp/pir/pattern_rewrite/drr_same_type_binding_test.cc index d672cc4ae9c13..bf8f847b2a877 100644 --- a/test/cpp/pir/pattern_rewrite/drr_same_type_binding_test.cc +++ b/test/cpp/pir/pattern_rewrite/drr_same_type_binding_test.cc @@ -30,7 +30,7 @@ / | \ \ \ / | \ \ \ full / | | \ \ full_tmp - / | transpose1 | trans2 trans3 \ / | + / | transpose1 | trans2 trans3 \ / | / | / | | | | \ / | softmax1 | / | | | | \ / | \ | / softmax2 | | | add1 | diff --git a/test/ir/inference/test_trt_convert_conv2d.py b/test/ir/inference/test_trt_convert_conv2d.py index 84b44adc574ef..3fa99a078ddd7 100644 --- a/test/ir/inference/test_trt_convert_conv2d.py +++ b/test/ir/inference/test_trt_convert_conv2d.py @@ -65,7 +65,7 @@ def generate_weight1(attrs: List[Dict[str, Any]]): strides_options = [[2, 2], [1, 2]] paddings_options = [[0, 3], [1, 2, 3, 4]] groups_options = [1, 3] - padding_altorithm_options = ['EXPLICIT', 'SAME', 'VALID'] + padding_algorithm_options = ['EXPLICIT', 'SAME', 'VALID'] dilations_options = [[1, 2]] data_format_options = ['NCHW'] @@ -74,7 +74,7 @@ def generate_weight1(attrs: List[Dict[str, Any]]): strides_options, paddings_options, groups_options, - padding_altorithm_options, + padding_algorithm_options, dilations_options, data_format_options, ] @@ -90,7 +90,6 @@ def generate_weight1(attrs: List[Dict[str, Any]]): ) in itertools.product(*configurations): attrs = [ { - "data_fromat": data_format, "dilations": dilations, "padding_algorithm": padding_algorithm, "groups": groups, @@ -252,7 +251,7 @@ def generate_data(attrs: List[Dict[str, Any]]): strides_options = [[2, 2]] paddings_options = [[1, 1]] groups_options = [1] - padding_altorithm_options = ['EXPLICIT'] + padding_algorithm_options = ['EXPLICIT'] dilations_options = [[1, 1]] data_format_options = ['NCHW'] @@ -263,7 +262,7 @@ def generate_data(attrs: List[Dict[str, Any]]): strides_options, paddings_options, groups_options, - padding_altorithm_options, + padding_algorithm_options, dilations_options, data_format_options, ] @@ -282,14 +281,13 @@ def generate_data(attrs: List[Dict[str, Any]]): ic = input_shape[1] attrs = [ { - "data_fromat": data_format, "dilations": dilations, "padding_algorithm": padding_algorithm, "groups": groups, "paddings": paddings, "strides": strides, "data_format": data_format, - # below attrs are used for my convience. + # below attrs are used for my convenience. "input_shape": input_shape, "weight_shape": [ oc, diff --git a/test/ir/inference/test_trt_convert_conv2d_transpose.py b/test/ir/inference/test_trt_convert_conv2d_transpose.py index 57c973b466732..c493377ebf976 100644 --- a/test/ir/inference/test_trt_convert_conv2d_transpose.py +++ b/test/ir/inference/test_trt_convert_conv2d_transpose.py @@ -80,7 +80,6 @@ def generate_weight1(num_channels, attrs: List[Dict[str, Any]]): self.num_channels = num_channels dics = [ { - "data_fromat": data_format, "dilations": dilations, "padding_algorithm": padding_algorithm, "groups": groups, @@ -272,7 +271,7 @@ def generate_weight1(num_channels, attrs: List[Dict[str, Any]]): self.num_channels = num_channels dics = [ { - "data_fromat": 'NCHW', + "data_format": 'NCHW', "dilations": [1, 1], "padding_algorithm": 'EXPLICIT', "groups": 1, diff --git a/test/ir/inference/test_trt_convert_conv3d_transpose.py b/test/ir/inference/test_trt_convert_conv3d_transpose.py index ba545275d805b..b2d15d3643f57 100644 --- a/test/ir/inference/test_trt_convert_conv3d_transpose.py +++ b/test/ir/inference/test_trt_convert_conv3d_transpose.py @@ -48,7 +48,7 @@ def generate_weight1(num_channels, attrs: List[Dict[str, Any]]): self.num_channels = num_channels dics = [ { - "data_fromat": 'NCHW', + "data_format": 'NCHW', "dilations": [1, 1, 1], "padding_algorithm": 'EXPLICIT', "groups": 1, diff --git a/test/ir/inference/test_trt_convert_depthwise_conv2d.py b/test/ir/inference/test_trt_convert_depthwise_conv2d.py index 5a38347d8e646..5fd25562f33a9 100644 --- a/test/ir/inference/test_trt_convert_depthwise_conv2d.py +++ b/test/ir/inference/test_trt_convert_depthwise_conv2d.py @@ -84,7 +84,7 @@ def generate_weight1(attrs: List[Dict[str, Any]]): "groups": groups, "padding_algorithm": padding_algorithm, "dilations": dilations, - "data_fromat": data_format, + "data_format": data_format, } ] diff --git a/test/ir/inference/test_trt_convert_depthwise_conv2d_transpose.py b/test/ir/inference/test_trt_convert_depthwise_conv2d_transpose.py index e570fe77f1c2e..4a0b0faf4df65 100644 --- a/test/ir/inference/test_trt_convert_depthwise_conv2d_transpose.py +++ b/test/ir/inference/test_trt_convert_depthwise_conv2d_transpose.py @@ -87,13 +87,12 @@ def generate_weight1(attrs: list[dict[str, Any]]): ): dics = [ { - "data_fromat": data_format, + "data_format": data_format, "dilations": dilations, "padding_algorithm": padding_algorithm, "groups": groups, "paddings": paddings, "strides": strides, - "data_format": data_format, "output_size": [], "output_padding": [], } diff --git a/test/ir/inference/test_trt_convert_elementwiseadd_transpose.py b/test/ir/inference/test_trt_convert_elementwiseadd_transpose.py index dd9c495d49772..012aaa00241b4 100644 --- a/test/ir/inference/test_trt_convert_elementwiseadd_transpose.py +++ b/test/ir/inference/test_trt_convert_elementwiseadd_transpose.py @@ -22,7 +22,7 @@ import paddle.inference as paddle_infer -class TrtConvertElementwiseaddTransposeTest(TrtLayerAutoScanTest): +class TrtConvertElementwiseAddTransposeTest(TrtLayerAutoScanTest): def is_program_valid(self, program_config: ProgramConfig) -> bool: return True diff --git a/test/ir/inference/test_trt_convert_expand_as_v2.py b/test/ir/inference/test_trt_convert_expand_as_v2.py index 50bb2ac34ca27..16c689573eeaa 100644 --- a/test/ir/inference/test_trt_convert_expand_as_v2.py +++ b/test/ir/inference/test_trt_convert_expand_as_v2.py @@ -247,7 +247,7 @@ def clear_dynamic_shape(): generate_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 program_config.set_input_type(np.float32) - # fill_constant will be folded by constnt folding pass! + # fill_constant will be folded by constant folding pass! yield self.create_inference_config(), (1, 2), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half program_config.set_input_type(np.float16) diff --git a/test/ir/inference/test_trt_convert_expand_v2.py b/test/ir/inference/test_trt_convert_expand_v2.py index 0961dac427699..03d8dd32df6b5 100644 --- a/test/ir/inference/test_trt_convert_expand_v2.py +++ b/test/ir/inference/test_trt_convert_expand_v2.py @@ -253,7 +253,7 @@ def clear_dynamic_shape(): generate_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 program_config.set_input_type(np.float32) - # fill_constant will be folded by constnt folding pass! + # fill_constant will be folded by constant folding pass! yield self.create_inference_config(), (1, 2), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half program_config.set_input_type(np.float16) @@ -399,7 +399,7 @@ def clear_dynamic_shape(): generate_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 program_config.set_input_type(np.float32) - # fill_constant will be folded by constnt folding pass! + # fill_constant will be folded by constant folding pass! yield self.create_inference_config(), (1, 2), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half program_config.set_input_type(np.float16) diff --git a/test/ir/inference/test_trt_ops_fp32_mix_precision.py b/test/ir/inference/test_trt_ops_fp32_mix_precision.py index 3347cfb6d06db..6a7a6051dea61 100644 --- a/test/ir/inference/test_trt_ops_fp32_mix_precision.py +++ b/test/ir/inference/test_trt_ops_fp32_mix_precision.py @@ -56,7 +56,7 @@ def generate_elementwise_weight(op_type): attrs = [ { - "data_fromat": 'NCHW', + "data_format": 'NCHW', "dilations": [1, 2], "padding_algorithm": 'EXPLICIT', "groups": 1, diff --git a/test/ir/inference/test_xpu_convert_mixed_precision.py b/test/ir/inference/test_xpu_convert_mixed_precision.py index f09d00440ac64..cce33ca3bc9dc 100644 --- a/test/ir/inference/test_xpu_convert_mixed_precision.py +++ b/test/ir/inference/test_xpu_convert_mixed_precision.py @@ -27,7 +27,7 @@ from paddle.vision.models import resnet50 -class ConvertMixedPrecison(unittest.TestCase): +class ConvertMixedPrecision(unittest.TestCase): def test(self): self.temp_dir = tempfile.TemporaryDirectory() model = resnet50(True) diff --git a/test/ir/pir/cinn/symbolic/test_cinn_broadcast_symbolic.py b/test/ir/pir/cinn/symbolic/test_cinn_broadcast_symbolic.py index 63009a9704d7c..96f8fbfebd24b 100644 --- a/test/ir/pir/cinn/symbolic/test_cinn_broadcast_symbolic.py +++ b/test/ir/pir/cinn/symbolic/test_cinn_broadcast_symbolic.py @@ -74,7 +74,7 @@ def eval_symbolic(self, use_cinn): self.check_jit_kernel_info(net.forward) return out - def test_eval_symolic(self): + def test_eval_symbolic(self): cinn_out = self.eval_symbolic(use_cinn=True) dy_out = self.eval_symbolic(use_cinn=False) np.testing.assert_allclose(cinn_out.numpy(), dy_out.numpy(), atol=1e-8) diff --git a/test/ir/pir/cinn/symbolic/test_cinn_reduce_symbolic_demo.py b/test/ir/pir/cinn/symbolic/test_cinn_reduce_symbolic_demo.py index bb2e1c789e22f..dede8a2083efc 100644 --- a/test/ir/pir/cinn/symbolic/test_cinn_reduce_symbolic_demo.py +++ b/test/ir/pir/cinn/symbolic/test_cinn_reduce_symbolic_demo.py @@ -70,7 +70,7 @@ def eval_symbolic(self, use_cinn): return out - def test_eval_symolic(self): + def test_eval_symbolic(self): cinn_out = self.eval_symbolic(use_cinn=True) # dy_out = self.eval_symbolic(use_cinn=False) # np.testing.assert_allclose(cinn_out.numpy(), dy_out.numpy(), atol=1e-8) diff --git a/test/ir/pir/test_ir_pybind.py b/test/ir/pir/test_ir_pybind.py index 22e03fdc1bf95..460e5e489eb35 100644 --- a/test/ir/pir/test_ir_pybind.py +++ b/test/ir/pir/test_ir_pybind.py @@ -71,7 +71,7 @@ def test_operation(self): self.assertEqual(len(matmul_op.get_input_names()), 2) self.assertEqual(len(matmul_op.get_attr_names()), 2) self.assertEqual(len(matmul_op.get_output_names()), 1) - # test oprand.index + # test operand.index self.assertEqual(matmul_op.operand(0).index(), 0) self.assertEqual(matmul_op.operand(1).index(), 1) self.assertEqual(add_op.operand(0).index(), 0) From e4518a8b00d3a7f1505bd7dd7b10c337e4d1383b Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 26 Feb 2024 10:35:37 +0800 Subject: [PATCH 057/282] Update skip_files.py (#62004) --- python/paddle/jit/sot/opcode_translator/skip_files.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/paddle/jit/sot/opcode_translator/skip_files.py b/python/paddle/jit/sot/opcode_translator/skip_files.py index ca7f3552ad6ac..f3517d1419c38 100644 --- a/python/paddle/jit/sot/opcode_translator/skip_files.py +++ b/python/paddle/jit/sot/opcode_translator/skip_files.py @@ -54,7 +54,7 @@ import paddle -NEED_SKIP_THIRD_PARTIY_MODULES = { +NEED_SKIP_THIRD_PARTY_MODULES = { abc, collections, contextlib, @@ -98,13 +98,13 @@ import sre_compile import sre_parse - NEED_SKIP_THIRD_PARTIY_MODULES.add(sre_compile) - NEED_SKIP_THIRD_PARTIY_MODULES.add(sre_parse) + NEED_SKIP_THIRD_PARTY_MODULES.add(sre_compile) + NEED_SKIP_THIRD_PARTY_MODULES.add(sre_parse) if sys.version_info < (3, 12): import distutils - NEED_SKIP_THIRD_PARTIY_MODULES.add(distutils) + NEED_SKIP_THIRD_PARTY_MODULES.add(distutils) def _strip_init_py(s): @@ -115,7 +115,7 @@ def _module_dir(m: types.ModuleType): return _strip_init_py(m.__file__) -skip_file_names = {_module_dir(m) for m in NEED_SKIP_THIRD_PARTIY_MODULES} +skip_file_names = {_module_dir(m) for m in NEED_SKIP_THIRD_PARTY_MODULES} sot_path = os.path.dirname(__file__).rpartition(os.sep)[0] + os.sep From 5508f55d6e0861d2ee4ff87f847c5e6e3b0f8e39 Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 26 Feb 2024 10:36:04 +0800 Subject: [PATCH 058/282] Fix typos(attetion, etc) (#62044) --- .../ir/mkldnn/cpu_quantize_squash_pass_tester.cc | 2 +- .../fluid/framework/ir/mkldnn/self_attention_fuse_pass.h | 2 +- .../ir/mkldnn/shuffle_channel_mkldnn_detect_pass.cc | 2 +- .../ir/multi_devices_graph_pass/all_reduce_deps_pass.cc | 4 ++-- .../backward_optimizer_op_deps_pass.cc | 2 +- .../multi_devices_graph_check_pass.cc | 6 +++--- .../multi_devices_graph_print_pass.cc | 4 ++-- .../fluid/framework/ir/xpu/add_layernorm_xpu_fuse_pass.cc | 2 +- .../ir/xpu/fused_multi_transformer_int8_xpu_quant_pass.cc | 2 +- .../ir/xpu/multi_encoder_xpu_adaptive_seqlen_fuse_pass.h | 8 ++++---- 10 files changed, 17 insertions(+), 17 deletions(-) diff --git a/paddle/fluid/framework/ir/mkldnn/cpu_quantize_squash_pass_tester.cc b/paddle/fluid/framework/ir/mkldnn/cpu_quantize_squash_pass_tester.cc index 5b1cd5fe87aed..90ed3009749ad 100644 --- a/paddle/fluid/framework/ir/mkldnn/cpu_quantize_squash_pass_tester.cc +++ b/paddle/fluid/framework/ir/mkldnn/cpu_quantize_squash_pass_tester.cc @@ -1018,7 +1018,7 @@ TEST(CpuQuantizeSquashPass, fc_dequant_more_than_one_op_after_dequant) { // a->Concat1->b // b->Concat2 -// b->Quatize1(Scale)->c +// b->Quantize1(Scale)->c // c->Fc1 // c->Fc2 TEST(CpuQuantizeSquashPass, quatize_with_same_scale) { diff --git a/paddle/fluid/framework/ir/mkldnn/self_attention_fuse_pass.h b/paddle/fluid/framework/ir/mkldnn/self_attention_fuse_pass.h index ade48f398e3b6..a264795bd78fb 100644 --- a/paddle/fluid/framework/ir/mkldnn/self_attention_fuse_pass.h +++ b/paddle/fluid/framework/ir/mkldnn/self_attention_fuse_pass.h @@ -24,7 +24,7 @@ namespace paddle { namespace framework { namespace ir { -// Fusing of self-attetion structure +// Fusing of self-attention structure class Graph; diff --git a/paddle/fluid/framework/ir/mkldnn/shuffle_channel_mkldnn_detect_pass.cc b/paddle/fluid/framework/ir/mkldnn/shuffle_channel_mkldnn_detect_pass.cc index 5cf9a973061f0..764712a2fcd8a 100644 --- a/paddle/fluid/framework/ir/mkldnn/shuffle_channel_mkldnn_detect_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/shuffle_channel_mkldnn_detect_pass.cc @@ -174,7 +174,7 @@ void ShuffleChannelMKLDNNDetectPass::ApplyImpl(ir::Graph* graph) const { } } - // shuffle_channel dosen't change shape + // shuffle_channel doesn't change shape if ((reshape2_shape[0] != -1) && (x_shape1[0] != reshape2_shape[0])) { return; } diff --git a/paddle/fluid/framework/ir/multi_devices_graph_pass/all_reduce_deps_pass.cc b/paddle/fluid/framework/ir/multi_devices_graph_pass/all_reduce_deps_pass.cc index 0dcf316c33c69..6327929663ab4 100644 --- a/paddle/fluid/framework/ir/multi_devices_graph_pass/all_reduce_deps_pass.cc +++ b/paddle/fluid/framework/ir/multi_devices_graph_pass/all_reduce_deps_pass.cc @@ -150,7 +150,7 @@ class AllReduceDepsPass : public ir::Pass { const std::vector& all_reduce_op_handles) const { // get vars order std::map> vars = - GetSoredGradientsFromStaleProgram(graph); + GetSortedGradientsFromStaleProgram(graph); std::stringstream out; size_t grads_of_stale_program = 0; out << "Get Order From details::kStaleProgramOpDescs: "; @@ -188,7 +188,7 @@ class AllReduceDepsPass : public ir::Pass { } } - std::map> GetSoredGradientsFromStaleProgram( + std::map> GetSortedGradientsFromStaleProgram( const ir::Graph& graph) const { std::map> vars; auto ops = diff --git a/paddle/fluid/framework/ir/multi_devices_graph_pass/backward_optimizer_op_deps_pass.cc b/paddle/fluid/framework/ir/multi_devices_graph_pass/backward_optimizer_op_deps_pass.cc index 82e8dada11556..a4feed4693a62 100644 --- a/paddle/fluid/framework/ir/multi_devices_graph_pass/backward_optimizer_op_deps_pass.cc +++ b/paddle/fluid/framework/ir/multi_devices_graph_pass/backward_optimizer_op_deps_pass.cc @@ -68,7 +68,7 @@ class BackWardOpDepsPass : public ir::Pass { AddDep(graph, opt_handles[i - 1], opt_handles[i]); } - VLOG(10) << "add deps between backward and optimze:"; + VLOG(10) << "add deps between backward and optimize:"; AddDep(graph, backward_op_handles[backward_op_handles.size() - 1], opt_handles[0]); diff --git a/paddle/fluid/framework/ir/multi_devices_graph_pass/multi_devices_graph_check_pass.cc b/paddle/fluid/framework/ir/multi_devices_graph_pass/multi_devices_graph_check_pass.cc index d08bbc1720de6..95f64e896f77b 100644 --- a/paddle/fluid/framework/ir/multi_devices_graph_pass/multi_devices_graph_check_pass.cc +++ b/paddle/fluid/framework/ir/multi_devices_graph_pass/multi_devices_graph_check_pass.cc @@ -19,14 +19,14 @@ namespace paddle { namespace framework { namespace ir { -class SSAGraghBuilderWithChecker : public ir::Pass { +class SSAGraphBuilderWithChecker : public ir::Pass { protected: void ApplyImpl(ir::Graph *graph) const override { PADDLE_ENFORCE_EQ( IsValidGraph(graph), true, platform::errors::InvalidArgument( - "In SSAGraghBuilderWithChecker, invalid Graph input.")); + "In SSAGraphBuilderWithChecker, invalid Graph input.")); } bool IsValidGraph(const ir::Graph *graph) const { @@ -99,6 +99,6 @@ class SSAGraghBuilderWithChecker : public ir::Pass { } // namespace paddle REGISTER_PASS(multi_devices_check_pass, - paddle::framework::ir::SSAGraghBuilderWithChecker) + paddle::framework::ir::SSAGraphBuilderWithChecker) .RequireGraphAttr(paddle::framework::details::kGraphVars) .RequireGraphAttr(paddle::framework::details::kGraphDepVars); diff --git a/paddle/fluid/framework/ir/multi_devices_graph_pass/multi_devices_graph_print_pass.cc b/paddle/fluid/framework/ir/multi_devices_graph_pass/multi_devices_graph_print_pass.cc index 97454b7583de2..6005c7de5c551 100644 --- a/paddle/fluid/framework/ir/multi_devices_graph_pass/multi_devices_graph_print_pass.cc +++ b/paddle/fluid/framework/ir/multi_devices_graph_pass/multi_devices_graph_print_pass.cc @@ -19,7 +19,7 @@ namespace paddle { namespace framework { namespace ir { -class SSAGraghBuilderWithPrinterPass : public ir::Pass { +class SSAGraphBuilderWithPrinterPass : public ir::Pass { protected: void ApplyImpl(ir::Graph *graph) const override { std::unique_ptr fout( @@ -102,5 +102,5 @@ void GraphvizSSAGraphPrinter::Print(const ir::Graph &graph, } // namespace paddle REGISTER_PASS(multi_devices_print_pass, - paddle::framework::ir::SSAGraghBuilderWithPrinterPass) + paddle::framework::ir::SSAGraphBuilderWithPrinterPass) .RequirePassAttr(paddle::framework::ir::kGraphvizPath); diff --git a/paddle/fluid/framework/ir/xpu/add_layernorm_xpu_fuse_pass.cc b/paddle/fluid/framework/ir/xpu/add_layernorm_xpu_fuse_pass.cc index 5db915c4e17f5..55e8786f73c4d 100644 --- a/paddle/fluid/framework/ir/xpu/add_layernorm_xpu_fuse_pass.cc +++ b/paddle/fluid/framework/ir/xpu/add_layernorm_xpu_fuse_pass.cc @@ -59,7 +59,7 @@ After the pass is applied: scale---- add_layernorm_fusion ---- bias / | \ \ / | \ \ - variance | meam z_add + variance | mean z_add Output */ struct AddLayernormXPUPattern : public PatternBase { diff --git a/paddle/fluid/framework/ir/xpu/fused_multi_transformer_int8_xpu_quant_pass.cc b/paddle/fluid/framework/ir/xpu/fused_multi_transformer_int8_xpu_quant_pass.cc index 4f9af98495c37..7c5b7c9e5e4e7 100755 --- a/paddle/fluid/framework/ir/xpu/fused_multi_transformer_int8_xpu_quant_pass.cc +++ b/paddle/fluid/framework/ir/xpu/fused_multi_transformer_int8_xpu_quant_pass.cc @@ -525,7 +525,7 @@ int FusedMultiTransformerInt8XPUQuantPass::FusedMultiTransformerInt8( id++; } }; - // genereate input node + // generate input node attr2weight( "qkv_in_scale", &(input_max_nodes_vec[0]), &(input_max_names_vec[0])); attr2weight("out_linear_in_scale", diff --git a/paddle/fluid/framework/ir/xpu/multi_encoder_xpu_adaptive_seqlen_fuse_pass.h b/paddle/fluid/framework/ir/xpu/multi_encoder_xpu_adaptive_seqlen_fuse_pass.h index a21d6498dea8e..22910c2120530 100644 --- a/paddle/fluid/framework/ir/xpu/multi_encoder_xpu_adaptive_seqlen_fuse_pass.h +++ b/paddle/fluid/framework/ir/xpu/multi_encoder_xpu_adaptive_seqlen_fuse_pass.h @@ -40,7 +40,7 @@ class MultiEncoderXPUAdaptiveSeqlenFusePass : public FusePassBase { /* adaptive seqlen V1, before: - inpu_var* mask_var* + input_var* mask_var* | | | | embedding_xpu matmul @@ -59,7 +59,7 @@ class MultiEncoderXPUAdaptiveSeqlenFusePass : public FusePassBase { after: - inpu_var* mask_var* + input_var* mask_var* \ / \ / embedding_xpu @@ -81,7 +81,7 @@ class MultiEncoderXPUAdaptiveSeqlenFusePass : public FusePassBase { /* adaptive seqlen V2, before: - inpu_var* mask_var* + input_var* mask_var* | | | | embedding_xpu not_equal @@ -115,7 +115,7 @@ class MultiEncoderXPUAdaptiveSeqlenFusePass : public FusePassBase { after: - inpu_var* mask_var* + input_var* mask_var* \ / \ / embedding_xpu From 67bd48f9724735eaadddc71d8dd934a7972de470 Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 26 Feb 2024 10:36:31 +0800 Subject: [PATCH 059/282] Fix some typos(ingest_externel_file, etc) (#62040) --- .../auto_parallel/spmd_rules/replicated_spmd_rule.cc | 2 +- paddle/fluid/distributed/ps/service/server.h | 2 +- paddle/fluid/distributed/ps/table/depends/rocksdb_warpper.h | 2 +- paddle/fluid/distributed/ps/table/ssd_sparse_table.cc | 4 ++-- paddle/fluid/distributed/ps/wrapper/fleet.cc | 4 ++-- paddle/fluid/distributed/ps/wrapper/fleet.h | 2 +- paddle/fluid/distributed/test/graph_node_test.cc | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/paddle/fluid/distributed/auto_parallel/spmd_rules/replicated_spmd_rule.cc b/paddle/fluid/distributed/auto_parallel/spmd_rules/replicated_spmd_rule.cc index b0ffb47c99234..5227a82a4b8b5 100644 --- a/paddle/fluid/distributed/auto_parallel/spmd_rules/replicated_spmd_rule.cc +++ b/paddle/fluid/distributed/auto_parallel/spmd_rules/replicated_spmd_rule.cc @@ -30,7 +30,7 @@ ReplicatedSPMDRule::InferForward(const std::vector& input_specs, } // TODO(ljz): we need to know num of output and size of each output before - // generate the excat replicated dist tensor attr for the current op. + // generate the exact replicated dist tensor attr for the current op. // here we just assume that only one output tensor and has the same size as // the first input tensor. return {intput_dist_attrs, {ReplicatedOnMesh(input_specs[0].dist_attr())}}; diff --git a/paddle/fluid/distributed/ps/service/server.h b/paddle/fluid/distributed/ps/service/server.h index 5a0764b11e8a1..bae9ab652ff74 100644 --- a/paddle/fluid/distributed/ps/service/server.h +++ b/paddle/fluid/distributed/ps/service/server.h @@ -194,7 +194,7 @@ class PsBaseService : public PsService { const char *err_msg) { response.set_err_msg(err_msg); response.set_err_code(err_code); - LOG(WARNING) << "Resonse err_code:" << err_code << " msg:" << err_msg; + LOG(WARNING) << "Response err_code:" << err_code << " msg:" << err_msg; } virtual int32_t Initialize() = 0; diff --git a/paddle/fluid/distributed/ps/table/depends/rocksdb_warpper.h b/paddle/fluid/distributed/ps/table/depends/rocksdb_warpper.h index 873644f8ca416..b090ef778d2ac 100644 --- a/paddle/fluid/distributed/ps/table/depends/rocksdb_warpper.h +++ b/paddle/fluid/distributed/ps/table/depends/rocksdb_warpper.h @@ -237,7 +237,7 @@ class RocksDBHandler { Uint64Comparator* get_comparator() { return &_comparator; } - int ingest_externel_file(int id, + int ingest_external_file(int id, const std::vector& sst_filelist) { rocksdb::IngestExternalFileOptions ifo; ifo.move_files = true; diff --git a/paddle/fluid/distributed/ps/table/ssd_sparse_table.cc b/paddle/fluid/distributed/ps/table/ssd_sparse_table.cc index 13577bde3e145..d72b4ee1c3d3f 100644 --- a/paddle/fluid/distributed/ps/table/ssd_sparse_table.cc +++ b/paddle/fluid/distributed/ps/table/ssd_sparse_table.cc @@ -2896,7 +2896,7 @@ int32_t SSDSparseTable::LoadWithBinary(const std::string& path, int param) { auto sst_filelist = _afs_client.list(::paddle::string::format_string( "%s_%d/part-*", FLAGS_rocksdb_path.c_str(), shard_idx)); if (!sst_filelist.empty()) { - int ret = _db->ingest_externel_file(shard_idx, sst_filelist); + int ret = _db->ingest_external_file(shard_idx, sst_filelist); if (ret) { VLOG(0) << "ingest file failed"; abort(); @@ -3038,7 +3038,7 @@ int32_t SSDSparseTable::CacheTable(uint16_t pass_id) { } VLOG(0) << "write sst_file shard " << shard_id << ": " << butil::gettimeofday_ms() - show_begin << " ms"; - int ret = _db->ingest_externel_file(shard_id, {filename}); + int ret = _db->ingest_external_file(shard_id, {filename}); if (ret) { VLOG(0) << "ingest file failed" << ", " << status.getState(); diff --git a/paddle/fluid/distributed/ps/wrapper/fleet.cc b/paddle/fluid/distributed/ps/wrapper/fleet.cc index ac404f5d9e70c..44043bc65501c 100644 --- a/paddle/fluid/distributed/ps/wrapper/fleet.cc +++ b/paddle/fluid/distributed/ps/wrapper/fleet.cc @@ -918,8 +918,8 @@ std::default_random_engine& FleetWrapper::LocalRandomEngine() { clock_gettime(CLOCK_REALTIME, &tp); double cur_time = tp.tv_sec + tp.tv_nsec * 1e-9; static std::atomic x(0); - std::seed_seq sseq = {x++, x++, x++, (uint64_t)(cur_time * 1000)}; - engine.seed(sseq); + std::seed_seq s_seq = {x++, x++, x++, (uint64_t)(cur_time * 1000)}; + engine.seed(s_seq); } }; thread_local engine_wrapper_t r; diff --git a/paddle/fluid/distributed/ps/wrapper/fleet.h b/paddle/fluid/distributed/ps/wrapper/fleet.h index 208e94ec12102..95504ede00fad 100644 --- a/paddle/fluid/distributed/ps/wrapper/fleet.h +++ b/paddle/fluid/distributed/ps/wrapper/fleet.h @@ -288,7 +288,7 @@ class FleetWrapper { std::string GetDistDesc() const { CHECK(is_initialized_ == true) - << "fleetwrapper should be initialized first!!!"; + << "FleetWrapper should be initialized first!!!"; return dist_desc_; } diff --git a/paddle/fluid/distributed/test/graph_node_test.cc b/paddle/fluid/distributed/test/graph_node_test.cc index 26207a9ad8c9e..8c29c2bf1df3f 100644 --- a/paddle/fluid/distributed/test/graph_node_test.cc +++ b/paddle/fluid/distributed/test/graph_node_test.cc @@ -416,7 +416,7 @@ void RunBrpcPushSparse() { // auto ph_host2 = paddle::distributed::PSHost(ip2, port2, 1); // host_sign_list_.push_back(ph_host2.SerializeToString()); // // test-end - // // Srart Server + // // Start Server // std::thread* server_thread = new std::thread(RunServer); // std::thread* server_thread2 = new std::thread(RunServer2); // sleep(1); From 59980808241e7432de2ba6dd43444908bce37208 Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 26 Feb 2024 10:39:21 +0800 Subject: [PATCH 060/282] Fix typos(oprations, etc) (#62043) --- paddle/fluid/framework/ir/fusion_group/operation.cc | 2 +- paddle/fluid/framework/ir/fusion_group/operation.h | 2 +- paddle/fluid/framework/ir/fusion_group/subgraph.h | 2 +- .../fluid/framework/ir/ipu/optimizer_extract_pass.cc | 10 +++++----- .../buffer_shared_cross_op_memory_reuse_pass.cc | 4 ++-- .../memory_optimization_var_info.h | 2 +- .../share_varinfo_into_cinn_pass.cc | 2 +- .../while_op_eager_deletion_pass.cc | 2 +- .../ir/mkldnn/cpu_quantize_squash_pass_tester.cc | 4 ++-- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/paddle/fluid/framework/ir/fusion_group/operation.cc b/paddle/fluid/framework/ir/fusion_group/operation.cc index 908aa6d62b6f7..75b0d8d631f8a 100644 --- a/paddle/fluid/framework/ir/fusion_group/operation.cc +++ b/paddle/fluid/framework/ir/fusion_group/operation.cc @@ -152,7 +152,7 @@ void OperationMap::InsertUnaryElementwiseOperations() { } void OperationMap::InsertBinaryElementwiseOperations() { - // For binary elementwise oprations: + // For binary elementwise operations: // ${0} - x // ${1} - y // ${2} - out diff --git a/paddle/fluid/framework/ir/fusion_group/operation.h b/paddle/fluid/framework/ir/fusion_group/operation.h index 3edf2f598525a..dd939cd3cbbf1 100644 --- a/paddle/fluid/framework/ir/fusion_group/operation.h +++ b/paddle/fluid/framework/ir/fusion_group/operation.h @@ -55,7 +55,7 @@ struct Operation { return false; } if (IsGradOp() && exprs.size() != static_cast(num_operands)) { - // When it is a backward opertion, it should hold a expression for each + // When it is a backward operation, it should hold a expression for each // operand. return false; } diff --git a/paddle/fluid/framework/ir/fusion_group/subgraph.h b/paddle/fluid/framework/ir/fusion_group/subgraph.h index 057fc7efffb30..97caa43249002 100644 --- a/paddle/fluid/framework/ir/fusion_group/subgraph.h +++ b/paddle/fluid/framework/ir/fusion_group/subgraph.h @@ -150,7 +150,7 @@ class SubGraph { !IsInputOfExternalOp(n)) { // When the outputs size is 0, it is also considered a intermidiate // output. It maybe an unused output or the fetching vars, so that we - // cannot eleiminate it directly here. + // cannot eliminate it directly here. intermediate_out_vars.push_back(n); } } diff --git a/paddle/fluid/framework/ir/ipu/optimizer_extract_pass.cc b/paddle/fluid/framework/ir/ipu/optimizer_extract_pass.cc index 284d144bf7534..d09519dfa5b04 100644 --- a/paddle/fluid/framework/ir/ipu/optimizer_extract_pass.cc +++ b/paddle/fluid/framework/ir/ipu/optimizer_extract_pass.cc @@ -100,8 +100,8 @@ void IpuOptimizerExtractPass::ApplyImpl(ir::Graph* graph) const { // bool is_optimizer = is_optimizer_op(op_namescope); bool is_regularization = is_regularization_op(op_namescope); - VLOG(10) << "found optimizer releated op: " << op_type; - // initial larning_rate will be set in ipu_backend + VLOG(10) << "found optimizer related op: " << op_type; + // initial learning_rate will be set in ipu_backend set_ops.insert(op_type); if (op_type == "sgd") { auto type = std::string{"sgd"}; @@ -267,10 +267,10 @@ void IpuOptimizerExtractPass::ApplyImpl(ir::Graph* graph) const { auto value = PADDLE_GET_CONST(float, op->GetAttr("value")); new_op.SetAttr("clip_norm", value); } else if (ignored_ops.count(op_type)) { - VLOG(10) << "Ignore optimizer releated op: " << op_type; + VLOG(10) << "Ignore optimizer related op: " << op_type; } else { PADDLE_THROW(platform::errors::InvalidArgument( - "Unknown optimizer releated op_type: %s", op_type)); + "Unknown optimizer related op_type: %s", op_type)); } } else if (op_role == OpRole::kLoss) { VLOG(10) << "found loss op type: " << op->Type(); @@ -312,7 +312,7 @@ void IpuOptimizerExtractPass::ApplyImpl(ir::Graph* graph) const { new_op.SetAttr("weight_decay_mode", std::string{"l2_regularization"}); } } else { - VLOG(10) << "No weight deacy setting found"; + VLOG(10) << "No weight decay setting found"; } // setup grad clip diff --git a/paddle/fluid/framework/ir/memory_optimize_pass/buffer_shared_cross_op_memory_reuse_pass.cc b/paddle/fluid/framework/ir/memory_optimize_pass/buffer_shared_cross_op_memory_reuse_pass.cc index 2656580228049..b41b76c99aff6 100644 --- a/paddle/fluid/framework/ir/memory_optimize_pass/buffer_shared_cross_op_memory_reuse_pass.cc +++ b/paddle/fluid/framework/ir/memory_optimize_pass/buffer_shared_cross_op_memory_reuse_pass.cc @@ -245,7 +245,7 @@ void BufferSharedCrossOpMemoryReusePass::RunOnScopeIdx(size_t idx) const { } } - // After all output args have been transversed, we should check whether + // After all output args have been traversed, we should check whether // there is new unlived var after `op` runs. for (auto op_iter = var_to_ops.begin(); op_iter != var_to_ops.end();) { // erase op from `var_to_ops` first @@ -355,7 +355,7 @@ void BufferSharedCrossOpMemoryReusePass::BuildOpDependencyMap() const { // BFS to fill `preceding_ops` graph_view.BreadthFirstVisit([&](OpHandleBase *cur_op) { // All preceding ops of cur_op should be: - // - preceding ops of cur_op, that is connected to cur_op directely + // - preceding ops of cur_op, that is connected to cur_op directly // - all preceding ops of `direct preceding ops of cur_op` auto &all_preceding_ops_of_cur_op = preceding_ops[cur_op]; for (auto &preceding_op : graph_view.PrecedingOps(cur_op)) { diff --git a/paddle/fluid/framework/ir/memory_optimize_pass/memory_optimization_var_info.h b/paddle/fluid/framework/ir/memory_optimize_pass/memory_optimization_var_info.h index 2980fa4e34a81..38238d8c7c307 100644 --- a/paddle/fluid/framework/ir/memory_optimize_pass/memory_optimization_var_info.h +++ b/paddle/fluid/framework/ir/memory_optimize_pass/memory_optimization_var_info.h @@ -89,7 +89,7 @@ class MemOptVarInfo { * scheduled in many threads inside ParallelExecutor, runtime_ref_cnt_ * must be an atomic integer to guarantee the thread safety and visibility. * - * Speciallly, if ref_cnt_ is 1, we do not need to reset runtime_ref_cnt_ + * Specially, if ref_cnt_ is 1, we do not need to reset runtime_ref_cnt_ * after iteration ends. */ size_t ref_cnt_; diff --git a/paddle/fluid/framework/ir/memory_optimize_pass/share_varinfo_into_cinn_pass.cc b/paddle/fluid/framework/ir/memory_optimize_pass/share_varinfo_into_cinn_pass.cc index 2bc3d839af549..d9ea00e3935cc 100644 --- a/paddle/fluid/framework/ir/memory_optimize_pass/share_varinfo_into_cinn_pass.cc +++ b/paddle/fluid/framework/ir/memory_optimize_pass/share_varinfo_into_cinn_pass.cc @@ -111,7 +111,7 @@ static void TakeVarInfoFromMainGraph( } // This pass will be applied on both the main graph and all cinn subgraphs, -// and it distinguishs them according to whether the graph has the +// and it distinguishes them according to whether the graph has the // kMemOptVarInfoFromMainGraph attribute or not. // On the main graph, it finds all cinn_launch ops and shares MemOptVarInfos // to their subgraphs. diff --git a/paddle/fluid/framework/ir/memory_optimize_pass/while_op_eager_deletion_pass.cc b/paddle/fluid/framework/ir/memory_optimize_pass/while_op_eager_deletion_pass.cc index 42f395da7c8a8..2d26587fdb24f 100644 --- a/paddle/fluid/framework/ir/memory_optimize_pass/while_op_eager_deletion_pass.cc +++ b/paddle/fluid/framework/ir/memory_optimize_pass/while_op_eager_deletion_pass.cc @@ -55,7 +55,7 @@ class WhileOpEagerDeletionPass : public ir::Pass { } } if (graph->IsConstructedByPartialProgram()) { - VLOG(4) << "Is Paritial Program"; + VLOG(4) << "Is Partial Program"; PADDLE_ENFORCE_LE( target_ops.size(), 1, diff --git a/paddle/fluid/framework/ir/mkldnn/cpu_quantize_squash_pass_tester.cc b/paddle/fluid/framework/ir/mkldnn/cpu_quantize_squash_pass_tester.cc index 90ed3009749ad..d2c6d981c3a2e 100644 --- a/paddle/fluid/framework/ir/mkldnn/cpu_quantize_squash_pass_tester.cc +++ b/paddle/fluid/framework/ir/mkldnn/cpu_quantize_squash_pass_tester.cc @@ -1021,7 +1021,7 @@ TEST(CpuQuantizeSquashPass, fc_dequant_more_than_one_op_after_dequant) { // b->Quantize1(Scale)->c // c->Fc1 // c->Fc2 -TEST(CpuQuantizeSquashPass, quatize_with_same_scale) { +TEST(CpuQuantizeSquashPass, quantize_with_same_scale) { auto first_scale = 1.2345f; auto second_scale = 1.2345f; auto use_mkldnn = true; @@ -1033,7 +1033,7 @@ TEST(CpuQuantizeSquashPass, quatize_with_same_scale) { } // if scales are not the same, do not fuse -TEST(CpuQuantizeSquashPass, quatize_with_different_scale) { +TEST(CpuQuantizeSquashPass, quantize_with_different_scale) { auto first_scale = 1.2345f; auto second_scale = 1.5432f; auto use_mkldnn = true; From 1e8a77cfa5452a92491e68052032c87104a07135 Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 26 Feb 2024 10:40:12 +0800 Subject: [PATCH 061/282] Fix some typos(faied, etc) (#62042) --- paddle/fluid/framework/fleet/gloo_wrapper.cc | 2 +- paddle/fluid/framework/fleet/gloo_wrapper.h | 2 +- .../fluid/framework/fleet/heter_ps/feature_value.h | 6 +++--- .../framework/fleet/heter_ps/gpu_graph_utils.h | 2 +- paddle/fluid/framework/fleet/heter_ps/heter_comm.h | 2 +- paddle/fluid/framework/io/crypto/aes_cipher.cc | 13 ------------- paddle/fluid/framework/io/crypto/aes_cipher.h | 13 ------------- .../framework/ir/fusion_group/code_generator.cc | 2 +- .../framework/ir/fusion_group/fusion_group_pass.cc | 2 +- 9 files changed, 9 insertions(+), 35 deletions(-) diff --git a/paddle/fluid/framework/fleet/gloo_wrapper.cc b/paddle/fluid/framework/fleet/gloo_wrapper.cc index 140de78bf01dc..277004b6dc164 100644 --- a/paddle/fluid/framework/fleet/gloo_wrapper.cc +++ b/paddle/fluid/framework/fleet/gloo_wrapper.cc @@ -141,7 +141,7 @@ std::vector HdfsStore::get(const std::string& key) { PADDLE_ENFORCE_EQ(read_status, 0, paddle::platform::errors::Fatal( - "HdfsStore::get, path read faied: " + path)); + "HdfsStore::get, path read failed: " + path)); #endif return result; } diff --git a/paddle/fluid/framework/fleet/gloo_wrapper.h b/paddle/fluid/framework/fleet/gloo_wrapper.h index b1f236fb60548..fa352fb8eb99f 100644 --- a/paddle/fluid/framework/fleet/gloo_wrapper.h +++ b/paddle/fluid/framework/fleet/gloo_wrapper.h @@ -95,7 +95,7 @@ class ParallelConnectContext : public gloo::rendezvous::Context { : gloo::rendezvous::Context(rank, size, base) {} virtual ~ParallelConnectContext() {} // in gloo::rendezvous::Context wait&get one by one, - // slowly in case big size, especialy in HdfsStore + // slowly in case big size, especially in HdfsStore void connectFullMesh(Store& store, // NOLINT std::shared_ptr& dev); // NOLINT struct Impl { diff --git a/paddle/fluid/framework/fleet/heter_ps/feature_value.h b/paddle/fluid/framework/fleet/heter_ps/feature_value.h index 7cdb20bb978bc..0ab35f4a2beca 100644 --- a/paddle/fluid/framework/fleet/heter_ps/feature_value.h +++ b/paddle/fluid/framework/fleet/heter_ps/feature_value.h @@ -75,7 +75,7 @@ class CommonFeatureValueAccessor { __host__ __device__ int EmbedDim() const { return embed_sgd_dim; } __host__ __device__ int EmbedXDim() const { return embedx_sgd_dim; } __host__ __device__ int EmbedWDim() const { return embedx_dim; } - __host__ __device__ int CpuPtrIndex() const { return 0; } // cpuprt uint64 + __host__ __device__ int CpuPtrIndex() const { return 0; } // cpu_ptr uint64 __host__ __device__ int DeltaScoreIndex() const { return CpuPtrIndex() + 2; } @@ -686,7 +686,7 @@ class CommonFeatureValueAccessor { std::vector embedx_w; */ std::stringstream os; - os << "cpuptr: " << common_feature_value.CpuPtr(const_cast(v)) + os << "cpu_ptr: " << common_feature_value.CpuPtr(const_cast(v)) << " delta_score: " << v[2] << " show: " << v[3] << " click: " << v[4] << " embed_w:" << v[5] << " embed_g2sum:"; for (int i = common_feature_value.EmbedG2SumIndex(); @@ -732,7 +732,7 @@ struct FeatureValue { friend std::ostream& operator<<(std::ostream& out, FeatureValue& val) { out << "show: " << val.show << " clk: " << val.clk << " slot: " << val.slot << " lr: " << val.lr << " mf_dim: " << val.mf_dim - << "cpuptr: " << val.cpu_ptr << " mf_size: " << val.mf_size << " mf:"; + << "cpu_ptr: " << val.cpu_ptr << " mf_size: " << val.mf_size << " mf:"; for (int i = 0; i < val.mf_dim + 1; ++i) { out << " " << val.mf[i]; } diff --git a/paddle/fluid/framework/fleet/heter_ps/gpu_graph_utils.h b/paddle/fluid/framework/fleet/heter_ps/gpu_graph_utils.h index 7523b349f1190..ed0c23a0fa8dc 100644 --- a/paddle/fluid/framework/fleet/heter_ps/gpu_graph_utils.h +++ b/paddle/fluid/framework/fleet/heter_ps/gpu_graph_utils.h @@ -96,7 +96,7 @@ inline void debug_gpu_memory_info(int gpu_id, const char* desc) { err, cudaSuccess, platform::errors::InvalidArgument("cudaMemGetInfo failed!")); - VLOG(0) << "updatex gpu memory on device " << gpu_id << ", " + VLOG(0) << "update gpu memory on device " << gpu_id << ", " << "avail=" << avail / 1024.0 / 1024.0 / 1024.0 << "g, " << "total=" << total / 1024.0 / 1024.0 / 1024.0 << "g, " << "use_rate=" << (total - avail) / static_cast(total) diff --git a/paddle/fluid/framework/fleet/heter_ps/heter_comm.h b/paddle/fluid/framework/fleet/heter_ps/heter_comm.h index 18e3966b220c0..1ccca57cd2979 100644 --- a/paddle/fluid/framework/fleet/heter_ps/heter_comm.h +++ b/paddle/fluid/framework/fleet/heter_ps/heter_comm.h @@ -278,7 +278,7 @@ class HeterComm { h_push_fea_sizes.resize(node_size * node_size); } }; - // pull parition shard key by devices + // pull partition shard key by devices struct PullResource { size_t h_recv_fea_num = 0; uint32_t* d_restore_keys_idx = nullptr; diff --git a/paddle/fluid/framework/io/crypto/aes_cipher.cc b/paddle/fluid/framework/io/crypto/aes_cipher.cc index 5a1258fa432f7..8802dc1b12158 100644 --- a/paddle/fluid/framework/io/crypto/aes_cipher.cc +++ b/paddle/fluid/framework/io/crypto/aes_cipher.cc @@ -12,19 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - #include "paddle/fluid/framework/io/crypto/aes_cipher.h" #include diff --git a/paddle/fluid/framework/io/crypto/aes_cipher.h b/paddle/fluid/framework/io/crypto/aes_cipher.h index a60fe1de45d7c..5dfdbd49f4f96 100644 --- a/paddle/fluid/framework/io/crypto/aes_cipher.h +++ b/paddle/fluid/framework/io/crypto/aes_cipher.h @@ -12,19 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - #pragma once #include diff --git a/paddle/fluid/framework/ir/fusion_group/code_generator.cc b/paddle/fluid/framework/ir/fusion_group/code_generator.cc index f18b5fd668659..e59c495f2dd8d 100644 --- a/paddle/fluid/framework/ir/fusion_group/code_generator.cc +++ b/paddle/fluid/framework/ir/fusion_group/code_generator.cc @@ -366,7 +366,7 @@ std::unordered_map CodeGenerator::EncodeVarNodes( // Encoding output vars. for (auto* out : output_var_nodes) { - VLOG(3) << "Ecoding output names:" << out->Name() << "(" << out + VLOG(3) << "Encoding output names:" << out->Name() << "(" << out << "), id:" << id; if (var_ids.find(out) == var_ids.end()) { var_ids[out] = id++; diff --git a/paddle/fluid/framework/ir/fusion_group/fusion_group_pass.cc b/paddle/fluid/framework/ir/fusion_group/fusion_group_pass.cc index 1f8f3b63598ce..4eba0a73ae787 100644 --- a/paddle/fluid/framework/ir/fusion_group/fusion_group_pass.cc +++ b/paddle/fluid/framework/ir/fusion_group/fusion_group_pass.cc @@ -37,7 +37,7 @@ void FusionGroupPass::ApplyImpl(ir::Graph* graph) const { // if (!phi::GPUDeviceCode::IsAvailable()) { // LOG(WARNING) // << "Disable fusion_group because CUDA Driver or NVRTC is not - // avaiable."; + // available."; // return 0; // } From 41fb7f0f2db75878c2a1365714a271c1dace328e Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 26 Feb 2024 10:41:36 +0800 Subject: [PATCH 062/282] Fix some typos(supoort, etc) (#62041) --- .../generator/codegen_utils.py | 4 +-- .../generator/eager_gen.py | 10 +++--- .../generator/python_c_gen.py | 36 ++++++++++--------- paddle/fluid/eager/grad_node_info.h | 6 ++-- 4 files changed, 29 insertions(+), 27 deletions(-) diff --git a/paddle/fluid/eager/auto_code_generator/generator/codegen_utils.py b/paddle/fluid/eager/auto_code_generator/generator/codegen_utils.py index fd19005cec39a..c13fb1cb4848c 100644 --- a/paddle/fluid/eager/auto_code_generator/generator/codegen_utils.py +++ b/paddle/fluid/eager/auto_code_generator/generator/codegen_utils.py @@ -118,7 +118,7 @@ def ReadFwdFile(filepath): # empty file loaded by yaml is None contents = yaml.load(f, Loader=yaml.FullLoader) f.close() - # not all fused ops supoort dygraph + # not all fused ops support dygraph if filepath.endswith("fused_ops.yaml") is True: new_apis = [ api @@ -134,7 +134,7 @@ def ReadBwdFile(filepath, bw_ops=None): f = open(filepath, 'r') if bw_ops is None: contents = yaml.load(f, Loader=yaml.FullLoader) - # not all fused ops supoort dygraph + # not all fused ops support dygraph if filepath.endswith("fused_backward.yaml") is True: new_apis = [ api diff --git a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py index dad46949d70ea..62327c5aa8785 100644 --- a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py +++ b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py @@ -382,7 +382,7 @@ class {} : public egr::GradNodeBase {{ }} """ -HIHGER_ORDER_DERIVATIVE_VALUE_TEMPLATE = """ if(trace_backward) {{ +HIGHER_ORDER_DERIVATIVE_VALUE_TEMPLATE = """ if(trace_backward) {{ {} // Node Construction {} @@ -1254,7 +1254,7 @@ def GenerateNodeCreationCodes(self, for_backward=False, is_inplaced=False): ) else: self.node_creation_str = ( - HIHGER_ORDER_DERIVATIVE_VALUE_TEMPLATE.format( + HIGHER_ORDER_DERIVATIVE_VALUE_TEMPLATE.format( node_creation_event_str, node_construction_str, set_attributes_str, @@ -2266,7 +2266,7 @@ def GenerateNodeDefinition( backward_attrs_list = self.backward_attrs_list backward_inplace_map = self.backward_inplace_map indent = GetIndent(1) - need_gen_trace_backard_for_inplace = False + need_gen_trace_backward_for_inplace = False # Construct grad_api function args # Order: TensorWrappers, GradTensors, Attributes @@ -2519,7 +2519,7 @@ def GenerateNodeDefinition( }} else {{ {inplace_str} }}""" - need_gen_trace_backard_for_inplace = True + need_gen_trace_backward_for_inplace = True else: inplace_for_grad_outs_str += inplace_str @@ -2623,7 +2623,7 @@ def GenerateNodeDefinition( if ( len(next_grad_node_creation_str) > 0 or is_invoke_forward_api - or need_gen_trace_backard_for_inplace + or need_gen_trace_backward_for_inplace ): compute_require_next_grad_str = f"{indent}bool trace_backward = egr::Controller::Instance().HasGrad() && create_graph;\n" diff --git a/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py b/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py index daf16f446ab12..777eea1221429 100644 --- a/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py +++ b/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py @@ -325,7 +325,7 @@ def __init__(self, forward_api_contents, namespace): # Generated Results self.python_c_function_str = "" self.python_c_function_reg_str = "" - self.python_c_funcion_declare_str = "" + self.python_c_function_declare_str = "" def CollectIsForwardOnly(self): forward_api_contents = self.forward_api_contents @@ -515,7 +515,7 @@ def GeneratePythonCFunction(self): dygraph_function_call_str, ) - # Generate Python-C Function Definetion + # Generate Python-C Function Definition self.python_c_function_str = PYTHON_C_FUNCTION_TEMPLATE.format( forward_api_name, pythonc_record_event_str, @@ -526,7 +526,7 @@ def GeneratePythonCFunction(self): noamp_dygraph_function_str, return_str, ) - self.python_c_funcion_declare_str = ( + self.python_c_function_declare_str = ( PYTHON_C_FUNCTION_DECLARE_TEMPLATE.format(name=forward_api_name) ) @@ -572,7 +572,7 @@ def GeneratePythonCFunction(self): " return ToPyObject(out, args, inplace_var_idx_map);" ) - # Generate Python-C Function Definetion + # Generate Python-C Function Definition python_c_inplace_func_str = PYTHON_C_FUNCTION_TEMPLATE.format( inplaced_forward_api_name, pythonc_record_event_str, @@ -584,7 +584,7 @@ def GeneratePythonCFunction(self): return_str, ) - python_c_funcion_declare_str = ( + python_c_function_declare_str = ( PYTHON_C_FUNCTION_DECLARE_TEMPLATE.format( name=inplaced_forward_api_name ) @@ -603,13 +603,15 @@ def GeneratePythonCFunction(self): # self.forward_api_name ending with '_' means it only has inplace api if self.forward_api_name[-1] == '_': self.python_c_function_str = python_c_inplace_func_str - self.python_c_funcion_declare_str = python_c_funcion_declare_str + self.python_c_function_declare_str = ( + python_c_function_declare_str + ) # Generate Python-C Function Registration self.python_c_function_reg_str = python_c_inplace_func_reg_str else: self.python_c_function_str += python_c_inplace_func_str - self.python_c_funcion_declare_str += ( - python_c_funcion_declare_str + self.python_c_function_declare_str += ( + python_c_function_declare_str ) # Generate Python-C Function Registration self.python_c_function_reg_str += python_c_inplace_func_reg_str @@ -652,7 +654,7 @@ def __init__(self, path): # Generated Result self.python_c_functions_str = "" self.python_c_functions_reg_str = "" - self.python_c_funcion_declare_str = "" + self.python_c_function_declare_str = "" def GeneratePythonCFunctions(self): namespace = self.namespace @@ -671,8 +673,8 @@ def GeneratePythonCFunctions(self): self.python_c_functions_reg_str += ( f_generator.python_c_function_reg_str ) - self.python_c_funcion_declare_str += ( - f_generator.python_c_funcion_declare_str + self.python_c_function_declare_str += ( + f_generator.python_c_function_declare_str ) def AttachNamespace(self): @@ -685,9 +687,9 @@ def AttachNamespace(self): self.python_c_functions_str = NAMESPACE_WRAPPER_TEMPLATE.format( namespace, python_c_functions_str ) - self.python_c_funcion_declare_str = ( + self.python_c_function_declare_str = ( NAMESPACE_WRAPPER_TEMPLATE.format( - namespace, self.python_c_funcion_declare_str + namespace, self.python_c_function_declare_str ) ) @@ -766,20 +768,20 @@ def GeneratePythonCFile(filepath, python_c_str): py_c_generator.python_c_functions_reg_str ) generated_python_c_functions_header += ( - py_c_generator.python_c_funcion_declare_str + py_c_generator.python_c_function_declare_str ) python_c_str = GeneratePythonCWrappers( generated_python_c_functions, generated_python_c_registration ) - soucre_path = args.source_path + source_path = args.source_path header_path = args.header_path - for path in [soucre_path, header_path]: + for path in [source_path, header_path]: if os.path.exists(path): os.remove(path) - GeneratePythonCFile(soucre_path, python_c_str) + GeneratePythonCFile(source_path, python_c_str) GeneratePythonCFile( header_path, PYTHON_C_H_TEMPLATE.format(body=generated_python_c_functions_header), diff --git a/paddle/fluid/eager/grad_node_info.h b/paddle/fluid/eager/grad_node_info.h index 248b4b88af4c0..7b5e36f4d5cdc 100644 --- a/paddle/fluid/eager/grad_node_info.h +++ b/paddle/fluid/eager/grad_node_info.h @@ -36,8 +36,8 @@ namespace egr { * TODO(yangzhanlue): GradNodeBase will also in charge of get the correct input * from GradOpDescMaker to GradNodeBase. * - * NOTE: GradNodeBase has a method named run, this method should be overrided by - * the specific derived class, it will prepare backward inputs and double + * NOTE: GradNodeBase has a method named run, this method should be overridden + *by the specific derived class, it will prepare backward inputs and double * backward's depends. Then, it will call C++ API of backward kernel functions * to finish backward computation. * @@ -203,7 +203,7 @@ class GradNodeBase { /** * operator() designed to contain the real backward execution logic, it should - * be overrided by derived class defined for each operator. It accepts a + * be overridden by derived class defined for each operator. It accepts a * vector of Tensor which contains grads input of current operator * * Note: why we need backward inputs and outputs construct as vector of vector From 63753f6d9255810252f9205d74c17068c9d052c1 Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 26 Feb 2024 10:42:37 +0800 Subject: [PATCH 063/282] Fix some typos(retring, etc) (#62039) --- paddle/fluid/distributed/fleet_executor/message_bus.cc | 2 +- paddle/fluid/distributed/ps/service/heter_client.cc | 2 +- paddle/fluid/distributed/ps/service/heter_server.cc | 10 +++++----- paddle/fluid/distributed/ps/service/heter_server.h | 8 ++++---- .../fluid/operators/pscore/heter_listen_and_serv_op.cc | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/paddle/fluid/distributed/fleet_executor/message_bus.cc b/paddle/fluid/distributed/fleet_executor/message_bus.cc index b5786e2393393..1860a8f3bf110 100644 --- a/paddle/fluid/distributed/fleet_executor/message_bus.cc +++ b/paddle/fluid/distributed/fleet_executor/message_bus.cc @@ -193,7 +193,7 @@ void MessageBus::ListenPort() { int interval = 100; while (server_.Start(ip_for_brpc, &options) != 0) { ++retry_times; - LOG(INFO) << "Message bus is retring for starting brpc for " << retry_times + LOG(INFO) << "Message bus is retrying for starting brpc for " << retry_times << " times. And will retry after " << interval / 1000 << " seconds."; std::this_thread::sleep_for(std::chrono::milliseconds(interval)); diff --git a/paddle/fluid/distributed/ps/service/heter_client.cc b/paddle/fluid/distributed/ps/service/heter_client.cc index e0744f96f91fa..8bdb749b0ecea 100644 --- a/paddle/fluid/distributed/ps/service/heter_client.cc +++ b/paddle/fluid/distributed/ps/service/heter_client.cc @@ -32,7 +32,7 @@ int GetMicroId(const platform::DeviceContext& ctx, PADDLE_ENFORCE_EQ(var->IsType(), true, platform::errors::InvalidArgument( - "the type of micro id shoulde be LoDTensor.")); + "the type of micro id should be LoDTensor.")); auto micro_id = -1; auto* tensor = var->GetMutable(); if (platform::is_cpu_place(tensor->place())) { diff --git a/paddle/fluid/distributed/ps/service/heter_server.cc b/paddle/fluid/distributed/ps/service/heter_server.cc index eb4d9b8304513..26dd4e6052c9b 100644 --- a/paddle/fluid/distributed/ps/service/heter_server.cc +++ b/paddle/fluid/distributed/ps/service/heter_server.cc @@ -28,10 +28,10 @@ void HeterServer::RegisterServiceHandler(std::string message_name, service_.RegisterServiceHandler(message_name, func); } -void HeterServer::StartHeterService(bool neeed_encrypt) { +void HeterServer::StartHeterService(bool need_encrypt) { server_.AddService(&service_, brpc::SERVER_DOESNT_OWN_SERVICE); brpc::ServerOptions options; - if (neeed_encrypt) { + if (need_encrypt) { options.mutable_ssl_options()->default_cert.certificate = "/cert.pem"; options.mutable_ssl_options()->default_cert.private_key = "/key.pem"; } @@ -63,10 +63,10 @@ void HeterServer::StartHeterService(bool neeed_encrypt) { VLOG(4) << "start service done"; } -void HeterServer::StartHeterInterService(bool neeed_encrypt) { +void HeterServer::StartHeterInterService(bool need_encrypt) { server_inter_.AddService(&service_, brpc::SERVER_DOESNT_OWN_SERVICE); brpc::ServerOptions options; - if (neeed_encrypt) { + if (need_encrypt) { options.mutable_ssl_options()->default_cert.certificate = "/cert.pem"; options.mutable_ssl_options()->default_cert.private_key = "/key.pem"; } @@ -99,7 +99,7 @@ void HeterServer::StartHeterInterService(bool neeed_encrypt) { VLOG(4) << "start service done"; } -void HeterServer::SetFanin(const int& fan_in) { service_.SetFanin(fan_in); } +void HeterServer::SetFanIn(const int& fan_in) { service_.SetFanIn(fan_in); } void HeterServer::WaitServerReady() { std::unique_lock lock(this->mutex_ready_); diff --git a/paddle/fluid/distributed/ps/service/heter_server.h b/paddle/fluid/distributed/ps/service/heter_server.h index f556322eb7479..61531749e95a0 100644 --- a/paddle/fluid/distributed/ps/service/heter_server.h +++ b/paddle/fluid/distributed/ps/service/heter_server.h @@ -162,7 +162,7 @@ class SendAndRecvVariableHandler final : public ServiceHandlerBase { /* timeline_.Pause(); if (timeline_.ElapsedSec() > FLAGS_switch_send_recv_timeout_s) { - VLOG(0) << "vars not consumed exceed 10 miniutes"; + VLOG(0) << "vars not consumed exceed 10 minutes"; break; } */ @@ -182,7 +182,7 @@ class SendAndRecvVariableHandler final : public ServiceHandlerBase { /* timeline_.Pause(); if (timeline_.ElapsedSec() > FLAGS_switch_send_recv_timeout_s) { - VLOG(0) << "vars not produced exceed 10 miniutes"; + VLOG(0) << "vars not produced exceed 10 minutes"; break; } */ @@ -524,7 +524,7 @@ class HeterService : public PsService { peer_endpoints_ = peer_endpoints; } - void SetFanin(const int& fan_in) { fan_in_ = fan_in; } + void SetFanIn(const int& fan_in) { fan_in_ = fan_in; } void ForceExit() { VLOG(3) << "heter service force exit"; @@ -626,7 +626,7 @@ class HeterServer { service_.SetPeerEndPoints(peer_endpoints); } - void SetFanin(const int& fan_in); + void SetFanIn(const int& fan_in); void SetServiceHandler( std::shared_ptr request_handler) { diff --git a/paddle/fluid/operators/pscore/heter_listen_and_serv_op.cc b/paddle/fluid/operators/pscore/heter_listen_and_serv_op.cc index 5dafe26464d3d..978981a6fcdf3 100644 --- a/paddle/fluid/operators/pscore/heter_listen_and_serv_op.cc +++ b/paddle/fluid/operators/pscore/heter_listen_and_serv_op.cc @@ -144,7 +144,7 @@ void HeterListenAndServOp::RunImpl(const framework::Scope &scope, heter_server_ = distributed::HeterServer::GetInstance(); heter_server_->SetEndPoint(endpoint); - heter_server_->SetFanin(fan_in); + heter_server_->SetFanIn(fan_in); auto optimize_blocks = Attr>("optimize_blocks"); From 96fa4dcfc13726397b83b63902241bee9f76a22d Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 26 Feb 2024 10:43:40 +0800 Subject: [PATCH 064/282] Fix some typos(is_taged, etc) (#62019) --- python/setup.py.in | 10 +++++----- python/setup_cinn.py.in | 6 +++--- setup.py | 34 ++++++++++++++++---------------- test/legacy_test/test_version.py | 4 ++-- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/python/setup.py.in b/python/setup.py.in index 329f092d44801..f140b66bd1c44 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -94,7 +94,7 @@ def get_xpu_xhpc_version(): else: return 'False' -def is_taged(): +def is_tagged(): try: cmd = ['git', 'describe', '--exact-match', '--tags', 'HEAD', '2>/dev/null'] git_tag = subprocess.Popen(cmd, stdout = subprocess.PIPE, cwd="@PADDLE_SOURCE_DIR@").communicate()[0].strip() @@ -125,7 +125,7 @@ cudnn_version = '%(cudnn)s' xpu_version = '%(xpu)s' xpu_xccl_version = '%(xpu_xccl)s' xpu_xhpc_version = '%(xpu_xhpc)s' -istaged = %(istaged)s +is_tagged = %(is_tagged)s commit = '%(commit)s' with_mkl = '%(with_mkl)s' cinn_version = '%(cinn)s' @@ -195,7 +195,7 @@ def show(): >>> # doctest: -SKIP """ - if istaged: + if is_tagged: print('full_version:', full_version) print('major:', major) print('minor:', minor) @@ -344,7 +344,7 @@ def cinn(): 'xpu_xccl': get_xpu_xccl_version(), 'xpu_xhpc': get_xpu_xhpc_version(), 'commit': commit, - 'istaged': is_taged(), + 'is_tagged': is_tagged(), 'with_mkl': '@WITH_MKL@', 'cinn': get_cinn_version()}) @@ -828,7 +828,7 @@ if '${CMAKE_BUILD_TYPE}' == 'Release': if('${WITH_SHARED_IR}' == 'ON'): # change rpath of pir.ext for loading 3rd party lib commands.append("patchelf --set-rpath '$ORIGIN:$ORIGIN/../libs' ${PADDLE_BINARY_DIR}/python/paddle/libs/${IR_NAME}") - # The sw_64 not suppot patchelf, so we just disable that. + # The sw_64 not support patchelf, so we just disable that. if platform.machine() != 'sw_64' and platform.machine() != 'mips64': for command in commands: if os.system(command) != 0: diff --git a/python/setup_cinn.py.in b/python/setup_cinn.py.in index cbdef191c4cd3..18d94a1629d27 100644 --- a/python/setup_cinn.py.in +++ b/python/setup_cinn.py.in @@ -63,7 +63,7 @@ def get_cudnn_version(): else: return 'False' -def is_taged(): +def is_tagged(): try: cmd = ['git', 'describe', '--exact-match', '--tags', 'HEAD', '2>/dev/null'] git_tag = subprocess.Popen(cmd, stdout = subprocess.PIPE, cwd="${PROJECT_SOURCE_DIR}").communicate()[0].strip() @@ -85,7 +85,7 @@ minor = '%(minor)d' patch = '%(patch)s' cuda_version = '%(cuda)s' cudnn_version = '%(cudnn)s' -istaged = %(istaged)s +is_tagged = %(is_tagged)s commit = '%(commit)s' with_mkl = '%(with_mkl)s' ''' @@ -108,7 +108,7 @@ with_mkl = '%(with_mkl)s' 'cuda': get_cuda_version(), 'cudnn': get_cudnn_version(), 'commit': commit, - 'istaged': is_taged(), + 'is_tagged': is_tagged(), 'with_mkl': '${WITH_MKL}'}) write_version_py(filename='${CMAKE_BINARY_DIR}/python/cinn/version/info.py') diff --git a/setup.py b/setup.py index f19c22f909d07..215f767b73d53 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,7 @@ def filter_setup_args(input_args): filter_args_list = [] for arg in input_args: if arg == 'rerun-cmake': - rerun_cmake = True # delete Cmakecache.txt and rerun cmake + rerun_cmake = True # delete CMakeCache.txt and rerun cmake continue if arg == 'only-cmake': only_cmake = True # only cmake and do not make, leave a chance for users to adjust build options @@ -314,7 +314,7 @@ def git_commit(): def _get_version_detail(idx): assert ( idx < 3 - ), "vesion info consists of %(major)d.%(minor)d.%(patch)d, \ + ), "version info consists of %(major)d.%(minor)d.%(patch)d, \ so detail index must less than 3" tag_version_regex = env_dict.get("TAG_VERSION_REGEX") paddle_version = env_dict.get("PADDLE_VERSION") @@ -400,7 +400,7 @@ def get_xpu_xhpc_version(): return 'False' -def is_taged(): +def is_tagged(): try: cmd = [ 'git', @@ -447,7 +447,7 @@ def write_version_py(filename='paddle/version/__init__.py'): xpu_version = '%(xpu)s' xpu_xccl_version = '%(xpu_xccl)s' xpu_xhpc_version = '%(xpu_xhpc)s' -istaged = %(istaged)s +is_tagged = %(is_tagged)s commit = '%(commit)s' with_mkl = '%(with_mkl)s' cinn_version = '%(cinn)s' @@ -516,7 +516,7 @@ def show(): cinn: False >>> # doctest: -SKIP """ - if istaged: + if is_tagged: print('full_version:', full_version) print('major:', major) print('minor:', minor) @@ -667,7 +667,7 @@ def cinn(): 'xpu_xccl': get_xpu_xccl_version(), 'xpu_xhpc': get_xpu_xhpc_version(), 'commit': commit, - 'istaged': is_taged(), + 'is_tagged': is_tagged(), 'with_mkl': env_dict.get("WITH_MKL"), 'cinn': get_cinn_version(), } @@ -824,13 +824,13 @@ def cmake_run(build_path): subprocess.check_call(cmake_args) -def build_run(args, build_path, envrion_var): +def build_run(args, build_path, environ_var): with cd(build_path): build_args = [] build_args.append(CMAKE) build_args += args try: - subprocess.check_call(build_args, cwd=build_path, env=envrion_var) + subprocess.check_call(build_args, cwd=build_path, env=environ_var) except (CalledProcessError, KeyboardInterrupt) as e: sys.exit(1) @@ -870,7 +870,7 @@ def build_steps(): print("build_dir:", build_dir) # run cmake to generate native build files cmake_cache_file_path = os.path.join(build_path, "CMakeCache.txt") - # if rerun_cmake is True,remove CMakeCache.txt and rerun camke + # if rerun_cmake is True,remove CMakeCache.txt and rerun cmake if os.path.isfile(cmake_cache_file_path) and rerun_cmake is True: os.remove(cmake_cache_file_path) @@ -880,13 +880,13 @@ def build_steps(): if os.path.exists(cmake_cache_file_path) and not ( bool_ninja and not os.path.exists(build_ninja_file_path) ): - print("Do not need rerun camke, everything is ready, run build now") + print("Do not need rerun cmake, everything is ready, run build now") else: cmake_run(build_path) # make if only_cmake: print( - "You have finished running cmake, the program exited,run 'ccmake build' to adjust build options and 'python setup.py install to build'" + "You have finished running cmake, the program exited,run 'cmake build' to adjust build options and 'python setup.py install to build'" ) sys.exit() run_cmake_build(build_path) @@ -1269,7 +1269,7 @@ def get_package_data_and_package_dir(): + '/python/paddle/libs/' + env_dict.get("IR_NAME") ) - # The sw_64 not suppot patchelf, so we just disable that. + # The sw_64 not support patchelf, so we just disable that. if platform.machine() != 'sw_64' and platform.machine() != 'mips64': for command in commands: if os.system(command) != 0: @@ -1610,11 +1610,11 @@ def check_build_dependency(): f.read().splitlines() ) # Specify the dependencies to install - python_dependcies_module = [] + python_dependencies_module = [] installed_packages = [] for dependency in build_dependencies: - python_dependcies_module.append( + python_dependencies_module.append( re.sub("_|-", '', re.sub(r"==.*|>=.*|<=.*", '', dependency)) ) reqs = subprocess.check_output([sys.executable, '-m', 'pip', 'freeze']) @@ -1624,7 +1624,7 @@ def check_build_dependency(): re.sub("_|-", '', r.decode().split('==')[0]).lower() ) - for dependency in python_dependcies_module: + for dependency in python_dependencies_module: if dependency.lower() not in installed_packages: raise RuntimeError(missing_modules.format(dependency=dependency)) @@ -1633,7 +1633,7 @@ def install_cpp_dist_and_build_test(install_dir, lib_test_dir, headers, libs): """install cpp distribution and build test target TODO(huangjiyi): - 1. This function will be moved when seperating C++ distribution + 1. This function will be moved when separating C++ distribution installation from python package installation. 2. Reduce the header and library files to be installed. """ @@ -1705,7 +1705,7 @@ def submodules_not_exists_or_empty(folder): end = time.time() print(f' --- Submodule initialization took {end - start:.2f} sec') except Exception: - print(' --- Submodule initalization failed') + print(' --- Submodule initialization failed') print('Please run:\n\tgit submodule update --init --recursive') sys.exit(1) diff --git a/test/legacy_test/test_version.py b/test/legacy_test/test_version.py index 2b6d8f599c582..830a0cc0f290c 100644 --- a/test/legacy_test/test_version.py +++ b/test/legacy_test/test_version.py @@ -30,10 +30,10 @@ def setUp(self): def test_check_output(self): # check commit format self.assertTrue(re.match(self._commit_regex, base_version.commit)) - self.assertTrue(isinstance(base_version.istaged, bool)) + self.assertTrue(isinstance(base_version.is_tagged, bool)) # check version format - if base_version.istaged: + if base_version.is_tagged: self.assertTrue(re.match(self._major_regex, base_version.major)) self.assertTrue(re.match(self._minor_regex, base_version.minor)) self.assertTrue(re.match(self._patch_regex, base_version.patch)) From 508c6ffa81d64215e1338426915012214b539ad6 Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Mon, 26 Feb 2024 10:53:50 +0800 Subject: [PATCH 065/282] [PIR] Fix a typo in `instruction_base` (#62052) --- .../framework/new_executor/instruction/instruction_base.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/framework/new_executor/instruction/instruction_base.cc b/paddle/fluid/framework/new_executor/instruction/instruction_base.cc index bb58d01f91d8e..cfdd5d3f9d7a9 100644 --- a/paddle/fluid/framework/new_executor/instruction/instruction_base.cc +++ b/paddle/fluid/framework/new_executor/instruction/instruction_base.cc @@ -288,7 +288,7 @@ void InstructionBase::InitInputsOutputsIds( ::pir::Operation* op, const ValueExecutionInfo& value_exec_info) { auto op_attributes = op->attributes(); std::string op_name; - if (op_attributes.count("op_name ")) { + if (op_attributes.count("op_name")) { op_name = op_attributes.at("op_name").dyn_cast().AsString(); } From e97328da23351c02ad71b68eb8b14da13fa74042 Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 26 Feb 2024 11:11:46 +0800 Subject: [PATCH 066/282] Rename paddle/cinn/common/arithmatic.h (#61910) * Fix * ci * ci * Fix --- paddle/cinn/common/CMakeLists.txt | 4 ++-- .../cinn/common/{arithmatic.cc => arithmetic.cc} | 2 +- .../cinn/common/{arithmatic.h => arithmetic.h} | 16 ++++++++-------- .../{arithmatic_test.cc => arithmetic_test.cc} | 2 +- paddle/cinn/common/cas.cc | 2 +- paddle/cinn/common/integer_set.cc | 2 +- paddle/cinn/ir/tensor.cc | 2 +- paddle/cinn/optim/ir_simplify.cc | 2 +- paddle/cinn/optim/transform_polyfor_to_for.cc | 2 +- 9 files changed, 17 insertions(+), 17 deletions(-) rename paddle/cinn/common/{arithmatic.cc => arithmetic.cc} (99%) rename paddle/cinn/common/{arithmatic.h => arithmetic.h} (87%) rename paddle/cinn/common/{arithmatic_test.cc => arithmetic_test.cc} (98%) diff --git a/paddle/cinn/common/CMakeLists.txt b/paddle/cinn/common/CMakeLists.txt index 123c848ac8358..e9c4523edd323 100644 --- a/paddle/cinn/common/CMakeLists.txt +++ b/paddle/cinn/common/CMakeLists.txt @@ -16,7 +16,7 @@ gather_srcs( ir_util.cc test_helper.cc # cuda_test_helper.cc - arithmatic.cc + arithmetic.cc cas.cc union_find.cc python_interpreter_guard.cc @@ -36,7 +36,7 @@ cinn_cc_test(test_topo_walker SRCS topo_walker_test.cc DEPS gtest glog) cinn_cc_test(test_cinn_value SRCS cinn_value_test.cc DEPS cinncore) cinn_cc_test(test_shared SRCS shared_test.cc DEPS cinncore) cinn_cc_test(test_graph_utils SRCS graph_utils_test.cc DEPS cinncore) -cinn_cc_test(test_arithmatic SRCS arithmatic_test.cc DEPS cinncore) +cinn_cc_test(test_arithmetic SRCS arithmetic_test.cc DEPS cinncore) cinn_cc_test(test_cas SRCS cas_test.cc DEPS cinncore) cinn_cc_test(test_type SRCS type_test.cc DEPS cinncore) cinn_cc_test(test_axis SRCS axis_test.cc DEPS cinncore) diff --git a/paddle/cinn/common/arithmatic.cc b/paddle/cinn/common/arithmetic.cc similarity index 99% rename from paddle/cinn/common/arithmatic.cc rename to paddle/cinn/common/arithmetic.cc index 5cabe56dff2db..e2c4ed1b8a6a7 100644 --- a/paddle/cinn/common/arithmatic.cc +++ b/paddle/cinn/common/arithmetic.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "paddle/cinn/common/arithmatic.h" +#include "paddle/cinn/common/arithmetic.h" #include #include diff --git a/paddle/cinn/common/arithmatic.h b/paddle/cinn/common/arithmetic.h similarity index 87% rename from paddle/cinn/common/arithmatic.h rename to paddle/cinn/common/arithmetic.h index e73a9bac3ede2..f90b795c8c8ff 100644 --- a/paddle/cinn/common/arithmatic.h +++ b/paddle/cinn/common/arithmetic.h @@ -13,20 +13,20 @@ // limitations under the License. /** - * This file includes some arithmatic utilities, such as simplifying/solving a + * This file includes some arithmetic utilities, such as simplifying/solving a * math equation/CINN expression. */ #pragma once -#include "paddle/cinn/ir/ir.h" +#include "paddle/cinn/ir/ir.h" // NOLINT, Should be in front of other header files -#include +#include // NOLINT -#include -#include -#include -#include -#include +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT #ifdef As #undef As diff --git a/paddle/cinn/common/arithmatic_test.cc b/paddle/cinn/common/arithmetic_test.cc similarity index 98% rename from paddle/cinn/common/arithmatic_test.cc rename to paddle/cinn/common/arithmetic_test.cc index 32eb30f9f6965..4382f279bc43a 100644 --- a/paddle/cinn/common/arithmatic_test.cc +++ b/paddle/cinn/common/arithmetic_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "paddle/cinn/common/arithmatic.h" +#include "paddle/cinn/common/arithmetic.h" #include #include diff --git a/paddle/cinn/common/cas.cc b/paddle/cinn/common/cas.cc index ddfbfce983fcb..f2e93286a04a7 100644 --- a/paddle/cinn/common/cas.cc +++ b/paddle/cinn/common/cas.cc @@ -19,7 +19,7 @@ #include #include -#include "paddle/cinn/common/arithmatic.h" +#include "paddle/cinn/common/arithmetic.h" #include "paddle/cinn/common/ir_util.h" #include "paddle/cinn/ir/ir_mutator.h" #include "paddle/cinn/ir/ir_printer.h" diff --git a/paddle/cinn/common/integer_set.cc b/paddle/cinn/common/integer_set.cc index 2a344eb00d5a5..f6d6446b9bb24 100644 --- a/paddle/cinn/common/integer_set.cc +++ b/paddle/cinn/common/integer_set.cc @@ -14,7 +14,7 @@ #include "paddle/cinn/common/integer_set.h" -#include "paddle/cinn/common/arithmatic.h" +#include "paddle/cinn/common/arithmetic.h" #include "paddle/cinn/ir/ir_mutator.h" #include "paddle/cinn/ir/op/ir_operators.h" #include "paddle/cinn/ir/utils/ir_copy.h" diff --git a/paddle/cinn/ir/tensor.cc b/paddle/cinn/ir/tensor.cc index 7b3f15c6ed0be..5224a2172ac5c 100644 --- a/paddle/cinn/ir/tensor.cc +++ b/paddle/cinn/ir/tensor.cc @@ -18,7 +18,7 @@ #include "paddle/cinn/ast_gen_ius/tensor_group.h" #include "paddle/cinn/cinn.h" -#include "paddle/cinn/common/arithmatic.h" +#include "paddle/cinn/common/arithmetic.h" #include "paddle/cinn/common/axis.h" #include "paddle/cinn/common/cas.h" #include "paddle/cinn/common/common.h" diff --git a/paddle/cinn/optim/ir_simplify.cc b/paddle/cinn/optim/ir_simplify.cc index fa69dd19ff0c6..c92ac15e5deb6 100644 --- a/paddle/cinn/optim/ir_simplify.cc +++ b/paddle/cinn/optim/ir_simplify.cc @@ -21,7 +21,7 @@ #include #include -#include "paddle/cinn/common/arithmatic.h" +#include "paddle/cinn/common/arithmetic.h" #include "paddle/cinn/common/cas.h" #include "paddle/cinn/common/ir_util.h" #include "paddle/cinn/ir/ir_mutator.h" diff --git a/paddle/cinn/optim/transform_polyfor_to_for.cc b/paddle/cinn/optim/transform_polyfor_to_for.cc index 8a7392ed5d54b..ff29bb0058801 100644 --- a/paddle/cinn/optim/transform_polyfor_to_for.cc +++ b/paddle/cinn/optim/transform_polyfor_to_for.cc @@ -17,7 +17,7 @@ #include #include -#include "paddle/cinn/common/arithmatic.h" +#include "paddle/cinn/common/arithmetic.h" #include "paddle/cinn/common/cas.h" #include "paddle/cinn/common/ir_util.h" #include "paddle/cinn/common/type.h" From 72135041fdff8489cc62ca887a1158c665a2de39 Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 26 Feb 2024 11:13:20 +0800 Subject: [PATCH 067/282] Fix some typos (protocal, staic, etc.), test=document_fix (#61882) * Fix, test=document_fix * ci --- cmake/cinn/core.cmake | 2 +- cmake/cinn/external/absl.cmake | 2 +- cmake/cuda.cmake | 2 +- cmake/external/cub.cmake | 2 +- cmake/external/lapack.cmake | 2 +- cmake/external/mklml.cmake | 2 +- cmake/external/protobuf.cmake | 2 +- cmake/external/pybind11.cmake | 2 +- cmake/flags.cmake | 2 +- cmake/generic.cmake | 18 +++++++++--------- cmake/init.cmake | 4 ++-- cmake/phi.cmake | 4 ++-- cmake/third_party.cmake | 12 ++++++------ cmake/unity_build.cmake | 4 ++-- 14 files changed, 30 insertions(+), 30 deletions(-) diff --git a/cmake/cinn/core.cmake b/cmake/cinn/core.cmake index dedefc57a698b..66741e7f8182b 100644 --- a/cmake/cinn/core.cmake +++ b/cmake/cinn/core.cmake @@ -242,7 +242,7 @@ function(cinn_merge_static_libs TARGET_NAME) COMMAND ${CMAKE_COMMAND} -E touch ${target_SRCS} DEPENDS ${libs}) - # Generate dummy staic lib + # Generate dummy static lib file(WRITE ${target_SRCS} "const char *dummy_${TARGET_NAME} = \"${target_SRCS}\";") add_library(${TARGET_NAME} STATIC ${target_SRCS}) diff --git a/cmake/cinn/external/absl.cmake b/cmake/cinn/external/absl.cmake index 0b3f3d685ed80..8d9e0e45b45ba 100644 --- a/cmake/cinn/external/absl.cmake +++ b/cmake/cinn/external/absl.cmake @@ -50,7 +50,7 @@ ExternalProject_Add( BUILD_BYPRODUCTS ${ABSL_INSTALL_DIR}/lib/libabsl_bad_variant_access.a BUILD_BYPRODUCTS ${ABSL_INSTALL_DIR}/lib/libabsl_raw_hash_set.a) -# It may be more convinent if we just include all absl libs +# It may be more convenient if we just include all absl libs set(ABSL_LIB_NAMES hash wyhash diff --git a/cmake/cuda.cmake b/cmake/cuda.cmake index 8cd7b835629d3..81a7228629d25 100644 --- a/cmake/cuda.cmake +++ b/cmake/cuda.cmake @@ -300,7 +300,7 @@ set(CUDA_PROPAGATE_HOST_FLAGS OFF) # So, don't set these flags here. set(CMAKE_CUDA_STANDARD 17) -# (Note) For windows, if delete /W[1-4], /W1 will be added defaultly and conflic with -w +# (Note) For windows, if delete /W[1-4], /W1 will be added defaultly and conflict with -w # So replace /W[1-4] with /W0 if(WIN32) string(REGEX REPLACE "/W[1-4]" " /W0 " CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS}") diff --git a/cmake/external/cub.cmake b/cmake/external/cub.cmake index 74f0d3188b534..681cfb2cfa6cf 100644 --- a/cmake/external/cub.cmake +++ b/cmake/external/cub.cmake @@ -29,7 +29,7 @@ if(${CMAKE_CUDA_COMPILER_VERSION} GREATER_EQUAL 11.6) set(CUB_TAG 1.16.0) execute_process(COMMAND git --git-dir=${CUB_SOURCE_DIR}/.git --work-tree=${CUB_SOURCE_DIR} checkout ${CUB_TAG}) - # cub 1.16.0 is not compitable with current thrust version + # cub 1.16.0 is not compatible with current thrust version add_definitions(-DTHRUST_IGNORE_CUB_VERSION_CHECK) else() set(CUB_TAG 1.8.0) diff --git a/cmake/external/lapack.cmake b/cmake/external/lapack.cmake index 1b5032ab6ff1b..62da0987085d1 100644 --- a/cmake/external/lapack.cmake +++ b/cmake/external/lapack.cmake @@ -20,7 +20,7 @@ set(LAPACK_DOWNLOAD_DIR set(LAPACK_INSTALL_DIR ${THIRD_PARTY_PATH}/install/lapack) set(LAPACK_LIB_DIR ${LAPACK_INSTALL_DIR}/lib) -# Note(zhouwei): lapack need fortan compiler which many machines don't have, so use precompiled library. +# Note(zhouwei): lapack need fortran compiler which many machines don't have, so use precompiled library. # use lapack tag v3.10.0 on 06/28/2021 https://github.com/Reference-LAPACK/lapack if(LINUX) set(LAPACK_FILE diff --git a/cmake/external/mklml.cmake b/cmake/external/mklml.cmake index d5e3fa16bf0e2..f7c2035cd0a1f 100644 --- a/cmake/external/mklml.cmake +++ b/cmake/external/mklml.cmake @@ -35,7 +35,7 @@ if(WIN32) else() #TODO(intel-huying): # Now enable csrmm function in mklml library temporarily, - # it will be updated as offical version later. + # it will be updated as official version later. set(MKLML_FILE "csrmm_mklml_lnx_2019.0.5.tgz" CACHE STRING "" FORCE) diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake index ab5ab5e47604e..09aa9a2b0726e 100755 --- a/cmake/external/protobuf.cmake +++ b/cmake/external/protobuf.cmake @@ -322,7 +322,7 @@ if(WITH_IPU) elseif(WITH_ARM_BRPC) set(PROTOBUF_VERSION 21.12-baidu-ee-common) elseif(WIN32) - #Lower version prootbuf is used for widows + #Lower version protobuf is used for widows set(PROTOBUF_VERSION 21.12) else() set(PROTOBUF_VERSION 21.12) diff --git a/cmake/external/pybind11.cmake b/cmake/external/pybind11.cmake index 6ce8290d72f42..dcb890b294cfb 100644 --- a/cmake/external/pybind11.cmake +++ b/cmake/external/pybind11.cmake @@ -28,7 +28,7 @@ if(NOT WIN32) file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/pybind/cast.h.patch native_dst) # Note: [Why calling some `git` commands before `patch`?] - # Paddle's CI uses cache to accelarate the make process. However, error might raise when patch codes in two scenarios: + # Paddle's CI uses cache to accelerate the make process. However, error might raise when patch codes in two scenarios: # 1. Patch to the wrong version: the tag version of CI's cache falls behind PYBIND_TAG, use `git checkout ${PYBIND_TAG}` to solve this. # 2. Patch twice: the tag version of cache == PYBIND_TAG, but patch has already applied to cache. set(PYBIND_PATCH_COMMAND diff --git a/cmake/flags.cmake b/cmake/flags.cmake index bb94e5627bd62..ee60dd1485818 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -17,7 +17,7 @@ function(CheckCompilerCXX14Flag) elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang") # cmake >= 3.0 compiler id "AppleClang" on Mac OS X, otherwise "Clang" - # Apple Clang is a different compiler than upstream Clang which havs different version numbers. + # Apple Clang is a different compiler than upstream Clang which has different version numbers. # https://gist.github.com/yamaya/2924292 if(APPLE) # cmake < 3.0 compiler id "Clang" on Mac OS X if(${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 5.1) diff --git a/cmake/generic.cmake b/cmake/generic.cmake index 49eec5ba879e0..c18e25fa84a64 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -44,7 +44,7 @@ # # nv_library(example SRCS example.cu) # -# To specify that a library new_example.a depends on other libraies: +# To specify that a library new_example.a depends on other libraries: # # cc_library(new_example SRCS new_example.cc DEPS example) # @@ -72,7 +72,7 @@ # nv_test(example_test SRCS example_test.cu DEPS example) # # It is pretty often that executable and test binaries depend on -# pre-defined external libaries like glog and gflags defined in +# pre-defined external libraries like glog and gflags defined in # /cmake/external/*.cmake: # # cc_test(example_test SRCS example_test.cc DEPS example glog gflags) @@ -257,7 +257,7 @@ function(merge_static_libs TARGET_NAME) COMMAND ${CMAKE_COMMAND} -E touch ${target_SRCS} DEPENDS ${libs}) - # Generate dummy staic lib + # Generate dummy static lib generate_dummy_static_lib(LIB_NAME ${TARGET_NAME} FILE_PATH ${target_SRCS} GENERATOR "generic.cmake:merge_static_libs") target_link_libraries(${TARGET_NAME} ${libs_deps}) @@ -310,7 +310,7 @@ function(merge_static_libs TARGET_NAME) foreach(lib ${libs}) set(libfiles ${libfiles} $) endforeach() - # msvc compiler will put libarary in directory of "/Release/xxxlib" by default + # msvc compiler will put library in directory of "/Release/xxxlib" by default add_custom_command( TARGET ${TARGET_NAME} POST_BUILD @@ -530,7 +530,7 @@ function(cc_test TARGET_NAME) "${multiValueArgs}" ${ARGN}) if(WIN32) # NOTE(zhiqiu): on windows platform, the symbols should be exported - # explicitly by __declspec(dllexport), however, there are serveral + # explicitly by __declspec(dllexport), however, there are several # symbols not exported, and link error occurs. # so, the tests are not built against dynamic libraries now. cc_test_old( @@ -577,7 +577,7 @@ function(cc_test_old TARGET_NAME) cmake_parse_arguments(cc_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) cc_test_build(${TARGET_NAME} SRCS ${cc_test_SRCS} DEPS ${cc_test_DEPS}) - # we dont test hcom op, because it need complex configuration + # we donot test hcom op, because it need complex configuration # with more than one machine cc_test_run(${TARGET_NAME} COMMAND ${TARGET_NAME} ARGS ${cc_test_ARGS}) elseif(WITH_TESTING AND NOT TEST ${TARGET_NAME}) @@ -809,7 +809,7 @@ function(hip_binary TARGET_NAME) set(multiValueArgs SRCS DEPS) cmake_parse_arguments(hip_binary "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - # FindHIP.cmake defined hip_add_executable, HIP_SOURCE_PROPERTY_FORMAT is requried for .cc files + # FindHIP.cmake defined hip_add_executable, HIP_SOURCE_PROPERTY_FORMAT is required for .cc files hip_add_executable(${TARGET_NAME} ${hip_binary_SRCS}) if(hip_binary_DEPS) target_link_libraries(${TARGET_NAME} ${hip_binary_DEPS}) @@ -826,7 +826,7 @@ function(hip_test TARGET_NAME) cmake_parse_arguments(hip_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) # FindHIP.cmake defined hip_add_executable, - # HIP_SOURCE_PROPERTY_FORMAT is requried for .cc files + # HIP_SOURCE_PROPERTY_FORMAT is required for .cc files hip_add_executable(${TARGET_NAME} ${hip_test_SRCS}) # "-pthread -ldl -lrt" is defined in CMAKE_CXX_LINK_EXECUTABLE target_link_options(${TARGET_NAME} PRIVATE -pthread -ldl -lrt) @@ -1231,7 +1231,7 @@ function(grpc_library TARGET_NAME) get_filename_component(PROTO_WE ${grpc_library_PROTO} NAME_WE) get_filename_component(PROTO_PATH ${ABS_PROTO} PATH) - # FIXME(putcn): the follwoing line is supposed to generate *.pb.h and cc, but + # FIXME(putcn): the following line is supposed to generate *.pb.h and cc, but # somehow it didn't. line 602 to 604 is to patching this. Leaving this here # for now to enable dist CI. paddle_protobuf_generate_cpp(grpc_proto_srcs grpc_proto_hdrs "${ABS_PROTO}") diff --git a/cmake/init.cmake b/cmake/init.cmake index 86c43cb233bfc..201f66be82f72 100644 --- a/cmake/init.cmake +++ b/cmake/init.cmake @@ -43,8 +43,8 @@ else() set(CMAKE_CUDA_FLAGS_MINSIZEREL "-Xcompiler=\"-MD -O1 -Ob1\" -DNDEBUG") endif() - # It can specify CUDA compile flag manualy, - # its use is to remvoe /Zi to reduce GPU static library size. But it's dangerous + # It can specify CUDA compile flag manually, + # its use is to remove /Zi to reduce GPU static library size. But it's dangerous # because CUDA will update by nvidia, then error will occur. # Now, it's only used in VS2015 + CUDA:[10.0, 10.2] set(WIN_PROPS ${CMAKE_SOURCE_DIR}/cmake/paddle_win.props) diff --git a/cmake/phi.cmake b/cmake/phi.cmake index bfb6a88eb62a7..4aabcbb0f7607 100644 --- a/cmake/phi.cmake +++ b/cmake/phi.cmake @@ -119,7 +119,7 @@ function(kernel_declare TARGET_LIST) is_all_backend "${first_registry}") if(NOT is_all_backend STREQUAL "") - # parse the registerd kernel message + # parse the registered kernel message string( REPLACE "PD_REGISTER_KERNEL_FOR_ALL_BACKEND_DTYPE_EXCEPT_CUSTOM(" "" kernel_msg "${first_registry}") @@ -131,7 +131,7 @@ function(kernel_declare TARGET_LIST) is_all_backend "${first_registry}") - # parse the registerd kernel message + # parse the registered kernel message string(REPLACE "PD_REGISTER_KERNEL_FOR_ALL_BACKEND_DTYPE(" "" kernel_msg "${first_registry}") endif() diff --git a/cmake/third_party.cmake b/cmake/third_party.cmake index 17c428660b223..2d8020adcf7d0 100755 --- a/cmake/third_party.cmake +++ b/cmake/third_party.cmake @@ -13,7 +13,7 @@ # limitations under the License. include(ExternalProject) -# Creat a target named "third_party", which can compile external dependencies on all platform(windows/linux/mac) +# Create a target named "third_party", which can compile external dependencies on all platform(windows/linux/mac) set(THIRD_PARTY_PATH "${CMAKE_BINARY_DIR}/third_party" @@ -55,8 +55,8 @@ if(NOT WITH_SETUP_INSTALL) endif() endif() -# cache funciton to avoid repeat download code of third_party. -# This function has 4 parameters, URL / REPOSITOR / TAG / DIR: +# cache function to avoid repeat download code of third_party. +# This function has 4 parameters, URL / REPOSITORY / TAG / DIR: # 1. URL: specify download url of 3rd party # 2. REPOSITORY: specify git REPOSITORY of 3rd party # 3. TAG: specify git tag/branch/commitID of 3rd party @@ -64,7 +64,7 @@ endif() # # The function Return 1 PARENT_SCOPE variables: # - ${TARGET}_DOWNLOAD_CMD: Simply place "${TARGET}_DOWNLOAD_CMD" in ExternalProject_Add, -# and you no longer need to set any donwnload steps in ExternalProject_Add. +# and you no longer need to set any download steps in ExternalProject_Add. # For example: # Cache_third_party(${TARGET} # REPOSITORY ${TARGET_REPOSITORY} @@ -145,10 +145,10 @@ macro(UNSET_VAR VAR_NAME) unset(${VAR_NAME}) endmacro() -# Funciton to Download the dependencies during compilation +# Function to Download the dependencies during compilation # This function has 2 parameters, URL / DIRNAME: # 1. URL: The download url of 3rd dependencies -# 2. NAME: The name of file, that determin the dirname +# 2. NAME: The name of file, that determine the dirname # function(file_download_and_uncompress URL NAME) set(options "") diff --git a/cmake/unity_build.cmake b/cmake/unity_build.cmake index d1b97cf08f60c..f7c88a6bb4a4e 100644 --- a/cmake/unity_build.cmake +++ b/cmake/unity_build.cmake @@ -74,7 +74,7 @@ endfunction() # If the source file does not hit any registed groups, use itself. # This function put the actual combination relationship in variables instead of # writing the unity source file. The reason is that writing unity source file -# will change the timestampe and affect the effect of retaining the build +# will change the timestamp and affect the effect of retaining the build # directory on Windows. # Here you need to specify the source type which belongs to cc or cu. function(compose_unity_target_sources TARGET TYPE) @@ -84,7 +84,7 @@ function(compose_unity_target_sources TARGET TYPE) PROPERTY ${TARGET}_${TYPE}_group_index) foreach(src ${ARGN}) set(unity_file "") - # Note(zhouwei25): UB use the path releative to CMAKE_SOURCE_DIR. + # Note(zhouwei25): UB use the path relative to CMAKE_SOURCE_DIR. # If use absolute path, sccache/ccache hit rate will be reduced. if(IS_ABSOLUTE ${src}) set(src_absolute_path ${src}) From 665f97bb6c020c53ec7951547875a41299b038cd Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 26 Feb 2024 11:17:40 +0800 Subject: [PATCH 068/282] Fix some typos (optioanl, etc.) (#61836) --- .../event_garbage_collector.cc | 8 +++---- .../pir_adaptor/pir_adaptor_util.cc | 23 ++++++++++--------- .../pir_adaptor/pir_adaptor_util.h | 4 ++-- .../pir/transforms/constant_folding_pass.cc | 12 +++++----- paddle/phi/api/yaml/generator/dist_api_gen.py | 6 ++--- 5 files changed, 27 insertions(+), 26 deletions(-) diff --git a/paddle/fluid/framework/new_executor/garbage_collector/event_garbage_collector.cc b/paddle/fluid/framework/new_executor/garbage_collector/event_garbage_collector.cc index e8bee7705fe30..1b4f5128589d6 100644 --- a/paddle/fluid/framework/new_executor/garbage_collector/event_garbage_collector.cc +++ b/paddle/fluid/framework/new_executor/garbage_collector/event_garbage_collector.cc @@ -31,8 +31,8 @@ InterpreterCoreEventGarbageCollector::InterpreterCoreEventGarbageCollector( /*allow_spinning*/ true, /*track_task*/ false); queue_ = CreateSingleThreadedWorkQueue(options); - for (auto& instruc : vec_instruction) { - gc_event_.emplace_back(instruc.DeviceContext().GetPlace(), + for (auto& instruct : vec_instruction) { + gc_event_.emplace_back(instruct.DeviceContext().GetPlace(), platform::GenerateDeviceEventFlag()); } } @@ -44,8 +44,8 @@ InterpreterCoreEventGarbageCollector::InterpreterCoreEventGarbageCollector( /*allow_spinning*/ true, /*track_task*/ false); queue_ = CreateSingleThreadedWorkQueue(options); - for (auto& instruc : vec_instruction) { - gc_event_.emplace_back(instruc->DeviceContext().GetPlace(), + for (auto& instruct : vec_instruction) { + gc_event_.emplace_back(instruct->DeviceContext().GetPlace(), platform::GenerateDeviceEventFlag()); } } diff --git a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc index aa9003cb164f9..4894e64a8f4d1 100644 --- a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc +++ b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc @@ -497,7 +497,7 @@ void HandleForSpecialOp(pir::Operation* op, } PADDLE_ENFORCE(var, paddle::platform::errors::InvalidArgument( - "The variable %s shoud exist", name)); + "The variable %s should exist", name)); value_exe_info->Add(value, name); } else if (op->isa()) { @@ -531,7 +531,7 @@ void HandleForSpecialOp(pir::Operation* op, .AsString(); auto value = op->operand_source(0); - // change opreand name to param_name + // change operand name to param_name auto orig_name = value_exe_info->GetValue2VarName().at(value); if (param_name == orig_name) { @@ -547,22 +547,23 @@ void HandleForSpecialOp(pir::Operation* op, value_exe_info->Rename(param_name, orig_name); } else if (op->isa()) { - VLOG(6) << "Handle for builtin.shadow_ouptut"; + VLOG(6) << "Handle for builtin.shadow_output"; auto var_name = op->attributes() .at("output_name") .dyn_cast() .AsString(); auto value = op->operand_source(0); + Scope* scope = const_cast(value_exe_info->GetScope()); if (value.defining_op()->HasAttribute(kAttrIsPersistable) && value.attribute(kAttrIsPersistable).data()) { - VLOG(6) << "Handle for builtin.shadow_ouptut persistable value:" + VLOG(6) << "Handle for builtin.shadow_output persistable value:" << var_name; scope = const_cast(value_exe_info->GetScope()->root()); } - // change opreand name to param_name + // change operand name to param_name auto orig_name = value_exe_info->GetValue2VarName().at(value); if (var_name == orig_name) { @@ -603,7 +604,7 @@ void HandleForSpecialOp(pir::Operation* op, PADDLE_ENFORCE_EQ(value_exe_info->GetValue2VarName().count(in_value), true, phi::errors::PreconditionNotMet( - "input of buildin slice not in name map")); + "input of builtin slice not in name map")); int index = op->attributes().at("index").dyn_cast().data(); @@ -626,7 +627,7 @@ void HandleForSpecialOp(pir::Operation* op, PADDLE_ENFORCE_EQ(value_exe_info->GetValue2VarName().count(in_value), true, phi::errors::PreconditionNotMet( - "input of buildin split not in name map")); + "input of builtin split not in name map")); auto in_var = value_exe_info->GetVarByValue(in_value); auto variable_array = in_var->Get(); @@ -817,7 +818,7 @@ void BuildRuntimeContext(pir::Operation* op, pir::Value ptr = op->operand_source(index); if (!IsInvalid(ptr)) { - VLOG(8) << "ctx->EmplaceBackInput : an optioanl input " << name; + VLOG(8) << "ctx->EmplaceBackInput : an optional input " << name; continue; } @@ -845,7 +846,7 @@ void BuildRuntimeContext(pir::Operation* op, auto legacy_arg_name = op_normalizer.GetLegacyArgName(fluid_op_name, name); if (!IsInvalid(ptr)) { - VLOG(8) << "ctx->EmplaceBackOutput : an optioanl output " << name; + VLOG(8) << "ctx->EmplaceBackOutput : an optional output " << name; continue; } @@ -906,7 +907,7 @@ std::shared_ptr BuildOperatorBase( auto legacy_attr_name = op_normalizer.GetLegacyArgName(fluid_op_name, name); if (!IsInvalid(ptr)) { - VLOG(8) << "Push back inputs to VariableNameMap : an optioanl input " + VLOG(8) << "Push back inputs to VariableNameMap : an optional input " << name; continue; } @@ -1004,7 +1005,7 @@ std::shared_ptr BuildOperatorBase( op_normalizer.GetLegacyArgName(fluid_op_name, output_name_list[i]); if (!IsInvalid(ptr)) { - VLOG(8) << "Push back outputs to VariableNameMap : an optioanl output " + VLOG(8) << "Push back outputs to VariableNameMap : an optional output " << legacy_arg_name; continue; } diff --git a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.h b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.h index 92072e6c9807f..0aa3fa0f80db2 100644 --- a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.h +++ b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.h @@ -190,7 +190,7 @@ void BuildPhiContext(pir::Operation* op, InType optional_input(temp); ctx->EmplaceBackInput(optional_input); } - VLOG(8) << "ctx->EmplaceBackInput : an optioanl input " << t; + VLOG(8) << "ctx->EmplaceBackInput : an optional input " << t; continue; } @@ -441,7 +441,7 @@ void BuildPhiContext(pir::Operation* op, OutType optional_input(temp); ctx->EmplaceBackOutput(optional_input); } - VLOG(8) << "ctx->EmplaceBackOutput : an optioanl output"; + VLOG(8) << "ctx->EmplaceBackOutput : an optional output"; continue; } diff --git a/paddle/fluid/pir/transforms/constant_folding_pass.cc b/paddle/fluid/pir/transforms/constant_folding_pass.cc index 71c836cdcf96d..d7834f9195bfd 100644 --- a/paddle/fluid/pir/transforms/constant_folding_pass.cc +++ b/paddle/fluid/pir/transforms/constant_folding_pass.cc @@ -74,7 +74,7 @@ class ConstantFoldingPattern : public pir::RewritePattern { } bool Match(pir::Operation* op) const override { - VLOG(4) << "constant_folding_pass applys match on [" << op->name() + VLOG(4) << "constant_folding_pass applies match on [" << op->name() << "] op"; // 1. Some ops do not need to be processed if (op->HasTrait() || @@ -143,7 +143,7 @@ class ConstantFoldingPattern : public pir::RewritePattern { } } - // 7. maybe affect performence + // 7. maybe affect performance if (op->isa()) { auto next_ops = pir::GetUseOpsForOutput(op, 0); for (auto [next_op, _] : next_ops) { @@ -161,7 +161,7 @@ class ConstantFoldingPattern : public pir::RewritePattern { void Rewrite(pir::Operation* op, pir::PatternRewriter& rewriter) const override { // NOLINT - VLOG(4) << "constant_folding_pass applys rewrite on [" << op->name() + VLOG(4) << "constant_folding_pass applies rewrite on [" << op->name() << "] op"; auto output_var_names = RunOp(op, rewriter); @@ -410,7 +410,7 @@ class ConstantFoldingPatternForTrain : public ConstantFoldingPattern { context, suffix, place, scope, exe_config, deleted_vars) {} bool Match(pir::Operation* op) const override { - VLOG(4) << "constant_folding_pass applys match on [" << op->name() + VLOG(4) << "constant_folding_pass applies match on [" << op->name() << "] op"; if (!ConstantFoldingPattern::Match(op)) { return false; @@ -427,7 +427,7 @@ class ConstantFoldingPatternForTrain : public ConstantFoldingPattern { void Rewrite(pir::Operation* op, pir::PatternRewriter& rewriter) const override { // NOLINT - VLOG(4) << "constant_folding_pass for train applys rewrite on [" + VLOG(4) << "constant_folding_pass for train applies rewrite on [" << op->name() << "] op"; auto output_var_names = RunOp(op, rewriter); @@ -454,7 +454,7 @@ class ConstantFoldingPatternForTrain : public ConstantFoldingPattern { rewriter.ReplaceAllUsesWith(op->result(i), constant_op->result(0)); } rewriter.EraseOp(op); - VLOG(4) << "constant_folding_pass for traun applied rewrite on [" + VLOG(4) << "constant_folding_pass for train applied rewrite on [" << op->name() << "] op"; } }; diff --git a/paddle/phi/api/yaml/generator/dist_api_gen.py b/paddle/phi/api/yaml/generator/dist_api_gen.py index 958551d96e34f..03d65a920b9d2 100644 --- a/paddle/phi/api/yaml/generator/dist_api_gen.py +++ b/paddle/phi/api/yaml/generator/dist_api_gen.py @@ -36,7 +36,7 @@ // Kernel Dispatch Body{} }} """ -DIPATCH_END_GUARD_TEMPLATE = """ +DISPATCH_END_GUARD_TEMPLATE = """ PADDLE_THROW(phi::errors::Unimplemented( "The kernel of ({}) for input tensors is unimplemented, please check the type of input tensors.")); """ @@ -1899,7 +1899,7 @@ def gene_base_api_code(self, inplace_flag=False): self.get_define_args(inplace_flag), self.gene_kernel_select(), kernel_dispatch_code - + DIPATCH_END_GUARD_TEMPLATE.format(self.api), + + DISPATCH_END_GUARD_TEMPLATE.format(self.api), ) else: dist_branch_code = "" @@ -1947,7 +1947,7 @@ def generate_api( if is_fused_ops_yaml is True else "paddle/phi/api/include/api.h" ) - # not all fused ops supoort dygraph + # not all fused ops support dygraph if is_fused_ops_yaml is True: new_apis = [ api From 2ea42ce2847781cc2d68a5c8a07afa33bc645119 Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 26 Feb 2024 11:18:03 +0800 Subject: [PATCH 069/282] Fix some typos(kernel_dialtion, etc) (#62013) --- .../group_with_group_merge_pass.cc | 2 +- .../group_merge/group_with_group_merge_util.h | 2 +- paddle/cinn/hlir/framework/graph_compiler.cc | 4 ++-- paddle/cinn/hlir/framework/graph_compiler.h | 2 +- paddle/cinn/hlir/framework/instruction.cc | 18 ++++++++--------- paddle/cinn/hlir/framework/pir_compiler.cc | 6 +++--- .../hlir/framework/print_graph_pass_test.cc | 2 +- paddle/cinn/hlir/op/nn.cc | 18 ++++++++--------- paddle/cinn/hlir/op/reduction.cc | 12 +++++------ paddle/cinn/hlir/pass/fusion_merge_pass.cc | 20 +++++++++---------- .../cinn/hlir/pass/fusion_merge_pass_util.h | 2 +- .../hlir/pass/general_fusion_merge_pass.cc | 16 +++++++-------- .../graph_group_fuse_helper.h | 2 +- paddle/cinn/hlir/pass/op_fusion_pass_test.cc | 12 +++++------ paddle/cinn/hlir/pe/ir_schedule_pe.cc | 4 ++-- paddle/cinn/hlir/pe/transform.cc | 2 +- 16 files changed, 62 insertions(+), 62 deletions(-) diff --git a/paddle/cinn/hlir/dialect/operator/transforms/group_merge/group_with_group_merge_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/group_merge/group_with_group_merge_pass.cc index 40e03a6574832..7ee55cc7c9396 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/group_merge/group_with_group_merge_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/group_merge/group_with_group_merge_pass.cc @@ -355,7 +355,7 @@ bool GraphGroupFuseHelper::AllOutputsSameSize( template bool GraphGroupFuseHelper::HorizontalElementwiseFuseReduce( const OpGroupPtr& src, const OpGroupPtr& dst) const { - return honrizontal_elementwise_fuse_reduce(src.GetGroup(), dst.GetGroup()); + return horizontal_elementwise_fuse_reduce(src.GetGroup(), dst.GetGroup()); } template diff --git a/paddle/cinn/hlir/dialect/operator/transforms/group_merge/group_with_group_merge_util.h b/paddle/cinn/hlir/dialect/operator/transforms/group_merge/group_with_group_merge_util.h index f1094dc78e796..f6c17ae28ebfb 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/group_merge/group_with_group_merge_util.h +++ b/paddle/cinn/hlir/dialect/operator/transforms/group_merge/group_with_group_merge_util.h @@ -127,7 +127,7 @@ inline bool elementwise_fuse_broadcast( return true; } -inline bool honrizontal_elementwise_fuse_reduce( +inline bool horizontal_elementwise_fuse_reduce( const std::shared_ptr& first, const std::shared_ptr& second) { std::shared_ptr ele_group, reduce_group; diff --git a/paddle/cinn/hlir/framework/graph_compiler.cc b/paddle/cinn/hlir/framework/graph_compiler.cc index ffa599805f13e..4ed9ff14d217b 100644 --- a/paddle/cinn/hlir/framework/graph_compiler.cc +++ b/paddle/cinn/hlir/framework/graph_compiler.cc @@ -210,7 +210,7 @@ static void BufferMallocWithCallback(void* args, int num_args) { for (int i = 0; i < num_args; ++i) { cinn_buffer_t* buffer = static_cast(pod_args[i]); CHECK(buffer->external_malloc) - << "external_malloc is nullptr at " << i << "-th argumemnts"; + << "external_malloc is nullptr at " << i << "-th arguments"; buffer->external_malloc->operator()(nullptr, buffer); } } @@ -282,7 +282,7 @@ void GraphCompiler::InsertBufferHandlers( malloc_var_names, std::vector({}), function_name); - VLOG(4) << "seting malloc function " << function_name << " for var " + VLOG(4) << "setting malloc function " << function_name << " for var " << cinn::utils::Join(malloc_var_names, ", "); malloc_instr->SetLoweredFunc( reinterpret_cast(BufferMallocWithCallback), function_name); diff --git a/paddle/cinn/hlir/framework/graph_compiler.h b/paddle/cinn/hlir/framework/graph_compiler.h index d972fc856c825..01dca3e3f65a4 100644 --- a/paddle/cinn/hlir/framework/graph_compiler.h +++ b/paddle/cinn/hlir/framework/graph_compiler.h @@ -85,7 +85,7 @@ class GraphCompiler final { const std::vector>& instructions); // find the first and last instruction where a variable used, and mark the - // variable should allocate buffer before the first instruction runing and + // variable should allocate buffer before the first instruction running and // can release the buffer after the last instruction finished. void AnalyzeVariableLifeTime( const std::vector>& instructions, diff --git a/paddle/cinn/hlir/framework/instruction.cc b/paddle/cinn/hlir/framework/instruction.cc index 7a85318654efc..c7185223843d5 100644 --- a/paddle/cinn/hlir/framework/instruction.cc +++ b/paddle/cinn/hlir/framework/instruction.cc @@ -168,9 +168,9 @@ void Instruction::Run( pod_args[2], static_cast(stream)); } else { - VLOG(3) << "Runing extern function " << function_name_; + VLOG(3) << "Running extern function " << function_name_; for (int idx = 0; idx < fn_ptrs_.size(); ++idx) { - VLOG(3) << "Runing func name: " << fn_names_[idx]; + VLOG(3) << "Running func name: " << fn_names_[idx]; auto& pod_args = args_cached_[idx]; CHECK(fn_ptrs_[idx]) << "The LoweredFunc address should be set first by " "calling SetLoweredFunc method"; @@ -184,7 +184,7 @@ void Instruction::Run( } } } - VLOG(3) << "Done Runing extern function " << function_name_; + VLOG(3) << "Done Running extern function " << function_name_; } #elif defined(CINN_WITH_CUDNN) auto& pod_args = args_cached_[0]; @@ -315,9 +315,9 @@ void Instruction::Run( pod_args[2], static_cast(stream)); } else { - VLOG(3) << "Runing extern function " << function_name_; + VLOG(3) << "Running extern function " << function_name_; for (int idx = 0; idx < fn_ptrs_.size(); ++idx) { - VLOG(3) << "Runing func name: " << fn_names_[idx]; + VLOG(3) << "Running func name: " << fn_names_[idx]; auto& pod_args = args_cached_[idx]; CHECK(fn_ptrs_[idx]) << "The LoweredFunc address should be set first by " "calling SetLoweredFunc method"; @@ -331,12 +331,12 @@ void Instruction::Run( } } } - VLOG(3) << "Done Runing extern function " << function_name_; + VLOG(3) << "Done Running extern function " << function_name_; } #else - VLOG(3) << "Runing extern function " << function_name_; + VLOG(3) << "Running extern function " << function_name_; for (int idx = 0; idx < fn_ptrs_.size(); ++idx) { - VLOG(3) << "Runing func name: " << fn_names_[idx]; + VLOG(3) << "Running func name: " << fn_names_[idx]; auto& pod_args = args_cached_[idx]; CHECK(fn_ptrs_[idx]) << "The LoweredFunc address should be set first by " "calling SetLoweredFunc method"; @@ -350,7 +350,7 @@ void Instruction::Run( } } } - VLOG(3) << "Done Runing extern function " << function_name_; + VLOG(3) << "Done Running extern function " << function_name_; #endif if (!cinn::runtime::CheckStringFlagFalse(FLAGS_cinn_self_check_accuracy)) { diff --git a/paddle/cinn/hlir/framework/pir_compiler.cc b/paddle/cinn/hlir/framework/pir_compiler.cc index f180a480c91dd..1cd7b0220b496 100644 --- a/paddle/cinn/hlir/framework/pir_compiler.cc +++ b/paddle/cinn/hlir/framework/pir_compiler.cc @@ -28,7 +28,7 @@ namespace cinn { namespace hlir { namespace framework { -// TODO(Aurelius84): Clear usless Build Interface. +// TODO(Aurelius84): Clear useless Build Interface. std::unique_ptr PirCompiler::Build() { m_builder_.Clear(); // NOTE(Aurelius84): Currently only support each op for one group @@ -213,8 +213,8 @@ std::shared_ptr BuildScope(const Target& target, }; for (auto& op : *program.block()) { - for (auto oprand : op.operands()) { - create_var(oprand.source()); + for (auto operand : op.operands()) { + create_var(operand.source()); } for (auto result : op.results()) { diff --git a/paddle/cinn/hlir/framework/print_graph_pass_test.cc b/paddle/cinn/hlir/framework/print_graph_pass_test.cc index b26c60c716d0c..0bb21aa41cb5b 100644 --- a/paddle/cinn/hlir/framework/print_graph_pass_test.cc +++ b/paddle/cinn/hlir/framework/print_graph_pass_test.cc @@ -46,7 +46,7 @@ void PrintGraphPass(Graph* src) { CINN_REGISTER_PASS(PrintGraph) .describe( - "This pass just save the visulization Graph to " + "This pass just save the visualization Graph to " "g.attrs[\"print_graph\"].") .set_change_structure(false) .provide_graph_attr("print_graph") diff --git a/paddle/cinn/hlir/op/nn.cc b/paddle/cinn/hlir/op/nn.cc index 8aebede272568..60cbc1c89e222 100644 --- a/paddle/cinn/hlir/op/nn.cc +++ b/paddle/cinn/hlir/op/nn.cc @@ -662,7 +662,7 @@ std::shared_ptr StrategyForConv2dNCHWc( std::vector kernel_shape = inputs[1]->shape; // kernel_h == 1 && kernel_w == 1 CHECK_EQ(kernel_shape.size(), 6U) - << "kernel_dialtion shape size should be 6"; + << "kernel_dilation shape size should be 6"; bool is_1x1 = (is_zero(kernel_shape[2] - 1)) && (is_zero(kernel_shape[3] - 1)); ir::Tensor res; @@ -2224,18 +2224,18 @@ std::vector InferShapeForBatchNormTrain( CHECK_EQ(inputs_shape[0][1], inputs_shape[2][0]) << "x and bias dimension size is not equal!"; CHECK_EQ(inputs_shape[0][1], inputs_shape[3][0]) - << "x and moveing_mean dimension size is not equal!"; + << "x and moving_mean dimension size is not equal!"; CHECK_EQ(inputs_shape[0][1], inputs_shape[4][0]) - << "x and moveing_variance dimension size is not equal!"; + << "x and moving_variance dimension size is not equal!"; } else if (data_layout == "NHWC") { CHECK_EQ(inputs_shape[0][3], inputs_shape[1][0]) << "x and scale dimension is not equal!"; CHECK_EQ(inputs_shape[0][3], inputs_shape[2][0]) << "x and bias dimension size is not equal!"; CHECK_EQ(inputs_shape[0][3], inputs_shape[3][0]) - << "x and moveing_mean dimension size is not equal!"; + << "x and moving_mean dimension size is not equal!"; CHECK_EQ(inputs_shape[0][3], inputs_shape[4][0]) - << "x and moveing_variance dimension size is not equal!"; + << "x and moving_variance dimension size is not equal!"; } else { LOG(FATAL) << "data_layout " << data_layout << " is not support!"; } @@ -2302,16 +2302,16 @@ std::vector InferShapeForBatchNormGrad( CHECK_EQ(inputs_shape[0][1], inputs_shape[2][0]) << "dy and bias dimension size is not equal!"; CHECK_EQ(inputs_shape[0][1], inputs_shape[3][0]) - << "dy and moveing_mean dimension size is not equal!"; + << "dy and moving_mean dimension size is not equal!"; CHECK_EQ(inputs_shape[0][1], inputs_shape[4][0]) - << "dy and moveing_variance dimension size is not equal!"; + << "dy and moving_variance dimension size is not equal!"; } else if (data_layout == "NHWC") { CHECK_EQ(inputs_shape[0][3], inputs_shape[2][0]) << "dy and bias dimension size is not equal!"; CHECK_EQ(inputs_shape[0][3], inputs_shape[3][0]) - << "dy and moveing_mean dimension size is not equal!"; + << "dy and moving_mean dimension size is not equal!"; CHECK_EQ(inputs_shape[0][3], inputs_shape[4][0]) - << "dy and moveing_variance dimension size is not equal!"; + << "dy and moving_variance dimension size is not equal!"; } else { LOG(FATAL) << "data_layout " << data_layout << " is not support!"; } diff --git a/paddle/cinn/hlir/op/reduction.cc b/paddle/cinn/hlir/op/reduction.cc index b50b6b108f954..a8fda43e0ceb5 100644 --- a/paddle/cinn/hlir/op/reduction.cc +++ b/paddle/cinn/hlir/op/reduction.cc @@ -188,7 +188,7 @@ std::shared_ptr StrategyForReduce( for (int i = 0; i < arg_pack.size(); i++) { if (arg_pack[i].is_expr()) { Expr temp = arg_pack[i]; - // TODO(zhhsplendid): old reducetion schedule assumes all length-1 + // TODO(zhhsplendid): old reduction schedule assumes all length-1 // for loops are simplified, but it is not after we add length-1 // back. Reduction schedule is complex and we haven't changed it to // support the length-1 for loop yet. So we simplify here. The todo @@ -651,16 +651,16 @@ std::vector> InferLayoutForBnOptimize( } // namespace cinn CINN_REGISTER_HELPER(reduce_ops) { -#define CINN_REGISTER_REDUCTION_WITH_DTYPE(op__, op_stragegy__, dtype__) \ +#define CINN_REGISTER_REDUCTION_WITH_DTYPE(op__, op_strategy__, dtype__) \ CINN_REGISTER_OP(op__) \ .describe(#op__ " function") \ .set_num_inputs(1) \ .set_num_outputs(1) \ .set_attr( \ - "CINNStrategy", cinn::hlir::op::StrategyFor##op_stragegy__) \ + "CINNStrategy", cinn::hlir::op::StrategyFor##op_strategy__) \ .set_attr( \ "CINNStrategySymbolic", \ - cinn::hlir::op::StrategyFor##op_stragegy__##Symbolic) \ + cinn::hlir::op::StrategyFor##op_strategy__##Symbolic) \ .set_attr("infershape", \ MakeOpFunction(cinn::hlir::op::InferShapeForReduction)) \ .set_attr( \ @@ -674,8 +674,8 @@ CINN_REGISTER_HELPER(reduce_ops) { "OpPattern", cinn::hlir::framework::OpPatternKind::kReduction) \ .set_support_level(4); -#define CINN_REGISTER_REDUCTION(op__, op_stragegy__) \ - CINN_REGISTER_REDUCTION_WITH_DTYPE(op__, op_stragegy__, ) +#define CINN_REGISTER_REDUCTION(op__, op_strategy__) \ + CINN_REGISTER_REDUCTION_WITH_DTYPE(op__, op_strategy__, ) CINN_REGISTER_REDUCTION(reduce_sum, ReduceSum); CINN_REGISTER_REDUCTION(reduce_prod, ReduceProd); diff --git a/paddle/cinn/hlir/pass/fusion_merge_pass.cc b/paddle/cinn/hlir/pass/fusion_merge_pass.cc index 86c0e5360fc0d..eb251fca8608e 100644 --- a/paddle/cinn/hlir/pass/fusion_merge_pass.cc +++ b/paddle/cinn/hlir/pass/fusion_merge_pass.cc @@ -233,7 +233,7 @@ class FusionMergePassHelper : public FusionHelperBase { break; } - // if can't fuse to othors Groups, new Groups. + // if can't fuse to other Groups, new Groups. if (!fusionable) { fusionable_consumers.push_back({candidate}); } @@ -488,7 +488,7 @@ class FusionMergePassHelper : public FusionHelperBase { fusionable_consumers) { VLOG(3) << "VerticalFuse...!"; GroupList fused_groups; - GroupPtr master_fuesd_group(nullptr); + GroupPtr master_fused_group(nullptr); for (auto& consumer : fusionable_consumers) { auto fused_group = std::make_shared(); // update depth using consumer depth. @@ -623,8 +623,8 @@ class FusionMergePassHelper : public FusionHelperBase { fusion_groups_[postion] = fused_group; fusion_groups_index_[fused_group] = postion; - if (!master_fuesd_group.get()) { - master_fuesd_group = fused_group; + if (!master_fused_group.get()) { + master_fused_group = fused_group; } CHECK(fused_group->output_nodes.size()) << "No output node is found, " << fused_group->group_id; @@ -654,8 +654,8 @@ class FusionMergePassHelper : public FusionHelperBase { if (be_output) { VLOG(4) << "Insert Id " << node->id() << " Into Group " - << master_fuesd_group->group_id; - master_fuesd_group->output_nodes.insert(node); + << master_fused_group->group_id; + master_fused_group->output_nodes.insert(node); } } // insert unfusionable consumer groups @@ -663,10 +663,10 @@ class FusionMergePassHelper : public FusionHelperBase { if (fusionable_consumers.count(consumer)) { continue; } - master_fuesd_group->mut_consumer_groups()->insert(consumer); + master_fused_group->mut_consumer_groups()->insert(consumer); // update consumer's producer consumer->mut_producer_groups()->erase(producer); - consumer->mut_producer_groups()->insert(master_fuesd_group); + consumer->mut_producer_groups()->insert(master_fused_group); } } @@ -979,7 +979,7 @@ class FusionMergePassHelper : public FusionHelperBase { // element-wise and injective op must be horizontal relation. {OpPatternKind::kInjective, is_same_size}, // element-wise and reduce op must be horizontal relation. - {OpPatternKind::kReduction, honrizontal_elementwise_fuse_reduce}}; + {OpPatternKind::kReduction, horizontal_elementwise_fuse_reduce}}; // vertical relation.vertical_relation = { {OpPatternKind::kElementWise, is_same_size}, @@ -1044,7 +1044,7 @@ class FusionMergePassHelper : public FusionHelperBase { // horizontal relation.horizontal_relation = { // reduce and element-wise op must be horizontal relation. - {OpPatternKind::kElementWise, honrizontal_elementwise_fuse_reduce}, + {OpPatternKind::kElementWise, horizontal_elementwise_fuse_reduce}, // reduce and broadcast op must be horizontal relation. {OpPatternKind::kBroadcast, is_same_size}, // reduce and injective op must be horizontal relation. diff --git a/paddle/cinn/hlir/pass/fusion_merge_pass_util.h b/paddle/cinn/hlir/pass/fusion_merge_pass_util.h index bc14748f5f648..219d08d7d08e6 100644 --- a/paddle/cinn/hlir/pass/fusion_merge_pass_util.h +++ b/paddle/cinn/hlir/pass/fusion_merge_pass_util.h @@ -105,7 +105,7 @@ CONDITION_FUNC(elementwise_fuse_broadcast) { return true; } -CONDITION_FUNC(honrizontal_elementwise_fuse_reduce) { +CONDITION_FUNC(horizontal_elementwise_fuse_reduce) { std::shared_ptr ele_group, reduce_group; if (first->op_pattern_kind == framework::kReduction) { ele_group = second; diff --git a/paddle/cinn/hlir/pass/general_fusion_merge_pass.cc b/paddle/cinn/hlir/pass/general_fusion_merge_pass.cc index cf1b91fcc1357..65d0d9eb7c243 100644 --- a/paddle/cinn/hlir/pass/general_fusion_merge_pass.cc +++ b/paddle/cinn/hlir/pass/general_fusion_merge_pass.cc @@ -69,7 +69,7 @@ class GeneralFusionMergePassHelper : public FusionHelperBase { } GroupList operator()() { - // run fusion merge untill no update. + // run fusion merge until no update. DoFusionMerge(); for (auto& group : fusion_groups_) { VLOG(3) << "Fusion Group -> " << group->group_id; @@ -564,7 +564,7 @@ class GeneralFusionMergePassHelper : public FusionHelperBase { fusionable_consumers) { VLOG(3) << "VerticalFuse...!"; GroupList fused_groups; - GroupPtr master_fuesd_group(nullptr); + GroupPtr master_fused_group(nullptr); for (auto& consumer : fusionable_consumers) { auto fused_group = std::make_shared(graph_); // update depth using consumer depth. @@ -700,8 +700,8 @@ class GeneralFusionMergePassHelper : public FusionHelperBase { fusion_groups_[postion] = fused_group; fusion_groups_index_[fused_group] = postion; - if (!master_fuesd_group.get()) { - master_fuesd_group = fused_group; + if (!master_fused_group.get()) { + master_fused_group = fused_group; } CHECK(fused_group->output_nodes.size()) << "No output node is found, " << fused_group->group_id; @@ -731,8 +731,8 @@ class GeneralFusionMergePassHelper : public FusionHelperBase { if (be_output) { VLOG(4) << "Insert Id " << node->id() << " Into Group " - << master_fuesd_group->group_id; - master_fuesd_group->output_nodes.insert(node); + << master_fused_group->group_id; + master_fused_group->output_nodes.insert(node); } } // insert unfusionable consumer groups @@ -740,10 +740,10 @@ class GeneralFusionMergePassHelper : public FusionHelperBase { if (fusionable_consumers.count(consumer)) { continue; } - master_fuesd_group->mut_consumer_groups()->insert(consumer); + master_fused_group->mut_consumer_groups()->insert(consumer); // update consumer's producer consumer->mut_producer_groups()->erase(producer); - consumer->mut_producer_groups()->insert(master_fuesd_group); + consumer->mut_producer_groups()->insert(master_fused_group); } } diff --git a/paddle/cinn/hlir/pass/general_fusion_merge_pass/graph_group_fuse_helper.h b/paddle/cinn/hlir/pass/general_fusion_merge_pass/graph_group_fuse_helper.h index 3859ad88ff016..f3f2802ac3007 100644 --- a/paddle/cinn/hlir/pass/general_fusion_merge_pass/graph_group_fuse_helper.h +++ b/paddle/cinn/hlir/pass/general_fusion_merge_pass/graph_group_fuse_helper.h @@ -138,7 +138,7 @@ bool GraphGroupFuseHelper::AllOutputsSameSize( template bool GraphGroupFuseHelper::HorizontalElementwiseFuseReduce( const OpGroupPtr& src, const OpGroupPtr& dst) const { - return honrizontal_elementwise_fuse_reduce( + return horizontal_elementwise_fuse_reduce( &ctx_->graph_group_fusion_helper(), src.GetGroup(), dst.GetGroup()); } diff --git a/paddle/cinn/hlir/pass/op_fusion_pass_test.cc b/paddle/cinn/hlir/pass/op_fusion_pass_test.cc index 885afd929ba87..c9d723c91be50 100755 --- a/paddle/cinn/hlir/pass/op_fusion_pass_test.cc +++ b/paddle/cinn/hlir/pass/op_fusion_pass_test.cc @@ -66,9 +66,9 @@ TEST(OpFusionPass, ElementWise_Fusion_1) { CHECK_EQ(graph->fusion_groups.size(), 1); } -TEST(OpFusionPass, Brodcast_Test_0) { +TEST(OpFusionPass, Broadcast_Test_0) { int h = 32, w = 32; - NetBuilder net_builder("Brodcast_Test_0"); + NetBuilder net_builder("Broadcast_Test_0"); // create model { auto A = net_builder.CreateInput(Float(32), {w}, "A"); @@ -89,9 +89,9 @@ TEST(OpFusionPass, Brodcast_Test_0) { CHECK_EQ(graph->fusion_groups.size(), 1); } -TEST(OpFusionPass, Brodcast_Test_1) { +TEST(OpFusionPass, Broadcast_Test_1) { int h = 32, w = 32; - NetBuilder net_builder("Brodcast_Test_1"); + NetBuilder net_builder("Broadcast_Test_1"); // create model { auto A = net_builder.CreateInput(Float(32), {w}, "A"); @@ -114,9 +114,9 @@ TEST(OpFusionPass, Brodcast_Test_1) { CHECK_EQ(graph->fusion_groups.size(), 1); } -TEST(OpFusionPass, Brodcast_Test_2) { +TEST(OpFusionPass, Broadcast_Test_2) { int n = 2, c = 16, h = 32, w = 32; - NetBuilder net_builder("Brodcast_Test_2"); + NetBuilder net_builder("Broadcast_Test_2"); // create model { auto A = net_builder.CreateInput(Float(32), {c}, "A"); diff --git a/paddle/cinn/hlir/pe/ir_schedule_pe.cc b/paddle/cinn/hlir/pe/ir_schedule_pe.cc index 2c27c98d5faf9..36052d25f8a44 100644 --- a/paddle/cinn/hlir/pe/ir_schedule_pe.cc +++ b/paddle/cinn/hlir/pe/ir_schedule_pe.cc @@ -611,7 +611,7 @@ void IRCudaScheduleBlockReduce(ir::IRSchedule &ir_sch, // NOLINT } // bind block and thread for reduce. - // as outer loop range should be eqaul, get loop size. + // as outer loop range should be equal, get loop size. auto b_loop = ir::GetLoopExtent(ir_sch.GetLoops(out->name)[0]); // reduce_tmp_out { @@ -784,7 +784,7 @@ void IRCudaScheduleBlockShuffleReduce(ir::IRSchedule &ir_sch, // NOLINT } return loop_var_count; } - LOG(FATAL) << "Can't find var in tensor indeces!"; + LOG(FATAL) << "Can't find var in tensor indexes!"; }; auto loop_var_count = get_loop_index(ir_sch.GetLoops(reduce_out->name).back(), ir_sch.GetBlock(reduce_out->name)); diff --git a/paddle/cinn/hlir/pe/transform.cc b/paddle/cinn/hlir/pe/transform.cc index 324e886195f60..2e78caca83206 100644 --- a/paddle/cinn/hlir/pe/transform.cc +++ b/paddle/cinn/hlir/pe/transform.cc @@ -1174,7 +1174,7 @@ ir::Tensor SliceAssign(const ir::Tensor& input, new_strides[i] = -new_strides[i]; } else { CHECK_LT(new_starts[i], new_ends[i]) - << "[ends] shoould greater than [starts] when [strides] > 0"; + << "[ends] should greater than [starts] when [strides] > 0"; } } From 1b68a51dbdc6b4e93a0c8e28df74e8d881272501 Mon Sep 17 00:00:00 2001 From: JYChen Date: Mon, 26 Feb 2024 11:23:20 +0800 Subject: [PATCH 070/282] fix shape error in combine-getitem (#61922) --- paddle/fluid/pybind/eager_method.cc | 10 ++++---- python/paddle/base/variable_index.py | 4 ++-- test/indexing/test_getitem.py | 35 ++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/paddle/fluid/pybind/eager_method.cc b/paddle/fluid/pybind/eager_method.cc index 9dc8897a10a41..09fb067f41dee 100644 --- a/paddle/fluid/pybind/eager_method.cc +++ b/paddle/fluid/pybind/eager_method.cc @@ -1404,14 +1404,14 @@ static PyObject* tensor__getitem_dygraph(TensorObject* self, if (pos_of_new_dim != 0) { std::vector perm(out.shape().size(), 0); - int tmp1 = pos_of_new_dim, tmp2 = 0, + int tmp1 = rank_of_new_dim, tmp2 = 0, tmp3 = pos_of_new_dim + rank_of_new_dim; for (int i = 0; i < static_cast(out.shape().size()); ++i) { - if (i < rank_of_new_dim) { + if (i < pos_of_new_dim) { perm[i] = - tmp1++; // range(pos_of_new_dim, pos_of_new_dim + rank_of_new_dim) - } else if (i >= rank_of_new_dim && i < pos_of_new_dim + rank_of_new_dim) { - perm[i] = tmp2++; // range(0, pos_of_new_dim) + tmp1++; // range(rank_of_new_dim, pos_of_new_dim + rank_of_new_dim) + } else if (i >= pos_of_new_dim && i < pos_of_new_dim + rank_of_new_dim) { + perm[i] = tmp2++; // range(0, rank_of_new_dim) } else { perm[i] = tmp3++; // range(pos_of_new_dim + rank_of_new_dim, out.ndim) } diff --git a/python/paddle/base/variable_index.py b/python/paddle/base/variable_index.py index 6ccfe1c6164d2..0d7704272df61 100644 --- a/python/paddle/base/variable_index.py +++ b/python/paddle/base/variable_index.py @@ -908,8 +908,8 @@ def _getitem_static(x, indices): if pos_of_new_dim != 0: perm = ( - list(range(pos_of_new_dim, pos_of_new_dim + rank_of_new_dim)) - + list(range(0, pos_of_new_dim)) + list(range(rank_of_new_dim, pos_of_new_dim + rank_of_new_dim)) + + list(range(0, rank_of_new_dim)) + list(range(pos_of_new_dim + rank_of_new_dim, out.ndim)) ) out = out.transpose(perm) diff --git a/test/indexing/test_getitem.py b/test/indexing/test_getitem.py index f840042c57e09..bf700e4986e57 100644 --- a/test/indexing/test_getitem.py +++ b/test/indexing/test_getitem.py @@ -234,6 +234,26 @@ def test_combined_index_11(self): np.testing.assert_allclose(y.numpy(), np_res) + def test_combined_index_12(self): + np_data = ( + np.arange(3 * 4 * 5 * 6).reshape((3, 4, 5, 6)).astype(self.ndtype) + ) + + if self.dtype == 'bfloat16': + np_data = convert_uint16_to_float(convert_float_to_uint16(np_data)) + if self.dtype == 'complex64' or self.dtype == 'complex128': + np_data = np_data + 1j * np_data + + np_res = np_data[:, :, [2, 4], :] + + x = paddle.to_tensor(np_data, dtype=self.dtype) + y = x[:, :, [2, 4], :] + + if self.dtype == 'bfloat16': + y = paddle.cast(y, dtype='float32') + + np.testing.assert_allclose(y.numpy(), np_res) + def test_index_has_range(self): np_data = ( np.arange(3 * 4 * 5 * 6).reshape((3, 4, 5, 6)).astype(self.ndtype) @@ -982,6 +1002,21 @@ def test_combined_index_11(self): np.testing.assert_allclose(res[0], np_res) + @test_with_pir_api + def test_combined_index_12(self): + np_data = np.arange(3 * 4 * 5 * 6).reshape((3, 4, 5, 6)) + np_res = np_data[:, :, [2, 4], :] + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + x = paddle.to_tensor(np_data) + y = _getitem_static( + x, (slice(None), slice(None), [2, 4], slice(None)) + ) + res = self.exe.run(fetch_list=[y]) + + np.testing.assert_allclose(res[0], np_res) + @test_with_pir_api def test_index_has_range(self): # only one bool tensor with all False From e19e3c9435ee71ac844d78f98a34265ac7a73589 Mon Sep 17 00:00:00 2001 From: feifei-111 <2364819892@qq.com> Date: Mon, 26 Feb 2024 11:23:52 +0800 Subject: [PATCH 071/282] [SOT] rewrite resume function generation (#62012) --- .../executor/function_graph.py | 71 +- .../executor/opcode_executor.py | 850 ++++++++++-------- .../executor/opcode_inline_executor.py | 10 - .../executor/pycode_generator.py | 217 ++--- .../sot/opcode_translator/executor/tracker.py | 2 +- .../instruction_utils/__init__.py | 5 +- .../instruction_utils/instruction_pass.py | 2 + .../instruction_utils/instruction_utils.py | 8 + .../instruction_utils/opcode_analysis.py | 109 +-- test/sot/test_11_jumps.py | 12 + test/sot/test_analysis_inputs.py | 8 +- 11 files changed, 609 insertions(+), 685 deletions(-) diff --git a/python/paddle/jit/sot/opcode_translator/executor/function_graph.py b/python/paddle/jit/sot/opcode_translator/executor/function_graph.py index 8f87e19cd4d28..dc57b252e00c2 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/function_graph.py +++ b/python/paddle/jit/sot/opcode_translator/executor/function_graph.py @@ -39,6 +39,7 @@ from ...utils import ( ENV_SHOW_TRACKERS, NameGenerator, + SotUndefinedVar, inner_error_default_handler, is_inplace_api, is_paddle_api, @@ -140,6 +141,20 @@ def get_params_and_non_param_symbol(*args, **kwargs): return params, non_params +class VariableLoader: + def __init__(self, store_var_info, pycode_gen): + self._store_var_info = store_var_info + self._pycode_gen: PyCodeGen = pycode_gen + + def load(self, var): + if var is SotUndefinedVar(): + self._pycode_gen.gen_load_const(SotUndefinedVar()) + elif isinstance(var, NullVariable): + var.reconstruct(self._pycode_gen) + else: + self._pycode_gen.gen_load(self._store_var_info[var.id]) + + class FunctionGraph: """ A Graph representation corresponding to each FunctionFrame @@ -281,17 +296,6 @@ def guard_fn(self) -> Guard: return make_guard(guards) def _restore_origin_opcode(self, stack_vars, store_var_info, instr_idx): - class VariableLoader: - def __init__(self, store_var_info, pycode_gen): - self._store_var_info = store_var_info - self._pycode_gen: PyCodeGen = pycode_gen - - def load(self, var): - if isinstance(var, NullVariable): - var.reconstruct(self._pycode_gen) - return - self._pycode_gen.gen_load(self._store_var_info[var.id]) - origin_instrs = get_instructions(self.pycode_gen._origin_code) is_precall = origin_instrs[instr_idx].opname == "PRECALL" current_idx = instr_idx @@ -308,7 +312,7 @@ def load(self, var): restore_instr_names = restore_instr_names[:-1] self.pycode_gen.extend_instrs(restore_instrs) - nop = self.pycode_gen._add_instr("NOP") + nop = self.pycode_gen.add_instr("NOP") for instr in origin_instrs: if instr.jump_to == origin_instrs[current_idx]: @@ -324,26 +328,21 @@ def load(self, var): name_gen = NameGenerator("__start_compile_saved_orig_") + # here is not update changed values, it just give names to stack vars + # and want keep same interface as _build_compile_fn_with_name_store for var in stack_vars[::-1]: - store_var_info[var.id] = name_gen.next() - self.pycode_gen.gen_store_fast(store_var_info[var.id]) + if store_var_info[var.id] is None: + store_var_info[var.id] = name_gen.next() + self.pycode_gen.gen_store_fast(store_var_info[var.id]) + else: + self.pycode_gen.gen_store( + store_var_info[var.id], self.pycode_gen._origin_code + ) return VariableLoader(store_var_info, self.pycode_gen) - def _build_compile_fn_with_name_store(self, to_store_vars): - class VariableLoader: - def __init__(self, index_for_load, pycode_gen): - self._index_for_load = index_for_load - self._pycode_gen: PyCodeGen = pycode_gen - - def load(self, var, allow_push_null=True): - if isinstance(var, NullVariable): - var.reconstruct(self._pycode_gen) - return - self._pycode_gen.gen_load(self._index_for_load[var.id]) - + def _build_compile_fn_with_name_store(self, to_store_vars, store_var_info): # var_id -> local_name mapping - index_for_load = {} to_store_vars = list( filter(lambda x: not isinstance(x, NullVariable), to_store_vars) ) @@ -351,19 +350,15 @@ def load(self, var, allow_push_null=True): name_gen = NameGenerator("__start_compile_saved_") for var in to_store_vars[::-1]: - index_for_load[var.id] = name_gen.next() - - def _log_fn(): - print( - f"[StartCompile] saved var: {index_for_load[var.id]} = ", - var, + if store_var_info[var.id] is None: + store_var_info[var.id] = name_gen.next() + self.pycode_gen.gen_store_fast(store_var_info[var.id]) + else: + self.pycode_gen.gen_store( + store_var_info[var.id], self.pycode_gen._origin_code ) - log_do(4, _log_fn) - - self.pycode_gen.gen_store_fast(index_for_load[var.id]) - - return VariableLoader(index_for_load, self.pycode_gen) + return VariableLoader(store_var_info, self.pycode_gen) def get_compiled_fn(self, *ret_vars): ret_items = [ diff --git a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py index e9a985e5b728c..e0ada6a9b74fa 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py +++ b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py @@ -27,8 +27,6 @@ import opcode -from paddle.jit.utils import OrderedSet - from ...profiler import EventGuard, event_register from ...psdb import NO_BREAKGRAPH_CODES from ...utils import ( @@ -45,8 +43,7 @@ from ..instruction_utils import ( Instruction, Space, - analysis_inputs, - analysis_used_names_with_space, + analysis_used_names, calc_stack_effect, get_instructions, ) @@ -416,7 +413,34 @@ def transform(self): """ raise NotImplementedError() - def get_var(self, name: str): + def find_space_of_var_name(self, name): + code = self._graph.pycode_gen._origin_code + if name in (code.co_freevars + code.co_cellvars): + return Space.cells + elif name in code.co_varnames: + return Space.locals + elif name in code.co_names: + return Space.globals + else: + return Space.not_found + + def has_var(self, name: str): + space = self.find_space_of_var_name(name) + + if space == Space.locals: + return name in self._locals + elif space == Space.cells: + return name in self._cells + elif space == Space.globals: + return name in set( + chain( + self._globals.keys(), + self._builtins.keys(), + ) + ) + return False + + def get_var(self, name: str, allow_undefined=False): """ Gets the variable with the given name. @@ -438,31 +462,27 @@ def get_var(self, name: str): return self._globals.get(name) elif name in self._builtins.keys(): return self._builtins[name] + elif allow_undefined: + return SotUndefinedVar() else: raise InnerError(f'Can not get var: {name}') - def has_var(self, name: str, space: str = "any"): - if space == "any": - return name in set( - chain( - self._locals.keys(), - self._cells.keys(), - self._globals.keys(), - self._builtins.keys(), - ) - ) - elif space == Space.locals: - return name in self._locals + def set_var(self, name: str, value: VariableBase): + space = self.find_space_of_var_name(name) + + # if name is new created, we always place it to locals + if space in (Space.locals, Space.not_found): + self._locals[name] = value elif space == Space.cells: - return name in self._cells + self._cells[name].set_value(value) elif space == Space.globals: - return name in set( - chain( - self._globals.keys(), - self._builtins.keys(), - ) - ) - return False + self._globals[name] = value + + def _find_names_in_space(self, names, space): + target_names = [ + name for name in names if self.find_space_of_var_name(name) in space + ] + return target_names def pop_call_stack_until_self(self): """ @@ -1511,6 +1531,31 @@ def __init__(self, frame: types.FrameType, **kwargs): super().__init__(frame.f_code, graph) Dispatcher.graph = graph + def transform(self): + static_function = get_static_function(self._frame, "eval_frame") + if static_function is not None: + code = self._frame.f_code + inputs = [] + for i in range(code.co_argcount): + arg_name = code.co_varnames[i] + value = self._locals[arg_name] + inputs.append(value) + output = self._graph.call_ast(static_function, *inputs) + if output is not None: + self.stack.push(output) + self.RETURN_VALUE(None) + return ( + CustomCode(self.new_code, self.new_code is None), + self.guard_fn, + ) + self.run() + if self.new_code is self.empty_code: + raise InnerError("OpExecutor return a empty new_code.") + return ( + CustomCode(self.new_code, self.new_code is None), + self.guard_fn, + ) + def cleanup(self): self._graph.pycode_gen = None Dispatcher.graph = None @@ -1560,56 +1605,99 @@ def _prepare_virtual_env(self): ) ) - def gen_compute_in_break_with_name_store(self, restore_names, instr_idx): + def FOR_ITER(self, instr): + iterator = self.stack.pop() + backup_iter_idx = None + + start = self.indexof(instr) + end = self.indexof(instr.jump_to) + for i in range(start, end): + if self._instructions[i].opname == "RETURN_VALUE": + raise FallbackError("Found RETURN_VALUE in for loop body.") + + self._graph.add_global_guarded_variable(iterator) + + try: + if not isinstance(iterator, SequenceIterVariable): + raise BreakGraphError( + f"Can not simulate iterator of {type(iterator)}." + ) + + backup_iter_idx = iterator.idx + + self._inline_call_for_loop(iterator, instr) + self._lasti = self.indexof(instr.jump_to) + next_instr = self._instructions[self._lasti] + self._lasti += int(next_instr.opname == 'END_FOR') + except BreakGraphError as e: + log(3, f"[BreakGraph] FOR_ITER sim for loop failed for: {e}\n") + if backup_iter_idx: + iterator.idx = backup_iter_idx + self._graph.remove_global_guarded_variable(iterator) + self.stack.push(iterator) + self._break_graph_when_for_loop(iterator, instr) + return Stop(state="BreakGraph") + + def RETURN_VALUE(self, instr: Instruction): + assert ( + len(self.stack) == 1 + ), f"Stack must have one element, but get {len(self.stack)} elements." + ret_val = self.stack.pop() + return self.compile_return(ret_val) + + def RETURN_CONST(self, instr: Instruction): + ret_const = self._co_consts[instr.arg] + return self.compile_return(ret_const) + + def compile_return(self, ret_val): + compile_fn = self._graph.get_compiled_fn(ret_val) + if compile_fn.graph_size() < ENV_MIN_GRAPH_SIZE.get(): + self.new_code = None + else: + self._graph.start_compile(ret_val) + self._graph.pycode_gen.gen_return() + self.new_code = self._graph.pycode_gen.gen_pycode() + self.guard_fn = self._graph.guard_fn + return Stop(state="Return") + + def get_compute_fn_and_update_changed_vars( + self, restore_names, stack, end_idx + ): """ - branch 1: if the graph size is too small, just run in dygraph - branch 2: if the graph is big enough, create compiled_fn - - This api will generator opcodes in different situation, the generated codes - will do the same thing as origin code. - - restore_names: - the names used in resume functions, branch 2 will restore these values, - branch 1 also need these names for generating opcode, but they are not - needed to be restored - instr_idx: - the index for branch 1 to find the boundary and copy origin opcode + this function will: + 1. add opcodes to self._graph.pycode_gen, which do the same thing as origin code. + 2. update the value of whom would be changed in generated codes + + This api will generator opcodes in different situation, + branch 1: if the graph size is too small, just run in dygraph. + branch 2: if the graph is big enough, create compiled_fn. + + Params: + restore_names: the names used in resume functions. + end_idx: instruction index where simulation get break. + stack: current stack """ - # if we want get compiled fn, and do not do ast twice, - # we must give retval to get_compiled_fn which strictly same as start_compile - store_vars = list(self.stack) - store_var_info = {} + store_vars = list(stack) + store_var_info = {var.id: None for var in stack} for name in restore_names: - _var = self.get_var(name) - if _var not in self.stack: + _var = self.get_var(name, allow_undefined=True) + if _var is SotUndefinedVar(): + continue + if _var not in stack: store_vars.append(_var) - store_var_info[_var.id] = name + store_var_info[_var.id] = name compile_fn = self._graph.get_compiled_fn(*store_vars) if compile_fn.graph_size() < ENV_MIN_GRAPH_SIZE.get(): return self._graph._restore_origin_opcode( - list(self.stack), store_var_info, instr_idx + list(stack), store_var_info, end_idx ) else: - return self._graph._build_compile_fn_with_name_store(store_vars) - - def _create_resume_fn(self, index, stack_size): - """ - Create a resume function and its inputs at the specified index. - - Args: - index: The index at which the resume function is created. - stack_size: The size of the stack. - - Returns: - The resume function and its inputs. - - """ - pycode_gen = PyCodeGen(self._frame) - fn, inputs = pycode_gen.gen_resume_fn_at(index, stack_size) - return fn, inputs + return self._graph._build_compile_fn_with_name_store( + store_vars, store_var_info + ) @fallback_when_occur_error def _break_graph_when_if(self, result: TensorVariable, instr: Instruction): @@ -1622,66 +1710,105 @@ def _break_graph_when_if(self, result: TensorVariable, instr: Instruction): """ self._graph.add_global_guarded_variable(result) - # minus the bool value - stack_size = len(self.stack) - 1 - # gen call static fn opcode - if_fn, if_inputs = self._create_resume_fn( - self.indexof(instr) + 1, stack_size + # 1. analyse info + cur_index = self.indexof(instr) + true_fn_start_index = cur_index + 1 + false_fn_start_index = self.indexof(instr.jump_to) + stack_size_after_if = len(self.stack) - 1 + + # 2. create true_fn and false_fn + def create_if_branch_fn(start_idx, input_var_names): + if self._instructions[start_idx].opname == "RETURN_VALUE": + return None + pycode_gen = PyCodeGen(self._frame) + origin_instrs = get_instructions(pycode_gen._origin_code) + pycode_gen.set_function_inputs( + input_var_names, stack_size=stack_size_after_if + ) + pycode_gen.extend_instrs(origin_instrs[start_idx:]) + # the resume_fn contains return code, so we don't need set output here + # global vars are updated correctly, and need local vars will return + resume_fn = pycode_gen.create_function() + return resume_fn + + true_fn_read_names, _ = analysis_used_names( + self._instructions, self.indexof(instr) + 1 + ) + true_fn_input_var_names = self._find_names_in_space( + true_fn_read_names, (Space.locals, Space.cells) + ) + + true_fn = create_if_branch_fn( + start_idx=true_fn_start_index, + input_var_names=true_fn_input_var_names, + ) + + false_fn_read_names, _ = analysis_used_names( + self._instructions, self.indexof(instr.jump_to) + ) + false_fn_input_var_names = self._find_names_in_space( + false_fn_read_names, (Space.locals, Space.cells) ) - else_fn, else_inputs = self._create_resume_fn( - self.indexof(instr.jump_to), stack_size + + false_fn = create_if_branch_fn( + start_idx=false_fn_start_index, + input_var_names=false_fn_input_var_names, ) - inputs_names = if_inputs | else_inputs + # 4. setup vars which is created in loop as Undefind + for name in true_fn_input_var_names[:-1]: + if not self.has_var(name): + self._graph.pycode_gen.gen_load_const(SotUndefinedVar()) + self._graph.pycode_gen.gen_store(name, self._code) + for name in false_fn_input_var_names: + if not self.has_var(name): + self._graph.pycode_gen.gen_load_const(SotUndefinedVar()) + self._graph.pycode_gen.gen_store(name, self._code) - var_loader = self.gen_compute_in_break_with_name_store( - inputs_names, self.indexof(instr) + # 4. compile codes before if + update_var_names = list(true_fn_read_names | false_fn_read_names) + var_loader = self.get_compute_fn_and_update_changed_vars( + update_var_names, self.stack, cur_index ) + # 5. create if sturcture and call true_fn and false_fn var_loader.load(result) - # the result is used by if opcode, and should not be input of resume_fn - self.stack.pop() + if_code = self._graph.pycode_gen.add_instr(instr.opname) - # gen call if/else resume fn opcode - if if_fn is not None: - self._graph.pycode_gen.gen_load_object( - if_fn, if_fn.__code__.co_name - ) - insert_index = len(self._graph.pycode_gen._instructions) - 1 - for i, stack_arg in enumerate(self.stack): - var_loader.load(stack_arg) - for name in if_inputs: - var_loader.load(self.get_var(name)) - self._graph.pycode_gen.gen_call_function( - argc=if_fn.__code__.co_argcount, - ) - self._graph.pycode_gen.gen_return() - else: - insert_index = len(self._graph.pycode_gen._instructions) - 1 - self._graph.pycode_gen.gen_return() + assert true_fn is not None - if else_fn is not None: - self._graph.pycode_gen.gen_load_object( - else_fn, else_fn.__code__.co_name + self._graph.pycode_gen.gen_load_object( + true_fn, true_fn.__code__.co_name + ) + for stack_arg in list(self.stack)[:-1]: + var_loader.load(stack_arg) + + for name in true_fn_input_var_names: + var_loader.load(self.get_var(name, allow_undefined=True)) + + self._graph.pycode_gen.gen_call_function( + argc=true_fn.__code__.co_argcount, + ) + self._graph.pycode_gen.gen_return() + + if false_fn is not None: + false_start_code = self._graph.pycode_gen.gen_load_object( + false_fn, false_fn.__code__.co_name ) - jump_to = self._graph.pycode_gen._instructions[-1] - for i, stack_arg in enumerate(self.stack): + for stack_arg in list(self.stack)[:-1]: var_loader.load(stack_arg) - for name in else_inputs: - var_loader.load(self.get_var(name)) + for name in false_fn_input_var_names: + var_loader.load(self.get_var(name, allow_undefined=True)) + self._graph.pycode_gen.gen_call_function( - argc=else_fn.__code__.co_argcount, + argc=false_fn.__code__.co_argcount, ) self._graph.pycode_gen.gen_return() else: - self._graph.pycode_gen.gen_return() - jump_to = self._graph.pycode_gen._instructions[-1] + false_start_code = self._graph.pycode_gen.gen_return() - # gen jump opcode - self._graph.pycode_gen._insert_instr( - insert_index, instr.opname, jump_to=jump_to - ) + if_code.jump_to = false_start_code self.new_code = self._graph.pycode_gen.gen_pycode() self.guard_fn = self._graph.guard_fn @@ -1702,41 +1829,60 @@ def _break_graph_when_call( push_n: The number of elements to be pushed onto the stack. """ + self.stack = origin_stack + + # 1. collect infomations push_n = push_n(instr.arg) if callable(push_n) else push_n is_precall = instr.opname == "PRECALL" - index = self.indexof(instr) + cur_index = self.indexof(instr) # Use CALL instead of PRECALL to calculate the real stack effect - call_instr = self._instructions[index + int(is_precall)] + call_instr = self._instructions[cur_index + int(is_precall)] # skip CALL if current instr is PRECALL - next_index = index + 1 + int(is_precall) - self.stack = origin_stack - - # gen call static fn opcode + next_index = cur_index + 1 + int(is_precall) + stack_effect = calc_stack_effect(call_instr) + pop_n = push_n - stack_effect + stack_size_after_call = len(self.stack) - pop_n + push_n - resume_input_name = analysis_inputs(self._instructions, next_index) + # 2. create resume function + read_names, _ = analysis_used_names(self._instructions, next_index) - var_loader = self.gen_compute_in_break_with_name_store( - resume_input_name, index + input_var_names = self._find_names_in_space( + read_names, (Space.locals, Space.cells) ) - # gen graph break call fn opcode - stack_effect = calc_stack_effect(call_instr) - pop_n = push_n - stack_effect + def create_resume_fn(): + if self._instructions[next_index].opname == "RETURN_VALUE": + return None + pycode_gen = PyCodeGen(self._frame) + origin_instrs = get_instructions(pycode_gen._origin_code) + pycode_gen.set_function_inputs( + input_var_names, stack_size=stack_size_after_call + ) + pycode_gen.extend_instrs(origin_instrs[next_index:]) + # the resume_fn contains return code, so we don't need set output here + # global vars are updated correctly, and need local vars will return + resume_fn = pycode_gen.create_function() + return resume_fn - for i, stack_arg in enumerate(self.stack): + resume_fn = create_resume_fn() + + # 3. compile sub graph before call + var_loader = self.get_compute_fn_and_update_changed_vars( + read_names, self.stack, cur_index + ) + + # 4. recover stack + for stack_arg in self.stack: var_loader.load(stack_arg) - # gen call resume fn opcode + # 5. run the break CALL with origin python # NOTE(SigureMo): In Python 3.11，we need generate KW_NAMES if the call shape is not None. self._graph.pycode_gen.gen_kw_names(self._call_shape) self._graph.pycode_gen.extend_instrs( - self._instructions[index:next_index] + self._instructions[cur_index:next_index] ) - self.stack.pop_n(pop_n) - stack_size = len(self.stack) + push_n - - resume_fn, _ = self._create_resume_fn(next_index, stack_size) + # 6. run resume fn if resume_fn: self._graph.pycode_gen.gen_load_object( resume_fn, resume_fn.__code__.co_name @@ -1744,9 +1890,11 @@ def _break_graph_when_call( # NOTE(zrr1999): We need to shift the resume_fn under its arguments. # In Python 3.11+, NULL + resume_fn should be shifted together. shift_n = 2 if sys.version_info >= (3, 11) else 1 - self._graph.pycode_gen.gen_shift_n(shift_n, stack_size + shift_n) - for name in resume_input_name: - var_loader.load(self.get_var(name)) + self._graph.pycode_gen.gen_shift_n( + shift_n, stack_size_after_call + shift_n + ) + for name in input_var_names: + var_loader.load(self.get_var(name, allow_undefined=True)) self._graph.pycode_gen.gen_call_function( argc=resume_fn.__code__.co_argcount, ) @@ -1757,112 +1905,14 @@ def _break_graph_when_call( self.new_code = self._graph.pycode_gen.gen_pycode() self.guard_fn = self._graph.guard_fn - def transform(self): - static_function = get_static_function(self._frame, "eval_frame") - if static_function is not None: - code = self._frame.f_code - inputs = [] - for i in range(code.co_argcount): - arg_name = code.co_varnames[i] - value = self._locals[arg_name] - inputs.append(value) - output = self._graph.call_ast(static_function, *inputs) - if output is not None: - self.stack.push(output) - self.RETURN_VALUE(None) - return ( - CustomCode(self.new_code, self.new_code is None), - self.guard_fn, - ) - self.run() - if self.new_code is self.empty_code: - raise InnerError("OpExecutor return a empty new_code.") - return ( - CustomCode(self.new_code, self.new_code is None), - self.guard_fn, - ) - - def _gen_loop_body_between( - self, inputs: list, for_iter_idx: int, start: int, end: int - ) -> types.FunctionType: - """ - Generates the loop body between the specified indices in the instruction list. - - Args: - inputs: function inputs infos - for_iter_idx (int): For find the for_iter opcode - start (int): The start index of the loop body. - end (int): The end index of the loop body. - - Returns: - tuple: The generated loop body function object and its inputs. - - """ - pycode_gen = PyCodeGen(self._frame) - origin_instrs = get_instructions(pycode_gen._origin_code) - - for_iter = origin_instrs[for_iter_idx] - - # for balance the stack (the loop body will pop iter first before break or return) - # this None is used for replace the iterator obj in stack top - pycode_gen.gen_load_const(None) - - # extend loop body main logic - pycode_gen.extend_instrs(origin_instrs[start:end]) - - # break should jump to this nop - nop_for_break = pycode_gen._add_instr("NOP") - - # need do additional operates when break - pycode_gen.gen_load_const(False) - pycode_gen.gen_store_fast(inputs[-1]) - pycode_gen.gen_load_const(None) # keep stack balance - - # continue should jump to this nop - nop_for_continue = pycode_gen._add_instr("NOP") - pycode_gen.gen_pop_top() - - # relocate jump - out_loop = for_iter.jump_to - for instr in pycode_gen._instructions: - if instr.jump_to == for_iter: - instr.jump_to = nop_for_continue - if instr.jump_to == out_loop: - instr.jump_to = nop_for_break - - # outputs is the same as inputs - pycode_gen.gen_outputs_and_return(inputs) - return pycode_gen.create_fn_with_inputs(inputs) - @fallback_when_occur_error def _break_graph_when_for_loop( self, iterator: VariableBase, for_iter: Instruction ): - ''' - for_iter: the FOR_ITER opcode - - need find out opcodes which unpack value from FOR_ITER, by analysing stack - - case 1: - for i in iter: - - FOR_ITER - STORE_FAST i - - case 2: - for i,j in iter: - - FOR_ITER - UNPACK_SEQUENCE 2 - STORE_FAST i - STORE_FAST j - - TODO: check var is in globals or builtins, only locals considered now - ''' - # 0. prepare sub functions - # 0.1 find the range of loop body + # 1. find the range of loop body assert for_iter.jump_to is not None - loop_body_start_idx = self.indexof(for_iter) + 1 + for_iter_idx = self.indexof(for_iter) + loop_body_start_idx = for_iter_idx + 1 loop_body_end_idx = self.indexof(for_iter.jump_to) curent_stack = 1 @@ -1877,122 +1927,170 @@ def _break_graph_when_for_loop( if curent_stack == 0: break - # 0.2 create loop body function - all_used_vars = analysis_used_names_with_space( + # 2. create loop body function + loop_body_read_names, loop_body_write_names = analysis_used_names( self._instructions, loop_body_start_idx, loop_body_end_idx ) - loop_body_inputs = [ - k - for k, v in all_used_vars.items() - if v in (Space.locals, Space.cells) - ] + ["_break_flag"] - - loop_body_fn = self._gen_loop_body_between( - loop_body_inputs, - self.indexof(for_iter), - loop_body_start_idx, - loop_body_end_idx, - ) + loop_body_inputs = self._find_names_in_space( + loop_body_read_names | loop_body_write_names, + (Space.locals, Space.cells), + ) + ["_break_flag"] + loop_body_outputs = list(loop_body_write_names) + ["_break_flag"] - log(3, "[Resumed Function]: break graph in loop create loop body as\n") - log_do(3, lambda: dis.dis(loop_body_fn)) + def create_loop_body(): + pycode_gen = PyCodeGen(self._frame) - # 0.3 create after loop part function, minus 1 for iterator - after_loop_fn, fn_inputs = self._create_resume_fn( - loop_body_end_idx, len(self.stack) - 1 - ) + pycode_gen.set_function_inputs(loop_body_inputs, stack_size=0) - total_inputs = OrderedSet(list(fn_inputs) + list(loop_body_inputs[:-1])) + origin_instrs = get_instructions(pycode_gen._origin_code) + for_iter = origin_instrs[for_iter_idx] - # 1. part before for-loop, start compile - ret_names = [ - name - for name in total_inputs - if name in chain(self._locals, self._cells) - ] + # for balance the stack (the loop body will pop iter first before break or return) + # this None is used for replace the iterator obj in stack top + pycode_gen.gen_load_const(None) + + # extend loop body main logic + pycode_gen.extend_instrs( + origin_instrs[loop_body_start_idx:loop_body_end_idx] + ) + + # break should jump to this nop + nop_for_break = pycode_gen.add_instr("NOP") + + # need do additional operates when break + pycode_gen.gen_load_const(False) + pycode_gen.gen_store_fast(loop_body_inputs[-1]) + pycode_gen.gen_load_const(None) # keep stack balance + + # continue should jump to this nop + nop_for_continue = pycode_gen.add_instr("NOP") + pycode_gen.gen_pop_top() + + # relocate jump + out_loop = for_iter.jump_to + for instr in pycode_gen._instructions: + if instr.jump_to == for_iter: + instr.jump_to = nop_for_continue + if instr.jump_to == out_loop: + instr.jump_to = nop_for_break + + # outputs is the same as inputs + pycode_gen.set_function_outputs(loop_body_outputs) + loop_body_fn = pycode_gen.create_function() + + log( + 3, + "[Resumed Function]: break graph in loop create loop body as\n", + ) + log_do(3, lambda: dis.dis(loop_body_fn)) - var_loader = self.gen_compute_in_break_with_name_store( - ret_names, self.indexof(for_iter) + return loop_body_fn + + loop_body_fn = create_loop_body() + + # 3. create after loop part function, stack size minus 1 for iterator + after_loop_read_names, _ = analysis_used_names( + self._instructions, loop_body_end_idx, len(self._instructions) + ) + after_loop_fn_inputs = self._find_names_in_space( + after_loop_read_names, (Space.locals, Space.cells) ) - # 2. restore vars with origin name - for name in ret_names: - var_loader.load(self.get_var(name)) - self._graph.pycode_gen.gen_store(name, self._code) + def create_after_loop_fn(): + if self._instructions[loop_body_end_idx].opname == "RETURN_VALUE": + return None + pycode_gen = PyCodeGen(self._frame) + origin_instrs = get_instructions(pycode_gen._origin_code) + pycode_gen.set_function_inputs( + after_loop_fn_inputs, stack_size=len(self.stack) - 1 + ) + pycode_gen.extend_instrs(origin_instrs[loop_body_end_idx:]) + # the resume_fn contains return code, so we don't need set output here + # global vars are updated correctly, and need local vars will return + after_loop_fn = pycode_gen.create_function() + return after_loop_fn - # 3. setup vars which is created in loop as Undefind - undefined_names = set() + after_loop_fn = create_after_loop_fn() + + # 4. setup vars which is created in loop as Undefind for name in loop_body_inputs[:-1]: - if not self.has_var(name, all_used_vars[name]): - undefined_names.add(name) + if not self.has_var(name): + self._graph.pycode_gen.gen_load_const(SotUndefinedVar()) + self._graph.pycode_gen.gen_store(name, self._code) + for name in after_loop_fn_inputs: + if not self.has_var(name): self._graph.pycode_gen.gen_load_const(SotUndefinedVar()) self._graph.pycode_gen.gen_store(name, self._code) - # 4.1 load iterator + # 5. compile sub graph before for-loop + update_names = list(loop_body_read_names | after_loop_read_names) + var_loader = self.get_compute_fn_and_update_changed_vars( + update_names, self.stack, self.indexof(for_iter) + ) + + # 6. prepare a new loop and call loop body + # 6.1. load iterator, it is in stack, so we can load it with var_loader var_loader.load(iterator) self.stack.pop() - # 4.2 gen FOR_ITER and unpack data + # 6.2. copy FOR_ITER and unpack logic self._graph.pycode_gen.extend_instrs( - self._instructions[self.indexof(for_iter) : loop_body_start_idx] + self._instructions[for_iter_idx:loop_body_start_idx] ) - # 5. call loop body - # 5.1 load loop body + # 6.3 load loop body, prepare inputs and call self._graph.pycode_gen.gen_load_object( loop_body_fn, loop_body_fn.__code__.co_name ) - # 5.2 load loop body inputs for name in loop_body_inputs[:-1]: self._graph.pycode_gen.gen_load(name) - # 5.3 load break flag + # this is the _break_flag self._graph.pycode_gen.gen_load_const(True) - # 5.4 call loop body self._graph.pycode_gen.gen_call_function( argc=loop_body_fn.__code__.co_argcount ) - # 5.5 unpack and store retval, keep break_flag in stack - self._graph.pycode_gen.gen_unpack_sequence(len(loop_body_inputs)) + # 7. unpack and update changed vars, keep break_flag in stack + self._graph.pycode_gen.gen_unpack_sequence(len(loop_body_outputs)) - for name in loop_body_inputs[:-1]: + for name in loop_body_outputs[:-1]: self._graph.pycode_gen.gen_store(name, self._code) - # 6. add jump if break + # 8. create the tail of a for loop, jump back to FOR_ITER + # and process case if break jump_if_break = self._graph.pycode_gen.gen_pop_jump( direction=JumpDirection.FORWARD, suffix=PopJumpCond.FALSE ) - # 7. jump back to FOR_ITER self._graph.pycode_gen.gen_jump( for_iter, direction=JumpDirection.BACKWARD ) - nop = self._graph.pycode_gen._add_instr("NOP") + nop = self._graph.pycode_gen.add_instr("NOP") for_iter.jump_to = nop jump_if_break.jump_to = nop - # 8. call after_loop_fn - self._graph.pycode_gen.gen_load_object( - after_loop_fn, after_loop_fn.__code__.co_name - ) + # 9. prepare inputs and call after_loop_fn + if after_loop_fn is not None: + self._graph.pycode_gen.gen_load_object( + after_loop_fn, after_loop_fn.__code__.co_name + ) - for stack_arg in self.stack: - var_loader.load(stack_arg) - for name in fn_inputs: - if not self.has_var(name) and name not in undefined_names: - undefined_names.add(name) - self._graph.pycode_gen.gen_load_const(SotUndefinedVar()) - self._graph.pycode_gen.gen_store(name, self._code) - self._graph.pycode_gen.gen_load(name) + for stack_arg in self.stack: + var_loader.load(stack_arg) - self._graph.pycode_gen.gen_call_function( - argc=after_loop_fn.__code__.co_argcount - ) + for name in after_loop_fn_inputs: + self._graph.pycode_gen.gen_load(name) + + self._graph.pycode_gen.gen_call_function( + argc=after_loop_fn.__code__.co_argcount + ) + # return what after_loop_fn return self._graph.pycode_gen.gen_return() + self.new_code = self._graph.pycode_gen.gen_pycode() self.guard_fn = self._graph.guard_fn @@ -2000,135 +2098,95 @@ def _inline_call_for_loop( self, iterator: VariableBase, for_iter: Instruction ): assert for_iter.jump_to is not None - pycode_gen = PyCodeGen(self._frame) - origin_instrs = get_instructions(pycode_gen._origin_code) + # 1. analyse input and output start_idx = self.indexof(for_iter) end_idx = self.indexof(for_iter.jump_to) - all_used_vars = analysis_used_names_with_space( - origin_instrs, start_idx, end_idx + read_names, write_names = analysis_used_names( + self._instructions, start_idx, end_idx ) - inputs = [ - k - for k, v in all_used_vars.items() - if v in (Space.locals, Space.cells) - ] + [iterator.id] + # why add write_names as input? check case in test/sot/test_12_for_loop.py + # test_for_without_zero_iter + input_var_names = self._find_names_in_space( + read_names | write_names, (Space.locals, Space.cells) + ) + [iterator.id] + output_var_names = list(write_names) + [iterator.id] - # 1. load iter - pycode_gen.gen_load_fast(iterator.id) + # 2. create inline call loop fn + def create_inline_call_fn(): + pycode_gen = PyCodeGen(self._frame) + origin_instrs = get_instructions(pycode_gen._origin_code) - # 2. copy main logic - pycode_gen.extend_instrs(origin_instrs[start_idx:end_idx]) + pycode_gen.set_function_inputs(input_var_names, stack_size=0) - # 3. add break, continue marker and relocate jump - for_iter_instr = origin_instrs[start_idx] - assert for_iter_instr.jump_to is not None - out_loop_instr = for_iter_instr.jump_to + # 2.1. load iter, it is a input of loop fn + pycode_gen.gen_load_fast(iterator.id) - pycode_gen.gen_jump(out_loop_instr, direction=JumpDirection.FORWARD) - nop_for_continue = pycode_gen._add_instr("NOP") + # 2.2. copy main logic + pycode_gen.extend_instrs(origin_instrs[start_idx:end_idx]) - jump = pycode_gen.gen_jump( - for_iter_instr, direction=JumpDirection.BACKWARD - ) + # 2.3. add break, continue marker and relocate jump + for_iter_instr = origin_instrs[start_idx] + assert for_iter_instr.jump_to is not None + out_loop_instr = for_iter_instr.jump_to - nop_for_break = pycode_gen._add_instr("NOP") + pycode_gen.gen_jump(out_loop_instr, direction=JumpDirection.FORWARD) + nop_for_continue = pycode_gen.add_instr("NOP") - for instr in pycode_gen._instructions: - if instr.jump_to == for_iter_instr: - instr.jump_to = nop_for_continue + jump = pycode_gen.gen_jump( + for_iter_instr, direction=JumpDirection.BACKWARD + ) - if ( - instr.jump_to in origin_instrs - and origin_instrs.index(instr.jump_to) >= end_idx - ): - instr.jump_to = nop_for_break + nop_for_break = pycode_gen.add_instr("NOP") - jump.jump_to = for_iter_instr - pycode_gen.gen_outputs_and_return(inputs) - inline_call_fn = pycode_gen.create_fn_with_inputs(inputs) + # 2.4. relocate jumps + for instr in pycode_gen._instructions: + if instr.jump_to == for_iter_instr: + instr.jump_to = nop_for_continue - log( - 3, - f"[Resumed Function]: Inline call for loop function {inline_call_fn.__code__.co_name}\n", - ) - log_do(3, lambda: dis.dis(inline_call_fn)) + if ( + instr.jump_to in origin_instrs + and origin_instrs.index(instr.jump_to) >= end_idx + ): + instr.jump_to = nop_for_break + + jump.jump_to = for_iter_instr + + pycode_gen.set_function_outputs(output_var_names) + inline_call_fn = pycode_gen.create_function() - # TODO: update globals builtins + log( + 3, + f"[Resumed Function]: Inline call for loop function {inline_call_fn.__code__.co_name}\n", + ) + log_do(3, lambda: dis.dis(inline_call_fn)) + + return inline_call_fn + + inline_call_fn = create_inline_call_fn() + + # 3. create function variable fn = UserDefinedFunctionVariable( inline_call_fn, self._graph, DanglingTracker(), ) + # 4. prepare input datas and call input_vars = [ - self.get_var(name) - if self.has_var(name, all_used_vars[name]) - else SotUndefinedVar() - for name in inputs[:-1] + self.get_var(name, allow_undefined=True) + for name in input_var_names[:-1] ] + [iterator] + ret = fn(*input_vars) - # slice_variable is [:-1] + + # 5. update changed vars slice_const = slice(None, -1, None) slice_variable = SliceVariable( slice_const, self._graph, ConstTracker(slice_const) ) - for name, val in zip(inputs[:-1], ret[slice_variable]): - self._locals[name] = val - - def FOR_ITER(self, instr): - iterator = self.stack.pop() - backup_iter_idx = None - - start = self.indexof(instr) - end = self.indexof(instr.jump_to) - for i in range(start, end): - if self._instructions[i].opname == "RETURN_VALUE": - raise FallbackError("Found RETURN_VALUE in for loop body.") - - self._graph.add_global_guarded_variable(iterator) - - try: - if not isinstance(iterator, SequenceIterVariable): - raise BreakGraphError( - f"Can not simulate iterator of {type(iterator)}." - ) - - backup_iter_idx = iterator.idx - - self._inline_call_for_loop(iterator, instr) - self._lasti = self.indexof(instr.jump_to) - next_instr = self._instructions[self._lasti] - self._lasti += int(next_instr.opname == 'END_FOR') - except BreakGraphError as e: - log(3, f"[BreakGraph] FOR_ITER sim for loop failed for: {e}\n") - if backup_iter_idx: - iterator.idx = backup_iter_idx - self._graph.remove_global_guarded_variable(iterator) - self.stack.push(iterator) - self._break_graph_when_for_loop(iterator, instr) - return Stop(state="BreakGraph") - - def RETURN_VALUE(self, instr: Instruction): - assert ( - len(self.stack) == 1 - ), f"Stack must have one element, but get {len(self.stack)} elements." - ret_val = self.stack.pop() - return self.compile_return(ret_val) - - def RETURN_CONST(self, instr: Instruction): - ret_const = self._co_consts[instr.arg] - return self.compile_return(ret_const) - def compile_return(self, ret_val): - compile_fn = self._graph.get_compiled_fn(ret_val) - if compile_fn.graph_size() < ENV_MIN_GRAPH_SIZE.get(): - self.new_code = None - else: - self._graph.start_compile(ret_val) - self._graph.pycode_gen.gen_return() - self.new_code = self._graph.pycode_gen.gen_pycode() - self.guard_fn = self._graph.guard_fn - return Stop(state="Return") + for name, var in zip(output_var_names[:-1], ret[slice_variable]): + self.set_var(name, var) diff --git a/python/paddle/jit/sot/opcode_translator/executor/opcode_inline_executor.py b/python/paddle/jit/sot/opcode_translator/executor/opcode_inline_executor.py index 3832d05f04448..306166aa7d872 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/opcode_inline_executor.py +++ b/python/paddle/jit/sot/opcode_translator/executor/opcode_inline_executor.py @@ -299,16 +299,6 @@ def _break_graph_when_if(self, result, instr: Instruction): "OpcodeInlineExecutor want break graph when simulate `if`." ) - def _create_resume_fn(self, index: int, stack_size: int = 0): - """ - Helper method to create a resume function for the executor. - - Args: - index (int): The index of the instruction to resume execution from. - stack_size (int, optional): The size of the stack. Defaults to 0. - """ - raise BreakGraphError("_create_resume_fn.") - def FOR_ITER(self, instr: Instruction): iterator = self.stack.top assert isinstance(iterator, IterVariable) diff --git a/python/paddle/jit/sot/opcode_translator/executor/pycode_generator.py b/python/paddle/jit/sot/opcode_translator/executor/pycode_generator.py index 69e174818d662..2ada3f7228f11 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/pycode_generator.py +++ b/python/paddle/jit/sot/opcode_translator/executor/pycode_generator.py @@ -39,11 +39,9 @@ no_eval_frame, ) from ..instruction_utils import ( - analysis_inputs, apply_instr_pass, calc_stack_effect, gen_instr, - get_instructions, instrs_info, modify_instrs, modify_vars, @@ -437,6 +435,42 @@ def __init__( self.hooks = [] if self.disable_eval_frame: self.gen_disable_eval_frame() + self.fn_name = ResumeFnNameFactory().next() + + def set_function_inputs(self, inputs: list[str], stack_size: int): + stack_arg_str = self.fn_name + '_stack_{}' + + self._code_options['co_argcount'] = len(inputs) + stack_size + self._code_options['co_varnames'] = list( + [stack_arg_str.format(i) for i in range(stack_size)] + + inputs + + [ + var_name + for var_name in self._origin_code.co_varnames + if var_name not in inputs + ] + ) + + self._instructions.extend( + [ + gen_instr('LOAD_FAST', argval=stack_arg_str.format(i)) + for i in range(stack_size) + ] + ) + + def set_function_outputs(self, outputs: list[str]): + for name in outputs: + self.gen_load(name) + self.gen_build_tuple(len(outputs)) + self.gen_return() + + def create_function(self) -> types.FunctionType: + self.update_code_name(self.fn_name, is_resumed_fn=True) + new_code = self.gen_pycode() + if len(new_code.co_freevars) + len(new_code.co_cellvars) > 0: + raise FallbackError("Break graph in closure is not support.") + fn = types.FunctionType(new_code, self._f_globals, new_code.co_name) + return fn def insert_prefix_instructions(self): """ @@ -509,58 +543,6 @@ def gen_pycode(self) -> types.CodeType: return new_code - def gen_resume_fn_at( - self, index: int, stack_size: int - ) -> tuple[None | types.FunctionType, OrderedSet[str]]: - """ - Generates a resume function at the specified index in the instruction list. - - Args: - index (int): The index in the instruction list to generate the resume function. - stack_size (int): The size of the stack. Defaults to 0. - - Returns: - tuple: The resume function object and the inputs to the function. - - """ - - self._instructions = get_instructions(self._origin_code) - # TODO(dev): could give an example code here? - if self._instructions[index].opname == 'RETURN_VALUE': - return None, OrderedSet() - inputs = analysis_inputs(self._instructions, index) - fn_name = ResumeFnNameFactory().next() - stack_arg_str = fn_name + '_stack_{}' - - self._instructions = ( - [ - gen_instr('LOAD_FAST', argval=stack_arg_str.format(i)) - for i in range(stack_size) - ] - + [gen_instr('JUMP_FORWARD', jump_to=self._instructions[index])] - + self._instructions - ) - - self._code_options['co_argcount'] = len(inputs) + stack_size - # inputs should be at the front of the co_varnames - self._code_options['co_varnames'] = list( - [stack_arg_str.format(i) for i in range(stack_size)] - + list(inputs) - + [ - var_name - for var_name in self._code_options['co_varnames'] - if var_name not in inputs - ] - ) - - self.update_code_name(fn_name, is_resumed_fn=True) - new_code = self.gen_pycode() - if len(new_code.co_freevars) + len(new_code.co_cellvars) > 0: - raise FallbackError("Break graph in closure is not support.") - fn = types.FunctionType(new_code, self._f_globals, new_code.co_name) - - return fn, inputs - @cached_property def global_null_variable(self): from .variables.basic import NullVariable @@ -593,39 +575,6 @@ def gen_enable_eval_frame(self): self.gen_call_function(1) self.gen_pop_top() - def gen_outputs_and_return(self, outputs): - for name in outputs: - self.gen_load(name) - self.gen_build_tuple(len(outputs)) - self.gen_return() - - def create_fn_with_inputs(self, inputs: list) -> types.FunctionType: - """ - Creates a function with specific input and output variables. - - Args: - inputs (list): The input variables. - - Returns: - function: The created function object. - """ - self._code_options['co_argcount'] = len(inputs) - self._code_options['co_varnames'] = list( - list(inputs) - + [ - var_name - for var_name in self._origin_code.co_varnames - if var_name not in inputs - ] - ) - fn_name = ResumeFnNameFactory().next() - self.update_code_name(fn_name, is_resumed_fn=True) - new_code = self.gen_pycode() - if len(new_code.co_freevars) + len(new_code.co_cellvars) > 0: - raise FallbackError("Break graph in closure is not support.") - fn = types.FunctionType(new_code, self._f_globals, new_code.co_name) - return fn - def gen_load_const(self, value: Any): """ Generates instructions to load a constant value. @@ -636,7 +585,7 @@ def gen_load_const(self, value: Any): if not list_contain_by_id(self._code_options["co_consts"], value): self._code_options["co_consts"].append(value) idx = list_find_index_by_id(self._code_options["co_consts"], value) - self._add_instr("LOAD_CONST", arg=idx, argval=value) + return self.add_instr("LOAD_CONST", arg=idx, argval=value) def gen_print_log(self, message): """print a log""" @@ -745,7 +694,7 @@ def gen_load_global(self, name, push_null=False): idx <<= 1 if push_null: idx |= 1 - self._add_instr("LOAD_GLOBAL", arg=idx, argval=name) + return self.add_instr("LOAD_GLOBAL", arg=idx, argval=name) def gen_load_object(self, obj, obj_name: str, push_null: bool = True): """ @@ -758,14 +707,14 @@ def gen_load_object(self, obj, obj_name: str, push_null: bool = True): if obj_name not in self._f_globals: self._f_globals[obj_name] = obj - self.gen_load_global(obj_name, push_null=push_null) + return self.gen_load_global(obj_name, push_null=push_null) def gen_load_null_variable(self): """ Generate the bytecode for loading a null variable. """ null_var = self.global_null_variable - self.gen_load_object(null_var, "___null_var", push_null=False) + return self.gen_load_object(null_var, "___null_var", push_null=False) def gen_load_fast(self, name): """ @@ -777,7 +726,7 @@ def gen_load_fast(self, name): if name not in self._code_options["co_varnames"]: self._code_options["co_varnames"].append(name) idx = self._code_options["co_varnames"].index(name) - self._add_instr("LOAD_FAST", arg=idx, argval=name) + return self.add_instr("LOAD_FAST", arg=idx, argval=name) def gen_load_deref(self, name): if name not in self.cell_free_storage: @@ -791,7 +740,7 @@ def gen_load_deref(self, name): ).index(name) else: idx = self.cell_free_storage.index(name) - self._add_instr("LOAD_DEREF", arg=idx, argval=name) + return self.add_instr("LOAD_DEREF", arg=idx, argval=name) def gen_load_attr(self, name: str): if name not in self._code_options["co_names"]: @@ -799,49 +748,49 @@ def gen_load_attr(self, name: str): idx = self._code_options["co_names"].index(name) if sys.version_info >= (3, 12): idx <<= 1 - self._add_instr("LOAD_ATTR", arg=idx, argval=name) + return self.add_instr("LOAD_ATTR", arg=idx, argval=name) def gen_store_attr(self, name: str): if name not in self._code_options["co_names"]: self._code_options["co_names"].append(name) idx = self._code_options["co_names"].index(name) - self._add_instr("STORE_ATTR", arg=idx, argval=name) + return self.add_instr("STORE_ATTR", arg=idx, argval=name) def gen_delete_attr(self, name: str): if name not in self._code_options["co_names"]: self._code_options["co_names"].append(name) idx = self._code_options["co_names"].index(name) - self._add_instr("DELETE_ATTR", arg=idx, argval=name) + return self.add_instr("DELETE_ATTR", arg=idx, argval=name) def gen_load_method(self, name: str): if name not in self._code_options["co_names"]: self._code_options["co_names"].append(name) idx = self._code_options["co_names"].index(name) - self._add_instr("LOAD_METHOD", arg=idx, argval=name) + return self.add_instr("LOAD_METHOD", arg=idx, argval=name) def gen_delete_global(self, name: str): if name not in self._code_options["co_names"]: self._code_options["co_names"].append(name) idx = self._code_options["co_names"].index(name) - self._add_instr("DELETE_GLOBAL", arg=idx, argval=name) + return self.add_instr("DELETE_GLOBAL", arg=idx, argval=name) def gen_import_name(self, name: str): if name not in self._code_options["co_names"]: self._code_options["co_names"].append(name) idx = self._code_options["co_names"].index(name) - self._add_instr("IMPORT_NAME", arg=idx, argval=name) + return self.add_instr("IMPORT_NAME", arg=idx, argval=name) def gen_store_fast(self, name): if name not in self._code_options["co_varnames"]: self._code_options["co_varnames"].append(name) idx = self._code_options["co_varnames"].index(name) - self._add_instr("STORE_FAST", arg=idx, argval=name) + return self.add_instr("STORE_FAST", arg=idx, argval=name) def gen_store_global(self, name): if name not in self._code_options["co_names"]: self._code_options["co_names"].append(name) idx = self._code_options["co_names"].index(name) - self._add_instr("STORE_GLOBAL", arg=idx, argval=name) + return self.add_instr("STORE_GLOBAL", arg=idx, argval=name) def gen_store_deref(self, name): if name not in self.cell_free_storage: @@ -855,50 +804,50 @@ def gen_store_deref(self, name): ).index(name) else: idx = self.cell_free_storage.index(name) - self._add_instr("STORE_DEREF", arg=idx, argval=name) + return self.add_instr("STORE_DEREF", arg=idx, argval=name) def gen_store_subscr(self): - self._add_instr("STORE_SUBSCR") + return self.add_instr("STORE_SUBSCR") def gen_subscribe(self): - self._add_instr("BINARY_SUBSCR") + return self.add_instr("BINARY_SUBSCR") def gen_build_tuple(self, count): - self._add_instr("BUILD_TUPLE", arg=count, argval=count) + return self.add_instr("BUILD_TUPLE", arg=count, argval=count) def gen_build_list(self, count): - self._add_instr("BUILD_LIST", arg=count, argval=count) + return self.add_instr("BUILD_LIST", arg=count, argval=count) def gen_build_map(self, count): - self._add_instr("BUILD_MAP", arg=count, argval=count) + return self.add_instr("BUILD_MAP", arg=count, argval=count) def gen_build_slice(self, argc): - self._add_instr("BUILD_SLICE", arg=argc, argval=argc) + return self.add_instr("BUILD_SLICE", arg=argc, argval=argc) def gen_unpack_sequence(self, count): - self._add_instr("UNPACK_SEQUENCE", arg=count, argval=count) + return self.add_instr("UNPACK_SEQUENCE", arg=count, argval=count) def gen_call_function(self, argc=0): if sys.version_info >= (3, 11): if sys.version_info < (3, 12): - self._add_instr("PRECALL", arg=argc, argval=argc) - self._add_instr("CALL", arg=argc, argval=argc) + self.add_instr("PRECALL", arg=argc, argval=argc) + self.add_instr("CALL", arg=argc, argval=argc) else: - self._add_instr("CALL_FUNCTION", arg=argc, argval=argc) + self.add_instr("CALL_FUNCTION", arg=argc, argval=argc) def gen_call_function_ex(self, has_kwargs): flag = 0 if has_kwargs: flag |= CALL_FUNCTION_EX_FLAG.CFE_HAS_KWARGS - self._add_instr("CALL_FUNCTION_EX", arg=flag, argval=flag) + self.add_instr("CALL_FUNCTION_EX", arg=flag, argval=flag) def gen_call_method(self, argc=0): if sys.version_info >= (3, 11): if sys.version_info < (3, 12): - self._add_instr("PRECALL", arg=argc, argval=argc) - self._add_instr("CALL", arg=argc, argval=argc) + self.add_instr("PRECALL", arg=argc, argval=argc) + self.add_instr("CALL", arg=argc, argval=argc) else: - self._add_instr("CALL_METHOD", arg=argc, argval=argc) + self.add_instr("CALL_METHOD", arg=argc, argval=argc) def gen_kw_names(self, kw_names: tuple[str, ...] | None): if kw_names is None: @@ -908,22 +857,22 @@ def gen_kw_names(self, kw_names: tuple[str, ...] | None): if kw_names not in self._code_options["co_consts"]: self._code_options["co_consts"].append(kw_names) idx = self._code_options["co_consts"].index(kw_names) - self._add_instr("KW_NAMES", arg=idx, argval=kw_names) + self.add_instr("KW_NAMES", arg=idx, argval=kw_names) def gen_pop_top(self): - self._add_instr("POP_TOP") + return self.add_instr("POP_TOP") def gen_rot_n(self, n): if n <= 1: return if sys.version_info >= (3, 11): for i in range(n, 1, -1): - self._add_instr("SWAP", arg=i) + self.add_instr("SWAP", arg=i) elif sys.version_info >= (3, 10): - self._add_instr("ROT_N", arg=n) + self.add_instr("ROT_N", arg=n) else: if n <= 4: - self._add_instr("ROT_" + ["TWO", "THREE", "FOUR"][n - 2]) + self.add_instr("ROT_" + ["TWO", "THREE", "FOUR"][n - 2]) else: def rot_n_fn(n): @@ -937,7 +886,7 @@ def rot_n_fn(n): self.gen_build_tuple(n) self.gen_load_const(rot_n_fn(n)) self.gen_rot_n(2) - self._add_instr("CALL_FUNCTION_EX", arg=0) + self.add_instr("CALL_FUNCTION_EX", arg=0) self.gen_unpack_sequence(n) def gen_shift_n(self, s: int, n: int): @@ -970,7 +919,7 @@ def gen_shift_n(self, s: int, n: int): # NOTE: s=-1, n=3 [1,2,3,4,5] -> [1,2,4,5,3] if s == -1: for i in range(2, n + 1): - self._add_instr("SWAP", arg=i) + self.add_instr("SWAP", arg=i) else: self.gen_shift_n(-1, n) self.gen_shift_n(s + 1, n) @@ -981,7 +930,7 @@ def gen_shift_n(self, s: int, n: int): def gen_swap(self, n): if sys.version_info >= (3, 11): - self._add_instr("SWAP", arg=n) + self.add_instr("SWAP", arg=n) else: raise NotImplementedError("swap is not supported before python3.11") @@ -992,9 +941,9 @@ def gen_jump( direction: JumpDirection = JumpDirection.FORWARD, ) -> Instruction: if sys.version_info >= (3, 11): - return self._add_instr(f"JUMP_{direction.value}", jump_to=jump_to) + return self.add_instr(f"JUMP_{direction.value}", jump_to=jump_to) else: - return self._add_instr("JUMP_ABSOLUTE", jump_to=jump_to) + return self.add_instr("JUMP_ABSOLUTE", jump_to=jump_to) def gen_pop_jump( self, @@ -1004,33 +953,33 @@ def gen_pop_jump( suffix: PopJumpCond = PopJumpCond.NONE, ) -> Instruction: if sys.version_info >= (3, 11): - return self._add_instr( + return self.add_instr( f"POP_JUMP_{direction.value}_IF_{suffix.value}", jump_to=jump_to ) else: - return self._add_instr( + return self.add_instr( f"POP_JUMP_IF_{suffix.value}", jump_to=jump_to ) def gen_return(self): - self._add_instr("RETURN_VALUE") + return self.add_instr("RETURN_VALUE") def gen_get_iter(self): - self._add_instr("GET_ITER") + return self.add_instr("GET_ITER") def gen_operator_only(self, op_name): """ only generator operator instruction, do nothing for operands. """ - self._add_instr(op_name) + return self.add_instr(op_name) def gen_operator(self, op_name): """ only generator operator instruction, do nothing for operands. """ - self._add_instr(op_name) + return self.add_instr(op_name) def gen_compare(self, cmp_op): """ @@ -1039,9 +988,9 @@ def gen_compare(self, cmp_op): """ if sys.version_info >= (3, 12): cmp_op <<= 4 - self._add_instr("COMPARE_OP", cmp_op) + return self.add_instr("COMPARE_OP", cmp_op) - def _add_instr(self, *args, **kwargs): + def add_instr(self, *args, **kwargs): instr = gen_instr(*args, **kwargs) self._instructions.append(instr) return instr diff --git a/python/paddle/jit/sot/opcode_translator/executor/tracker.py b/python/paddle/jit/sot/opcode_translator/executor/tracker.py index fd7168f4e5957..51d21a5572129 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/tracker.py +++ b/python/paddle/jit/sot/opcode_translator/executor/tracker.py @@ -393,7 +393,7 @@ def __init__(self, iter_source: VariableBase): def gen_instructions(self, codegen: PyCodeGen): self.iter_source.tracker.gen_instructions(codegen) - codegen._add_instr("GET_ITER") + codegen.add_instr("GET_ITER") def trace_value_from_frame(self): iter_source_tracer = self.iter_source.tracker.trace_value_from_frame() diff --git a/python/paddle/jit/sot/opcode_translator/instruction_utils/__init__.py b/python/paddle/jit/sot/opcode_translator/instruction_utils/__init__.py index 0b9429e078ec7..833fd3c207e88 100644 --- a/python/paddle/jit/sot/opcode_translator/instruction_utils/__init__.py +++ b/python/paddle/jit/sot/opcode_translator/instruction_utils/__init__.py @@ -15,6 +15,7 @@ from .instruction_pass import apply_instr_pass # noqa: F401 from .instruction_utils import ( # noqa: F401 Instruction, + Space, calc_offset_from_bytecode_offset, calc_stack_effect, convert_instruction, @@ -29,7 +30,5 @@ reset_offset, ) from .opcode_analysis import ( # noqa: F401 - Space, - analysis_inputs, - analysis_used_names_with_space, + analysis_used_names, ) diff --git a/python/paddle/jit/sot/opcode_translator/instruction_utils/instruction_pass.py b/python/paddle/jit/sot/opcode_translator/instruction_utils/instruction_pass.py index 8725aa55c3213..5b0cc17fc808f 100644 --- a/python/paddle/jit/sot/opcode_translator/instruction_utils/instruction_pass.py +++ b/python/paddle/jit/sot/opcode_translator/instruction_utils/instruction_pass.py @@ -90,6 +90,8 @@ def find_related_local_opcodes(instrs, code_options): if len(stack) > 0 and stack[-1] is not None: opcode_pairs.append((stack[-1], instr)) stack.pop() + elif "ROT" in instr.opname: + return [] else: try: pop_n, push_n = StackAnalyser().stack_effect(instr) diff --git a/python/paddle/jit/sot/opcode_translator/instruction_utils/instruction_utils.py b/python/paddle/jit/sot/opcode_translator/instruction_utils/instruction_utils.py index 05e6dcfc91e7d..2965c8e6bc056 100644 --- a/python/paddle/jit/sot/opcode_translator/instruction_utils/instruction_utils.py +++ b/python/paddle/jit/sot/opcode_translator/instruction_utils/instruction_utils.py @@ -17,6 +17,7 @@ import dataclasses import dis import sys +from enum import Enum from typing import TYPE_CHECKING, Any from ...utils import InnerError @@ -410,3 +411,10 @@ def calc_stack_effect(instr: Instruction, *, jump: bool | None = None) -> int: assert instr.arg is not None return -instr.arg - 1 return dis.stack_effect(instr.opcode, instr.arg, jump=jump) + + +class Space(Enum): + locals = 1 + globals = 2 + cells = 3 + not_found = 4 diff --git a/python/paddle/jit/sot/opcode_translator/instruction_utils/opcode_analysis.py b/python/paddle/jit/sot/opcode_translator/instruction_utils/opcode_analysis.py index f0211167f4449..2e8ded5d2ac5e 100644 --- a/python/paddle/jit/sot/opcode_translator/instruction_utils/opcode_analysis.py +++ b/python/paddle/jit/sot/opcode_translator/instruction_utils/opcode_analysis.py @@ -15,11 +15,9 @@ from __future__ import annotations import dataclasses -from enum import Enum from paddle.jit.utils import OrderedSet -from ...utils import InnerError from .instruction_utils import Instruction from .opcode_info import ALL_JUMP, HAS_FREE, HAS_LOCAL, UNCONDITIONAL_JUMP @@ -30,6 +28,11 @@ class State: writes: OrderedSet[str] visited: OrderedSet[int] + def __or__(self, other): + reads = self.reads | other.reads + writes = self.writes | other.writes + return State(reads, writes, OrderedSet()) + def is_read_opcode(opname): if opname in [ @@ -63,7 +66,7 @@ def is_write_opcode(opname): return False -def analysis_inputs( +def analysis_used_names( instructions: list[Instruction], current_instr_idx: int, stop_instr_idx: int | None = None, @@ -97,7 +100,7 @@ def walk(state: State, start: int) -> OrderedSet[str]: end = len(instructions) if stop_instr_idx is None else stop_instr_idx for i in range(start, end): if i in state.visited: - return state.reads + return state state.visited.add(i) instr = instructions[i] @@ -116,104 +119,12 @@ def walk(state: State, start: int) -> OrderedSet[str]: not_jump_branch = ( fork(state, i, False, target_idx) if instr.opname not in UNCONDITIONAL_JUMP - else OrderedSet() - ) - return jump_branch | not_jump_branch - elif instr.opname == "RETURN_VALUE": - return state.reads - return state.reads - - return walk(root_state, current_instr_idx) - - -@dataclasses.dataclass -class SpaceState: - reads: dict[str, Space] - writes: dict[str, Space] - visited: OrderedSet[int] - - def __or__(self, other): - reads = {} - reads.update(other.reads) - reads.update(self.reads) - writes = {} - writes.update(other.writes) - writes.update(self.writes) - return SpaceState(reads, writes, OrderedSet()) - - -class Space(Enum): - locals = 1 - globals = 2 - cells = 3 - all = 4 - - -def get_space(opname: str): - if "FAST" in opname: - return Space.locals - elif "GLOBAL" in opname: - return Space.globals - elif "DEREF" in opname or "CLOSURE" in opname: - return Space.cells - elif "NAME" in opname: - return Space.all - else: - raise InnerError(f"Unknown space for {opname}") - - -def analysis_used_names_with_space( - instructions: list[Instruction], - start_instr_idx: int, - stop_instr_idx: int | None = None, -): - root_state = SpaceState({}, {}, OrderedSet()) - - def fork( - state: SpaceState, start: int, jump: bool, jump_target: int - ) -> SpaceState: - new_start = start + 1 if not jump else jump_target - new_state = SpaceState( - dict(state.reads), - dict(state.writes), - OrderedSet(state.visited), - ) - return walk(new_state, new_start) - - def walk(state: SpaceState, start: int) -> SpaceState: - end = len(instructions) if stop_instr_idx is None else stop_instr_idx - for i in range(start, end): - if i in state.visited: - return state - state.visited.add(i) - - instr = instructions[i] - if instr.opname in HAS_LOCAL | HAS_FREE: - if is_read_opcode(instr.opname) and instr.argval not in ( - state.writes - ): - space = get_space(instr.opname) - state.reads[instr.argval] = space - elif is_write_opcode(instr.opname): - space = get_space(instr.opname) - state.writes[instr.argval] = space - elif instr.opname in ALL_JUMP: - assert instr.jump_to is not None - target_idx = instructions.index(instr.jump_to) - # Fork to two branches, jump or not - jump_branch = fork(state, i, True, target_idx) - not_jump_branch = ( - fork(state, i, False, target_idx) - if instr.opname not in UNCONDITIONAL_JUMP - else SpaceState({}, {}, OrderedSet()) + else State(OrderedSet(), OrderedSet(), OrderedSet()) ) return jump_branch | not_jump_branch elif instr.opname == "RETURN_VALUE": return state return state - state = walk(root_state, start_instr_idx) - all_used_vars = {} - all_used_vars.update(state.writes) - all_used_vars.update(state.reads) - return all_used_vars + state = walk(root_state, current_instr_idx) + return state.reads, state.writes diff --git a/test/sot/test_11_jumps.py b/test/sot/test_11_jumps.py index 80fa1f4a4eb02..6073766e8b60f 100644 --- a/test/sot/test_11_jumps.py +++ b/test/sot/test_11_jumps.py @@ -114,5 +114,17 @@ def test_breakgraph(self): self.assert_results(pop_jump_if_not_none, true_tensor, a) +def new_var_in_if(): + x = paddle.to_tensor(1) + if x > 0: + y = 1 + return y + + +class TestCreateVarInIf(TestCaseBase): + def test_case(self): + self.assert_results(new_var_in_if) + + if __name__ == "__main__": unittest.main() diff --git a/test/sot/test_analysis_inputs.py b/test/sot/test_analysis_inputs.py index 20b32c2225324..880de6060d400 100644 --- a/test/sot/test_analysis_inputs.py +++ b/test/sot/test_analysis_inputs.py @@ -20,7 +20,7 @@ import paddle from paddle.jit.sot.opcode_translator.instruction_utils import ( - analysis_inputs, + analysis_used_names, calc_offset_from_bytecode_offset, get_instructions, ) @@ -36,12 +36,12 @@ def assert_inputs_equals(instruction_offset: int, expected_inputs: set[str]): current_instr_idx = calc_offset_from_bytecode_offset( test_frame.f_lasti + 2, instructions ) - actual_inputs = analysis_inputs( + reads, writes = analysis_used_names( instructions, current_instr_idx + instruction_offset ) assert ( - set(actual_inputs) == expected_inputs - ), f"actual_inputs: {actual_inputs}, expected_inputs: {expected_inputs}" + set(reads) == expected_inputs + ), f"actual_inputs: {reads}, expected_inputs: {expected_inputs}" def case1(x): From f4abfbec2489c269ec0082f87f5ba53e90eb5f6e Mon Sep 17 00:00:00 2001 From: cyber-pioneer <116002591+cyber-pioneer@users.noreply.github.com> Date: Mon, 26 Feb 2024 11:33:48 +0800 Subject: [PATCH 072/282] support decomp swiglu (#62026) * support swiglu decomp * support decomp swiglu * add swiglu test case * update prim op list * fix test case * fix bn test case --- .../decomp_interface_gen_op_list.py | 2 + paddle/fluid/primitive/base/primitive_ops.h | 2 + paddle/fluid/primitive/composite/composite.h | 13 +++++ .../paddle/jit/sot/utils/paddle_api_config.py | 1 + .../test_batch_norm_op_prim_nchw.py | 5 +- .../test_batch_norm_op_prim_nhwc.py | 1 + .../test_prim_sub_graph_dynamic_shape.py | 50 ++++++++++++++++--- 7 files changed, 65 insertions(+), 9 deletions(-) diff --git a/paddle/fluid/pir/dialect/op_generator/decomp_interface_gen_op_list.py b/paddle/fluid/pir/dialect/op_generator/decomp_interface_gen_op_list.py index b40e8b4d3dea2..9af8dfa12d702 100644 --- a/paddle/fluid/pir/dialect/op_generator/decomp_interface_gen_op_list.py +++ b/paddle/fluid/pir/dialect/op_generator/decomp_interface_gen_op_list.py @@ -42,6 +42,7 @@ "rsqrt", "sigmoid", "silu", + "swiglu", "softmax", "sqrt", "square", @@ -74,6 +75,7 @@ "rsqrt", "sigmoid", "silu", + "swiglu", "softmax", "sqrt", "square", diff --git a/paddle/fluid/primitive/base/primitive_ops.h b/paddle/fluid/primitive/base/primitive_ops.h index 947970ed92790..d477c32a62258 100644 --- a/paddle/fluid/primitive/base/primitive_ops.h +++ b/paddle/fluid/primitive/base/primitive_ops.h @@ -69,6 +69,7 @@ const std::set& GetPrimitiveOpNames() { "pd_op.cos", "pd_op.where", "pd_op.split", + "pd_op.split_with_num", "pd_op.reshape", "pd_op.erf", "pd_op.tanh", @@ -79,6 +80,7 @@ const std::set& GetPrimitiveOpNames() { "pd_op.shape", "pd_op.full", "pd_op.full_int_array", + "pd_op.full_with_tensor", "pd_op.if", "pd_op.while", /* basic ops by PIR*/ diff --git a/paddle/fluid/primitive/composite/composite.h b/paddle/fluid/primitive/composite/composite.h index 28983fa3cfd63..4fe8ec04a6031 100644 --- a/paddle/fluid/primitive/composite/composite.h +++ b/paddle/fluid/primitive/composite/composite.h @@ -335,6 +335,19 @@ Tensor silu_decomp(const Tensor& x) { } } +template +Tensor swiglu_decomp(const Tensor& x, const paddle::optional& y) { + auto y_ptr = y.get_ptr(); + if (y_ptr) { + return silu_decomp(x) * y.get(); + } else { + int axis = x.shape().size() - 1; + int num = 2; + std::vector xs = backend::split_with_num(x, num, axis); + return silu_decomp(xs[0]) * xs[1]; + } +} + template Tensor relu_decomp(const Tensor& x) { return maximum(x, full(empty_shape, 0.0, x.dtype())); diff --git a/python/paddle/jit/sot/utils/paddle_api_config.py b/python/paddle/jit/sot/utils/paddle_api_config.py index e21648f1a6ce6..8a5cde9e65716 100644 --- a/python/paddle/jit/sot/utils/paddle_api_config.py +++ b/python/paddle/jit/sot/utils/paddle_api_config.py @@ -34,6 +34,7 @@ def get_paddle_api(): modules = [ paddle, paddle.nn.functional, + paddle.incubate.nn.functional, paddle.linalg, paddle.signal, paddle.fft, diff --git a/test/legacy_test/test_batch_norm_op_prim_nchw.py b/test/legacy_test/test_batch_norm_op_prim_nchw.py index 3520e1e25aa77..06c42f221dfa2 100644 --- a/test/legacy_test/test_batch_norm_op_prim_nchw.py +++ b/test/legacy_test/test_batch_norm_op_prim_nchw.py @@ -67,7 +67,7 @@ def setUp(self): self.python_out_sig = ["Y"] # (Todo: CZ) random error self.check_prim_pir = False - self.check_prim_pir_grad = True + self.check_prim_pir_grad = False self.check_cpu_prim_pir_grad = False self.initConfig() @@ -284,6 +284,7 @@ def initConfig(self): self.use_global_stats = None self.check_prim_pir = True self.check_cpu_prim_pir_grad = True + self.check_prim_pir_grad = True class TestBatchNormOpNCHWTestModeFp64(TestBatchNormOp): @@ -357,7 +358,7 @@ def initConfig(self): self.use_global_stats = None # Todo(CZ): open this self.check_prim_pir = False - self.check_cpu_prim_pir_grad = True + self.check_cpu_prim_pir_grad = False @unittest.skipIf( diff --git a/test/legacy_test/test_batch_norm_op_prim_nhwc.py b/test/legacy_test/test_batch_norm_op_prim_nhwc.py index 01ad65e41b657..cefacd02b2a91 100644 --- a/test/legacy_test/test_batch_norm_op_prim_nhwc.py +++ b/test/legacy_test/test_batch_norm_op_prim_nhwc.py @@ -131,6 +131,7 @@ def initConfig(self): self.data_format = "NHWC" self.use_global_stats = None self.check_prim_pir = True + self.check_prim_pir_grad = True self.check_cpu_prim_pir_grad = True diff --git a/test/prim/pir_prim/test_prim_sub_graph_dynamic_shape.py b/test/prim/pir_prim/test_prim_sub_graph_dynamic_shape.py index 6be76dd54af38..155cfbdeeb268 100644 --- a/test/prim/pir_prim/test_prim_sub_graph_dynamic_shape.py +++ b/test/prim/pir_prim/test_prim_sub_graph_dynamic_shape.py @@ -84,7 +84,15 @@ def index_sample_net(x, index): return paddle.index_sample(x, index) -class TestPrimOne(unittest.TestCase): +def swiglu_net1(x, y): + return paddle.incubate.nn.functional.swiglu(x, y) + + +def swiglu_net2(x): + return paddle.incubate.nn.functional.swiglu(x) + + +class TestPrimBase(unittest.TestCase): def setUp(self): np.random.seed(2023) self.dtype = "float32" @@ -130,7 +138,7 @@ def test_prim_all_dynamic(self): np.testing.assert_allclose(ref, actual, rtol=1e-6) -class TestPrimOne2(TestPrimOne): +class TestPrimAny(TestPrimBase): def setUp(self): np.random.seed(2023) self.dtype = "bool" @@ -142,7 +150,7 @@ def setUp(self): self.enable_cinn = False -class TestEmbeddingPrimOne3(TestPrimOne): +class TestEmbedding(TestPrimBase): def setUp(self): np.random.seed(2023) self.dtype = "int" @@ -154,7 +162,7 @@ def setUp(self): self.enable_cinn = False -class TestPrimOne3(TestPrimOne): +class TestPrimFullLike(TestPrimBase): def setUp(self): np.random.seed(2023) self.dtype = "float32" @@ -166,7 +174,7 @@ def setUp(self): self.enable_cinn = False -class TestPrimOne4(TestPrimOne): +class TestPrimStack(TestPrimBase): def setUp(self): np.random.seed(2023) self.dtype = "float32" @@ -178,7 +186,7 @@ def setUp(self): self.enable_cinn = False -class TestPrimOne5(TestPrimOne): +class TestPrimTile(TestPrimBase): def setUp(self): np.random.seed(2023) self.dtype = "float32" @@ -190,7 +198,7 @@ def setUp(self): self.enable_cinn = False -class TestPrimOne6(TestPrimOne): +class TestPrimTile2(TestPrimBase): def setUp(self): np.random.seed(2023) self.dtype = "float32" @@ -269,5 +277,33 @@ def setUp(self): self.enable_cinn = False +class TestPrimSwiglu1(TestPrimTwo): + def setUp(self): + np.random.seed(2023) + self.shape_x = [300, 4096] + self.shape_y = [300, 4096] + self.dtype_x = "float32" + self.dtype_y = "float32" + self.init_x_shape = [None, None] + self.init_y_shape = [None, None] + self.x = np.random.random(self.shape_x).astype(self.dtype_x) + self.y = np.random.random(self.shape_y).astype(self.dtype_y) + self.net = swiglu_net1 + self.necessary_ops = "pd_op.swiglu" + self.enable_cinn = False + + +class TestPrimSwiglu2(TestPrimBase): + def setUp(self): + np.random.seed(2023) + self.shape_x = [300, 4096] + self.dtype_x = "float32" + self.init_x_shape = [None, 4096] + self.x = np.random.random(self.shape_x).astype(self.dtype_x) + self.net = swiglu_net2 + self.necessary_ops = "pd_op.swiglu" + self.enable_cinn = False + + if __name__ == "__main__": unittest.main() From 467c94bcbb607193e477fac10db53957ec4cdf0d Mon Sep 17 00:00:00 2001 From: cyberslack_lee Date: Mon, 26 Feb 2024 11:39:03 +0800 Subject: [PATCH 073/282] [clang-tidy] NO.38-40 enable `trivially-destructible`, `modernize-make-unique`,`modernize-avoid-bind` (#61556) * fix * fix * fix * fix * fix * fix --- .clang-tidy | 2 +- paddle/fluid/framework/new_executor/new_executor_defs.cc | 2 -- paddle/fluid/framework/new_executor/new_executor_defs.h | 2 +- paddle/fluid/imperative/amp_auto_cast.cc | 2 -- paddle/fluid/imperative/amp_auto_cast.h | 2 +- paddle/pir/src/core/parser/ir_parser.cc | 4 ++-- 6 files changed, 5 insertions(+), 9 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 2ed67098e2a02..1653cef5fa1aa 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -198,7 +198,7 @@ performance-move-const-arg, -performance-move-constructor-init, -performance-no-automatic-move, performance-noexcept-move-constructor, --performance-trivially-destructible, +performance-trivially-destructible, -performance-type-promotion-in-math-fn, -performance-unnecessary-copy-initialization, readability-container-size-empty, diff --git a/paddle/fluid/framework/new_executor/new_executor_defs.cc b/paddle/fluid/framework/new_executor/new_executor_defs.cc index bfa7542b65b75..b3ec52029bb5b 100644 --- a/paddle/fluid/framework/new_executor/new_executor_defs.cc +++ b/paddle/fluid/framework/new_executor/new_executor_defs.cc @@ -39,8 +39,6 @@ VariableScope::VariableScope(Scope* scope) { "You have passed a nullptr to construct VariableScope.")); } -VariableScope::~VariableScope() = default; - Scope* VariableScope::GetMutableScope() const { return scope_; } Scope* VariableScope::GetMutableLocalScope() const { return local_scope_; } diff --git a/paddle/fluid/framework/new_executor/new_executor_defs.h b/paddle/fluid/framework/new_executor/new_executor_defs.h index df82aedfcec5f..c416b151aef03 100644 --- a/paddle/fluid/framework/new_executor/new_executor_defs.h +++ b/paddle/fluid/framework/new_executor/new_executor_defs.h @@ -82,7 +82,7 @@ class VariableScope { void SetLocalScope(Scope* local_scope); - ~VariableScope(); + ~VariableScope() = default; // Get variable id by name, return -1 if not found int GetIdByName(const std::string& name) const; diff --git a/paddle/fluid/imperative/amp_auto_cast.cc b/paddle/fluid/imperative/amp_auto_cast.cc index 0dd5bc5779d21..50df994014004 100644 --- a/paddle/fluid/imperative/amp_auto_cast.cc +++ b/paddle/fluid/imperative/amp_auto_cast.cc @@ -162,8 +162,6 @@ AmpOperators::AmpOperators() << unsupported_bf16_ops_->size(); } -AmpOperators::~AmpOperators() = default; - AmpOperators& AmpOperators::Instance() { static AmpOperators instance; return instance; diff --git a/paddle/fluid/imperative/amp_auto_cast.h b/paddle/fluid/imperative/amp_auto_cast.h index 1864f990576b1..eda10499f90d6 100644 --- a/paddle/fluid/imperative/amp_auto_cast.h +++ b/paddle/fluid/imperative/amp_auto_cast.h @@ -45,7 +45,7 @@ class Tracer; // Singleton implementation with C++ 11 class AmpOperators { public: - ~AmpOperators(); + ~AmpOperators() = default; AmpOperators(const AmpOperators& o) = delete; const AmpOperators& operator=(const AmpOperators& o) = delete; diff --git a/paddle/pir/src/core/parser/ir_parser.cc b/paddle/pir/src/core/parser/ir_parser.cc index 5d52da81e8582..3f45573509305 100644 --- a/paddle/pir/src/core/parser/ir_parser.cc +++ b/paddle/pir/src/core/parser/ir_parser.cc @@ -18,9 +18,9 @@ namespace pir { IrParser::IrParser(IrContext* ctx, std::istream& is) { - lexer.reset(new Lexer{is}); + lexer = std::make_unique(is); this->ctx = ctx; - builder.reset(new Builder{ctx}); + builder = std::make_unique(ctx); } Token IrParser::ConsumeToken() { return lexer->ConsumeToken(); } From d7285b15ea6feb9cf350ec8838ce1867bb1f899c Mon Sep 17 00:00:00 2001 From: cyberslack_lee Date: Mon, 26 Feb 2024 11:40:10 +0800 Subject: [PATCH 074/282] [clang-tidy] NO.25 enable modernize-use-transparent-functors (#61689) * clangtidy 25 * codestyle * codestyle * fix * fix --- paddle/common/ddim.cc | 2 +- paddle/phi/infermeta/spmd_rules/reshape.cc | 6 +++--- paddle/pir/src/core/ir_printer.cc | 16 ++++++++-------- .../fluid/memory/thread_local_allocator_test.cc | 2 +- .../api/analyzer_capi_exp_pd_tensor_tester.cc | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/paddle/common/ddim.cc b/paddle/common/ddim.cc index 1f83a1c93b88d..58ccbe17d8df0 100644 --- a/paddle/common/ddim.cc +++ b/paddle/common/ddim.cc @@ -248,7 +248,7 @@ DDim DDim::reshape(std::vector& shape) const { if (it != shape.end()) { int index = static_cast(std::distance(shape.begin(), it)); int reshape_out_product = - std::accumulate(shape.begin(), shape.end(), -1, std::multiplies()); + std::accumulate(shape.begin(), shape.end(), -1, std::multiplies<>()); shape[index] = static_cast(product(in_dims)) / reshape_out_product; } diff --git a/paddle/phi/infermeta/spmd_rules/reshape.cc b/paddle/phi/infermeta/spmd_rules/reshape.cc index 41a263a1ae35b..3d3bfb38d22fa 100644 --- a/paddle/phi/infermeta/spmd_rules/reshape.cc +++ b/paddle/phi/infermeta/spmd_rules/reshape.cc @@ -44,8 +44,8 @@ std::vector InferTargetShape(const std::vector& shape, } } - int64_t product = std::accumulate( - shape.begin(), shape.end(), 1, std::multiplies()); + int64_t product = + std::accumulate(shape.begin(), shape.end(), 1, std::multiplies<>()); if (product > 0) { PADDLE_ENFORCE_EQ( product, @@ -72,7 +72,7 @@ std::vector> MakeReshapeDimTrans( const std::vector& tgt_shape) { std::vector> ret; int64_t total_elem_num_src = std::accumulate( - src_shape.begin(), src_shape.end(), 1, std::multiplies()); + src_shape.begin(), src_shape.end(), 1, std::multiplies<>()); std::vector inferred_tgt_shape = InferTargetShape(tgt_shape, total_elem_num_src); diff --git a/paddle/pir/src/core/ir_printer.cc b/paddle/pir/src/core/ir_printer.cc index b7cf6404818b5..c1a0fcd905ac8 100644 --- a/paddle/pir/src/core/ir_printer.cc +++ b/paddle/pir/src/core/ir_printer.cc @@ -69,7 +69,7 @@ void BasicIrPrinter::PrintType(Type type) { } else if (type.isa()) { os << "vec["; auto inner_types = type.dyn_cast().data(); - detail::PrintInterleave( + pir::detail::PrintInterleave( inner_types.begin(), inner_types.end(), [this](Type v) { this->PrintType(v); }, @@ -132,7 +132,7 @@ void BasicIrPrinter::PrintAttribute(Attribute attr) { } else if (auto arr = attr.dyn_cast()) { const auto& vec = arr.AsVector(); os << "["; - detail::PrintInterleave( + pir::detail::PrintInterleave( vec.begin(), vec.end(), [this](Attribute v) { this->PrintAttribute(v); }, @@ -256,7 +256,7 @@ void IrPrinter::PrintOpResult(Operation* op) { for (size_t idx = 0; idx < num_op_result; idx++) { op_results.push_back(op->result(idx)); } - detail::PrintInterleave( + pir::detail::PrintInterleave( op_results.begin(), op_results.end(), [this](Value v) { this->PrintValue(v); }, @@ -266,11 +266,11 @@ void IrPrinter::PrintOpResult(Operation* op) { void IrPrinter::PrintAttributeMap(Operation* op) { AttributeMap attributes = op->attributes(); - std::map> order_attributes( + std::map> order_attributes( attributes.begin(), attributes.end()); os << " {"; - detail::PrintInterleave( + pir::detail::PrintInterleave( order_attributes.begin(), order_attributes.end(), [this](std::pair it) { @@ -291,7 +291,7 @@ void IrPrinter::PrintOpOperands(Operation* op) { for (size_t idx = 0; idx < num_op_operands; idx++) { op_operands.push_back(op->operand_source(idx)); } - detail::PrintInterleave( + pir::detail::PrintInterleave( op_operands.begin(), op_operands.end(), [this](Value v) { this->PrintValue(v); }, @@ -312,7 +312,7 @@ void IrPrinter::PrintOperandsType(Operation* op) { } } os << " ("; - detail::PrintInterleave( + pir::detail::PrintInterleave( op_operand_types.begin(), op_operand_types.end(), [this](Type t) { this->PrintType(t); }, @@ -332,7 +332,7 @@ void IrPrinter::PrintOpReturnType(Operation* op) { op_result_types.emplace_back(nullptr); } } - detail::PrintInterleave( + pir::detail::PrintInterleave( op_result_types.begin(), op_result_types.end(), [this](Type t) { this->PrintType(t); }, diff --git a/test/cpp/fluid/memory/thread_local_allocator_test.cc b/test/cpp/fluid/memory/thread_local_allocator_test.cc index c322295892da3..c6cb4cf0acf20 100644 --- a/test/cpp/fluid/memory/thread_local_allocator_test.cc +++ b/test/cpp/fluid/memory/thread_local_allocator_test.cc @@ -78,7 +78,7 @@ TEST(ThreadLocalAllocator, cross_scope_release) { for (auto &addresses : allocator_addresses) { std::sort(addresses.begin(), addresses.end()); ASSERT_EQ(std::adjacent_find( - addresses.begin(), addresses.end(), std::equal_to()), + addresses.begin(), addresses.end(), std::equal_to<>()), addresses.end()); } diff --git a/test/cpp/inference/api/analyzer_capi_exp_pd_tensor_tester.cc b/test/cpp/inference/api/analyzer_capi_exp_pd_tensor_tester.cc index 0b69c235a03fc..7a32aefb16d30 100644 --- a/test/cpp/inference/api/analyzer_capi_exp_pd_tensor_tester.cc +++ b/test/cpp/inference/api/analyzer_capi_exp_pd_tensor_tester.cc @@ -69,7 +69,7 @@ void PD_run() { int32_t out_num = std::accumulate(output_shape->data, output_shape->data + output_shape->size, 1, - std::multiplies()); + std::multiplies<>()); out_data.resize(out_num); PD_TensorCopyToCpuFloat(output_tensor, out_data.data()); LOG(INFO) << "Output tensor name is: " << PD_TensorGetName(output_tensor); From 2823a59f63af9190d7df93d36c394a0b886c333e Mon Sep 17 00:00:00 2001 From: Sonder <55493212+AndSonder@users.noreply.github.com> Date: Mon, 26 Feb 2024 12:30:30 +0800 Subject: [PATCH 075/282] [AutoParallel] Fit allreduce_matmul_grad_overlapping when using master grad (#61865) * remove sync_with_cpp * fix allreduce matmul grad overlaping when open master_grad * add annotation * update universal codes --- .../allreduce_matmul_grad_overlapping.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/python/paddle/distributed/passes/allreduce_matmul_grad_overlapping.py b/python/paddle/distributed/passes/allreduce_matmul_grad_overlapping.py index 48e837fa3c46f..89e6c20ad03c9 100644 --- a/python/paddle/distributed/passes/allreduce_matmul_grad_overlapping.py +++ b/python/paddle/distributed/passes/allreduce_matmul_grad_overlapping.py @@ -133,6 +133,53 @@ def _split_matmul_grad_and_multi_streaming_allreduce( matmul_grad_op = ops[matmul_grad_id] allreduce_op = ops[allreduce_id] + # NOTE(Sonder): Why move those operations to the back of matmul_v2? + # When using amp_master_grad, the cast operation is inserted after matmul_grad. + # However, when employing allreduce_matmul_grad_overlapping, the matmul_grad is + # split into two matmul operations. In this case, some operations would access + # uninitialized tensors. Therefore, we move the cast operation to the back of the + # second matmul operation to avoid this problem. + skip_overlapping = False + moved_ops_idx = [] + moved_ops_output = [] + matmul_grad_output = matmul_grad_op.output('Y@GRAD')[0] + + for idx in range(matmul_grad_id + 1, allreduce_id): + if matmul_grad_output in ops[idx].desc.input_arg_names(): + moved_ops_idx.append(idx) + moved_ops_output.extend(ops[idx].desc.output_arg_names()) + else: + for input_name in ops[idx].desc.input_arg_names(): + if input_name in moved_ops_output: + skip_overlapping = True + + if skip_overlapping: + continue + + for i, idx in enumerate(moved_ops_idx): + op = ops[idx] + dist_attr = self.dist_context.get_op_dist_attr_for_program(op) + + op_inputs = op.desc.input_names() + op_outputs = op.desc.output_names() + + op_inputs = {name: op.input(name) for name in op_inputs} + op_outputs = {name: op.output(name) for name in op_outputs} + + op = block._insert_op_without_sync( + index=allreduce_id + 1 + i, + type=op.type, + inputs=op_inputs, + outputs=op_outputs, + attrs=op.all_attrs(), + ) + + self.dist_context.set_op_dist_attr_for_program(op, dist_attr) + + for i, idx in enumerate(moved_ops_idx): + block._remove_op(idx - i, sync=False) + allreduce_id -= 1 + tran_x = matmul_grad_op.attr("trans_x") assert ( not tran_x From 488f2d536f0f794fdbb787785af3e14f95d767c5 Mon Sep 17 00:00:00 2001 From: ShenLiang <1422485404@qq.com> Date: Mon, 26 Feb 2024 13:15:06 +0800 Subject: [PATCH 076/282] set default in p2p_overlap (#62051) --- paddle/fluid/framework/distributed_strategy.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/framework/distributed_strategy.proto b/paddle/fluid/framework/distributed_strategy.proto index 58460fcf9064b..27c7a7a7af276 100755 --- a/paddle/fluid/framework/distributed_strategy.proto +++ b/paddle/fluid/framework/distributed_strategy.proto @@ -82,7 +82,7 @@ message PpConfig { optional bool sharding_comm_overlap = 4 [ default = false ]; optional bool profiling = 5 [ default = false ]; optional bool release_gradients = 6 [ default = false ]; - optional bool overlap_p2p_comm = 7 [default = false]; + optional bool overlap_p2p_comm = 7 [default = true]; } message DygraphShardingConfig { From 082f95490d5c347a69d2566a62126771755245ea Mon Sep 17 00:00:00 2001 From: Ghost Screaming Date: Mon, 26 Feb 2024 14:20:32 +0800 Subject: [PATCH 077/282] [AutoParallel] Fix problems of pp. (#61840) * [AutoParallel] Fix inplace full_ in pp. * [AutoParallel] Fix problem of PHI::DatatYPE::UNDEFINED. * Polish code. * Fix problem of split_with_num FillZeroForEmptyGradInput. --- .../auto_code_generator/generator/eager_gen.py | 11 +++++------ paddle/fluid/eager/grad_node_info.cc | 2 +- paddle/phi/api/yaml/generator/dist_api_gen.py | 7 ++++++- paddle/phi/infermeta/multiary.cc | 14 ++++++++++---- paddle/phi/infermeta/ternary.cc | 5 +++++ 5 files changed, 27 insertions(+), 12 deletions(-) diff --git a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py index 62327c5aa8785..13ddbca4c9ef5 100644 --- a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py +++ b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py @@ -554,8 +554,10 @@ class {} : public egr::GradNodeBase {{ }} """ CREATE_PLAIN_OPTIONAL_TENSOR_TEMPLATE = """ - paddle::optional {}_optional; - if({}.initialized()) {}_optional = paddle::make_optional({}); + paddle::optional {name}_optional; + if({name}.initialized() || + ({name}.defined() && {name}.is_dist_tensor() && + phi::distributed::NeedComputationClipForPP({name}.impl()))) {name}_optional = paddle::make_optional({name}); """ CREATE_RECOVER_OPTIONAL_TENSOR_TEMPLATE = """ @@ -2434,10 +2436,7 @@ def GenerateNodeDefinition( get_tensor_str += ( "\n" + CREATE_PLAIN_OPTIONAL_TENSOR_TEMPLATE.format( - transformed_tensor_name, - transformed_tensor_name, - transformed_tensor_name, - transformed_tensor_name, + name=transformed_tensor_name ) ) grad_api_args[ diff --git a/paddle/fluid/eager/grad_node_info.cc b/paddle/fluid/eager/grad_node_info.cc index 78e3dd32fd40e..2a97f5bf35e90 100644 --- a/paddle/fluid/eager/grad_node_info.cc +++ b/paddle/fluid/eager/grad_node_info.cc @@ -196,7 +196,7 @@ void GradNodeBase::SetGradInMeta(const std::vector& fwd_out, if (!fwd_out_tensor.initialized()) { if (fwd_out_tensor.defined() && fwd_out_tensor.is_dist_tensor() && - !phi::distributed::NeedComputationClipForPP(fwd_out_tensor.impl())) { + phi::distributed::NeedComputationClipForPP(fwd_out_tensor.impl())) { VLOG(3) << "Tensor " << fwd_out_tensor.name() << " is DistTensor," << " and needs computation clip for pipeline parallel." << " Still SetGradInMeta for it."; diff --git a/paddle/phi/api/yaml/generator/dist_api_gen.py b/paddle/phi/api/yaml/generator/dist_api_gen.py index 03d65a920b9d2..e199c5c1a520d 100644 --- a/paddle/phi/api/yaml/generator/dist_api_gen.py +++ b/paddle/phi/api/yaml/generator/dist_api_gen.py @@ -596,6 +596,7 @@ def parse_infer_meta(self, infer_meta_config): def need_to_generate_code_for_inplace_impl(self, i): return ( self.inplace_flag + and self.kernel['func'][0] != 'full' and self.inplace_map is not None and self.outputs['names'][i] in self.inplace_map ) @@ -1023,7 +1024,11 @@ def generate_output_creation_code(self) -> str: output_creation_code += "\n phi::DeviceContext* dev_ctx = nullptr;" if output_num == 1: # api output generate - if self.need_to_generate_code_for_inplace_impl(0): + if ( + self.inplace_flag + and self.inplace_map is not None + and self.outputs['names'][0] in self.inplace_map + ): inplace_assign_code = ( " = " + self.inplace_map[self.outputs['names'][0]] ) diff --git a/paddle/phi/infermeta/multiary.cc b/paddle/phi/infermeta/multiary.cc index 978a80674272f..b7a5dd51de901 100644 --- a/paddle/phi/infermeta/multiary.cc +++ b/paddle/phi/infermeta/multiary.cc @@ -4512,17 +4512,23 @@ void FusedRopeInferMeta(const MetaTensor& q, "Input should be a 4-D tensor of format [N, C, H, W] " "or [N, H, W, C], but got %u.", input_dims.size())); - if (q) { - out_q->set_dims(q.dims()); - out_q->set_dtype(q.dtype()); - } + out_q->set_dims(q.dims()); + out_q->set_dtype(q.dtype()); if (k) { out_k->set_dims(k.dims()); out_k->set_dtype(k.dtype()); + } else { + if (out_k) { + out_k->set_dtype(q.dtype()); + } } if (v) { out_v->set_dims(v.dims()); out_v->set_dtype(v.dtype()); + } else { + if (out_v) { + out_v->set_dtype(q.dtype()); + } } } diff --git a/paddle/phi/infermeta/ternary.cc b/paddle/phi/infermeta/ternary.cc index 5701ffd4da5d2..b728c33abf2e2 100644 --- a/paddle/phi/infermeta/ternary.cc +++ b/paddle/phi/infermeta/ternary.cc @@ -293,6 +293,11 @@ void FlashAttnInferMeta(const MetaTensor& q, out->set_dims(out_dims); out->set_dtype(q.dtype()); out->set_layout(q.layout()); + softmax->set_dtype(q.dtype()); + softmax_lse->set_dtype(q.dtype()); + if (seed_offset) { + seed_offset->set_dtype(phi::DataType::INT64); + } } void ArangeTensorInferMeta(const MetaTensor& start, From 194ef8baa4a6e94d9ef70e05554e5d3e2ac909f9 Mon Sep 17 00:00:00 2001 From: RichardWooSJTU <37864677+RichardWooSJTU@users.noreply.github.com> Date: Mon, 26 Feb 2024 14:28:35 +0800 Subject: [PATCH 078/282] Fix llm.int8 unit test (#61591) * fix llm.int8 unit test * fix llm.int8 unnittest when cpu * fix numerical mismatch * code clean --- .../phi/kernels/cpu/weight_quantize_kernel.cc | 17 ++- .../phi/kernels/gpu/weight_quantize_kernel.cu | 5 +- .../impl/weight_quantize_kernel_gpu_impl.h | 12 +- .../impl/weight_quantize_kernel_impl.h | 20 +-- test/quantization/test_llm_int8_linear.py | 129 +++++++++--------- 5 files changed, 100 insertions(+), 83 deletions(-) diff --git a/paddle/phi/kernels/cpu/weight_quantize_kernel.cc b/paddle/phi/kernels/cpu/weight_quantize_kernel.cc index 313c59e2e6676..61304e43d4e85 100644 --- a/paddle/phi/kernels/cpu/weight_quantize_kernel.cc +++ b/paddle/phi/kernels/cpu/weight_quantize_kernel.cc @@ -22,7 +22,11 @@ limitations under the License. */ namespace phi { -template +template void quant_compute(const DeviceContext& dev_ctx, const DenseTensor& x, DenseTensor* out, @@ -48,7 +52,7 @@ void quant_compute(const DeviceContext& dev_ctx, DDim dims = {num}; const T* x_data = x.data(); D* out_data = out->data(); - T* scale_data = scale->data(); + ScaleT* scale_data = scale->data(); DenseTensor x_int(out->type()); @@ -121,11 +125,16 @@ void WeightQuantizeKernel(const Context& dev_ctx, DenseTensor* out, DenseTensor* scale) { dev_ctx.template Alloc(out); - dev_ctx.template Alloc(scale); - if (algo == "weight_only_int8" || algo == "llm.int8") { + if (algo == "weight_only_int8") { + dev_ctx.template Alloc(scale); quant_compute( dev_ctx, x, out, scale, algo, arch, group_size); + } else if (algo == "llm.int8") { + dev_ctx.template Alloc(scale); + quant_compute( + dev_ctx, x, out, scale, algo, arch, group_size); } else if (algo == "weight_only_int4") { + dev_ctx.template Alloc(scale); quant_compute( dev_ctx, x, out, scale, algo, arch, group_size); } else { diff --git a/paddle/phi/kernels/gpu/weight_quantize_kernel.cu b/paddle/phi/kernels/gpu/weight_quantize_kernel.cu index 8cd5598e2e92a..103691f9cd8a4 100644 --- a/paddle/phi/kernels/gpu/weight_quantize_kernel.cu +++ b/paddle/phi/kernels/gpu/weight_quantize_kernel.cu @@ -37,7 +37,6 @@ void WeightQuantizeKernel(const Context& dev_ctx, DenseTensor quanted_x; dev_ctx.template Alloc(out); - dev_ctx.template Alloc(scale); size_t m = x.dims()[0]; size_t n = x.dims()[1]; quanted_x.Resize({static_cast(m), static_cast(n)}); @@ -51,15 +50,17 @@ void WeightQuantizeKernel(const Context& dev_ctx, "Currently, arch only support 70, 75, 80, 86.")); if (algo == "llm.int8") { + dev_ctx.template Alloc(scale); std::vector axis = {1, 0}; funcs::Transpose trans; weight_quant_gpu(dev_ctx, x.data(), quanted_x.data(), - scale->data(), + scale->data(), weight_shape); trans(dev_ctx, quanted_x, out, axis); } else if (algo == "weight_only_int8") { + dev_ctx.template Alloc(scale); weight_quant_gpu(dev_ctx, x.data(), quanted_x.data(), diff --git a/paddle/phi/kernels/impl/weight_quantize_kernel_gpu_impl.h b/paddle/phi/kernels/impl/weight_quantize_kernel_gpu_impl.h index 201dd403270f3..05d0e47b31455 100644 --- a/paddle/phi/kernels/impl/weight_quantize_kernel_gpu_impl.h +++ b/paddle/phi/kernels/impl/weight_quantize_kernel_gpu_impl.h @@ -106,10 +106,10 @@ void weight_permute_gpu(const GPUContext& dev_ctx, } } -template +template __global__ void per_channel_quant_gpu(const T* weight_data, int8_t* quanted_weight_data, - T* scale_data, + ScaleT* scale_data, int total_k, int total_vec_n) { int n = blockIdx.x * blockDim.x + threadIdx.x; @@ -133,10 +133,10 @@ __global__ void per_channel_quant_gpu(const T* weight_data, abs_max[i] = fmaxf((abs_max[i]), fabsf((weight[i]))); } } - phi::AlignedVector scale; + phi::AlignedVector scale; #pragma unroll for (int i = 0; i < VectorSize; ++i) { - scale[i] = static_cast(abs_max[i] / static_cast(127.0f)); + scale[i] = static_cast(abs_max[i] / static_cast(127.0f)); } *reinterpret_cast(scale_data + VectorSize * n) = *reinterpret_cast(&scale); @@ -161,11 +161,11 @@ __global__ void per_channel_quant_gpu(const T* weight_data, } } } -template +template void weight_quant_gpu(const GPUContext& dev_ctx, const T* weight_data, int8_t* quanted_weight_data, - T* scale_data, + ScaleT* scale_data, const std::vector& shape) { int total_k = shape[0]; int total_n = shape[1]; diff --git a/paddle/phi/kernels/impl/weight_quantize_kernel_impl.h b/paddle/phi/kernels/impl/weight_quantize_kernel_impl.h index 2905fd14e6b33..6f7fc1e9c0680 100644 --- a/paddle/phi/kernels/impl/weight_quantize_kernel_impl.h +++ b/paddle/phi/kernels/impl/weight_quantize_kernel_impl.h @@ -42,9 +42,9 @@ inline T xabs(const T x) { return x < static_cast(0.0) ? -x : x; } -template +template void per_channel_scale( - T* scale, const T* input, size_t m, size_t n, float bound) { + ScaleT* scale, const T* input, size_t m, size_t n, float bound) { for (size_t i = 0; i < n; ++i) { float max = static_cast(input[i]); for (size_t j = 0; j < m; ++j) { @@ -52,12 +52,12 @@ void per_channel_scale( ? static_cast(xabs(input[j * n + i])) : max; } - scale[i] = static_cast(max / bound); + scale[i] = static_cast(max / bound); } } -template -void group_wise_scale(T* scale, +template +void group_wise_scale(ScaleT* scale, const T* input, size_t m, size_t n, @@ -72,15 +72,15 @@ void group_wise_scale(T* scale, : max; } scale[static_cast(j / group_size) * n + i] = - static_cast(max / bound); + static_cast(max / bound); } } } -template +template void per_channel_quant(int8_t* output, const T* input, - const T* scale, + const ScaleT* scale, size_t num_rows, size_t num_cols) { size_t bytes_per_out_col = num_cols * quant_bit / 8; @@ -123,10 +123,10 @@ void per_channel_quant(int8_t* output, } } -template +template void group_wise_quant(int8_t* output, const T* input, - const T* scale, + const ScaleT* scale, size_t num_rows, size_t num_cols, const int group_size) { diff --git a/test/quantization/test_llm_int8_linear.py b/test/quantization/test_llm_int8_linear.py index 972c41bd31f52..909f44c0ca404 100644 --- a/test/quantization/test_llm_int8_linear.py +++ b/test/quantization/test_llm_int8_linear.py @@ -24,9 +24,6 @@ from paddle.framework import set_default_dtype from paddle.pir_utils import test_with_pir_api -np.random.seed(123) -paddle.seed(42) - @unittest.skipIf( not core.is_compiled_with_cuda() @@ -43,11 +40,13 @@ def config(self): self.batch = 1 self.token = 32 self.in_features = 64 - self.out_features = 256 + self.out_features = 128 self.threshold = 6.0 self.static = False def setUp(self): + np.random.seed(123) + paddle.seed(42) self.config() x = np.random.random((self.batch, self.token, self.in_features)) self.x = paddle.to_tensor(x, dtype=self.dtype) @@ -64,49 +63,89 @@ def setUp(self): self.in_features, self.out_features, bias_attr=bias_attr ) - self.bias = self.linear.bias self.weight = self.linear.weight self.weight_scale = None self.weight, self.weight_scale = Q.weight_quantize( self.weight, algo="llm.int8" ) + def dynamic_quant(self, x): + row_ranges = paddle.max(x, axis=[-1]).astype('float32') + row_ranges = row_ranges.unsqueeze(-1) + quant_x = paddle.round( + paddle.clip( + x.astype('float32') * 127.0 * (1 / row_ranges), + min=-127.0, + max=127.0, + ) + ).astype('int8') + return quant_x, row_ranges + def get_linear_out(self): - out = self.linear(self.x) + outlier_cols = ( + paddle.nonzero(paddle.max(self.x, axis=[0, 1]) > self.threshold) + .reshape([-1]) + .numpy() + .tolist() + ) + + x_int8 = self.x + if len(outlier_cols) > 0: + x_fp = self.x[:, :, outlier_cols] + w_fp = self.linear.weight[outlier_cols] + res_fp = paddle.matmul(x_fp, w_fp) + + x_int8[:, :, outlier_cols] = 0 + x_int8, row_ranges = self.dynamic_quant(x_int8) + + res_int8 = paddle.matmul(x_int8, self.weight.transpose((1, 0))) + dequant_scale = row_ranges * self.weight_scale / 127.0 + res_dequant = (res_int8.astype('float32') * dequant_scale).astype( + self.dtype + ) + + if len(outlier_cols) > 0: + out = res_dequant + res_fp + else: + out = res_dequant + + if self.bias: + out += self.bias + return out.numpy() def get_llm_int8_linear_out(self): out = Q.llm_int8_linear( self.x, self.weight, - bias=self.bias, + bias=self.linear.bias, weight_scale=self.weight_scale, threshold=self.threshold, ) return out.numpy() @test_with_pir_api - def get_llm_int8_linear_out_static(self): + def llm_int8_linear_out_static(self, out_expect): paddle.enable_static() - main = base.static.Program() - start = base.static.Program() - with base.static.program_guard(main, start): - x = paddle.static.data("x", self.x.shape, dtype=self.x.dtype) + main = paddle.static.Program() + start = paddle.static.Program() + with paddle.static.program_guard(main, start): + x = paddle.static.data("x", self.x.shape, dtype=self.dtype) weight = paddle.static.data( - "weight", self.weight.shape, dtype=self.weight.dtype + "weight", self.weight.shape, dtype='int8' ) bias = paddle.static.data( - "bias", self.bias.shape, dtype=self.bias.dtype + "bias", self.linear.bias.shape, dtype=self.dtype ) x_np = self.x.numpy() weight_np = self.weight.numpy() - bias_np = self.bias.numpy() + bias_np = self.linear.bias.numpy() if self.weight_scale is not None: weight_scale = paddle.static.data( "weight_scale", self.weight_scale.shape, - dtype=self.weight_scale.dtype, + dtype='float32', ) weight_scale_np = self.weight_scale.numpy() else: @@ -128,20 +167,30 @@ def get_llm_int8_linear_out_static(self): } exe = base.Executor(paddle.CUDAPlace(0)) exe.run(start) - (out,) = exe.run(main, feed=feed_dict, fetch_list=[out]) + (out_real,) = exe.run(main, feed=feed_dict, fetch_list=[out]) + paddle.disable_static() - return out + + if self.dtype == "bfloat16": + out_real = convert_uint16_to_float(out_real) + out_expect = convert_uint16_to_float(out_expect) + + np.testing.assert_allclose( + out_real, out_expect, rtol=self.rtol, atol=self.atol + ) def test_llm_int8_linear(self): out_expect = self.get_linear_out() if self.static: - out_real = self.get_llm_int8_linear_out_static() + self.llm_int8_linear_out_static(out_expect) + return else: out_real = self.get_llm_int8_linear_out() if self.dtype == "bfloat16": out_real = convert_uint16_to_float(out_real) out_expect = convert_uint16_to_float(out_expect) + np.testing.assert_allclose( out_real, out_expect, rtol=self.rtol, atol=self.atol ) @@ -174,19 +223,6 @@ def config(self): self.weight_dtype = "int8" -@unittest.skipIf( - not core.is_compiled_with_cuda() - or get_cuda_version() < 11020 - or paddle.device.cuda.get_device_capability()[0] < 8, - "quantized_matmul requires CUDA >= 11.2 and CUDA_ARCH >= 8", -) -class LLMInt8LinearTestCase3(LLMInt8LinearTestCase): - def config(self): - super().config() - self.dtype = 'bfloat16' - self.weight_dtype = "int8" - - @unittest.skipIf( not core.is_compiled_with_cuda() or get_cuda_version() < 11020 @@ -215,20 +251,6 @@ def config(self): self.weight_dtype = "int4" -@unittest.skipIf( - not core.is_compiled_with_cuda() - or get_cuda_version() < 11020 - or paddle.device.cuda.get_device_capability()[0] < 8 - or not core.is_bfloat16_supported(core.CUDAPlace(0)), - "quantized_matmul requires CUDA >= 11.2 and CUDA_ARCH >= 8 or core is not support bfloat16", -) -class LLMInt8LinearTestCase6(LLMInt8LinearTestCase): - def config(self): - super().config() - self.dtype = 'bfloat16' - self.weight_dtype = "int4" - - @unittest.skipIf( not core.is_compiled_with_cuda() or get_cuda_version() < 11020 @@ -260,21 +282,6 @@ def config(self): self.token = 1 -@unittest.skipIf( - not core.is_compiled_with_cuda() - or get_cuda_version() < 11020 - or paddle.device.cuda.get_device_capability()[0] < 8, - "quantized_matmul requires CUDA >= 11.2 and CUDA_ARCH >= 8", -) -class LLMInt8LinearTestCase9(LLMInt8LinearTestCase): - def config(self): - super().config() - self.dtype = 'bfloat16' - self.weight_dtype = "int8" - self.batch = 1 - self.token = 1 - - @unittest.skipIf( not core.is_compiled_with_cuda() or get_cuda_version() < 11020 From 9247adeb0fb62cb91d27ee0acb5bd9c30ce854ce Mon Sep 17 00:00:00 2001 From: YuanRisheng Date: Mon, 26 Feb 2024 14:30:40 +0800 Subject: [PATCH 079/282] fix eb4 (#62032) --- .../distributed/fleet/layers/mpu/mp_layers.py | 8 ++---- .../fleet/utils/sequence_parallel_utils.py | 3 +- .../nn/functional/fused_matmul_bias.py | 28 ++++++++++++++++--- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/python/paddle/distributed/fleet/layers/mpu/mp_layers.py b/python/paddle/distributed/fleet/layers/mpu/mp_layers.py index a24bbd3321439..fd66927ced6db 100644 --- a/python/paddle/distributed/fleet/layers/mpu/mp_layers.py +++ b/python/paddle/distributed/fleet/layers/mpu/mp_layers.py @@ -16,6 +16,7 @@ import paddle from paddle.autograd import PyLayer +from paddle.base import core from paddle.distributed import fleet from paddle.nn import functional as F @@ -33,7 +34,7 @@ def is_fused_matmul_bias_supported(): - return hasattr(paddle._C_ops, 'fused_gemm_epilogue') + return hasattr(core.eager.ops.legacy, 'fused_gemm_epilogue') def is_fused_linear_param_grad_add_supported(): @@ -213,10 +214,7 @@ def forward( if not fuse_matmul_bias: return paddle._C_ops.linear(x, weight, bias) else: - result, _ = paddle._C_ops.fused_gemm_epilogue( - x, weight, bias, False, False, "none" - ) - return result + return paddle._legacy_C_ops.fused_gemm_epilogue(x, weight, bias) @staticmethod def backward(ctx, dy): diff --git a/python/paddle/distributed/fleet/utils/sequence_parallel_utils.py b/python/paddle/distributed/fleet/utils/sequence_parallel_utils.py index f499054bc8496..940d7408ff5be 100644 --- a/python/paddle/distributed/fleet/utils/sequence_parallel_utils.py +++ b/python/paddle/distributed/fleet/utils/sequence_parallel_utils.py @@ -17,6 +17,7 @@ import paddle from paddle import distributed as dist from paddle.autograd import PyLayer +from paddle.base import core from paddle.distributed import fleet from paddle.distributed.fleet.meta_parallel import get_rng_state_tracker from paddle.distributed.fleet.utils.hybrid_parallel_util import ( @@ -221,7 +222,7 @@ def is_fused_matmul_bias_supported(): and not paddle.is_compiled_with_rocm() or paddle.is_compiled_with_xpu() ): - return hasattr(paddle._C_ops, "fused_gemm_epilogue") + return hasattr(core.eager.ops.legacy, "fused_gemm_epilogue") else: return False diff --git a/python/paddle/incubate/nn/functional/fused_matmul_bias.py b/python/paddle/incubate/nn/functional/fused_matmul_bias.py index 1b894ce297a1c..56d5e30a506ab 100644 --- a/python/paddle/incubate/nn/functional/fused_matmul_bias.py +++ b/python/paddle/incubate/nn/functional/fused_matmul_bias.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from paddle import _C_ops +from paddle import _C_ops, _legacy_C_ops from paddle.base.layer_helper import LayerHelper -from paddle.framework import in_dynamic_or_pir_mode +from paddle.framework import ( + in_dynamic_mode, + in_pir_mode, +) from paddle.tensor.linalg import matmul @@ -56,7 +59,11 @@ def fused_matmul_bias( """ if bias is None: return matmul(x, y, transpose_x, transpose_y, name) - if in_dynamic_or_pir_mode(): + if in_dynamic_mode(): + return _legacy_C_ops.fused_gemm_epilogue( + x, y, bias, 'trans_x', transpose_x, 'trans_y', transpose_y + ) + if in_pir_mode(): out, _ = _C_ops.fused_gemm_epilogue( x, y, bias, transpose_x, transpose_y, "none" ) @@ -146,7 +153,20 @@ def fused_linear_activation( if activation is None: activation = "none" - if in_dynamic_or_pir_mode(): + if in_dynamic_mode(): + return _legacy_C_ops.fused_gemm_epilogue( + x, + y, + bias, + 'trans_x', + trans_x, + 'trans_y', + trans_y, + 'activation', + activation, + ) + + if in_pir_mode(): out, _ = _C_ops.fused_gemm_epilogue( x, y, From 1c5bbe428b7cf62a8f74e8f50386d515a9e10838 Mon Sep 17 00:00:00 2001 From: diadestiny <44188454+diadestiny@users.noreply.github.com> Date: Mon, 26 Feb 2024 14:46:03 +0800 Subject: [PATCH 080/282] [SOT][3.12] Support `BINARY_SLICE` and `STORE_SLICE` opcode in Python 3.12 (#62028) --- .../executor/opcode_executor.py | 36 +++++++++++++++++-- test/sot/skip_files_py312 | 1 - 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py index e0ada6a9b74fa..3b40633a73e25 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py +++ b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py @@ -696,6 +696,21 @@ def BINARY_OP(self, instr: Instruction): def BINARY_SUBSCR(self, instr: Instruction): key = self.stack.pop() container = self.stack.pop() + self.binary_subscr_operation(key, container, instr.opname) + + @call_break_graph_decorator(push_n=1) + def BINARY_SLICE(self, instr: Instruction): + end = self.stack.pop() + start = self.stack.pop() + container = self.stack.pop() + key = SliceVariable( + slice(start, end), + graph=self._graph, + tracker=DummyTracker([start, end]), + ) + self.binary_subscr_operation(key, container, instr.opname) + + def binary_subscr_operation(self, key, container, opname): assert isinstance(key, VariableBase) # TODO(xiongkun): getitem / getattr support key and attr as variable. if isinstance(key, TensorVariable) and isinstance( @@ -710,7 +725,7 @@ def BINARY_SUBSCR(self, instr: Instruction): if isinstance(key, TensorVariable): raise BreakGraphError( - f"Key is a TensorVariable in BINARY_SUBSCR, {container}[{key}]" + f"Key is a TensorVariable in {opname}, {container}[{key}]" ) result = BuiltinVariable( @@ -884,11 +899,28 @@ def STORE_SUBSCR(self, instr: Instruction): key = self.stack.pop() container = self.stack.pop() value = self.stack.pop() + self.store_subscr_operation(key, container, value, instr.opname) + + @call_break_graph_decorator(push_n=0) + def STORE_SLICE(self, instr: Instruction): + end = self.stack.pop() + start = self.stack.pop() + container = self.stack.pop() + value = self.stack.pop() + + key = SliceVariable( + slice(start, end), + graph=self._graph, + tracker=DummyTracker([start, end]), + ) + self.store_subscr_operation(key, container, value, instr.opname) + + def store_subscr_operation(self, key, container, value, opname): assert isinstance(key, VariableBase) self._graph.add_global_guarded_variable(key) if isinstance(key, TensorVariable): raise BreakGraphError( - f"Key is a TensorVariable in STORE_SUBSCR, {container}[{key}] = {value}" + f"Key is a TensorVariable in {opname}, {container}[{key}] = {value}" ) # TODO(xiongkun): support tensor[tensor] = tensor, dy2static is not the same with dygraph. container[key.get_py_value()] = value diff --git a/test/sot/skip_files_py312 b/test/sot/skip_files_py312 index 815f3a9e68b49..59cd1a37055f4 100644 --- a/test/sot/skip_files_py312 +++ b/test/sot/skip_files_py312 @@ -2,7 +2,6 @@ ./test_11_jumps.py ./test_12_for_loop.py ./test_14_operators.py -./test_15_slice.py ./test_21_global.py ./test_analysis_inputs.py ./test_break_graph.py From 2598a16e30a56b2c430b01a32ca1bb40c46bd488 Mon Sep 17 00:00:00 2001 From: ming1753 <61511741+ming1753@users.noreply.github.com> Date: Mon, 26 Feb 2024 14:49:59 +0800 Subject: [PATCH 081/282] Inference with optimized model (#61598) * Inference with optimized model * clear gpu mem * delete scale and zero_point in qdq ops * add test * modify API * fix bug --- .../ir/delete_quant_dequant_linear_op_pass.cc | 14 +++ ...rt_delete_weight_dequant_linear_op_pass.cc | 14 +++ paddle/fluid/inference/analysis/argument.h | 3 + .../passes/save_optimized_model_pass.cc | 29 ++--- paddle/fluid/inference/api/analysis_config.cc | 6 + .../fluid/inference/api/analysis_predictor.cc | 80 +++++++++++- .../fluid/inference/api/analysis_predictor.h | 2 + .../inference/api/paddle_analysis_config.h | 10 +- .../fluid/inference/api/paddle_pass_builder.h | 15 +-- paddle/fluid/pybind/inference_api.cc | 3 + .../inference/test_use_optimized_model_api.py | 116 ++++++++++++++++++ 11 files changed, 265 insertions(+), 27 deletions(-) create mode 100644 test/ir/inference/test_use_optimized_model_api.py diff --git a/paddle/fluid/framework/ir/delete_quant_dequant_linear_op_pass.cc b/paddle/fluid/framework/ir/delete_quant_dequant_linear_op_pass.cc index 317beb100acb1..7358a82c6ca3c 100644 --- a/paddle/fluid/framework/ir/delete_quant_dequant_linear_op_pass.cc +++ b/paddle/fluid/framework/ir/delete_quant_dequant_linear_op_pass.cc @@ -126,6 +126,13 @@ void DeleteQuantDequantLinearOpPass::ApplyImpl(ir::Graph* graph) const { */ std::unordered_set nodes2rm = {}; + // delete Scale and ZeroPoint tensor in scope + std::vector vars2rm = {}; + vars2rm.emplace_back(quantize_linear_op->Op()->Input("Scale")[0]); + vars2rm.emplace_back(quantize_linear_op->Op()->Input("ZeroPoint")[0]); + vars2rm.emplace_back(dequantize_linear_op->Op()->Input("Scale")[0]); + vars2rm.emplace_back(dequantize_linear_op->Op()->Input("ZeroPoint")[0]); + // Get input scale from tensor const phi::DenseTensor& input_scale_tensor = scope->GetVar(quantize_linear_op_scale->Name()) @@ -175,6 +182,13 @@ void DeleteQuantDequantLinearOpPass::ApplyImpl(ir::Graph* graph) const { nodes2rm.insert(dequantize_linear_op); nodes2rm.insert(dequantize_linear_op_out); GraphSafeRemoveNodes(graph, nodes2rm); + + for (auto& var_name : vars2rm) { + if (scope->FindVar(var_name)) { + scope->EraseVars({var_name}); + } + } + found_count++; }; gpd(graph, handler); diff --git a/paddle/fluid/framework/ir/trt_delete_weight_dequant_linear_op_pass.cc b/paddle/fluid/framework/ir/trt_delete_weight_dequant_linear_op_pass.cc index 99aa8a5002e85..6e12933f0f4d5 100644 --- a/paddle/fluid/framework/ir/trt_delete_weight_dequant_linear_op_pass.cc +++ b/paddle/fluid/framework/ir/trt_delete_weight_dequant_linear_op_pass.cc @@ -232,6 +232,13 @@ void TrtDeleteWeightQuantDequantLinearOpPass::ApplyImpl( } */ std::unordered_set nodes2rm = {}; + + // delete Scale and ZeroPoint tensor in scope + std::vector vars2rm = {}; + vars2rm.emplace_back(weight_dequantize_linear_op->Op()->Input("Scale")[0]); + vars2rm.emplace_back( + weight_dequantize_linear_op->Op()->Input("ZeroPoint")[0]); + int bit_length = PADDLE_GET_CONST( int, weight_dequantize_linear_op->Op()->GetAttr("bit_length")); int range = ((1 << (bit_length - 1)) - 1); @@ -356,6 +363,13 @@ void TrtDeleteWeightQuantDequantLinearOpPass::ApplyImpl( } GraphSafeRemoveNodes(graph, nodes2rm); + + for (auto& var_name : vars2rm) { + if (scope->FindVar(var_name)) { + scope->EraseVars({var_name}); + } + } + found_count++; }; gpd(graph, handler); diff --git a/paddle/fluid/inference/analysis/argument.h b/paddle/fluid/inference/analysis/argument.h index 22a420f2de347..69b78b1ef33f3 100644 --- a/paddle/fluid/inference/analysis/argument.h +++ b/paddle/fluid/inference/analysis/argument.h @@ -165,6 +165,9 @@ struct Argument { DECL_ARGUMENT_FIELD(model_params_path, ModelParamsPath, std::string); DECL_ARGUMENT_FIELD(model_from_memory, ModelFromMemory, bool); DECL_ARGUMENT_FIELD(save_optimized_model, SaveOptimizedModel, bool); + DECL_ARGUMENT_FIELD(optimized_model_save_path, + OptimizedModelSavePath, + std::string); DECL_ARGUMENT_FIELD(optim_cache_dir, OptimCacheDir, std::string); DECL_ARGUMENT_FIELD(enable_ir_optim, EnableIrOptim, bool); diff --git a/paddle/fluid/inference/analysis/passes/save_optimized_model_pass.cc b/paddle/fluid/inference/analysis/passes/save_optimized_model_pass.cc index cad0296369479..cc463ce45f105 100644 --- a/paddle/fluid/inference/analysis/passes/save_optimized_model_pass.cc +++ b/paddle/fluid/inference/analysis/passes/save_optimized_model_pass.cc @@ -24,22 +24,7 @@ namespace inference { namespace analysis { void SaveOptimizedModelPass::SaveOptimizedModel(Argument* argument) { - std::string model_opt_cache_dir = argument->optim_cache_dir(); - if (!model_opt_cache_dir.empty()) { - if (!PathExists(model_opt_cache_dir)) { - PADDLE_ENFORCE_NE( - MKDIR(model_opt_cache_dir.c_str()), - -1, - platform::errors::PreconditionNotMet( - "Can not create optimize cache directory: %s, Make sure you " - "have permission to write", - model_opt_cache_dir)); - } - } else { - model_opt_cache_dir = argument->Has("model_dir") - ? argument->model_dir() - : GetDirRoot(argument->model_program_path()); - } + std::string model_opt_cache_dir = argument->optimized_model_save_path(); auto& scope = argument->scope(); auto* graph = argument->main_graph_ptr(); @@ -52,6 +37,14 @@ void SaveOptimizedModelPass::SaveOptimizedModel(Argument* argument) { framework::ir::GraphToProgram(*graph, &optimized_program_desc); + // Some vars may be deleted by pass, so we need to remove them in block + framework::BlockDesc* block = optimized_program_desc.MutableBlock(0); + for (auto& var_desc : block->AllVars()) { + if (var_desc->Persistable() && !scope.FindVar(var_desc->Name())) { + block->RemoveVar(var_desc->Name()); + } + } + auto IsPersistable = [](const framework::VarDesc* var) { if (var->Persistable() && var->GetType() != framework::proto::VarType::FEED_MINIBATCH && @@ -81,7 +74,7 @@ void SaveOptimizedModelPass::SaveOptimizedModel(Argument* argument) { } } - std::string save_params_path = path + "/" + "_optimized.pdiparams"; + std::string save_params_path = path + ".pdiparams"; std::vector save_var_list(save_var_set.begin(), save_var_set.end()); std::sort(save_var_list.begin(), save_var_list.end()); @@ -112,7 +105,7 @@ void SaveOptimizedModelPass::SaveOptimizedModel(Argument* argument) { } } } - std::string save_model_path = path + "/" + "_optimized.pdmodel"; + std::string save_model_path = path + ".pdmodel"; auto str = optimized_program_desc.Proto()->SerializeAsString(); std::ofstream file(save_model_path.c_str(), std::ios::binary); file.write(str.c_str(), str.size()); // NOLINT diff --git a/paddle/fluid/inference/api/analysis_config.cc b/paddle/fluid/inference/api/analysis_config.cc index 17841b8be5bad..7c1dad8a0d2b3 100644 --- a/paddle/fluid/inference/api/analysis_config.cc +++ b/paddle/fluid/inference/api/analysis_config.cc @@ -544,6 +544,8 @@ AnalysisConfig::AnalysisConfig(const AnalysisConfig &other) { CP_MEMBER(ir_debug_); CP_MEMBER(specify_input_name_); + CP_MEMBER(use_optimized_model_); + CP_MEMBER(cpu_math_library_num_threads_); CP_MEMBER(serialized_info_cache_); @@ -1152,6 +1154,8 @@ std::string AnalysisConfig::SerializeInfoCache() { ss << enable_ir_optim_; ss << ir_debug_; + ss << use_optimized_model_; + ss << specify_input_name_; ss << cpu_math_library_num_threads_; @@ -1471,6 +1475,8 @@ std::string AnalysisConfig::Summary() { {"save_optimized_model", save_optimized_model_ ? "true" : "false"}); os.InsertRow({"ir_optim", enable_ir_optim_ ? "true" : "false"}); os.InsertRow({"ir_debug", ir_debug_ ? "true" : "false"}); + os.InsertRow( + {"use_optimized_model", use_optimized_model_ ? "true" : "false"}); os.InsertRow({"memory_optim", enable_memory_optim_ ? "true" : "false"}); os.InsertRow({"enable_profile", with_profile_ ? "true" : "false"}); os.InsertRow({"enable_log", with_glog_info_ ? "true" : "false"}); diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index dd922cd45dd34..4ff85d08ff1d1 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -415,6 +415,24 @@ bool AnalysisPredictor::Init( // no matter with or without MKLDNN paddle::platform::SetNumThreads(config_.cpu_math_library_num_threads()); + // Use Optimized model to inference + if (config_.use_optimized_model_) { + std::string optimized_model_path = GetOptimizedModelPath(); + std::string optimized_model = optimized_model_path + ".pdmodel"; + std::string optimized_params = optimized_model_path + ".pdiparams"; + if (FileExists(optimized_model) && FileExists(optimized_params)) { + config_.SetModel(optimized_model, optimized_params); + LOG(INFO) << "Load Optimized model from " << optimized_model_path; + } else { + LOG(WARNING) + << "The optimized model is not found, fallback to original model. " + "EnableSaveOptimModel will be turned on and the optimized model " + "can be available next time."; + config_.EnableSaveOptimModel(true); + config_.UseOptimizedModel(false); + } + } + if (!PrepareScope(parent_scope)) { return false; } @@ -554,6 +572,55 @@ void AnalysisPredictor::InitPlace() { } } +std::string AnalysisPredictor::GetOptimizedModelPath() { + std::string model_opt_cache_dir = config_.opt_cache_dir_; + if (!model_opt_cache_dir.empty()) { + if (!PathExists(model_opt_cache_dir)) { + PADDLE_ENFORCE_NE( + MKDIR(model_opt_cache_dir.c_str()), + -1, + platform::errors::PreconditionNotMet( + "Can not create optimize cache directory: %s, Make sure you " + "have permission to write", + model_opt_cache_dir)); + } + } else { + model_opt_cache_dir = + !config_.model_dir().empty() + ? config_.model_dir() + : inference::analysis::GetDirRoot(config_.prog_file()); + } + return model_opt_cache_dir + "/" + "_optimized"; +} + +void AnalysisPredictor::ClearExtraParams() { + auto var_names = scope_->LocalVarNames(); + std::vector trt_repetitive_params; + for (auto &op_desc : inference_program_->Block(0).AllOps()) { + if (op_desc->Type() == "tensorrt_engine") { + auto trt_params = PADDLE_GET_CONST(std::vector, + op_desc->GetAttr("parameters")); + trt_repetitive_params.insert( + trt_repetitive_params.end(), trt_params.begin(), trt_params.end()); + } + } + + std::vector extra_params; + for (auto &var_desc : inference_program_->Block(0).AllVars()) { + if (var_desc->Persistable()) { + // Clear repetitive parameters in tensorrt + if (scope_->FindVar(var_desc->Name()) && + std::count(trt_repetitive_params.begin(), + trt_repetitive_params.end(), + var_desc->Name())) { + extra_params.emplace_back(var_desc->Name()); + } + } + } + scope_->EraseVars(extra_params); + VLOG(1) << "Clear " << extra_params.size() << " extra params."; +} + void AnalysisPredictor::InitResourceManager(void *stream) { #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) predictor_stream_ = @@ -701,7 +768,17 @@ bool AnalysisPredictor::PrepareProgram( // not be executed. model_precision_ = paddle::inference::GetModelPrecision(*inference_program_); - OptimizeInferenceProgram(); + if (config_.use_optimized_model_) { + LoadParameters(); + ClearExtraParams(); +#ifdef PADDLE_WITH_CUDA + if (config_.use_gpu()) { + paddle::platform::EmptyCache(); + } +#endif + } else { + OptimizeInferenceProgram(); + } } else { // If the program is passed from external, no need to optimize it, this // logic is used in the clone scenario. @@ -1600,6 +1677,7 @@ void AnalysisPredictor::PrepareArgument() { argument_->SetModelProgramPath(config_.prog_file()); argument_->SetModelParamsPath(config_.params_file()); } + argument_->SetOptimizedModelSavePath(GetOptimizedModelPath()); // For JITLayer argument_->SetSkipLoadParams(config_.skip_load_params_); diff --git a/paddle/fluid/inference/api/analysis_predictor.h b/paddle/fluid/inference/api/analysis_predictor.h index 24e8cc1cbe850..1c107e936d69a 100644 --- a/paddle/fluid/inference/api/analysis_predictor.h +++ b/paddle/fluid/inference/api/analysis_predictor.h @@ -494,6 +494,8 @@ class AnalysisPredictor : public PaddlePredictor { void InitPlace(); void InitDeviceContexts(); void InitResourceManager(void *stream); + std::string GetOptimizedModelPath(); + void ClearExtraParams(); #if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE) // fleet exe related diff --git a/paddle/fluid/inference/api/paddle_analysis_config.h b/paddle/fluid/inference/api/paddle_analysis_config.h index e69710e93c8f5..5f187e3cb7a22 100644 --- a/paddle/fluid/inference/api/paddle_analysis_config.h +++ b/paddle/fluid/inference/api/paddle_analysis_config.h @@ -606,7 +606,6 @@ struct PD_INFER_DECL AnalysisConfig { /// \return bool Whether to use ir graph optimization. /// bool ir_optim() const { return enable_ir_optim_; } - /// /// \brief INTERNAL Determine whether to use the feed and fetch operators. /// Just for internal development, not stable yet. @@ -881,6 +880,13 @@ struct PD_INFER_DECL AnalysisConfig { bool new_executor_enabled() const { return use_new_executor_; } + /// + /// \brief Control whether to use optimized model to inference. + /// + /// \param x whether to use optimized model. + /// + void UseOptimizedModel(bool x = true) { use_optimized_model_ = x; } + void EnableDlnne( int min_subgraph_size = 3, int max_batch_size = 1, @@ -1316,6 +1322,8 @@ struct PD_INFER_DECL AnalysisConfig { bool enable_ir_optim_{true}; bool ir_debug_{false}; + bool use_optimized_model_{false}; + bool use_new_executor_{false}; bool specify_input_name_{false}; diff --git a/paddle/fluid/inference/api/paddle_pass_builder.h b/paddle/fluid/inference/api/paddle_pass_builder.h index ece8506159921..2318c88741f28 100644 --- a/paddle/fluid/inference/api/paddle_pass_builder.h +++ b/paddle/fluid/inference/api/paddle_pass_builder.h @@ -113,13 +113,14 @@ class PD_INFER_DECL PaddlePassBuilder { protected: /// \cond Protected - std::vector analysis_passes_{ - {"ir_graph_build_pass", - "ir_analysis_pass", - "save_optimized_model_pass", - "ir_params_sync_among_devices_pass", - "adjust_cudnn_workspace_size_pass", - "inference_op_replace_pass"}}; + std::vector analysis_passes_{{ + "ir_graph_build_pass", + "ir_analysis_pass", + "ir_params_sync_among_devices_pass", + "adjust_cudnn_workspace_size_pass", + "inference_op_replace_pass", + "save_optimized_model_pass", + }}; std::vector passes_; std::unordered_set deleted_passes_; /// \endcond diff --git a/paddle/fluid/pybind/inference_api.cc b/paddle/fluid/pybind/inference_api.cc index 524cb46f21a60..2072bb3802cdd 100644 --- a/paddle/fluid/pybind/inference_api.cc +++ b/paddle/fluid/pybind/inference_api.cc @@ -861,6 +861,9 @@ void BindAnalysisConfig(py::module *m) { &AnalysisConfig::SwitchIrOptim, py::arg("x") = true) .def("ir_optim", &AnalysisConfig::ir_optim) + .def("use_optimized_model", + &AnalysisConfig::UseOptimizedModel, + py::arg("x") = true) .def("enable_memory_optim", &AnalysisConfig::EnableMemoryOptim, py::arg("x") = true) diff --git a/test/ir/inference/test_use_optimized_model_api.py b/test/ir/inference/test_use_optimized_model_api.py new file mode 100644 index 0000000000000..cdfcb705e8a9c --- /dev/null +++ b/test/ir/inference/test_use_optimized_model_api.py @@ -0,0 +1,116 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np +from inference_pass_test import InferencePassTest + +import paddle +from paddle.inference import Config, create_predictor + +# -------------------------- TestNet -------------------------- +# x +# / \ +# conv2d \ x +# | \ IR/Pass / \ +# batch_norm conv2d ——————> tensorrt_engine conv2d +# | / \ / +# relu / elemenwise_add +# \ / | +# elemenwise_add y +# | +# y +# ------------------------------------------------------------- + + +class TestNet(paddle.nn.Layer): + def __init__(self): + super().__init__() + self.conv1 = paddle.nn.Conv2D(3, 6, kernel_size=3, bias_attr=False) + self.bn1 = paddle.nn.BatchNorm2D(6) + self.relu = paddle.nn.ReLU() + self.conv2 = paddle.nn.Conv2D(3, 6, kernel_size=3, bias_attr=False) + + def forward(self, x): + x1 = self.conv1(x) + x1 = self.bn1(x1) + x1 = self.relu(x1) + x2 = self.conv2(x) + y = paddle.add(x1, x2) + return y + + +class UseOptimizedModel(InferencePassTest): + def setUp(self): + paddle.disable_static() + self.test_model = TestNet() + self.input_data = (np.ones([1, 3, 32, 32])).astype('float32') + self.path_prefix = "inference_test_models/use_optimized_model_test" + self.cache_dir = "inference_test_models/cache" + paddle.jit.save( + self.test_model, + self.path_prefix, + input_spec=[ + paddle.static.InputSpec(shape=[1, 3, 32, 32], dtype='float32') + ], + ) + + def test_check_output(self): + out_origin_model = self.inference() + out_optimized_model = self.inference() + np.testing.assert_allclose( + out_origin_model, out_optimized_model, rtol=1e-5, atol=1e-2 + ) + + def inference(self): + # Config + config = Config( + self.path_prefix + ".pdmodel", self.path_prefix + ".pdiparams" + ) + # if core.is_compiled_with_cuda(): + config.enable_use_gpu(100, 0) + config.enable_tensorrt_engine( + workspace_size=1 << 30, + max_batch_size=1, + min_subgraph_size=1, + precision_mode=paddle.inference.PrecisionType.Float32, + use_static=True, + use_calib_mode=False, + ) + config.enable_tuned_tensorrt_dynamic_shape() + config.exp_disable_tensorrt_ops(["elementwise_add"]) + config.set_optim_cache_dir(self.cache_dir) + config.use_optimized_model(True) + + # predictor + predictor = create_predictor(config) + + # inference + input_tensor = predictor.get_input_handle( + predictor.get_input_names()[0] + ) + input_tensor.reshape(self.input_data.shape) + input_tensor.copy_from_cpu(self.input_data.copy()) + predictor.run() + output_tensor = predictor.get_output_handle( + predictor.get_output_names()[0] + ) + out = output_tensor.copy_to_cpu() + out = np.array(out).flatten() + return out + + +if __name__ == "__main__": + unittest.main() From 4300e32c27ebbf6cb18964ce3ddc29fc4ffa9626 Mon Sep 17 00:00:00 2001 From: lanxianghit <47554610+lanxianghit@users.noreply.github.com> Date: Mon, 26 Feb 2024 15:07:32 +0800 Subject: [PATCH 082/282] [PIR][DynamicShape] Add InferSymbolicShape for top_p_sampling & feed op (#62011) * Add InferSymbolicShape for top_p_sampling, feed, select_input, where --- .../paddle_op_infer_sym.cc | 42 +++++++++++++++++-- .../dialect/operator/ir/control_flow_op.cc | 42 +++++++++++++++++++ .../pir/dialect/operator/ir/control_flow_op.h | 4 +- 3 files changed, 84 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc index 0b1dff55f4c41..86580325ba12a 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h" +#include "paddle/common/ddim.h" #include "paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.h" #include "paddle/fluid/pir/dialect/operator/ir/op_attribute.h" @@ -1005,8 +1006,9 @@ bool MaxOpInferSymbolicShape(pir::Operation *op, bool WhereOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + shape_analysis->SetShapeOrDataForValue( + op->result(0), + shape_analysis->GetShapeOrDataForValue(op->operand_source(0))); return true; } @@ -1017,7 +1019,21 @@ bool Where_OpInferSymbolicShape( bool FeedOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - // This Op has NO InferMeta in yaml, just return true + const common::DDim &result_dims = + op->result(0).type().dyn_cast().dims(); + std::vector out_dims; + for (int i = 0; i < result_dims.size(); i++) { + if (result_dims[i] == -1) { + out_dims.emplace_back(shape_analysis->GetNextSymName()); + } else { + out_dims.emplace_back(result_dims[i]); + } + } + + shape_analysis->SetShapeOrDataForValue( + op->result(0), + symbol::ShapeOrDataDimExprs{symbol::TensorShapeOrDataDimExprs(out_dims)}); + return true; } @@ -1025,6 +1041,26 @@ bool TopPSamplingOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { PADDLE_THROW(phi::errors::Unimplemented( op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + + const auto &x_dims = [op, shape_analysis] { + const auto &shape_or_data = + shape_analysis->GetShapeOrDataForValue(op->operand_source(0)); + if (shape_or_data.data().has_value()) { + return shape_or_data.data().value(); + } else { + return shape_or_data.shape(); + } + }(); + + // all the result have the same shape + for (uint32_t rst_idx = 0; rst_idx < op->num_results(); rst_idx++) { + const std::vector out_dims{x_dims[0], 1}; + shape_analysis->SetShapeOrDataForValue( + op->result(rst_idx), + symbol::ShapeOrDataDimExprs{ + symbol::TensorShapeOrDataDimExprs(out_dims)}); + } + return true; } diff --git a/paddle/fluid/pir/dialect/operator/ir/control_flow_op.cc b/paddle/fluid/pir/dialect/operator/ir/control_flow_op.cc index 7f3929d0b9967..92ec95b6b65f6 100644 --- a/paddle/fluid/pir/dialect/operator/ir/control_flow_op.cc +++ b/paddle/fluid/pir/dialect/operator/ir/control_flow_op.cc @@ -989,6 +989,48 @@ void SelectInputOp::VerifySig() { VLOG(4) << "End Verifying for: AssignArray_Op."; } +bool SelectInputOp::InferSymbolicShape( + pir::ShapeConstraintIRAnalysis *shape_analysis) { + auto GetSymExprForValue = + [shape_analysis](pir::Value val) -> const std::vector & { + const auto &shape_or_data = shape_analysis->GetShapeOrDataForValue(val); + if (shape_or_data.data().has_value()) { + return shape_or_data.data().value(); + } else { + return shape_or_data.shape(); + } + }; + + const auto &input1_dims = GetSymExprForValue(operand_source(0)); + const auto &input2_dims = GetSymExprForValue(operand_source(1)); + + std::vector out_dims = input1_dims; + // merge shape for input1 and input2, since we don't know which will be + // selected in compile time, the strategy is same with IfOp, see IfOp's + // comments for details and examples + if (input2_dims.size() != 0) { + // now only support input1 and input2 have same rank. + PADDLE_ENFORCE_EQ(input1_dims.size(), + input2_dims.size(), + phi::errors::PreconditionNotMet( + "The true and false block should have same rank, " + "but got true_rank(%d) and false_rank(%d)", + input1_dims.size(), + input2_dims.size())); + for (size_t i = 0; i < input1_dims.size(); i++) { + if (input1_dims[i] != input2_dims[i]) { + out_dims[i] = symbol::DimExpr{shape_analysis->GetNextSymName()}; + } + } + } + + shape_analysis->SetShapeOrDataForValue( + result(0), + symbol::ShapeOrDataDimExprs{symbol::TensorShapeOrDataDimExprs(out_dims)}); + + return true; +} + void SelectOutputOp::VerifySig() { VLOG(4) << "Verifying inputs, outputs and attributes for: SelectOutputOp."; VLOG(4) << "Verifying inputs:"; diff --git a/paddle/fluid/pir/dialect/operator/ir/control_flow_op.h b/paddle/fluid/pir/dialect/operator/ir/control_flow_op.h index f8a6bbb9f3b0f..8b5af449d4820 100644 --- a/paddle/fluid/pir/dialect/operator/ir/control_flow_op.h +++ b/paddle/fluid/pir/dialect/operator/ir/control_flow_op.h @@ -193,7 +193,8 @@ class AssertOp : public pir::Op { pir::Value data() { return operand_source(1); } }; -class SelectInputOp : public pir::Op { +class SelectInputOp + : public pir::Op { public: using Op::Op; static const char *name() { return "pd_op.select_input"; } @@ -202,6 +203,7 @@ class SelectInputOp : public pir::Op { void VerifySig(); pir::Value mask() { return operand_source(0); } pir::Value out() { return result(0); } + bool InferSymbolicShape(pir::ShapeConstraintIRAnalysis *shape_analysis); }; class SelectOutputOp : public pir::Op { From ae2d4b96cde2160d4abf606b1701486f70df5868 Mon Sep 17 00:00:00 2001 From: Yuanle Liu Date: Mon, 26 Feb 2024 15:08:29 +0800 Subject: [PATCH 083/282] [PIR] Fix conv2d_add_(act)_fuse_pass (#61979) * fix conv2d_add_fuse_pass * fix * add comment --- .../fluid/inference/api/analysis_predictor.cc | 4 +- .../fluid/pir/dialect/op_generator/op_gen.py | 58 ++++--- .../pir/dialect/op_generator/ops_api_gen.py | 4 +- .../fusion/conv2d_add_act_fuse_pass.cc | 4 + .../transforms/fusion/conv2d_add_fuse_pass.cc | 7 + .../fusion/fused_weight_only_linear_pass.cc | 3 +- .../transforms/transform_general_functions.cc | 20 ++- .../transforms/transform_general_functions.h | 10 ++ .../test_conv2d_add_act_fuse_pass.py | 28 ++- .../fused_pass/test_conv2d_add_fuse_pass.py | 23 ++- .../fused_pass/test_conv2d_bias_fuse_pass.py | 163 ------------------ 11 files changed, 109 insertions(+), 215 deletions(-) delete mode 100644 test/ir/pir/fused_pass/test_conv2d_bias_fuse_pass.py diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index 4ff85d08ff1d1..299e69d628745 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -904,13 +904,13 @@ bool AnalysisPredictor::PrepareExecutor() { params_sync_among_devices_pass->SetNotOwned(pir::kPlaceAttr, &place_); params_sync_among_devices_pass->SetNotOwned(pir::kParamScopeAttr, sub_scope_); + gpu_pm.AddPass(std::move(params_sync_among_devices_pass)); auto constant_folding_pass = ::pir::CreateConstantFoldingPass(); constant_folding_pass->SetNotOwned(pir::kPlaceAttr, &place_); constant_folding_pass->SetNotOwned(pir::kParamScopeAttr, sub_scope_); - - gpu_pm.AddPass(std::move(params_sync_among_devices_pass)); gpu_pm.AddPass(std::move(constant_folding_pass)); + gpu_pm.AddPass(::pir::CreateDeadCodeEliminationPass()); gpu_pm.AddPass(::pir::CreateReplaceFetchWithShadowOutputPass()); //----------------------------------------------------------------------------------------------// diff --git a/paddle/fluid/pir/dialect/op_generator/op_gen.py b/paddle/fluid/pir/dialect/op_generator/op_gen.py index 55da686d2a3b1..40dc916d4f4ad 100644 --- a/paddle/fluid/pir/dialect/op_generator/op_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/op_gen.py @@ -86,25 +86,27 @@ #pragma once #include -#include "paddle/pir/include/core/builder.h" -#include "paddle/pir/include/core/operation_utils.h" +#include "paddle/fluid/pir/dialect/operator/interface/decomp.h" #include "paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_symbolic_shape.h" -#include "paddle/pir/include/core/op_base.h" -#include "paddle/pir/include/core/op_trait.h" -#include "paddle/fluid/pir/dialect/operator/utils/utils.h" -#include "paddle/fluid/pir/dialect/operator/utils/op_yaml_info_util.h" -#include "paddle/fluid/pir/dialect/operator/interface/op_yaml_info.h" #include "paddle/fluid/pir/dialect/operator/interface/infermeta.h" -#include "paddle/fluid/pir/dialect/operator/interface/vjp.h" +#include "paddle/fluid/pir/dialect/operator/interface/op_yaml_info.h" #include "paddle/fluid/pir/dialect/operator/interface/parse_kernel_key.h" -#include "paddle/fluid/pir/dialect/operator/interface/decomp.h" +#include "paddle/fluid/pir/dialect/operator/interface/vjp.h" #include "paddle/fluid/pir/dialect/operator/trait/inplace.h" +#include "paddle/fluid/pir/dialect/operator/utils/op_yaml_info_util.h" +#include "paddle/fluid/pir/dialect/operator/utils/utils.h" +#include "paddle/pir/include/core/builder.h" +#include "paddle/pir/include/core/op_base.h" +#include "paddle/pir/include/core/op_trait.h" +#include "paddle/pir/include/core/operation_utils.h" +#ifdef PADDLE_WITH_DNNL #include "paddle/fluid/pir/dialect/operator/trait/onednn.h" -#include "paddle/fluid/pir/dialect/operator/trait/custom_vjp.h" +#endif #include "paddle/fluid/framework/infershape_utils.h" -#include "paddle/phi/core/infermeta_utils.h" -#include "paddle/fluid/pir/dialect/operator/ir/manual_op.h" #include "paddle/fluid/ir_adaptor/translator/utils.h" +#include "paddle/fluid/pir/dialect/operator/ir/manual_op.h" +#include "paddle/fluid/pir/dialect/operator/trait/custom_vjp.h" +#include "paddle/phi/core/infermeta_utils.h" {only_pd_op_header_files} {other_info} @@ -176,27 +178,27 @@ class {TEST_API} {op_name} : public pir::Op<{op_name}{interfaces}{traits}> {{ # ===================================== CC_FILE_TEMPLATE = """// This file is generated by "paddle/fluid/pir/dialect/op_generator/op_gen.py" #include "{h_file}" -#include "paddle/fluid/pir/dialect/operator/ir/op_type.h" #include "paddle/fluid/pir/dialect/kernel/ir/kernel_type.h" -#include "paddle/fluid/pir/dialect/operator/ir/op_attribute.h" -#include "paddle/fluid/pir/dialect/operator/ir/ir_tensor.h" -#include "paddle/fluid/pir/dialect/operator/ir/ir_selected_rows.h" #include "paddle/fluid/pir/dialect/operator/ir/ir_meta_tensor.h" -#include "paddle/pir/include/core/builtin_attribute.h" -#include "paddle/pir/include/core/builtin_type.h" -#include "paddle/pir/include/core/builtin_op.h" -#include "paddle/pir/include/core/ir_context.h" -#include "paddle/phi/core/enforce.h" +#include "paddle/fluid/pir/dialect/operator/ir/ir_selected_rows.h" +#include "paddle/fluid/pir/dialect/operator/ir/ir_tensor.h" +#include "paddle/fluid/pir/dialect/operator/ir/op_attribute.h" +#include "paddle/fluid/pir/dialect/operator/ir/op_type.h" +#include "paddle/fluid/primitive/rule/vjp/vjp.h" +#include "paddle/phi/api/lib/utils/allocator.h" #include "paddle/phi/core/dense_tensor.h" +#include "paddle/phi/core/enforce.h" +#include "paddle/phi/infermeta/backward.h" #include "paddle/phi/infermeta/binary.h" +#include "paddle/phi/infermeta/fusion.h" #include "paddle/phi/infermeta/multiary.h" #include "paddle/phi/infermeta/nullary.h" -#include "paddle/phi/infermeta/unary.h" #include "paddle/phi/infermeta/ternary.h" -#include "paddle/phi/infermeta/backward.h" -#include "paddle/phi/infermeta/fusion.h" -#include "paddle/phi/api/lib/utils/allocator.h" -#include "paddle/fluid/primitive/rule/vjp/vjp.h" +#include "paddle/phi/infermeta/unary.h" +#include "paddle/pir/include/core/builtin_attribute.h" +#include "paddle/pir/include/core/builtin_op.h" +#include "paddle/pir/include/core/builtin_type.h" +#include "paddle/pir/include/core/ir_context.h" #include "paddle/pir/include/core/op_base.h" using namespace paddle::dialect; @@ -237,12 +239,12 @@ class {TEST_API} {op_name} : public pir::Op<{op_name}{interfaces}{traits}> {{ VJP_CC_FILE_TEMPLATE = """// This file is generated by "paddle/fluid/pir/dialect/op_generator/op_gen.py" #include "paddle/fluid/pir/dialect/operator/ir/op_attribute.h" #include "paddle/fluid/pir/dialect/operator/ir/pd_op.h" +#include "paddle/fluid/pir/dialect/operator/utils/utils.h" #include "paddle/fluid/primitive/rule/vjp/vjp.h" #include "paddle/fluid/primitive/type/lazy_tensor.h" +#include "paddle/phi/common/int_array.h" #include "paddle/pir/include/core/builtin_op.h" #include "paddle/pir/include/core/op_base.h" -#include "paddle/phi/common/int_array.h" -#include "paddle/fluid/pir/dialect/operator/utils/utils.h" namespace paddle {{ namespace dialect {{ diff --git a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py index 8328e406ae0e6..5dbbf62642e97 100644 --- a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py @@ -20,11 +20,11 @@ CPP_FILE_TEMPLATE = """ #include -#include "paddle/fluid/pybind/static_op_function.h" +#include "paddle/fluid/eager/api/utils/global_utils.h" #include "paddle/fluid/pybind/eager_op_function.h" #include "paddle/fluid/pybind/manual_static_op_function.h" +#include "paddle/fluid/pybind/static_op_function.h" #include "paddle/phi/core/enforce.h" -#include "paddle/fluid/eager/api/utils/global_utils.h" {body} diff --git a/paddle/fluid/pir/transforms/fusion/conv2d_add_act_fuse_pass.cc b/paddle/fluid/pir/transforms/fusion/conv2d_add_act_fuse_pass.cc index c7710721350eb..9e950dc2d11b9 100644 --- a/paddle/fluid/pir/transforms/fusion/conv2d_add_act_fuse_pass.cc +++ b/paddle/fluid/pir/transforms/fusion/conv2d_add_act_fuse_pass.cc @@ -45,6 +45,8 @@ class Conv2dAddActFusePattern pir::Value add_input = op.x(); IR_ENFORCE(add_input == conv2d_out); + if (!pir::ValueIsPersitable(op.y())) return false; + pir::Value add_out = op.out(); if (!add_out.HasOneUse()) return false; @@ -117,6 +119,8 @@ class Conv2dAdd2ActFusePattern ->dyn_cast(); if (!add1_op) return false; + if (!pir::ValueIsPersitable(add1_op.y())) return false; + pir::Value add1_out = add1_op.out(); if (!add1_out.HasOneUse()) return false; diff --git a/paddle/fluid/pir/transforms/fusion/conv2d_add_fuse_pass.cc b/paddle/fluid/pir/transforms/fusion/conv2d_add_fuse_pass.cc index c5eb8134d05c4..9c1cec5b9b645 100644 --- a/paddle/fluid/pir/transforms/fusion/conv2d_add_fuse_pass.cc +++ b/paddle/fluid/pir/transforms/fusion/conv2d_add_fuse_pass.cc @@ -19,6 +19,9 @@ #include "paddle/fluid/pir/dialect/operator/ir/pd_op.h" #include "paddle/fluid/pir/drr/include/drr_pattern_base.h" +#include "paddle/fluid/pir/transforms/transform_general_functions.h" +#include "paddle/pir/include/core/builtin_op.h" +#include "paddle/pir/include/core/value.h" #include "paddle/pir/include/pass/pass.h" #include "paddle/pir/include/pass/pass_registry.h" @@ -44,6 +47,10 @@ class Conv2dAddFusePattern : public paddle::drr::DrrPatternBase { pat.Tensor("add_out") = add(pat.Tensor("conv2d_out"), pat.Tensor("bias")); pat.RequireNativeCall( [](const paddle::drr::MatchContext &match_ctx) -> bool { + if (!pir::ValueIsPersitable(match_ctx.Tensor("bias"))) { + return false; + } + auto padding_algorithm = match_ctx.Attr("padding_algorithm"); if (padding_algorithm != "EXPLICIT" && padding_algorithm != "SAME" && diff --git a/paddle/fluid/pir/transforms/fusion/fused_weight_only_linear_pass.cc b/paddle/fluid/pir/transforms/fusion/fused_weight_only_linear_pass.cc index 6922691684d30..bf4ea92af67b2 100644 --- a/paddle/fluid/pir/transforms/fusion/fused_weight_only_linear_pass.cc +++ b/paddle/fluid/pir/transforms/fusion/fused_weight_only_linear_pass.cc @@ -50,8 +50,7 @@ class FusedWeightOnlyLinearPattern : public paddle::drr::DrrPatternBase { src.Op(paddle::dialect::MatmulOp::name(), {{"transpose_x", src.Attr("matmul_transpose_x")}, {"transpose_y", src.Attr("matmul_transpose_y")}}); - const auto ¶meter = src.Op( - pir::ParameterOp::name(), {{"parameter_name", src.Attr("param_name")}}); + const auto ¶meter = src.Op(pir::ParameterOp::name()); src.Tensor("w") = parameter(); src.Tensor("matmul_out") = matmul(src.Tensor("x"), src.Tensor("w")); const auto &add = src.Op(paddle::dialect::AddOp::name()); diff --git a/paddle/fluid/pir/transforms/transform_general_functions.cc b/paddle/fluid/pir/transforms/transform_general_functions.cc index 92fc8ba68ddb7..55a1dc463dc6d 100644 --- a/paddle/fluid/pir/transforms/transform_general_functions.cc +++ b/paddle/fluid/pir/transforms/transform_general_functions.cc @@ -20,6 +20,7 @@ #include "paddle/fluid/pir/dialect/operator/ir/op_dialect.h" #include "paddle/fluid/pir/dialect/operator/ir/op_type.h" +#include "paddle/fluid/pir/dialect/operator/ir/pd_op.h" #include "paddle/pir/include/core/builtin_op.h" #include "paddle/pir/include/core/op_operand.h" #include "paddle/pir/include/core/parameter.h" @@ -116,7 +117,7 @@ std::vector> GetUseOpsForOutput( auto result = op->result(index); std::vector> use_ops; for (auto it = result.use_begin(); it != result.use_end(); ++it) { - use_ops.push_back(std::make_pair(it->owner(), it->index())); + use_ops.emplace_back(it->owner(), it->index()); } return use_ops; } @@ -138,4 +139,21 @@ std::vector GetUsedExternalValue(const pir::Block& block) { return used_values; } +bool ValueIsPersitable(pir::Value value) { + if (value.defining_op()->num_operands() > 0) { + for (const auto& source_value : value.defining_op()->operands_source()) { + if (!ValueIsPersitable(source_value)) { + return false; + } + } + } else { + if (!value.defining_op()->isa() && + !value.defining_op()->isa() && + !value.defining_op()->isa()) { + return false; + } + } + return true; +} + } // namespace pir diff --git a/paddle/fluid/pir/transforms/transform_general_functions.h b/paddle/fluid/pir/transforms/transform_general_functions.h index 8b9ffdd8cf477..d34c6d6863802 100644 --- a/paddle/fluid/pir/transforms/transform_general_functions.h +++ b/paddle/fluid/pir/transforms/transform_general_functions.h @@ -95,4 +95,14 @@ std::vector GetUsedExternalValue(const Operation& op); */ std::vector GetUsedExternalValue(const Block& block); +/** + * @brief Determine whether a value comes from a weight or has no input op. That + is to say, it is permissible. + * + * @param pir::Value + + * @return bool + */ +bool ValueIsPersitable(pir::Value value); + } // namespace pir diff --git a/test/ir/pir/fused_pass/test_conv2d_add_act_fuse_pass.py b/test/ir/pir/fused_pass/test_conv2d_add_act_fuse_pass.py index ca397cbebce82..aaaf7cb175497 100644 --- a/test/ir/pir/fused_pass/test_conv2d_add_act_fuse_pass.py +++ b/test/ir/pir/fused_pass/test_conv2d_add_act_fuse_pass.py @@ -19,21 +19,22 @@ import paddle from paddle.base import core +from paddle.pir.core import create_parameter paddle.enable_static() class TestConv2dAddActFusePattern(PassTest): r""" - x_var f_var + x_var f_var(w) \ / conv2d | - conv2d_var y_var + conv2d_var y_var(w) \ / elementwise_add | - elementwise_add_var + add_var | act | @@ -59,8 +60,14 @@ def build_ir_program(self): data_format='NCHW', bias_attr=False, ) - y = paddle.static.data( - name="y", shape=[3, 32, 28, 28], dtype="float32" + + y = create_parameter( + name="y", + shape=[3, 32, 28, 28], + dtype='float32', + initializer=paddle.nn.initializer.Assign( + np.random.random((3, 32, 28, 28)).astype("float32") + ), ) act_op = paddle.nn.ReLU() out = act_op(paddle.add(conv2d(x), y)) @@ -68,7 +75,6 @@ def build_ir_program(self): self.pass_list = ['conv2d_add_act_fuse_pass'] self.feeds = { "x": np.random.random((3, 1, 28, 28)).astype("float32"), - "y": np.random.random((3, 32, 28, 28)).astype("float32"), } self.fetch_list = [out] self.valid_op_map = { @@ -130,8 +136,13 @@ def build_ir_program(self): data_format='NCHW', bias_attr=False, ) - y = paddle.static.data( - name="y", shape=[3, 32, 28, 28], dtype="float32" + y = create_parameter( + name="y", + shape=[3, 32, 28, 28], + dtype='float32', + initializer=paddle.nn.initializer.Assign( + np.random.random((3, 32, 28, 28)).astype("float32") + ), ) residual_data = paddle.static.data( name="residual_data", shape=[3, 32, 28, 28], dtype="float32" @@ -144,7 +155,6 @@ def build_ir_program(self): self.pass_list = ['conv2d_add_act_fuse_pass'] self.feeds = { "x": np.random.random((3, 1, 28, 28)).astype("float32"), - "y": np.random.random((3, 32, 28, 28)).astype("float32"), "residual_data": np.random.random((3, 32, 28, 28)).astype( "float32" ), diff --git a/test/ir/pir/fused_pass/test_conv2d_add_fuse_pass.py b/test/ir/pir/fused_pass/test_conv2d_add_fuse_pass.py index cff8c6addd0e6..1e19364e3ba8d 100644 --- a/test/ir/pir/fused_pass/test_conv2d_add_fuse_pass.py +++ b/test/ir/pir/fused_pass/test_conv2d_add_fuse_pass.py @@ -19,17 +19,20 @@ import paddle from paddle.base import core +from paddle.pir.core import create_parameter paddle.enable_static() class TestConv2dAddFusePass(PassTest): r""" - x_var f_var + x_var filter(w) \ / - conv2d - | - add + conv2d bias(w) + | / + add + | + out_var """ def is_program_valid(self, program=None): @@ -43,8 +46,13 @@ def build_ir_program(self): x = paddle.static.data( name='x', shape=[3, 1, 28, 28], dtype='float32' ) - y = paddle.static.data( - name="y", shape=[3, 32, 28, 28], dtype="float32" + bias = create_parameter( + name="bias", + shape=[3, 32, 28, 28], + dtype='float32', + initializer=paddle.nn.initializer.Assign( + np.random.random((3, 32, 28, 28)).astype("float32") + ), ) conv2d = paddle.nn.Conv2D( in_channels=1, @@ -54,12 +62,11 @@ def build_ir_program(self): data_format='NCHW', bias_attr=False, ) - out = paddle.add(conv2d(x), y) + out = paddle.add(conv2d(x), bias) out = paddle.assign(out) self.pass_list = ['conv2d_add_fuse_pass'] self.feeds = { "x": np.random.random((3, 1, 28, 28)).astype("float32"), - "y": np.random.random((3, 32, 28, 28)).astype("float32"), } self.fetch_list = [out] self.valid_op_map = { diff --git a/test/ir/pir/fused_pass/test_conv2d_bias_fuse_pass.py b/test/ir/pir/fused_pass/test_conv2d_bias_fuse_pass.py deleted file mode 100644 index 1751f58818f3f..0000000000000 --- a/test/ir/pir/fused_pass/test_conv2d_bias_fuse_pass.py +++ /dev/null @@ -1,163 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest - -import numpy as np -from pass_test import PassTest - -import paddle - -paddle.enable_static() - - -@unittest.skipIf( - not paddle.base.core.is_compiled_with_mkldnn(), - "Test case only for OneDNN pass.", -) -class TestConv2dAddFusePass(PassTest): - def is_program_valid(self, program=None): - return True - - def build_ir_program(self): - with paddle.pir_utils.IrGuard(): - main_prog = paddle.static.Program() - start_prog = paddle.static.Program() - with paddle.pir.core.program_guard(main_prog, start_prog): - x = paddle.static.data( - name='x', shape=[5, 5, 5, 5], dtype='float32' - ) - bias_attr = paddle.ParamAttr( - learning_rate=0.0, - initializer=paddle.nn.initializer.Normal(mean=0.0, std=2.0), - ) - bias = paddle.static.create_parameter( - shape=[1], dtype='float32', attr=bias_attr, is_bias=False - ) - w_attr = paddle.ParamAttr( - learning_rate=0.0, - initializer=paddle.nn.initializer.Normal(mean=0.0, std=2.0), - ) - conv2d = paddle.nn.Conv2D( - in_channels=5, - out_channels=1, - kernel_size=[1, 1], - groups=1, - stride=[1, 1], - padding=[1, 1, 1, 1], - dilation=[1, 1], - data_format='NCHW', - bias_attr=False, - weight_attr=w_attr, - ) - - out = paddle.add(conv2d(x), bias) - out = paddle.assign(out) - self.pass_list = ['conv2d_bias_fuse_pass'] - self.feeds = { - "x": np.random.random((5, 5, 5, 5)).astype("float32"), - "bias": np.random.random(1).astype("float32"), - } - self.fetch_list = [out] - self.valid_op_map = { - "onednn_op.fused_conv2d": 1, - "pd_op.conv2d": 0, - "pd_op.add": 0, - } - return [main_prog, start_prog] - - def sample_program(self): - yield self.build_ir_program(), False - - def setUp(self): - self.places.append(paddle.CPUPlace()) - - def test_check_output(self): - self.check_pass_correct() - - -@unittest.skipIf( - not paddle.base.core.is_compiled_with_mkldnn(), - "Test case only for OneDNN pass.", -) -class TestConv2dAddFusePassWithAddParam(PassTest): - def is_program_valid(self, program=None): - return True - - def build_ir_program(self): - with paddle.pir_utils.IrGuard(): - main_prog = paddle.static.Program() - start_prog = paddle.static.Program() - with paddle.pir.core.program_guard(main_prog, start_prog): - x = paddle.static.data( - name='x', shape=[5, 5, 5, 5], dtype='float32' - ) - bias_attr = paddle.ParamAttr( - learning_rate=0.0, - initializer=paddle.nn.initializer.Normal(mean=0.0, std=2.0), - ) - bias = paddle.static.create_parameter( - shape=[1], dtype='float32', attr=bias_attr, is_bias=False - ) - w_attr = paddle.ParamAttr( - learning_rate=0.0, - initializer=paddle.nn.initializer.Normal(mean=0.0, std=2.0), - ) - conv2d = paddle.nn.Conv2D( - in_channels=5, - out_channels=1, - kernel_size=[1, 1], - groups=1, - stride=[1, 1], - padding=[1, 1, 1, 1], - dilation=[1, 1], - data_format='NCHW', - bias_attr=False, - weight_attr=w_attr, - ) - add_out = paddle.add(conv2d(x), bias) - other_param_attr = paddle.ParamAttr( - learning_rate=0.0, - initializer=paddle.nn.initializer.Normal(mean=0.0, std=2.0), - ) - other_param = paddle.static.create_parameter( - shape=[1], dtype='float32', attr=bias_attr, is_bias=False - ) - out = paddle.add(add_out, other_param) - out = paddle.assign(out) - self.pass_list = ['conv2d_bias_fuse_pass'] - self.feeds = { - "x": np.random.random((5, 5, 5, 5)).astype("float32"), - "bias": np.random.random(1).astype("float32"), - } - self.fetch_list = [out] - self.valid_op_map = { - "onednn_op.fused_conv2d": 1, - "pd_op.conv2d": 0, - "pd_op.add": 1, - } - return [main_prog, start_prog] - - def sample_program(self): - yield self.build_ir_program(), False - - def setUp(self): - self.places.append(paddle.CPUPlace()) - - def test_check_output(self): - self.check_pass_correct() - - -if __name__ == "__main__": - unittest.main() From 0b5ae5b2a1e787ef04b00f185bb8ccec993cfff8 Mon Sep 17 00:00:00 2001 From: Difer <707065510@qq.com> Date: Mon, 26 Feb 2024 16:09:40 +0800 Subject: [PATCH 084/282] [AutoConfig]add buffer mechanism and set best cfg log dir (#61499) * add buffer and best_cfg_log for autotuner * fix some errors * fix some typo errors * fix some error in recoder --- .../paddle/distributed/auto_tuner/recorder.py | 52 +++++++++++++------ python/paddle/distributed/launch/main.py | 27 +++++++--- 2 files changed, 57 insertions(+), 22 deletions(-) diff --git a/python/paddle/distributed/auto_tuner/recorder.py b/python/paddle/distributed/auto_tuner/recorder.py index 69c5300183262..006093a348d4a 100644 --- a/python/paddle/distributed/auto_tuner/recorder.py +++ b/python/paddle/distributed/auto_tuner/recorder.py @@ -51,33 +51,53 @@ def sort_metric(self, direction, metric_name) -> None: reverse=False, ) - def get_best(self, metric, direction, mode=None) -> Tuple[dict, bool]: + def get_best( + self, metric, direction, buffer=None, max_mem_usage=None + ) -> Tuple[dict, bool]: self.sort_metric(direction=direction, metric_name=metric) if len(self.history) == 0: - return (self.history[0], True) - if mode == "SFT" or mode == "LoRA" or mode == "Pretrain": - best_cfg = self.history[0] - if ( - isinstance(best_cfg["max_mem_usage"], str) - or best_cfg["time"] == -1 - ): - return (best_cfg, True) - first_few = 0 + return (None, True) + + best_cfg = self.history[0] + if isinstance(best_cfg["max_mem_usage"], str) or best_cfg["time"] == -1: + return (best_cfg, True) + + if buffer is not None: + if buffer < 0: + raise ValueError("The buffer should be not less than 0.") + assert ( + max_mem_usage is not None + ), "max_mem_usage cannot be None when buffer is greater than 0." + if max_mem_usage <= 0: + raise ValueError("max_mem_usage should be greater than 0.") + for cfg in self.history: + if ( + not best_cfg["max_mem_usage"] + and cfg["max_mem_usage"] + and not isinstance(cfg["max_mem_usage"], str) + and cfg["time"] != -1 + ): + best_cfg = cfg + continue + if ( not isinstance(cfg["max_mem_usage"], str) + and cfg["max_mem_usage"] and cfg["max_mem_usage"] < best_cfg["max_mem_usage"] and cfg["time"] != -1 ): best_cfg = cfg - first_few += 1 - if first_few >= 3: + + if ( + not isinstance(cfg["max_mem_usage"], str) + and cfg["max_mem_usage"] + and cfg["max_mem_usage"] < max_mem_usage - buffer + and cfg["time"] != -1 + ): break return (best_cfg, False) - if isinstance(self.history[0]["max_mem_usage"], str) or ( - "time" in self.history[0] and self.history[0]["time"] == -1 - ): - return (self.history[0], True) + return (self.history[0], False) def _store_history_impl(self, data, path="./history.csv"): diff --git a/python/paddle/distributed/launch/main.py b/python/paddle/distributed/launch/main.py index f5c0f8d7f1671..80f082260d110 100644 --- a/python/paddle/distributed/launch/main.py +++ b/python/paddle/distributed/launch/main.py @@ -436,6 +436,10 @@ def launch(): # max_search_time max_search_time = tuner_cfg.get("max_search_time", None) + # buffer and memory + buffer = tuner_cfg.get("buffer", None) + max_mem_usage = tuner_cfg.get("max_mem_usage", None) + is_first_task = True # build history recorder recorder = HistoryRecorder(tuner_cfg) @@ -681,6 +685,8 @@ def launch(): cur_best_cfgs, err = recorder.get_best( metric=tuner_cfg['metric_cfg']['name'], direction=tuner_cfg['metric_cfg']['OptimizationDirection'], + buffer=buffer, + max_mem_usage=max_mem_usage, ) if not err: ctx.logger.info(f"Current best config: {cur_best_cfgs}") @@ -781,6 +787,8 @@ def launch(): direction=tuner_cfg['metric_cfg'][ 'OptimizationDirection' ], + buffer=buffer, + max_mem_usage=max_mem_usage, ) if not err: ctx.logger.info(f"Current best config: {cur_best_cfgs}") @@ -1158,7 +1166,8 @@ def launch(): cur_best_cfgs, err = recorder.get_best( metric=tuner_cfg['metric_cfg']['name'], direction=tuner_cfg['metric_cfg']['OptimizationDirection'], - mode=mode, + buffer=buffer, + max_mem_usage=max_mem_usage, ) if not err: ctx.logger.info(f"Current best config: {cur_best_cfgs}") @@ -1206,7 +1215,8 @@ def launch(): best_cfg, err = recorder.get_best( metric=tuner_cfg['metric_cfg']['name'], direction=tuner_cfg['metric_cfg']['OptimizationDirection'], - mode=mode, + buffer=buffer, + max_mem_usage=max_mem_usage, ) if err: raise ValueError( @@ -1232,7 +1242,8 @@ def launch(): best_cfg, err = recorder.get_best( metric=tuner_cfg['metric_cfg']['name'], direction=tuner_cfg['metric_cfg']['OptimizationDirection'], - mode=mode, + buffer=buffer, + max_mem_usage=max_mem_usage, ) if err: raise ValueError( @@ -1255,9 +1266,13 @@ def launch(): ctx.args.job_id = "best_cfg" ctx.logger.info(f"Launch best cfg: {best_cfg}") logger.info(f"Launch best cfg: {best_cfg}") - ctx.args.log_dir = ctx.args.log_dir = os.path.join( - os.path.dirname(ctx.args.auto_tuner_json), "best_cfg" - ) + + if tuner_cfg.get("best_cfg_dir", None): + ctx.args.log_dir = tuner_cfg["best_cfg_dir"] + else: + ctx.args.log_dir = os.path.join( + os.path.dirname(ctx.args.auto_tuner_json), "best_cfg" + ) # run best cfg c = controllers.init(ctx) c.run() From cc63252b90e688226cd85ec235777117831613cb Mon Sep 17 00:00:00 2001 From: wanghuancoder Date: Mon, 26 Feb 2024 16:12:32 +0800 Subject: [PATCH 085/282] [PIR] pir onednn support mul (#61662) * pir onednn support mul --- .../instruction/onednn/onednn_instruction.cc | 6 +++-- .../onednn/onednn_legacy_instruction.cc | 6 +++-- .../pir_adaptor/pir_adaptor_util.cc | 6 +++-- .../ir_adaptor/translator/op_translator.cc | 26 +++++++++++++++++++ .../pir/dialect/op_generator/ops_api_gen.py | 1 + paddle/fluid/pir/dialect/operator/ir/ops.yaml | 10 +++++++ .../pir/dialect/operator/ir/ops_backward.yaml | 10 +++++++ .../dialect/operator/ir/ops_onednn_extra.yaml | 6 +++-- test/mkldnn/test_mul_int8_mkldnn_op.py | 2 +- test/mkldnn/test_mul_mkldnn_op.py | 26 ++++++++++++++++--- 10 files changed, 86 insertions(+), 13 deletions(-) diff --git a/paddle/fluid/framework/new_executor/instruction/onednn/onednn_instruction.cc b/paddle/fluid/framework/new_executor/instruction/onednn/onednn_instruction.cc index fab561df7a6e9..aa3df67535747 100644 --- a/paddle/fluid/framework/new_executor/instruction/onednn/onednn_instruction.cc +++ b/paddle/fluid/framework/new_executor/instruction/onednn/onednn_instruction.cc @@ -116,7 +116,8 @@ void TensorNameMap(pir::Operation* op, auto& name2id = op_yaml_info.InputName2Id(); - std::string fluid_op_name = op_yaml_info.GetOriginOpName(); + std::string fluid_op_name = + phi::TransToFluidOpName(op_yaml_info.OpRuntimeInfo().kernel_func); auto& op_normalizer = paddle::translator::OpNameNormalizer::instance(); @@ -327,7 +328,8 @@ OneDNNPhiKernelInstruction::OneDNNPhiKernelInstruction( .dyn_cast() .AsVector(); auto& op_normalizer = paddle::translator::OpNameNormalizer::instance(); - std::string fluid_op_name = yaml_info_parser.GetOriginOpName(); + std::string fluid_op_name = + phi::TransToFluidOpName(yaml_info_parser.OpRuntimeInfo().kernel_func); for (auto& attr : extra_args_attr) { auto attr_name = attr.dyn_cast().AsString(); diff --git a/paddle/fluid/framework/new_executor/instruction/onednn/onednn_legacy_instruction.cc b/paddle/fluid/framework/new_executor/instruction/onednn/onednn_legacy_instruction.cc index 0d14d59bcd35b..b006c11bf783a 100644 --- a/paddle/fluid/framework/new_executor/instruction/onednn/onednn_legacy_instruction.cc +++ b/paddle/fluid/framework/new_executor/instruction/onednn/onednn_legacy_instruction.cc @@ -219,7 +219,8 @@ OneDNNLegacyKernelInstruction::OneDNNLegacyKernelInstruction( .AsVector(); auto& op_normalizer = paddle::translator::OpNameNormalizer::instance(); - std::string fluid_op_name = yaml_info_parser.GetOriginOpName(); + std::string fluid_op_name = + phi::TransToFluidOpName(yaml_info_parser.OpRuntimeInfo().kernel_func); for (auto& attr : data_format_tensors_attr) { auto input_name = attr.dyn_cast().AsString(); data_format_tensors_.insert( @@ -241,7 +242,8 @@ OneDNNLegacyKernelInstruction::OneDNNLegacyKernelInstruction( .AsVector(); auto& op_normalizer = paddle::translator::OpNameNormalizer::instance(); - std::string fluid_op_name = yaml_info_parser.GetOriginOpName(); + std::string fluid_op_name = + phi::TransToFluidOpName(yaml_info_parser.OpRuntimeInfo().kernel_func); for (auto& input : skip_transform_inputs) { auto input_name = input.dyn_cast().AsString(); diff --git a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc index 4894e64a8f4d1..a56868482223a 100644 --- a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc +++ b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc @@ -805,7 +805,8 @@ void BuildRuntimeContext(pir::Operation* op, auto& name2id = op_yaml_info.InputName2Id(); - std::string fluid_op_name = op_yaml_info.GetOriginOpName(); + std::string fluid_op_name = + phi::TransToFluidOpName(op_yaml_info.OpRuntimeInfo().kernel_func); auto& op_normalizer = paddle::translator::OpNameNormalizer::instance(); @@ -890,7 +891,8 @@ std::shared_ptr BuildOperatorBase( auto& name2id = op_yaml_info.InputName2Id(); - std::string fluid_op_name = op_yaml_info.GetOriginOpName(); + std::string fluid_op_name = + phi::TransToFluidOpName(op_yaml_info.OpRuntimeInfo().kernel_func); auto& op_normalizer = paddle::translator::OpNameNormalizer::instance(); diff --git a/paddle/fluid/ir_adaptor/translator/op_translator.cc b/paddle/fluid/ir_adaptor/translator/op_translator.cc index 3acb0f4fc0718..c01df4d6e236c 100644 --- a/paddle/fluid/ir_adaptor/translator/op_translator.cc +++ b/paddle/fluid/ir_adaptor/translator/op_translator.cc @@ -1451,6 +1451,19 @@ ValueInfo GetTensorInfoByVarName(const OpDesc& op_desc, } struct MulOpTranscriber : public OpTranscriber { + pir::Operation* operator()(pir::IrContext* ctx, + TranslationContext* param_map, + const OpDesc& op_desc, + pir::Block* block) override { +#ifdef PADDLE_WITH_DNNL + if (op_desc.GetAttrIfExists("use_mkldnn")) { + return static_cast(*this).operator()( + ctx, param_map, op_desc, block); + } +#endif + return OpTranscriber::operator()(ctx, param_map, op_desc, block); + } + pir::OpInfo LookUpOpInfo(pir::IrContext* ctx, const OpDesc& op_desc) override { const std::string& target_op_name = paddle::dialect::MatmulOp::name(); @@ -1605,6 +1618,19 @@ struct MulOpTranscriber : public OpTranscriber { }; struct MulGradOpTranscriber : public OpTranscriber { + pir::Operation* operator()(pir::IrContext* ctx, + TranslationContext* param_map, + const OpDesc& op_desc, + pir::Block* block) override { +#ifdef PADDLE_WITH_DNNL + if (op_desc.GetAttrIfExists("use_mkldnn")) { + return static_cast(*this).operator()( + ctx, param_map, op_desc, block); + } +#endif + return OpTranscriber::operator()(ctx, param_map, op_desc, block); + } + pir::OpInfo LookUpOpInfo(pir::IrContext* ctx, const OpDesc& op_desc) override { const std::string& target_op_name = paddle::dialect::MatmulGradOp::name(); diff --git a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py index 5dbbf62642e97..b141f1ecfa879 100644 --- a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py @@ -107,6 +107,7 @@ 'onednn_to_paddle_layout', 'lrn', 'multi_gru', + 'matmul_with_flatten', ] NO_NEED_GEN_STATIC_ONLY_APIS = [ diff --git a/paddle/fluid/pir/dialect/operator/ir/ops.yaml b/paddle/fluid/pir/dialect/operator/ir/ops.yaml index 4fcd90c99fe0a..f1e20326d59de 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops.yaml @@ -848,6 +848,16 @@ backward : matmul_grad interfaces : paddle::dialect::InferSymbolicShapeInterface +- op : matmul_with_flatten + args : (Tensor x, Tensor y, int x_num_col_dims = 1, int y_num_col_dims = 1) + output : Tensor + infer_meta : + func : MatmulWithFlattenInferMeta + kernel : + func : matmul_with_flatten + data_type : x + backward : matmul_with_flatten_grad + - op : matrix_rank args : (Tensor x, float tol, bool use_default_tol=true, bool hermitian=false) output : Tensor(out) diff --git a/paddle/fluid/pir/dialect/operator/ir/ops_backward.yaml b/paddle/fluid/pir/dialect/operator/ir/ops_backward.yaml index 3c0ce287fe173..7b3068a8ab6c9 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops_backward.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops_backward.yaml @@ -412,6 +412,16 @@ func : matmul_grad backward : matmul_double_grad +- backward_op : matmul_with_flatten_grad + forward : matmul_with_flatten (Tensor x, Tensor y, int x_num_col_dims=1, int y_num_col_dims=1) -> Tensor(out) + args : (Tensor x, Tensor y, Tensor out_grad, int x_num_col_dims=1, int y_num_col_dims=1) + output : Tensor(x_grad), Tensor(y_grad) + infer_meta : + func : GeneralBinaryGradInferMeta + param : [x, y] + kernel : + func : matmul_with_flatten_grad + - backward_op : max_grad forward: max (Tensor x, IntArray axis={}, bool keepdim=false) -> Tensor(out) args : (Tensor x, Tensor out, Tensor out_grad, IntArray axis={}, bool keepdim=false, bool reduce_all=false) diff --git a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml index 4d3c49f4af840..4f9cc8706bf23 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml @@ -156,9 +156,11 @@ extra_args : str mkldnn_data_type="float32" data_format_tensors : x, y, out_grad -# - op : matmul_with_flatten +- op : matmul_with_flatten + extra_args : float scale_x=1.0, float[] scale_y={1.0}, float scale_out=1.0, bool force_fp32_output=false -# - op : matmul_with_flatten_grad +- op : matmul_with_flatten_grad + extra_args : float scale_x=1.0, float[] scale_y={1.0}, float scale_out=1.0, bool force_fp32_output=false - op : max dynamic_fallback : True diff --git a/test/mkldnn/test_mul_int8_mkldnn_op.py b/test/mkldnn/test_mul_int8_mkldnn_op.py index 75cb7eb9a604f..56b9966cbeaea 100644 --- a/test/mkldnn/test_mul_int8_mkldnn_op.py +++ b/test/mkldnn/test_mul_int8_mkldnn_op.py @@ -79,7 +79,7 @@ def init_data(self): def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode self.check_output_with_place( - core.CPUPlace(), atol=0, check_dygraph=False + core.CPUPlace(), atol=0, check_dygraph=False, check_pir_onednn=True ) diff --git a/test/mkldnn/test_mul_mkldnn_op.py b/test/mkldnn/test_mul_mkldnn_op.py index 68a8899b853ac..9759a581dbb4c 100644 --- a/test/mkldnn/test_mul_mkldnn_op.py +++ b/test/mkldnn/test_mul_mkldnn_op.py @@ -57,16 +57,28 @@ def init_inputs_dtype(self): pass def test_check_output(self): - self.check_output_with_place(core.CPUPlace()) + self.check_output_with_place( + core.CPUPlace(), check_pir_onednn=True, check_dygraph=False + ) def test_check_grad(self): - self.check_grad_with_place(core.CPUPlace(), ['X', 'Y'], 'Out') + self.check_grad_with_place( + core.CPUPlace(), + ['X', 'Y'], + 'Out', + check_pir_onednn=True, + check_dygraph=False, + ) def test_check_grad_ignore_x(self): - self.check_grad_with_place(core.CPUPlace(), ['Y'], 'Out', set('X')) + self.check_grad_with_place( + core.CPUPlace(), ['Y'], 'Out', set('X'), check_pir_onednn=True + ) def test_check_grad_ignore_y(self): - self.check_grad_with_place(core.CPUPlace(), ['X'], 'Out', set('Y')) + self.check_grad_with_place( + core.CPUPlace(), ['X'], 'Out', set('Y'), check_pir_onednn=True + ) class TestMulXNumColDims2OneDNNOp(TestMulOneDNNOp): @@ -135,6 +147,8 @@ def test_check_grad(self): 'Out', user_defined_grads=[self.dx, self.dy], user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + check_pir_onednn=True, + check_dygraph=False, ) def test_check_grad_ignore_x(self): @@ -146,6 +160,8 @@ def test_check_grad_ignore_x(self): set('X'), user_defined_grads=[self.dy], user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + check_pir_onednn=True, + check_dygraph=False, ) def test_check_grad_ignore_y(self): @@ -157,6 +173,8 @@ def test_check_grad_ignore_y(self): set('Y'), user_defined_grads=[self.dx], user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + check_pir_onednn=True, + check_dygraph=False, ) From e4b4aa2c562146b7dc084e7ef90116c8d87b3621 Mon Sep 17 00:00:00 2001 From: Difer <707065510@qq.com> Date: Mon, 26 Feb 2024 16:13:02 +0800 Subject: [PATCH 086/282] keep cluster exit consistency (#61898) --- python/paddle/distributed/launch/main.py | 28 +++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/python/paddle/distributed/launch/main.py b/python/paddle/distributed/launch/main.py index 80f082260d110..c1d99d49e9b63 100644 --- a/python/paddle/distributed/launch/main.py +++ b/python/paddle/distributed/launch/main.py @@ -1199,10 +1199,36 @@ def launch(): os.system("kill -9 " + pid) time.sleep(3) end_time = time.time() + + # keep cluster exit consistency + path = f"auto_tuner/exit/{job_id}/{ip}" if max_search_time and (end_time - start_time) > int( max_search_time ): - break + if nnodes > 1: + while not client.put(path, "error".encode('latin-1')): + time.sleep(1) + else: + break + else: + if nnodes > 1: + while not client.put(path, "ok".encode('latin-1')): + time.sleep(1) + + if nnodes > 1: + result = list(client.get_prefix(f"auto_tuner/exit/{job_id}")) + size = len(result) + while size != nnodes: + time.sleep(1) + result = list( + client.get_prefix(f"auto_tuner/exit/{job_id}/") + ) + size = len(result) + status = [i[0].decode() for i in result] + + if "error" in status: + break + recorder.store_history(history_file_path) # get best config to run From a73f982ba5d910f76fbb0cc1e7f9415accf62f40 Mon Sep 17 00:00:00 2001 From: wanghuancoder Date: Mon, 26 Feb 2024 16:19:33 +0800 Subject: [PATCH 087/282] pir onednn support concat (#62016) --- .../dialect/operator/ir/ops_onednn_extra.yaml | 6 ++++-- paddle/phi/kernels/onednn/concat_kernel.cc | 14 ++++++++++++- test/mkldnn/test_concat_bf16_mkldnn_op.py | 3 ++- test/mkldnn/test_concat_int8_mkldnn_op.py | 2 +- test/mkldnn/test_concat_mkldnn_op.py | 20 ++++++++++++++----- 5 files changed, 35 insertions(+), 10 deletions(-) diff --git a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml index 4f9cc8706bf23..7527a2f395e6a 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml @@ -37,9 +37,11 @@ - op : clip_grad extra_args : str mkldnn_data_type="float32" -# - op : concat +- op : concat + extra_args : bool use_quantizer=false, str mkldnn_data_type="float32" -# - op : concat_grad +- op : concat_grad + extra_args : bool use_quantizer=false, str mkldnn_data_type="float32" - op : conv2d extra_args : bool is_test=false diff --git a/paddle/phi/kernels/onednn/concat_kernel.cc b/paddle/phi/kernels/onednn/concat_kernel.cc index f3ff30e2fa861..c7c258ea88001 100644 --- a/paddle/phi/kernels/onednn/concat_kernel.cc +++ b/paddle/phi/kernels/onednn/concat_kernel.cc @@ -72,6 +72,16 @@ class ConcatOneDNNHandler : public OneDNNHandlerNoCachingT { }; } // namespace funcs +bool ConcatCheckIfOneDNNSupport(const KernelContext* ctx) { + auto input0 = ctx->InputAt(0); + int batch_size = + !input0.lod().empty() ? input0.lod()[0].size() - 1 : input0.dims()[0]; + if (ctx->InputsSize() > 64 && batch_size < 1000) { + return false; + } + return true; +} + static void EnforceLayouts(const std::vector inputs) { for (auto* input : inputs) { PADDLE_ENFORCE_EQ( @@ -151,4 +161,6 @@ PD_REGISTER_KERNEL(concat, float, phi::dtype::bfloat16, int8_t, - uint8_t) {} + uint8_t) { + kernel->check_if_onednn_kernel_support_ = phi::ConcatCheckIfOneDNNSupport; +} diff --git a/test/mkldnn/test_concat_bf16_mkldnn_op.py b/test/mkldnn/test_concat_bf16_mkldnn_op.py index 0e316b533ca02..82582b4c03370 100644 --- a/test/mkldnn/test_concat_bf16_mkldnn_op.py +++ b/test/mkldnn/test_concat_bf16_mkldnn_op.py @@ -52,7 +52,7 @@ def calculate_grads(self): self.dxs = np.split(self.dout, self.sections, self.axis) def test_check_output(self): - self.check_output_with_place(core.CPUPlace()) + self.check_output_with_place(core.CPUPlace(), check_pir_onednn=True) def test_check_grad(self): self.calculate_grads() @@ -62,6 +62,7 @@ def test_check_grad(self): "Out", user_defined_grads=[self.dxs[0], self.dxs[1], self.dxs[2]], user_defined_grad_outputs=[self.dout], + check_pir_onednn=True, ) # --------------------test concat bf16 in with axis 0-------------------- diff --git a/test/mkldnn/test_concat_int8_mkldnn_op.py b/test/mkldnn/test_concat_int8_mkldnn_op.py index 5ad5046f1c23e..546f6d4978f50 100644 --- a/test/mkldnn/test_concat_int8_mkldnn_op.py +++ b/test/mkldnn/test_concat_int8_mkldnn_op.py @@ -37,7 +37,7 @@ def setUp(self): def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_output(check_dygraph=False) + self.check_output(check_dygraph=False, check_pir_onednn=True) # --------------------test concat s8 in with axis 0-------------------- diff --git a/test/mkldnn/test_concat_mkldnn_op.py b/test/mkldnn/test_concat_mkldnn_op.py index 64c7747a00196..bc3400d8d0a77 100644 --- a/test/mkldnn/test_concat_mkldnn_op.py +++ b/test/mkldnn/test_concat_mkldnn_op.py @@ -47,12 +47,20 @@ def configure_datatype(self): self.dtype = np.float32 def test_check_output(self): - self.check_output_with_place(core.CPUPlace(), check_dygraph=False) + self.check_output_with_place( + core.CPUPlace(), check_dygraph=False, check_pir_onednn=True + ) def test_check_grad(self): - self.check_grad(['x0'], 'Out', check_dygraph=False) - self.check_grad(['x1'], 'Out', check_dygraph=False) - self.check_grad(['x2'], 'Out', check_dygraph=False) + self.check_grad( + ['x0'], 'Out', check_dygraph=False, check_pir_onednn=True + ) + self.check_grad( + ['x1'], 'Out', check_dygraph=False, check_pir_onednn=True + ) + self.check_grad( + ['x2'], 'Out', check_dygraph=False, check_pir_onednn=True + ) def init_test_data(self): self.x0 = np.random.random(self.x0_shape).astype(np.float32) @@ -124,7 +132,9 @@ def configure_datatype(self): self.dtype = np.float32 def test_check_output(self): - self.check_output_with_place(core.CPUPlace(), check_dygraph=False) + self.check_output_with_place( + core.CPUPlace(), check_dygraph=False, check_pir_onednn=True + ) def init_test_data(self): self.x = np.ones(self.shape).astype(np.float32) From 3be438979c4b8499f2243f75c779252a5b90108a Mon Sep 17 00:00:00 2001 From: Siming Dai <908660116@qq.com> Date: Mon, 26 Feb 2024 16:22:49 +0800 Subject: [PATCH 088/282] remove graph_reindex_test (#62058) --- test/legacy_test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/legacy_test/CMakeLists.txt b/test/legacy_test/CMakeLists.txt index 4d6f01a354329..2f729cc1f3b9d 100644 --- a/test/legacy_test/CMakeLists.txt +++ b/test/legacy_test/CMakeLists.txt @@ -555,9 +555,9 @@ if((NOT WITH_GPU) endif() list(REMOVE_ITEM TEST_OPS "test_stride") +list(REMOVE_ITEM TEST_OPS "test_graph_reindex") if(WITH_COVERAGE) list(REMOVE_ITEM TEST_OPS test_weight_decay) - list(REMOVE_ITEM TEST_OPS test_graph_reindex) list(REMOVE_ITEM TEST_OPS test_cuda_graphed_layer) list(REMOVE_ITEM TEST_OPS test_cuda_graph_partial_graph_static_run) list(REMOVE_ITEM DIST_TEST_OPS test_dist_fleet_geo) From 3538e34da10789b93419489d39720082f50d1e0a Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 26 Feb 2024 16:23:00 +0800 Subject: [PATCH 089/282] Update data_feed.cu (#62055) --- paddle/fluid/framework/data_feed.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/framework/data_feed.cu b/paddle/fluid/framework/data_feed.cu index a86ea409abeb4..a4d9de9f1b05a 100644 --- a/paddle/fluid/framework/data_feed.cu +++ b/paddle/fluid/framework/data_feed.cu @@ -490,7 +490,7 @@ int AcquireInstance(BufState *state) { if (state->GetNextStep()) { DEBUG_STATE(state); return state->len; - } else if (state->GetNextCentrolWord()) { + } else if (state->GetNextCentralWord()) { DEBUG_STATE(state); return state->len; } else if (state->GetNextBatch()) { From 255478f049e5061767dd59804f9b5d4dc2a83fa9 Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 26 Feb 2024 16:23:44 +0800 Subject: [PATCH 090/282] Fix end_patter_layernorms end_pattern_layernorms (#62047) --- .../ir/embedding_eltwise_layernorm_fuse_pass.cc | 10 +++++----- .../preln_embedding_eltwise_layernorm_fuse_pass.cc | 12 ++++++------ .../ir/trt_embedding_eltwise_layernorm_fuse_pass.cc | 12 ++++++------ ...t_tuning_embedding_eltwise_layernorm_fuse_pass.cc | 12 ++++++------ 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/paddle/fluid/framework/ir/embedding_eltwise_layernorm_fuse_pass.cc b/paddle/fluid/framework/ir/embedding_eltwise_layernorm_fuse_pass.cc index 1c2b35c691a08..68ec0492a42da 100644 --- a/paddle/fluid/framework/ir/embedding_eltwise_layernorm_fuse_pass.cc +++ b/paddle/fluid/framework/ir/embedding_eltwise_layernorm_fuse_pass.cc @@ -230,7 +230,7 @@ int EmbeddingEltwiseLayerNormFusePass::BuildFusion( std::vector end_pattern_scales; std::vector end_pattern_biases; std::vector end_pattern_out; - std::vector end_patter_layernorms; + std::vector end_pattern_layernorms; std::vector> end_pattern_remove_nodes; GraphPatternDetector gpd3; auto* pattern3 = gpd3.mutable_pattern(); @@ -264,7 +264,7 @@ int EmbeddingEltwiseLayerNormFusePass::BuildFusion( end_pattern_biases.push_back(layer_norm_bias); end_pattern_scales.push_back(layer_norm_scale); end_pattern_out.push_back(layer_norm_out); - end_patter_layernorms.push_back(layer_norm); + end_pattern_layernorms.push_back(layer_norm); }; gpd3(graph, handler3); @@ -376,13 +376,13 @@ int EmbeddingEltwiseLayerNormFusePass::BuildFusion( new_op_desc.SetInput("Scale", {end_pattern_scales[k]->Name()}); new_op_desc.SetOutput("Out", {end_pattern_out[k]->Name()}); new_op_desc.SetAttr("epsilon", - end_patter_layernorms[k]->Op()->GetAttr("epsilon")); + end_pattern_layernorms[k]->Op()->GetAttr("epsilon")); - if (end_patter_layernorms[k]->Op()->HasAttr("out_threshold")) { + if (end_pattern_layernorms[k]->Op()->HasAttr("out_threshold")) { new_op_desc.SetAttr("enable_int8", true); new_op_desc.SetAttr( "out_threshold", - end_patter_layernorms[k]->Op()->GetAttr("out_threshold")); + end_pattern_layernorms[k]->Op()->GetAttr("out_threshold")); } auto* embedding_eltwise_layernorm = graph->CreateOpNode(&new_op_desc); diff --git a/paddle/fluid/framework/ir/preln_embedding_eltwise_layernorm_fuse_pass.cc b/paddle/fluid/framework/ir/preln_embedding_eltwise_layernorm_fuse_pass.cc index 5f8113a169959..1734e7d675755 100644 --- a/paddle/fluid/framework/ir/preln_embedding_eltwise_layernorm_fuse_pass.cc +++ b/paddle/fluid/framework/ir/preln_embedding_eltwise_layernorm_fuse_pass.cc @@ -240,7 +240,7 @@ int PrelnEmbeddingEltwiseLayerNormFusePass::BuildFusion( std::vector end_pattern_scales; std::vector end_pattern_biases; std::vector end_pattern_out; - std::vector end_patter_layernorms; + std::vector end_pattern_layernorms; std::vector end_patter_elementwise; std::vector> end_pattern_remove_nodes; GraphPatternDetector gpd3; @@ -275,7 +275,7 @@ int PrelnEmbeddingEltwiseLayerNormFusePass::BuildFusion( end_pattern_biases.push_back(layer_norm_bias); end_pattern_scales.push_back(layer_norm_scale); end_pattern_out.push_back(layer_norm_out); - end_patter_layernorms.push_back(layer_norm); + end_pattern_layernorms.push_back(layer_norm); end_patter_elementwise.push_back(eltwise_add); }; gpd3(graph, handler3); @@ -328,7 +328,7 @@ int PrelnEmbeddingEltwiseLayerNormFusePass::BuildFusion( embs.push_back(inner_pattern_ins[item].second->Name()); } - OpDesc new_op_desc(end_patter_layernorms[0]->Op()->Block()); + OpDesc new_op_desc(end_pattern_layernorms[0]->Op()->Block()); new_op_desc.SetType("fused_preln_embedding_eltwise_layernorm"); new_op_desc.SetInput("Ids", ids); new_op_desc.SetInput("Embs", embs); @@ -340,14 +340,14 @@ int PrelnEmbeddingEltwiseLayerNormFusePass::BuildFusion( new_op_desc.SetOutput("Out_0", {end_pattern_out[k]->Name()}); new_op_desc.SetOutput("Out_1", {inner_pattern_out[k]->Name()}); new_op_desc.SetAttr("epsilon", - end_patter_layernorms[k]->Op()->GetAttr("epsilon")); + end_pattern_layernorms[k]->Op()->GetAttr("epsilon")); - if (end_patter_layernorms[k]->Op()->HasAttr("out_threshold") && + if (end_pattern_layernorms[k]->Op()->HasAttr("out_threshold") && end_patter_elementwise[k]->Op()->HasAttr("out_threshold")) { new_op_desc.SetAttr("enable_int8", true); new_op_desc.SetAttr( "out_0_threshold", - end_patter_layernorms[k]->Op()->GetAttr("out_threshold")); + end_pattern_layernorms[k]->Op()->GetAttr("out_threshold")); new_op_desc.SetAttr( "out_1_threshold", end_patter_elementwise[k]->Op()->GetAttr("out_threshold")); diff --git a/paddle/fluid/framework/ir/trt_embedding_eltwise_layernorm_fuse_pass.cc b/paddle/fluid/framework/ir/trt_embedding_eltwise_layernorm_fuse_pass.cc index 6b761b68e8c2f..e07073d64042b 100644 --- a/paddle/fluid/framework/ir/trt_embedding_eltwise_layernorm_fuse_pass.cc +++ b/paddle/fluid/framework/ir/trt_embedding_eltwise_layernorm_fuse_pass.cc @@ -234,7 +234,7 @@ int TrtEmbeddingEltwiseLayerNormFusePass::BuildFusion( std::vector end_pattern_scales; std::vector end_pattern_biases; std::vector end_pattern_out; - std::vector end_patter_layernorms; + std::vector end_pattern_layernorms; std::vector> end_pattern_remove_nodes; GraphPatternDetector gpd3; auto* pattern3 = gpd3.mutable_pattern(); @@ -268,7 +268,7 @@ int TrtEmbeddingEltwiseLayerNormFusePass::BuildFusion( end_pattern_biases.push_back(layer_norm_bias); end_pattern_scales.push_back(layer_norm_scale); end_pattern_out.push_back(layer_norm_out); - end_patter_layernorms.push_back(layer_norm); + end_pattern_layernorms.push_back(layer_norm); }; gpd3(graph, handler3); @@ -354,7 +354,7 @@ int TrtEmbeddingEltwiseLayerNormFusePass::BuildFusion( } if (flag) { - OpDesc new_op_desc(end_patter_layernorms[0]->Op()->Block()); + OpDesc new_op_desc(end_pattern_layernorms[0]->Op()->Block()); new_op_desc.SetType("fused_embedding_eltwise_layernorm"); new_op_desc.SetInput("Ids", ids); new_op_desc.SetInput("Embs", embs); @@ -366,13 +366,13 @@ int TrtEmbeddingEltwiseLayerNormFusePass::BuildFusion( new_op_desc.SetInput("Scale", {end_pattern_scales[k]->Name()}); new_op_desc.SetOutput("Out", {end_pattern_out[k]->Name()}); new_op_desc.SetAttr("epsilon", - end_patter_layernorms[k]->Op()->GetAttr("epsilon")); + end_pattern_layernorms[k]->Op()->GetAttr("epsilon")); - if (end_patter_layernorms[k]->Op()->HasAttr("out_threshold")) { + if (end_pattern_layernorms[k]->Op()->HasAttr("out_threshold")) { new_op_desc.SetAttr("enable_int8", true); new_op_desc.SetAttr( "out_threshold", - end_patter_layernorms[k]->Op()->GetAttr("out_threshold")); + end_pattern_layernorms[k]->Op()->GetAttr("out_threshold")); } auto* embedding_eltwise_layernorm = graph->CreateOpNode(&new_op_desc); diff --git a/paddle/fluid/framework/ir/trt_prompt_tuning_embedding_eltwise_layernorm_fuse_pass.cc b/paddle/fluid/framework/ir/trt_prompt_tuning_embedding_eltwise_layernorm_fuse_pass.cc index 6bdd56dff2087..f93a42a7dbab8 100644 --- a/paddle/fluid/framework/ir/trt_prompt_tuning_embedding_eltwise_layernorm_fuse_pass.cc +++ b/paddle/fluid/framework/ir/trt_prompt_tuning_embedding_eltwise_layernorm_fuse_pass.cc @@ -300,7 +300,7 @@ int TrtPromptTuningEmbeddingEltwiseLayerNormFusePass::BuildFusion( std::vector end_pattern_scales; std::vector end_pattern_biases; std::vector end_pattern_out; - std::vector end_patter_layernorms; + std::vector end_pattern_layernorms; std::vector> end_pattern_remove_nodes; GraphPatternDetector gpd3; auto* pattern3 = gpd3.mutable_pattern(); @@ -342,7 +342,7 @@ int TrtPromptTuningEmbeddingEltwiseLayerNormFusePass::BuildFusion( end_pattern_biases.push_back(layer_norm_bias); end_pattern_scales.push_back(layer_norm_scale); end_pattern_out.push_back(layer_norm_out); - end_patter_layernorms.push_back(layer_norm); + end_pattern_layernorms.push_back(layer_norm); }; gpd3(graph, handler3); @@ -428,7 +428,7 @@ int TrtPromptTuningEmbeddingEltwiseLayerNormFusePass::BuildFusion( } if (flag) { - OpDesc new_op_desc(end_patter_layernorms[0]->Op()->Block()); + OpDesc new_op_desc(end_pattern_layernorms[0]->Op()->Block()); new_op_desc.SetType("prompt_tuning_emb_eltwise_layernorm"); new_op_desc.SetInput("Ids", ids); new_op_desc.SetInput("Embs", embs); @@ -440,13 +440,13 @@ int TrtPromptTuningEmbeddingEltwiseLayerNormFusePass::BuildFusion( new_op_desc.SetInput("DenseVector", {end_pattern_eltadd1_out[k]->Name()}); new_op_desc.SetOutput("Out", {end_pattern_out[k]->Name()}); new_op_desc.SetAttr("epsilon", - end_patter_layernorms[k]->Op()->GetAttr("epsilon")); + end_pattern_layernorms[k]->Op()->GetAttr("epsilon")); - if (end_patter_layernorms[k]->Op()->HasAttr("out_threshold")) { + if (end_pattern_layernorms[k]->Op()->HasAttr("out_threshold")) { new_op_desc.SetAttr("enable_int8", true); new_op_desc.SetAttr( "out_threshold", - end_patter_layernorms[k]->Op()->GetAttr("out_threshold")); + end_pattern_layernorms[k]->Op()->GetAttr("out_threshold")); } auto* embedding_eltwise_layernorm = graph->CreateOpNode(&new_op_desc); From 9d1196cf1c57008cbd6b248d17099426ea035b65 Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 26 Feb 2024 16:26:10 +0800 Subject: [PATCH 091/282] Fix some typos (distribuion, funciotn, etc.) (#62000) --- python/paddle/distribution/binomial.py | 2 +- .../distribution/continuous_bernoulli.py | 2 +- python/paddle/distribution/dirichlet.py | 4 ++-- python/paddle/distribution/exponential.py | 4 ++-- .../paddle/distribution/exponential_family.py | 2 +- python/paddle/distribution/gamma.py | 4 ++-- python/paddle/distribution/geometric.py | 4 ++-- python/paddle/distribution/kl.py | 16 ++++++------- python/paddle/distribution/lognormal.py | 2 +- python/paddle/distribution/multinomial.py | 2 +- .../distribution/multivariate_normal.py | 24 +++++++++---------- python/paddle/distribution/normal.py | 2 +- python/paddle/distribution/poisson.py | 2 +- python/paddle/distribution/transform.py | 20 ++++++++-------- 14 files changed, 45 insertions(+), 45 deletions(-) diff --git a/python/paddle/distribution/binomial.py b/python/paddle/distribution/binomial.py index 9bf5ec41faaad..c14899e01ea7a 100644 --- a/python/paddle/distribution/binomial.py +++ b/python/paddle/distribution/binomial.py @@ -116,7 +116,7 @@ def _check_constraint(self, total_count, probs): @property def mean(self): - """Mean of binomial distribuion. + """Mean of binomial distribution. Returns: Tensor: mean value. diff --git a/python/paddle/distribution/continuous_bernoulli.py b/python/paddle/distribution/continuous_bernoulli.py index 1df7653f0103a..3fd0335fc4076 100644 --- a/python/paddle/distribution/continuous_bernoulli.py +++ b/python/paddle/distribution/continuous_bernoulli.py @@ -210,7 +210,7 @@ def _log_constant(self): @property def mean(self): - """Mean of Continuous Bernoulli distribuion. + """Mean of Continuous Bernoulli distribution. Returns: Tensor: mean value. diff --git a/python/paddle/distribution/dirichlet.py b/python/paddle/distribution/dirichlet.py index 4a0dd3a46d63b..8560a8e6f0e04 100644 --- a/python/paddle/distribution/dirichlet.py +++ b/python/paddle/distribution/dirichlet.py @@ -51,7 +51,7 @@ class Dirichlet(exponential_family.ExponentialFamily): distribution, also called :math:`\alpha`. When it's over one dimension, the last axis denotes the parameter of distribution, ``event_shape=concentration.shape[-1:]`` , axes other than last are - condsider batch dimensions with ``batch_shape=concentration.shape[:-1]`` . + consider batch dimensions with ``batch_shape=concentration.shape[:-1]`` . Examples: @@ -79,7 +79,7 @@ def __init__(self, concentration): @property def mean(self): - """Mean of Dirichelt distribution. + """Mean of Dirichlet distribution. Returns: Mean value of distribution. diff --git a/python/paddle/distribution/exponential.py b/python/paddle/distribution/exponential.py index abf88bc3c4c37..943850ea6e176 100644 --- a/python/paddle/distribution/exponential.py +++ b/python/paddle/distribution/exponential.py @@ -83,7 +83,7 @@ def __init__(self, rate): @property def mean(self): - """Mean of exponential distribuion. + """Mean of exponential distribution. Returns: Tensor: mean value. @@ -134,7 +134,7 @@ def rsample(self, shape=()): return -paddle.log(uniform) / self.rate def prob(self, value): - r"""Probability density funciotn evaluated at value. + r"""Probability density function evaluated at value. .. math:: diff --git a/python/paddle/distribution/exponential_family.py b/python/paddle/distribution/exponential_family.py index fbe7e71ebb1e9..2e668501acdab 100644 --- a/python/paddle/distribution/exponential_family.py +++ b/python/paddle/distribution/exponential_family.py @@ -48,7 +48,7 @@ def _mean_carrier_measure(self): raise NotImplementedError def entropy(self): - """caculate entropy use `bregman divergence` + """calculate entropy use `bregman divergence` https://www.lix.polytechnique.fr/~nielsen/EntropyEF-ICIP2010.pdf """ entropy_value = -self._mean_carrier_measure diff --git a/python/paddle/distribution/gamma.py b/python/paddle/distribution/gamma.py index e1ae3a1f63658..696f77d926e39 100644 --- a/python/paddle/distribution/gamma.py +++ b/python/paddle/distribution/gamma.py @@ -113,7 +113,7 @@ def __init__(self, concentration, rate): @property def mean(self): - """Mean of gamma distribuion. + """Mean of gamma distribution. Returns: Tensor: mean value. @@ -130,7 +130,7 @@ def variance(self): return self.concentration / self.rate.pow(2) def prob(self, value): - """Probability density funciotn evaluated at value + """Probability density function evaluated at value Args: value (float|Tensor): Value to be evaluated. diff --git a/python/paddle/distribution/geometric.py b/python/paddle/distribution/geometric.py index 6df855b168143..e4ba916fb58d2 100644 --- a/python/paddle/distribution/geometric.py +++ b/python/paddle/distribution/geometric.py @@ -124,7 +124,7 @@ def stddev(self): return paddle.sqrt(self.variance) def pmf(self, k): - r"""Probability mass funciotn evaluated at k. + r"""Probability mass function evaluated at k. .. math:: @@ -341,5 +341,5 @@ def kl_divergence(self, other): ) else: raise TypeError( - f"Exected type of other is geometric.Geometric, but got {type(other)}" + f"Exacted type of other is geometric.Geometric, but got {type(other)}" ) diff --git a/python/paddle/distribution/kl.py b/python/paddle/distribution/kl.py index 44474b4ab5f79..64b8f568b08db 100644 --- a/python/paddle/distribution/kl.py +++ b/python/paddle/distribution/kl.py @@ -73,13 +73,13 @@ def kl_divergence(p, q): def register_kl(cls_p, cls_q): - """Decorator for register a KL divergence implemention function. + """Decorator for register a KL divergence implementation function. - The ``kl_divergence(p, q)`` function will search concrete implemention + The ``kl_divergence(p, q)`` function will search concrete implementation functions registered by ``register_kl``, according to multi-dispatch pattern. - If an implemention function is found, it will return the result, otherwise, + If an implementation function is found, it will return the result, otherwise, it will raise ``NotImplementError`` exception. Users can register - implemention function by the decorator. + implementation function by the decorator. Args: cls_p (Distribution): The Distribution type of Instance p. Subclass derived from ``Distribution``. @@ -110,16 +110,16 @@ def _dispatch(cls_p, cls_q): """Multiple dispatch into concrete implement function.""" # find all matched super class pair of p and q - matchs = [ + matches = [ (super_p, super_q) for super_p, super_q in _REGISTER_TABLE if issubclass(cls_p, super_p) and issubclass(cls_q, super_q) ] - if not matchs: + if not matches: raise NotImplementedError - left_p, left_q = min(_Compare(*m) for m in matchs).classes - right_p, right_q = min(_Compare(*reversed(m)) for m in matchs).classes + left_p, left_q = min(_Compare(*m) for m in matches).classes + right_p, right_q = min(_Compare(*reversed(m)) for m in matches).classes if _REGISTER_TABLE[left_p, left_q] is not _REGISTER_TABLE[right_p, right_q]: warnings.warn( diff --git a/python/paddle/distribution/lognormal.py b/python/paddle/distribution/lognormal.py index 98f9f5670eb3e..bcf4a103ee756 100644 --- a/python/paddle/distribution/lognormal.py +++ b/python/paddle/distribution/lognormal.py @@ -97,7 +97,7 @@ def __init__(self, loc, scale): @property def mean(self): - """Mean of lognormal distribuion. + """Mean of lognormal distribution. Returns: Tensor: mean value. diff --git a/python/paddle/distribution/multinomial.py b/python/paddle/distribution/multinomial.py index f89e773373d4e..7cc44ccece36d 100644 --- a/python/paddle/distribution/multinomial.py +++ b/python/paddle/distribution/multinomial.py @@ -87,7 +87,7 @@ def __init__(self, total_count, probs): @property def mean(self): - """mean of multinomial distribuion. + """mean of multinomial distribution. Returns: Tensor: mean value. diff --git a/python/paddle/distribution/multivariate_normal.py b/python/paddle/distribution/multivariate_normal.py index 7e8e0ccf513a8..a576b3357d9ed 100644 --- a/python/paddle/distribution/multivariate_normal.py +++ b/python/paddle/distribution/multivariate_normal.py @@ -150,7 +150,7 @@ def __init__( batch_shape + [precision_matrix.shape[-2], precision_matrix.shape[-1]] ) - self._check_constriants() + self._check_constraints() self.loc = loc.expand( batch_shape + [ @@ -223,16 +223,16 @@ def _check_positive_definite(self, value): raise ValueError( "covariance_matrix or precision_matrix must be a symmetric matrix" ) - is_postive_definite = ( + is_positive_definite = ( paddle.cast(paddle.linalg.eigvalsh(value), dtype=self.dtype) > 0 ).all() - return is_postive_definite + return is_positive_definite - def _check_constriants(self): - """Check whether the matrix satisfy corresponding constriant + def _check_constraints(self): + """Check whether the matrix satisfy corresponding constraint Return: - Tensor: indicator for the pass of constriants check + Tensor: indicator for the pass of constraints check """ if self.scale_tril is not None: check = self._check_lower_triangular(self.scale_tril) @@ -241,25 +241,25 @@ def _check_constriants(self): "scale_tril matrix must be a lower triangular matrix with positive diagonals" ) elif self.covariance_matrix is not None: - is_postive_definite = self._check_positive_definite( + is_positive_definite = self._check_positive_definite( self.covariance_matrix ) - if not is_postive_definite: + if not is_positive_definite: raise ValueError( "covariance_matrix must be a symmetric positive definite matrix" ) else: - is_postive_definite = self._check_positive_definite( + is_positive_definite = self._check_positive_definite( self.precision_matrix ) - if not is_postive_definite: + if not is_positive_definite: raise ValueError( "precision_matrix must be a symmetric positive definite matrix" ) @property def mean(self): - """Mean of Multivariate Normal distribuion. + """Mean of Multivariate Normal distribution. Returns: Tensor: mean value. @@ -451,7 +451,7 @@ def precision_to_scale_tril(P): def batch_mahalanobis(bL, bx): r""" - Computes the squared Mahalanobis distance of the Multivariate Normal distribution with cholesky decomposition of the covatiance matrix. + Computes the squared Mahalanobis distance of the Multivariate Normal distribution with cholesky decomposition of the covariance matrix. Accepts batches for both bL and bx. Args: diff --git a/python/paddle/distribution/normal.py b/python/paddle/distribution/normal.py index a985556a5c7fd..3ac8282a4a2b0 100644 --- a/python/paddle/distribution/normal.py +++ b/python/paddle/distribution/normal.py @@ -159,7 +159,7 @@ def __init__(self, loc, scale, name=None): @property def mean(self): - """Mean of normal distribuion. + """Mean of normal distribution. Returns: Tensor: mean value. diff --git a/python/paddle/distribution/poisson.py b/python/paddle/distribution/poisson.py index 4cd50962f085d..a160b44b9ccf6 100644 --- a/python/paddle/distribution/poisson.py +++ b/python/paddle/distribution/poisson.py @@ -112,7 +112,7 @@ def _check_constraint(self, value): @property def mean(self): - """Mean of poisson distribuion. + """Mean of poisson distribution. Returns: Tensor: mean value. diff --git a/python/paddle/distribution/transform.py b/python/paddle/distribution/transform.py index cdb612aea2c0f..230d0e225fea1 100644 --- a/python/paddle/distribution/transform.py +++ b/python/paddle/distribution/transform.py @@ -131,7 +131,7 @@ def _is_injective(cls): def __call__(self, input): """Make this instance as a callable object. The return value is - depening on the input type. + depending on the input type. * If the input is a ``Tensor`` instance, return ``self.forward(input)`` . @@ -160,7 +160,7 @@ def forward(self, x): Useful for turning one random outcome into another. Args: - x (Tensos): Input parameter, generally is a sample generated + x (Tensor): Input parameter, generally is a sample generated from ``Distribution``. Returns: @@ -299,7 +299,7 @@ def _codomain(self): return variable.real def _forward(self, x): - """Inner method for publid API ``forward``, subclass should + """Inner method for public API ``forward``, subclass should overwrite this method for supporting forward transformation. """ raise NotImplementedError('Forward not implemented') @@ -340,8 +340,8 @@ def _forward_shape(self, shape): return shape def _inverse_shape(self, shape): - """Inner method called by ``inverse_shape``, whic is used to infer the - invese shape. Subclass should overwrite this method for supporting + """Inner method called by ``inverse_shape``, which is used to infer the + inverse shape. Subclass should overwrite this method for supporting ``inverse_shape``. """ return shape @@ -355,7 +355,7 @@ class AbsTransform(Transform): distributions with the absolute value function, which maps ``(-inf, inf)`` to ``[0, inf)`` . - * For ``y`` in ``(0, inf)`` , ``AbsTransform.inverse(y)`` returns the set invese + * For ``y`` in ``(0, inf)`` , ``AbsTransform.inverse(y)`` returns the set inverse ``{x in (-inf, inf) : |x| = y}`` as a tuple, ``-y, y`` . * For ``y`` equal ``0`` , ``AbsTransform.inverse(0)`` returns ``0, 0``, which is not the set inverse (the set inverse is the singleton {0}), but "works" in @@ -681,13 +681,13 @@ class IndependentTransform(Transform): some of the rightmost batch axes as event axes. Generally, it is used to expand the event axes. This has no effect on the - forward or inverse transformaion, but does sum out the - ``reinterpretd_bach_rank`` rightmost dimensions in computing the determinant + forward or inverse transformation, but does sum out the + ``reinterpreted_batch_rank`` rightmost dimensions in computing the determinant of Jacobian matrix. To see this, consider the ``ExpTransform`` applied to a Tensor which has sample, batch, and event ``(S,B,E)`` shape semantics. Suppose the Tensor's - paritioned-shape is ``(S=[4], B=[2, 2], E=[3])`` , reinterpreted_batch_rank + partitioned-shape is ``(S=[4], B=[2, 2], E=[3])`` , reinterpreted_batch_rank is 1. Then the reinterpreted Tensor's shape is ``(S=[4], B=[2], E=[2, 3])`` . The shape returned by ``forward`` and ``inverse`` is unchanged, ie, ``[4,2,2,3]`` . However the shape returned by ``inverse_log_det_jacobian`` @@ -875,7 +875,7 @@ def __init__(self, in_event_shape, out_event_shape): ): raise TypeError( f"Expected type of 'in_event_shape' and 'out_event_shape' is " - f"Squence[int], but got 'in_event_shape': {in_event_shape}, " + f"Sequence[int], but got 'in_event_shape': {in_event_shape}, " f"'out_event_shape': {out_event_shape}" ) in_size = 1 From 1a625d8bd05a00e392010709af14c811a69531ad Mon Sep 17 00:00:00 2001 From: Difer <707065510@qq.com> Date: Mon, 26 Feb 2024 16:27:32 +0800 Subject: [PATCH 092/282] [AutoConfig]add jobid in logdir name (#61895) * add jobid in logdir name * fix id from 0 --- python/paddle/distributed/launch/main.py | 31 ++++++++++++++---------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/python/paddle/distributed/launch/main.py b/python/paddle/distributed/launch/main.py index c1d99d49e9b63..ee4987e22888f 100644 --- a/python/paddle/distributed/launch/main.py +++ b/python/paddle/distributed/launch/main.py @@ -461,9 +461,16 @@ def launch(): gbs_cur_cfg = gbs_tuner.search_once() best_gbs = None + + # every task has own job id + job_id += 1 + task_job_id = "gbs_tuner_" + str(job_id) + ctx.args.job_id = task_job_id + while gbs_cur_cfg: ctx = copy.deepcopy(raw_ctx) - log_dir = "GBSSearch/GBS{}_DP{}_MP{}_PP{}_Sharding_degree_{}_stage_{}_MBS{}_Recompute_{}_granularity_{}".format( + log_dir = "Job{}_GBSSearch/GBS{}_DP{}_MP{}_PP{}_Sharding_degree_{}_stage_{}_MBS{}_Recompute_{}_granularity_{}".format( + job_id, gbs_cur_cfg["global_batch_size"], gbs_cur_cfg["dp_degree"], gbs_cur_cfg["mp_degree"], @@ -476,11 +483,6 @@ def launch(): ) ctx.args.log_dir = log_dir - # every task has own job id - job_id += 1 - task_job_id = "gbs_tuner_" + str(job_id) - ctx.args.job_id = task_job_id - # generate script args of task gbs_new_args = gen_new_args( raw_args, gbs_cur_cfg, gbs_tuner_cfg @@ -620,8 +622,15 @@ def launch(): ) cur_cfg["acc_steps"] = acc_steps cur_cfg["global_batch_size"] = global_batch_size + + # every task has own job id + job_id += 1 + task_job_id = "auto_tuner_" + str(job_id) + ctx.args.job_id = task_job_id + if "sharding_overlap" in cur_cfg: - log_dir = "GBS{}_DP{}_MP{}_PP{}_VPP{}_Sharding{}_Stage{}_MBS{}_Recompute_{}_Granularity_{}_AccStep{}_Overlap_{}".format( + log_dir = "Job{}_GBS{}_DP{}_MP{}_PP{}_VPP{}_Sharding{}_Stage{}_MBS{}_Recompute_{}_Granularity_{}_AccStep{}_Overlap_{}".format( + job_id, global_batch_size, cur_cfg["dp_degree"], cur_cfg["mp_degree"], @@ -636,7 +645,8 @@ def launch(): cur_cfg["sharding_overlap"], ) else: - log_dir = "GBS{}_DP{}_MP{}_PP{}_VPP{}_Sharding{}_Stage{}_MBS{}_Recompute_{}_Granularity_{}_AccStep{}".format( + log_dir = "Job{}_GBS{}_DP{}_MP{}_PP{}_VPP{}_Sharding{}_Stage{}_MBS{}_Recompute_{}_Granularity_{}_AccStep{}".format( + job_id, global_batch_size, cur_cfg["dp_degree"], cur_cfg["mp_degree"], @@ -653,11 +663,6 @@ def launch(): os.path.dirname(ctx.args.auto_tuner_json), log_dir ) - # every task has own job id - job_id += 1 - task_job_id = "auto_tuner_" + str(job_id) - ctx.args.job_id = task_job_id - # generate script args of task new_args = gen_new_args(raw_args, cur_cfg, tuner_cfg) ctx.args.training_script_args = new_args From 2a472d9ad1d67d63d89bc79b9e27e880387e48f8 Mon Sep 17 00:00:00 2001 From: winter-wang <78149749+winter-wang@users.noreply.github.com> Date: Mon, 26 Feb 2024 16:41:29 +0800 Subject: [PATCH 093/282] [PIR] fix bug in dy2st split program. (#62030) --- .../pir_adaptor/pir_adaptor_util.cc | 12 +++--- paddle/fluid/pybind/pir.cc | 43 +++++++++++++------ .../include/dialect/control_flow/ir/cf_op.h | 2 + paddle/pir/src/core/ir_printer.cc | 19 ++++++-- paddle/pir/src/core/value_impl.cc | 9 ++-- .../pir/src/dialect/control_flow/ir/cf_op.cc | 16 +++++-- .../jit/dy2static/pir_partial_program.py | 8 ++++ 7 files changed, 77 insertions(+), 32 deletions(-) diff --git a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc index a56868482223a..1e2fa3269bb41 100644 --- a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc +++ b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc @@ -556,11 +556,13 @@ void HandleForSpecialOp(pir::Operation* op, auto value = op->operand_source(0); Scope* scope = const_cast(value_exe_info->GetScope()); - if (value.defining_op()->HasAttribute(kAttrIsPersistable) && - value.attribute(kAttrIsPersistable).data()) { - VLOG(6) << "Handle for builtin.shadow_output persistable value:" - << var_name; - scope = const_cast(value_exe_info->GetScope()->root()); + if (auto bool_atttr = + value.attribute(kAttrIsPersistable)) { + if (bool_atttr.data()) { + VLOG(6) << "Handle for builtin.shadow_ouptut persistable value:" + << var_name; + scope = const_cast(value_exe_info->GetScope()->root()); + } } // change operand name to param_name diff --git a/paddle/fluid/pybind/pir.cc b/paddle/fluid/pybind/pir.cc index 723ff501450c0..35c19c8f00c76 100644 --- a/paddle/fluid/pybind/pir.cc +++ b/paddle/fluid/pybind/pir.cc @@ -74,6 +74,7 @@ #include "paddle/pir/include/core/program.h" #include "paddle/pir/include/core/type.h" #include "paddle/pir/include/core/value.h" +#include "paddle/pir/include/core/visitors.h" #include "paddle/pir/include/dialect/control_flow/ir/cf_dialect.h" #include "paddle/pir/include/dialect/shape/ir/shape_attribute.h" #include "paddle/pir/include/dialect/shape/ir/shape_dialect.h" @@ -749,6 +750,12 @@ void BindValue(py::module *m) { } else if (auto data_op = self.defining_op()) { return data_op.attribute("name").AsString(); + } else if (auto block_arg = self.dyn_cast()) { + if (block_arg.is_kwarg()) { + return block_arg.keyword(); + } else { + return "arg_" + std::to_string(block_arg.index()); + } } else { PADDLE_THROW(phi::errors::InvalidArgument( "Currently, we can only get name of Value that " @@ -952,9 +959,11 @@ AnalysisMiddleVariable(const Program &program, forward_inputs.end()); range_block_do( program.block(), backward_range, [&backward_inputs](Operation *op) { - for (auto &t : op->operands()) { - backward_inputs.insert(t.source()); - } + pir::Walk(op, [&](Operation *inner_op) { + for (auto &t : inner_op->operands()) { + backward_inputs.insert(t.source()); + } + }); }); range_block_do( @@ -1215,28 +1224,34 @@ SplitedResult SplitForwardBackward( // counter = 0; if (has_backward) { - VLOG(4) << "start create backward inputs, inserting pd.data ops."; - VLOG(4) << "Create pd.data for backward program: fo, start with input_" - << counter; + VLOG(4) << "start create backward inputs, creating keyword argument."; + VLOG(4) + << "Create keyword argument for backward program: fo, start with input_" + << counter; std::for_each( forward_outputs.begin(), forward_outputs.end(), create_kwarg_fn); - VLOG(4) << "Create pd.data for backward program: fx, start with input_" - << counter; + VLOG(4) + << "Create keyword argument for backward program: fx, start with input_" + << counter; std::for_each( forward_inputs.begin(), forward_inputs.end(), create_kwarg_fn); - VLOG(4) << "Create pd.data for backward program: fp, start with input_" - << counter; + VLOG(4) + << "Create keyword argument for backward program: fp, start with input_" + << counter; std::for_each( forward_params.begin(), forward_params.end(), create_kwarg_fn); - VLOG(4) << "Create pd.data for backward program: fm, start with input_" - << counter; + VLOG(4) + << "Create keyword argument for backward program: fm, start with input_" + << counter; std::for_each(middle_values.begin(), middle_values.end(), create_kwarg_fn); - VLOG(4) << "Create pd.data for backward program: fo_g, start with input_" + VLOG(4) << "Create keyword argument for backward program: fo_g, start with " + "input_" << counter; std::for_each(forward_outputs_grads.begin(), forward_outputs_grads.end(), create_kwarg_fn); - VLOG(4) << "Create pd.data for backward program end. input_" << counter; + VLOG(4) << "Create keyword argument for backward program end. input_" + << counter; } // counter = 0; diff --git a/paddle/pir/include/dialect/control_flow/ir/cf_op.h b/paddle/pir/include/dialect/control_flow/ir/cf_op.h index ed3e51df121c4..f56c920bec5fb 100644 --- a/paddle/pir/include/dialect/control_flow/ir/cf_op.h +++ b/paddle/pir/include/dialect/control_flow/ir/cf_op.h @@ -57,6 +57,7 @@ class IR_API TuplePushOp : public Op { Value inlet, std::initializer_list element_list); void VerifySig(); + void VerifyRegion(); Value container() { return container_interface().container(); } Value inlet() { return operand_source(0); } @@ -84,6 +85,7 @@ class IR_API TuplePopOp : public Op { OperationArgument &argument, // NOLINT Value outlet); void VerifySig(); + void VerifyRegion(); Value container() { return container_interface().container(); } Value inlet() { return container_interface().inlet(); } diff --git a/paddle/pir/src/core/ir_printer.cc b/paddle/pir/src/core/ir_printer.cc index c1a0fcd905ac8..de75d6d2fc603 100644 --- a/paddle/pir/src/core/ir_printer.cc +++ b/paddle/pir/src/core/ir_printer.cc @@ -216,6 +216,16 @@ void IrPrinter::PrintRegion(const Region& region) { void IrPrinter::PrintBlock(const Block& block) { os << indentation() << "{\n"; AddIndentation(); + if (!block.kwargs_empty()) { + os << indentation() << "^kw:"; + auto cur = block.kwargs_begin(), end = block.kwargs_end(); + PrintValue(cur->second); + while (++cur != end) { + os << ", "; + PrintValue(cur->second); + } + os << "\n"; + } for (auto& item : block) { PrintOperation(&item); os << "\n"; @@ -241,10 +251,11 @@ void IrPrinter::PrintValue(Value v) { aliases_[key] = new_name; os << new_name; } else { - std::string new_name = "%arg" + std::to_string(cur_block_argument_number_); - cur_block_argument_number_++; - aliases_[key] = new_name; - os << new_name; + auto arg = v.dyn_cast(); + os << (aliases_[key] = + arg.is_kwarg() + ? "%kwarg_" + arg.keyword() + : "%arg_" + std::to_string(cur_block_argument_number_++)); } } diff --git a/paddle/pir/src/core/value_impl.cc b/paddle/pir/src/core/value_impl.cc index 8c55a50ac7946..37dcb48370b6e 100644 --- a/paddle/pir/src/core/value_impl.cc +++ b/paddle/pir/src/core/value_impl.cc @@ -27,9 +27,9 @@ void ValueImpl::set_first_use(OpOperandImpl *first_use) { uint32_t offset = kind(); first_use_offseted_by_kind_ = reinterpret_cast( reinterpret_cast(first_use) + offset); - VLOG(10) << "The index of this value is " << offset - << ". Offset and set first use: " << first_use << " -> " - << first_use_offseted_by_kind_ << "."; + VLOG(10) << "The index of this value is: " << offset + << ". The address of this value is: " << this + << ". This value first use is: " << first_use << "."; } std::string ValueImpl::PrintUdChain() { @@ -56,8 +56,7 @@ ValueImpl::ValueImpl(Type type, uint32_t kind) : id_(GenerateId()) { first_use_offseted_by_kind_ = reinterpret_cast( reinterpret_cast(nullptr) + kind); VLOG(10) << "Construct a ValueImpl whose's kind is " << kind - << ". The offset first_use address is: " - << first_use_offseted_by_kind_; + << ". The value_impl address is: " << this; } } // namespace detail diff --git a/paddle/pir/src/dialect/control_flow/ir/cf_op.cc b/paddle/pir/src/dialect/control_flow/ir/cf_op.cc index c203fbafb5a02..0006974a8b70c 100644 --- a/paddle/pir/src/dialect/control_flow/ir/cf_op.cc +++ b/paddle/pir/src/dialect/control_flow/ir/cf_op.cc @@ -54,8 +54,6 @@ void TuplePushOp::VerifySig() { IR_ENFORCE(num_operands() >= 1u, "The size of inputs must no less than 1."); IR_ENFORCE(operand_source(0).type().isa(), "The first input of cf.tuple_push must be inlet_type."); - IR_ENFORCE(operand_source(0).HasOneUse(), - "The inlet value of cf.tuple_push can only be used once."); // No attributes should be verify. @@ -64,6 +62,13 @@ void TuplePushOp::VerifySig() { VLOG(4) << "End Verifying for TuplePushOp."; } +void TuplePushOp::VerifyRegion() { + // Note(winter-wang):Constraints on the number of uses can only can be placed + // in VerifyRegion, Otherwise cloning would fail. + IR_ENFORCE(operand_source(0).HasOneUse(), + "The inlet value of cf.tuple_push can only be used once."); +} + size_t TuplePushOp::tuple_size() { auto operands_size = num_operands(); IR_ENFORCE(operands_size >= 1u, @@ -96,12 +101,15 @@ void TuplePopOp::VerifySig() { IR_ENFORCE(num_operands() == 1u, "The size of inputs must equal to 1."); IR_ENFORCE(operand_source(0).type().isa(), "The first input of cf.tuple_pop must be outlet_type."); - IR_ENFORCE(operand_source(0).HasOneUse(), - "The outlet value of cf.tuple_pop can only be used once."); // No attributes should be verify. // Verify outputs: +} + +void TuplePopOp::VerifyRegion() { + IR_ENFORCE(operand_source(0).HasOneUse(), + "The outlet value of cf.tuple_pop can only be used once."); // Verify stack validity: auto pop_op = container_interface().tuple_pop_op(); diff --git a/python/paddle/jit/dy2static/pir_partial_program.py b/python/paddle/jit/dy2static/pir_partial_program.py index 2a55277fd77b4..10d16bb215741 100644 --- a/python/paddle/jit/dy2static/pir_partial_program.py +++ b/python/paddle/jit/dy2static/pir_partial_program.py @@ -746,6 +746,9 @@ def _append_backward_desc(self, train_runnable_program: RunableProgram): params = train_runnable_program.param_values combined_inputs = list(itertools.chain(inputs, params)) forward_end_idx = len(program.global_block().ops) + forward_end_op = None + if forward_end_idx > 0: + forward_end_op = program.global_block().ops[-1] grad_info_map = [None] * len(combined_inputs) with backend_guard(self._backend): check_type( @@ -798,6 +801,11 @@ def _append_backward_desc(self, train_runnable_program: RunableProgram): ) ), ) + if forward_end_op is not None: + for idx, op in enumerate(program.global_block().ops): + if op == forward_end_op: + forward_end_idx = idx + 1 + break if self._hooker: ( From ae8717b73f4028dfd7c0ed6e01a219683168dab5 Mon Sep 17 00:00:00 2001 From: cyberslack_lee Date: Mon, 26 Feb 2024 17:09:24 +0800 Subject: [PATCH 094/282] [clang-tidy] NO.19 enable readability-container-size-empty (#61713) * clangtidy 19 * fix --- paddle/fluid/framework/ir/generate_pass.cc | 4 ++-- paddle/fluid/pir/dialect/operator/utils/utils.cc | 2 +- paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc | 2 +- paddle/phi/infermeta/spmd_rules/dim_trans.cc | 2 +- paddle/phi/infermeta/spmd_rules/reshape.cc | 2 +- paddle/phi/kernels/stride/slice_kernel.cc | 3 +-- paddle/phi/kernels/stride/strided_slice_kernel.cc | 2 +- paddle/pir/src/core/builtin_op.cc | 2 +- 8 files changed, 9 insertions(+), 10 deletions(-) diff --git a/paddle/fluid/framework/ir/generate_pass.cc b/paddle/fluid/framework/ir/generate_pass.cc index cf2b7595b8859..42897c0a4a63d 100644 --- a/paddle/fluid/framework/ir/generate_pass.cc +++ b/paddle/fluid/framework/ir/generate_pass.cc @@ -348,7 +348,7 @@ GraphPatternDetector::handle_t GetGenerateRewrite( std::vector arguments; for (const std::string& argument : var.arguments()) { // The input may be mapped on the operator of pattern subgraph. - if (var_node_maps[argument].size() == 0) { + if (var_node_maps[argument].empty()) { VarDesc var_desc(patterns::UniqueKey(argument)); var_node_maps[argument].emplace_back( graph->CreateVarNode(&var_desc)); @@ -363,7 +363,7 @@ GraphPatternDetector::handle_t GetGenerateRewrite( std::vector arguments; for (const std::string& argument : var.arguments()) { // The output may be mapped on the operator of pattern subgraph. - if (var_node_maps[argument].size() == 0) { + if (var_node_maps[argument].empty()) { VarDesc var_desc(patterns::UniqueKey(argument)); var_node_maps[argument].emplace_back( graph->CreateVarNode(&var_desc)); diff --git a/paddle/fluid/pir/dialect/operator/utils/utils.cc b/paddle/fluid/pir/dialect/operator/utils/utils.cc index b4bad427567b7..299a047694e50 100644 --- a/paddle/fluid/pir/dialect/operator/utils/utils.cc +++ b/paddle/fluid/pir/dialect/operator/utils/utils.cc @@ -181,7 +181,7 @@ static std::unordered_map< {AttrType::ARRAY, [](const pir::Attribute& attr) { auto attr_vec = attr.dyn_cast().AsVector(); - if (attr_vec.size() == 0) { + if (attr_vec.empty()) { return VariantType{std::vector()}; } AttrType element_type = GetAttributeType(attr_vec[0]); diff --git a/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc b/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc index 469ab96a3c0cb..3450140741e21 100644 --- a/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc +++ b/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc @@ -786,7 +786,7 @@ static phi::Backend GetKernelBackendByYaml( } } - if (backend_info.size() > 0 && kernel_backend == phi::Backend::UNDEFINED) { + if (!backend_info.empty() && kernel_backend == phi::Backend::UNDEFINED) { kernel_backend = paddle::experimental::ParseBackend(place); } diff --git a/paddle/phi/infermeta/spmd_rules/dim_trans.cc b/paddle/phi/infermeta/spmd_rules/dim_trans.cc index feb9927dc1ef5..400e1c125b1f9 100644 --- a/paddle/phi/infermeta/spmd_rules/dim_trans.cc +++ b/paddle/phi/infermeta/spmd_rules/dim_trans.cc @@ -125,7 +125,7 @@ std::string Split::to_string() { std::shared_ptr make_flatten( const std::vector>& dims) { std::shared_ptr ptr; - if (dims.size() == 0) { + if (dims.empty()) { ptr = std::make_shared(); } else if (dims.size() == 1) { ptr = dims[0]; diff --git a/paddle/phi/infermeta/spmd_rules/reshape.cc b/paddle/phi/infermeta/spmd_rules/reshape.cc index 3d3bfb38d22fa..2e8d79e14bf49 100644 --- a/paddle/phi/infermeta/spmd_rules/reshape.cc +++ b/paddle/phi/infermeta/spmd_rules/reshape.cc @@ -120,7 +120,7 @@ std::vector> MakeReshapeDimTrans( } } - if (tgt_splitted_shape.size() > 0) { + if (!tgt_splitted_shape.empty()) { std::vector> input_dims; for (int i = 0, n = static_cast(src_dims.size()); i < n; i++) { int64_t in_dim = src_dims[i]; diff --git a/paddle/phi/kernels/stride/slice_kernel.cc b/paddle/phi/kernels/stride/slice_kernel.cc index fa3322cb6d826..3e21360ce09d0 100644 --- a/paddle/phi/kernels/stride/slice_kernel.cc +++ b/paddle/phi/kernels/stride/slice_kernel.cc @@ -20,7 +20,6 @@ #include "paddle/phi/core/kernel_registry.h" #include "paddle/phi/kernels/funcs/slice_utils.h" - namespace phi { template @@ -59,7 +58,7 @@ void SliceStridedKernel(const Context& ctx, } std::vector decrease_flag(output_dims.size(), 0); - if (decrease_axis.size() > 0) { + if (!decrease_axis.empty()) { for (int i = 0; i < static_cast(decrease_axis.size()); ++i) { int64_t axis = decrease_axis[i]; decrease_flag[axis] = 1; diff --git a/paddle/phi/kernels/stride/strided_slice_kernel.cc b/paddle/phi/kernels/stride/strided_slice_kernel.cc index 77919f8d000a0..f3b36565def3e 100644 --- a/paddle/phi/kernels/stride/strided_slice_kernel.cc +++ b/paddle/phi/kernels/stride/strided_slice_kernel.cc @@ -90,7 +90,7 @@ void StridedSliceRawStridedKernel(const Context& dev_ctx, } // generate new shape - if (decrease_axis.size() > 0) { + if (!decrease_axis.empty()) { std::vector new_out_shape; std::vector new_out_stride; for (size_t i = 0; i < decrease_axis.size(); ++i) { diff --git a/paddle/pir/src/core/builtin_op.cc b/paddle/pir/src/core/builtin_op.cc index 079294dea9446..24b7624dafc63 100644 --- a/paddle/pir/src/core/builtin_op.cc +++ b/paddle/pir/src/core/builtin_op.cc @@ -353,7 +353,7 @@ void SplitOp::PassStopGradients(OperationArgument &argument) { if (auto input = argument.inputs[0]) { auto *defining_op = input.defining_op(); if (defining_op && defining_op->isa()) { - IR_ENFORCE(argument.output_types.size(), + IR_ENFORCE(!argument.output_types.empty(), defining_op->num_operands(), "Required SplitOp.output.size() == CombineOp.input.size(), " "but received %d != %d", From d0b2081cbcdedeb618234b22d4fbe51a7c3f058c Mon Sep 17 00:00:00 2001 From: wanghuancoder Date: Mon, 26 Feb 2024 18:20:14 +0800 Subject: [PATCH 095/282] [PIR] pir onednn support expand (#62054) * pir onednn support expand --- paddle/fluid/pir/dialect/CMakeLists.txt | 5 +- .../op_generator/op_creator_drr_gen.py | 3 + .../fluid/pir/dialect/op_generator/op_gen.py | 9 +- .../dialect/operator/ir/manual_onednn_op.cc | 346 ++++++++++++++++++ .../dialect/operator/ir/manual_onednn_op.h | 94 +++++ .../pir/dialect/operator/ir/op_dialect.cc | 9 + .../dialect/operator/ir/ops_onednn_extra.yaml | 6 +- test/mkldnn/test_expand_v2_mkldnn_op.py | 7 +- 8 files changed, 472 insertions(+), 7 deletions(-) create mode 100644 paddle/fluid/pir/dialect/operator/ir/manual_onednn_op.cc create mode 100644 paddle/fluid/pir/dialect/operator/ir/manual_onednn_op.h diff --git a/paddle/fluid/pir/dialect/CMakeLists.txt b/paddle/fluid/pir/dialect/CMakeLists.txt index 535f8cdc7c818..2955a6d57afb5 100644 --- a/paddle/fluid/pir/dialect/CMakeLists.txt +++ b/paddle/fluid/pir/dialect/CMakeLists.txt @@ -250,8 +250,9 @@ set(op_dialect_srcs ${PADDLE_SOURCE_DIR}/paddle/fluid/pir/transforms/shape_optimization_pass.cc) if(WITH_MKLDNN) - set(op_dialect_srcs ${op_dialect_srcs} ${onednn_op_source_file} - ${op_onednn_info_file}) + set(op_dialect_srcs + ${op_dialect_srcs} ${onednn_op_source_file} ${op_onednn_info_file} + ${CMAKE_CURRENT_SOURCE_DIR}/operator/ir/manual_onednn_op.cc) endif() set(op_dialect_deps phi common pir type_info string_helper) diff --git a/paddle/fluid/pir/dialect/op_generator/op_creator_drr_gen.py b/paddle/fluid/pir/dialect/op_generator/op_creator_drr_gen.py index 04fb7002ce6f4..e205e295fd9ef 100644 --- a/paddle/fluid/pir/dialect/op_generator/op_creator_drr_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/op_creator_drr_gen.py @@ -26,6 +26,9 @@ {op_header} #include "paddle/fluid/pir/dialect/operator/ir/manual_op.h" +#ifdef PADDLE_WITH_DNNL +#include "paddle/fluid/pir/dialect/operator/ir/manual_onednn_op.h" +#endif namespace paddle {{ namespace drr {{ diff --git a/paddle/fluid/pir/dialect/op_generator/op_gen.py b/paddle/fluid/pir/dialect/op_generator/op_gen.py index 40dc916d4f4ad..67462983fbf0a 100644 --- a/paddle/fluid/pir/dialect/op_generator/op_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/op_gen.py @@ -320,6 +320,11 @@ class {TEST_API} {op_name} : public pir::Op<{op_name}{interfaces}{traits}> {{ 'assign_out_', } +ONEDNN_MANUAL_OP_LIST = { + 'split_grad', + 'expand', +} + attr_types_map = { 'IntArray': ['paddle::dialect::IntArrayAttribute', 'IntArray'], 'Scalar': ['paddle::dialect::ScalarAttribute', 'Scalar'], @@ -1347,7 +1352,9 @@ def AutoCodeGen(op_info_items, all_op_info_items, namespaces, dialect_name): if len(op_traits) > 0: op_traits_str = "," + ",".join(op_traits) - if op_name in PD_MANUAL_OP_LIST: + if dialect_name == "onednn_op" and op_name in ONEDNN_MANUAL_OP_LIST: + continue + elif dialect_name != "onednn_op" and op_name in PD_MANUAL_OP_LIST: continue if op_kernel_map is None: func_list = [None] diff --git a/paddle/fluid/pir/dialect/operator/ir/manual_onednn_op.cc b/paddle/fluid/pir/dialect/operator/ir/manual_onednn_op.cc new file mode 100644 index 0000000000000..352677f0047c8 --- /dev/null +++ b/paddle/fluid/pir/dialect/operator/ir/manual_onednn_op.cc @@ -0,0 +1,346 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#ifdef GET_OP_LIST +#undef GET_OP_LIST +paddle::onednn::dialect::ExpandOp +#else + +#include "paddle/fluid/pir/dialect/operator/ir/manual_onednn_op.h" +#include "paddle/fluid/pir/dialect/kernel/ir/kernel_type.h" +#include "paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h" +#include "paddle/fluid/pir/dialect/operator/ir/ir_meta_tensor.h" +#include "paddle/fluid/pir/dialect/operator/ir/ir_selected_rows.h" +#include "paddle/fluid/pir/dialect/operator/ir/ir_tensor.h" +#include "paddle/fluid/pir/dialect/operator/ir/onednn_op.h" +#include "paddle/fluid/pir/dialect/operator/ir/op_attribute.h" +#include "paddle/fluid/pir/dialect/operator/ir/op_type.h" +#include "paddle/fluid/pir/dialect/operator/ir/pd_op.h" +#include "paddle/fluid/primitive/rule/vjp/vjp.h" +#include "paddle/phi/api/lib/data_type_set.h" +#include "paddle/phi/api/lib/utils/allocator.h" +#include "paddle/phi/core/dense_tensor.h" +#include "paddle/phi/core/enforce.h" +#include "paddle/phi/infermeta/backward.h" +#include "paddle/phi/infermeta/binary.h" +#include "paddle/phi/infermeta/fusion.h" +#include "paddle/phi/infermeta/multiary.h" +#include "paddle/phi/infermeta/nullary.h" +#include "paddle/phi/infermeta/ternary.h" +#include "paddle/phi/infermeta/unary.h" +#include "paddle/pir/include/core/builtin_attribute.h" +#include "paddle/pir/include/core/builtin_op.h" +#include "paddle/pir/include/core/builtin_type.h" +#include "paddle/pir/include/core/ir_context.h" +#include "paddle/pir/include/core/op_base.h" + +namespace paddle { +namespace onednn { +namespace dialect { + +const char* ExpandOp::attributes_name[1] = {"mkldnn_data_type"}; + +OpInfoTuple ExpandOp::GetOpInfo() { + std::vector inputs = { + paddle::dialect::OpInputInfo( + "x", "paddle::dialect::DenseTensorType", false, false, false, true), + paddle::dialect::OpInputInfo("shape", + "paddle::dialect::IntArrayAttribute", + false, + false, + true, + false)}; + std::vector attributes = { + paddle::dialect::OpAttributeInfo( + "mkldnn_data_type", "pir::StrAttribute", "")}; + std::vector outputs = { + paddle::dialect::OpOutputInfo( + "out", "paddle::dialect::DenseTensorType", false, false)}; + pir::AttributeMap extra_attr_default_value; + pir::Attribute attr_mkldnn_data_type = + pir::StrAttribute::get(pir::IrContext::Instance(), "float32"); + extra_attr_default_value["mkldnn_data_type"] = attr_mkldnn_data_type; + + paddle::dialect::OpRunTimeInfo run_time_info = + paddle::dialect::OpRunTimeInfo("ExpandInferMeta", + {"x", "shape"}, + "expand", + {"x", "shape"}, + {"x"}, + {}, + {}, + {}, + {"mkldnn_data_type"}, + {}, + extra_attr_default_value, + {}, + false, + false); + return std::make_tuple(inputs, attributes, outputs, run_time_info, "expand"); +} + +void ExpandOp::Build(pir::Builder& builder, + pir::OperationArgument& argument, + pir::Value x_, + const std::vector& shape, + const std::string& mkldnn_data_type) { + VLOG(4) << "Start build ExpandOp"; + + // Generate int_array mutable attribute: shape + paddle::dialect::FullIntArrayOp full_shape_op = + builder.Build( + shape, phi::DataType::INT64, phi::CPUPlace()); + pir::Value shape_ = full_shape_op->result(0); + + VLOG(4) << "Builder construction inputs"; + std::vector argument_inputs = {x_, shape_}; + argument.AddInputs(argument_inputs); + + VLOG(4) << "Builder construction attributes"; + pir::AttributeMap argument_attributes = {}; + pir::Attribute attr_mkldnn_data_type = + pir::StrAttribute::get(pir::IrContext::Instance(), mkldnn_data_type); + argument.AddAttribute("mkldnn_data_type", attr_mkldnn_data_type); + argument_attributes.insert({"mkldnn_data_type", attr_mkldnn_data_type}); + + std::vector argument_outputs = + ExpandOp::InferMeta(argument_inputs, argument_attributes); + argument.AddOutputs(argument_outputs.begin(), argument_outputs.end()); + ::pir::PassStopGradientsDefaultly(argument); +} + +void ExpandOp::Build(pir::Builder& builder, + pir::OperationArgument& argument, + pir::Value x_, + pir::AttributeMap attributes) { + VLOG(4) << "Start build ExpandOp"; + + IR_ENFORCE(attributes.find("shape") != attributes.end(), + "'shape' Attribute is expected for ExpandOp. "); + std::vector shape = + attributes.at("shape") + .dyn_cast() + .data() + .GetData(); + + IR_ENFORCE(attributes.find("mkldnn_data_type") != attributes.end(), + "'mkldnn_data_type' Attribute is expected for ExpandOp. "); + std::string mkldnn_data_type = attributes.at("mkldnn_data_type") + .dyn_cast() + .AsString(); + + // Generate int_array mutable attribute: shape + paddle::dialect::FullIntArrayOp full_shape_op = + builder.Build( + shape, phi::DataType::INT64, phi::CPUPlace()); + pir::Value shape_ = full_shape_op->result(0); + + VLOG(4) << "Builder construction inputs"; + std::vector argument_inputs = {x_, shape_}; + argument.AddInputs(argument_inputs); + + VLOG(4) << "Builder construction attributes"; + pir::AttributeMap argument_attributes = {}; + pir::Attribute attr_mkldnn_data_type = + pir::StrAttribute::get(pir::IrContext::Instance(), mkldnn_data_type); + argument.AddAttribute("mkldnn_data_type", attr_mkldnn_data_type); + argument_attributes.insert({"mkldnn_data_type", attr_mkldnn_data_type}); + + std::vector argument_outputs = + ExpandOp::InferMeta(argument_inputs, argument_attributes); + argument.AddOutputs(argument_outputs.begin(), argument_outputs.end()); + ::pir::PassStopGradientsDefaultly(argument); +} + +void ExpandOp::Build(pir::Builder& builder, + pir::OperationArgument& argument, + pir::Value x_, + pir::Value shape_, + const std::string& mkldnn_data_type) { + VLOG(4) << "Start build ExpandOp"; + + VLOG(4) << "Builder construction inputs"; + std::vector argument_inputs = {x_, shape_}; + argument.AddInputs(argument_inputs); + + VLOG(4) << "Builder construction attributes"; + pir::AttributeMap argument_attributes = {}; + pir::Attribute attr_mkldnn_data_type = + pir::StrAttribute::get(pir::IrContext::Instance(), mkldnn_data_type); + argument.AddAttribute("mkldnn_data_type", attr_mkldnn_data_type); + argument_attributes.insert({"mkldnn_data_type", attr_mkldnn_data_type}); + + std::vector argument_outputs = + ExpandOp::InferMeta(argument_inputs, argument_attributes); + argument.AddOutputs(argument_outputs.begin(), argument_outputs.end()); + ::pir::PassStopGradientsDefaultly(argument); +} + +void ExpandOp::VerifySig() { + VLOG(4) << "Start Verifying inputs, outputs and attributes for: ExpandOp."; + VLOG(4) << "Verifying inputs:"; + { + auto input_size = num_operands(); + IR_ENFORCE(input_size == 2u, + "The size %d of inputs must be equal to 2.", + input_size); + IR_ENFORCE((*this) + ->operand_source(0) + .type() + .isa(), + "Type validation failed for the 0th input, got %s.", + (*this)->operand_source(0).type()); + if (auto vec_type = + (*this)->operand_source(1).type().dyn_cast()) { + for (size_t i = 0; i < vec_type.size(); ++i) { + IR_ENFORCE(vec_type[i].isa(), + "Type validation failed for the 1th input, got %s.", + (*this)->operand_source(1).type()); + } + } else { + IR_ENFORCE((*this) + ->operand_source(1) + .type() + .isa(), + "Type validation failed for the 1th input, got %s.", + (*this)->operand_source(1).type()); + } + } + VLOG(4) << "Verifying attributes:"; + { + auto& attributes = this->attributes(); + IR_ENFORCE(attributes.count("mkldnn_data_type") > 0, + "mkldnn_data_type does not exist."); + IR_ENFORCE(attributes.at("mkldnn_data_type").isa(), + "Type of attribute: mkldnn_data_type is not pir::StrAttribute."); + } + VLOG(4) << "Verifying outputs:"; + { + auto output_size = num_results(); + IR_ENFORCE(output_size == 1u, + "The size %d of outputs must be equal to 1.", + output_size); + IR_ENFORCE( + (*this)->result(0).type().isa(), + "Type validation failed for the 0th output."); + } + VLOG(4) << "End Verifying for: ExpandOp."; +} + +void ExpandOp::InferMeta(phi::InferMetaContext* infer_meta) { + auto fn = PD_INFER_META(phi::ExpandInferMeta); + fn(infer_meta); +} + +std::vector ExpandOp::InferMeta( + const std::vector& input_values, + const pir::AttributeMap& attributes) { + IR_ENFORCE(input_values.size() == 2, + "Num of inputs is expected to be 2 but got %d.", + input_values.size()); + + pir::Value x_ = input_values[0]; + pir::Value shape_ = input_values[1]; + VLOG(4) << "Builder construction outputs"; + + paddle::dialect::DenseTensorType x; + if (x_.type().isa()) { + x = x_.type().dyn_cast(); + } else if (x_.type().isa()) { + paddle::dialect::AllocatedDenseTensorType allocated_x = + x_.type().dyn_cast(); + x = paddle::dialect::DenseTensorType::get(pir::IrContext::Instance(), + allocated_x.dtype(), + allocated_x.dims(), + allocated_x.data_layout(), + allocated_x.lod(), + allocated_x.offset()); + } else { + PADDLE_THROW(phi::errors::Unimplemented( + "Only support paddle::dialect::DenseTensorType or " + "paddle::dialect::AllocatedDenseTensorType")); + } + + phi::IntArray shape; + if (shape_.defining_op()->isa()) { + shape = std::move(phi::IntArray(paddle::dialect::GetInt64Vector( + shape_.defining_op() + ->dyn_cast() + .attribute("value")))); + } else if (shape_.type().isa()) { + size_t shape_size = shape_.type().dyn_cast().size(); + // In ExpandInferMeta use -2 to represent the element in expand_shape is a + // var. + shape = std::move(phi::IntArray(std::vector(shape_size, -2))); + shape.SetFromTensor(true); + } else if (shape_.type().isa()) { + size_t shape_size = common::product( + shape_.type().dyn_cast().dims()); + // In ExpandInferMeta use -2 to represent the element in expand_shape is a + // var. + shape = std::move(phi::IntArray(std::vector(shape_size, -2))); + shape.SetFromTensor(true); + } else { + PADDLE_THROW(phi::errors::Unimplemented( + "Only support VectorType or DenseTensorType")); + } + + VLOG(4) << "Builder construction dense_x"; + paddle::dialect::IrTensor ir_tensor_x( + paddle::dialect::TransToPhiDataType(x.dtype()), + x.dims(), + x.data_layout(), + x.lod(), + x.offset()); + VLOG(4) << "Builder construction meta_x"; + paddle::dialect::IrMetaTensor meta_x(&ir_tensor_x); + paddle::dialect::IrTensor dense_out; + paddle::dialect::IrMetaTensor meta_out(&dense_out); + + phi::ExpandInferMeta(meta_x, shape, &meta_out); + + std::vector argument_outputs; + pir::Type out_dense_tensor_type = paddle::dialect::DenseTensorType::get( + pir::IrContext::Instance(), + paddle::dialect::TransToIrDataType(dense_out.dtype()), + dense_out.dims(), + dense_out.layout(), + dense_out.lod(), + dense_out.offset()); + argument_outputs.push_back(out_dense_tensor_type); + + return argument_outputs; +} + +phi::DataType ExpandOp::GetKernelTypeForVar( + const std::string& var_name, + const phi::DataType& tensor_dtype, + const phi::DataType& expected_kernel_dtype) { + VLOG(4) << "Get KernelType for Var of op: ExpandOp"; + + return expected_kernel_dtype; +} + +bool ExpandOp::InferSymbolicShape( + pir::ShapeConstraintIRAnalysis* shape_analysis) { + VLOG(4) << "Infer symbolic shape for op: ExpandOp"; + return paddle::dialect::ExpandOpInferSymbolicShape(this->operation(), + shape_analysis); +} + +} // namespace dialect +} // namespace onednn +} // namespace paddle + +IR_DEFINE_EXPLICIT_TYPE_ID(paddle::onednn::dialect::ExpandOp) +#endif diff --git a/paddle/fluid/pir/dialect/operator/ir/manual_onednn_op.h b/paddle/fluid/pir/dialect/operator/ir/manual_onednn_op.h new file mode 100644 index 0000000000000..3c8050480ade9 --- /dev/null +++ b/paddle/fluid/pir/dialect/operator/ir/manual_onednn_op.h @@ -0,0 +1,94 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include + +#include "paddle/fluid/framework/infershape_utils.h" +#include "paddle/fluid/ir_adaptor/translator/utils.h" +#include "paddle/fluid/pir/dialect/operator/interface/decomp.h" +#include "paddle/fluid/pir/dialect/operator/interface/get_kernel_type_for_var.h" +#include "paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_symbolic_shape.h" +#include "paddle/fluid/pir/dialect/operator/interface/infermeta.h" +#include "paddle/fluid/pir/dialect/operator/interface/op_yaml_info.h" +#include "paddle/fluid/pir/dialect/operator/interface/parse_kernel_key.h" +#include "paddle/fluid/pir/dialect/operator/interface/vjp.h" +#include "paddle/fluid/pir/dialect/operator/ir/manual_op.h" +#include "paddle/fluid/pir/dialect/operator/trait/custom_vjp.h" +#include "paddle/fluid/pir/dialect/operator/trait/inplace.h" +#include "paddle/fluid/pir/dialect/operator/trait/onednn.h" +#include "paddle/fluid/pir/dialect/operator/utils/op_yaml_info_util.h" +#include "paddle/fluid/pir/dialect/operator/utils/utils.h" +#include "paddle/phi/core/infermeta_utils.h" +#include "paddle/pir/include/core/builder.h" +#include "paddle/pir/include/core/ir_printer.h" +#include "paddle/pir/include/core/op_base.h" +#include "paddle/pir/include/core/op_trait.h" +#include "paddle/pir/include/core/operation_utils.h" + +namespace paddle { +namespace onednn { +namespace dialect { + +class ExpandOp : public pir::Op { + public: + using Op::Op; + static const char* name() { return "onednn_op.expand"; } + static const char* attributes_name[1]; + static constexpr uint32_t attributes_num = 1; + static OpInfoTuple GetOpInfo(); + static void Build(pir::Builder& builder, // NOLINT + pir::OperationArgument& argument, // NOLINT + pir::Value x_, + const std::vector& shape = {}, + const std::string& mkldnn_data_type = "float32"); + static void Build(pir::Builder& builder, // NOLINT + pir::OperationArgument& argument, // NOLINT + pir::Value x_, + pir::Value shape_, + const std::string& mkldnn_data_type = "float32"); + static void Build(pir::Builder& builder, // NOLINT + pir::OperationArgument& argument, // NOLINT + pir::Value x_, + pir::AttributeMap attributes); + + void VerifySig(); + + static phi::DataType GetKernelTypeForVar( + const std::string& var_name, + const phi::DataType& tensor_dtype, + const phi::DataType& expected_kernel_dtype); + + bool InferSymbolicShape(pir::ShapeConstraintIRAnalysis* shape_analysis); + + pir::Value x() { return operand_source(0); } + pir::Value shape() { return operand_source(1); } + pir::Value out() { return result(0); } + + static void InferMeta(phi::InferMetaContext* infer_meta); + static std::vector InferMeta( + const std::vector& input_values, + const pir::AttributeMap& attributes); +}; + +} // namespace dialect +} // namespace onednn +} // namespace paddle + +IR_DECLARE_EXPLICIT_TYPE_ID(paddle::onednn::dialect::ExpandOp) diff --git a/paddle/fluid/pir/dialect/operator/ir/op_dialect.cc b/paddle/fluid/pir/dialect/operator/ir/op_dialect.cc index 26f88ead120f6..98391f36cddd9 100644 --- a/paddle/fluid/pir/dialect/operator/ir/op_dialect.cc +++ b/paddle/fluid/pir/dialect/operator/ir/op_dialect.cc @@ -31,6 +31,9 @@ #include "paddle/pir/include/dialect/control_flow/ir/cf_dialect.h" #include "paddle/pir/include/dialect/control_flow/ir/cf_op.h" #include "paddle/pir/include/dialect/shape/ir/shape_attribute.h" +#ifdef PADDLE_WITH_DNNL +#include "paddle/fluid/pir/dialect/operator/ir/manual_onednn_op.h" +#endif namespace paddle { namespace dialect { @@ -258,6 +261,12 @@ void OperatorDialect::initialize() { #include "paddle/fluid/pir/dialect/operator/ir/manual_op.cc" // NOLINT >(); +#ifdef PADDLE_WITH_DNNL + RegisterOps< +#define GET_OP_LIST +#include "paddle/fluid/pir/dialect/operator/ir/manual_onednn_op.cc" // NOLINT + >(); +#endif RegisterInterfaces(); } diff --git a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml index 7527a2f395e6a..c63c96e28c433 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml @@ -77,9 +77,11 @@ - op : exp_grad -# - op : expand +- op : expand + extra_args : str mkldnn_data_type="float32" -# - op : expand_grad +- op : expand_grad + extra_args : str mkldnn_data_type="float32" - op : fc extra_args : bool ALL_KERNELS_MUST_COMPUTE_RUNTIME_SHAPE=true, bool use_quantizer=false, str mkldnn_data_type="float32", float scale_in=1.0, float[] scale_weights={1.0f}, float scale_out=1.0, bool force_fp32_output=false diff --git a/test/mkldnn/test_expand_v2_mkldnn_op.py b/test/mkldnn/test_expand_v2_mkldnn_op.py index 4c8b9d0221a4e..3855e9060ff20 100644 --- a/test/mkldnn/test_expand_v2_mkldnn_op.py +++ b/test/mkldnn/test_expand_v2_mkldnn_op.py @@ -48,10 +48,12 @@ def init_data(self): self.expand_times = [2, 3, 4, 1] def test_check_output(self): - self.check_output_with_place(core.CPUPlace()) + self.check_output_with_place(core.CPUPlace(), check_pir_onednn=True) def test_check_grad(self): - self.check_grad_with_place(core.CPUPlace(), ["X"], "Out") + self.check_grad_with_place( + core.CPUPlace(), ["X"], "Out", check_pir_onednn=True + ) class TestExpandV2ExpandDimOneDNNOp(TestExpandV2OneDNNOp): @@ -153,6 +155,7 @@ def test_check_grad(self): "Out", user_defined_grads=[convert_float_to_uint16(self.dx)], user_defined_grad_outputs=[self.dout], + check_pir_onednn=True, ) cls_name = "{}_{}".format(parent.__name__, "Expand_v2_BF16") From 133023dd7034dcb4e55de5d96739247ec89967f2 Mon Sep 17 00:00:00 2001 From: Travis-Lee Date: Mon, 26 Feb 2024 18:47:23 +0800 Subject: [PATCH 096/282] Fix Bug Fallback To Cpu With Memmory Leak, test=develop (#62062) --- paddle/fluid/framework/operator.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc index c9d7af6a44cea..bf2badc5a82cf 100644 --- a/paddle/fluid/framework/operator.cc +++ b/paddle/fluid/framework/operator.cc @@ -2081,7 +2081,7 @@ void OperatorWithKernel::RunImpl(const Scope& scope, ExecutionContext(*this, exec_scope, *dev_ctx, *runtime_ctx)); } if (fallback_to_cpu) { - [[maybe_unused]] auto released_kernel = phi_kernel_.release(); + phi_kernel_.reset(); } } From cfdd8133070bd928ae028d39345394d0f8cfdefa Mon Sep 17 00:00:00 2001 From: WangZhen <23097963+0x45f@users.noreply.github.com> Date: Mon, 26 Feb 2024 18:54:04 +0800 Subject: [PATCH 097/282] [PIR AMP]Use common AMP code for PIR and Dygraph mode (#61496) --- paddle/fluid/eager/amp_utils.h | 165 --------- .../eager_manual/forwards/add_n_fwd_func.cc | 9 +- .../forwards/conv2d_fwd_function.cc | 12 +- .../forwards/multiply_fwd_func.cc | 21 +- .../forwards/sync_batch_norm_fwd_func.cc | 2 - .../forwards/fused_attention_fwd_func.cc | 6 +- ...as_dropout_residual_layer_norm_fwd_func.cc | 4 +- .../forwards/fused_feedforward_fwd_func.cc | 6 +- .../forwards/fused_gate_attention_fwd_func.cc | 6 +- .../forwards/fused_gemm_epilogue_fwd_func.cc | 6 +- .../auto_code_generator/eager_generator.cc | 4 +- .../generator/eager_gen.py | 16 +- .../generator/python_c_gen.py | 2 - paddle/fluid/eager/eager_amp_auto_cast.h | 143 -------- paddle/fluid/eager/type_promotion_utils.h | 3 +- paddle/fluid/imperative/amp_utils.h | 340 ++++++++++++++++++ .../fluid/pir/dialect/op_generator/api_gen.py | 17 +- .../pir/dialect/operator/utils/amp_utils.cc | 219 ----------- .../pir/dialect/operator/utils/amp_utils.h | 60 ---- paddle/fluid/pybind/CMakeLists.txt | 4 +- paddle/fluid/pybind/eager_method.cc | 17 +- 21 files changed, 412 insertions(+), 650 deletions(-) delete mode 100644 paddle/fluid/eager/amp_utils.h delete mode 100644 paddle/fluid/eager/eager_amp_auto_cast.h create mode 100644 paddle/fluid/imperative/amp_utils.h delete mode 100644 paddle/fluid/pir/dialect/operator/utils/amp_utils.cc delete mode 100644 paddle/fluid/pir/dialect/operator/utils/amp_utils.h diff --git a/paddle/fluid/eager/amp_utils.h b/paddle/fluid/eager/amp_utils.h deleted file mode 100644 index f53f82e2ab6b5..0000000000000 --- a/paddle/fluid/eager/amp_utils.h +++ /dev/null @@ -1,165 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#include - -#include "paddle/fluid/eager/api/utils/global_utils.h" -#include "paddle/fluid/imperative/amp_auto_cast.h" - -namespace egr { - -static inline phi::DataType GetPromoteType( - const std::string& op_name, - const paddle::small_vector, - kSlotSmallVectorSize>& amp_tensors_vector, - const phi::DataType& amp_dtype) { - auto dst_type = amp_dtype; - // only consider the dtype of input(X). - if (op_name == "batch_norm" || op_name == "layer_norm" || - op_name == "sync_batch_norm" || - op_name == "moving_average_abs_max_scale") { - if (amp_tensors_vector[0][0].dtype() == phi::DataType::FLOAT32) { - dst_type = phi::DataType::FLOAT32; - } - return dst_type; - } - - if (egr::Controller::Instance().GetCurrentTracer()->GetAmpDtype() == - "float16") { - if (op_name == "fused_attention") { - for (size_t i = 0; i < amp_tensors_vector.size(); i++) { - if (i != 3 || i != 4 || i != 9 || i != 10) { - if (amp_tensors_vector[i][0].dtype() == phi::DataType::FLOAT32) { - dst_type = phi::DataType::FLOAT32; - return dst_type; - } - } - } - } else if (op_name == "fused_feedforward") { - for (size_t i = 0; i < amp_tensors_vector.size(); i++) { - if (i != 7 || i != 8 || i != 9 || i != 10) { - if (amp_tensors_vector[i][0].dtype() == phi::DataType::FLOAT32) { - dst_type = phi::DataType::FLOAT32; - return dst_type; - } - } - } - } - } - - for (const auto& tensors : amp_tensors_vector) { - for (const auto& tensor : tensors) { - if (tensor.dtype() == phi::DataType::FLOAT32) { - dst_type = tensor.dtype(); - break; - } - } - } - - return dst_type; -} - -inline phi::DataType GetDtypeWithPlace( - const std::string& op_name, - const paddle::small_vector, - kSlotSmallVectorSize>& amp_tensors_vector, - const phi::DataType amp_dtype) { - if (amp_dtype == phi::DataType::FLOAT32) { - return amp_dtype; - } - bool is_right_place = false; - for (const auto& tensors : amp_tensors_vector) { - for (const auto& tensor : tensors) { - auto place = tensor.place(); - // TODO(lizhiyu): If the tensor is a dist-tensor, it's place may be - // `unknown` in the no-calculation rank right now. - // We use `is_dist_tensor()` to avoid the bug temporarily. The - // dist-tensor in the no-calculation rank should have the right - // place. - is_right_place = - (tensor.is_dist_tensor() || paddle::platform::is_gpu_place(place) || - paddle::platform::is_cuda_pinned_place(place) || - paddle::platform::is_xpu_place(place) || - paddle::platform::is_custom_place(place)); - if (is_right_place) { - break; - } - } - } - - if (!is_right_place) { - VLOG(6) << "Change " << op_name << "'s AMP type from " << amp_dtype - << " to FP32"; - return phi::DataType::FLOAT32; - } - return amp_dtype; -} - -inline phi::DataType GetAmpDestDtype( - const std::string& op_name, - const paddle::small_vector, - kSlotSmallVectorSize>& amp_tensors_vector) { - auto amp_level = egr::Controller::Instance().GetAMPLevel(); - auto amp_setting_dtype = - egr::Controller::Instance().GetCurrentTracer()->GetAmpPhiDtype(); - auto dst_type = amp_setting_dtype; - - bool use_promote = true; - if (amp_level == paddle::imperative::AmpLevel::O2) { - use_promote = - egr::Controller::Instance().GetCurrentTracer()->GetUsePromote(); - } - - if (use_promote) { - if (paddle::imperative::AmpOperators::Instance() - .GetMutableAllowOps() - ->count(op_name)) { - dst_type = amp_setting_dtype; - } else if (paddle::imperative::AmpOperators::Instance() - .GetMutableBlockOps() - ->count(op_name)) { - dst_type = phi::DataType::FLOAT32; - } else { - if (amp_level == paddle::imperative::AmpLevel::OD) { - dst_type = phi::DataType::FLOAT32; - } else { - dst_type = - GetPromoteType(op_name, amp_tensors_vector, amp_setting_dtype); - } - } - } else { - // use_promote can be set to false only for O2 training. - if (paddle::imperative::AmpOperators::Instance() - .GetMutableBlockOps() - ->count(op_name)) { - dst_type = phi::DataType::FLOAT32; - } - } - - if (dst_type == amp_setting_dtype && - (paddle::imperative::AmpOperators::Instance() - .GetMutableUnsupportedOps(amp_setting_dtype) - ->count(op_name))) { - dst_type = phi::DataType::FLOAT32; - } - - dst_type = GetDtypeWithPlace(op_name, amp_tensors_vector, dst_type); - VLOG(6) << "AMP GetAmpDestDtype:" - << " op(" << op_name << ") amp_dtype(" << dst_type << ") amp_level(" - << static_cast(amp_level) << ")."; - return dst_type; -} - -} // namespace egr diff --git a/paddle/fluid/eager/api/manual/eager_manual/forwards/add_n_fwd_func.cc b/paddle/fluid/eager/api/manual/eager_manual/forwards/add_n_fwd_func.cc index d27ca1d242953..0fdfb88192808 100644 --- a/paddle/fluid/eager/api/manual/eager_manual/forwards/add_n_fwd_func.cc +++ b/paddle/fluid/eager/api/manual/eager_manual/forwards/add_n_fwd_func.cc @@ -13,12 +13,11 @@ // limitations under the License. #include "paddle/common/flags.h" -#include "paddle/fluid/eager/amp_utils.h" #include "paddle/fluid/eager/api/manual/eager_manual/dygraph_forward_api.h" #include "paddle/fluid/eager/api/manual/eager_manual/nodes/nodes.h" #include "paddle/fluid/eager/api/utils/global_utils.h" -#include "paddle/fluid/eager/eager_amp_auto_cast.h" #include "paddle/fluid/eager/nan_inf_utils.h" +#include "paddle/fluid/imperative/amp_utils.h" #include "paddle/fluid/platform/profiler/event_tracing.h" COMMON_DECLARE_bool(check_nan_inf); @@ -36,9 +35,11 @@ paddle::Tensor add_n_ad_func(const std::vector& x) { paddle::small_vector, egr::kSlotSmallVectorSize> amp_tensors_vector = {x}; - auto amp_dst_dtype = egr::GetAmpDestDtype(op_name, amp_tensors_vector); + auto amp_dst_dtype = + paddle::imperative::GetAmpDestDtype(op_name, amp_tensors_vector); - auto NEW_x = egr::EagerAmpAutoCasts("x", x, amp_dst_dtype, op_name); + auto NEW_x = + paddle::imperative::AmpAutoCasts("x", x, amp_dst_dtype, op_name); { paddle::imperative::AutoCastGuard guard( diff --git a/paddle/fluid/eager/api/manual/eager_manual/forwards/conv2d_fwd_function.cc b/paddle/fluid/eager/api/manual/eager_manual/forwards/conv2d_fwd_function.cc index 7cf3ee807b685..b794363d8e015 100644 --- a/paddle/fluid/eager/api/manual/eager_manual/forwards/conv2d_fwd_function.cc +++ b/paddle/fluid/eager/api/manual/eager_manual/forwards/conv2d_fwd_function.cc @@ -13,13 +13,12 @@ // limitations under the License. #include "paddle/common/flags.h" -#include "paddle/fluid/eager/amp_utils.h" #include "paddle/fluid/eager/api/manual/eager_manual/dygraph_forward_api.h" #include "paddle/fluid/eager/api/manual/eager_manual/nodes/nodes.h" #include "paddle/fluid/eager/api/utils/global_utils.h" -#include "paddle/fluid/eager/eager_amp_auto_cast.h" #include "paddle/fluid/eager/eager_layout_auto_tune.h" #include "paddle/fluid/eager/nan_inf_utils.h" +#include "paddle/fluid/imperative/amp_utils.h" #include "paddle/fluid/platform/profiler/event_tracing.h" COMMON_DECLARE_bool(check_nan_inf); @@ -44,12 +43,13 @@ paddle::Tensor conv2d_ad_func(const paddle::Tensor& input, paddle::small_vector, egr::kSlotSmallVectorSize> amp_tensors_vector = {{input}, {filter}}; - auto amp_dst_dtype = egr::GetAmpDestDtype(op_name, amp_tensors_vector); + auto amp_dst_dtype = + paddle::imperative::GetAmpDestDtype(op_name, amp_tensors_vector); auto new_input = - egr::EagerAmpAutoCast("input", input, amp_dst_dtype, op_name); - auto new_filter = - egr::EagerAmpAutoCast("filter", filter, amp_dst_dtype, op_name); + paddle::imperative::AmpAutoCast("input", input, amp_dst_dtype, op_name); + auto new_filter = paddle::imperative::AmpAutoCast( + "filter", filter, amp_dst_dtype, op_name); { paddle::imperative::AutoCastGuard guard( diff --git a/paddle/fluid/eager/api/manual/eager_manual/forwards/multiply_fwd_func.cc b/paddle/fluid/eager/api/manual/eager_manual/forwards/multiply_fwd_func.cc index 856407c58e96c..9d1451c74e65f 100644 --- a/paddle/fluid/eager/api/manual/eager_manual/forwards/multiply_fwd_func.cc +++ b/paddle/fluid/eager/api/manual/eager_manual/forwards/multiply_fwd_func.cc @@ -13,14 +13,13 @@ // limitations under the License. #include "paddle/common/flags.h" -#include "paddle/fluid/eager/amp_utils.h" #include "paddle/fluid/eager/api/manual/eager_manual/dygraph_forward_api.h" #include "paddle/fluid/eager/api/manual/eager_manual/nodes/nodes.h" #include "paddle/fluid/eager/api/utils/global_utils.h" -#include "paddle/fluid/eager/eager_amp_auto_cast.h" #include "paddle/fluid/eager/eager_layout_auto_tune.h" #include "paddle/fluid/eager/nan_inf_utils.h" #include "paddle/fluid/eager/type_promotion_utils.h" +#include "paddle/fluid/imperative/amp_utils.h" #include "paddle/fluid/platform/place.h" #include "paddle/fluid/platform/profiler/event_tracing.h" #include "paddle/phi/api/include/sparse_api.h" @@ -45,10 +44,13 @@ paddle::Tensor multiply_ad_func(const paddle::Tensor& x, paddle::small_vector, egr::kSlotSmallVectorSize> amp_tensors_vector = {{x}, {y}}; - auto amp_dst_dtype = egr::GetAmpDestDtype(op_name, amp_tensors_vector); + auto amp_dst_dtype = + paddle::imperative::GetAmpDestDtype(op_name, amp_tensors_vector); - auto new_x = egr::EagerAmpAutoCast("x", x, amp_dst_dtype, op_name); - auto new_y = egr::EagerAmpAutoCast("y", y, amp_dst_dtype, op_name); + auto new_x = + paddle::imperative::AmpAutoCast("x", x, amp_dst_dtype, op_name); + auto new_y = + paddle::imperative::AmpAutoCast("y", y, amp_dst_dtype, op_name); { paddle::imperative::AutoCastGuard guard( @@ -392,10 +394,13 @@ paddle::Tensor multiply_ad_func(const paddle::Tensor& x, paddle::small_vector, egr::kSlotSmallVectorSize> amp_tensors_vector = {{x}, {y}}; - auto amp_dst_dtype = egr::GetAmpDestDtype(op_name, amp_tensors_vector); + auto amp_dst_dtype = + paddle::imperative::GetAmpDestDtype(op_name, amp_tensors_vector); - auto new_x = egr::EagerAmpAutoCast("x", x, amp_dst_dtype, op_name); - auto new_y = egr::EagerAmpAutoCast("y", y, amp_dst_dtype, op_name); + auto new_x = + paddle::imperative::AmpAutoCast("x", x, amp_dst_dtype, op_name); + auto new_y = + paddle::imperative::AmpAutoCast("y", y, amp_dst_dtype, op_name); { paddle::imperative::AutoCastGuard guard( diff --git a/paddle/fluid/eager/api/manual/eager_manual/forwards/sync_batch_norm_fwd_func.cc b/paddle/fluid/eager/api/manual/eager_manual/forwards/sync_batch_norm_fwd_func.cc index c4e007801c66c..7c30de5ca75f3 100644 --- a/paddle/fluid/eager/api/manual/eager_manual/forwards/sync_batch_norm_fwd_func.cc +++ b/paddle/fluid/eager/api/manual/eager_manual/forwards/sync_batch_norm_fwd_func.cc @@ -13,11 +13,9 @@ // limitations under the License. #include "paddle/common/flags.h" -#include "paddle/fluid/eager/amp_utils.h" #include "paddle/fluid/eager/api/manual/eager_manual/dygraph_forward_api.h" #include "paddle/fluid/eager/api/manual/eager_manual/nodes/nodes.h" #include "paddle/fluid/eager/api/utils/global_utils.h" -#include "paddle/fluid/eager/eager_amp_auto_cast.h" #include "paddle/fluid/eager/eager_layout_auto_tune.h" #include "paddle/fluid/eager/nan_inf_utils.h" #include "paddle/fluid/platform/profiler/event_tracing.h" diff --git a/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_attention_fwd_func.cc b/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_attention_fwd_func.cc index 6130b79059f65..82ffaf5251a7d 100644 --- a/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_attention_fwd_func.cc +++ b/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_attention_fwd_func.cc @@ -14,10 +14,10 @@ #include "paddle/fluid/eager/accumulation/accumulation_node.h" #include "paddle/fluid/eager/amp_auto_cast.h" -#include "paddle/fluid/eager/amp_utils.h" #include "paddle/fluid/eager/api/manual/fluid_manual/dygraph_forward_api.h" #include "paddle/fluid/eager/api/manual/fluid_manual/nodes/nodes.h" #include "paddle/fluid/eager/api/utils/global_utils.h" +#include "paddle/fluid/imperative/amp_utils.h" #include "paddle/fluid/platform/profiler/event_tracing.h" std::tuple, egr::kSlotSmallVectorSize> amp_tensors_vector = {{X}, {Y}, {Bias}}; - auto amp_dst_dtype = - egr::GetAmpDestDtype("fused_gemm_epilogue", amp_tensors_vector); + auto amp_dst_dtype = paddle::imperative::GetAmpDestDtype( + "fused_gemm_epilogue", amp_tensors_vector); auto NEW_X = egr::AmpAutoCast("X", X, amp_dst_dtype, "fused_gemm_epilogue"); auto NEW_Y = egr::AmpAutoCast("Y", Y, amp_dst_dtype, "fused_gemm_epilogue"); diff --git a/paddle/fluid/eager/auto_code_generator/eager_generator.cc b/paddle/fluid/eager/auto_code_generator/eager_generator.cc index 66b4d05f68bf0..33d6da07f81a7 100644 --- a/paddle/fluid/eager/auto_code_generator/eager_generator.cc +++ b/paddle/fluid/eager/auto_code_generator/eager_generator.cc @@ -1751,7 +1751,7 @@ static std::pair GenerateForwardFunctionContents( amp_logic_str += "\n"; const char* GET_AMP_GET_DST_DTYPE_CONTEXT = " auto amp_dst_dtype = " - "egr::GetAmpDestDtype(\"%s\", " + "paddle::imperative::GetAmpDestDtype(\"%s\", " "amp_tensors_vector);\n"; amp_logic_str += paddle::string::Sprintf(GET_AMP_GET_DST_DTYPE_CONTEXT, op_type); @@ -3018,7 +3018,7 @@ static void GenerateForwardDygraphFile(const std::string& forward_cc_path, "#include " "\"paddle/fluid/eager/api/generated/fluid_generated/nodes/nodes.h\"\n" "#include \"paddle/fluid/eager/api/utils/global_utils.h\"\n" - "#include \"paddle/fluid/eager/amp_utils.h\"\n" + "#include \"paddle/fluid/imperative/amp_utils.h\"\n" "#include \"paddle/fluid/eager/amp_auto_cast.h\"\n" "#include \"paddle/fluid/platform/profiler/event_tracing.h\"\n\n"; diff --git a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py index 13ddbca4c9ef5..74fc6b9a7dbc6 100644 --- a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py +++ b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py @@ -452,8 +452,6 @@ class {} : public egr::GradNodeBase {{ #include "paddle/phi/api/include/sparse_api.h" #include "paddle/fluid/eager/api/utils/global_utils.h" #include "paddle/fluid/platform/profiler/event_tracing.h" -#include "paddle/fluid/eager/amp_utils.h" -#include "paddle/fluid/eager/eager_amp_auto_cast.h" #include "paddle/phi/backends/gpu/gpu_info.h" #include "paddle/fluid/eager/nan_inf_utils.h" #include "paddle/fluid/eager/api/manual/eager_manual/dygraph_forward_api.h" @@ -461,6 +459,8 @@ class {} : public egr::GradNodeBase {{ #include "paddle/phi/api/lib/data_transform.h" #include "paddle/fluid/eager/type_promotion_utils.h" #include "paddle/phi/common/type_promotion.h" +#include "paddle/fluid/imperative/amp_utils.h" + COMMON_DECLARE_bool(check_nan_inf); COMMON_DECLARE_string(tensor_operants_mode); {} @@ -1523,7 +1523,7 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): f"if ({name}) amp_tensors_vector.push_back({{ *{name} }});\n" ) amp_autocast_optional_list.append( - f"auto new_{name} = egr::EagerAmpAutoCast(\"{name}\", {name}, amp_dst_dtype, op_name);\n" + f"auto new_{name} = paddle::imperative::AmpAutoCast(\"{name}\", {name}, amp_dst_dtype, op_name);\n" ) layout_tensors_vector_optional_list.append( f"if ({name}) tensors_vector.push_back({{ *{name} }});\n" @@ -1540,13 +1540,13 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): arg_str = f"paddle::Tensor& {name}" amp_tensors_vector_list.append(f"{{{name}}}") amp_autocast_list.append( - f"auto new_{name} = egr::EagerAmpAutoCast(\"{name}\", {name}, amp_dst_dtype, op_name);\n" + f"auto new_{name} = paddle::imperative::AmpAutoCast(\"{name}\", {name}, amp_dst_dtype, op_name);\n" ) else: arg_str = f"const paddle::Tensor& {name}" amp_tensors_vector_list.append(f"{{{name}}}") amp_autocast_list.append( - f"auto new_{name} = egr::EagerAmpAutoCast(\"{name}\", {name}, amp_dst_dtype, op_name);\n" + f"auto new_{name} = paddle::imperative::AmpAutoCast(\"{name}\", {name}, amp_dst_dtype, op_name);\n" ) layout_autotune_list.append( f"auto new_{name} = transformer->TransInTensor(\"{name}\", {name});\n" @@ -1567,7 +1567,7 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): f"if ({name}) amp_tensors_vector.push_back( *{name} );\n" ) amp_autocast_optional_list.append( - f"auto new_{name} = egr::EagerAmpAutoCasts(\"{name}\", {name}, amp_dst_dtype, op_name);\n" + f"auto new_{name} = paddle::imperative::AmpAutoCasts(\"{name}\", {name}, amp_dst_dtype, op_name);\n" ) layout_autotune_optional_list.append( f"auto new_{name} = transformer->TransInTensors(\"{name}\", {name});\n" @@ -1583,7 +1583,7 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): arg_str = f"const std::vector& {name}" amp_tensors_vector_list.append(f"{name}") amp_autocast_list.append( - f"auto new_{name} = egr::EagerAmpAutoCasts(\"{name}\", {name}, amp_dst_dtype, op_name);\n" + f"auto new_{name} = paddle::imperative::AmpAutoCasts(\"{name}\", {name}, amp_dst_dtype, op_name);\n" ) layout_autotune_list.append( f"auto new_{name} = transformer->TransInTensors(\"{name}\", {name});\n" @@ -1817,7 +1817,7 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): amp_tensors_vector_optional_list_str = " ".join( amp_tensors_vector_optional_list ) - amp_get_dst_dtype_str = "auto amp_dst_dtype = egr::GetAmpDestDtype(op_name, amp_tensors_vector);\n" + amp_get_dst_dtype_str = "auto amp_dst_dtype = paddle::imperative::GetAmpDestDtype(op_name, amp_tensors_vector);\n" amp_autocast_list_str = ( " ".join(amp_autocast_list) + " " diff --git a/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py b/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py index 777eea1221429..ce160306e13cc 100644 --- a/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py +++ b/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py @@ -193,8 +193,6 @@ def FindParsingFunctionFromAttributeType(atype): #include "paddle/fluid/eager/api/manual/eager_manual/dygraph_forward_api.h" #include "paddle/fluid/pybind/eager_custom_python_api.h" #include "paddle/fluid/pybind/eager.h" -#include "paddle/fluid/eager/amp_utils.h" -#include "paddle/fluid/eager/eager_amp_auto_cast.h" #include "paddle/fluid/pybind/eager_op_function.h" namespace paddle {{ namespace pybind {{ diff --git a/paddle/fluid/eager/eager_amp_auto_cast.h b/paddle/fluid/eager/eager_amp_auto_cast.h deleted file mode 100644 index 08c0a51b1b972..0000000000000 --- a/paddle/fluid/eager/eager_amp_auto_cast.h +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include "paddle/fluid/eager/api/generated/eager_generated/forwards/dygraph_functions.h" - -namespace egr { - -static inline bool NeedCast(const paddle::Tensor& tensor, - const phi::DataType& dst_dtype) { - auto place = tensor.place(); - auto data_type = tensor.dtype(); - // Except CPU judgment, other conditions should be consistent with - // amp_utils.h's judgment - if (paddle::platform::is_gpu_place(place) || - paddle::platform::is_cuda_pinned_place(place) || - paddle::platform::is_xpu_place(place) || - paddle::platform::is_custom_place(place) || - paddle::platform::is_cpu_place(place)) { - // CudaPinnedPlace is added for varbase created by dataloader - // Cpu place is for different place tensor, when input1 is cpu and input2 is - // gpu - if ((data_type == phi::DataType::FLOAT32 || - data_type == phi::DataType::FLOAT16 || - data_type == phi::DataType::BFLOAT16) && - (data_type != dst_dtype)) { - return true; - } - } - return false; -} - -inline paddle::Tensor Cast(const paddle::Tensor& input, - const phi::DataType& dst_dtype, - const bool trace_backward = true) { - if (input.is_sparse_coo_tensor() || input.is_sparse_csr_tensor()) { - if (trace_backward) { - return sparse::cast_ad_func(input, phi::DataType::UNDEFINED, dst_dtype); - } else { - return paddle::experimental::sparse::cast( - input, phi::DataType::UNDEFINED, dst_dtype); - } - } else { - if (trace_backward) { - return cast_ad_func(input, dst_dtype); - } else { - return paddle::experimental::cast(input, dst_dtype); - } - } -} - -inline std::vector EagerAmpAutoCasts( - const std::string& inputs_name, - const std::vector& inputs, - const phi::DataType& dst_dtype, - std::string op_name UNUSED, - bool trace_backward UNUSED = true) { - VLOG(6) << "AMP AmpAutoCasts:" - << " inputs(" << inputs_name << ") dst_dtype(" - << phi::DataTypeToString(dst_dtype) << ")."; - std::vector inputs_casted; - for (auto& input : inputs) { - if (NeedCast(input, dst_dtype)) { - inputs_casted.emplace_back(std::move(Cast(input, dst_dtype))); - } else { - inputs_casted.emplace_back(input); - } - } - return inputs_casted; -} - -inline paddle::Tensor EagerAmpAutoCast(const std::string& input_name, - const paddle::Tensor& input, - const phi::DataType& dst_dtype, - const std::string& op_name, - bool trace_backward = true) { - VLOG(6) << "AMP AmpAutoCasts:" - << " input(" << egr::EagerUtils::TensorStr(input) << " to dst_dtype(" - << phi::DataTypeToString(dst_dtype) << ")."; - if ((op_name == "batch_norm" || op_name == "layer_norm" || - op_name == "sync_batch_norm" || op_name == "weight_only_linear") && - input_name != "x") { - return input; - } - - if (dst_dtype == phi::DataType::FLOAT16) { - if (op_name == "run_program") { - return input; - } - if ((op_name == "fused_attention" || op_name == "fused_feedforward")) { - if (input_name == "LnScale" || input_name == "LnBias" || - input_name == "Ln2Scale" || input_name == "Ln2Bias" || - input_name == "Ln1Scale" || input_name == "Ln1Bias") { - return input; - } - } - } - if (NeedCast(input, dst_dtype)) { - VLOG(6) << "Input : " << input.impl() << "NeedCast"; - return Cast(input, dst_dtype, trace_backward); - } - return input; -} - -inline paddle::optional EagerAmpAutoCast( - const std::string& input_name, - const paddle::optional& input, - const phi::DataType& dst_dtype, - const std::string& op_name, - bool trace_backward = true) { - if (input) { - return EagerAmpAutoCast( - input_name, *input, dst_dtype, op_name, trace_backward); - } - return paddle::none; -} - -inline paddle::optional> EagerAmpAutoCasts( - const std::string& inputs_name, - const paddle::optional>& inputs, - const phi::DataType& dst_dtype, - std::string op_name, - bool trace_backward = true) { - if (inputs) { - return EagerAmpAutoCasts( - inputs_name, *inputs, dst_dtype, op_name, trace_backward); - } - return paddle::optional>(); -} - -} // namespace egr diff --git a/paddle/fluid/eager/type_promotion_utils.h b/paddle/fluid/eager/type_promotion_utils.h index 0a1193fd6bf48..3ef732bac78bf 100644 --- a/paddle/fluid/eager/type_promotion_utils.h +++ b/paddle/fluid/eager/type_promotion_utils.h @@ -14,6 +14,7 @@ #pragma once #include "paddle/fluid/eager/api/utils/global_utils.h" +#include "paddle/fluid/imperative/amp_utils.h" #include "paddle/phi/common/type_promotion.h" namespace egr { @@ -23,7 +24,7 @@ inline paddle::Tensor PromoteCast(const std::string& input_name, const phi::DataType& dst_dtype, bool trace_backward = true) { if (input.dtype() != dst_dtype) { - return Cast(input, dst_dtype, trace_backward); + return paddle::imperative::Cast(input, dst_dtype, trace_backward); } else { return input; } diff --git a/paddle/fluid/imperative/amp_utils.h b/paddle/fluid/imperative/amp_utils.h new file mode 100644 index 0000000000000..37dcd48359e34 --- /dev/null +++ b/paddle/fluid/imperative/amp_utils.h @@ -0,0 +1,340 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include + +#if !(defined(PADDLE_NO_PYTHON) && defined(PADDLE_ON_INFERENCE)) +#include "paddle/fluid/eager/api/generated/eager_generated/forwards/dygraph_functions.h" +#endif +#include "paddle/fluid/eager/api/utils/global_utils.h" +#include "paddle/fluid/eager/type_defs.h" +#include "paddle/fluid/imperative/amp_auto_cast.h" +#include "paddle/fluid/pir/dialect/operator/ir/pd_api.h" +#include "paddle/fluid/pir/dialect/operator/utils/utils.h" +#include "paddle/phi/api/include/api.h" +#include "paddle/phi/api/include/sparse_api.h" +#include "paddle/utils/small_vector.h" + +namespace paddle { +namespace imperative { +static inline phi::DataType GetDataType(const pir::Value& value) { + return paddle::dialect::GetValueDataType(value); +} + +static inline phi::DataType GetDataType(const paddle::Tensor& tensor) { + return tensor.dtype(); +} + +template +static inline phi::DataType GetPromoteType( + const std::string& op_name, + const paddle::small_vector, egr::kSlotSmallVectorSize>& + amp_tensors_vector, + const phi::DataType& amp_dtype) { + auto dst_type = amp_dtype; + // only consider the dtype of input(X). + if (op_name == "batch_norm" || op_name == "layer_norm" || + op_name == "sync_batch_norm" || + op_name == "moving_average_abs_max_scale") { + if (GetDataType(amp_tensors_vector[0][0]) == phi::DataType::FLOAT32) { + dst_type = phi::DataType::FLOAT32; + } + return dst_type; + } + + if (egr::Controller::Instance().GetCurrentTracer()->GetAmpDtype() == + "float16") { + if (op_name == "fused_attention") { + for (size_t i = 0; i < amp_tensors_vector.size(); i++) { + if (i != 3 || i != 4 || i != 9 || i != 10) { + if (GetDataType(amp_tensors_vector[i][0]) == phi::DataType::FLOAT32) { + dst_type = phi::DataType::FLOAT32; + return dst_type; + } + } + } + } else if (op_name == "fused_feedforward") { + for (size_t i = 0; i < amp_tensors_vector.size(); i++) { + if (i != 7 || i != 8 || i != 9 || i != 10) { + if (GetDataType(amp_tensors_vector[i][0]) == phi::DataType::FLOAT32) { + dst_type = phi::DataType::FLOAT32; + return dst_type; + } + } + } + } + } + + for (const auto& tensors : amp_tensors_vector) { + for (const auto& tensor : tensors) { + if (GetDataType(tensor) == phi::DataType::FLOAT32) { + dst_type = GetDataType(tensor); + break; + } + } + } + + return dst_type; +} + +static inline phi::DataType GetDtypeWithPlace( + const std::string& op_name, + const paddle::small_vector, + egr::kSlotSmallVectorSize>& amp_tensors_vector, + const phi::DataType amp_dtype) { + if (amp_dtype == phi::DataType::FLOAT32) { + return amp_dtype; + } + bool is_right_place = false; + for (const auto& tensors : amp_tensors_vector) { + for (const auto& tensor : tensors) { + auto place = tensor.place(); + // TODO(lizhiyu): If the tensor is a dist-tensor, it's place may be + // `unknown` in the no-calculation rank right now. + // We use `is_dist_tensor()` to avoid the bug temporarily. The + // dist-tensor in the no-calculation rank should have the right + // place. + is_right_place = + (tensor.is_dist_tensor() || paddle::platform::is_gpu_place(place) || + paddle::platform::is_cuda_pinned_place(place) || + paddle::platform::is_xpu_place(place) || + paddle::platform::is_custom_place(place)); + if (is_right_place) { + break; + } + } + } + + if (!is_right_place) { + VLOG(6) << "Change " << op_name << "'s AMP type from " << amp_dtype + << " to FP32"; + return phi::DataType::FLOAT32; + } + return amp_dtype; +} + +static inline phi::DataType GetDtypeWithPlace( + const std::string& op_name UNUSED, + const paddle::small_vector, + egr::kSlotSmallVectorSize>& amp_tensors_vector + UNUSED, + const phi::DataType amp_dtype) { + return amp_dtype; +} + +template +inline phi::DataType GetAmpDestDtype( + const std::string& op_name, + const paddle::small_vector, egr::kSlotSmallVectorSize>& + amp_tensors_vector) { + auto amp_level = egr::Controller::Instance().GetAMPLevel(); + auto amp_setting_dtype = + egr::Controller::Instance().GetCurrentTracer()->GetAmpPhiDtype(); + auto dst_type = amp_setting_dtype; + + bool use_promote = true; + if (amp_level == paddle::imperative::AmpLevel::O2) { + use_promote = + egr::Controller::Instance().GetCurrentTracer()->GetUsePromote(); + } + + if (use_promote) { + if (paddle::imperative::AmpOperators::Instance() + .GetMutableAllowOps() + ->count(op_name)) { + dst_type = amp_setting_dtype; + } else if (paddle::imperative::AmpOperators::Instance() + .GetMutableBlockOps() + ->count(op_name)) { + dst_type = phi::DataType::FLOAT32; + } else { + if (amp_level == paddle::imperative::AmpLevel::OD) { + dst_type = phi::DataType::FLOAT32; + } else { + dst_type = + GetPromoteType(op_name, amp_tensors_vector, amp_setting_dtype); + } + } + } else { + // use_promote can be set to false only for O2 training. + if (paddle::imperative::AmpOperators::Instance() + .GetMutableBlockOps() + ->count(op_name)) { + dst_type = phi::DataType::FLOAT32; + } + } + + if (dst_type == amp_setting_dtype && + (paddle::imperative::AmpOperators::Instance() + .GetMutableUnsupportedOps(amp_setting_dtype) + ->count(op_name))) { + dst_type = phi::DataType::FLOAT32; + } + + dst_type = GetDtypeWithPlace(op_name, amp_tensors_vector, dst_type); + VLOG(6) << "AMP GetAmpDestDtype:" + << " op(" << op_name << ") amp_dtype(" << dst_type << ") amp_level(" + << static_cast(amp_level) << ")."; + return dst_type; +} + +static inline bool NeedCast(const paddle::Tensor& tensor, + const phi::DataType& dst_dtype) { + auto place = tensor.place(); + auto data_type = tensor.dtype(); + // Except CPU judgment, other conditions should be consistent with + // amp_utils.h's judgment + if (paddle::platform::is_gpu_place(place) || + paddle::platform::is_cuda_pinned_place(place) || + paddle::platform::is_xpu_place(place) || + paddle::platform::is_custom_place(place) || + paddle::platform::is_cpu_place(place)) { + // CudaPinnedPlace is added for varbase created by dataloader + // Cpu place is for different place tensor, when input1 is cpu and input2 + // is gpu + if ((data_type == phi::DataType::FLOAT32 || + data_type == phi::DataType::FLOAT16 || + data_type == phi::DataType::BFLOAT16) && + (data_type != dst_dtype)) { + return true; + } + } + return false; +} + +static inline bool NeedCast(const pir::Value& value, + const phi::DataType& dst_dtype) { + auto data_type = paddle::dialect::GetValueDataType(value); + if ((data_type == phi::DataType::FLOAT32 || + data_type == phi::DataType::FLOAT16 || + data_type == phi::DataType::BFLOAT16) && + (data_type != dst_dtype)) { + return true; + } + return false; +} + +#if !(defined(PADDLE_NO_PYTHON) && defined(PADDLE_ON_INFERENCE)) +static inline paddle::Tensor Cast(const paddle::Tensor& input, + const phi::DataType& dst_dtype, + const bool trace_backward = true) { + if (input.is_sparse_coo_tensor() || input.is_sparse_csr_tensor()) { + if (trace_backward) { + return sparse::cast_ad_func(input, phi::DataType::UNDEFINED, dst_dtype); + } else { + return paddle::experimental::sparse::cast( + input, phi::DataType::UNDEFINED, dst_dtype); + } + } else { + if (trace_backward) { + return cast_ad_func(input, dst_dtype); + } else { + return paddle::experimental::cast(input, dst_dtype); + } + } +} +#endif + +static inline pir::Value Cast(const pir::Value& input, + const phi::DataType& dst_dtype, + const bool trace_backward UNUSED = true) { + paddle::imperative::AutoCastGuard guard( + egr::Controller::Instance().GetCurrentAmpAttrs(), + paddle::imperative::AmpLevel::O0); + return paddle::dialect::cast(input, dst_dtype); +} + +template +inline std::vector AmpAutoCasts(const std::string& inputs_name, + const std::vector& inputs, + const phi::DataType& dst_dtype, + std::string op_name UNUSED, + bool trace_backward UNUSED = true) { + VLOG(6) << "AMP AmpAutoCasts:" + << " inputs(" << inputs_name << ") dst_dtype(" + << phi::DataTypeToString(dst_dtype) << ")."; + std::vector inputs_casted; + for (auto& input : inputs) { + if (NeedCast(input, dst_dtype)) { + inputs_casted.emplace_back(std::move(Cast(input, dst_dtype))); + } else { + inputs_casted.emplace_back(input); + } + } + return inputs_casted; +} + +template +inline T AmpAutoCast(const std::string& input_name, + const T& input, + const phi::DataType& dst_dtype, + const std::string& op_name, + bool trace_backward = true) { + VLOG(6) << "AMP AmpAutoCasts:" + << " input(" << input_name << ") dst_dtype(" + << phi::DataTypeToString(dst_dtype) << ")."; + if ((op_name == "batch_norm" || op_name == "layer_norm" || + op_name == "sync_batch_norm" || op_name == "weight_only_linear") && + input_name != "x") { + return input; + } + + if (dst_dtype == phi::DataType::FLOAT16) { + if (op_name == "run_program") { + return input; + } + if ((op_name == "fused_attention" || op_name == "fused_feedforward")) { + if (input_name == "LnScale" || input_name == "LnBias" || + input_name == "Ln2Scale" || input_name == "Ln2Bias" || + input_name == "Ln1Scale" || input_name == "Ln1Bias") { + return input; + } + } + } + if (NeedCast(input, dst_dtype)) { + VLOG(6) << "Input : " << input.impl() << "NeedCast"; + return Cast(input, dst_dtype, trace_backward); + } + return input; +} + +template +inline paddle::optional AmpAutoCast(const std::string& input_name, + const paddle::optional& input, + const phi::DataType& dst_dtype, + const std::string& op_name, + bool trace_backward = true) { + if (input) { + return AmpAutoCast(input_name, *input, dst_dtype, op_name, trace_backward); + } + return paddle::none; +} + +template +inline paddle::optional> AmpAutoCasts( + const std::string& inputs_name, + const paddle::optional>& inputs, + const phi::DataType& dst_dtype, + std::string op_name, + bool trace_backward = true) { + if (inputs) { + return AmpAutoCasts( + inputs_name, *inputs, dst_dtype, op_name, trace_backward); + } + return paddle::optional>(); +} + +} // namespace imperative +} // namespace paddle diff --git a/paddle/fluid/pir/dialect/op_generator/api_gen.py b/paddle/fluid/pir/dialect/op_generator/api_gen.py index 1d62d14213dda..d3c1a718a61b3 100644 --- a/paddle/fluid/pir/dialect/op_generator/api_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/api_gen.py @@ -58,8 +58,9 @@ #include "paddle/pir/include/core/builtin_op.h" #include "paddle/fluid/pir/dialect/operator/utils/utils.h" #include "paddle/fluid/imperative/amp_auto_cast.h" -#include "paddle/fluid/pir/dialect/operator/utils/amp_utils.h" +#include "paddle/fluid/imperative/amp_utils.h" #include "paddle/fluid/eager/api/utils/global_utils.h" +#include "paddle/fluid/eager/type_defs.h" {body} @@ -102,9 +103,9 @@ if (egr::Controller::Instance().GetCurrentAmpAttrs()->GetAmpLevel() != paddle::imperative::AmpLevel::O0){{ VLOG(5) << "Check and Prepare For AMP"; auto op_name = phi::TransToFluidOpName("{op_name}"); - std::vector> amp_values_vector = {{ {no_optional_inputs} }}; + paddle::small_vector, egr::kSlotSmallVectorSize> amp_values_vector = {{ {no_optional_inputs} }}; {optional_inputs} - auto amp_dst_dtype = paddle::dialect::GetAmpDestDtype("{op_name}", amp_values_vector); + auto amp_dst_dtype = paddle::imperative::GetAmpDestDtype("{op_name}", amp_values_vector); {new_inputs} {{ paddle::imperative::AutoCastGuard guard(egr::Controller::Instance().GetCurrentAmpAttrs(), paddle::imperative::AmpLevel::O0); @@ -116,7 +117,7 @@ AMP_OPTIONAL_INPUTS_TEMPLATE = """if ({optional_input}) amp_values_vector.push_back({vec_optional_input}); """ -AMP_NEW_INPUTS_TEMPLATE = """auto new_{input} = paddle::dialect::PirAmpAutoCast("{input}", {input}, amp_dst_dtype, op_name); +AMP_NEW_INPUTS_TEMPLATE = """auto new_{input} = paddle::imperative::{cast_func}("{input}", {input}, amp_dst_dtype, op_name); """ OP_DISPATCH_TEMPLATE = """ @@ -629,9 +630,13 @@ def _gen_amp_optional_inputs(self, op_info): def _gen_amp_new_inputs(self, op_info, op_name): name_list = op_info.input_name_list + type_list = op_info.input_type_list ret = '' - for name in name_list: - ret += AMP_NEW_INPUTS_TEMPLATE.format(input=name, op_name=op_name) + for name, type in zip(name_list, type_list): + cast_func = 'AmpAutoCasts' if VECTOR_TYPE in type else 'AmpAutoCast' + ret += AMP_NEW_INPUTS_TEMPLATE.format( + input=name, cast_func=cast_func + ) return ret def _gen_amp_args(self, op_info, is_mutable_attr): diff --git a/paddle/fluid/pir/dialect/operator/utils/amp_utils.cc b/paddle/fluid/pir/dialect/operator/utils/amp_utils.cc deleted file mode 100644 index ac631baf66ee6..0000000000000 --- a/paddle/fluid/pir/dialect/operator/utils/amp_utils.cc +++ /dev/null @@ -1,219 +0,0 @@ -// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "paddle/fluid/pir/dialect/operator/utils/amp_utils.h" -#include "paddle/fluid/eager/api/utils/global_utils.h" -#include "paddle/fluid/imperative/amp_auto_cast.h" -#include "paddle/fluid/pir/dialect/operator/ir/pd_api.h" -#include "paddle/fluid/pir/dialect/operator/utils/utils.h" - -namespace paddle { -namespace dialect { - -phi::DataType GetPromoteType( - const std::string& op_name, - const std::vector>& amp_values_vector, - const phi::DataType& amp_dtype) { - auto dst_type = amp_dtype; - // only consider the dtype of input(X). - if (op_name == "batch_norm" || op_name == "layer_norm" || - op_name == "sync_batch_norm" || - op_name == "moving_average_abs_max_scale") { - if (GetValueDataType(amp_values_vector[0][0]) == phi::DataType::FLOAT32) { - dst_type = phi::DataType::FLOAT32; - } - return dst_type; - } - - if (egr::Controller::Instance().GetCurrentAmpAttrs()->GetAmpDtype() == - "float16") { - if (op_name == "fused_attention") { - for (size_t i = 0; i < amp_values_vector.size(); i++) { - if (i != 3 || i != 4 || i != 9 || i != 10) { - if (GetValueDataType(amp_values_vector[i][0]) == - phi::DataType::FLOAT32) { - dst_type = phi::DataType::FLOAT32; - return dst_type; - } - } - } - } else if (op_name == "fused_feedforward") { - for (size_t i = 0; i < amp_values_vector.size(); i++) { - if (i != 7 || i != 8 || i != 9 || i != 10) { - if (GetValueDataType(amp_values_vector[i][0]) == - phi::DataType::FLOAT32) { - dst_type = phi::DataType::FLOAT32; - return dst_type; - } - } - } - } - } - - for (const auto& values : amp_values_vector) { - for (const auto& value : values) { - if (GetValueDataType(value) == phi::DataType::FLOAT32) { - dst_type = GetValueDataType(value); - break; - } - } - } - - return dst_type; -} - -pir::Value Cast(const pir::Value& input, const phi::DataType& dst_dtype) { - paddle::imperative::AutoCastGuard guard( - egr::Controller::Instance().GetCurrentAmpAttrs(), - paddle::imperative::AmpLevel::O0); - return paddle::dialect::cast(input, dst_dtype); -} - -bool NeedCast(const pir::Value& value, const phi::DataType& dst_dtype) { - auto data_type = GetValueDataType(value); - if ((data_type == phi::DataType::FLOAT32 || - data_type == phi::DataType::FLOAT16 || - data_type == phi::DataType::BFLOAT16) && - (data_type != dst_dtype)) { - return true; - } - return false; -} - -pir::Value PirAmpAutoCast(const std::string& input_name, - const pir::Value& input, - const phi::DataType& dst_dtype, - const std::string& op_name) { - VLOG(6) << "AMP AmpAutoCasts:" - << " input(" << input_name << " to dst_dtype(" - << phi::DataTypeToString(dst_dtype) << ")."; - if ((op_name == "batch_norm" || op_name == "layer_norm" || - op_name == "sync_batch_norm" || op_name == "weight_only_linear") && - input_name != "x") { - return input; - } - - if (dst_dtype == phi::DataType::FLOAT16) { - if (op_name == "run_program") { - return input; - } - if ((op_name == "fused_attention" || op_name == "fused_feedforward")) { - if (input_name == "LnScale" || input_name == "LnBias" || - input_name == "Ln2Scale" || input_name == "Ln2Bias" || - input_name == "Ln1Scale" || input_name == "Ln1Bias") { - return input; - } - } - } - if (NeedCast(input, dst_dtype)) { - VLOG(6) << "Input : " << input_name << "NeedCast"; - return Cast(input, dst_dtype); - } - return input; -} - -paddle::optional PirAmpAutoCast( - const std::string& input_name, - const paddle::optional& input, - const phi::DataType& dst_dtype, - const std::string& op_name) { - if (input) { - return PirAmpAutoCast(input_name, *input, dst_dtype, op_name); - } - return paddle::none; -} - -std::vector PirAmpAutoCast(const std::string& inputs_name, - const std::vector& inputs, - const phi::DataType& dst_dtype, - const std::string& op_name) { - VLOG(6) << "AMP AmpAutoCasts:" - << " inputs(" << inputs_name << ") dst_dtype(" - << phi::DataTypeToString(dst_dtype) << ")."; - std::vector inputs_casted; - for (auto& input : inputs) { - if (NeedCast(input, dst_dtype)) { - inputs_casted.emplace_back(std::move(Cast(input, dst_dtype))); - } else { - inputs_casted.emplace_back(input); - } - } - return inputs_casted; -} - -paddle::optional> PirAmpAutoCast( - const std::string& inputs_name, - const paddle::optional>& inputs, - const phi::DataType& dst_dtype, - const std::string& op_name) { - if (inputs) { - return PirAmpAutoCast(inputs_name, *inputs, dst_dtype, op_name); - } - return paddle::optional>(); -} - -phi::DataType GetAmpDestDtype( - const std::string& op_name, - const std::vector>& amp_values_vector) { - auto amp_level = egr::Controller::Instance().GetAMPLevel(); - auto amp_setting_dtype = - egr::Controller::Instance().GetCurrentAmpAttrs()->GetAmpPhiDtype(); - auto dst_type = amp_setting_dtype; - - bool use_promote = true; - if (amp_level == paddle::imperative::AmpLevel::O2) { - use_promote = - egr::Controller::Instance().GetCurrentAmpAttrs()->GetUsePromote(); - } - - if (use_promote) { - if (paddle::imperative::AmpOperators::Instance() - .GetMutableAllowOps() - ->count(op_name)) { - dst_type = amp_setting_dtype; - } else if (paddle::imperative::AmpOperators::Instance() - .GetMutableBlockOps() - ->count(op_name)) { - dst_type = phi::DataType::FLOAT32; - } else { - if (amp_level == paddle::imperative::AmpLevel::OD) { - dst_type = phi::DataType::FLOAT32; - } else { - dst_type = - GetPromoteType(op_name, amp_values_vector, amp_setting_dtype); - } - } - } else { - // use_promote can be set to false only for O2 training. - if (paddle::imperative::AmpOperators::Instance() - .GetMutableBlockOps() - ->count(op_name)) { - dst_type = phi::DataType::FLOAT32; - } - } - - if (dst_type == amp_setting_dtype && - (paddle::imperative::AmpOperators::Instance() - .GetMutableUnsupportedOps(amp_setting_dtype) - ->count(op_name))) { - dst_type = phi::DataType::FLOAT32; - } - - VLOG(6) << "AMP GetAmpDestDtype:" - << " op(" << op_name << ") amp_dtype(" << dst_type << ") amp_level(" - << static_cast(amp_level) << ")."; - return dst_type; -} -} // namespace dialect -} // namespace paddle diff --git a/paddle/fluid/pir/dialect/operator/utils/amp_utils.h b/paddle/fluid/pir/dialect/operator/utils/amp_utils.h deleted file mode 100644 index e8f44fe584c49..0000000000000 --- a/paddle/fluid/pir/dialect/operator/utils/amp_utils.h +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include "paddle/phi/common/data_type.h" -#include "paddle/pir/include/core/value.h" -#include "paddle/utils/optional.h" - -namespace paddle { -namespace dialect { - -phi::DataType GetPromoteType( - const std::string& op_name, - const std::vector>& amp_tensors_vector, - const phi::DataType& amp_dtype); - -pir::Value Cast(const pir::Value& input, const phi::DataType& dst_dtype); - -bool NeedCast(const pir::Value& value, const phi::DataType& dst_dtype); - -pir::Value PirAmpAutoCast(const std::string& input_name, - const pir::Value& input, - const phi::DataType& dst_dtype, - const std::string& op_name); - -paddle::optional PirAmpAutoCast( - const std::string& input_name, - const paddle::optional& input, - const phi::DataType& dst_dtype, - const std::string& op_name); - -std::vector PirAmpAutoCast(const std::string& inputs_name, - const std::vector& inputs, - const phi::DataType& dst_dtype, - const std::string& op_name); - -paddle::optional> PirAmpAutoCast( - const std::string& inputs_name, - const paddle::optional>& inputs, - const phi::DataType& dst_dtype, - const std::string& op_name); - -phi::DataType GetAmpDestDtype( - const std::string& op_name, - const std::vector>& amp_values_vector); - -} // namespace dialect -} // namespace paddle diff --git a/paddle/fluid/pybind/CMakeLists.txt b/paddle/fluid/pybind/CMakeLists.txt index b72be65e98e75..27b47485f29df 100755 --- a/paddle/fluid/pybind/CMakeLists.txt +++ b/paddle/fluid/pybind/CMakeLists.txt @@ -119,8 +119,6 @@ set(PYBIND_SRCS pybind.cc imperative.cc inference_api.cc - ops_api.cc - static_op_function.cc control_flow_api.cc pir.cc graph.cc @@ -509,6 +507,8 @@ if(WITH_PYTHON) set(PYBIND_SRCS eager_legacy_op_function.cc ${PYBIND_SRCS}) set(PYBIND_SRCS eager_op_function.cc ${PYBIND_SRCS}) set(PYBIND_SRCS eager_math_op_patch.cc ${PYBIND_SRCS}) + set(PYBIND_SRCS ops_api.cc ${PYBIND_SRCS}) + set(PYBIND_SRCS static_op_function.cc ${PYBIND_SRCS}) list(APPEND PYBIND_DEPS eager_api) list(APPEND PYBIND_DEPS autograd_meta) list(APPEND PYBIND_DEPS backward) diff --git a/paddle/fluid/pybind/eager_method.cc b/paddle/fluid/pybind/eager_method.cc index 09fb067f41dee..6cefe60285f05 100644 --- a/paddle/fluid/pybind/eager_method.cc +++ b/paddle/fluid/pybind/eager_method.cc @@ -55,11 +55,10 @@ typedef SSIZE_T ssize_t; #pragma GCC diagnostic ignored "-Wmissing-field-initializers" #include "paddle/common/ddim.h" #include "paddle/common/flags.h" -#include "paddle/fluid/eager/amp_utils.h" #include "paddle/fluid/eager/api/generated/eager_generated/backwards/nodes.h" #include "paddle/fluid/eager/api/generated/eager_generated/forwards/dygraph_functions.h" -#include "paddle/fluid/eager/eager_amp_auto_cast.h" #include "paddle/fluid/framework/python_headers.h" +#include "paddle/fluid/imperative/amp_utils.h" #include "paddle/fluid/memory/allocation/mmap_allocator.h" #include "paddle/fluid/pybind/tensor_py.h" #include "paddle/phi/core/distributed/auto_parallel/dist_tensor.h" @@ -1613,10 +1612,11 @@ static PyObject* tensor__setitem_dygraph(TensorObject* self, paddle::small_vector, egr::kSlotSmallVectorSize> tmps = {{self->tensor}, {value_tensor}}; - auto amp_dtype = egr::GetAmpDestDtype("set_value", tmps); - self->tensor = egr::EagerAmpAutoCast( + auto amp_dtype = + paddle::imperative::GetAmpDestDtype("set_value", tmps); + self->tensor = paddle::imperative::AmpAutoCast( self->tensor.name(), self->tensor, amp_dtype, "set_value"); - value_tensor = egr::EagerAmpAutoCast( + value_tensor = paddle::imperative::AmpAutoCast( value_tensor.name(), value_tensor, amp_dtype, "set_value"); } if (self->tensor.dtype() != value_tensor.dtype()) { @@ -1707,10 +1707,11 @@ static PyObject* tensor__setitem_dygraph(TensorObject* self, paddle::small_vector, egr::kSlotSmallVectorSize> tmps = {{self->tensor}, {value_tensor}}; - auto amp_dtype = egr::GetAmpDestDtype("index_put", tmps); - self->tensor = egr::EagerAmpAutoCast( + auto amp_dtype = + paddle::imperative::GetAmpDestDtype("index_put", tmps); + self->tensor = paddle::imperative::AmpAutoCast( self->tensor.name(), self->tensor, amp_dtype, "index_put"); - value_tensor = egr::EagerAmpAutoCast( + value_tensor = paddle::imperative::AmpAutoCast( value_tensor.name(), value_tensor, amp_dtype, "index_put"); } if (self->tensor.dtype() != value_tensor.dtype()) { From 229d94539cfacc50f3895c05c1385b783a152297 Mon Sep 17 00:00:00 2001 From: Jia Wenxuan <64853160+JiaWenxuan@users.noreply.github.com> Date: Mon, 26 Feb 2024 19:10:25 +0800 Subject: [PATCH 098/282] Fix build error when switching cmake option (#61991) * fix some bugs * fix code style problem * modify .gitignore --- .gitignore | 1 - paddle/fluid/pir/dialect/op_generator/op_creator_drr_gen.py | 5 +++++ paddle/fluid/pir/drr/CMakeLists.txt | 6 +++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 70088635f954e..8f87c4094fce1 100644 --- a/.gitignore +++ b/.gitignore @@ -107,4 +107,3 @@ paddle/phi/kernels/fusion/cutlass/cutlass_kernels/fpA_intB_gemm/autogen/* paddle/phi/kernels/fusion/cutlass/cutlass_kernels/fpA_intB_gemm/autogen_tmp/* paddle/fluid/pybind/static_op_function.* paddle/fluid/pybind/ops_api.cc -paddle/fluid/pir/drr/src/*_op_factory_generated.* diff --git a/paddle/fluid/pir/dialect/op_generator/op_creator_drr_gen.py b/paddle/fluid/pir/dialect/op_generator/op_creator_drr_gen.py index e205e295fd9ef..63eccc8eb018f 100644 --- a/paddle/fluid/pir/dialect/op_generator/op_creator_drr_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/op_creator_drr_gen.py @@ -13,6 +13,7 @@ # limitations under the License. import argparse +import os import yaml from op_gen import ( @@ -199,6 +200,10 @@ def gen_cpp_file_code(self, cpp_file_path): ), ) + directory_path = os.path.dirname(cpp_file_path) + if not os.path.exists(directory_path): + os.makedirs(directory_path, exist_ok=True) + with open(cpp_file_path, 'w') as f: f.write( CPP_FILE_TEMPLATE.format( diff --git a/paddle/fluid/pir/drr/CMakeLists.txt b/paddle/fluid/pir/drr/CMakeLists.txt index fa65eb1e4bf9e..512e3927004e4 100644 --- a/paddle/fluid/pir/drr/CMakeLists.txt +++ b/paddle/fluid/pir/drr/CMakeLists.txt @@ -29,7 +29,7 @@ set(op_yaml_files ) set(pd_op_creator_file - ${PADDLE_SOURCE_DIR}/paddle/fluid/pir/drr/src/pd_op_factory_generated.cc) + ${PADDLE_BINARY_DIR}/paddle/fluid/pir/drr/src/pd_op_factory_generated.cc) set(pd_op_creator_file_tmp ${pd_op_creator_file}.tmp) set(pd_dialect_name pd_op) @@ -62,7 +62,7 @@ if(WITH_CINN AND NOT CINN_ONLY) ${PADDLE_SOURCE_DIR}/paddle/cinn/hlir/dialect/operator/ir/ops.yaml) set(cinn_op_creator_file - ${PADDLE_SOURCE_DIR}/paddle/fluid/pir/drr/src/cinn_op_factory_generated.cc + ${PADDLE_BINARY_DIR}/paddle/fluid/pir/drr/src/cinn_op_factory_generated.cc ) set(cinn_op_creator_file_tmp ${cinn_op_creator_file}.tmp) set(cinn_dialect_name cinn_op) @@ -96,7 +96,7 @@ if(WITH_MKLDNN) ${PADDLE_SOURCE_DIR}/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml ) set(onednn_op_creator_file - ${PADDLE_SOURCE_DIR}/paddle/fluid/pir/drr/src/onednn_op_factory_generated.cc + ${PADDLE_BINARY_DIR}/paddle/fluid/pir/drr/src/onednn_op_factory_generated.cc ) set(onednn_op_creator_file_tmp ${onednn_op_creator_file}.tmp) From 044dfe1e3ae0f089873be419c65c287fe5c288a3 Mon Sep 17 00:00:00 2001 From: LiYuRio <63526175+LiYuRio@users.noreply.github.com> Date: Mon, 26 Feb 2024 19:23:30 +0800 Subject: [PATCH 099/282] add dense to dist inplace api (#62014) --- paddle/fluid/pybind/eager_method.cc | 25 ++++++++++ .../paddle/distributed/auto_parallel/api.py | 6 +-- .../semi_dense_tensor_to_dist_api.py | 50 +++++++++++++++++++ test/auto_parallel/test_dist_tensor_api.py | 10 ++++ 4 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 test/auto_parallel/semi_dense_tensor_to_dist_api.py diff --git a/paddle/fluid/pybind/eager_method.cc b/paddle/fluid/pybind/eager_method.cc index 6cefe60285f05..6fe07282a2223 100644 --- a/paddle/fluid/pybind/eager_method.cc +++ b/paddle/fluid/pybind/eager_method.cc @@ -973,6 +973,27 @@ static PyObject* tensor__zero_grads(TensorObject* self, EAGER_CATCH_AND_THROW_RETURN_NULL } +static PyObject* tensor__to_dist(TensorObject* self, + PyObject* args, + PyObject* kwargs) { + EAGER_TRY + const auto& placements = + CastPyArg2VectorOfPlacement(PyTuple_GET_ITEM(args, 0), 0); + const auto& mesh = CastPyArg2ProcessMesh(PyTuple_GET_ITEM(args, 1), 1); + + if (self->tensor.is_dense_tensor()) { + const auto& dense_tensor_ptr = + std::static_pointer_cast(self->tensor.impl()); + auto dist_tensor_ptr = std::make_shared( + dense_tensor_ptr, mesh, placements); + self->tensor.set_impl(dist_tensor_ptr); + } + + RETURN_PY_NONE + + EAGER_CATCH_AND_THROW_RETURN_NULL +} + static PyObject* tensor__share_buffer_to(TensorObject* self, PyObject* args, PyObject* kwargs) { @@ -3218,6 +3239,10 @@ PyMethodDef variable_methods[] = { // NOLINT (PyCFunction)(void (*)())tensor__zero_grads, METH_VARARGS | METH_KEYWORDS, nullptr}, + {"_to_dist_", + (PyCFunction)(void (*)())tensor__to_dist, + METH_VARARGS | METH_KEYWORDS, + nullptr}, {"_share_buffer_to", (PyCFunction)(void (*)())tensor__share_buffer_to, METH_VARARGS | METH_KEYWORDS, diff --git a/python/paddle/distributed/auto_parallel/api.py b/python/paddle/distributed/auto_parallel/api.py index 73a69c91b74a4..d5de3545b2ea6 100644 --- a/python/paddle/distributed/auto_parallel/api.py +++ b/python/paddle/distributed/auto_parallel/api.py @@ -864,15 +864,11 @@ def __init__(self, mesh): self._mesh = mesh def _shard_parameter(self, param): - # TODO(liyurui): remove this trick dense to dist convert after adding - # dense_tensor.to_dist method. if param.is_dense(): - zero_dense = paddle.zeros(param.shape) placements = [] for _ in range(len(self._mesh.shape)): placements.append(dist.Replicate()) - zero_dist = dist.shard_tensor(zero_dense, self._mesh, placements) - res = param + zero_dist + param._to_dist_(placements, self._mesh) new_placements = get_placement_with_sharding(param) shard_param = dist.reshard(param, param.process_mesh, new_placements) diff --git a/test/auto_parallel/semi_dense_tensor_to_dist_api.py b/test/auto_parallel/semi_dense_tensor_to_dist_api.py new file mode 100644 index 0000000000000..da09846eadbd7 --- /dev/null +++ b/test/auto_parallel/semi_dense_tensor_to_dist_api.py @@ -0,0 +1,50 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import unittest + +import numpy as np + +import paddle +import paddle.distributed as dist + + +class TestDenseTensorToDistAPI(unittest.TestCase): + def setUp(self): + self._shape = eval(os.getenv("shape")) + self._dtype = os.getenv("dtype") + self._seed = 2023 + self._backend = os.getenv("backend") + self._mesh = dist.ProcessMesh([0, 1], dim_names=["x"]) + paddle.seed(self._seed) + np.random.seed(self._seed) + + def run_test_dense_tensor_to_dist_api(self): + if self._backend == "cpu": + paddle.set_device("cpu") + place = paddle.CPUPlace() + elif self._backend == "gpu": + place = paddle.CUDAPlace(dist.get_rank()) + + dense_dist_tensor = paddle.rand([4, 10]) + dense_dist_tensor._to_dist_([dist.Replicate()], self._mesh) + assert dense_dist_tensor.is_dist() + + def test_case(self): + self.run_test_dense_tensor_to_dist_api() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/auto_parallel/test_dist_tensor_api.py b/test/auto_parallel/test_dist_tensor_api.py index 73d45a2e019fa..2328cc4c532d5 100644 --- a/test/auto_parallel/test_dist_tensor_api.py +++ b/test/auto_parallel/test_dist_tensor_api.py @@ -50,6 +50,16 @@ def test_dtensor_from_local_api(self): user_defined_envs=envs, ) + def test_dense_tensor_to_dist_api(self): + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) + for envs in envs_list: + self.run_test_case( + "semi_dense_tensor_to_dist_api.py", + user_defined_envs=envs, + ) + if __name__ == "__main__": unittest.main() From eea10b17b24b80dcad2a6c955ad6cc1925adaa0b Mon Sep 17 00:00:00 2001 From: gouzil <66515297+gouzil@users.noreply.github.com> Date: Mon, 26 Feb 2024 20:41:39 +0800 Subject: [PATCH 100/282] [SOT][3.12] Support `CALL_INTRINSIC_1` opcode in Python 3.12 (#61995) --------- Co-authored-by: Nyakku Shigure --- .../opcode_translator/executor/instr_flag.py | 17 +++++++++++++++++ .../executor/opcode_executor.py | 14 ++++++++++++++ test/sot/skip_files_py312 | 2 -- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/python/paddle/jit/sot/opcode_translator/executor/instr_flag.py b/python/paddle/jit/sot/opcode_translator/executor/instr_flag.py index 1dd795439d459..448eac6ff95c2 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/instr_flag.py +++ b/python/paddle/jit/sot/opcode_translator/executor/instr_flag.py @@ -13,6 +13,7 @@ # limitations under the License. # flags for instructions +from enum import Enum class FORMAT_VALUE_FLAG: @@ -34,3 +35,19 @@ class MAKE_FUNCTION_FLAG: class CALL_FUNCTION_EX_FLAG: CFE_HAS_KWARGS = 0x01 + + +# see https://github.com/python/cpython/blob/3.12/Python/intrinsics.c#L211-L225 +class IntrinsicsUnaryFunctions(Enum): + INTRINSIC_1_INVALID = 0 + INTRINSIC_PRINT = 1 # no support, only non-interactive mode + INTRINSIC_IMPORT_STAR = 2 # no support, `from module import *` + INTRINSIC_STOPITERATION_ERROR = 3 # no support, generator or coroutine + INTRINSIC_ASYNC_GEN_WRAP = 4 # no support, async + INTRINSIC_UNARY_POSITIVE = 5 + INTRINSIC_LIST_TO_TUPLE = 6 + INTRINSIC_TYPEVAR = 7 # no support, PEP 695 + INTRINSIC_PARAMSPEC = 8 # no support, PEP 695 + INTRINSIC_TYPEVARTUPLE = 9 # no support, PEP 695 + INTRINSIC_SUBSCRIPT_GENERIC = 10 # no support, PEP 695 + INTRINSIC_TYPEALIAS = 11 # no support, PEP 695 diff --git a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py index 3b40633a73e25..bd4dde84918ff 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py +++ b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py @@ -62,6 +62,7 @@ CALL_FUNCTION_EX_FLAG as CFE, FORMAT_VALUE_FLAG as FV, MAKE_FUNCTION_FLAG as MF, + IntrinsicsUnaryFunctions, ) from .pycode_generator import PyCodeGen from .tracker import ( @@ -1545,6 +1546,19 @@ def LIST_TO_TUPLE(self, instr: Instruction): ) ) + def CALL_INTRINSIC_1(self, instr: Instruction): + intrinsic_func = IntrinsicsUnaryFunctions(instr.arg) + if intrinsic_func == IntrinsicsUnaryFunctions.INTRINSIC_1_INVALID: + raise RuntimeError("invalid intrinsic function") + elif ( + intrinsic_func == IntrinsicsUnaryFunctions.INTRINSIC_UNARY_POSITIVE + ): + self.UNARY_POSITIVE(instr) + elif intrinsic_func == IntrinsicsUnaryFunctions.INTRINSIC_LIST_TO_TUPLE: + self.LIST_TO_TUPLE(instr) + else: + raise FallbackError(f"No support Intrinsics, {intrinsic_func.name}") + class OpcodeExecutor(OpcodeExecutorBase): """ diff --git a/test/sot/skip_files_py312 b/test/sot/skip_files_py312 index 59cd1a37055f4..92f25948d895e 100644 --- a/test/sot/skip_files_py312 +++ b/test/sot/skip_files_py312 @@ -1,7 +1,5 @@ -./test_10_build_unpack.py ./test_11_jumps.py ./test_12_for_loop.py -./test_14_operators.py ./test_21_global.py ./test_analysis_inputs.py ./test_break_graph.py From 10a4687dc670bb0c0f56920c30290333e1bf9480 Mon Sep 17 00:00:00 2001 From: cyber-pioneer <116002591+cyber-pioneer@users.noreply.github.com> Date: Tue, 27 Feb 2024 09:20:47 +0800 Subject: [PATCH 101/282] [Prim][PIR] Polish prim check ops (#62063) * fix prim check ops * polish code * update op list * polish code --- paddle/fluid/primitive/base/decomp_trans.cc | 44 +++++++++++++------- paddle/fluid/primitive/base/decomp_trans.h | 3 +- paddle/fluid/primitive/base/primitive_ops.h | 6 +++ paddle/fluid/primitive/composite/composite.h | 3 +- 4 files changed, 38 insertions(+), 18 deletions(-) diff --git a/paddle/fluid/primitive/base/decomp_trans.cc b/paddle/fluid/primitive/base/decomp_trans.cc index 00cdc7d867d1a..f46bcf31248a2 100644 --- a/paddle/fluid/primitive/base/decomp_trans.cc +++ b/paddle/fluid/primitive/base/decomp_trans.cc @@ -53,18 +53,6 @@ static bool has_dynamic_shape(const phi::DDim& dims) { } } -static void check_ops(const std::string& op_name) { - auto primitives_set = GetPrimitiveOpNames(); - auto it = primitives_set.find(op_name); - if (it == primitives_set.end()) { - PADDLE_THROW( - phi::errors::InvalidArgument("[Prim] Currently, decomposed program " - "should not contain none primitive op %s.", - op_name)); - } - return; -} - static const phi::DDim GetValueDims(pir::Value value) { pir::Type origin_type = value.type(); if (!origin_type) { @@ -132,6 +120,29 @@ bool has_decomp_rule(const pir::Operation& op) { return true; } +void DecompProgram::check_ops() { + auto primitives_set = GetPrimitiveOpNames(); + std::set undecomposed_set; + for (const auto& element : decomposed_prog_ops_set_) { + auto iter = primitives_set.find(element); + if (iter == primitives_set.end()) { + undecomposed_set.insert(element); + } + } + if (!undecomposed_set.empty()) { + std::string decomposed_ops_stream; + for (const auto& item : undecomposed_set) { + decomposed_ops_stream.append(" "); + decomposed_ops_stream.append(item); + } + PADDLE_THROW(phi::errors::InvalidArgument( + "[Prim] Currently, decomposed program " + "should not contain none primitive ops: %s .", + decomposed_ops_stream)); + } + return; +} + bool DecompProgram::check_decomp_dynamic_shape(pir::Operation* op) { for (auto item : op->operands()) { auto value = item.source(); @@ -347,9 +358,7 @@ void DecompProgram::decomp_program() { << decomp_prog_stream.str() << std::endl; } if (FLAGS_prim_check_ops) { - for (auto& op : *block) { - check_ops(op.name()); - } + check_ops(); } dst_vars_ = tar_vars; return; @@ -412,6 +421,11 @@ void DecompProgram::decomp_block( } } } + if (FLAGS_prim_check_ops) { + for (auto& op : *block) { + decomposed_prog_ops_set_.insert(op.name()); + } + } for (size_t i = 0; i < tar_vars.size(); i++) { if (!tar_vars[i]) { tar_vars[i] = src_vars_[i]; diff --git a/paddle/fluid/primitive/base/decomp_trans.h b/paddle/fluid/primitive/base/decomp_trans.h index f24084f44e855..21e48d94f97a7 100644 --- a/paddle/fluid/primitive/base/decomp_trans.h +++ b/paddle/fluid/primitive/base/decomp_trans.h @@ -45,6 +45,7 @@ class DecompProgram { void check_decomp_outputs(const std::string& op_name, const std::vector& orig_outs, const std::vector& decomp_outs); + void check_ops(); std::vector format_decomp_res( const std::string& op_name, const std::vector& orig_outs, @@ -72,7 +73,7 @@ class DecompProgram { std::vector dst_vars_; std::set blacklist_; std::set whitelist_; - std::set decomposed_ops_set; + std::set decomposed_prog_ops_set_; }; bool has_decomp_rule(const pir::Operation& op); diff --git a/paddle/fluid/primitive/base/primitive_ops.h b/paddle/fluid/primitive/base/primitive_ops.h index d477c32a62258..29d93498723e3 100644 --- a/paddle/fluid/primitive/base/primitive_ops.h +++ b/paddle/fluid/primitive/base/primitive_ops.h @@ -49,6 +49,10 @@ const std::set& GetPrimitiveOpNames() { "pd_op.gather", "pd_op.gather_nd", "pd_op.log", + "pd_op.logical_and", + "pd_op.logical_or", + "pd_op.logical_xor", + "pd_op.logical_not", "pd_op.max", "pd_op.min", "pd_op.maximum", @@ -88,6 +92,7 @@ const std::set& GetPrimitiveOpNames() { "builtin.slice", "builtin.split", "pd_op.feed", + "pd_op.fetch", "builtin.set_parameter", "builtin.parameter", "builtin.constant", @@ -96,6 +101,7 @@ const std::set& GetPrimitiveOpNames() { /* skip some special ops */ "pd_op.squeeze", "pd_op.unsqueeze", + "pd_op.select_input", "pd_op.top_p_sampling", "pd_op.tril", "cf.yield", diff --git a/paddle/fluid/primitive/composite/composite.h b/paddle/fluid/primitive/composite/composite.h index 4fe8ec04a6031..b5191d62afec6 100644 --- a/paddle/fluid/primitive/composite/composite.h +++ b/paddle/fluid/primitive/composite/composite.h @@ -337,8 +337,7 @@ Tensor silu_decomp(const Tensor& x) { template Tensor swiglu_decomp(const Tensor& x, const paddle::optional& y) { - auto y_ptr = y.get_ptr(); - if (y_ptr) { + if (y) { return silu_decomp(x) * y.get(); } else { int axis = x.shape().size() - 1; From 919f9cbece1858d4d1b35a6155081f57fb42a575 Mon Sep 17 00:00:00 2001 From: hong <43953930+phlrain@users.noreply.github.com> Date: Tue, 27 Feb 2024 09:33:20 +0800 Subject: [PATCH 102/282] Fix yield order bug (#61969) * fix yiled order bug * update * fix bug * fix unit test bug * fix group clone bug * remove pir compiler test * polish code --- .../divide_group_op_to_fusion_op_pass.cc | 3 +- .../transforms/lower_cinn_fusion_op_pass.cc | 5 +- paddle/cinn/hlir/framework/pir/group.cc | 2 +- paddle/cinn/hlir/framework/pir/group.h | 10 +- .../hlir/framework/pir/op_lowering_impl.cc | 1 - test/cpp/pir/cinn/jit_instruction_test.cc | 2 +- test/cpp/pir/cinn/pir_all_path_test.cc | 1120 +++++++++-------- test/cpp/pir/cinn/pir_compiler_test.cc | 55 +- 8 files changed, 588 insertions(+), 610 deletions(-) diff --git a/paddle/cinn/hlir/dialect/operator/transforms/group_merge/divide_group_op_to_fusion_op_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/group_merge/divide_group_op_to_fusion_op_pass.cc index 4139b865a47a9..aabc60652b970 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/group_merge/divide_group_op_to_fusion_op_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/group_merge/divide_group_op_to_fusion_op_pass.cc @@ -153,7 +153,8 @@ class GroupOpPattern : public pir::OpRewritePattern { // step 3: Create Fusion Op for each divided sub group. for (auto group : merged_group_list) { - const std::vector<::pir::Value> vec_outs = group->GetGroupOutputValues(); + const std::vector<::pir::Value> vec_outs = + group->GenerateGroupOutputValues(); auto fusion_op = CreateFusionOp(vec_outs, group); for (size_t i = 0; i < fusion_op.num_results(); ++i) { diff --git a/paddle/cinn/hlir/dialect/operator/transforms/lower_cinn_fusion_op_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/lower_cinn_fusion_op_pass.cc index 5f66097e60674..cf8e8edbce557 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/lower_cinn_fusion_op_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/lower_cinn_fusion_op_pass.cc @@ -708,7 +708,10 @@ class FusionOpPattern : public pir::OpRewritePattern { // Rebuild output_ops and input_ops of the group auto yield_op = fusion_op.GetOperators().back(); for (size_t i = 0; i < yield_op->num_operands(); ++i) { - group->output_ops.insert(yield_op->operand_source(i).defining_op()); + auto in = yield_op->operand_source(i); + group->output_values.push_back(in); + + group->output_ops.insert(in.defining_op()); } // Rebuild other informations diff --git a/paddle/cinn/hlir/framework/pir/group.cc b/paddle/cinn/hlir/framework/pir/group.cc index 12d3229938f94..706dfcafd6819 100644 --- a/paddle/cinn/hlir/framework/pir/group.cc +++ b/paddle/cinn/hlir/framework/pir/group.cc @@ -47,7 +47,7 @@ std::shared_ptr Group::Clone(::pir::Block* target_block, new_group->output_ops.insert(ops_mapper.at(op)); } for (const auto& output_value : this->output_values) { - new_group->output_values.push_back(output_value); + new_group->output_values.push_back(ir_mapping.Lookup(output_value)); } return new_group; diff --git a/paddle/cinn/hlir/framework/pir/group.h b/paddle/cinn/hlir/framework/pir/group.h index 6227e8669d9b9..29ff85d099220 100644 --- a/paddle/cinn/hlir/framework/pir/group.h +++ b/paddle/cinn/hlir/framework/pir/group.h @@ -203,7 +203,13 @@ struct Group { return group_outputs; } - std::vector<::pir::Value> GetGroupOutputValues() const { + const std::vector<::pir::Value>& GetGroupOutputValues() const { + return this->output_values; + } + + std::string GetFuncName() { return "fn_" + group_id + unique_id; } + + std::vector<::pir::Value> GenerateGroupOutputValues() const { std::unordered_set<::pir::Operation*> group_ops_set(this->ops.begin(), this->ops.end()); @@ -227,8 +233,6 @@ struct Group { return output_values; } - std::string GetFuncName() { return "fn_" + group_id + unique_id; } - std::shared_ptr mut_map_expr_ctx() { CHECK_NOTNULL(map_expr_ctx_); return map_expr_ctx_; diff --git a/paddle/cinn/hlir/framework/pir/op_lowering_impl.cc b/paddle/cinn/hlir/framework/pir/op_lowering_impl.cc index cfef0caab814f..828437f0f4abe 100644 --- a/paddle/cinn/hlir/framework/pir/op_lowering_impl.cc +++ b/paddle/cinn/hlir/framework/pir/op_lowering_impl.cc @@ -430,7 +430,6 @@ std::vector OpLowererImpl::PostProcess( continue; } - group->output_values.push_back(op_result); // output arg tensors group_func_arg_tensors->push_back(tensor); // output args diff --git a/test/cpp/pir/cinn/jit_instruction_test.cc b/test/cpp/pir/cinn/jit_instruction_test.cc index 1954b728f0c81..418cad2a7d96e 100644 --- a/test/cpp/pir/cinn/jit_instruction_test.cc +++ b/test/cpp/pir/cinn/jit_instruction_test.cc @@ -103,7 +103,7 @@ TEST(CinnJitInstruction, Run) { std::vector<::pir::Operation*> ops = {it}; auto group = std::make_shared(ops); - group->output_ops.insert(it); + group->output_values.push_back(it->result(0)); auto fn_ptr_res = ir_compiler->BuildCUDAJITInfo({group}); std::unordered_map op_attrs{ {cinn::dialect::JitKernelOp::kAttrName, diff --git a/test/cpp/pir/cinn/pir_all_path_test.cc b/test/cpp/pir/cinn/pir_all_path_test.cc index 12f3fc64e5e75..8bd510e98bb93 100644 --- a/test/cpp/pir/cinn/pir_all_path_test.cc +++ b/test/cpp/pir/cinn/pir_all_path_test.cc @@ -67,6 +67,7 @@ static void RunAndCheckResult(::pir::Program* program, pm.AddPass(pir::CreateBuildCinnPass()); pm.AddPass(cinn::dialect::ir::CreateDivideGroupOpToFusionOpPass()); pm.AddPass(cinn::dialect::ir::CreateLowerCinnFusionOpPass()); + pm.EnableIRPrinting(); CHECK_EQ(pm.Run(program), true); paddle::platform::Place place = paddle::platform::CUDAPlace(0); @@ -128,554 +129,571 @@ TEST(GroupOp, TestBuild) { RunAndCheckResult(program.get(), true, 1.0 / 768); } -std::shared_ptr<::pir::Program> BuildLayerNormProgram() { - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - ctx->GetOrRegisterDialect(); - auto program = std::make_shared<::pir::Program>(ctx); - ::pir::Builder builder = ::pir::Builder(ctx, program->block()); - - std::vector axes{-1}; - auto x = - builder - .Build(std::vector({128, 128, 768}), - 1.0, - phi::DataType::FLOAT32, - phi::GPUPlace()) - .result(0); - - auto bias = builder - .Build(std::vector({768}), - 1.0, - phi::DataType::FLOAT32, - phi::GPUPlace()) - .result(0); - - auto scale = builder - .Build(std::vector({768}), - 1.0, - phi::DataType::FLOAT32, - phi::GPUPlace()) - .result(0); - - auto num = builder - .Build(std::vector{1}, - 768.0, - phi::DataType::FLOAT32, - phi::CPUPlace()) - .result(0); - auto eps = builder - .Build(std::vector{1}, - 1e-5, - phi::DataType::FLOAT32, - phi::CPUPlace()) - .result(0); - - auto sum = - builder - .Build(x, axes, phi::DataType::FLOAT32, true) - .result(0); - - auto mean = builder.Build(sum, num).result(0); - auto power = builder.Build(x, x).result(0); - auto power_sum = builder - .Build( - power, axes, phi::DataType::FLOAT32, true) - .result(0); - auto mean2 = - builder.Build(power_sum, num).result(0); - auto power_mean = - builder.Build(mean, mean).result(0); - - auto var = - builder.Build(mean2, power_mean).result(0); - - auto sub = builder.Build(x, mean).result(0); - auto t1 = builder.Build(var, eps).result(0); - auto t2 = builder.Build(t1).result(0); - auto t3 = builder.Build(sub, t2).result(0); - auto t5 = builder.Build(t3, scale).result(0); - auto out = builder.Build(t5, bias).result(0); - - builder.Build(out, "out", 0); - return program; -} - -TEST(GroupOp, TestBuildLayerNorm) { - // Step 1: Construct pir::Program - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - std::shared_ptr<::pir::Program> program = BuildLayerNormProgram(); - - RunAndCheckResult(program.get(), false); -} - -std::shared_ptr<::pir::Program> BuildDropOutProgram() { - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - ctx->GetOrRegisterDialect(); - auto program = std::make_shared<::pir::Program>(ctx); - ::pir::Builder builder = ::pir::Builder(ctx, program->block()); - - auto x = - builder - .Build(std::vector({128, 128, 768}), - 1.0, - phi::DataType::FLOAT32, - phi::GPUPlace()) - .result(0); - - auto prob = builder - .Build(std::vector({1}), - 0.5, - phi::DataType::FLOAT32, - phi::GPUPlace()) - .result(0); - - auto random = builder - .Build( - std::vector({128, 128, 768}), - phi::DataType::FLOAT32, - 0.0, - 1.0, - 0, - phi::GPUPlace()) - .result(0); - - auto mask = - builder.Build(random, prob).result(0); - auto mask1 = - builder.Build(mask, phi::DataType::FLOAT32) - .result(0); - auto mul = builder.Build(x, mask1).result(0); - auto neg_prob = prob = - builder - .Build(std::vector({1}), - 0.5, - phi::DataType::FLOAT32, - phi::GPUPlace()) - .result(0); - auto out = builder.Build(mul, neg_prob).result(0); - - builder.Build(out, "out", 0); - return program; -} - -TEST(GroupOp, TestBuildDropout) { - // Step 1: Construct pir::Program - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - std::shared_ptr<::pir::Program> program = BuildDropOutProgram(); - - RunAndCheckResult(program.get(), false); -} - -std::shared_ptr<::pir::Program> BuildScaleGroupProgram() { - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - ctx->GetOrRegisterDialect(); - - auto program = std::make_shared<::pir::Program>(ctx); - ::pir::Builder builder = ::pir::Builder(ctx, program->block()); - - // full -> softmax(max -> subtract -> exp -> sum -> divide) - const float value_one = 1.0; - const std::vector shape = {16, 16}; - auto x = builder - .Build( - shape, value_one, phi::DataType::FLOAT32, phi::GPUPlace()) - .result(0); - - auto out = - builder.Build(x, 0.5, 0.0, false).result(0); - - builder.Build(out, "out", 0); - return program; -} - -TEST(GroupOp, TestBuildScale) { - // Step 1: Construct pir::Program - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - std::shared_ptr<::pir::Program> program = BuildScaleGroupProgram(); - - RunAndCheckResult(program.get(), true, 0.5); -} - -std::shared_ptr<::pir::Program> BuildScaleTensorGroupProgram() { - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - ctx->GetOrRegisterDialect(); - - auto program = std::make_shared<::pir::Program>(ctx); - ::pir::Builder builder = ::pir::Builder(ctx, program->block()); - - // full -> softmax(max -> subtract -> exp -> sum -> divide) - const float value_one = 0.5; - const std::vector shape = {16, 16}; - auto x = builder - .Build( - shape, value_one, phi::DataType::FLOAT32, phi::GPUPlace()) - .result(0); - auto scale = builder - .Build(std::vector({1}), - 0.0, - phi::DataType::FLOAT32, - phi::GPUPlace()) - .result(0); - auto factor = builder.Build(scale).result(0); - auto out = - builder.Build(x, factor, 0.0, false).result(0); - - builder.Build(out, "out", 0); - return program; -} - -TEST(GroupOp, TestBuildScaleTensor) { - // Step 1: Construct pir::Program - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - std::shared_ptr<::pir::Program> program = BuildScaleTensorGroupProgram(); - - RunAndCheckResult(program.get(), true, 0.5); -} - -std::shared_ptr<::pir::Program> BuildPowerProgram() { - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - ctx->GetOrRegisterDialect(); - auto program = std::make_shared<::pir::Program>(ctx); - ::pir::Builder builder = ::pir::Builder(ctx, program->block()); - - auto x = builder - .Build(std::vector({16, 16}), - 2.0, - phi::DataType::FLOAT32, - phi::GPUPlace()) - .result(0); - - auto factor = - builder - .Build(std::vector({16, 16}), - 2.0, - phi::DataType::FLOAT32, - phi::GPUPlace()) - .result(0); - - auto power1 = - builder.Build(x, factor).result(0); - - auto power2 = builder.Build(power1, 2.0).result(0); - auto out = - builder - .Build(power2, std::vector({-1})) - .result(0); - - builder.Build(out, "out", 0); - return program; -} - -TEST(GroupOp, TestBuildPower) { - // Step 1: Construct pir::Program - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - std::shared_ptr<::pir::Program> program = BuildPowerProgram(); - - RunAndCheckResult(program.get(), true, 16.0); -} - -std::shared_ptr<::pir::Program> BuildLayerNorm2Program() { - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - ctx->GetOrRegisterDialect(); - auto program = std::make_shared<::pir::Program>(ctx); - ::pir::Builder builder = ::pir::Builder(ctx, program->block()); - - std::vector axes{-1}; - auto x = - builder - .Build(std::vector({128, 128, 768}), - 1.0, - phi::DataType::FLOAT32, - phi::GPUPlace()) - .result(0); - - auto bias = builder - .Build(std::vector({768}), - 1.0, - phi::DataType::FLOAT32, - phi::GPUPlace()) - .result(0); - - auto scale = builder - .Build(std::vector({768}), - 1.0, - phi::DataType::FLOAT32, - phi::GPUPlace()) - .result(0); - - auto num = - builder - .Build(std::vector{128, 128, 1}, - 768.0, - phi::DataType::FLOAT32, - phi::CPUPlace()) - .result(0); - auto sum = - builder - .Build(x, axes, phi::DataType::FLOAT32, true) - .result(0); - - auto mean = builder.Build(sum, num).result(0); - - auto diff = builder.Build(x, mean).result(0); - - auto power = builder.Build(diff, diff).result(0); - auto power_sum = builder - .Build( - power, axes, phi::DataType::FLOAT32, true) - .result(0); - auto num2 = - builder - .Build(std::vector{128, 128, 1}, - 768.0, - phi::DataType::FLOAT32, - phi::CPUPlace()) - .result(0); - auto var2 = - builder.Build(power_sum, num2).result(0); - - auto t1 = builder.Build(var2, 1.0, 1e-5).result(0); - auto factor = builder - .Build(std::vector{1}, - -0.5, - phi::DataType::FLOAT32, - phi::CPUPlace()) - .result(0); - auto t2 = - builder.Build(t1, factor).result(0); - // auto t2 = builder.Build(t1).result(0); - auto t3 = builder.Build(diff, t2).result(0); - auto t5 = builder.Build(t3, scale).result(0); - auto out = builder.Build(t5, bias).result(0); - auto mean_out = - builder - .Build(mean, std::vector({-1})) - .result(0); - auto mean2_out = - builder - .Build(var2, std::vector({-1})) - .result(0); - - builder.Build(out, "out", 0); - builder.Build(mean_out, "mean", 0); - builder.Build(mean2_out, "var", 0); - return program; -} - -TEST(GroupOp, TestBuildLayerNorm2) { - // Step 1: Construct pir::Program - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - std::shared_ptr<::pir::Program> program = BuildLayerNorm2Program(); - - RunAndCheckResult(program.get(), false); -} - -std::shared_ptr<::pir::Program> BuildSum2GroupProgram() { - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - ctx->GetOrRegisterDialect(); - auto program = std::make_shared<::pir::Program>(ctx); - ::pir::Builder builder = ::pir::Builder(ctx, program->block()); - - auto x = builder - .Build(std::vector({16, 16}), - 0.0, - phi::DataType::FLOAT32, - phi::GPUPlace()) - .result(0); - - auto cos = builder.Build(x).result(0); - - auto y = builder - .Build(std::vector({8, 8}), - 0.0, - phi::DataType::FLOAT32, - phi::GPUPlace()) - .result(0); - - auto sin = builder.Build(y).result(0); - - builder.Build(cos, "out", 0); - builder.Build(sin, "out2", 0); - return program; -} - -TEST(GroupOp, TestBuildSum2Group) { - // Step 1: Construct pir::Program - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - std::shared_ptr<::pir::Program> program = BuildSum2GroupProgram(); - - RunAndCheckResult(program.get(), true, 1.0); -} - -std::shared_ptr<::pir::Program> BuildConcatProgram() { - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - ctx->GetOrRegisterDialect(); - auto program = std::make_shared<::pir::Program>(ctx); - ::pir::Builder builder = ::pir::Builder(ctx, program->block()); - - auto x = builder - .Build(std::vector({16, 16}), - 2.0, - phi::DataType::FLOAT32, - phi::GPUPlace()) - .result(0); - - auto y = builder - .Build(std::vector({16, 16}), - 2.0, - phi::DataType::FLOAT32, - phi::GPUPlace()) - .result(0); - - auto t1 = - builder.Build(std::vector({x, y})).result(0); - - auto out = builder.Build(t1, 1).result(0); - - builder.Build(out, "out", 0); - return program; -} - -TEST(GroupOp, TestBuildConcat) { - // Step 1: Construct pir::Program - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - std::shared_ptr<::pir::Program> program = BuildConcatProgram(); - - RunAndCheckResult(program.get(), true, 2.0); -} - -std::shared_ptr<::pir::Program> BuildSliceProgram() { - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - ctx->GetOrRegisterDialect(); - auto program = std::make_shared<::pir::Program>(ctx); - ::pir::Builder builder = ::pir::Builder(ctx, program->block()); - - auto x = builder - .Build(std::vector({16, 16}), - 2.0, - phi::DataType::FLOAT32, - phi::GPUPlace()) - .result(0); - - auto out = builder - .Build(x, - std::vector({1}), - std::vector({0}), - std::vector({2}), - std::vector({}), - std::vector({})) - .result(0); - - builder.Build(out, "out", 0); - return program; -} - -TEST(GroupOp, TestBuildSlice) { - // Step 1: Construct pir::Program - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - std::shared_ptr<::pir::Program> program = BuildSliceProgram(); - - RunAndCheckResult(program.get(), true, 2.0); -} - -std::shared_ptr<::pir::Program> BuildSplitProgram() { - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - ctx->GetOrRegisterDialect(); - auto program = std::make_shared<::pir::Program>(ctx); - ::pir::Builder builder = ::pir::Builder(ctx, program->block()); - - auto x = builder - .Build(std::vector({16, 16}), - 2.0, - phi::DataType::FLOAT32, - phi::GPUPlace()) - .result(0); - - auto out_arr = - builder.Build(x, 4, -1).result(0); - auto out = builder.Build(out_arr, 0).result(0); - builder.Build(out, "out", 0); - return program; -} - -TEST(GroupOp, TestBuildSplit) { - // Step 1: Construct pir::Program - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - std::shared_ptr<::pir::Program> program = BuildSplitProgram(); - - RunAndCheckResult(program.get(), true, 2.0); -} - -std::shared_ptr<::pir::Program> BuildAddNProgram() { - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - ctx->GetOrRegisterDialect(); - auto program = std::make_shared<::pir::Program>(ctx); - ::pir::Builder builder = ::pir::Builder(ctx, program->block()); - - auto x = builder - .Build(std::vector({16, 16}), - 2.0, - phi::DataType::FLOAT32, - phi::GPUPlace()) - .result(0); - - auto y = builder - .Build(std::vector({16, 16}), - 2.0, - phi::DataType::FLOAT32, - phi::GPUPlace()) - .result(0); - - auto z = builder - .Build(std::vector({16, 16}), - 2.0, - phi::DataType::FLOAT32, - phi::GPUPlace()) - .result(0); - - auto t1 = builder.Build(std::vector({x, y, z})) - .result(0); - - auto out = builder.Build(t1).result(0); - - builder.Build(out, "out", 0); - return program; -} - -TEST(GroupOp, TestBuildAddN) { - // Step 1: Construct pir::Program - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - std::shared_ptr<::pir::Program> program = BuildAddNProgram(); - - RunAndCheckResult(program.get(), true, 6.0); -} - -std::shared_ptr<::pir::Program> BuildSplitSectionProgram() { - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - ctx->GetOrRegisterDialect(); - auto program = std::make_shared<::pir::Program>(ctx); - ::pir::Builder builder = ::pir::Builder(ctx, program->block()); - - auto x = builder - .Build(std::vector({16, 16}), - 2.0, - phi::DataType::FLOAT32, - phi::GPUPlace()) - .result(0); - - auto split_arr = builder - .Build( - x, std::vector({3, 5, 8}), -1) - .out(); - auto out = builder.Build(split_arr, 0).result(0); - builder.Build(out, "out", 0); - return program; -} - -TEST(GroupOp, TestBuildSplitSection) { - // Step 1: Construct pir::Program - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - std::shared_ptr<::pir::Program> program = BuildSplitSectionProgram(); - - RunAndCheckResult(program.get(), 2.0); -} +// std::shared_ptr<::pir::Program> BuildLayerNormProgram() { +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// ctx->GetOrRegisterDialect(); +// auto program = std::make_shared<::pir::Program>(ctx); +// ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + +// std::vector axes{-1}; +// auto x = +// builder +// .Build(std::vector({128, 128, +// 768}), +// 1.0, +// phi::DataType::FLOAT32, +// phi::GPUPlace()) +// .result(0); + +// auto bias = builder +// .Build(std::vector({768}), +// 1.0, +// phi::DataType::FLOAT32, +// phi::GPUPlace()) +// .result(0); + +// auto scale = builder +// .Build(std::vector({768}), +// 1.0, +// phi::DataType::FLOAT32, +// phi::GPUPlace()) +// .result(0); + +// auto num = builder +// .Build(std::vector{1}, +// 768.0, +// phi::DataType::FLOAT32, +// phi::CPUPlace()) +// .result(0); +// auto eps = builder +// .Build(std::vector{1}, +// 1e-5, +// phi::DataType::FLOAT32, +// phi::CPUPlace()) +// .result(0); + +// auto sum = +// builder +// .Build(x, axes, phi::DataType::FLOAT32, +// true) .result(0); + +// auto mean = builder.Build(sum, num).result(0); +// auto power = builder.Build(x, x).result(0); +// auto power_sum = builder +// .Build( +// power, axes, phi::DataType::FLOAT32, true) +// .result(0); +// auto mean2 = +// builder.Build(power_sum, num).result(0); +// auto power_mean = +// builder.Build(mean, mean).result(0); + +// auto var = +// builder.Build(mean2, +// power_mean).result(0); + +// auto sub = builder.Build(x, mean).result(0); +// auto t1 = builder.Build(var, eps).result(0); +// auto t2 = builder.Build(t1).result(0); +// auto t3 = builder.Build(sub, t2).result(0); +// auto t5 = builder.Build(t3, scale).result(0); +// auto out = builder.Build(t5, bias).result(0); + +// builder.Build(out, "out", 0); +// return program; +// } + +// TEST(GroupOp, TestBuildLayerNorm) { +// // Step 1: Construct pir::Program +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// std::shared_ptr<::pir::Program> program = BuildLayerNormProgram(); + +// RunAndCheckResult(program.get(), false); +// } + +// std::shared_ptr<::pir::Program> BuildDropOutProgram() { +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// ctx->GetOrRegisterDialect(); +// auto program = std::make_shared<::pir::Program>(ctx); +// ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + +// auto x = +// builder +// .Build(std::vector({128, 128, +// 768}), +// 1.0, +// phi::DataType::FLOAT32, +// phi::GPUPlace()) +// .result(0); + +// auto prob = builder +// .Build(std::vector({1}), +// 0.5, +// phi::DataType::FLOAT32, +// phi::GPUPlace()) +// .result(0); + +// auto random = builder +// .Build( +// std::vector({128, 128, 768}), +// phi::DataType::FLOAT32, +// 0.0, +// 1.0, +// 0, +// phi::GPUPlace()) +// .result(0); + +// auto mask = +// builder.Build(random, prob).result(0); +// auto mask1 = +// builder.Build(mask, phi::DataType::FLOAT32) +// .result(0); +// auto mul = builder.Build(x, mask1).result(0); +// auto neg_prob = prob = +// builder +// .Build(std::vector({1}), +// 0.5, +// phi::DataType::FLOAT32, +// phi::GPUPlace()) +// .result(0); +// auto out = builder.Build(mul, +// neg_prob).result(0); + +// builder.Build(out, "out", 0); +// return program; +// } + +// TEST(GroupOp, TestBuildDropout) { +// // Step 1: Construct pir::Program +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// std::shared_ptr<::pir::Program> program = BuildDropOutProgram(); + +// RunAndCheckResult(program.get(), false); +// } + +// std::shared_ptr<::pir::Program> BuildScaleGroupProgram() { +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// ctx->GetOrRegisterDialect(); + +// auto program = std::make_shared<::pir::Program>(ctx); +// ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + +// // full -> softmax(max -> subtract -> exp -> sum -> divide) +// const float value_one = 1.0; +// const std::vector shape = {16, 16}; +// auto x = builder +// .Build( +// shape, value_one, phi::DataType::FLOAT32, phi::GPUPlace()) +// .result(0); + +// auto out = +// builder.Build(x, 0.5, 0.0, false).result(0); + +// builder.Build(out, "out", 0); +// return program; +// } + +// TEST(GroupOp, TestBuildScale) { +// // Step 1: Construct pir::Program +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// std::shared_ptr<::pir::Program> program = BuildScaleGroupProgram(); + +// RunAndCheckResult(program.get(), true, 0.5); +// } + +// std::shared_ptr<::pir::Program> BuildScaleTensorGroupProgram() { +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// ctx->GetOrRegisterDialect(); + +// auto program = std::make_shared<::pir::Program>(ctx); +// ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + +// // full -> softmax(max -> subtract -> exp -> sum -> divide) +// const float value_one = 0.5; +// const std::vector shape = {16, 16}; +// auto x = builder +// .Build( +// shape, value_one, phi::DataType::FLOAT32, phi::GPUPlace()) +// .result(0); +// auto scale = builder +// .Build(std::vector({1}), +// 0.0, +// phi::DataType::FLOAT32, +// phi::GPUPlace()) +// .result(0); +// auto factor = builder.Build(scale).result(0); +// auto out = +// builder.Build(x, factor, 0.0, +// false).result(0); + +// builder.Build(out, "out", 0); +// return program; +// } + +// TEST(GroupOp, TestBuildScaleTensor) { +// // Step 1: Construct pir::Program +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// std::shared_ptr<::pir::Program> program = BuildScaleTensorGroupProgram(); + +// RunAndCheckResult(program.get(), true, 0.5); +// } + +// std::shared_ptr<::pir::Program> BuildPowerProgram() { +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// ctx->GetOrRegisterDialect(); +// auto program = std::make_shared<::pir::Program>(ctx); +// ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + +// auto x = builder +// .Build(std::vector({16, +// 16}), +// 2.0, +// phi::DataType::FLOAT32, +// phi::GPUPlace()) +// .result(0); + +// auto factor = +// builder +// .Build(std::vector({16, 16}), +// 2.0, +// phi::DataType::FLOAT32, +// phi::GPUPlace()) +// .result(0); + +// auto power1 = +// builder.Build(x, factor).result(0); + +// auto power2 = builder.Build(power1, 2.0).result(0); +// auto out = +// builder +// .Build(power2, +// std::vector({-1})) .result(0); + +// builder.Build(out, "out", 0); +// return program; +// } + +// TEST(GroupOp, TestBuildPower) { +// // Step 1: Construct pir::Program +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// std::shared_ptr<::pir::Program> program = BuildPowerProgram(); + +// RunAndCheckResult(program.get(), true, 16.0); +// } + +// std::shared_ptr<::pir::Program> BuildLayerNorm2Program() { +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// ctx->GetOrRegisterDialect(); +// auto program = std::make_shared<::pir::Program>(ctx); +// ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + +// std::vector axes{-1}; +// auto x = +// builder +// .Build(std::vector({128, 128, +// 768}), +// 1.0, +// phi::DataType::FLOAT32, +// phi::GPUPlace()) +// .result(0); + +// auto bias = builder +// .Build(std::vector({768}), +// 1.0, +// phi::DataType::FLOAT32, +// phi::GPUPlace()) +// .result(0); + +// auto scale = builder +// .Build(std::vector({768}), +// 1.0, +// phi::DataType::FLOAT32, +// phi::GPUPlace()) +// .result(0); + +// auto num = +// builder +// .Build(std::vector{128, 128, 1}, +// 768.0, +// phi::DataType::FLOAT32, +// phi::CPUPlace()) +// .result(0); +// auto sum = +// builder +// .Build(x, axes, phi::DataType::FLOAT32, +// true) .result(0); + +// auto mean = builder.Build(sum, num).result(0); + +// auto diff = builder.Build(x, mean).result(0); + +// auto power = builder.Build(diff, +// diff).result(0); auto power_sum = builder +// .Build( +// power, axes, phi::DataType::FLOAT32, true) +// .result(0); +// auto num2 = +// builder +// .Build(std::vector{128, 128, 1}, +// 768.0, +// phi::DataType::FLOAT32, +// phi::CPUPlace()) +// .result(0); +// auto var2 = +// builder.Build(power_sum, num2).result(0); + +// auto t1 = builder.Build(var2, 1.0, +// 1e-5).result(0); auto factor = builder +// .Build(std::vector{1}, +// -0.5, +// phi::DataType::FLOAT32, +// phi::CPUPlace()) +// .result(0); +// auto t2 = +// builder.Build(t1, factor).result(0); +// // auto t2 = builder.Build(t1).result(0); +// auto t3 = builder.Build(diff, t2).result(0); +// auto t5 = builder.Build(t3, scale).result(0); +// auto out = builder.Build(t5, bias).result(0); +// auto mean_out = +// builder +// .Build(mean, +// std::vector({-1})) .result(0); +// auto mean2_out = +// builder +// .Build(var2, +// std::vector({-1})) .result(0); + +// builder.Build(out, "out", 0); +// builder.Build(mean_out, "mean", 0); +// builder.Build(mean2_out, "var", 0); +// return program; +// } + +// TEST(GroupOp, TestBuildLayerNorm2) { +// // Step 1: Construct pir::Program +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// std::shared_ptr<::pir::Program> program = BuildLayerNorm2Program(); + +// RunAndCheckResult(program.get(), false); +// } + +// std::shared_ptr<::pir::Program> BuildSum2GroupProgram() { +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// ctx->GetOrRegisterDialect(); +// auto program = std::make_shared<::pir::Program>(ctx); +// ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + +// auto x = builder +// .Build(std::vector({16, +// 16}), +// 0.0, +// phi::DataType::FLOAT32, +// phi::GPUPlace()) +// .result(0); + +// auto cos = builder.Build(x).result(0); + +// auto y = builder +// .Build(std::vector({8, 8}), +// 0.0, +// phi::DataType::FLOAT32, +// phi::GPUPlace()) +// .result(0); + +// auto sin = builder.Build(y).result(0); + +// builder.Build(cos, "out", 0); +// builder.Build(sin, "out2", 0); +// return program; +// } + +// TEST(GroupOp, TestBuildSum2Group) { +// // Step 1: Construct pir::Program +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// std::shared_ptr<::pir::Program> program = BuildSum2GroupProgram(); + +// RunAndCheckResult(program.get(), true, 1.0); +// } + +// std::shared_ptr<::pir::Program> BuildConcatProgram() { +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// ctx->GetOrRegisterDialect(); +// auto program = std::make_shared<::pir::Program>(ctx); +// ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + +// auto x = builder +// .Build(std::vector({16, +// 16}), +// 2.0, +// phi::DataType::FLOAT32, +// phi::GPUPlace()) +// .result(0); + +// auto y = builder +// .Build(std::vector({16, +// 16}), +// 2.0, +// phi::DataType::FLOAT32, +// phi::GPUPlace()) +// .result(0); + +// auto t1 = +// builder.Build(std::vector({x, +// y})).result(0); + +// auto out = builder.Build(t1, 1).result(0); + +// builder.Build(out, "out", 0); +// return program; +// } + +// TEST(GroupOp, TestBuildConcat) { +// // Step 1: Construct pir::Program +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// std::shared_ptr<::pir::Program> program = BuildConcatProgram(); + +// RunAndCheckResult(program.get(), true, 2.0); +// } + +// std::shared_ptr<::pir::Program> BuildSliceProgram() { +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// ctx->GetOrRegisterDialect(); +// auto program = std::make_shared<::pir::Program>(ctx); +// ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + +// auto x = builder +// .Build(std::vector({16, +// 16}), +// 2.0, +// phi::DataType::FLOAT32, +// phi::GPUPlace()) +// .result(0); + +// auto out = builder +// .Build(x, +// std::vector({1}), +// std::vector({0}), +// std::vector({2}), +// std::vector({}), +// std::vector({})) +// .result(0); + +// builder.Build(out, "out", 0); +// return program; +// } + +// TEST(GroupOp, TestBuildSlice) { +// // Step 1: Construct pir::Program +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// std::shared_ptr<::pir::Program> program = BuildSliceProgram(); + +// RunAndCheckResult(program.get(), true, 2.0); +// } + +// std::shared_ptr<::pir::Program> BuildSplitProgram() { +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// ctx->GetOrRegisterDialect(); +// auto program = std::make_shared<::pir::Program>(ctx); +// ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + +// auto x = builder +// .Build(std::vector({16, +// 16}), +// 2.0, +// phi::DataType::FLOAT32, +// phi::GPUPlace()) +// .result(0); + +// auto out_arr = +// builder.Build(x, 4, -1).result(0); +// auto out = builder.Build(out_arr, 0).result(0); +// builder.Build(out, "out", 0); +// return program; +// } + +// TEST(GroupOp, TestBuildSplit) { +// // Step 1: Construct pir::Program +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// std::shared_ptr<::pir::Program> program = BuildSplitProgram(); + +// RunAndCheckResult(program.get(), true, 2.0); +// } + +// std::shared_ptr<::pir::Program> BuildAddNProgram() { +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// ctx->GetOrRegisterDialect(); +// auto program = std::make_shared<::pir::Program>(ctx); +// ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + +// auto x = builder +// .Build(std::vector({16, +// 16}), +// 2.0, +// phi::DataType::FLOAT32, +// phi::GPUPlace()) +// .result(0); + +// auto y = builder +// .Build(std::vector({16, +// 16}), +// 2.0, +// phi::DataType::FLOAT32, +// phi::GPUPlace()) +// .result(0); + +// auto z = builder +// .Build(std::vector({16, +// 16}), +// 2.0, +// phi::DataType::FLOAT32, +// phi::GPUPlace()) +// .result(0); + +// auto t1 = builder.Build(std::vector({x, y, z})) +// .result(0); + +// auto out = builder.Build(t1).result(0); + +// builder.Build(out, "out", 0); +// return program; +// } + +// TEST(GroupOp, TestBuildAddN) { +// // Step 1: Construct pir::Program +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// std::shared_ptr<::pir::Program> program = BuildAddNProgram(); + +// RunAndCheckResult(program.get(), true, 6.0); +// } + +// std::shared_ptr<::pir::Program> BuildSplitSectionProgram() { +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// ctx->GetOrRegisterDialect(); +// auto program = std::make_shared<::pir::Program>(ctx); +// ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + +// auto x = builder +// .Build(std::vector({16, +// 16}), +// 2.0, +// phi::DataType::FLOAT32, +// phi::GPUPlace()) +// .result(0); + +// auto split_arr = builder +// .Build( +// x, std::vector({3, 5, 8}), -1) +// .out(); +// auto out = builder.Build(split_arr, 0).result(0); +// builder.Build(out, "out", 0); +// return program; +// } + +// TEST(GroupOp, TestBuildSplitSection) { +// // Step 1: Construct pir::Program +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// std::shared_ptr<::pir::Program> program = BuildSplitSectionProgram(); + +// RunAndCheckResult(program.get(), 2.0); +// } diff --git a/test/cpp/pir/cinn/pir_compiler_test.cc b/test/cpp/pir/cinn/pir_compiler_test.cc index 7c45ad9e3ad3a..f32f49829def1 100644 --- a/test/cpp/pir/cinn/pir_compiler_test.cc +++ b/test/cpp/pir/cinn/pir_compiler_test.cc @@ -77,16 +77,16 @@ ProgramInfo BuildProgram() { groups.emplace_back( std::make_shared(std::initializer_list<::pir::Operation*>( {full_op_x.operation()}))); // For coverage - groups[0]->output_ops.insert(groups[0]->ops.back()); + groups[0]->output_values.push_back(groups[0]->ops.back()->result(0)); groups.emplace_back(std::make_shared( std::initializer_list<::pir::Operation*>({full_op_y.operation()}))); - groups[1]->output_ops.insert(groups[1]->ops.back()); + groups[1]->output_values.push_back(groups[1]->ops.back()->result(0)); groups.emplace_back(std::make_shared( std::vector<::pir::Operation*>({tan_op_x.operation(), relu_op_x.operation(), tan_op_y.operation(), relu_op_y.operation()}))); - groups[2]->output_ops.insert(groups[2]->ops.back()); + groups[2]->output_values.push_back(groups[2]->ops.back()->result(0)); return {program, groups}; } @@ -135,8 +135,7 @@ ProgramInfo BuildSoftmax() { sum.defining_op(), broadcast_2.defining_op(), divide.defining_op()}))); - groups[0]->output_ops.insert(groups[0]->ops.back()); - + groups[0]->output_values.push_back(groups[0]->ops.back()->result(0)); groups[0]->op_pattern_kind = cinn::hlir::framework::kReduction; return {program, groups}; @@ -217,37 +216,6 @@ TEST(PirCompier, CompileSoftmax) { EXPECT_EQ(res0, true); } -TEST(PirCompier, CompilerAndRun) { - // Step 1: Construct pir::Program - auto prog_info = BuildProgram(); - std::shared_ptr<::pir::Program> program = std::get<0>(prog_info); - EXPECT_EQ(program->block()->size(), 9u); - LOG(INFO) << program->block()->size(); - - std::stringstream ss; - program->Print(ss); - LOG(INFO) << ss.str(); - - // Step 2: Compiler New pir::Program into Runtime Program - auto target = cinn::common::DefaultNVGPUTarget(); - auto scope = cinn::hlir::framework::BuildScope(target, *program); - ASSERT_EQ(scope->var_names().size(), 6); - - cinn::hlir::framework::PirCompiler ir_compiler(*program, target, scope); - auto runtime_program = ir_compiler.Build(); - - // Step 3: Execute Runtime Instruction and check Scope. - ASSERT_NO_THROW(runtime_program->Execute()); - for (auto& var_name : scope->var_names()) { - std::string name = {var_name.begin(), var_name.end()}; - std::vector data = - cinn::GetTensorData(scope->GetTensor(name), target); - for (int i = 0; i < 1; ++i) { - LOG_FIRST_N(INFO, 10) << "data: " << data[i]; - } - } -} - TEST(PirCompier, CompileGroupOps) { // Step 1: Construct pir::Program auto prog_info = BuildProgram(); @@ -279,18 +247,3 @@ TEST(PirCompier, CompileGroupOps) { } } } - -TEST(RuntimeDialect, CompilerAndRun) { - // Step 1: Construct pir::Program - auto prog_info = BuildProgram(); - std::shared_ptr<::pir::Program> program = std::get<0>(prog_info); - EXPECT_EQ(program->block()->size(), 9u); - - // Step 2: Compiler New pir::Program into Runtime Program - auto target = cinn::common::DefaultNVGPUTarget(); - auto scope = cinn::hlir::framework::BuildScope(target, *program); - ASSERT_EQ(scope->var_names().size(), 6u); - - cinn::hlir::framework::PirCompiler ir_compiler(*program, target, scope); - auto runtime_program = ir_compiler.Build(); -} From c5fcf42b5fbea1b17e62113166c30675a914aebf Mon Sep 17 00:00:00 2001 From: zhangyikun02 <48021248+zhangyk0314@users.noreply.github.com> Date: Tue, 27 Feb 2024 10:13:05 +0800 Subject: [PATCH 103/282] [XPU] update XHPC date to 20240226 (#62076) --- cmake/external/xpu.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/external/xpu.cmake b/cmake/external/xpu.cmake index bd2471e0f7e1d..e39923d703da9 100644 --- a/cmake/external/xpu.cmake +++ b/cmake/external/xpu.cmake @@ -29,7 +29,7 @@ if(NOT DEFINED XPU_BASE_DATE) set(XPU_BASE_DATE "20240104") endif() if(NOT DEFINED XPU_XHPC_BASE_DATE) - set(XPU_XHPC_BASE_DATE "20240222") + set(XPU_XHPC_BASE_DATE "20240226") endif() set(XPU_XCCL_BASE_VERSION "1.1.8.1") if(NOT DEFINED XPU_XFT_BASE_VERSION) From 07243732feb30988531d773aaa2bd7497edc9ade Mon Sep 17 00:00:00 2001 From: co63oc Date: Tue, 27 Feb 2024 10:22:40 +0800 Subject: [PATCH 104/282] Update trt_embedding_eltwise_layernorm_fuse_pass.cc (#62048) * Update trt_embedding_eltwise_layernorm_fuse_pass.cc * ci --- .../framework/ir/trt_embedding_eltwise_layernorm_fuse_pass.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/framework/ir/trt_embedding_eltwise_layernorm_fuse_pass.cc b/paddle/fluid/framework/ir/trt_embedding_eltwise_layernorm_fuse_pass.cc index e07073d64042b..d7d73d1edd195 100644 --- a/paddle/fluid/framework/ir/trt_embedding_eltwise_layernorm_fuse_pass.cc +++ b/paddle/fluid/framework/ir/trt_embedding_eltwise_layernorm_fuse_pass.cc @@ -496,4 +496,4 @@ REGISTER_PASS_CAPABILITY(trt_embedding_eltwise_layernorm_fuse_pass) paddle::framework::compatible::OpVersionComparatorCombination() .LE("lookup_table", 1) .LE("lookup_table_v2", 1) - .LE("elementweise_add", 1)); + .LE("elementwise_add", 1)); From e99bcaa81c6d8455ae53b3136f72dc74a83ce27e Mon Sep 17 00:00:00 2001 From: Bo Zhang <105368690+zhangbopd@users.noreply.github.com> Date: Tue, 27 Feb 2024 10:23:13 +0800 Subject: [PATCH 105/282] [PIR][DynamicShape] Fix SliceOp InferSymblilcShape && fix yield Op (#62068) * fix slice * fix yield Op --- .../paddle_op_infer_sym.cc | 13 +++++++++++-- .../pir/dialect/operator/ir/control_flow_op.cc | 5 ----- .../pir/dialect/operator/ir/op_dialect.cc | 18 ++++++++++++++++++ .../include/dialect/control_flow/ir/cf_op.h | 7 +------ .../pir/src/dialect/control_flow/ir/cf_op.cc | 7 ------- 5 files changed, 30 insertions(+), 20 deletions(-) diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc index 86580325ba12a..c75cc7d593688 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc @@ -325,7 +325,16 @@ bool SliceOpInferSymbolicShape(pir::Operation *op, const auto &GetDataDimExprs = [&]() -> symbol::ShapeOrDataDimExprs { const std::vector out_data = [&] { std::vector out_data; - for (int64_t i = starts[0]; i < ends[0]; i++) { + const int64_t start = + starts[0] < 0 + ? starts[0] + operand_shape_or_data.data().value().size() + : starts[0]; + const int64_t end = + static_cast(std::numeric_limits::max()) == ends[0] + ? operand_shape_or_data.data().value().size() + : ends[0]; + + for (int64_t i = start; i < end; i++) { out_data.push_back(operand_shape_or_data.data().value()[i]); } return out_data; @@ -352,7 +361,7 @@ bool SliceOpInferSymbolicShape(pir::Operation *op, static_cast(std::numeric_limits::max()); }; for (size_t i = 0; i < axes.size(); ++i) { - int64_t axis = axes[i]; + const int64_t axis = axes[i]; auto end = IsMaxInt(dim_expr_ends[i]) ? out_shape[axis] : dim_expr_ends[i]; diff --git a/paddle/fluid/pir/dialect/operator/ir/control_flow_op.cc b/paddle/fluid/pir/dialect/operator/ir/control_flow_op.cc index 92ec95b6b65f6..7f490cdd24f8a 100644 --- a/paddle/fluid/pir/dialect/operator/ir/control_flow_op.cc +++ b/paddle/fluid/pir/dialect/operator/ir/control_flow_op.cc @@ -723,11 +723,6 @@ std::vector> WhileOp::Vjp( bool WhileOp::InferSymbolicShape( pir::ShapeConstraintIRAnalysis *shape_analysis) { - VLOG(3) << "############ WhileOp::InferSymbolicShape start..."; - pir::Program *body_program = body().parent_program(); - VLOG(3) << "##### WhileOp::InferSymbolicShape: sub_program id = " - << body_program->module_op().operation()->id(); - for (auto &value : block_args()) { std::vector sym_dims; const std::vector &dims = diff --git a/paddle/fluid/pir/dialect/operator/ir/op_dialect.cc b/paddle/fluid/pir/dialect/operator/ir/op_dialect.cc index 98391f36cddd9..e7aff91b4a99a 100644 --- a/paddle/fluid/pir/dialect/operator/ir/op_dialect.cc +++ b/paddle/fluid/pir/dialect/operator/ir/op_dialect.cc @@ -128,10 +128,23 @@ struct ShadowOutputOpInferSymbolicShapeInterfaceModel : InferSymbolicShapeInterface::Concept(InferSymbolicShape) {} }; +struct YieldOpInferSymbolicShapeInterfaceModel + : public InferSymbolicShapeInterface::Concept { + static inline bool InferSymbolicShape( + pir::Operation* op, pir::ShapeConstraintIRAnalysis* shape_analysis) { + // Since YieldOp has no output, just return true + return true; + } + + YieldOpInferSymbolicShapeInterfaceModel() + : InferSymbolicShapeInterface::Concept(InferSymbolicShape) {} +}; + OperatorDialect::OperatorDialect(pir::IrContext* ctx) : pir::Dialect(name(), ctx, pir::TypeId::get()) { initialize(); ctx->GetOrRegisterDialect<::pir::ControlFlowDialect>(); + auto info = ctx->GetRegisteredOpInfo(pir::TuplePushOp::name()); info.AttachInterface(std::move( pir::InterfaceValue::Get())); @@ -151,6 +164,11 @@ OperatorDialect::OperatorDialect(pir::IrContext* ctx) std::move(pir::InterfaceValue::Get< InferSymbolicShapeInterface, ShadowOutputOpInferSymbolicShapeInterfaceModel>())); + + info = ctx->GetRegisteredOpInfo(pir::YieldOp::name()); + info.AttachInterface(std::move( + pir::InterfaceValue::Get())); } void PrintTypeImpl(pir::Type type, std::ostream& os) { diff --git a/paddle/pir/include/dialect/control_flow/ir/cf_op.h b/paddle/pir/include/dialect/control_flow/ir/cf_op.h index f56c920bec5fb..0d6e60a017ab3 100644 --- a/paddle/pir/include/dialect/control_flow/ir/cf_op.h +++ b/paddle/pir/include/dialect/control_flow/ir/cf_op.h @@ -14,16 +14,13 @@ #pragma once #include -#include "paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_symbolic_shape.h" #include "paddle/pir/include/core/builder.h" #include "paddle/pir/include/core/op_base.h" #include "paddle/pir/include/core/op_trait.h" #include "paddle/pir/include/dialect/control_flow/ir/cf_interface.h" namespace pir { -class IR_API YieldOp : public Op { +class IR_API YieldOp : public Op { public: using Op::Op; static const char *name() { return "cf.yield"; } @@ -34,8 +31,6 @@ class IR_API YieldOp : public Op &Value); void VerifySig() {} - - bool InferSymbolicShape(pir::ShapeConstraintIRAnalysis *shape_analysis); }; /// diff --git a/paddle/pir/src/dialect/control_flow/ir/cf_op.cc b/paddle/pir/src/dialect/control_flow/ir/cf_op.cc index 0006974a8b70c..3ead6991b272a 100644 --- a/paddle/pir/src/dialect/control_flow/ir/cf_op.cc +++ b/paddle/pir/src/dialect/control_flow/ir/cf_op.cc @@ -25,13 +25,6 @@ void YieldOp::Build(Builder &builder, argument.AddInputs(inputs); } -bool YieldOp::InferSymbolicShape( - pir::ShapeConstraintIRAnalysis *shape_analysis) { - VLOG(3) << "############ YieldOp::InferSymbolicShape start..."; - // YieldOp has no output, just return true - return true; -} - void TuplePushOp::Build(Builder &builder, // NOLINT OperationArgument &argument, // NOLINT Value inlet, From ff5816bb9670cacc3c805dd224d80d63a76c62ab Mon Sep 17 00:00:00 2001 From: co63oc Date: Tue, 27 Feb 2024 10:23:45 +0800 Subject: [PATCH 106/282] Fix typos(RreplenishLayerAndOutput -> ReplenishLayerAndOutput) (#62075) --- .../tensorrt/convert/activation_op.cc | 2 +- .../tensorrt/convert/affine_channel_op.cc | 2 +- .../tensorrt/convert/anchor_generator_op.cc | 2 +- .../inference/tensorrt/convert/arg_max_op.cc | 10 ++--- .../inference/tensorrt/convert/arg_min_op.cc | 10 ++--- .../inference/tensorrt/convert/assign_op.cc | 2 +- .../tensorrt/convert/batch_norm_op.cc | 4 +- .../tensorrt/convert/bilinear_interp_v2_op.cc | 2 +- .../tensorrt/convert/bitwise_and_op.cc | 2 +- .../tensorrt/convert/bitwise_not_op.cc | 2 +- .../tensorrt/convert/bitwise_or_op.cc | 2 +- .../inference/tensorrt/convert/bmm_op.cc | 2 +- .../tensorrt/convert/c_allreduce_op.cc | 2 +- .../inference/tensorrt/convert/cast_op.cc | 2 +- .../inference/tensorrt/convert/celu_op.cc | 2 +- .../inference/tensorrt/convert/clip_op.cc | 2 +- .../inference/tensorrt/convert/concat_op.cc | 2 +- .../convert/cross_multihead_matmul_op.cc | 2 +- .../inference/tensorrt/convert/cumsum_op.cc | 4 +- .../tensorrt/convert/deformable_conv_op.cc | 4 +- .../tensorrt/convert/dequantize_linear_op.cc | 2 +- .../inference/tensorrt/convert/dropout_op.cc | 4 +- .../inference/tensorrt/convert/einsum_op.cc | 2 +- .../tensorrt/convert/elementwise_op.cc | 10 ++--- .../convert/elementwiseadd_transpose_op.cc | 8 ++-- .../tensorrt/convert/emb_eltwise_layernorm.cc | 14 +++---- .../inference/tensorrt/convert/equal_op.cc | 4 +- .../tensorrt/convert/expand_v2_op.cc | 2 +- .../tensorrt/convert/fill_any_like_op.cc | 2 +- .../fill_constant_batch_size_like_op.cc | 2 +- .../tensorrt/convert/fill_constant_op.cc | 2 +- .../convert/flash_multihead_matmul_op.cc | 4 +- .../convert/flatten_contiguous_range_op.cc | 2 +- .../inference/tensorrt/convert/flatten_op.cc | 2 +- .../inference/tensorrt/convert/flip_op.cc | 2 +- .../tensorrt/convert/gather_nd_op.cc | 2 +- .../inference/tensorrt/convert/gather_op.cc | 2 +- .../inference/tensorrt/convert/gelu_op.cc | 2 +- .../generic_and_custom_plugin_creater.cc | 6 +-- .../tensorrt/convert/grid_sampler_op.cc | 2 +- .../tensorrt/convert/group_norm_op.cc | 4 +- .../tensorrt/convert/hard_sigmoid_op.cc | 2 +- .../tensorrt/convert/hard_swish_op.cc | 2 +- .../tensorrt/convert/index_select_op.cc | 2 +- .../tensorrt/convert/instance_norm_op.cc | 2 +- .../tensorrt/convert/layer_norm_op.cc | 6 +-- .../convert/layernorm_shift_partition_op.cc | 2 +- .../tensorrt/convert/leaky_relu_op.cc | 2 +- .../tensorrt/convert/logsigmoid_op.cc | 2 +- .../tensorrt/convert/lookup_table_op.cc | 4 +- .../tensorrt/convert/matrix_multiply_op.cc | 2 +- .../tensorrt/convert/merge_layernorm_op.cc | 2 +- .../inference/tensorrt/convert/mish_op.cc | 2 +- .../tensorrt/convert/multiclass_nms3_op.cc | 6 +-- .../tensorrt/convert/multiclass_nms_op.cc | 2 +- .../tensorrt/convert/multihead_matmul_op.cc | 6 +-- .../convert/multihead_matmul_roformer_op.cc | 2 +- .../tensorrt/convert/nearest_interp_op.cc | 2 +- .../tensorrt/convert/nearest_interp_v2_op.cc | 2 +- .../inference/tensorrt/convert/one_hot_op.cc | 2 +- .../inference/tensorrt/convert/op_converter.h | 2 +- .../inference/tensorrt/convert/pad3d_op.cc | 2 +- .../inference/tensorrt/convert/pad_op.cc | 2 +- .../inference/tensorrt/convert/pool2d_op.cc | 2 +- .../inference/tensorrt/convert/pool3d_op.cc | 2 +- .../convert/preln_groupnorm_act_op.cc | 2 +- .../preln_layernorm_shift_partition_op.cc | 8 ++-- .../tensorrt/convert/preln_residual_bias.cc | 2 +- .../tensorrt/convert/preln_skip_layernorm.cc | 2 +- .../inference/tensorrt/convert/prelu_op.cc | 2 +- .../prompt_tuning_emb_eltwise_layernorm.cc | 16 +++---- .../convert/qk_multihead_matmul_op.cc | 2 +- .../tensorrt/convert/quantize_linear_op.cc | 2 +- .../inference/tensorrt/convert/range_op.cc | 2 +- .../tensorrt/convert/recover_padding_op.cc | 3 +- .../inference/tensorrt/convert/reduce_op.cc | 4 +- .../inference/tensorrt/convert/reshape_op.cc | 2 +- .../tensorrt/convert/reverse_roll_op.cc | 2 +- .../inference/tensorrt/convert/rnn_op.cc | 2 +- .../tensorrt/convert/roi_align_op.cc | 2 +- .../inference/tensorrt/convert/roll_op.cc | 2 +- .../inference/tensorrt/convert/scale_op.cc | 2 +- .../tensorrt/convert/set_value_op.cc | 2 +- .../inference/tensorrt/convert/shape_op.cc | 2 +- .../tensorrt/convert/share_data_op.cc | 2 +- .../tensorrt/convert/shuffle_channel_op.cc | 4 +- .../inference/tensorrt/convert/silu_op.cc | 2 +- .../inference/tensorrt/convert/size_op.cc | 2 +- .../tensorrt/convert/skip_groupnorm_act_op.cc | 2 +- .../tensorrt/convert/skip_layernorm.cc | 2 +- .../convert/skip_merge_layernorm_op.cc | 8 ++-- .../inference/tensorrt/convert/slice_op.cc | 2 +- .../inference/tensorrt/convert/softmax_op.cc | 4 +- .../tensorrt/convert/sparse_fc_op.cc | 42 +++++++++---------- .../convert/sparse_multihead_matmul_op.cc | 2 +- .../inference/tensorrt/convert/split_op.cc | 6 +-- .../inference/tensorrt/convert/square_op.cc | 2 +- .../inference/tensorrt/convert/squeeze2_op.cc | 2 +- .../inference/tensorrt/convert/stack_op.cc | 2 +- .../tensorrt/convert/strided_slice_op.cc | 2 +- .../inference/tensorrt/convert/sum_op.cc | 2 +- .../inference/tensorrt/convert/swish_op.cc | 2 +- .../tensorrt/convert/take_along_axis_op.cc | 3 +- .../tensorrt/convert/tanhshrink_op.cc | 2 +- .../tensorrt/convert/temporal_shift_op.cc | 4 +- .../inference/tensorrt/convert/tile_op.cc | 4 +- .../tensorrt/convert/trans_layernorm_op.cc | 8 ++-- .../convert/transformer_input_convert_op.cc | 8 ++-- .../tensorrt/convert/transpose_op.cc | 2 +- .../inference/tensorrt/convert/unary_op.cc | 2 +- .../inference/tensorrt/convert/unbind_op.cc | 2 +- .../tensorrt/convert/unsqueeze2_op.cc | 2 +- .../inference/tensorrt/convert/where_op.cc | 2 +- .../tensorrt/convert/yolo_box_head_op.cc | 2 +- .../inference/tensorrt/convert/yolo_box_op.cc | 2 +- 115 files changed, 198 insertions(+), 200 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/convert/activation_op.cc b/paddle/fluid/inference/tensorrt/convert/activation_op.cc index 43ef00ef1dfdc..f09e5091ae9b1 100644 --- a/paddle/fluid/inference/tensorrt/convert/activation_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/activation_op.cc @@ -112,7 +112,7 @@ class ActivationOpConverter : public OpConverter { auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, op_type_, {output_name}, test_mode); + ReplenishLayerAndOutput(layer, op_type_, {output_name}, test_mode); } protected: diff --git a/paddle/fluid/inference/tensorrt/convert/affine_channel_op.cc b/paddle/fluid/inference/tensorrt/convert/affine_channel_op.cc index 7cc3e053505d1..d7699c7c1003c 100644 --- a/paddle/fluid/inference/tensorrt/convert/affine_channel_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/affine_channel_op.cc @@ -72,7 +72,7 @@ class AffineChannelOpConverter : public OpConverter { power_weights.get(), channel_axis); - RreplenishLayerAndOutput(layer, "affine_channel", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "affine_channel", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/anchor_generator_op.cc b/paddle/fluid/inference/tensorrt/convert/anchor_generator_op.cc index 86d8809a6037f..6c386b1ba08f7 100644 --- a/paddle/fluid/inference/tensorrt/convert/anchor_generator_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/anchor_generator_op.cc @@ -80,7 +80,7 @@ class AnchorGeneratorOpConverter : public OpConverter { anchor_generator_inputs.size(), *anchor_generator_plugin); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( anchor_generator_layer, "anchor_generator", output_names, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/arg_max_op.cc b/paddle/fluid/inference/tensorrt/convert/arg_max_op.cc index 8d95810e15794..3b2301256cd05 100644 --- a/paddle/fluid/inference/tensorrt/convert/arg_max_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/arg_max_op.cc @@ -42,10 +42,10 @@ class ArgMaxOpConverter : public OpConverter { auto output_name = op_desc.Output("Out")[0]; bool keepdims = PADDLE_GET_CONST(bool, op_desc.GetAttr("keepdims")); if (keepdims) { - RreplenishLayerAndOutput(topk_layer, - "arg_max", - {output_name + "_value", output_name}, - test_mode); + ReplenishLayerAndOutput(topk_layer, + "arg_max", + {output_name + "_value", output_name}, + test_mode); } else { auto squeeze_layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *topk_layer->getOutput(1)); @@ -55,7 +55,7 @@ class ArgMaxOpConverter : public OpConverter { dims.d[i] = dims.d[i + 1]; } squeeze_layer->setReshapeDimensions(dims); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( squeeze_layer, "arg_max", {output_name}, test_mode); } } diff --git a/paddle/fluid/inference/tensorrt/convert/arg_min_op.cc b/paddle/fluid/inference/tensorrt/convert/arg_min_op.cc index 760e398c41e2b..c7e8b483fab76 100644 --- a/paddle/fluid/inference/tensorrt/convert/arg_min_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/arg_min_op.cc @@ -42,10 +42,10 @@ class ArgMinOpConverter : public OpConverter { auto output_name = op_desc.Output("Out")[0]; bool keepdims = PADDLE_GET_CONST(bool, op_desc.GetAttr("keepdims")); if (keepdims) { - RreplenishLayerAndOutput(topk_layer, - "arg_min", - {output_name + "_value", output_name}, - test_mode); + ReplenishLayerAndOutput(topk_layer, + "arg_min", + {output_name + "_value", output_name}, + test_mode); } else { auto squeeze_layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *topk_layer->getOutput(1)); @@ -55,7 +55,7 @@ class ArgMinOpConverter : public OpConverter { dims.d[i] = dims.d[i + 1]; } squeeze_layer->setReshapeDimensions(dims); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( squeeze_layer, "arg_min", {output_name}, test_mode); } } diff --git a/paddle/fluid/inference/tensorrt/convert/assign_op.cc b/paddle/fluid/inference/tensorrt/convert/assign_op.cc index 5f14d19ee132b..06534a90a76d8 100644 --- a/paddle/fluid/inference/tensorrt/convert/assign_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/assign_op.cc @@ -28,7 +28,7 @@ class AssignOpConverter : public OpConverter { auto* input = engine_->GetITensor(op_desc.Input("X")[0]); auto* layer = TRT_ENGINE_ADD_LAYER(engine_, Identity, *input); auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "assign", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "assign", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/batch_norm_op.cc b/paddle/fluid/inference/tensorrt/convert/batch_norm_op.cc index 01d6a156ce2b2..0b350c6c239d4 100644 --- a/paddle/fluid/inference/tensorrt/convert/batch_norm_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/batch_norm_op.cc @@ -170,10 +170,10 @@ class BatchNormOpConverter : public OpConverter { squeeze_layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *(layer->getOutput(0))); squeeze_layer->setReshapeDimensions(squeeze_shape); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( squeeze_layer, "batchnorm_add_scale", {output_name}, test_mode); } else { - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( layer, "batchnorm_add_scale", {output_name}, test_mode); } } diff --git a/paddle/fluid/inference/tensorrt/convert/bilinear_interp_v2_op.cc b/paddle/fluid/inference/tensorrt/convert/bilinear_interp_v2_op.cc index 4cd7378c17b44..854eaa4cc0b7e 100644 --- a/paddle/fluid/inference/tensorrt/convert/bilinear_interp_v2_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/bilinear_interp_v2_op.cc @@ -148,7 +148,7 @@ class BilinearInterpolateV2OpConverter : public OpConverter { layer->setScales(scales.data(), scales.size()); } - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( layer, "bilinear_interp_v2", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/bitwise_and_op.cc b/paddle/fluid/inference/tensorrt/convert/bitwise_and_op.cc index 4c8e60573d845..f01d019da00bd 100644 --- a/paddle/fluid/inference/tensorrt/convert/bitwise_and_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/bitwise_and_op.cc @@ -49,7 +49,7 @@ class BitwiseAndConverter : public OpConverter { } auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "bitwise_and", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "bitwise_and", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/bitwise_not_op.cc b/paddle/fluid/inference/tensorrt/convert/bitwise_not_op.cc index 08324bb5d3003..a944527313a02 100644 --- a/paddle/fluid/inference/tensorrt/convert/bitwise_not_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/bitwise_not_op.cc @@ -69,7 +69,7 @@ class BitwiseNotConverter : public OpConverter { } auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "bitwise_not", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "bitwise_not", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/bitwise_or_op.cc b/paddle/fluid/inference/tensorrt/convert/bitwise_or_op.cc index 33e82334d59e4..814ee8bd98551 100644 --- a/paddle/fluid/inference/tensorrt/convert/bitwise_or_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/bitwise_or_op.cc @@ -49,7 +49,7 @@ class BitwiseOrConverter : public OpConverter { } auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "bitwise_or", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "bitwise_or", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/bmm_op.cc b/paddle/fluid/inference/tensorrt/convert/bmm_op.cc index a7f4b15e7854d..861a4b05306c1 100644 --- a/paddle/fluid/inference/tensorrt/convert/bmm_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/bmm_op.cc @@ -38,7 +38,7 @@ class BMMOpConverter : public OpConverter { *input2, nvinfer1::MatrixOperation::kNONE); - RreplenishLayerAndOutput(layer, "bmm", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "bmm", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/c_allreduce_op.cc b/paddle/fluid/inference/tensorrt/convert/c_allreduce_op.cc index ae8d2196842a4..767cf996f7d7f 100644 --- a/paddle/fluid/inference/tensorrt/convert/c_allreduce_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/c_allreduce_op.cc @@ -84,7 +84,7 @@ class CAllReduceOpConverter : public OpConverter { #endif auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, name, {output_name}, test_mode); + ReplenishLayerAndOutput(layer, name, {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/cast_op.cc b/paddle/fluid/inference/tensorrt/convert/cast_op.cc index 34920cb62c794..77b0971a66979 100644 --- a/paddle/fluid/inference/tensorrt/convert/cast_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/cast_op.cc @@ -57,7 +57,7 @@ class CastOpConverter : public OpConverter { } auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "cast", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "cast", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/celu_op.cc b/paddle/fluid/inference/tensorrt/convert/celu_op.cc index c7ac2d989e038..837364a9feca7 100644 --- a/paddle/fluid/inference/tensorrt/convert/celu_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/celu_op.cc @@ -78,7 +78,7 @@ class CeluOpConverter : public OpConverter { nvinfer1::ElementWiseOperation::kSUM); auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "celu", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "celu", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/clip_op.cc b/paddle/fluid/inference/tensorrt/convert/clip_op.cc index e867f2832c8c1..5bd2a73c3fefb 100644 --- a/paddle/fluid/inference/tensorrt/convert/clip_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/clip_op.cc @@ -39,7 +39,7 @@ class ClipOpConverter : public OpConverter { layer->setBeta(max); auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "clip", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "clip", {output_name}, test_mode); #else PADDLE_THROW( platform::errors::Fatal("clip TRT converter is only supported on TRT " diff --git a/paddle/fluid/inference/tensorrt/convert/concat_op.cc b/paddle/fluid/inference/tensorrt/convert/concat_op.cc index 7f0afb5742074..6f4fdc30214b5 100644 --- a/paddle/fluid/inference/tensorrt/convert/concat_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/concat_op.cc @@ -49,7 +49,7 @@ class ConcatOpConverter : public OpConverter { engine_, Concatenation, itensors.data(), itensors.size()); layer->setAxis(axis); auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "concat", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "concat", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/cross_multihead_matmul_op.cc b/paddle/fluid/inference/tensorrt/convert/cross_multihead_matmul_op.cc index 0be73687858f5..6a1cf1951f9a6 100644 --- a/paddle/fluid/inference/tensorrt/convert/cross_multihead_matmul_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/cross_multihead_matmul_op.cc @@ -265,7 +265,7 @@ class CrossMultiheadMatMulOpConverter : public OpConverter { ("shuffle_last_multihead_matmul(Output: " + output_name + ")").c_str()); // return layer = reshape_after_mha_layer; - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( layer, "cross_multihead_matmul", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/cumsum_op.cc b/paddle/fluid/inference/tensorrt/convert/cumsum_op.cc index 7ed361cad178d..a18d16bd012fb 100644 --- a/paddle/fluid/inference/tensorrt/convert/cumsum_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/cumsum_op.cc @@ -45,7 +45,7 @@ class CumsumOpConverter : public OpConverter { cumsum_dim.d[0] = 1; } layer->setReshapeDimensions(cumsum_dim); - RreplenishLayerAndOutput(layer, "cumsum", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "cumsum", {output_name}, test_mode); } else { int axis = 0; if (op_desc.HasAttr("axis")) { @@ -161,7 +161,7 @@ class CumsumOpConverter : public OpConverter { nvinfer1::ILoopOutputLayer* loopOut = loop->addLoopOutput(*curSum->getOutput(0), reverseFlag, axis); loopOut->setInput(1, *tripLimit); - RreplenishLayerAndOutput(loopOut, "cumsum", {output_name}, test_mode); + ReplenishLayerAndOutput(loopOut, "cumsum", {output_name}, test_mode); } #else VLOG(3) << "Cumsum is not supported when TensorRT < 7.2.2"; diff --git a/paddle/fluid/inference/tensorrt/convert/deformable_conv_op.cc b/paddle/fluid/inference/tensorrt/convert/deformable_conv_op.cc index a126a6a5f06bf..4df2abb1a32ea 100644 --- a/paddle/fluid/inference/tensorrt/convert/deformable_conv_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/deformable_conv_op.cc @@ -105,7 +105,7 @@ class DeformableConvOpConverter : public OpConverter { std::vector output_names; output_names.push_back(op_desc.Output("Output").front()); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( deformable_conv_layer, "deformable_conv", output_names, test_mode); } else { auto* deformable_conv_plugin = new plugin::DeformableConvPluginDynamic( @@ -133,7 +133,7 @@ class DeformableConvOpConverter : public OpConverter { std::vector output_names; output_names.push_back(op_desc.Output("Output").front()); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( deformable_conv_layer, "deformable_conv", output_names, test_mode); } } diff --git a/paddle/fluid/inference/tensorrt/convert/dequantize_linear_op.cc b/paddle/fluid/inference/tensorrt/convert/dequantize_linear_op.cc index e0caca327a262..9b88e14fc9efe 100644 --- a/paddle/fluid/inference/tensorrt/convert/dequantize_linear_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/dequantize_linear_op.cc @@ -49,7 +49,7 @@ class DequantizeLinearOpConverter : public OpConverter { layer->setAxis(axis); } auto output_name = op_desc.Output("Y")[0]; - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( layer, "dequantize_linear", {output_name}, test_model); #else PADDLE_THROW( diff --git a/paddle/fluid/inference/tensorrt/convert/dropout_op.cc b/paddle/fluid/inference/tensorrt/convert/dropout_op.cc index bec18da482e41..b3a9da4d67ebf 100644 --- a/paddle/fluid/inference/tensorrt/convert/dropout_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/dropout_op.cc @@ -43,7 +43,7 @@ class DropoutOpConverter : public OpConverter { downgrade_in_infer == "upscale_in_train") { auto* layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *input1); auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "dropout", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "dropout", {output_name}, test_mode); return; } @@ -75,7 +75,7 @@ class DropoutOpConverter : public OpConverter { std::move(weight_tensor)); auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "dropout", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "dropout", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/einsum_op.cc b/paddle/fluid/inference/tensorrt/convert/einsum_op.cc index e43615da01c09..df7854acc1682 100644 --- a/paddle/fluid/inference/tensorrt/convert/einsum_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/einsum_op.cc @@ -39,7 +39,7 @@ class EinsumOpConverter : public OpConverter { engine_, Einsum, input_tensors.data(), input_num, equation.c_str()); auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "einsum", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "einsum", {output_name}, test_mode); #else VLOG(3) << "Einsum is not supported when TensorRT < 8.2.0"; #endif diff --git a/paddle/fluid/inference/tensorrt/convert/elementwise_op.cc b/paddle/fluid/inference/tensorrt/convert/elementwise_op.cc index f5f7e53cf4e0d..c02c0159c1ff9 100644 --- a/paddle/fluid/inference/tensorrt/convert/elementwise_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/elementwise_op.cc @@ -162,7 +162,7 @@ class ElementwiseTensorOpConverter : public OpConverter { *(less_layer->getOutput(0)), *(equal_layer->getOutput(0)), nvinfer1::ElementWiseOperation::kOR); - RreplenishLayerAndOutput(layer, "elementwise", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "elementwise", {output_name}, test_mode); } else if (op_type_ == "greater_equal") { auto* greater_layer = TRT_ENGINE_ADD_LAYER(engine_, @@ -181,7 +181,7 @@ class ElementwiseTensorOpConverter : public OpConverter { *(greater_layer->getOutput(0)), *(equal_layer->getOutput(0)), nvinfer1::ElementWiseOperation::kOR); - RreplenishLayerAndOutput(layer, "elementwise", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "elementwise", {output_name}, test_mode); } else if (op_type_ == "mod") { auto* div_layer = TRT_ENGINE_ADD_LAYER(engine_, @@ -203,7 +203,7 @@ class ElementwiseTensorOpConverter : public OpConverter { *(mul_layer->getOutput(0)), nvinfer1::ElementWiseOperation::kSUB); SupportFP32MixPrecision(output_name, op_desc.Type(), layer); - RreplenishLayerAndOutput(layer, "elementwise", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "elementwise", {output_name}, test_mode); } else { auto op_pair = ops.find(op_type_); PADDLE_ENFORCE_NE( @@ -217,7 +217,7 @@ class ElementwiseTensorOpConverter : public OpConverter { auto* layer = TRT_ENGINE_ADD_LAYER( engine_, ElementWise, *X, *reshape_y_tensor, op_pair->second); SupportFP32MixPrecision(output_name, op_desc.Type(), layer); - RreplenishLayerAndOutput(layer, "elementwise", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "elementwise", {output_name}, test_mode); } } @@ -350,7 +350,7 @@ class PowOpConverter : public OpConverter { auto* layer = TRT_ENGINE_ADD_LAYER( engine_, ElementWise, *X, *Y, nvinfer1::ElementWiseOperation::kPOW); SupportFP32MixPrecision(output_name, op_desc.Type(), layer); - RreplenishLayerAndOutput(layer, "elementwise", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "elementwise", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/elementwiseadd_transpose_op.cc b/paddle/fluid/inference/tensorrt/convert/elementwiseadd_transpose_op.cc index 71b00708edfcb..ed25619a01b0c 100644 --- a/paddle/fluid/inference/tensorrt/convert/elementwiseadd_transpose_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/elementwiseadd_transpose_op.cc @@ -40,10 +40,10 @@ class ElementwiseaddTransposeOpConverter : public OpConverter { engine_->AddDynamicPlugin(inputs.data(), 2, plugin); std::vector output_names; output_names.emplace_back(op_desc.Output("Out").front()); - RreplenishLayerAndOutput(elementwise_layer, - "fuse_elementwiseadd_transpose", - output_names, - test_mode); + ReplenishLayerAndOutput(elementwise_layer, + "fuse_elementwiseadd_transpose", + output_names, + test_mode); } } }; diff --git a/paddle/fluid/inference/tensorrt/convert/emb_eltwise_layernorm.cc b/paddle/fluid/inference/tensorrt/convert/emb_eltwise_layernorm.cc index 6ccb22e072f1b..340f16330a2e5 100644 --- a/paddle/fluid/inference/tensorrt/convert/emb_eltwise_layernorm.cc +++ b/paddle/fluid/inference/tensorrt/convert/emb_eltwise_layernorm.cc @@ -171,12 +171,12 @@ class EmbEltwiseLayerNormOpConverter : public OpConverter { } else { layer = plugin_layer; auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, - "ManyEmbLayerNormVarlenPluginDynamicV1", - {output_name, - std::string("qkv_plugin_mask"), - std::string("max_seqlen_tensor")}, - test_mode); + ReplenishLayerAndOutput(layer, + "ManyEmbLayerNormVarlenPluginDynamicV1", + {output_name, + std::string("qkv_plugin_mask"), + std::string("max_seqlen_tensor")}, + test_mode); } } else { for (int i = 0; i < input_num; i++) { @@ -247,7 +247,7 @@ class EmbEltwiseLayerNormOpConverter : public OpConverter { } layer = plugin_layer; auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( layer, "ManyEmbLayerNormPluginDynamicV1", {output_name}, test_mode); } } diff --git a/paddle/fluid/inference/tensorrt/convert/equal_op.cc b/paddle/fluid/inference/tensorrt/convert/equal_op.cc index 98db107752817..c1e196725c64c 100644 --- a/paddle/fluid/inference/tensorrt/convert/equal_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/equal_op.cc @@ -68,7 +68,7 @@ class EqualOpConverter : public OpConverter { layer = TRT_ENGINE_ADD_LAYER( engine_, ElementWise, *X, *Y, nvinfer1::ElementWiseOperation::kEQUAL); - RreplenishLayerAndOutput(layer, "equal", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "equal", {output_name}, test_mode); } }; @@ -125,7 +125,7 @@ class NotEqualOpConverter : public OpConverter { layer = TRT_ENGINE_ADD_LAYER( engine_, Unary, *layer->getOutput(0), nvinfer1::UnaryOperation::kNOT); - RreplenishLayerAndOutput(layer, "not_equal", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "not_equal", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/expand_v2_op.cc b/paddle/fluid/inference/tensorrt/convert/expand_v2_op.cc index 0efccce390a62..8b92d5b0a7116 100644 --- a/paddle/fluid/inference/tensorrt/convert/expand_v2_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/expand_v2_op.cc @@ -113,7 +113,7 @@ class ExpandOpConverter : public OpConverter { layer->setInput(2, *sizes_tensor); layer->setInput(3, *strides_tensor); - RreplenishLayerAndOutput(layer, op_type_, {output_name}, test_mode); + ReplenishLayerAndOutput(layer, op_type_, {output_name}, test_mode); } protected: diff --git a/paddle/fluid/inference/tensorrt/convert/fill_any_like_op.cc b/paddle/fluid/inference/tensorrt/convert/fill_any_like_op.cc index 0faf188e328f4..2b3efea9bd7bd 100644 --- a/paddle/fluid/inference/tensorrt/convert/fill_any_like_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/fill_any_like_op.cc @@ -77,7 +77,7 @@ class FillAnyLikeOpConverter : public OpConverter { layer->setInput(2, *sizes_tensor); layer->setInput(3, *strides_tensor); - RreplenishLayerAndOutput(layer, "fill_any_like", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "fill_any_like", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/fill_constant_batch_size_like_op.cc b/paddle/fluid/inference/tensorrt/convert/fill_constant_batch_size_like_op.cc index 69decf67d1e71..a0a3aa4e43934 100644 --- a/paddle/fluid/inference/tensorrt/convert/fill_constant_batch_size_like_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/fill_constant_batch_size_like_op.cc @@ -76,7 +76,7 @@ class FillConstantBatchSizeLikeOpConverter : public OpConverter { layer->setInput(1, *Add1DConstantLayer(value_vec, name + "alpha", true)); layer->setInput(2, *Add1DConstantLayer(beta_vec, name + "beta", false)); auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( layer, "fill_constant_batch_size_like", {output_name}, test_mode); #endif } diff --git a/paddle/fluid/inference/tensorrt/convert/fill_constant_op.cc b/paddle/fluid/inference/tensorrt/convert/fill_constant_op.cc index 533f44612c548..3b9cc9dd0d349 100644 --- a/paddle/fluid/inference/tensorrt/convert/fill_constant_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/fill_constant_op.cc @@ -120,7 +120,7 @@ class FillConstantOpConverter : public OpConverter { TRT_ENGINE_ADD_LAYER(engine_, Constant, trt_in_shape, weight.get()); } auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "fill_constant", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "fill_constant", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/flash_multihead_matmul_op.cc b/paddle/fluid/inference/tensorrt/convert/flash_multihead_matmul_op.cc index 91d3cfd10ae30..8b49127cb93db 100644 --- a/paddle/fluid/inference/tensorrt/convert/flash_multihead_matmul_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/flash_multihead_matmul_op.cc @@ -239,7 +239,7 @@ class FlashMultiheadMatMulOpConverter : public OpConverter { reshape_after_mha_layer->setName( ("shuffle_last_multihead_matmul(Output: " + output_name + ")").c_str()); layer = reshape_after_mha_layer; - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( layer, "flash_multihead_matmul", {output_name}, test_mode); } @@ -444,7 +444,7 @@ class FlashMultiheadMatMulOpConverter : public OpConverter { ")") .c_str()); std::vector output_names = {output_name}; - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( reshape_after_attention_layer, op_desc.Type(), output_names, test_mode); } diff --git a/paddle/fluid/inference/tensorrt/convert/flatten_contiguous_range_op.cc b/paddle/fluid/inference/tensorrt/convert/flatten_contiguous_range_op.cc index 32bee4fca9fa0..1d400b2f801cc 100644 --- a/paddle/fluid/inference/tensorrt/convert/flatten_contiguous_range_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/flatten_contiguous_range_op.cc @@ -169,7 +169,7 @@ class FlattenContiguousRangeOpConverter : public OpConverter { } auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( layer, "flatten_contiguous_range", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/flatten_op.cc b/paddle/fluid/inference/tensorrt/convert/flatten_op.cc index e6fd7aee5cde3..b778971ea05eb 100644 --- a/paddle/fluid/inference/tensorrt/convert/flatten_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/flatten_op.cc @@ -88,7 +88,7 @@ class FlattenOpConverter : public OpConverter { layer->setInput(1, *(concat_layer->getOutput(0))); } auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "flatten", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "flatten", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/flip_op.cc b/paddle/fluid/inference/tensorrt/convert/flip_op.cc index 53ac35e690551..0ac714507b5ce 100644 --- a/paddle/fluid/inference/tensorrt/convert/flip_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/flip_op.cc @@ -72,7 +72,7 @@ class FlipOpConverter : public OpConverter { auto* layer = TRT_ENGINE_ADD_LAYER(engine_, Identity, *input); auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "flip", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "flip", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/gather_nd_op.cc b/paddle/fluid/inference/tensorrt/convert/gather_nd_op.cc index 2b926ccdf1221..508d7a5f9b390 100644 --- a/paddle/fluid/inference/tensorrt/convert/gather_nd_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/gather_nd_op.cc @@ -36,7 +36,7 @@ class GatherNdOpConverter : public OpConverter { auto layer = TRT_ENGINE_ADD_LAYER( engine_, GatherV2, *input, *index, nvinfer1::GatherMode::kND); layer->setNbElementWiseDims(0); - RreplenishLayerAndOutput(layer, "gather_nd", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "gather_nd", {output_name}, test_mode); #else VLOG(4) << "convert a paddle gather_nd op to tensorrt gather_nd plugin"; diff --git a/paddle/fluid/inference/tensorrt/convert/gather_op.cc b/paddle/fluid/inference/tensorrt/convert/gather_op.cc index 71322f302fb72..06c43c663daee 100644 --- a/paddle/fluid/inference/tensorrt/convert/gather_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/gather_op.cc @@ -54,7 +54,7 @@ class GatherOpConverter : public OpConverter { engine_, Gather, *input_tensor, *reshape_layer->getOutput(0), axis); layer->setNbElementWiseDims(0); - RreplenishLayerAndOutput(layer, "gather", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "gather", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/gelu_op.cc b/paddle/fluid/inference/tensorrt/convert/gelu_op.cc index 539bcd6a3358e..54a11b4d54b52 100644 --- a/paddle/fluid/inference/tensorrt/convert/gelu_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/gelu_op.cc @@ -236,7 +236,7 @@ class GeluOpConverter : public OpConverter { #endif // if IS_TRT_VERSION_GE(7000) } auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "gelu", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "gelu", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc b/paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc index 98cff37d02fbe..5e4dfca1417f8 100644 --- a/paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc +++ b/paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc @@ -165,7 +165,7 @@ class CustomPluginCreater : public OpConverter { output_names.push_back(arg_name); } - RreplenishLayerAndOutput(layer, op_desc.Type(), output_names, test_mode); + ReplenishLayerAndOutput(layer, op_desc.Type(), output_names, test_mode); } }; @@ -248,7 +248,7 @@ class GenericPluginCreater : public OpConverter { new plugin::GenericPlugin(op, in_out_info, with_fp16); layer = engine_->AddDynamicPlugin(inputs.data(), inputs.size(), plugin); - RreplenishLayerAndOutput(layer, op_desc.Type(), output_names, test_mode); + ReplenishLayerAndOutput(layer, op_desc.Type(), output_names, test_mode); } }; @@ -334,7 +334,7 @@ class CustomGenericPluginCreater : public OpConverter { inputs.data(), inputs.size(), (plugin::DynamicPluginTensorRT *)plugin); CHECK(layer); - RreplenishLayerAndOutput(layer, op_desc.Type(), outputs, test_mode); + ReplenishLayerAndOutput(layer, op_desc.Type(), outputs, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/grid_sampler_op.cc b/paddle/fluid/inference/tensorrt/convert/grid_sampler_op.cc index fb73648485a0e..48bcdddcba875 100644 --- a/paddle/fluid/inference/tensorrt/convert/grid_sampler_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/grid_sampler_op.cc @@ -66,7 +66,7 @@ class GridSamplerOpConverter : public OpConverter { layer->setSampleMode(sampleMode); layer->setAlignCorners(align_corners); - RreplenishLayerAndOutput(layer, "grid_sampler", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "grid_sampler", {output_name}, test_mode); #else VLOG(3) << "grid_sampler is not supported when TensorRT < 8.5.1"; #endif diff --git a/paddle/fluid/inference/tensorrt/convert/group_norm_op.cc b/paddle/fluid/inference/tensorrt/convert/group_norm_op.cc index c9adf7c34bcf0..98f5d04cfa324 100644 --- a/paddle/fluid/inference/tensorrt/convert/group_norm_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/group_norm_op.cc @@ -81,7 +81,7 @@ class GroupNormOpConverter : public OpConverter { nvinfer1::ILayer* groupnorm_layer = engine_->AddDynamicPlugin(&input_itensor, 1, plugin); auto output_name = op_desc.Output("Y")[0]; - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( groupnorm_layer, "group_norm", {output_name}, test_mode); } else { int gn_num = input_itensor->getDimensions().d[0] * groups; @@ -100,7 +100,7 @@ class GroupNormOpConverter : public OpConverter { nvinfer1::ILayer* groupnorm_layer = engine_->AddPlugin(&input_itensor, 1, plugin); auto output_name = op_desc.Output("Y")[0]; - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( groupnorm_layer, "group_norm", {output_name}, test_mode); } } diff --git a/paddle/fluid/inference/tensorrt/convert/hard_sigmoid_op.cc b/paddle/fluid/inference/tensorrt/convert/hard_sigmoid_op.cc index eb8756da31853..efb0b2ae06bab 100644 --- a/paddle/fluid/inference/tensorrt/convert/hard_sigmoid_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/hard_sigmoid_op.cc @@ -40,7 +40,7 @@ class HardSigmoidOpConverter : public OpConverter { layer->setBeta(offset); auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "hard_sigmoid", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "hard_sigmoid", {output_name}, test_mode); #else PADDLE_THROW(platform::errors::Fatal( "Hard sigmoid TRT converter is only supported on TRT 5 or higher. " diff --git a/paddle/fluid/inference/tensorrt/convert/hard_swish_op.cc b/paddle/fluid/inference/tensorrt/convert/hard_swish_op.cc index e46a123494935..63e1f45b539dd 100644 --- a/paddle/fluid/inference/tensorrt/convert/hard_swish_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/hard_swish_op.cc @@ -85,7 +85,7 @@ class HardSwishOpConverter : public OpConverter { nvinfer1::ElementWiseOperation::kDIV); } auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "hard_swish", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "hard_swish", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/index_select_op.cc b/paddle/fluid/inference/tensorrt/convert/index_select_op.cc index 6f98371210c51..9ee875c92445e 100644 --- a/paddle/fluid/inference/tensorrt/convert/index_select_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/index_select_op.cc @@ -64,7 +64,7 @@ class IndexSelectConverter : public OpConverter { engine_, Gather, *input_tensor, *reshape_layer->getOutput(0), axis); layer->setNbElementWiseDims(0); - RreplenishLayerAndOutput(layer, "index_select", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "index_select", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/instance_norm_op.cc b/paddle/fluid/inference/tensorrt/convert/instance_norm_op.cc index 3a51a9f37e216..bd97df48309c7 100644 --- a/paddle/fluid/inference/tensorrt/convert/instance_norm_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/instance_norm_op.cc @@ -73,7 +73,7 @@ class InstanceNormOpConverter : public OpConverter { instance_norm_inputs.data(), instance_norm_inputs.size(), *plugin); auto output_name = op_desc.Output("Y")[0]; - RreplenishLayerAndOutput(layer, "instance_norm", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "instance_norm", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/layer_norm_op.cc b/paddle/fluid/inference/tensorrt/convert/layer_norm_op.cc index c39f9d3324243..50fa54bcf90c2 100644 --- a/paddle/fluid/inference/tensorrt/convert/layer_norm_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/layer_norm_op.cc @@ -70,7 +70,7 @@ class LayerNormOpConverter : public OpConverter { auto layer = TRT_ENGINE_ADD_LAYER( engine_, Normalization, *X, *Scale_reshape, *Bias_reshape, axisMask); layer->setEpsilon(eps); - RreplenishLayerAndOutput(layer, "layer_norm", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "layer_norm", {output_name}, test_mode); #endif #if IS_TRT_VERSION_LT(8600) // For dynamic shape & trt<8.6, @@ -113,7 +113,7 @@ class LayerNormOpConverter : public OpConverter { variance_shape, with_fp16); layernorm_layer = engine_->AddDynamicPlugin(&X, 1, plugin); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( layernorm_layer, "layer_norm", {output_name}, test_mode); #endif } else { @@ -160,7 +160,7 @@ class LayerNormOpConverter : public OpConverter { with_fp16); auto* layernorm_layer = engine_->AddPlugin( &X, 1, reinterpret_cast(plugin)); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( layernorm_layer, "layer_norm", {output_name}, test_mode); } } diff --git a/paddle/fluid/inference/tensorrt/convert/layernorm_shift_partition_op.cc b/paddle/fluid/inference/tensorrt/convert/layernorm_shift_partition_op.cc index 904d7aaf9045b..7cf5dea57d5d4 100644 --- a/paddle/fluid/inference/tensorrt/convert/layernorm_shift_partition_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/layernorm_shift_partition_op.cc @@ -96,7 +96,7 @@ class LayerNormShiftPartitionOpConverter : public OpConverter { } auto output_name = op_desc.Output("Y").front(); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( layernorm_layer, "layernorm_shift_partition", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/leaky_relu_op.cc b/paddle/fluid/inference/tensorrt/convert/leaky_relu_op.cc index 85a085aa221c4..d3fda4cb24e28 100644 --- a/paddle/fluid/inference/tensorrt/convert/leaky_relu_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/leaky_relu_op.cc @@ -116,7 +116,7 @@ class LeakyReluOpConverter : public OpConverter { engine_->SetWeights(alpha_name, std::move(alpha_tensor)); #endif auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( output_layer, "leaky_relu", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/logsigmoid_op.cc b/paddle/fluid/inference/tensorrt/convert/logsigmoid_op.cc index 6335f4a4b4408..8838bac48f9aa 100644 --- a/paddle/fluid/inference/tensorrt/convert/logsigmoid_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/logsigmoid_op.cc @@ -54,7 +54,7 @@ class LogSigmoidOpConverter : public OpConverter { *(sigmoid->getOutput(0)), nvinfer1::UnaryOperation::kLOG); auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "logsigmoid", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "logsigmoid", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/lookup_table_op.cc b/paddle/fluid/inference/tensorrt/convert/lookup_table_op.cc index ac11f2e2e3bbb..cdb49be72f50f 100644 --- a/paddle/fluid/inference/tensorrt/convert/lookup_table_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/lookup_table_op.cc @@ -47,7 +47,7 @@ class LookupTableOpConverter : public OpConverter { auto* gather_layer = TRT_ENGINE_ADD_LAYER( engine_, Gather, *w_tensor, *reshape_layer->getOutput(0), 0); - RreplenishLayerAndOutput(gather_layer, "gather", {out_name}, test_mode); + ReplenishLayerAndOutput(gather_layer, "gather", {out_name}, test_mode); } }; @@ -68,7 +68,7 @@ class LookupTableV2OpConverter : public OpConverter { auto* gather_layer = TRT_ENGINE_ADD_LAYER(engine_, Gather, *w_tensor, *ids_tensor, 0); - RreplenishLayerAndOutput(gather_layer, "gather", {out_name}, test_mode); + ReplenishLayerAndOutput(gather_layer, "gather", {out_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/matrix_multiply_op.cc b/paddle/fluid/inference/tensorrt/convert/matrix_multiply_op.cc index ebe4c724180d1..16d6f3f20750c 100644 --- a/paddle/fluid/inference/tensorrt/convert/matrix_multiply_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/matrix_multiply_op.cc @@ -261,7 +261,7 @@ class MatrixMultiplyOpConverter : public OpConverter { nvinfer1::ElementWiseOperation::kPROD); SupportFP32MixPrecision(output_name, op_desc.Type(), layer); } - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( layer, "matrix_multiply_op", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/merge_layernorm_op.cc b/paddle/fluid/inference/tensorrt/convert/merge_layernorm_op.cc index 6b425f7d4ea14..0b04132d30c2b 100644 --- a/paddle/fluid/inference/tensorrt/convert/merge_layernorm_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/merge_layernorm_op.cc @@ -76,7 +76,7 @@ class MergeLayernormOpConverter : public OpConverter { "mode.")); } auto output_name = op_desc.Output("Y").front(); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( merge_layernorm_layer, "merge_layernorm", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/mish_op.cc b/paddle/fluid/inference/tensorrt/convert/mish_op.cc index de3a7df82b0fd..e28e02de8b738 100644 --- a/paddle/fluid/inference/tensorrt/convert/mish_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/mish_op.cc @@ -54,7 +54,7 @@ class MishOpConverter : public OpConverter { } auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "mish", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "mish", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/multiclass_nms3_op.cc b/paddle/fluid/inference/tensorrt/convert/multiclass_nms3_op.cc index 72b456b5f7b8c..449593533820b 100644 --- a/paddle/fluid/inference/tensorrt/convert/multiclass_nms3_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/multiclass_nms3_op.cc @@ -163,11 +163,11 @@ class MultiClassNMS3OpConverter : public OpConverter { nvinfer1::Weights{ nvinfer1::DataType::kINT32, static_cast(index.data()), 1}); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( batch_nms_layer, "multiclass_nms3", {rois_num_name}, test_mode); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( nms_concat_layer, "multiclass_nms3", {output_name}, test_mode); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( constant_layer, "multiclass_nms3", {index_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/multiclass_nms_op.cc b/paddle/fluid/inference/tensorrt/convert/multiclass_nms_op.cc index 1eb103b958e51..e14ee099aa0f8 100644 --- a/paddle/fluid/inference/tensorrt/convert/multiclass_nms_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/multiclass_nms_op.cc @@ -151,7 +151,7 @@ class MultiClassNMSOpConverter : public OpConverter { int axis_index = engine_->with_dynamic_shape() ? 1 : 0; nms_concat_layer->setAxis(axis_index + 1); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( nms_concat_layer, "multiclass_nms", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/multihead_matmul_op.cc b/paddle/fluid/inference/tensorrt/convert/multihead_matmul_op.cc index 3c4ebc376be75..4e6cab4ff907e 100644 --- a/paddle/fluid/inference/tensorrt/convert/multihead_matmul_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/multihead_matmul_op.cc @@ -253,7 +253,7 @@ class MultiheadMatMulOpConverter : public OpConverter { max_seqlen_tensor); // max_seqlen, eval_placeholder_3 auto plugin_layer = engine_->network()->addPluginV2( plugin_inputs.data(), plugin_inputs.size(), *plugin); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( plugin_layer, "multihead_matmul", {output_name}, test_mode); } else { auto* reshape_before_matrix = @@ -757,7 +757,7 @@ class MultiheadMatMulOpConverter : public OpConverter { // return layer = reshape_after_mha_layer; - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( layer, "multihead_matmul", {output_name}, test_mode); } else { PADDLE_ENFORCE_EQ( @@ -867,7 +867,7 @@ class MultiheadMatMulOpConverter : public OpConverter { new plugin::QkvToContextPluginDynamic( hidden_in, head_number, head_size, scale, with_fp16); layer = engine_->AddDynamicPlugin(plugin_inputs.data(), 2, plugin); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( layer, "multihead_matmul", {output_name}, test_mode); } } diff --git a/paddle/fluid/inference/tensorrt/convert/multihead_matmul_roformer_op.cc b/paddle/fluid/inference/tensorrt/convert/multihead_matmul_roformer_op.cc index 5b657992e1549..517f5f1e7efc0 100644 --- a/paddle/fluid/inference/tensorrt/convert/multihead_matmul_roformer_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/multihead_matmul_roformer_op.cc @@ -193,7 +193,7 @@ class MultiheadMatMulRoformerOpConverter : public OpConverter { "You can use the config.SetTRTDynamicShapeInfo(...) interface to set " "the shape information to run the dynamic shape mode.")); } - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( layer, "multihead_matmul_roformer", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/nearest_interp_op.cc b/paddle/fluid/inference/tensorrt/convert/nearest_interp_op.cc index 4c0b1a027640b..d03940dd40aa5 100644 --- a/paddle/fluid/inference/tensorrt/convert/nearest_interp_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/nearest_interp_op.cc @@ -91,7 +91,7 @@ class NearestInterpolateOpConverter : public OpConverter { } layer->setScales(scales.data(), scales.size()); - RreplenishLayerAndOutput(layer, "nearest_interp", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "nearest_interp", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/nearest_interp_v2_op.cc b/paddle/fluid/inference/tensorrt/convert/nearest_interp_v2_op.cc index 6f33a71046977..feac662ff0441 100644 --- a/paddle/fluid/inference/tensorrt/convert/nearest_interp_v2_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/nearest_interp_v2_op.cc @@ -124,7 +124,7 @@ class NearestInterpolateV2OpConverter : public OpConverter { layer->setScales(scales.data(), scales.size()); } - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( layer, "nearest_interp_v2", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/one_hot_op.cc b/paddle/fluid/inference/tensorrt/convert/one_hot_op.cc index 3ab318de2df4d..2ec44e229e8b6 100644 --- a/paddle/fluid/inference/tensorrt/convert/one_hot_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/one_hot_op.cc @@ -79,7 +79,7 @@ class OneHotOpConverter : public OpConverter { engine_, OneHot, *indices_tensor, *values_tensor, *depth_tensor, -1); auto output_name = op_desc.Output("Out").front(); - RreplenishLayerAndOutput(layer, "one_hot", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "one_hot", {output_name}, test_mode); #else VLOG(3) << "one_hot is not supported when TensorRT < 8.5.1"; #endif diff --git a/paddle/fluid/inference/tensorrt/convert/op_converter.h b/paddle/fluid/inference/tensorrt/convert/op_converter.h index f2bf95306df8a..3b75a79d9b563 100644 --- a/paddle/fluid/inference/tensorrt/convert/op_converter.h +++ b/paddle/fluid/inference/tensorrt/convert/op_converter.h @@ -760,7 +760,7 @@ class OpConverter { return Add1DConstantLayer(input_data, weight_name, scalar); } - void RreplenishLayerAndOutput( + void ReplenishLayerAndOutput( nvinfer1::ILayer* layer, const std::string& layer_type, const std::vector& output_tensor_names, diff --git a/paddle/fluid/inference/tensorrt/convert/pad3d_op.cc b/paddle/fluid/inference/tensorrt/convert/pad3d_op.cc index a8a7a91e49a14..f1765f9967aad 100644 --- a/paddle/fluid/inference/tensorrt/convert/pad3d_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/pad3d_op.cc @@ -168,7 +168,7 @@ class Pad3dOpConverter : public OpConverter { } auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(slice_layer, "pad3d", {output_name}, test_mode); + ReplenishLayerAndOutput(slice_layer, "pad3d", {output_name}, test_mode); #else VLOG(3) << "pad3d is not supported when TensorRT < 8.2"; diff --git a/paddle/fluid/inference/tensorrt/convert/pad_op.cc b/paddle/fluid/inference/tensorrt/convert/pad_op.cc index 7e176587112c7..56a662a7254ee 100644 --- a/paddle/fluid/inference/tensorrt/convert/pad_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/pad_op.cc @@ -50,7 +50,7 @@ class PadOpConverter : public OpConverter { platform::errors::External( "add padding layer to tensorrt engine error")); auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "pad", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "pad", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/pool2d_op.cc b/paddle/fluid/inference/tensorrt/convert/pool2d_op.cc index 6c0ebc35f56d0..317a519b0cc44 100644 --- a/paddle/fluid/inference/tensorrt/convert/pool2d_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/pool2d_op.cc @@ -370,7 +370,7 @@ class Pool2dOpConverter : public OpConverter { layer = pool_layer; } auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "pool2d", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "pool2d", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/pool3d_op.cc b/paddle/fluid/inference/tensorrt/convert/pool3d_op.cc index af770af1e939b..c0f38cf79ff91 100644 --- a/paddle/fluid/inference/tensorrt/convert/pool3d_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/pool3d_op.cc @@ -224,7 +224,7 @@ class Pool3dOpConverter : public OpConverter { layer = pool_layer; } auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "pool3d", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "pool3d", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/preln_groupnorm_act_op.cc b/paddle/fluid/inference/tensorrt/convert/preln_groupnorm_act_op.cc index 7c65e32714cca..636e6b8785fd4 100644 --- a/paddle/fluid/inference/tensorrt/convert/preln_groupnorm_act_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/preln_groupnorm_act_op.cc @@ -74,7 +74,7 @@ class PrelnGroupnormActOpConverter : public OpConverter { std::vector output_names; output_names.emplace_back(op_desc.Output("Out_0").front()); output_names.emplace_back(op_desc.Output("Out_1").front()); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( groupnorm_layer, "preln_groupnorm_act", output_names, test_mode); } } diff --git a/paddle/fluid/inference/tensorrt/convert/preln_layernorm_shift_partition_op.cc b/paddle/fluid/inference/tensorrt/convert/preln_layernorm_shift_partition_op.cc index eff6a154046d7..6017edecaaa58 100644 --- a/paddle/fluid/inference/tensorrt/convert/preln_layernorm_shift_partition_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/preln_layernorm_shift_partition_op.cc @@ -76,10 +76,10 @@ class PrelnLayerNormShiftPartitionOpConverter : public OpConverter { std::vector output_names; output_names.emplace_back(op_desc.Output("Out_0").front()); output_names.emplace_back(op_desc.Output("Out_1").front()); - RreplenishLayerAndOutput(layernorm_layer, - "preln_layernorm_shift_partition", - output_names, - test_mode); + ReplenishLayerAndOutput(layernorm_layer, + "preln_layernorm_shift_partition", + output_names, + test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/preln_residual_bias.cc b/paddle/fluid/inference/tensorrt/convert/preln_residual_bias.cc index 9091cfd10e3e8..824f0ff902874 100644 --- a/paddle/fluid/inference/tensorrt/convert/preln_residual_bias.cc +++ b/paddle/fluid/inference/tensorrt/convert/preln_residual_bias.cc @@ -100,7 +100,7 @@ class PrelnResidualBiasOpConverter : public OpConverter { std::vector output_names; output_names.push_back(op_desc.Output("Y")[0]); output_names.push_back(op_desc.Output("BiasDropoutResidualOut")[0]); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( layer, "preln_residual_bias", output_names, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/preln_skip_layernorm.cc b/paddle/fluid/inference/tensorrt/convert/preln_skip_layernorm.cc index d21247e877cec..40fca0b317fb1 100644 --- a/paddle/fluid/inference/tensorrt/convert/preln_skip_layernorm.cc +++ b/paddle/fluid/inference/tensorrt/convert/preln_skip_layernorm.cc @@ -100,7 +100,7 @@ class PrelnSkipLayerNormOpConverter : public OpConverter { std::vector output_names; output_names.push_back(op_desc.Output("Out_0")[0]); output_names.push_back(op_desc.Output("Out_1")[0]); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( layer, "preln_skip_layernorm", {output_names}, test_mode); #else PADDLE_THROW(platform::errors::Fatal( diff --git a/paddle/fluid/inference/tensorrt/convert/prelu_op.cc b/paddle/fluid/inference/tensorrt/convert/prelu_op.cc index 2d2a804f394fd..98a0cc7a47866 100644 --- a/paddle/fluid/inference/tensorrt/convert/prelu_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/prelu_op.cc @@ -120,7 +120,7 @@ class PReluOpConverter : public OpConverter { engine_, ParametricReLU, *input, *real_alpha_tensor); auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "prelu", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "prelu", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/prompt_tuning_emb_eltwise_layernorm.cc b/paddle/fluid/inference/tensorrt/convert/prompt_tuning_emb_eltwise_layernorm.cc index 9fc4c96ab7b93..9250807662543 100644 --- a/paddle/fluid/inference/tensorrt/convert/prompt_tuning_emb_eltwise_layernorm.cc +++ b/paddle/fluid/inference/tensorrt/convert/prompt_tuning_emb_eltwise_layernorm.cc @@ -159,14 +159,14 @@ class PromptTuningEmbEltwiseLayerNormOpConverter : public OpConverter { engine_->DeleteITensor("pos_id", engine_->GetITensor("pos_id")); auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(plugin_layer, - "PromptTuningEmbLayerNormVarlenPluginDynamicV1", - {output_name, - std::string("qkv_plugin_mask"), - std::string("max_seqlen_tensor"), - std::string("mask_id"), - std::string("pos_id")}, - test_mode); + ReplenishLayerAndOutput(plugin_layer, + "PromptTuningEmbLayerNormVarlenPluginDynamicV1", + {output_name, + std::string("qkv_plugin_mask"), + std::string("max_seqlen_tensor"), + std::string("mask_id"), + std::string("pos_id")}, + test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/qk_multihead_matmul_op.cc b/paddle/fluid/inference/tensorrt/convert/qk_multihead_matmul_op.cc index 89b65e95bd8eb..4a24e7425068f 100644 --- a/paddle/fluid/inference/tensorrt/convert/qk_multihead_matmul_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/qk_multihead_matmul_op.cc @@ -289,7 +289,7 @@ class QkMultiheadMatMulOpConverter : public OpConverter { ("shuffle_last_multihead_matmul(Output: " + output_name + ")").c_str()); nvinfer1::ILayer* layer = nullptr; layer = reshape_after_mha_layer; - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( layer, "qk_multihead_matmul", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/quantize_linear_op.cc b/paddle/fluid/inference/tensorrt/convert/quantize_linear_op.cc index 4534fc0f53c68..b37a8f327e154 100644 --- a/paddle/fluid/inference/tensorrt/convert/quantize_linear_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/quantize_linear_op.cc @@ -50,7 +50,7 @@ class QuantizeLinearOpConverter : public OpConverter { layer->setAxis(axis); } auto output_name = op_desc.Output("Y")[0]; - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( layer, "quantize_linear", {output_name}, test_model); #else PADDLE_THROW( diff --git a/paddle/fluid/inference/tensorrt/convert/range_op.cc b/paddle/fluid/inference/tensorrt/convert/range_op.cc index 6e4db4fb2a4f8..b44d9d588744a 100644 --- a/paddle/fluid/inference/tensorrt/convert/range_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/range_op.cc @@ -59,7 +59,7 @@ class RangeOpConverter : public OpConverter { layer->setInput(1, *start1); layer->setInput(2, *step); - RreplenishLayerAndOutput(layer, "range", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "range", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/recover_padding_op.cc b/paddle/fluid/inference/tensorrt/convert/recover_padding_op.cc index 5e0a4ba0b563b..29507da6ebc50 100644 --- a/paddle/fluid/inference/tensorrt/convert/recover_padding_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/recover_padding_op.cc @@ -66,8 +66,7 @@ class RecoverPadding : public OpConverter { plugin::RecoverPaddingPlugin* plugin = new plugin::RecoverPaddingPlugin(); nvinfer1::ILayer* layer = engine_->AddDynamicPlugin(plugin_inputs.data(), input_num, plugin); - RreplenishLayerAndOutput( - layer, "recover_padding", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "recover_padding", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/reduce_op.cc b/paddle/fluid/inference/tensorrt/convert/reduce_op.cc index f3f96e9b02e49..fb1555087124c 100644 --- a/paddle/fluid/inference/tensorrt/convert/reduce_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/reduce_op.cc @@ -94,7 +94,7 @@ class ReduceOpConverter : public OpConverter { auto output_name = op_desc.Output("Out")[0]; // Ensure that the output type and input type are consistent. layer->getOutput(0)->setType(layer->getInput(0)->getType()); - RreplenishLayerAndOutput(layer, op_type, {output_name}, test_mode); + ReplenishLayerAndOutput(layer, op_type, {output_name}, test_mode); } protected: @@ -216,7 +216,7 @@ class ReduceAnyOpConverter : public ReduceOpConverter { // Ensure that the output type and input type are consistent. layer->getOutput(0)->setType(cast_layer->getInput(0)->getType()); - RreplenishLayerAndOutput(layer, op_type, {output_name}, test_mode); + ReplenishLayerAndOutput(layer, op_type, {output_name}, test_mode); }; }; diff --git a/paddle/fluid/inference/tensorrt/convert/reshape_op.cc b/paddle/fluid/inference/tensorrt/convert/reshape_op.cc index 92c592ee03212..c31cf1b012a49 100644 --- a/paddle/fluid/inference/tensorrt/convert/reshape_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/reshape_op.cc @@ -73,7 +73,7 @@ class ReshapeOpConverter : public OpConverter { "reshape2 op into " "Paddle-TRT.")); auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "reshape", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "reshape", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/reverse_roll_op.cc b/paddle/fluid/inference/tensorrt/convert/reverse_roll_op.cc index d974512557da6..dd7431a69c343 100644 --- a/paddle/fluid/inference/tensorrt/convert/reverse_roll_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/reverse_roll_op.cc @@ -68,7 +68,7 @@ class ReverseRollOpConverter : public OpConverter { "ReverseROll TRT Plugin should run in dynamic shape.")); } auto output_name = op_desc.Output("Out").front(); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( reverse_roll_layer, "reverse_roll", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/rnn_op.cc b/paddle/fluid/inference/tensorrt/convert/rnn_op.cc index c87a38803845c..3fdc5c5eb9050 100644 --- a/paddle/fluid/inference/tensorrt/convert/rnn_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/rnn_op.cc @@ -303,7 +303,7 @@ class RnnNativeOpConverter : public OpConverter { } auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(finally_layer, "rnn", {output_name}, test_mode); + ReplenishLayerAndOutput(finally_layer, "rnn", {output_name}, test_mode); // free if (is_bidirec) { for (auto& weight_bias : weight_bias_vec) delete[] weight_bias; diff --git a/paddle/fluid/inference/tensorrt/convert/roi_align_op.cc b/paddle/fluid/inference/tensorrt/convert/roi_align_op.cc index 7a0192b0f7b9c..8fbdea5edd4c9 100644 --- a/paddle/fluid/inference/tensorrt/convert/roi_align_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/roi_align_op.cc @@ -65,7 +65,7 @@ class RoiAlignOpConverter : public OpConverter { layer = roi_align_layer; std::vector output_names{output_name}; - RreplenishLayerAndOutput(layer, "roi_align", output_names, test_mode); + ReplenishLayerAndOutput(layer, "roi_align", output_names, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/roll_op.cc b/paddle/fluid/inference/tensorrt/convert/roll_op.cc index d0ff9e30d725f..ca42b3c34c3f8 100644 --- a/paddle/fluid/inference/tensorrt/convert/roll_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/roll_op.cc @@ -87,7 +87,7 @@ class RollOpConverter : public OpConverter { } } auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "roll", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "roll", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/scale_op.cc b/paddle/fluid/inference/tensorrt/convert/scale_op.cc index 0df7894929f8c..a4f54e216b9ce 100644 --- a/paddle/fluid/inference/tensorrt/convert/scale_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/scale_op.cc @@ -218,7 +218,7 @@ class ScaleOpConverter : public OpConverter { ("Scale: Shuffle_reshape (Output: " + out_name + ")").c_str()); } } - RreplenishLayerAndOutput(layer, "scale", {out_name}, test_mode); + ReplenishLayerAndOutput(layer, "scale", {out_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/set_value_op.cc b/paddle/fluid/inference/tensorrt/convert/set_value_op.cc index 6d1da3036de3b..1c734d791cdde 100644 --- a/paddle/fluid/inference/tensorrt/convert/set_value_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/set_value_op.cc @@ -249,7 +249,7 @@ class SetValueConverter : public OpConverter { layer->setAxis(axes); - RreplenishLayerAndOutput(layer, "set_value", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "set_value", {output_name}, test_mode); } else { PADDLE_THROW(platform::errors::Fatal( "static shape mode not supported in set value yet")); diff --git a/paddle/fluid/inference/tensorrt/convert/shape_op.cc b/paddle/fluid/inference/tensorrt/convert/shape_op.cc index cd7c5797a6f71..6b4e8e2b0829c 100644 --- a/paddle/fluid/inference/tensorrt/convert/shape_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/shape_op.cc @@ -30,7 +30,7 @@ class ShapeOpConverter : public OpConverter { auto* input = engine_->GetITensor(op_desc.Input("Input")[0]); nvinfer1::ILayer* layer = TRT_ENGINE_ADD_LAYER(engine_, Shape, *input); auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "shape", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "shape", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/share_data_op.cc b/paddle/fluid/inference/tensorrt/convert/share_data_op.cc index 644eeda8d102f..38fa1ff6e0c83 100644 --- a/paddle/fluid/inference/tensorrt/convert/share_data_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/share_data_op.cc @@ -28,7 +28,7 @@ class ShareDataOpConverter : public OpConverter { auto* input = engine_->GetITensor(op_desc.Input("X")[0]); auto* layer = TRT_ENGINE_ADD_LAYER(engine_, Identity, *input); auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "share_data", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "share_data", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/shuffle_channel_op.cc b/paddle/fluid/inference/tensorrt/convert/shuffle_channel_op.cc index 556744ed158c0..9aee6b112d223 100644 --- a/paddle/fluid/inference/tensorrt/convert/shuffle_channel_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/shuffle_channel_op.cc @@ -60,7 +60,7 @@ class ShuffleChannelOpConverter : public OpConverter { auto* output_layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *output); output_layer->setInput(1, *input_shape_tensor); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( output_layer, "shuffle_channel", {output_name}, test_mode); } #endif @@ -79,7 +79,7 @@ class ShuffleChannelOpConverter : public OpConverter { nvinfer1::Dims3 reshape_dim2(c, h, w); reshape_layer->setReshapeDimensions(reshape_dim2); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( reshape_layer, "shuffle_channel", {output_name}, test_mode); } } diff --git a/paddle/fluid/inference/tensorrt/convert/silu_op.cc b/paddle/fluid/inference/tensorrt/convert/silu_op.cc index d52b184390e96..b4b54e4bd3ba1 100644 --- a/paddle/fluid/inference/tensorrt/convert/silu_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/silu_op.cc @@ -56,7 +56,7 @@ class SiluOpConverter : public OpConverter { nvinfer1::ElementWiseOperation::kPROD); auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "silu", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "silu", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/size_op.cc b/paddle/fluid/inference/tensorrt/convert/size_op.cc index 1d4908d5bad93..f214a958b8d6b 100644 --- a/paddle/fluid/inference/tensorrt/convert/size_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/size_op.cc @@ -39,7 +39,7 @@ class SizeOpConverter : public OpConverter { reduce_dim, false); auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "size", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "size", {output_name}, test_mode); } }; } // namespace tensorrt diff --git a/paddle/fluid/inference/tensorrt/convert/skip_groupnorm_act_op.cc b/paddle/fluid/inference/tensorrt/convert/skip_groupnorm_act_op.cc index 3a822d71a8b1f..95bc327cac09d 100644 --- a/paddle/fluid/inference/tensorrt/convert/skip_groupnorm_act_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/skip_groupnorm_act_op.cc @@ -70,7 +70,7 @@ class SkipGroupnormActOpConverter : public OpConverter { nvinfer1::ILayer* groupnorm_layer = engine_->AddDynamicPlugin(inputs.data(), 2, plugin); auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( groupnorm_layer, "skip_groupnorm_act", {output_name}, test_mode); } } diff --git a/paddle/fluid/inference/tensorrt/convert/skip_layernorm.cc b/paddle/fluid/inference/tensorrt/convert/skip_layernorm.cc index 0cdb6088f3096..15ef380253949 100644 --- a/paddle/fluid/inference/tensorrt/convert/skip_layernorm.cc +++ b/paddle/fluid/inference/tensorrt/convert/skip_layernorm.cc @@ -257,7 +257,7 @@ class SkipLayerNormOpConverter : public OpConverter { layer = plugin_layer; } } - RreplenishLayerAndOutput(layer, "skip_layernorm", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "skip_layernorm", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/skip_merge_layernorm_op.cc b/paddle/fluid/inference/tensorrt/convert/skip_merge_layernorm_op.cc index f7768627d4334..4bb54de495b19 100644 --- a/paddle/fluid/inference/tensorrt/convert/skip_merge_layernorm_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/skip_merge_layernorm_op.cc @@ -80,10 +80,10 @@ class SkipMergeLayernormOpConverter : public OpConverter { "mode.")); } auto output_name = op_desc.Output("Out").front(); - RreplenishLayerAndOutput(skip_merge_layernorm_layer, - "skip_merge_layernorm", - {output_name}, - test_mode); + ReplenishLayerAndOutput(skip_merge_layernorm_layer, + "skip_merge_layernorm", + {output_name}, + test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/slice_op.cc b/paddle/fluid/inference/tensorrt/convert/slice_op.cc index 64ed3fd8da0ed..4a2d38d5e0736 100644 --- a/paddle/fluid/inference/tensorrt/convert/slice_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/slice_op.cc @@ -217,7 +217,7 @@ class SliceOpConverter : public OpConverter { layer = static_cast(reshape_layer); } } - RreplenishLayerAndOutput(layer, "slice", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "slice", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/softmax_op.cc b/paddle/fluid/inference/tensorrt/convert/softmax_op.cc index 9aefd7fb28b39..921402a9be5d2 100644 --- a/paddle/fluid/inference/tensorrt/convert/softmax_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/softmax_op.cc @@ -92,10 +92,10 @@ class SoftMaxOpConverter : public OpConverter { TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *layer->getOutput(0)); reshaped_layer->setReshapeDimensions( engine_->GetITensor(op_desc.Input("X")[0])->getDimensions()); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( reshaped_layer, "reshape_softmax_reshape", {output_name}, test_mode); } else { - RreplenishLayerAndOutput(layer, "softmax", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "softmax", {output_name}, test_mode); } } }; diff --git a/paddle/fluid/inference/tensorrt/convert/sparse_fc_op.cc b/paddle/fluid/inference/tensorrt/convert/sparse_fc_op.cc index 6bbcab93c40e9..bae9cccde6fa7 100644 --- a/paddle/fluid/inference/tensorrt/convert/sparse_fc_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/sparse_fc_op.cc @@ -203,15 +203,15 @@ class SparseFcOpConverter : public OpConverter { Activation, *(fc_layer_int8->getOutput(0)), nvinfer1::ActivationType::kRELU); - RreplenishLayerAndOutput(relu_layer_int8, - "relu_after_ernie_fc_int8", - {output_name}, - test_mode); + ReplenishLayerAndOutput(relu_layer_int8, + "relu_after_ernie_fc_int8", + {output_name}, + test_mode); } else { - RreplenishLayerAndOutput(fc_layer_int8, - "ernie_fc_op_int8: Convolution", - {output_name}, - test_mode); + ReplenishLayerAndOutput(fc_layer_int8, + "ernie_fc_op_int8: Convolution", + {output_name}, + test_mode); } } else { // add fc layer @@ -225,12 +225,12 @@ class SparseFcOpConverter : public OpConverter { Activation, *(fc_layer_float->getOutput(0)), nvinfer1::ActivationType::kRELU); - RreplenishLayerAndOutput(relu_layer_float, - "relu_after_ernie_fc_float", - {output_name}, - test_mode); + ReplenishLayerAndOutput(relu_layer_float, + "relu_after_ernie_fc_float", + {output_name}, + test_mode); } else { - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( fc_layer_float, "ernie_fc_op_float", {output_name}, test_mode); } } @@ -264,10 +264,10 @@ class SparseFcOpConverter : public OpConverter { auto* fc_after_reshape_int8 = reshape_after_fc( fc_layer_int8->getOutput(0), x_dim, x_num_col_dims); - RreplenishLayerAndOutput(fc_after_reshape_int8, - "sparse_fc_op_int8_reshape_after_fc: Shuffle", - {output_name}, - test_mode); + ReplenishLayerAndOutput(fc_after_reshape_int8, + "sparse_fc_op_int8_reshape_after_fc: Shuffle", + {output_name}, + test_mode); } else { plugin::SpmmPluginDynamic* plugin = new_spmm_plugin( weight, @@ -285,10 +285,10 @@ class SparseFcOpConverter : public OpConverter { auto* fc_after_reshape_float = reshape_after_fc( fc_layer_float->getOutput(0), x_dim, x_num_col_dims); - RreplenishLayerAndOutput(fc_after_reshape_float, - "shuffle_after_sparse_fc", - {output_name}, - test_mode); + ReplenishLayerAndOutput(fc_after_reshape_float, + "shuffle_after_sparse_fc", + {output_name}, + test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/sparse_multihead_matmul_op.cc b/paddle/fluid/inference/tensorrt/convert/sparse_multihead_matmul_op.cc index 075592807d7c5..74198b3066a88 100644 --- a/paddle/fluid/inference/tensorrt/convert/sparse_multihead_matmul_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/sparse_multihead_matmul_op.cc @@ -431,7 +431,7 @@ class SparseMultiheadMatMulOpConverter : public OpConverter { "You can use the config.SetTRTDynamicShapeInfo(...) interface to set " "the shape information to run the dynamic shape mode.")); } - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( layer, "multihead_matmul", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/split_op.cc b/paddle/fluid/inference/tensorrt/convert/split_op.cc index b12e58a536ef9..07e14759ddebf 100644 --- a/paddle/fluid/inference/tensorrt/convert/split_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/split_op.cc @@ -127,7 +127,7 @@ class SplitOpConverter : public OpConverter { layer->setInput(3, *stride_tensor); auto output_name = op_desc.Output("Out")[i]; - RreplenishLayerAndOutput(layer, "split", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "split", {output_name}, test_mode); } } else { auto chw_input_dims = input->getDimensions(); @@ -151,7 +151,7 @@ class SplitOpConverter : public OpConverter { trt_size_dims, trt_step_dims); auto output_name = op_desc.Output("Out")[i]; - RreplenishLayerAndOutput(layer, "split", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "split", {output_name}, test_mode); } } #else @@ -172,7 +172,7 @@ class SplitOpConverter : public OpConverter { for (int i = 0; i < output_num; i++) { output_names.push_back(op_desc.Output("Out")[i]); } - RreplenishLayerAndOutput(layer, "split", output_names, test_mode); + ReplenishLayerAndOutput(layer, "split", output_names, test_mode); #endif } }; diff --git a/paddle/fluid/inference/tensorrt/convert/square_op.cc b/paddle/fluid/inference/tensorrt/convert/square_op.cc index a59ec9d242de4..e06102d398cb8 100644 --- a/paddle/fluid/inference/tensorrt/convert/square_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/square_op.cc @@ -36,7 +36,7 @@ class SquareOpConverter : public OpConverter { nvinfer1::ElementWiseOperation::kPROD); auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "square", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "square", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/squeeze2_op.cc b/paddle/fluid/inference/tensorrt/convert/squeeze2_op.cc index 4fd8863a59145..ce11f4001767d 100644 --- a/paddle/fluid/inference/tensorrt/convert/squeeze2_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/squeeze2_op.cc @@ -85,7 +85,7 @@ class Squeeze2OpConverter : public OpConverter { } else { layer->setReshapeDimensions(trt_out_dims); } - RreplenishLayerAndOutput(layer, "squeeze2", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "squeeze2", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/stack_op.cc b/paddle/fluid/inference/tensorrt/convert/stack_op.cc index 1b0722a467a58..30ffcd88472d3 100644 --- a/paddle/fluid/inference/tensorrt/convert/stack_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/stack_op.cc @@ -76,7 +76,7 @@ class StackOpConverter : public OpConverter { engine_, Concatenation, inputs.data(), inputs.size()); layer->setAxis(axis); - RreplenishLayerAndOutput(layer, "stack", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "stack", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/strided_slice_op.cc b/paddle/fluid/inference/tensorrt/convert/strided_slice_op.cc index faef8f7977e37..6b721d37d205f 100644 --- a/paddle/fluid/inference/tensorrt/convert/strided_slice_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/strided_slice_op.cc @@ -180,7 +180,7 @@ class StridedSliceOpConverter : public OpConverter { layer = static_cast(reshape_layer); } } - RreplenishLayerAndOutput(layer, "strided_slice", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "strided_slice", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/sum_op.cc b/paddle/fluid/inference/tensorrt/convert/sum_op.cc index f9b8f83d13304..900a37126f1ce 100644 --- a/paddle/fluid/inference/tensorrt/convert/sum_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/sum_op.cc @@ -43,7 +43,7 @@ class SumOpConverter : public OpConverter { } } auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "sum", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "sum", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/swish_op.cc b/paddle/fluid/inference/tensorrt/convert/swish_op.cc index 10c30ec5a6f99..52f5cd0970edc 100644 --- a/paddle/fluid/inference/tensorrt/convert/swish_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/swish_op.cc @@ -74,7 +74,7 @@ class SwishOpConverter : public OpConverter { } auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "swish", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "swish", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/take_along_axis_op.cc b/paddle/fluid/inference/tensorrt/convert/take_along_axis_op.cc index af43d859bb78c..18cf1cfd362a0 100644 --- a/paddle/fluid/inference/tensorrt/convert/take_along_axis_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/take_along_axis_op.cc @@ -49,8 +49,7 @@ class TakeAlongAxisOpConverter : public OpConverter { nvinfer1::GatherMode::kELEMENT); layer->setGatherAxis(axis); - RreplenishLayerAndOutput( - layer, "take_along_axis", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "take_along_axis", {output_name}, test_mode); #endif } }; diff --git a/paddle/fluid/inference/tensorrt/convert/tanhshrink_op.cc b/paddle/fluid/inference/tensorrt/convert/tanhshrink_op.cc index 2a3a479fa1b9f..ef3afb25e97a3 100644 --- a/paddle/fluid/inference/tensorrt/convert/tanhshrink_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/tanhshrink_op.cc @@ -55,7 +55,7 @@ class TanhshrinkOpConverter : public OpConverter { *(tanh->getOutput(0)), nvinfer1::ElementWiseOperation::kSUB); auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "tanh_shrink", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "tanh_shrink", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/temporal_shift_op.cc b/paddle/fluid/inference/tensorrt/convert/temporal_shift_op.cc index 03983ff393033..edf659527c8ca 100644 --- a/paddle/fluid/inference/tensorrt/convert/temporal_shift_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/temporal_shift_op.cc @@ -205,10 +205,10 @@ class TemporalShiftOpConverter : public OpConverter { TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *reshape_layer3->getOutput(0)); nvinfer1::Permutation permute_order{0, 2, 3, 1}; transpose_layer2->setFirstTranspose(permute_order); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( transpose_layer2, "temporal_shift", {output_name}, test_mode); } else { - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( reshape_layer3, "temporal_shift", {output_name}, test_mode); } #else diff --git a/paddle/fluid/inference/tensorrt/convert/tile_op.cc b/paddle/fluid/inference/tensorrt/convert/tile_op.cc index 6b77fb1aae344..ffdc71e3af675 100644 --- a/paddle/fluid/inference/tensorrt/convert/tile_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/tile_op.cc @@ -91,7 +91,7 @@ class TileOpConverter : public OpConverter { layer->setInput(2, *output_shape_tensor); layer->setInput(3, *stride_tensor); layer->setMode(nvinfer1::SliceMode::kWRAP); - RreplenishLayerAndOutput(layer, "tile", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "tile", {output_name}, test_mode); } else { std::vector repeat_times = @@ -122,7 +122,7 @@ class TileOpConverter : public OpConverter { auto layer = TRT_ENGINE_ADD_LAYER( engine_, Slice, *input, input_shape, output_dim, output_stride); layer->setMode(nvinfer1::SliceMode::kWRAP); - RreplenishLayerAndOutput(layer, "tile", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "tile", {output_name}, test_mode); } #endif diff --git a/paddle/fluid/inference/tensorrt/convert/trans_layernorm_op.cc b/paddle/fluid/inference/tensorrt/convert/trans_layernorm_op.cc index 3db72738dc4c3..dc257beb14683 100644 --- a/paddle/fluid/inference/tensorrt/convert/trans_layernorm_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/trans_layernorm_op.cc @@ -77,10 +77,10 @@ class TransLayerNormOpConverter : public OpConverter { auto output_layernorm_name = op_desc.Output("Out_layernorm").front(); auto output_reshape_name = op_desc.Output("Out_reshape").front(); - RreplenishLayerAndOutput(layernorm_layer, - "trans_layernorm", - {output_layernorm_name, output_reshape_name}, - test_mode); + ReplenishLayerAndOutput(layernorm_layer, + "trans_layernorm", + {output_layernorm_name, output_reshape_name}, + test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/transformer_input_convert_op.cc b/paddle/fluid/inference/tensorrt/convert/transformer_input_convert_op.cc index 0aff0952aba62..1dca9bb818c38 100644 --- a/paddle/fluid/inference/tensorrt/convert/transformer_input_convert_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/transformer_input_convert_op.cc @@ -51,10 +51,10 @@ class TransformerInputConvert : public OpConverter { nvinfer1::ILayer* layer = engine_->AddDynamicPlugin(&input, input_num, plugin); - RreplenishLayerAndOutput(layer, - "transformer_input_convert", - {pos_id_name, max_seqlen_name}, - test_mode); + ReplenishLayerAndOutput(layer, + "transformer_input_convert", + {pos_id_name, max_seqlen_name}, + test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/transpose_op.cc b/paddle/fluid/inference/tensorrt/convert/transpose_op.cc index 0f0203fadc0c9..b16e8c2968714 100644 --- a/paddle/fluid/inference/tensorrt/convert/transpose_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/transpose_op.cc @@ -44,7 +44,7 @@ class TransposeOpConverter : public OpConverter { layer->setFirstTranspose(perm); auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "transpose", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "transpose", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/unary_op.cc b/paddle/fluid/inference/tensorrt/convert/unary_op.cc index 14881d1206357..ea78ec9292159 100644 --- a/paddle/fluid/inference/tensorrt/convert/unary_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/unary_op.cc @@ -70,7 +70,7 @@ class UnaryOpConverter : public OpConverter { } auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, op_type_, {output_name}, test_mode); + ReplenishLayerAndOutput(layer, op_type_, {output_name}, test_mode); } protected: diff --git a/paddle/fluid/inference/tensorrt/convert/unbind_op.cc b/paddle/fluid/inference/tensorrt/convert/unbind_op.cc index 539f682c23d1a..ca26d09afb5b3 100644 --- a/paddle/fluid/inference/tensorrt/convert/unbind_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/unbind_op.cc @@ -74,7 +74,7 @@ class UnbindOpConverter : public OpConverter { auto inputReshaped = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *inputSliced_out); inputReshaped->setInput(1, *newDims_tensor); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( inputReshaped, "unbind", {output_name}, test_mode); } } diff --git a/paddle/fluid/inference/tensorrt/convert/unsqueeze2_op.cc b/paddle/fluid/inference/tensorrt/convert/unsqueeze2_op.cc index 4dfad72e7d342..7cdc1b07fd04d 100644 --- a/paddle/fluid/inference/tensorrt/convert/unsqueeze2_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/unsqueeze2_op.cc @@ -90,7 +90,7 @@ class Unsqueeze2OpConverter : public OpConverter { } else { layer->setReshapeDimensions(trt_out_dims); } - RreplenishLayerAndOutput(layer, "unsqueeze2", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "unsqueeze2", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/where_op.cc b/paddle/fluid/inference/tensorrt/convert/where_op.cc index 19bd0b656e385..80887b7a3db00 100644 --- a/paddle/fluid/inference/tensorrt/convert/where_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/where_op.cc @@ -41,7 +41,7 @@ class WhereOpConverter : public OpConverter { auto layer = TRT_ENGINE_ADD_LAYER( engine_, Select, *condition_tensor, *input_x_tensor, *input_y_tensor); - RreplenishLayerAndOutput(layer, "where", {output_name}, test_mode); + ReplenishLayerAndOutput(layer, "where", {output_name}, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/yolo_box_head_op.cc b/paddle/fluid/inference/tensorrt/convert/yolo_box_head_op.cc index 6b58ef17a6a5a..eafb38221ecf3 100644 --- a/paddle/fluid/inference/tensorrt/convert/yolo_box_head_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/yolo_box_head_op.cc @@ -45,7 +45,7 @@ class YoloBoxHeadOpConverter : public OpConverter { yolo_box_inputs.data(), yolo_box_inputs.size(), *yolo_box_plugin); std::vector output_names; output_names.push_back(op_desc.Output("Out").front()); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( yolo_box_head_layer, "yolo_box_head", output_names, test_mode); } }; diff --git a/paddle/fluid/inference/tensorrt/convert/yolo_box_op.cc b/paddle/fluid/inference/tensorrt/convert/yolo_box_op.cc index 62d53789a5d5e..3ed413a913fd0 100644 --- a/paddle/fluid/inference/tensorrt/convert/yolo_box_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/yolo_box_op.cc @@ -75,7 +75,7 @@ class YoloBoxOpConverter : public OpConverter { output_names.push_back(op_desc.Output("Boxes").front()); output_names.push_back(op_desc.Output("Scores").front()); - RreplenishLayerAndOutput( + ReplenishLayerAndOutput( yolo_box_layer, "yolo_box", output_names, test_mode); } }; From 4078210c276e91f1c028c56c227ab655eff34e57 Mon Sep 17 00:00:00 2001 From: co63oc Date: Tue, 27 Feb 2024 10:24:49 +0800 Subject: [PATCH 107/282] Fix some typos(GetFuncion, mutli, etc) (#62038) --- paddle/cinn/hlir/framework/pir/utils.cc | 2 +- paddle/cinn/runtime/cuda/cuda_module.cc | 2 +- paddle/cinn/runtime/flags.cc | 6 ++--- paddle/cinn/runtime/intrinsic.h | 2 +- paddle/common/flags.cc | 32 ++++++++++++------------- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/paddle/cinn/hlir/framework/pir/utils.cc b/paddle/cinn/hlir/framework/pir/utils.cc index 1f5070d29fed7..83fe4ed5ef16c 100644 --- a/paddle/cinn/hlir/framework/pir/utils.cc +++ b/paddle/cinn/hlir/framework/pir/utils.cc @@ -406,7 +406,7 @@ OpPatternKind CompatibleInfo::OpKind(const ::pir::Operation& op) { auto kind = op_pattern_dict[cinn_op]; if (kind == hlir::framework::kBroadcast) { // As binary op was defined as broadcast, actually it should be - // element-wise. See fusion_hepler_base.h for detail. + // element-wise. See fusion_helper_base.h for detail. if (op_name != "broadcast_to") { kind = hlir::framework::kElementWise; } diff --git a/paddle/cinn/runtime/cuda/cuda_module.cc b/paddle/cinn/runtime/cuda/cuda_module.cc index 2df567c547cbc..430516d9168d3 100644 --- a/paddle/cinn/runtime/cuda/cuda_module.cc +++ b/paddle/cinn/runtime/cuda/cuda_module.cc @@ -79,7 +79,7 @@ void CUDAModule::LaunchKernel(int device_id, CUfunction CUDAModule::GetFunction(int device_id, const std::string& func_name) { - VLOG(5) << "GetFuncion : " << func_name << " with device_id : " << device_id; + VLOG(5) << "GetFunction : " << func_name << " with device_id : " << device_id; cinn::utils::RecordEvent record_run("cuLaunchKernel", cinn::utils::EventType::kOrdinary); if (!module_per_card_[device_id]) { diff --git a/paddle/cinn/runtime/flags.cc b/paddle/cinn/runtime/flags.cc index 1c21fe97eee25..89512913e8fa9 100644 --- a/paddle/cinn/runtime/flags.cc +++ b/paddle/cinn/runtime/flags.cc @@ -97,7 +97,7 @@ PD_DEFINE_bool(cinn_enable_map_expr_dynamic_shape, PD_DEFINE_bool(cinn_enable_map_expr_index_detail, BoolFromEnv("FLAGS_cinn_enable_map_expr_index_detail", false), - "It controls whether to display datail tensor index"); + "It controls whether to display detail tensor index"); PD_DEFINE_bool( cinn_use_custom_call, @@ -116,7 +116,7 @@ PD_DEFINE_string(cinn_check_fusion_accuracy_pass, PD_DEFINE_bool(cinn_use_cuda_vectorize, BoolFromEnv("FLAGS_cinn_use_cuda_vectorize", false), - "Whether use cuda vectroize on schedule config"); + "Whether use cuda vectorize on schedule config"); PD_DEFINE_bool(use_reduce_split_pass, BoolFromEnv("FLAGS_use_reduce_split_pass", false), @@ -141,7 +141,7 @@ PD_DEFINE_bool( BoolFromEnv("FLAGS_cinn_nvrtc_cubin_with_fmad", true), "Whether nvrtc enables fmad when compile to cubin. This flag only works " "when FLAGS_nvrtc_compile_to_cubin=true. Fmad is the cuda speed up " - "technique which contract fp mulitplication and addition/subtraction into " + "technique which contract fp multiplication and addition/subtraction into " "multiply-add operation. It may result in different fp precision."); // FLAGS for performance analysis and accuracy debug diff --git a/paddle/cinn/runtime/intrinsic.h b/paddle/cinn/runtime/intrinsic.h index c2db240de2d12..e37673099ce2d 100644 --- a/paddle/cinn/runtime/intrinsic.h +++ b/paddle/cinn/runtime/intrinsic.h @@ -127,7 +127,7 @@ static const char* parallel_launch = "cinn_backend_parallel_launch"; } // namespace intrinsic /** - * Call an intrnsic function. + * Call an intrinsic function. * @param type Return type of the function. * @param fn_name Name of the function. * @param args The arguments for the function. diff --git a/paddle/common/flags.cc b/paddle/common/flags.cc index 2cd5dc5c15421..e09c7c0e8316e 100644 --- a/paddle/common/flags.cc +++ b/paddle/common/flags.cc @@ -582,7 +582,7 @@ PHI_DEFINE_EXPORTED_uint64( "specified by FLAGS_reallocate_gpu_memory_in_mb until the gpu has " "no memory left for the additional trunk. Note: if you set this " "flag, the memory size set by " - "FLAGS_fraction_of_gpu_memory_to_use will be overrided by this " + "FLAGS_fraction_of_gpu_memory_to_use will be overridden by this " "flag. If you don't set this flag, PaddlePaddle will use " "FLAGS_fraction_of_gpu_memory_to_use to allocate gpu memory"); @@ -670,7 +670,7 @@ PHI_DEFINE_EXPORTED_bool(use_mkldnn, false, "Use MKLDNN to run"); * Value Range: int, default=2 * Example: * Note: Used to debug. Determine the call stack to print when error or - * exeception happens. + * exception happens. * If FLAGS_call_stack_level == 0, only the error message summary will be shown. * If FLAGS_call_stack_level == 1, the python stack and error message summary * will be shown. @@ -686,7 +686,7 @@ static const int32_t kDefaultCallStackLevel = 1; PHI_DEFINE_EXPORTED_int32( call_stack_level, kDefaultCallStackLevel, - "Determine the call stack to print when error or exeception happens." + "Determine the call stack to print when error or exception happens." // TODO(zhiqiu): implement logic of FLAGS_call_stack_level==0 // "If FLAGS_call_stack_level == 0, only the error message summary will be " // "shown. " @@ -852,13 +852,13 @@ PHI_DEFINE_EXPORTED_bool( * Since Version: 2.2.0 * Value Range: bool, default=false * Example: - * Note: Control whether load graph node and edge with multi threads parallely + * Note: Control whether load graph node and edge with multi threads parallelly * If it is not set, load graph data with one thread */ PHI_DEFINE_EXPORTED_bool(graph_load_in_parallel, false, "It controls whether load graph node and edge with " - "mutli threads parallely."); + "multi threads parallelly."); /** * Distributed related FLAG @@ -878,7 +878,7 @@ PHI_DEFINE_EXPORTED_bool(enable_neighbor_list_use_uva, * Since Version: 2.5.0 * Value Range: double, default=1.0 * Example: - * Note: Control whether load graph node and edge with multi threads parallely + * Note: Control whether load graph node and edge with multi threads parallelly * If it is not set, load graph data with one thread */ PHI_DEFINE_EXPORTED_double(graph_neighbor_size_percent, @@ -891,13 +891,13 @@ PHI_DEFINE_EXPORTED_double(graph_neighbor_size_percent, * Since Version: 2.2.0 * Value Range: bool, default=false * Example: - * Note: Control whether load graph node and edge with multi threads parallely + * Note: Control whether load graph node and edge with multi threads parallelly * If it is not set, load graph data with one thread */ PHI_DEFINE_EXPORTED_bool(graph_metapath_split_opt, false, "It controls whether load graph node and edge with " - "mutli threads parallely."); + "multi threads parallelly."); /** * Distributed related FLAG @@ -1393,7 +1393,7 @@ PHI_DEFINE_EXPORTED_bool(enable_pir_with_pt_in_dy2st, * Since Version: 2.6.0 * Value Range: bool, default=false * Example: - * Note: If Ture, New IR API will be used in Python + * Note: If True, New IR API will be used in Python */ PHI_DEFINE_EXPORTED_bool(enable_pir_api, false, "Enable new IR API in Python"); @@ -1403,7 +1403,7 @@ PHI_DEFINE_EXPORTED_bool(enable_pir_api, false, "Enable new IR API in Python"); * Since Version: 2.6.0 * Value Range: bool, default=false * Example: - * Note: If Ture, executor will use new IR and run in beta version by for trace + * Note: If True, executor will use new IR and run in beta version by for trace * version. */ PHI_DEFINE_EXPORTED_bool(enable_pir_in_executor_trace_run, @@ -1416,7 +1416,7 @@ PHI_DEFINE_EXPORTED_bool(enable_pir_in_executor_trace_run, * Since Version: 2.6.0 * Value Range: bool, default=true * Example: - * Note: If Ture, will apply inplace pass to new IR. + * Note: If True, will apply inplace pass to new IR. */ PHI_DEFINE_EXPORTED_bool(pir_apply_inplace_pass, true, @@ -1428,7 +1428,7 @@ PHI_DEFINE_EXPORTED_string( "", "It controls the ir inplace kernel subset do not use."); /** - * Specify the directory of saving PIR sugraph from @to_static + * Specify the directory of saving PIR subgraph from @to_static * Name: pir_subgraph_saving_dir * Since Version: 2.6.0 * Value Range: str, default="" @@ -1438,7 +1438,7 @@ PHI_DEFINE_EXPORTED_string( PHI_DEFINE_EXPORTED_string( pir_subgraph_saving_dir, "", - "Specify the directory of saving PIR sugraph from @to_static."); + "Specify the directory of saving PIR subgraph from @to_static."); PHI_DEFINE_EXPORTED_bool(enable_record_memory, false, "Enable memory recorder"); @@ -1464,7 +1464,7 @@ PHI_DEFINE_EXPORTED_int32( PHI_DEFINE_EXPORTED_bool(print_ir, false, "Whether print ir debug str."); PHI_DEFINE_EXPORTED_bool(prim_skip_dynamic, false, - "Whether to skip decomping op with dynamic shape."); + "Whether to skip decomposing op with dynamic shape."); PHI_DEFINE_EXPORTED_bool(prim_check_ops, false, "Whether to check the decomposed program, to ensure " @@ -1514,7 +1514,7 @@ PHI_DEFINE_EXPORTED_bool( PHI_DEFINE_EXPORTED_int64(alloc_fill_value, -1, "Whether to fill fixed value after allocation. " - "This is usefull for debugging."); + "This is useful for debugging."); /** * Apply shape optimization pass to new IR FLAG @@ -1577,7 +1577,7 @@ PHI_DEFINE_EXPORTED_string(lapack_dir, * Since Version: 3.0.0 * Value Range: bool, default=false * Example: - * Note: If Ture, will apply check_infer_symbolic pass. + * Note: If True, will apply check_infer_symbolic pass. */ PHI_DEFINE_EXPORTED_bool( check_infer_symbolic, From e2e280ec2388c825824ebe5019a228a79624ed2f Mon Sep 17 00:00:00 2001 From: co63oc Date: Tue, 27 Feb 2024 10:25:37 +0800 Subject: [PATCH 108/282] Fix some typos(TestUninitailzed, etc) (#62045) --- .../framework/ir/conv_elementwise_add2_act_fuse_pass.cc | 4 ++-- paddle/fluid/framework/ir/cost_model_test.cc | 2 +- paddle/fluid/framework/ir/delete_dropout_op_pass.cc | 2 +- paddle/fluid/framework/ir/graph.h | 4 ++-- paddle/fluid/framework/ir/graph_pattern_detector.cc | 2 +- paddle/fluid/framework/ir/graph_pattern_detector.h | 6 +++--- paddle/fluid/framework/ir/xpu/multi_encoder_xpu_fuse_pass.h | 4 ++-- .../fluid/framework/ir/xpu/squeeze_excitation_fuse_pass.h | 2 +- paddle/fluid/framework/ir/xpu/stack_fuse_pass.cc | 2 +- paddle/fluid/framework/ir/xpu/xpu_quantize_squash_pass.cc | 2 +- 10 files changed, 15 insertions(+), 15 deletions(-) diff --git a/paddle/fluid/framework/ir/conv_elementwise_add2_act_fuse_pass.cc b/paddle/fluid/framework/ir/conv_elementwise_add2_act_fuse_pass.cc index ce122e9668201..52ba852a730a5 100644 --- a/paddle/fluid/framework/ir/conv_elementwise_add2_act_fuse_pass.cc +++ b/paddle/fluid/framework/ir/conv_elementwise_add2_act_fuse_pass.cc @@ -35,7 +35,7 @@ namespace ir { GET_IR_NODE(act_op); \ GET_IR_NODE(act_out); -// Inherient the basic information from `base_desc`, and modify some fields. +// Inherit the basic information from `base_desc`, and modify some fields. framework::proto::OpDesc PrepareOpDesc( const framework::proto::OpDesc& base_desc, const std::string& bias, @@ -154,7 +154,7 @@ void ConvElementwiseAdd2ActFusePass::ApplyImpl(ir::Graph* graph) const { all_act_set.insert(cutlass_act_set.begin(), cutlass_act_set.end()); } - patterns::ConvElementwiseadd2Act pattern(gpd.mutable_pattern(), pattern_name); + patterns::ConvElementwiseAdd2Act pattern(gpd.mutable_pattern(), pattern_name); pattern(x, all_act_set); int found_count = 0; auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph, diff --git a/paddle/fluid/framework/ir/cost_model_test.cc b/paddle/fluid/framework/ir/cost_model_test.cc index ba2e0745263b4..b12b15f715aca 100644 --- a/paddle/fluid/framework/ir/cost_model_test.cc +++ b/paddle/fluid/framework/ir/cost_model_test.cc @@ -151,7 +151,7 @@ TEST(CostDataTest, TestGetGraphProgram) { EXPECT_EQ(cost_data.GetProgram(), nullptr); } -TEST(CostDataTest, TestUninitailzed) { +TEST(CostDataTest, TestUninitialized) { CostData cost_data; EXPECT_EQ(cost_data.GetWholeMemoryBytes(), CostData::NOT_MEASURED); EXPECT_EQ(cost_data.GetWholeTimeMs(), CostData::NOT_MEASURED); diff --git a/paddle/fluid/framework/ir/delete_dropout_op_pass.cc b/paddle/fluid/framework/ir/delete_dropout_op_pass.cc index 285c25c6a5e9d..c4a00934b6339 100644 --- a/paddle/fluid/framework/ir/delete_dropout_op_pass.cc +++ b/paddle/fluid/framework/ir/delete_dropout_op_pass.cc @@ -98,7 +98,7 @@ DeleteDropoutOpXPass::DeleteDropoutOpXPass() { } void DeleteDropoutOpXPass::ApplyImpl(ir::Graph* graph) const { - VLOG(3) << "delte dropout op."; + VLOG(3) << "delete dropout op."; std::unordered_set del_node_set; for (Node* n : graph->Nodes()) { if (n->IsOp() && n->Op()) { diff --git a/paddle/fluid/framework/ir/graph.h b/paddle/fluid/framework/ir/graph.h index 9856183b4e500..e0e434cfe5b8d 100644 --- a/paddle/fluid/framework/ir/graph.h +++ b/paddle/fluid/framework/ir/graph.h @@ -69,7 +69,7 @@ namespace ir { * Write-After-Read * a = op1(x) * x = op2(b) - * A control-dependency connection is created bettwen op1 and op2 such that + * A control-dependency connection is created between op1 and op2 such that * op1->op2, so as to ensure correct order. * * Write-After-Write @@ -477,7 +477,7 @@ class Graph { std::unordered_set node_set_; size_t num_node_created_{0}; // help to generate a unique node id. // NOTE(Aurelius84): Whether is constructed with partial ProgramDesc. - // In case of @to_static, whole trainning program is splited into two + // In case of @to_static, whole training program is splited into two // parts: forward graph and backward graph, which can be executed // independently. bool is_partial_{false}; diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.cc b/paddle/fluid/framework/ir/graph_pattern_detector.cc index 968bba9241361..df804cf0d4f7b 100644 --- a/paddle/fluid/framework/ir/graph_pattern_detector.cc +++ b/paddle/fluid/framework/ir/graph_pattern_detector.cc @@ -2981,7 +2981,7 @@ PDNode *patterns::SelfAttention::operator()(PDNode *in) { return transpose2_2_out; } -PDNode *patterns::ConvElementwiseadd2Act::operator()( +PDNode *patterns::ConvElementwiseAdd2Act::operator()( PDNode *conv_in, const std::unordered_set &conv_act_set) { auto conv_op = pattern->NewNode(conv_op_repr())->assert_is_op("conv2d"); auto conv_filter = pattern->NewNode(conv_filter_repr()) diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.h b/paddle/fluid/framework/ir/graph_pattern_detector.h index cf8db104c8910..22d88e96b2852 100644 --- a/paddle/fluid/framework/ir/graph_pattern_detector.h +++ b/paddle/fluid/framework/ir/graph_pattern_detector.h @@ -1613,8 +1613,8 @@ struct ConvElementwiseaddAct : public PatternBase { }; // Conv + ElementwiseAdd + ElementwiseAdd + Activation -struct ConvElementwiseadd2Act : public PatternBase { - ConvElementwiseadd2Act(PDPattern* pattern, const std::string& name_scope) +struct ConvElementwiseAdd2Act : public PatternBase { + ConvElementwiseAdd2Act(PDPattern* pattern, const std::string& name_scope) : PatternBase( pattern, name_scope, "conv_elementwiseadd2_elementwiseadd_act") {} @@ -1638,7 +1638,7 @@ struct ConvElementwiseadd2Act : public PatternBase { }; // Conv + ElementwiseAdd -// This pattern should be used after ConvElementwiseadd2Act or +// This pattern should be used after ConvElementwiseAdd2Act or // ConvElementwiseadd pass struct ConvElementwiseadd : public PatternBase { ConvElementwiseadd(PDPattern* pattern, const std::string& name_scope) diff --git a/paddle/fluid/framework/ir/xpu/multi_encoder_xpu_fuse_pass.h b/paddle/fluid/framework/ir/xpu/multi_encoder_xpu_fuse_pass.h index cb24ea8128451..6c45838073af6 100644 --- a/paddle/fluid/framework/ir/xpu/multi_encoder_xpu_fuse_pass.h +++ b/paddle/fluid/framework/ir/xpu/multi_encoder_xpu_fuse_pass.h @@ -33,7 +33,7 @@ namespace ir { /* step1: fuse single ops to single_encoder_xpu -step2: fuse mutitl single_encoder_xpu to multi_encoder_xpu +step2: fuse multi single_encoder_xpu to multi_encoder_xpu 1. step1 Origin subgraph: @@ -144,7 +144,7 @@ class MultiEncoderXPUFusePass : public FusePassBase { bool norm_before, bool with_q_scale, bool with_mask, - bool is_smooth_qunat) const; + bool is_smooth_quant) const; bool ApplyMultiEncoderXPUFuse(ir::Graph* graph) const; diff --git a/paddle/fluid/framework/ir/xpu/squeeze_excitation_fuse_pass.h b/paddle/fluid/framework/ir/xpu/squeeze_excitation_fuse_pass.h index 72249f6fdc087..9ba8c83c123a0 100644 --- a/paddle/fluid/framework/ir/xpu/squeeze_excitation_fuse_pass.h +++ b/paddle/fluid/framework/ir/xpu/squeeze_excitation_fuse_pass.h @@ -32,7 +32,7 @@ namespace paddle { namespace framework { namespace ir { /* -Squeeze and Excitaion Block Fusion for SE-ResNet +Squeeze and Excitation Block Fusion for SE-ResNet Origin subgraph Input | \ diff --git a/paddle/fluid/framework/ir/xpu/stack_fuse_pass.cc b/paddle/fluid/framework/ir/xpu/stack_fuse_pass.cc index ff7a0b30237d1..6229c7657e66a 100644 --- a/paddle/fluid/framework/ir/xpu/stack_fuse_pass.cc +++ b/paddle/fluid/framework/ir/xpu/stack_fuse_pass.cc @@ -87,7 +87,7 @@ StackPattern::StackPattern(PDPattern* pattern, const std::string& name_scope) /* "stack" can be replaced by "unsqueeze" if: 1. "stack inputs" are the same。 -1. "stack output" is "elementwise_add input" or "fused_multi_transformer +2. "stack output" is "elementwise_add input" or "fused_multi_transformer src_mask input". */ class StackFusePass : public FusePassBase { diff --git a/paddle/fluid/framework/ir/xpu/xpu_quantize_squash_pass.cc b/paddle/fluid/framework/ir/xpu/xpu_quantize_squash_pass.cc index 74026d50915b2..4bcd721dcc68b 100644 --- a/paddle/fluid/framework/ir/xpu/xpu_quantize_squash_pass.cc +++ b/paddle/fluid/framework/ir/xpu/xpu_quantize_squash_pass.cc @@ -142,7 +142,7 @@ void XPUQuantizeSquashPass::OpDequantSquash(Graph* graph) const { std::string branch_name = any_op->Op()->Input("branch")[0]; auto* branch_node = FindNodeWithName(graph, branch_name); // If branch datatype is not equal to dequant_out datatype, can not - // squash. Because phase1: dquantize + quantize squash maybe squash + // squash. Because phase1: dequantize + quantize squash maybe squash // branch quantize, if so, We judge the datatype to decide whether to // squash. If squash, the result will be wrong. if (branch_node->Var()->GetDataType() != From ef6b6409e5040860c7e5cdb5d2101d0cdfc9a2a2 Mon Sep 17 00:00:00 2001 From: feifei-111 <2364819892@qq.com> Date: Tue, 27 Feb 2024 10:26:24 +0800 Subject: [PATCH 109/282] [SOT] avoid trace create layer tracker (#61858) --------- Co-authored-by: SigureMo --- .../executor/opcode_executor.py | 6 ++++ .../sot/opcode_translator/executor/tracker.py | 32 +++++++++++++++++++ test/sot/test_simulate_initialize.py | 19 +++++++++++ 3 files changed, 57 insertions(+) diff --git a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py index bd4dde84918ff..e5740c9f29728 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py +++ b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py @@ -771,6 +771,12 @@ def LOAD_ATTR(self, instr: Instruction): )(obj, attr_name_var) ) + @call_break_graph_decorator(push_n=1) + def LOAD_SUPER_ATTR(self, instr: Instruction): + # This bytecode is for Python 3.12+, and it will break graph in Python 3.11-. + # We align it's behavior with Python 3.11-. + raise BreakGraphError("call super is not supported") + def LOAD_CONST(self, instr: Instruction): var = self._co_consts[instr.arg] self.stack.push(var) diff --git a/python/paddle/jit/sot/opcode_translator/executor/tracker.py b/python/paddle/jit/sot/opcode_translator/executor/tracker.py index 51d21a5572129..d8c8c54b88480 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/tracker.py +++ b/python/paddle/jit/sot/opcode_translator/executor/tracker.py @@ -17,6 +17,7 @@ import builtins import dis import sys +from itertools import chain from typing import TYPE_CHECKING from ...utils import InnerError, NameGenerator @@ -432,5 +433,36 @@ def gen_instructions(self, codegen: PyCodeGen): codegen.gen_build_map(len(self.kwargs)) codegen.gen_call_function_ex(has_kwargs=True) + def trace_value_from_frame(self): + class_tracer = self.layer_class.tracker.trace_value_from_frame() + arg_tracers = [ + arg.tracker.trace_value_from_frame() for arg in self.args + ] + kwarg_tracers_dict = { + k: v.tracker.trace_value_from_frame() + for k, v in self.kwargs.items() + } + kwarg_tracers = list(kwarg_tracers_dict.values()) + + expr = "{}(" + expr += ", ".join(["{}"] * len(arg_tracers)) + if len(arg_tracers) and len(kwarg_tracers) > 0: + expr += ", " + expr += ", ".join(f"{k}={{}}" for k in kwarg_tracers_dict.keys()) + expr += ")" + + return StringifyExpression( + expr, + [class_tracer] + arg_tracers + kwarg_tracers, + union_free_vars( + *( + tracer.free_vars + for tracer in chain( + [class_tracer], arg_tracers, kwarg_tracers + ) + ) + ), + ) + def __repr__(self) -> str: return f"CreateLayerTracker(Layer={self.layer_class}, args={self.args}, kwargs={self.kwargs})" diff --git a/test/sot/test_simulate_initialize.py b/test/sot/test_simulate_initialize.py index 08a30dfc5a696..4c7610319c67c 100644 --- a/test/sot/test_simulate_initialize.py +++ b/test/sot/test_simulate_initialize.py @@ -19,6 +19,7 @@ import paddle from paddle import nn from paddle.jit.sot import symbolic_translate +from paddle.jit.sot.utils import strict_mode_guard class A: @@ -43,6 +44,20 @@ def error_foo(x): return t(x) +class NopLayer(paddle.nn.Layer): + def __init__(self): + super().__init__() + self.weight = None + + +def created_layer_reconstruct(): + x = paddle.to_tensor([1, 2], dtype="float32") + weight = NopLayer().weight + if weight is not None: + x += 1 + return x + + def bar(x): a = A(x) t = paddle.to_tensor(x) @@ -66,6 +81,10 @@ def run(): self.assertRaises(paddle.jit.sot.utils.exceptions.InnerError, run) + @strict_mode_guard(False) + def test_created_layer_reconstruct(self): + self.assert_results(created_layer_reconstruct) + if __name__ == "__main__": unittest.main() From 7a97c10599e4666054e649f06e86a0ee1b1519f9 Mon Sep 17 00:00:00 2001 From: diadestiny <44188454+diadestiny@users.noreply.github.com> Date: Tue, 27 Feb 2024 10:29:24 +0800 Subject: [PATCH 110/282] [SOT][3.12] extract `RETURN_VALUE` and `RETURN_CONST` hard code (#62073) --- .../sot/opcode_translator/executor/opcode_executor.py | 8 +++++--- .../instruction_utils/opcode_analysis.py | 10 ++++++++-- .../opcode_translator/instruction_utils/opcode_info.py | 3 +++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py index e5740c9f29728..d5635c94d159c 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py +++ b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py @@ -47,7 +47,7 @@ calc_stack_effect, get_instructions, ) -from ..instruction_utils.opcode_info import JumpDirection, PopJumpCond +from ..instruction_utils.opcode_info import RETURN, JumpDirection, PopJumpCond from .dispatch_functions import ( operator_BAD, operator_exception_match, @@ -1664,8 +1664,10 @@ def FOR_ITER(self, instr): start = self.indexof(instr) end = self.indexof(instr.jump_to) for i in range(start, end): - if self._instructions[i].opname == "RETURN_VALUE": - raise FallbackError("Found RETURN_VALUE in for loop body.") + if self._instructions[i].opname in RETURN: + raise FallbackError( + f"Found {self._instructions[i].opname} in for loop body." + ) self._graph.add_global_guarded_variable(iterator) diff --git a/python/paddle/jit/sot/opcode_translator/instruction_utils/opcode_analysis.py b/python/paddle/jit/sot/opcode_translator/instruction_utils/opcode_analysis.py index 2e8ded5d2ac5e..eb8cb1735bddf 100644 --- a/python/paddle/jit/sot/opcode_translator/instruction_utils/opcode_analysis.py +++ b/python/paddle/jit/sot/opcode_translator/instruction_utils/opcode_analysis.py @@ -19,7 +19,13 @@ from paddle.jit.utils import OrderedSet from .instruction_utils import Instruction -from .opcode_info import ALL_JUMP, HAS_FREE, HAS_LOCAL, UNCONDITIONAL_JUMP +from .opcode_info import ( + ALL_JUMP, + HAS_FREE, + HAS_LOCAL, + RETURN, + UNCONDITIONAL_JUMP, +) @dataclasses.dataclass @@ -122,7 +128,7 @@ def walk(state: State, start: int) -> OrderedSet[str]: else State(OrderedSet(), OrderedSet(), OrderedSet()) ) return jump_branch | not_jump_branch - elif instr.opname == "RETURN_VALUE": + elif instr.opname in RETURN: return state return state diff --git a/python/paddle/jit/sot/opcode_translator/instruction_utils/opcode_info.py b/python/paddle/jit/sot/opcode_translator/instruction_utils/opcode_info.py index e9b4af9f03fb0..2dc69b7565672 100644 --- a/python/paddle/jit/sot/opcode_translator/instruction_utils/opcode_info.py +++ b/python/paddle/jit/sot/opcode_translator/instruction_utils/opcode_info.py @@ -28,6 +28,9 @@ UNCONDITIONAL_JUMP = {"JUMP_ABSOLUTE", "JUMP_FORWARD"} if sys.version_info >= (3, 11): UNCONDITIONAL_JUMP.add("JUMP_BACKWARD") +RETURN = {"RETURN_VALUE"} +if sys.version_info >= (3, 12): + RETURN.add("RETURN_CONST") class JumpDirection(Enum): From 9594c1f5e1501d541abaf07a50e452d0de70c32c Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Tue, 27 Feb 2024 11:02:19 +0800 Subject: [PATCH 111/282] fix reshard bug that input and output is same tensor (#62082) --- .../auto_parallel/reshard/p_to_s_reshard_function.cc | 4 +++- .../auto_parallel/reshard/s_to_s_reshard_function.cc | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/paddle/phi/core/distributed/auto_parallel/reshard/p_to_s_reshard_function.cc b/paddle/phi/core/distributed/auto_parallel/reshard/p_to_s_reshard_function.cc index 508ba644e2474..0acf5abf3eec8 100644 --- a/paddle/phi/core/distributed/auto_parallel/reshard/p_to_s_reshard_function.cc +++ b/paddle/phi/core/distributed/auto_parallel/reshard/p_to_s_reshard_function.cc @@ -57,7 +57,7 @@ void PToSReshardFunction::Eval(DeviceContext* dev_ctx, int out_split_axis = GetSplitAxisWithDimsMapping(out_dist_attr.dims_mapping()).begin()->first; - DenseTensor in_reduce_scatter = in.value(); + DenseTensor in_reduce_scatter; std::vector axis; if (out_split_axis != 0) { for (size_t i = 0; i < common::vectorize(logical_ddim).size(); ++i) { @@ -66,6 +66,8 @@ void PToSReshardFunction::Eval(DeviceContext* dev_ctx, std::swap(axis[0], axis[out_split_axis]); RESHARD_FUNCTOR( dev_ctx, Transpose, dtype, in.value(), axis, &in_reduce_scatter); + } else { + in_reduce_scatter.ShareDataWith(in.value()); } DenseTensor out_reduce_scatter; diff --git a/paddle/phi/core/distributed/auto_parallel/reshard/s_to_s_reshard_function.cc b/paddle/phi/core/distributed/auto_parallel/reshard/s_to_s_reshard_function.cc index 57b5e8209fce6..d1efe18cce149 100644 --- a/paddle/phi/core/distributed/auto_parallel/reshard/s_to_s_reshard_function.cc +++ b/paddle/phi/core/distributed/auto_parallel/reshard/s_to_s_reshard_function.cc @@ -63,7 +63,7 @@ void SToSReshardFunction::Eval(phi::DeviceContext* dev_ctx, int out_split_axis = GetSplitAxisWithDimsMapping(out_dist_attr.dims_mapping()).begin()->first; - DenseTensor in_all_to_all = in.value(); + DenseTensor in_all_to_all; // 1. preprocess, reshape and transpose the input tensor if (out_split_axis != 0) { // 1.1 calc the shape and reshape @@ -93,10 +93,11 @@ void SToSReshardFunction::Eval(phi::DeviceContext* dev_ctx, pre_shape_vec[in_split_axis] *= nranks; RESHARD_FUNCTOR( dev_ctx, Reshape, dtype, out_transpose, pre_shape_vec, &in_all_to_all); + } else { + in_all_to_all.ShareDataWith(in.value()); } // 2. use all to all to switch data to other ranks - DenseTensor out_all_to_all; RESHARD_FUNCTOR_WITH_COMM(dev_ctx, AllToAll, dtype, From c5d11a594a152ca3e0f7ef910bbf7cc1b6b75c49 Mon Sep 17 00:00:00 2001 From: zhangbo9674 <82555433+zhangbo9674@users.noreply.github.com> Date: Tue, 27 Feb 2024 11:08:59 +0800 Subject: [PATCH 112/282] [PIR] Enable pir executor by default in auto parallel (#62025) * enable pir executor by default in auto parallel * refine error msg * skip legacy executor * remove tests of deprecated api * fix * fix * fix * refine --------- Co-authored-by: zhiqiu --- .../ir_adaptor/translator/op_translator.cc | 21 +++ .../pir/dialect/op_generator/ops_api_gen.py | 6 + .../operator/interface/parse_kernel_key.cc | 4 + .../operator/interface/parse_kernel_key.h | 2 + paddle/fluid/pir/dialect/operator/ir/ops.yaml | 39 +++++ .../fluid/pir/dialect/operator/utils/utils.cc | 6 + paddle/phi/api/yaml/op_compat.yaml | 14 ++ paddle/phi/core/kernel_context.cc | 2 +- paddle/phi/infermeta/multiary.cc | 26 +++ paddle/phi/infermeta/multiary.h | 10 ++ paddle/phi/infermeta/ternary.cc | 20 +++ paddle/phi/infermeta/ternary.h | 8 + .../auto_parallel/static/engine.py | 5 + test/auto_parallel/engine_api.py | 151 ------------------ 14 files changed, 162 insertions(+), 152 deletions(-) diff --git a/paddle/fluid/ir_adaptor/translator/op_translator.cc b/paddle/fluid/ir_adaptor/translator/op_translator.cc index c01df4d6e236c..6e1ec454b6bab 100644 --- a/paddle/fluid/ir_adaptor/translator/op_translator.cc +++ b/paddle/fluid/ir_adaptor/translator/op_translator.cc @@ -3113,6 +3113,25 @@ struct CEmbeddingOpTranscriber : public OpTranscriber { } }; +struct QuantizeLinearOpTranscriber : public OpTranscriber { + void HandleNonexistentAttribute(pir::IrContext* ctx, + pir::AttributeMap* attribute_map, + const OpAttributeInfo& info) override { + if (info.name == "round_type") { + (*attribute_map)[info.name] = pir::Int32Attribute::get(ctx, 0); + } + if (info.name == "is_test") { + (*attribute_map)[info.name] = pir::BoolAttribute::get(ctx, true); + } + if (info.name == "only_observer") { + (*attribute_map)[info.name] = pir::BoolAttribute::get(ctx, false); + } + if (info.name == "moving_rate") { + (*attribute_map)[info.name] = pir::FloatAttribute::get(ctx, 0.9); + } + } +}; + OpTranslator::OpTranslator() { pir::IrContext* ctx = pir::IrContext::Instance(); ctx->GetOrRegisterDialect(); @@ -3185,6 +3204,8 @@ OpTranslator::OpTranslator() { special_handlers["elementwise_mod_grad"] = ElementwiseGradTranscriber(); special_handlers["elementwise_floordiv_grad"] = ElementwiseGradTranscriber(); special_handlers["c_embedding"] = CEmbeddingOpTranscriber(); + special_handlers["quantize_linear"] = QuantizeLinearOpTranscriber(); + special_handlers["dequantize_linear"] = QuantizeLinearOpTranscriber(); } } // namespace translator diff --git a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py index b141f1ecfa879..54b56a2e3c887 100644 --- a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py @@ -108,6 +108,12 @@ 'lrn', 'multi_gru', 'matmul_with_flatten', + 'moving_average_abs_max_scale', + 'moving_average_abs_max_scale_', + 'quantize_linear', + 'quantize_linear_', + 'dequantize_linear', + 'dequantize_linear_', ] NO_NEED_GEN_STATIC_ONLY_APIS = [ diff --git a/paddle/fluid/pir/dialect/operator/interface/parse_kernel_key.cc b/paddle/fluid/pir/dialect/operator/interface/parse_kernel_key.cc index 44183751b8ca1..5469237524880 100644 --- a/paddle/fluid/pir/dialect/operator/interface/parse_kernel_key.cc +++ b/paddle/fluid/pir/dialect/operator/interface/parse_kernel_key.cc @@ -28,6 +28,10 @@ KernelKeyTuple UniqueOpParseKernelKey(pir::Operation* op) { return {dtype, backend}; } +KernelKeyTuple SaveCombineOpParseKernelKey(pir::Operation* op) { + return {phi::DataType::FLOAT32, phi::Backend::UNDEFINED}; +} + } // namespace paddle::dialect IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::ParseKernelKeyInterface) diff --git a/paddle/fluid/pir/dialect/operator/interface/parse_kernel_key.h b/paddle/fluid/pir/dialect/operator/interface/parse_kernel_key.h index 2d101dbd310d5..7913893fdb7d7 100644 --- a/paddle/fluid/pir/dialect/operator/interface/parse_kernel_key.h +++ b/paddle/fluid/pir/dialect/operator/interface/parse_kernel_key.h @@ -57,6 +57,8 @@ class ParseKernelKeyInterface // Register the ParseKernelKeyInterface for unique op. KernelKeyTuple UniqueOpParseKernelKey(pir::Operation *op); +KernelKeyTuple SaveCombineOpParseKernelKey(pir::Operation *op); + } // namespace dialect } // namespace paddle diff --git a/paddle/fluid/pir/dialect/operator/ir/ops.yaml b/paddle/fluid/pir/dialect/operator/ir/ops.yaml index f1e20326d59de..594130926d569 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops.yaml @@ -366,6 +366,19 @@ data_type : x backward : depthwise_conv2d_transpose_grad +- op : dequantize_linear + args : (Tensor x, Tensor scale, Tensor zero_point, Tensor in_accum, Tensor in_state, int quant_axis = 0, int bit_length = 8, int round_type = 0, bool is_test = true, bool only_observer = false, float moving_rate=0.9f) + output : Tensor(y), Tensor(out_scale), Tensor(out_accum), Tensor(out_state) + infer_meta : + func : QuantizeLinearInferMeta + param : [x, scale, in_accum, in_state, quant_axis] + kernel : + func : quantize_linear + param : [x, scale, zero_point, in_accum, in_state, quant_axis, bit_length, round_type, is_test, only_observer, moving_rate] + data_type : x + optional : in_accum, in_state, out_scale, out_accum, out_state + inplace : (scale -> out_scale, in_accum -> out_accum, in_state -> out_state) + - op : disable_check_model_nan_inf args: (Tensor x, int flag = 0) output: Tensor(out) @@ -1083,6 +1096,19 @@ data_type : out_grad_in inplace: (out_grad_in -> out_grad_out) +- op : quantize_linear + args : (Tensor x, Tensor scale, Tensor zero_point, Tensor in_accum, Tensor in_state, int quant_axis = 0, int bit_length = 8, int round_type = 0, bool is_test = true, bool only_observer = false, float moving_rate=0.9f) + output : Tensor(y), Tensor(out_scale), Tensor(out_accum), Tensor(out_state) + infer_meta : + func : QuantizeLinearInferMeta + param : [x, scale, in_accum, in_state, quant_axis] + kernel : + func : quantize_linear + param : [x, scale, zero_point, in_accum, in_state, quant_axis, bit_length, round_type, is_test, only_observer, moving_rate] + data_type : x + optional : in_accum, in_state, out_scale, out_accum, out_state + inplace : (scale -> out_scale, in_accum -> out_accum, in_state -> out_state) + - op : randint args : (int low, int high, IntArray shape, DataType dtype=DataType::INT64, Place place={}) output : Tensor(out) @@ -1215,6 +1241,7 @@ func: save_combine_tensor param: [x, file_path, overwrite, save_as_fp16, save_to_memory] optional : out + interfaces : paddle::dialect::ParseKernelKeyInterface - op : seed args : (int seed, bool deterministic, str rng_name, bool force_cpu) @@ -1635,6 +1662,18 @@ func: match_matrix_tensor backward: match_matrix_tensor_grad +- op: moving_average_abs_max_scale + args: (Tensor x, Tensor in_accum, Tensor in_state, float moving_rate=0.9f, bool is_test=false) + output: Tensor(out), Tensor(out_scale), Tensor(out_state), Tensor(out_accum) + infer_meta: + func: MovingAverageAbsMaxScaleInferMeta + param: [x, in_accum, in_state] + kernel: + func: moving_average_abs_max_scale + param: [x, in_accum, in_state, moving_rate, is_test] + optional : in_accum, in_state, out, out_state, out_accum + inplace : (in_accum -> out_accum), (in_state -> out_state) + - op: nce args: (Tensor input, Tensor label, Tensor weight, Tensor bias, Tensor sample_weight, Tensor custom_dist_probs, Tensor custom_dist_alias, Tensor custom_dist_alias_probs, int num_total_classes, int[] custom_neg_classes={}, int num_neg_samples=10, int sampler=0, int seed=0, bool is_sparse=false, bool remote_prefetch=false, bool is_test=false) output: Tensor(cost), Tensor(sample_logits), Tensor(sample_labels) diff --git a/paddle/fluid/pir/dialect/operator/utils/utils.cc b/paddle/fluid/pir/dialect/operator/utils/utils.cc index 299a047694e50..9b450977814b6 100644 --- a/paddle/fluid/pir/dialect/operator/utils/utils.cc +++ b/paddle/fluid/pir/dialect/operator/utils/utils.cc @@ -72,6 +72,12 @@ const std::unordered_set LegacyOpList = { NceGradOp::name(), LrnOp::name(), LrnGradOp::name(), + MovingAverageAbsMaxScaleOp::name(), + MovingAverageAbsMaxScale_Op::name(), + QuantizeLinearOp::name(), + QuantizeLinear_Op::name(), + DequantizeLinearOp::name(), + DequantizeLinear_Op::name(), #ifdef PADDLE_WITH_DNNL paddle::onednn::dialect::LrnOp::name(), paddle::onednn::dialect::LrnGradOp::name(), diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml index 53e0cea953b87..1a3f86753fa7e 100755 --- a/paddle/phi/api/yaml/op_compat.yaml +++ b/paddle/phi/api/yaml/op_compat.yaml @@ -771,6 +771,10 @@ {scale : Scale, shift : Shift} - op : dequantize_linear + inputs : + {x : X, scale : Scale, zero_point : ZeroPoint, in_accum : InAccum, in_state : InState} + outputs : + {y : Y, out_scale : OutScale, out_accum : OutAccum, out_state : OutState} extra : attrs : [float moving_rate = 0.9] @@ -2197,6 +2201,12 @@ outputs : {param_out : ParamOut, velocity_out : VelocityOut, master_param_out : MasterParamOut} +- op : moving_average_abs_max_scale + inputs : + {x : X, in_accum : InAccum, in_state : InState} + outputs : + {out : Out, out_scale : OutScale, out_state : OutState, out_accum : OutAccum} + - op : multi_dot backward : multi_dot_grad inputs : @@ -2546,6 +2556,10 @@ {scale : Scale, shift : Shift, include_self: Include_self} - op : quantize_linear + inputs : + {x : X, scale : Scale, zero_point : ZeroPoint, in_accum : InAccum, in_state : InState} + outputs : + {y : Y, out_scale : OutScale, out_accum : OutAccum, out_state : OutState} extra : attrs : [float moving_rate = 0.9] diff --git a/paddle/phi/core/kernel_context.cc b/paddle/phi/core/kernel_context.cc index 07b05ccd2c760..4a6888fad7d86 100644 --- a/paddle/phi/core/kernel_context.cc +++ b/paddle/phi/core/kernel_context.cc @@ -119,7 +119,7 @@ const AttrType& KernelContext::AttrAt(size_t idx) const { return paddle::get(attrs_.at(idx)); } catch (paddle::bad_variant_access const& ex) { PADDLE_THROW(phi::errors::InvalidArgument( - "Attribute cast error in Op Kernel Context.")); + "Attribute %d cast error in Op Kernel Context.", idx)); } } diff --git a/paddle/phi/infermeta/multiary.cc b/paddle/phi/infermeta/multiary.cc index b7a5dd51de901..69214508ef3f9 100644 --- a/paddle/phi/infermeta/multiary.cc +++ b/paddle/phi/infermeta/multiary.cc @@ -3497,6 +3497,32 @@ void PsroiPoolInferMeta(const MetaTensor& x, out->set_dtype(x.dtype()); } +void QuantizeLinearInferMeta(const MetaTensor& x, + const MetaTensor& scale, + const MetaTensor& in_accum, + const MetaTensor& in_state, + int quant_axis, + MetaTensor* y, + MetaTensor* out_scale, + MetaTensor* out_accum, + MetaTensor* out_state) { + y->set_dims(x.dims()); + y->share_lod(x); + if (out_scale) { + if (quant_axis < 0) { + out_scale->set_dims(scale.dims()); + } else { + out_scale->set_dims({x.dims()[quant_axis]}); + } + } + if (out_accum) { + out_accum->set_dims(in_accum.dims()); + } + if (out_state) { + out_state->set_dims(in_state.dims()); + } +} + void RmsNormInferMeta(const MetaTensor& x, const MetaTensor& bias, const MetaTensor& residual, diff --git a/paddle/phi/infermeta/multiary.h b/paddle/phi/infermeta/multiary.h index 0774189dd8d4f..3d9b2539267e7 100644 --- a/paddle/phi/infermeta/multiary.h +++ b/paddle/phi/infermeta/multiary.h @@ -636,6 +636,16 @@ void PsroiPoolInferMeta(const MetaTensor& x, float spatial_scale, MetaTensor* out); +void QuantizeLinearInferMeta(const MetaTensor& x, + const MetaTensor& scale, + const MetaTensor& in_accum, + const MetaTensor& in_state, + int quant_axis, + MetaTensor* y, + MetaTensor* out_scale, + MetaTensor* out_accum, + MetaTensor* out_state); + void RmsNormInferMeta(const MetaTensor& x, const MetaTensor& bias, const MetaTensor& residual, diff --git a/paddle/phi/infermeta/ternary.cc b/paddle/phi/infermeta/ternary.cc index b728c33abf2e2..edd03e6b07513 100644 --- a/paddle/phi/infermeta/ternary.cc +++ b/paddle/phi/infermeta/ternary.cc @@ -896,6 +896,26 @@ void MultiClassNMSInferMeta(const MetaTensor& bboxes, nms_rois_num->set_dtype(DataType::INT32); } +void MovingAverageAbsMaxScaleInferMeta(const MetaTensor& x, + const MetaTensor& in_accum, + const MetaTensor& in_state, + MetaTensor* out, + MetaTensor* out_scale, + MetaTensor* out_state, + MetaTensor* out_accum) { + if (out) { + out->set_dims(x.dims()); + out->share_lod(x); + out_scale->set_dims({1}); + } + if (out_state) { + out_state->set_dims(in_state.dims()); + } + if (out_accum) { + out_accum->set_dims(in_accum.dims()); + } +} + void NllLossRawInferMeta(const MetaTensor& input, const MetaTensor& label, const MetaTensor& weight, diff --git a/paddle/phi/infermeta/ternary.h b/paddle/phi/infermeta/ternary.h index 7ffdc3d272069..d12378fe3a92c 100644 --- a/paddle/phi/infermeta/ternary.h +++ b/paddle/phi/infermeta/ternary.h @@ -140,6 +140,14 @@ void MatchMatrixTensorInferMeta(const MetaTensor& x, MetaTensor* tmp, MetaConfig config = MetaConfig()); +void MovingAverageAbsMaxScaleInferMeta(const MetaTensor& x, + const MetaTensor& in_accum, + const MetaTensor& in_state, + MetaTensor* out, + MetaTensor* out_scale, + MetaTensor* out_state, + MetaTensor* out_accum); + void MultiClassNMSInferMeta(const MetaTensor& bboxes, const MetaTensor& scores, const MetaTensor& rois_num, diff --git a/python/paddle/distributed/auto_parallel/static/engine.py b/python/paddle/distributed/auto_parallel/static/engine.py index 87497184392e6..401737bb13ac6 100644 --- a/python/paddle/distributed/auto_parallel/static/engine.py +++ b/python/paddle/distributed/auto_parallel/static/engine.py @@ -257,6 +257,11 @@ def __init__( paddle.framework.set_flags({'FLAGS_new_executor_sequential_run': 1}) paddle.framework.set_flags({'FLAGS_new_executor_static_build': 1}) + if auto_utils.use_new_executor(): + is_pir_mode = os.environ.get("FLAGS_enable_pir_in_executor", None) + if is_pir_mode is None: + paddle.framework.set_flags({'FLAGS_enable_pir_in_executor': 1}) + self.enable_job_schedule_profiler = False # get dist input spec from shard dataloader diff --git a/test/auto_parallel/engine_api.py b/test/auto_parallel/engine_api.py index cc921d41a74a9..7edcb9a9823cd 100644 --- a/test/auto_parallel/engine_api.py +++ b/test/auto_parallel/engine_api.py @@ -234,155 +234,6 @@ def train_low_level(): engine.load(model_filename) temp_dir.cleanup() - # Build dataloader from generator - # train - train_dataset = MyDataset(batch_num * batch_size) - train_dataloader = engine.dataloader_from_generator( - train_dataset, batch_size=batch_size, mode="train" - ) - engine.prepare(mode="train") - for data in train_dataloader: - outs = engine.run(data, feed=feed_dict, mode="train") - - # eval - engine.to_mode("eval") - eval_dataset2 = MyDataset(batch_size) - eval_dataloader = engine.dataloader_from_generator( - eval_dataset2, batch_size=batch_size - ) - engine.prepare() - for data in eval_dataloader: - outs = engine.run(data, feed=feed_dict) - - # predict - test_dataset = MyDataset(batch_size) - predict_dataloader = engine.dataloader_from_generator( - test_dataset, batch_size=batch_size, mode="predict" - ) - engine.prepare(mode="predict") - for data in predict_dataloader: - outs = engine.run(data, feed=feed_dict, mode="predict") - - # save - temp_dir = tempfile.TemporaryDirectory() - model_filename = os.path.join(temp_dir.name, 'mlp') - engine.save(model_filename, training=True) - engine.load(model_filename) - temp_dir.cleanup() - - -def train_builtin_data_vars(): - paddle.distributed.auto_parallel.static.dist_context.set_default_distributed_context( - None - ) - mlp = MLPLayer( - hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02, - ) - loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Adam( - learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None, - ) - metric = paddle.metric.Accuracy() - - strategy = auto.Strategy() - strategy.auto_mode = "semi" - - engine = auto.Engine(mlp, loss, optimizer, metric, strategy=strategy) - - # train - engine.to_mode("train") - - input_spec = static.InputSpec([batch_size, image_size], 'float32', 'input') - label_spec = static.InputSpec([batch_size, 1], 'int64', 'label') - engine.prepare(inputs_spec=[input_spec], labels_spec=[label_spec]) - - with static.program_guard(engine.main_program, engine.startup_program): - feed_list = engine.inputs + engine.labels - print(feed_list) - loader = paddle.base.io.DataLoader.from_generator( - feed_list=feed_list, capacity=4 * batch_size, iterable=False - ) - - places = static.cuda_places() - loader.set_batch_generator(batch_generator_creator(), places=places) - - for _ in range(epoch_num): - loader.start() # call DataLoader.start() before each epoch starts - try: - while True: - engine.run() - except paddle.base.core.EOFException: - loader.reset() # call DataLoader.reset() after catching EOFException - - -def train_non_builtin_data_vars(): - paddle.distributed.auto_parallel.static.dist_context.set_default_distributed_context( - None - ) - main_program = static.Program() - startup_program = static.Program() - with static.program_guard( - main_program, startup_program - ), utils.unique_name.guard(): - input = static.data( - name="input", shape=[batch_size, image_size], dtype='float32' - ) - label = static.data(name="label", shape=[batch_size, 1], dtype='int64') - - loader = paddle.base.io.DataLoader.from_generator( - feed_list=[input, label], capacity=4 * batch_size, iterable=False - ) - places = static.cuda_places() - loader.set_batch_generator(batch_generator_creator(), places=places) - - mlp = MLPLayer( - hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02, - ) - loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Adam( - learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None, - ) - metric = paddle.metric.Accuracy() - predict = mlp(input) - loss_var = loss(predict, label) - - strategy = auto.Strategy() - strategy.auto_mode = "semi" - - engine = auto.Engine( - loss=loss_var, optimizer=optimizer, metrics=metric, strategy=strategy - ) - - # train - engine.to_mode("train") - engine.prepare( - inputs=[input], - labels=[label], - main_program=main_program, - startup_program=startup_program, - ) - for _ in range(epoch_num): - loader.start() # call DataLoader.start() before each epoch starts - try: - while True: - engine.run() - except paddle.base.core.EOFException: - loader.reset() # call DataLoader.reset() after catching EOFException - def get_cost(): paddle.distributed.auto_parallel.static.dist_context.set_default_distributed_context( @@ -522,8 +373,6 @@ def get_cost_by_spec(): train_high_level(fetch=True) train_high_level(fetch=False) train_low_level() - train_builtin_data_vars() - train_non_builtin_data_vars() get_cost() get_cost_by_default_program() get_cost_by_spec() From 67570dc79601887531689c78736db912e65d0981 Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Tue, 27 Feb 2024 11:36:36 +0800 Subject: [PATCH 113/282] [PIR+CINN]Fix Insert GroupOp Position Bug and Open 13 test_sub_graph UT under with_cinn=True (#61884) * [PIR+CINN]Open 13 test_sub_graph UT under with_cinn=True * add UT * fix ReplaceWithGroupOp * fix build * fix insert group position bug * refine code * disable some UT --- .../pir/transforms/sub_graph_detector.cc | 69 ++++++++++++++----- test/ir/pir/cinn/sub_graphs/CMakeLists.txt | 2 +- .../pir/cinn/sub_graphs/test_sub_graph_42.py | 3 +- .../pir/cinn/sub_graphs/test_sub_graph_53.py | 5 +- .../pir/cinn/sub_graphs/test_sub_graph_55.py | 4 +- .../pir/cinn/sub_graphs/test_sub_graph_60.py | 5 +- .../pir/cinn/sub_graphs/test_sub_graph_68.py | 3 +- .../pir/cinn/sub_graphs/test_sub_graph_70.py | 5 +- .../pir/cinn/sub_graphs/test_sub_graph_71.py | 4 +- .../pir/cinn/sub_graphs/test_sub_graph_75.py | 5 +- .../pir/cinn/sub_graphs/test_sub_graph_76.py | 1 - .../pir/cinn/sub_graphs/test_sub_graph_77.py | 5 +- .../pir/cinn/sub_graphs/test_sub_graph_79.py | 5 +- .../pir/cinn/sub_graphs/test_sub_graph_80.py | 5 +- .../pir/cinn/sub_graphs/test_sub_graph_86.py | 4 +- 15 files changed, 80 insertions(+), 45 deletions(-) diff --git a/paddle/fluid/pir/transforms/sub_graph_detector.cc b/paddle/fluid/pir/transforms/sub_graph_detector.cc index dcb55412feb1f..e0cd23467cb04 100644 --- a/paddle/fluid/pir/transforms/sub_graph_detector.cc +++ b/paddle/fluid/pir/transforms/sub_graph_detector.cc @@ -478,6 +478,40 @@ std::vector AnalysisOutputs( return outputs; } +namespace { + +pir::Operation* FindInsertPoint(const GroupOpsVec& group_ops, + const std::vector& outputs) { + // Regard last op as insert position if there are no downstream ops between in + // group_ops. + pir::Operation* insert_point_op = group_ops.back(); + auto begin = group_ops.front()->operator Block::ConstIterator(); + auto end = ++(group_ops.back()->operator Block::ConstIterator()); + const std::unordered_set outputs_set(outputs.begin(), + outputs.end()); + const std::unordered_set group_ops_set( + group_ops.begin(), group_ops.end()); + + const auto& IsDownstreamOp = [&](const pir::Operation* op) -> bool { + if (group_ops_set.find(op) != group_ops_set.end()) return false; + for (auto& value : op->operands_source()) { + if (outputs_set.find(value) != outputs_set.end()) { + return true; + } + } + return false; + }; + // Find first downstream op as final insert position. + for (; begin != end; ++begin) { + if (IsDownstreamOp(begin)) { + insert_point_op = begin; + break; + } + } + return insert_point_op; +} +} // namespace + void ReplaceWithGroupOp(pir::Block* block, const GroupOpsVec& group_ops) { // NOLINT ::pir::IrContext* ctx = ::pir::IrContext::Instance(); @@ -486,25 +520,28 @@ void ReplaceWithGroupOp(pir::Block* block, ctx->GetOrRegisterDialect(); #endif ::pir::Builder builder = ::pir::Builder(ctx, block); - // step 1: Ensure the insert point and create GroupOp here. - auto* last_op = group_ops.back(); - builder.SetInsertionPointAfter(last_op); - std::vector output_types; - std::vector outputs = AnalysisOutputs(group_ops); - - for (auto& value : outputs) { - output_types.emplace_back(value.type()); - } + const std::vector outputs = AnalysisOutputs(group_ops); + + // step 1: Analysis and insert group op before insert_point. + auto* insert_point = FindInsertPoint(group_ops, outputs); + builder.set_insertion_point(insert_point); + VLOG(6) << "Insert GroupOp after " << insert_point->name(); + // step 2: Replace the old op with GroupOp. - auto new_group_op = builder.Build(output_types); - pir::Block* group_block = new_group_op.block(); + const auto& CreateGroupOp = [&]() -> cinn::dialect::GroupOp { + std::vector output_types; + for (auto& value : outputs) output_types.emplace_back(value.type()); - for (auto op : group_ops) { - op->MoveTo(group_block, group_block->end()); - } + auto new_group_op = builder.Build(output_types); + for (auto op : group_ops) { + op->MoveTo(new_group_op.block(), new_group_op.block()->end()); + } + return new_group_op; + }; + auto new_group_op = CreateGroupOp(); // step 3: Replace outputs of inner ops - std::vector group_outs = new_group_op->results(); + const std::vector group_outs = new_group_op->results(); std::unordered_set inner_ops(group_ops.begin(), group_ops.end()); for (size_t i = 0; i < outputs.size(); ++i) { @@ -515,7 +552,7 @@ void ReplaceWithGroupOp(pir::Block* block, } // step 4: Insert YieldOp for outputs - builder.SetInsertionPointToBlockEnd(group_block); + builder.SetInsertionPointToBlockEnd(new_group_op.block()); builder.Build<::pir::YieldOp>(outputs); } diff --git a/test/ir/pir/cinn/sub_graphs/CMakeLists.txt b/test/ir/pir/cinn/sub_graphs/CMakeLists.txt index f5c2bf6331537..2d166a44846f5 100644 --- a/test/ir/pir/cinn/sub_graphs/CMakeLists.txt +++ b/test/ir/pir/cinn/sub_graphs/CMakeLists.txt @@ -13,7 +13,7 @@ if(WITH_GPU) ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/python/:$ENV{PYTHONPATH} FLAGS_cinn_new_group_scheduler=1 FLAGS_enable_pir_api=1 - ${PYTHON_EXECUTABLE} + FLAGS_cudnn_deterministic=true ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/${cinn_sub_graph_test_name}.py WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_tests_properties(${cinn_sub_graph_test_name} PROPERTIES LABELS diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_42.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_42.py index b7b3d07001ca9..f7c6f47bd6337 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_42.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_42.py @@ -104,13 +104,14 @@ def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) # TODO(Aurelius84): cinn.gather will raise Check failed: input_args.size() == 3U (4 vs. 3) paddle.set_flags({"FLAGS_deny_cinn_ops": "gather"}) + # NOTE(Aurelius84): atol only satisfy 1e-5 under with_cinn=True. cinn_out = self.train( self.net, to_static=True, with_prim=True, with_cinn=True ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) ): - np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-6) + np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-5) if __name__ == '__main__': diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_53.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_53.py index 368d64ed09c65..91bc95ebf457b 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_53.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_53.py @@ -86,16 +86,15 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): outs = net(*self.inputs) return outs - # NOTE prim + cinn lead to error def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) cinn_out = self.train( - self.net, to_static=True, with_prim=True, with_cinn=False + self.net, to_static=True, with_prim=True, with_cinn=True ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) ): - np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) + np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-6) if __name__ == '__main__': diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_55.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_55.py index d274ca455a88b..c9781b8ae0e57 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_55.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_55.py @@ -91,8 +91,10 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): # NOTE prim + cinn lead to error def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) + # NOTE(Aurelius84): cinn_op.pool2d only support pool_type='avg' under adaptive=True + paddle.set_flags({"FLAGS_deny_cinn_ops": "pool2d"}) cinn_out = self.train( - self.net, to_static=True, with_prim=True, with_cinn=False + self.net, to_static=True, with_prim=True, with_cinn=True ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_60.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_60.py index 37a0c5228052a..c9fd19a3455c6 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_60.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_60.py @@ -108,11 +108,12 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): outs = net(*self.inputs) return outs - # NOTE prim + cinn lead to error def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) + # NOTE(Aurelius84): cinn_op.pool2d only support pool_type='avg' under adaptive=True + paddle.set_flags({"FLAGS_deny_cinn_ops": "pool2d"}) cinn_out = self.train( - self.net, to_static=True, with_prim=True, with_cinn=False + self.net, to_static=True, with_prim=True, with_cinn=True ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_68.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_68.py index 344306968e767..3ffa508fc23f5 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_68.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_68.py @@ -117,11 +117,10 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): outs = net(*self.inputs) return outs - # NOTE prim + cinn lead to error def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) cinn_out = self.train( - self.net, to_static=True, with_prim=False, with_cinn=False + self.net, to_static=True, with_prim=True, with_cinn=True ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_70.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_70.py index a6593682ec340..eeeca452b5e97 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_70.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_70.py @@ -68,11 +68,12 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): outs = net(*self.inputs) return outs - # NOTE prim + cinn lead to error def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) + # TODO(Aurelius84): disable gather op in CINN + paddle.set_flags({"FLAGS_deny_cinn_ops": "gather"}) cinn_out = self.train( - self.net, to_static=True, with_prim=True, with_cinn=False + self.net, to_static=True, with_prim=True, with_cinn=True ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_71.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_71.py index 35618f52a2d48..fd62209dc96c4 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_71.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_71.py @@ -244,16 +244,16 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): outs = net(*self.inputs) return outs - # NOTE prim + cinn lead to error def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) cinn_out = self.train( self.net, to_static=True, with_prim=True, with_cinn=False ) + # TODO(Aurelius84): atol only satisfy 1e-5 under with_cinn=True for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) ): - np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) + np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-5) if __name__ == '__main__': diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_75.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_75.py index 33f645fb013e5..965fa6021a673 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_75.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_75.py @@ -106,16 +106,15 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): outs = net(*self.inputs) return outs - # NOTE prim + cinn lead to error def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) cinn_out = self.train( - self.net, to_static=True, with_prim=False, with_cinn=False + self.net, to_static=True, with_prim=True, with_cinn=True ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) ): - np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) + np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-6) if __name__ == '__main__': diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_76.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_76.py index 1a879c53cbe0e..0b3d9fd560042 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_76.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_76.py @@ -125,7 +125,6 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): outs = net(*self.inputs) return outs - # NOTE prim + cinn lead to error def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) cinn_out = self.train( diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_77.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_77.py index 59834fca3fa5c..c892e461bcc9c 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_77.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_77.py @@ -171,16 +171,15 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): outs = net(*self.inputs) return outs - # NOTE prim + cinn lead to error def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) cinn_out = self.train( - self.net, to_static=True, with_prim=True, with_cinn=False + self.net, to_static=True, with_prim=True, with_cinn=True ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) ): - np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) + np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-6) if __name__ == '__main__': diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_79.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_79.py index 7109e845f0224..69b7847f2a096 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_79.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_79.py @@ -96,16 +96,15 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): outs = net(*self.inputs) return outs - # NOTE prim + cinn lead to error def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) cinn_out = self.train( - self.net, to_static=True, with_prim=True, with_cinn=False + self.net, to_static=True, with_prim=True, with_cinn=True ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) ): - np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) + np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-6) if __name__ == '__main__': diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_80.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_80.py index 90c9e4b6d8be9..9ce0cb50db21d 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_80.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_80.py @@ -93,16 +93,15 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): outs = net(*self.inputs) return outs - # NOTE prim + cinn lead to error def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) cinn_out = self.train( - self.net, to_static=True, with_prim=True, with_cinn=False + self.net, to_static=True, with_prim=True, with_cinn=True ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) ): - np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) + np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-6) if __name__ == '__main__': diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_86.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_86.py index e20c1ee4495f9..d117ee86a0aa8 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_86.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_86.py @@ -243,16 +243,16 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): outs = net(*self.inputs) return outs - # NOTE prim + cinn lead to error def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) + # NOTE(Aurelius84): atol only satisfy 1e-5 under with_cinn=True cinn_out = self.train( self.net, to_static=True, with_prim=True, with_cinn=False ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) ): - np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) + np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-5) if __name__ == '__main__': From 269de7f072ef8968ed385e4e53d4a6d4bdf45c3f Mon Sep 17 00:00:00 2001 From: AyaseNana <49900969+NKNaN@users.noreply.github.com> Date: Tue, 27 Feb 2024 12:08:40 +0800 Subject: [PATCH 114/282] API improvement for nn.functional.upsample (#61974) * improve upsample api * update test * update * update * refine upsample layer * update docs * fix test * udpate docs * update docs --- python/paddle/nn/functional/common.py | 73 ++++++++++++------- python/paddle/nn/layer/common.py | 39 +++++++--- .../legacy_test/test_bilinear_interp_v2_op.py | 23 ++++++ test/legacy_test/test_imperative_layers.py | 2 +- test/legacy_test/test_linear_interp_v2_op.py | 21 +++++- .../test_trilinear_interp_v2_op.py | 48 ++++++++++++ 6 files changed, 167 insertions(+), 39 deletions(-) diff --git a/python/paddle/nn/functional/common.py b/python/paddle/nn/functional/common.py index fcc79221d867d..de78e37d99fd9 100644 --- a/python/paddle/nn/functional/common.py +++ b/python/paddle/nn/functional/common.py @@ -182,15 +182,16 @@ def interpolate( mode='nearest', align_corners=False, align_mode=0, - data_format='NCHW', + data_format=None, name=None, ): """ This API resizes a batch of images. - The input must be a 3-D Tensor of the shape (num_batches, channels, in_w) - or 4-D (num_batches, channels, in_h, in_w), or a 5-D Tensor of the shape + The input must be be a 3-D Tensor of the shape (num_batches, channels, in_w) + or (num_batches, in_w, channels), or 4-D (num_batches, channels, in_h, in_w) or + (num_batches, in_h, in_w, channels), or a 5-D Tensor of the shape (num_batches, channels, in_d, in_h, in_w) or (num_batches, in_d, in_h, in_w, channels), Where in_w is width of the input tensor, in_h is the height of the input tensor, in_d is the depth of the input tensor. @@ -322,8 +323,9 @@ def interpolate( https://en.wikipedia.org/wiki/Bicubic_interpolation Parameters: - x (Tensor): 3-D, 4-D or 5-D Tensor, its data type is float32, float64, or uint8, - its data format is specified by :attr:`data_format`. + x (Tensor): 3-D, 4-D or 5-D Tensor, its data type is float32, float64, or uint8, its data format is + specified by :attr:`data_format`. If :attr:`data_format` is not provided, the data format will + be presumed according to its dimension. See details in :attr:`data_format`. size (list|tuple|Tensor|None): Output shape of image resize layer, the shape is (out_w, ) when input is a 3-D Tensor, the shape is (out_h, out_w) when input is a 4-D Tensor and is (out_d, out_h, out_w) when input is a 5-D Tensor. @@ -342,18 +344,21 @@ def interpolate( align_mode(int) : An optional for linear/bilinear/trilinear interpolation. Refer to the formula in the example above, it can be \'0\' for src_idx = scale_factor*(dst_index+0.5)-0.5 , can be \'1\' for src_idx = scale_factor*dst_index. - data_format (str, optional): Specify the data format of the input, and the data format of the output - will be consistent with that of the input. An optional string from:`NCW`, `NWC`, `"NCHW"`, `"NHWC"`, `"NCDHW"`, - `"NDHWC"`. The default is `"NCHW"`. When it is `"NCHW"`, the data is stored in the order of: - `[batch_size, input_channels, input_height, input_width]`. When it is `"NCHW"`, the data is stored - in the order of: `[batch_size, input_channels, input_depth, input_height, input_width]`. + data_format (str, optional): Specify the data format of the input, and the data format of + the output will be consistent with that of the input. An optional string from:`"NCW"`, + `"NWC"`, `"NCHW"`, `"NHWC"`, `"NCDHW"`, `"NDHWC"`. The default value is None. + When :attr:`data_format` is not specified, it will be automatically inferred from the + input dimension of :attr:`x`. When :attr:`x` is a 3-D Tensor, :attr:`data_format` will be + set to `"NCW"`; When :attr:`x` is a 4-D Tensor, :attr:`data_format` will be set to + `"NCHW"`; When :attr:`x` is a 5-D Tensor, :attr:`data_format` will be set to `"NCDHW"`. + When it is `"NCHW"`, the data should be stored in the order of: + `[batch_size, input_channels, input_height, input_width]`. When it is `"NCDHW"`, the + data should be stored in the order of: `[batch_size, input_channels, input_depth, input_height, input_width]`. name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name` Returns: - A 3-D Tensor of the shape (num_batches, channels, out_w) or (num_batches, out_w, channels), - A 4-D Tensor of the shape (num_batches, channels, out_h, out_w) or (num_batches, out_h, out_w, channels), - or 5-D Tensor of the shape (num_batches, channels, out_d, out_h, out_w) or (num_batches, out_d, out_h, out_w, channels). + A 3-D, 4-D or 5-D Tensor, with the same data format of the input :attr:`x`. Examples: @@ -375,6 +380,18 @@ def interpolate( >>> print(output_2.shape) [2, 3, 12, 10] """ + if data_format is None: + dim_size = len(x.shape) + if dim_size == 3: + data_format = 'NCW' + elif dim_size == 4: + data_format = 'NCHW' + elif dim_size == 5: + data_format = 'NCDHW' + else: + raise ValueError( + f"The dimension of the input tensor should only be 3-D, 4-D or 5-D, but the received dimension is {dim_size}." + ) data_format = data_format.upper() resample = mode.upper() resample_type = mode.lower() @@ -719,7 +736,7 @@ def upsample( mode='nearest', align_corners=False, align_mode=0, - data_format='NCHW', + data_format=None, name=None, ): """ @@ -727,7 +744,8 @@ def upsample( This API resizes a batch of images. The input must be a 3-D Tensor of the shape (num_batches, channels, in_w) - or 4-D (num_batches, channels, in_h, in_w), or a 5-D Tensor of the shape + or (num_batches, in_w, channels), or 4-D (num_batches, channels, in_h, in_w) or + (num_batches, in_h, in_w, channels), or a 5-D Tensor of the shape (num_batches, channels, in_d, in_h, in_w) or (num_batches, in_d, in_h, in_w, channels), Where in_w is width of the input tensor, in_h is the height of the input tensor, in_d is the depth of the input tensor. @@ -858,8 +876,9 @@ def upsample( https://en.wikipedia.org/wiki/Trilinear_interpolation. Parameters: - x (Tensor): 3-D, 4-D or 5-D Tensor, its data type is float32, float64, or uint8, - its data format is specified by :attr:`data_format`. + x (Tensor): 3-D, 4-D or 5-D Tensor, its data type is float32, float64, or uint8, its data format is + specified by :attr:`data_format`. If :attr:`data_format` is not provided, the data format will + be presumed according to its dimension. See details in :attr:`data_format`. size (list|tuple|Tensor|None, optional): Output shape of image resize layer, the shape is (out_w, ) when input is a 3-D Tensor, the shape is (out_h, out_w) when input is a 4-D Tensor and is (out_d, out_h, out_w) when input is a 5-D Tensor. @@ -879,19 +898,22 @@ def upsample( align_mode(int, optional) : An optional for linear/bilinear/trilinear interpolation. Refer to the formula in the example above, it can be \'0\' for src_idx = scale_factor*(dst_index+0.5)-0.5 , can be \'1\' for src_idx = scale_factor*dst_index. - data_format (str, optional): Specify the data format of the input, and the data format of the output - will be consistent with that of the input. An optional string from:`NCW`, `NWC`, `"NCHW"`, `"NHWC"`, `"NCDHW"`, - `"NDHWC"`. The default is `"NCHW"`. When it is `"NCHW"`, the data is stored in the order of: - `[batch_size, input_channels, input_height, input_width]`. When it is `"NCHW"`, the data is stored - in the order of: `[batch_size, input_channels, input_depth, input_height, input_width]`. + data_format (str, optional): Specify the data format of the input, and the data format of + the output will be consistent with that of the input. An optional string from:`"NCW"`, + `"NWC"`, `"NCHW"`, `"NHWC"`, `"NCDHW"`, `"NDHWC"`. The default value is None. + When :attr:`data_format` is not specified, it will be automatically inferred from the + input dimension of :attr:`x`. When :attr:`x` is a 3-D Tensor, :attr:`data_format` will be + set to `"NCW"`; When :attr:`x` is a 4-D Tensor, :attr:`data_format` will be set to + `"NCHW"`; When :attr:`x` is a 5-D Tensor, :attr:`data_format` will be set to `"NCDHW"`. + When it is `"NCHW"`, the data should be stored in the order of: + `[batch_size, input_channels, input_height, input_width]`. When it is `"NCDHW"`, the + data should be stored in the order of: `[batch_size, input_channels, input_depth, input_height, input_width]`. name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name` Returns: - A 3-D Tensor of the shape (num_batches, channels, out_w) or (num_batches, out_w, channels), - A 4-D Tensor of the shape (num_batches, channels, out_h, out_w) or (num_batches, out_h, out_w, channels), - or 5-D Tensor of the shape (num_batches, channels, out_d, out_h, out_w) or (num_batches, out_d, out_h, out_w, channels). + A 3-D, 4-D or 5-D Tensor, with the same data format of the input :attr:`x`. Examples: .. code-block:: python @@ -906,6 +928,7 @@ def upsample( [2, 3, 12, 12] """ + return interpolate( x, size, scale_factor, mode, align_corners, align_mode, data_format ) diff --git a/python/paddle/nn/layer/common.py b/python/paddle/nn/layer/common.py index 394fc3de6feb9..9dba25bb0043e 100644 --- a/python/paddle/nn/layer/common.py +++ b/python/paddle/nn/layer/common.py @@ -199,7 +199,8 @@ class Upsample(Layer): This op resizes a batch of images. The input must be a 3-D Tensor of the shape (num_batches, channels, in_w) - or 4-D (num_batches, channels, in_h, in_w), or a 5-D Tensor of the shape + or (num_batches, in_w, channels), or 4-D (num_batches, channels, in_h, in_w) or + (num_batches, in_h, in_w, channels), or a 5-D Tensor of the shape (num_batches, channels, in_d, in_h, in_w) or (num_batches, in_d, in_h, in_w, channels), Where in_w is width of the input tensor, in_h is the height of the input tensor, in_d is the depth of the input tensor. @@ -338,8 +339,7 @@ class Upsample(Layer): https://en.wikipedia.org/wiki/Trilinear_interpolation. Parameters: - x (Tensor): 3-D, 4-D or 5-D Tensor, its data type is float32, float64, or uint8, - its data format is specified by :attr:`data_format`. + size (list|tuple|Tensor|None): Output shape of image resize layer, the shape is (out_w, ) when input is a 3-D Tensor, the shape is (out_h, out_w) when input is a 4-D Tensor and is (out_d, out_h, out_w) when input is a 5-D Tensor. @@ -358,18 +358,21 @@ class Upsample(Layer): align_mode(int) : An optional for linear/bilinear/trilinear interpolation. Refer to the formula in the example above, it can be \'0\' for src_idx = scale_factor*(dst_indx+0.5)-0.5 , can be \'1\' for src_idx = scale_factor*dst_index. - data_format (str, optional): Specify the data format of the input, and the data format of the output - will be consistent with that of the input. An optional string from:`NCW`, `NWC`, `"NCHW"`, `"NHWC"`, `"NCDHW"`, - `"NDHWC"`. The default is `"NCHW"`. When it is `"NCHW"`, the data is stored in the order of: - `[batch_size, input_channels, input_height, input_width]`. When it is `"NCHW"`, the data is stored - in the order of: `[batch_size, input_channels, input_depth, input_height, input_width]`. + data_format (str, optional): Specify the data format of the input, and the data format of + the output will be consistent with that of the input. An optional string from:`"NCW"`, + `"NWC"`, `"NCHW"`, `"NHWC"`, `"NCDHW"`, `"NDHWC"`. The default value is None. + When :attr:`data_format` is not specified, it will be automatically inferred from the + input dimension of :attr:`x`. When :attr:`x` is a 3-D Tensor, :attr:`data_format` will be + set to `"NCW"`; When :attr:`x` is a 4-D Tensor, :attr:`data_format` will be set to + `"NCHW"`; When :attr:`x` is a 5-D Tensor, :attr:`data_format` will be set to `"NCDHW"`. + When it is `"NCHW"`, the data should be stored in the order of: + `[batch_size, input_channels, input_height, input_width]`. When it is `"NCDHW"`, the + data should be stored in the order of: `[batch_size, input_channels, input_depth, input_height, input_width]`. name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name` Returns: - A 3-D Tensor of the shape (num_batches, channels, out_w) or (num_batches, out_w, channels), - A 4-D Tensor of the shape (num_batches, channels, out_h, out_w) or (num_batches, out_h, out_w, channels), - or 5-D Tensor of the shape (num_batches, channels, out_d, out_h, out_w) or (num_batches, out_d, out_h, out_w, channels). + A callable object of Upsample. Examples: .. code-block:: python @@ -392,7 +395,7 @@ def __init__( mode='nearest', align_corners=False, align_mode=0, - data_format='NCHW', + data_format=None, name=None, ): super().__init__() @@ -405,6 +408,18 @@ def __init__( self.name = name def forward(self, x): + if self.data_format is None: + dim_size = len(x.shape) + if dim_size == 3: + self.data_format = 'NCW' + elif dim_size == 4: + self.data_format = 'NCHW' + elif dim_size == 5: + self.data_format = 'NCDHW' + else: + raise ValueError( + f"The dimension of the input tensor should only be 3-D, 4-D or 5-D, but the received dimension is {dim_size}." + ) out = F.interpolate( x, size=self.size, diff --git a/test/legacy_test/test_bilinear_interp_v2_op.py b/test/legacy_test/test_bilinear_interp_v2_op.py index 126cdaaf5da40..8e3061600b5a6 100755 --- a/test/legacy_test/test_bilinear_interp_v2_op.py +++ b/test/legacy_test/test_bilinear_interp_v2_op.py @@ -975,6 +975,29 @@ def test_case(self): np.testing.assert_allclose(out.numpy(), expect_res, rtol=1e-05) +class TestBilinearInterpOpAPI_dy5(unittest.TestCase): + def test_case(self): + import paddle + + if core.is_compiled_with_cuda(): + place = core.CUDAPlace(0) + else: + place = core.CPUPlace() + with base.dygraph.guard(place): + input_data = np.random.random((2, 3, 6, 6)).astype("float32") + scale_np = np.array([2, 2]).astype("int64") + input_x = paddle.to_tensor(input_data) + scale = paddle.to_tensor(scale_np) + expect_res = bilinear_interp_np( + input_data, out_h=12, out_w=12, align_corners=False + ) + up_layer = paddle.nn.Upsample( + scale_factor=scale, mode="bilinear", align_corners=False + ) + out = up_layer(input_x) + np.testing.assert_allclose(out.numpy(), expect_res, rtol=1e-05) + + @unittest.skipIf( not base.core.is_compiled_with_cuda(), "core is not compiled with CUDA" ) diff --git a/test/legacy_test/test_imperative_layers.py b/test/legacy_test/test_imperative_layers.py index b8c0445c9760e..9906d3ba0ede0 100644 --- a/test/legacy_test/test_imperative_layers.py +++ b/test/legacy_test/test_imperative_layers.py @@ -108,7 +108,7 @@ def test_layer_str(self): module = nn.Upsample(size=[12, 12]) self.assertEqual( str(module), - 'Upsample(size=[12, 12], mode=nearest, align_corners=False, align_mode=0, data_format=NCHW)', + 'Upsample(size=[12, 12], mode=nearest, align_corners=False, align_mode=0, data_format=None)', ) module = nn.UpsamplingNearest2D(size=[12, 12]) diff --git a/test/legacy_test/test_linear_interp_v2_op.py b/test/legacy_test/test_linear_interp_v2_op.py index 8aac88d69f97a..b6a37f4500b00 100755 --- a/test/legacy_test/test_linear_interp_v2_op.py +++ b/test/legacy_test/test_linear_interp_v2_op.py @@ -333,7 +333,6 @@ def test_case(self): mode='linear', align_mode=1, align_corners=False, - data_format='NCW', ) with base.dygraph.guard(): x = paddle.to_tensor(x_data) @@ -346,6 +345,26 @@ def test_case(self): np.testing.assert_allclose(interp.numpy(), expect, rtol=1e-05) +class TestLinearInterpOpAPI2_0_case2(unittest.TestCase): + def test_case(self): + # dygraph + x_data = np.random.random((1, 3, 128)).astype("float32") + with base.dygraph.guard(): + x = paddle.to_tensor(x_data) + interp = interpolate( + x, + size=[64], + mode='linear', + align_mode=1, + align_corners=False, + ) + expect = linear_interp_np( + x_data, out_w=64, align_mode=1, align_corners=False + ) + + np.testing.assert_allclose(interp.numpy(), expect, rtol=1e-05) + + class TestLinearInterpOpFP16(TestLinearInterpOp): def test_check_output(self): self.check_output(atol=1e-3, check_pir=True) diff --git a/test/legacy_test/test_trilinear_interp_v2_op.py b/test/legacy_test/test_trilinear_interp_v2_op.py index 45511da5754b0..4c30a3e6496d8 100755 --- a/test/legacy_test/test_trilinear_interp_v2_op.py +++ b/test/legacy_test/test_trilinear_interp_v2_op.py @@ -1021,5 +1021,53 @@ def init_test_case(self): self.data_layout = "NDHWC" +class TestTrilinearInterpOpAPI(unittest.TestCase): + def test_case(self): + import paddle + + if core.is_compiled_with_cuda(): + place = core.CUDAPlace(0) + else: + place = core.CPUPlace() + with base.dygraph.guard(place): + input_data = np.random.random((2, 3, 6, 6, 6)).astype("float32") + scale_np = np.array([2, 2, 2]).astype("int64") + input_x = paddle.to_tensor(input_data) + scale = paddle.to_tensor(scale_np) + expect_res = trilinear_interp_np( + input_data, out_d=12, out_h=12, out_w=12, align_corners=False + ) + up_layer = paddle.nn.Upsample( + scale_factor=scale, mode="trilinear", align_corners=False + ) + out = up_layer(input_x) + np.testing.assert_allclose(out.numpy(), expect_res, rtol=1e-05) + + +class TestTrilinearInterpOpAPI2(unittest.TestCase): + def test_case(self): + import paddle + + if core.is_compiled_with_cuda(): + place = core.CUDAPlace(0) + else: + place = core.CPUPlace() + with base.dygraph.guard(place): + input_data = np.random.random((2, 3, 6, 6, 6)).astype("float32") + scale_np = np.array([2, 2, 2]).astype("int64") + input_x = paddle.to_tensor(input_data) + scale = paddle.to_tensor(scale_np) + expect_res = trilinear_interp_np( + input_data, out_d=12, out_h=12, out_w=12, align_corners=False + ) + out = interpolate( + x=input_x, + scale_factor=scale, + mode="trilinear", + align_corners=False, + ) + np.testing.assert_allclose(out.numpy(), expect_res, rtol=1e-05) + + if __name__ == "__main__": unittest.main() From 1c5cd2795a1df84cfc6746b7a0c2d0a5158a47cf Mon Sep 17 00:00:00 2001 From: xiaoguoguo626807 <100397923+xiaoguoguo626807@users.noreply.github.com> Date: Tue, 27 Feb 2024 12:35:48 +0800 Subject: [PATCH 115/282] modify if nest pop_to_push_map (#62093) * modify if nest pop_to_push_map * Apply suggestions from code review --- python/paddle/autograd/ir_backward.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/python/paddle/autograd/ir_backward.py b/python/paddle/autograd/ir_backward.py index 042a541eb69f5..18f5054921ab7 100644 --- a/python/paddle/autograd/ir_backward.py +++ b/python/paddle/autograd/ir_backward.py @@ -264,6 +264,7 @@ def append_backward_ops( backward_ops, state, bwd_value_to_block_argument_map=ValueDict(), + control_flow_value_to_copyvalue_map=ValueDict(), ): ''' add grad_op in order of topological inverse sort @@ -558,11 +559,6 @@ def append_yield( # [builtin.combine , op3 , builtin.split] (op3's one input and one output are vectorType) # [op4] (op4's inputs and outputs are not vectorType) - # -----------------only for control flow-----------------# - # tuple_push value to pop value - control_flow_value_to_copyvalue_map = ValueDict() - control_flow_copyvalue_to_value_map = ValueDict() - if ( len(effective_forward_ops) > 1 and effective_forward_ops[-1].name() == "cf.yield" @@ -624,9 +620,6 @@ def append_yield( control_flow_value_to_copyvalue_map[ output[0] ] = copy_output[0] - control_flow_copyvalue_to_value_map[ - copy_output[0] - ] = output[0] else: # all(zero_flag) support this op has no contribution for grad # should be delete (prune sub_graph) @@ -671,6 +664,9 @@ def append_yield( ) sub_backward_ops = [] + sub_control_flow_value_to_copyvalue_map = ( + control_flow_value_to_copyvalue_map.copy() + ) append_backward_ops( op, [input[0] for input in inputs[1:]], @@ -681,6 +677,7 @@ def append_yield( no_grad_set, sub_backward_ops, sub_state, + control_flow_value_to_copyvalue_map=sub_control_flow_value_to_copyvalue_map, ) for input_tuple in inputs_used_by_other_op: state.value_to_valuegrad[ @@ -747,6 +744,10 @@ def append_yield( sub_bwd_value_to_block_argument_map.update( bwd_value_to_block_argument_map ) + sub_control_flow_value_to_copyvalue_map = ( + control_flow_value_to_copyvalue_map.copy() + ) + while_grad_block = grad_op.as_while_op().body() sub_backward_ops = [] append_backward_ops( @@ -760,6 +761,7 @@ def append_yield( sub_backward_ops, sub_state, sub_bwd_value_to_block_argument_map, + sub_control_flow_value_to_copyvalue_map, ) # update input_grad map update_input_grad_map(op, input_grads, origin_inputs) @@ -916,7 +918,6 @@ def calc_gradient_helper(outputs, inputs, grad_outputs, no_grad_set): state, ValueDict(), ) - # now value_to_valuegrad should be value <-> value (add sum op for the same values's grad value) outputs_set, inputs_set, no_gradvar_set = create_backward_prune_set( outputs_fwd_set, inputs_fwd_set, no_grad_set, state From 20a11fc18bc47a58a0cd692b3d2cc3ea4dfb28df Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Tue, 27 Feb 2024 14:03:12 +0800 Subject: [PATCH 116/282] [PHI] Fix a typo in infermeta error message (`bug` -> `but`) (#62108) --- paddle/phi/infermeta/multiary.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/paddle/phi/infermeta/multiary.cc b/paddle/phi/infermeta/multiary.cc index 69214508ef3f9..b091793ac5665 100644 --- a/paddle/phi/infermeta/multiary.cc +++ b/paddle/phi/infermeta/multiary.cc @@ -2208,7 +2208,7 @@ static void Interpolate1DInferShapeCheck( dim_x[i], 0, phi::errors::InvalidArgument("The shape of input(x) should be larger " - "than 0, bug received shape[%d] is %d ", + "than 0, but received shape[%d] is %d ", i, dim_x[i])); } @@ -2340,7 +2340,7 @@ static void Interpolate2DInferShapeCheck( dim_x[i], 0, phi::errors::InvalidArgument("The shape of input(x) should be larger " - "than 0, bug received shape[%d] is %d ", + "than 0, but received shape[%d] is %d ", i, dim_x[i])); } @@ -2493,7 +2493,7 @@ static void Interpolate3DInferShapeCheck( dim_x[i], 0, phi::errors::InvalidArgument("The shape of input(x) should be larger " - "than 0, bug received shape[%d] is %d ", + "than 0, but received shape[%d] is %d ", i, dim_x[i])); } From 46a60bf4af911bf62d5b593bff922ae0ff3023e1 Mon Sep 17 00:00:00 2001 From: HydrogenSulfate <490868991@qq.com> Date: Tue, 27 Feb 2024 14:13:57 +0800 Subject: [PATCH 117/282] [Prim] Optimize composite OP relu grad (#62059) * use cast instead of where to optimize composite OP relu_grad * Update composite_backward_api.h Revert change of silu_grad --- .../prim/api/composite_backward/composite_backward_api.h | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h index 1d9fbf176b664..7131d37dd5496 100644 --- a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h +++ b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h @@ -87,12 +87,9 @@ void silu_grad(const Tensor& x, template void relu_grad(const Tensor& out, const Tensor& out_grad, Tensor* x_grad) { if (x_grad) { - auto condition = greater_than( + auto mask = greater_than( out, full(common::vectorize(out.dims()), 0.0, out.dtype())); - auto res = - where(condition, - out_grad, - full(common::vectorize(out.dims()), 0.0, out.dtype())); + auto res = cast(mask, out.dtype()) * out_grad; set_output(res, x_grad); } } @@ -275,7 +272,6 @@ void add_grad(const Tensor& x, auto dy_tmp = reshape(dy_reduce_res, common::vectorize(y.dims())); set_output(dy_tmp, dy); } - } else { by_pass(out_grad, dy); } From 7071309a9af94d8227a8e4208e59d701ea7e0797 Mon Sep 17 00:00:00 2001 From: wawltor Date: Tue, 27 Feb 2024 14:25:06 +0800 Subject: [PATCH 118/282] fix the unqiue op that generate the wrong the inreverse result (#62103) --- paddle/phi/kernels/gpu/unique_kernel.cu | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/paddle/phi/kernels/gpu/unique_kernel.cu b/paddle/phi/kernels/gpu/unique_kernel.cu index 18b23dcab1c39..944502851db5d 100644 --- a/paddle/phi/kernels/gpu/unique_kernel.cu +++ b/paddle/phi/kernels/gpu/unique_kernel.cu @@ -173,8 +173,11 @@ UniqueFlattendCUDATensor(const Context& context, #ifdef PADDLE_WITH_HIP hipMemset(inv_loc_data_ptr, 0, sizeof(IndexT)); #else - cudaMemsetAsync(inv_loc_data_ptr, 0, sizeof(IndexT), context.stream()); + thrust::device_ptr inv_loc_data_dev(inv_loc_data_ptr); + inv_loc_data_dev[0] = 0; // without device_ptr, segmentation fault #endif + +#ifdef PADDLE_WITH_HIP size_t temp_storage_bytes = 0; cub::DeviceScan::InclusiveSum(NULL, temp_storage_bytes, @@ -190,6 +193,12 @@ UniqueFlattendCUDATensor(const Context& context, inv_loc_data_ptr, num_input, context.stream()); +#else + thrust::inclusive_scan(exec_policy, + inv_loc_data_ptr, + inv_loc_data_ptr + num_input, + inv_loc_data_ptr); +#endif thrust::scatter(exec_policy, inv_loc_data_ptr, inv_loc_data_ptr + num_input, From 4f0bf673e8e3122f5489314589f134c33cee2125 Mon Sep 17 00:00:00 2001 From: Bo Zhang <105368690+zhangbopd@users.noreply.github.com> Date: Tue, 27 Feb 2024 14:37:24 +0800 Subject: [PATCH 119/282] [PIR][DynamicShape] Fix test_llama_if_dy (#62095) * fix test_llama_if_dy * fix typo --- .../paddle_op_infer_sym.cc | 26 +++++++++++++++++++ test/ir/pir/cinn/symbolic/CMakeLists.txt | 5 ++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc index c75cc7d593688..1bbdf90d67fa4 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc @@ -181,6 +181,17 @@ bool ProdOpInferSymbolicShape(pir::Operation *op, bool ReshapeOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + pir::Value operand_source = op->operand_source(0); + if (shape_analysis->GetShapeOrDataForValue(operand_source) + .data() + .has_value()) { + const symbol::ShapeOrDataDimExprs &operand_shape_or_data = + shape_analysis->GetShapeOrDataForValue(operand_source); + shape_analysis->SetShapeOrDataForValue(op->result(0), + operand_shape_or_data); + return true; + } + pir::Value operand_source_shape = op->operand_source(1); const symbol::ShapeOrDataDimExprs &operand_shape_or_data = @@ -444,6 +455,21 @@ bool ConcatOpInferSymbolicShape( size_t rank = shape_data_list[0].shape().size(); axis = axis >= 0 ? axis : std::max(int64_t(0), int64_t(axis + rank)); + if (shape_data_list[0].data().has_value()) { + std::vector data; + data.reserve(shape_data_list.size()); + for (auto &data_elem : shape_data_list) { + data.push_back(data_elem.data().value()[0]); + } + const std::vector shape{std::int64_t(data.size())}; + symbol::ShapeOrDataDimExprs shape_data{ + symbol::TensorShapeOrDataDimExprs(shape, data)}; + pir::Value res = op->result(0); + shape_analysis->SetShapeOrDataForValue(res, shape_data); + + return true; + } + const std::vector &out_dims = [&] { std::vector out_dims = shape_data_list[0].shape(); for (size_t i = 0; i < rank; ++i) { diff --git a/test/ir/pir/cinn/symbolic/CMakeLists.txt b/test/ir/pir/cinn/symbolic/CMakeLists.txt index 8d9463d870fda..665d1a0b0461d 100644 --- a/test/ir/pir/cinn/symbolic/CMakeLists.txt +++ b/test/ir/pir/cinn/symbolic/CMakeLists.txt @@ -64,8 +64,9 @@ if(WITH_GPU) ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/python/:$ENV{PYTHONPATH} FLAGS_prim_all=true FLAGS_cinn_bucket_compile=false - FLAGS_pir_apply_shape_optimization_pass=0 FLAGS_enable_pir_api=1 - ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_llama_if_dy.py + FLAGS_pir_apply_shape_optimization_pass=true FLAGS_enable_pir_api=true + FLAGS_prim_enable_dynamic=true ${PYTHON_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/test_llama_if_dy.py WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_tests_properties(test_llama_if_dy PROPERTIES LABELS "RUN_TYPE=CINN") From c819304ff3279c5504e7ecfbe8f4a3f8cd9640a5 Mon Sep 17 00:00:00 2001 From: HydrogenSulfate <490868991@qq.com> Date: Tue, 27 Feb 2024 14:54:31 +0800 Subject: [PATCH 120/282] [Prim] Optimize composite OP subtract_double_grad (#61912) * optimize subtract_double_grad * Update composite_double_backward_api.h Fix --- .../composite_double_backward_api.h | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/paddle/fluid/prim/api/composite_backward/composite_double_backward_api.h b/paddle/fluid/prim/api/composite_backward/composite_double_backward_api.h index 8e0827eeb0b7e..02bd7e29443c0 100644 --- a/paddle/fluid/prim/api/composite_backward/composite_double_backward_api.h +++ b/paddle/fluid/prim/api/composite_backward/composite_double_backward_api.h @@ -609,18 +609,14 @@ void subtract_double_grad(const Tensor& y, Tensor* grad_out_grad) { if (grad_out_grad) { // ddout = ddx - ddy - if (!grad_x_grad && !grad_y_grad) { - grad_out_grad = nullptr; + if (grad_x_grad && grad_y_grad) { + set_output(grad_x_grad.get() - grad_y_grad.get(), grad_out_grad); + } else if (grad_x_grad) { + set_output(grad_x_grad.get(), grad_out_grad); + } else if (grad_y_grad) { + set_output(-grad_y_grad.get(), grad_out_grad); } else { - Tensor ddout = - full(common::vectorize(grad_out.dims()), 0.0, y.dtype()); - if (grad_x_grad) { - ddout = ddout + grad_x_grad.get(); - } - if (grad_y_grad) { - ddout = ddout - grad_y_grad.get(); - } - set_output(ddout, grad_out_grad); + grad_out_grad = nullptr; } } } From 579a12c15f7a2f708699bea97a67e8b9bf715005 Mon Sep 17 00:00:00 2001 From: danleifeng <52735331+danleifeng@users.noreply.github.com> Date: Tue, 27 Feb 2024 15:10:41 +0800 Subject: [PATCH 121/282] fix test_communicator_half_async random core;test=develop (#62078) --- .../run_server_for_communicator_half_async.py | 38 ++++++ .../fleet/test_communicator_half_async.py | 118 +++++++++--------- 2 files changed, 96 insertions(+), 60 deletions(-) create mode 100644 test/collective/fleet/run_server_for_communicator_half_async.py diff --git a/test/collective/fleet/run_server_for_communicator_half_async.py b/test/collective/fleet/run_server_for_communicator_half_async.py new file mode 100644 index 0000000000000..14d8fd80331b3 --- /dev/null +++ b/test/collective/fleet/run_server_for_communicator_half_async.py @@ -0,0 +1,38 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from test_communicator_half_async import TestCommunicatorHalfAsyncEnd2End + +import paddle + +paddle.enable_static() + +pipe_name = os.getenv("PIPE_FILE") + + +class RunServer(TestCommunicatorHalfAsyncEnd2End): + def runTest(self): + pass + + +os.environ["TRAINING_ROLE"] = "PSERVER" +os.environ["http_proxy"] = "" +os.environ["https_proxy"] = "" +half_run_server = RunServer() +with open(pipe_name, 'w') as pipe: + pipe.write('done') + +half_run_server.run_ut() diff --git a/test/collective/fleet/test_communicator_half_async.py b/test/collective/fleet/test_communicator_half_async.py index 25e5302fb444f..687337f25ab2a 100644 --- a/test/collective/fleet/test_communicator_half_async.py +++ b/test/collective/fleet/test_communicator_half_async.py @@ -15,6 +15,7 @@ import os import subprocess import sys +import tempfile import unittest import numpy @@ -23,6 +24,7 @@ from paddle import base from paddle.distributed import fleet from paddle.distributed.fleet.base import role_maker +from paddle.distributed.utils.launch_utils import find_free_ports paddle.enable_static() @@ -30,25 +32,44 @@ class TestCommunicatorHalfAsyncEnd2End(unittest.TestCase): def net(self): x = paddle.static.data(name='x', shape=[-1, 13], dtype='float32') - y_predict = paddle.static.nn.fc(x, size=1, activation=None) - y = paddle.static.data(name='y', shape=[-1, 1], dtype='float32') + x1 = paddle.static.data( + name='x1', shape=[-1, 1], dtype='int64', lod_level=1 + ) + emb = paddle.static.nn.embedding( + input=x1, + size=[10000, 10], + param_attr=base.ParamAttr( + name="embedding", + initializer=paddle.nn.initializer.Constant(value=0.01), + ), + is_sparse=True, + ) + + pool = paddle.static.nn.sequence_lod.sequence_pool( + input=emb.squeeze(-2), pool_type="sum" + ) + z = paddle.concat([x, pool], axis=1) + + y_predict = paddle.static.nn.fc(x=z, size=1) + y = paddle.static.data(name='y', shape=[-1, 1], dtype='float32') cost = paddle.nn.functional.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) - return avg_cost, x, y + return avg_cost, x, x1, y def fake_reader(self): def reader(): for i in range(10000): x = numpy.random.random((1, 13)).astype('float32') + z = numpy.random.randint(0, 9999, (1, 1)).astype('int64') y = numpy.random.randint(0, 2, (1, 1)).astype('int64') - yield x, y + yield x, z, y return reader def run_pserver(self, role, strategy): fleet.init(role) - avg_cost, x, y = self.net() + avg_cost, x, z, y = self.net() optimizer = paddle.optimizer.SGD(0.01) optimizer = fleet.distributed_optimizer(optimizer, strategy) optimizer.minimize(avg_cost) @@ -61,20 +82,20 @@ def run_trainer(self, role, strategy): exe = base.Executor(place) fleet.init(role) - avg_cost, x, y = self.net() + avg_cost, x, z, y = self.net() optimizer = paddle.optimizer.SGD(0.01) optimizer = fleet.distributed_optimizer(optimizer, strategy) optimizer.minimize(avg_cost) - exe.run(paddle.static.default_startup_program()) + exe.run(base.default_startup_program()) fleet.init_worker() train_reader = paddle.batch(self.fake_reader(), batch_size=24) - feeder = base.DataFeeder(place=place, feed_list=[x, y]) + feeder = base.DataFeeder(place=place, feed_list=[x, z, y]) for batch_id, data in enumerate(train_reader()): exe.run( - paddle.static.default_main_program(), + base.default_main_program(), feed=feeder.feed(data), fetch_list=[], ) @@ -82,19 +103,18 @@ def run_trainer(self, role, strategy): fleet.stop_worker() def run_ut(self): - strategy = paddle.distributed.fleet.DistributedStrategy() - strategy.a_sync = True - training_role = os.getenv("TRAINING_ROLE", "TRAINER") - role = role_maker.UserDefinedRoleMaker( - current_id=0, - role=role_maker.Role.WORKER - if training_role == "TRAINER" - else role_maker.Role.SERVER, - worker_num=1, - server_endpoints=["127.0.0.1:6002"], - ) + os.environ["PADDLE_PSERVER_NUMS"] = "1" + os.environ["PADDLE_TRAINERS_NUM"] = "1" + os.environ["PADDLE_TRAINER_ID"] = "0" + os.environ["PADDLE_TRAINERS_NUM"] = "1" + os.environ["POD_IP"] = "127.0.0.1" + + role = role_maker.PaddleCloudRoleMaker() + + strategy = paddle.distributed.fleet.DistributedStrategy() + strategy.a_sync = True if training_role == "TRAINER": self.run_trainer(role, strategy) @@ -102,61 +122,39 @@ def run_ut(self): self.run_pserver(role, strategy) def test_communicator(self): - run_server_cmd = """ + temp_dir = tempfile.TemporaryDirectory() + pipe_name = os.path.join(temp_dir.name, 'mypipe') + try: + os.mkfifo(pipe_name) + except OSError as oe: + print(f"Failed to create pipe: {oe}") -import sys -import os + port = find_free_ports(1).pop() -import time -import threading -import subprocess -import unittest -import numpy - -from test_communicator_half_async import TestCommunicatorHalfAsyncEnd2End - -import paddle -import paddle.base as base -import paddle.distributed.fleet as fleet -import paddle.distributed.fleet.base.role_maker as role_maker - -paddle.enable_static() - -class RunServer(TestCommunicatorHalfAsyncEnd2End): - def runTest(self): - pass - -os.environ["http_proxy"] = "" -os.environ["https_proxy"] = "" -os.environ["TRAINING_ROLE"] = "PSERVER" -half_run_server = RunServer() -half_run_server.run_ut() -""" - - server_file = "run_server_for_communicator_haflaysnc.py" - with open(server_file, "w") as wb: - wb.write(run_server_cmd) os.environ["TRAINING_ROLE"] = "PSERVER" - _python = sys.executable + os.environ["PADDLE_PORT"] = str(port) + os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = f"127.0.0.1:{port}" + os.environ["PIPE_FILE"] = pipe_name + _python = sys.executable + server_file = "run_server_for_communicator_half_async.py" ps_cmd = f"{_python} {server_file}" + ps_proc = subprocess.Popen( ps_cmd.strip().split(" "), stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) - os.environ["http_proxy"] = "" - os.environ["https_proxy"] = "" + with open(pipe_name, 'r') as pipe: + start_command = pipe.read() + os.environ["TRAINING_ROLE"] = "TRAINER" - os.environ["FLAGS_communicator_send_queue_size"] = "1" - os.environ["FLAGS_communicator_max_merge_var_num"] = "1" self.run_ut() ps_proc.kill() - - if os.path.exists(server_file): - os.remove(server_file) + ps_proc.wait() + outs, errs = ps_proc.communicate() if __name__ == '__main__': From 5d07c26f2d9a03f7b105d94ebd64b20f89bb18b0 Mon Sep 17 00:00:00 2001 From: WangZhen <23097963+0x45f@users.noreply.github.com> Date: Tue, 27 Feb 2024 15:15:06 +0800 Subject: [PATCH 122/282] [PIR AMP]Adapt auto_cast api for PIR AMP (#61859) --- python/paddle/amp/auto_cast.py | 295 +++++++++++++++++++-------------- test/amp/test_pir_amp.py | 27 +-- 2 files changed, 176 insertions(+), 146 deletions(-) diff --git a/python/paddle/amp/auto_cast.py b/python/paddle/amp/auto_cast.py index 1c84473a1e357..0286a668d10f5 100644 --- a/python/paddle/amp/auto_cast.py +++ b/python/paddle/amp/auto_cast.py @@ -17,7 +17,12 @@ import paddle from paddle.base import core -from paddle.base.framework import _dygraph_tracer, dygraph_only +from paddle.base.framework import ( + _dygraph_tracer, + dygraph_only, + in_dynamic_or_pir_mode, + in_pir_mode, +) from paddle.base.wrapped_decorator import signature_safe_contextmanager from .amp_lists import black_list, white_list @@ -271,7 +276,6 @@ def check_optimizers(optimizers): @signature_safe_contextmanager -@dygraph_only def amp_guard( enable=True, custom_white_list=None, @@ -325,6 +329,10 @@ def amp_guard( paddle.float32 >>> # doctest: -SKIP """ + assert ( + in_dynamic_or_pir_mode() + ), "We only support 'amp_guard' in dynamic or pir mode." + amp_state = locals() global _g_amp_state_ original_state = _g_amp_state_ @@ -343,59 +351,6 @@ def amp_guard( "If enable amp, dtype should be 'float16' or 'bfloat16'." ) - # check tracer - tracer = _dygraph_tracer() - if not tracer: - raise ValueError( - "current_tracer is None, maybe it is not in imperative mode." - ) - - # check device_type: - # NOTE: Now, amp only support gpu for float16 and bfloat16, xpu for float16, npu for float16. - # Maybe we will support cpu for bfloat16. - if enable and not ( - tracer._expected_place.is_gpu_place() - or tracer._expected_place.is_xpu_place() - or tracer._expected_place.is_custom_place() - ): - warnings.warn( - 'amp_guard can only be enabled on CUDAPlace, XPUPlace, and CustomPlace, current place is %s, so it makes no effect.' - % tracer._expected_place - ) - enable = False - if enable: - # For xpu: - if tracer._expected_place.is_xpu_place() and (dtype == 'bfloat16'): - warnings.warn('XPUPlace only support float16 amp.') - enable = False - # For custom device: - if tracer._expected_place.is_custom_place() and (dtype == 'bfloat16'): - warnings.warn('CustomPlace only support float16 amp.') - enable = False - # For gpu float16: Compute Capability should >= 7. - # For gpu bfloat16: Compute Capability should >= 8 & CUDA Version should >= 11. - if tracer._expected_place.is_gpu_place(): - if (dtype == 'float16') and not _is_gpu_float16_supported(): - prop = paddle.device.cuda.get_device_capability() - warnings.warn( - "For float16, amp only support NVIDIA GPU with Compute Capability 7.0 or higher, current GPU is: %s, with Compute Capability: %d.%d." - % (paddle.device.cuda.get_device_name(), prop[0], prop[1]) - ) - enable = False - elif (dtype == 'bfloat16') and not _is_gpu_bfloat16_supported(): - prop = paddle.device.cuda.get_device_capability() - cuda_version = paddle.version.cuda() - warnings.warn( - "For bfloat16, amp only support NVIDIA GPU with Compute Capability 8.0 or higher and CUDA Version 11.0 or higher, current GPU is: %s, with Compute Capability: %d.%d, current CUDA Version is: %s." - % ( - paddle.device.cuda.get_device_name(), - prop[0], - prop[1], - cuda_version, - ) - ) - enable = False - amp_dtype = dtype amp_global_state().amp_dtype = amp_dtype @@ -412,87 +367,179 @@ def amp_guard( custom_white_list, custom_black_list, level, dtype ) - if not enable: - amp_level = AMP_LEVEL.O0 - amp_dtype = "float32" - - # master_grad_hook will run at the end of backward. - # Since backward_final_hook will be cleared once they have been - # done, we should register the hook every step. - if ( - amp_global_state().use_master_grad - and not amp_global_state().already_register_final_backward_hook - ): + if in_pir_mode(): + if not enable: + amp_level = AMP_LEVEL.O0 + amp_dtype = "float32" + amp_attrs = core._get_amp_attrs() + # set amp level + original_amp_level = amp_attrs._amp_level + amp_attrs._amp_level = amp_level + # set amp op list + original_white_list, original_black_list = core._get_amp_op_list() + core._set_amp_op_list(_white_list, _black_list) + # set amp dtype + original_amp_dtype = amp_attrs._amp_dtype + amp_attrs._amp_dtype = amp_dtype + # switch promote + if amp_level == AMP_LEVEL.O2: + original_use_promote = amp_attrs._use_promote + amp_attrs._use_promote = use_promote - def master_grad_hook(): - # NOTE(lizhiyu): To support semi-auto of dygraph mode, we must - # classify the params of model into different calsses according to their process_mesh. - # Otherwise, fault will occur. - if not amp_global_state().already_classify_params_meshs: - for param in amp_global_state().model_parameters: - if param is not None and param.process_mesh is not None: - if ( - param.process_mesh - not in amp_global_state().mesh2params - ): - amp_global_state().mesh2params[ - param.process_mesh - ] = [param] - else: - amp_global_state().mesh2params[ - param.process_mesh - ].append(param) - amp_global_state().already_classify_params_meshs = True + try: + yield + finally: + _g_amp_state_ = original_state + amp_attrs._amp_level = original_amp_level + core._set_amp_op_list(original_white_list, original_black_list) + amp_attrs._amp_dtype = original_amp_dtype + if amp_level == AMP_LEVEL.O2: + amp_attrs._use_promote = original_use_promote - if len(amp_global_state().mesh2params): - for _, params in amp_global_state().mesh2params.items(): - core.eager.set_master_grads(params) - else: - core.eager.set_master_grads(amp_global_state().model_parameters) + else: + # check tracer + tracer = _dygraph_tracer() + if not tracer: + raise ValueError( + "current_tracer is None, maybe it is not in imperative mode." + ) + # check device_type: + # NOTE: Now, amp only support gpu for float16 and bfloat16, xpu for float16, npu for float16. + # Maybe we will support cpu for bfloat16. + if enable and not ( + tracer._expected_place.is_gpu_place() + or tracer._expected_place.is_xpu_place() + or tracer._expected_place.is_custom_place() + ): + warnings.warn( + 'amp_guard can only be enabled on CUDAPlace, XPUPlace, and CustomPlace, current place is %s, so it makes no effect.' + % tracer._expected_place + ) + enable = False + if enable: + # For xpu: + if tracer._expected_place.is_xpu_place() and (dtype == 'bfloat16'): + warnings.warn('XPUPlace only support float16 amp.') + enable = False + # For custom device: + if tracer._expected_place.is_custom_place() and ( + dtype == 'bfloat16' + ): + warnings.warn('CustomPlace only support float16 amp.') + enable = False + # For gpu float16: Compute Capability should >= 7. + # For gpu bfloat16: Compute Capability should >= 8 & CUDA Version should >= 11. + if tracer._expected_place.is_gpu_place(): + if (dtype == 'float16') and not _is_gpu_float16_supported(): + prop = paddle.device.cuda.get_device_capability() + warnings.warn( + "For float16, amp only support NVIDIA GPU with Compute Capability 7.0 or higher, current GPU is: %s, with Compute Capability: %d.%d." + % ( + paddle.device.cuda.get_device_name(), + prop[0], + prop[1], + ) + ) + enable = False + elif (dtype == 'bfloat16') and not _is_gpu_bfloat16_supported(): + prop = paddle.device.cuda.get_device_capability() + cuda_version = paddle.version.cuda() + warnings.warn( + "For bfloat16, amp only support NVIDIA GPU with Compute Capability 8.0 or higher and CUDA Version 11.0 or higher, current GPU is: %s, with Compute Capability: %d.%d, current CUDA Version is: %s." + % ( + paddle.device.cuda.get_device_name(), + prop[0], + prop[1], + cuda_version, + ) + ) + enable = False + + if not enable: + amp_level = AMP_LEVEL.O0 + amp_dtype = "float32" + + # master_grad_hook will run at the end of backward. + # Since backward_final_hook will be cleared once they have been + # done, we should register the hook every step. + if ( + amp_global_state().use_master_grad + and not amp_global_state().already_register_final_backward_hook + ): - amp_global_state().already_register_final_backward_hook = False + def master_grad_hook(): + # NOTE(lizhiyu): To support semi-auto of dygraph mode, we must + # classify the params of model into different calsses according to their process_mesh. + # Otherwise, fault will occur. + if not amp_global_state().already_classify_params_meshs: + for param in amp_global_state().model_parameters: + if param is not None and param.process_mesh is not None: + if ( + param.process_mesh + not in amp_global_state().mesh2params + ): + amp_global_state().mesh2params[ + param.process_mesh + ] = [param] + else: + amp_global_state().mesh2params[ + param.process_mesh + ].append(param) + amp_global_state().already_classify_params_meshs = True + + if len(amp_global_state().mesh2params): + for _, params in amp_global_state().mesh2params.items(): + core.eager.set_master_grads(params) + else: + core.eager.set_master_grads( + amp_global_state().model_parameters + ) - core.eager._add_backward_final_hook(master_grad_hook) - amp_global_state().already_register_final_backward_hook = True + amp_global_state().already_register_final_backward_hook = False - if tracer: - # enable auto_cast - original_amp_level = tracer._amp_level - tracer._amp_level = amp_level + core.eager._add_backward_final_hook(master_grad_hook) + amp_global_state().already_register_final_backward_hook = True - # set amp op list - original_white_list, original_black_list = tracer._get_amp_op_list() - tracer._set_amp_op_list(_white_list, _black_list) + if tracer: + # enable auto_cast + original_amp_level = tracer._amp_level + tracer._amp_level = amp_level - # TODO(zhiqiu) set amp related flags automatically in this guard - # Currently, if FLAGS_cudnn_batchnorm_spatial_persistent is set True in amp_guard, - # batch_norm can run in fast mode, but batch_norm_grad can not if backward if not executed inside amp_guard. - # So, users need to set related flags manually. + # set amp op list + original_white_list, original_black_list = tracer._get_amp_op_list() + tracer._set_amp_op_list(_white_list, _black_list) - # original_flags = get_flags(AMP_RELATED_FLAGS) - # set_flags(AMP_RELATED_FLAGS_SETTING) + # TODO(zhiqiu) set amp related flags automatically in this guard + # Currently, if FLAGS_cudnn_batchnorm_spatial_persistent is set True in amp_guard, + # batch_norm can run in fast mode, but batch_norm_grad can not if backward if not executed inside amp_guard. + # So, users need to set related flags manually. - # set amp dtype - original_amp_dtype = tracer._amp_dtype - tracer._amp_dtype = amp_dtype + # original_flags = get_flags(AMP_RELATED_FLAGS) + # set_flags(AMP_RELATED_FLAGS_SETTING) - # switch promote - if amp_level == AMP_LEVEL.O2: - original_use_promote = tracer._use_promote - tracer._use_promote = use_promote + # set amp dtype + original_amp_dtype = tracer._amp_dtype + tracer._amp_dtype = amp_dtype - # restore status - try: - yield - finally: - if tracer: - _g_amp_state_ = original_state - tracer._amp_level = original_amp_level - tracer._set_amp_op_list(original_white_list, original_black_list) - # set_flags(original_flags) - tracer._amp_dtype = original_amp_dtype + # switch promote if amp_level == AMP_LEVEL.O2: - tracer._use_promote = original_use_promote + original_use_promote = tracer._use_promote + tracer._use_promote = use_promote + + # restore status + try: + yield + finally: + if tracer: + _g_amp_state_ = original_state + tracer._amp_level = original_amp_level + tracer._set_amp_op_list( + original_white_list, original_black_list + ) + # set_flags(original_flags) + tracer._amp_dtype = original_amp_dtype + if amp_level == AMP_LEVEL.O2: + tracer._use_promote = original_use_promote class StateDictHook: diff --git a/test/amp/test_pir_amp.py b/test/amp/test_pir_amp.py index 6b4cd5e13c60d..214a68c0982bd 100644 --- a/test/amp/test_pir_amp.py +++ b/test/amp/test_pir_amp.py @@ -17,7 +17,6 @@ import numpy as np import paddle -from paddle.amp.auto_cast import _update_list from paddle.base import core @@ -48,22 +47,11 @@ def test_linear_amp_o1(self): with paddle.static.program_guard(main, startup): x = paddle.static.data('x', [3, 4], 'float32') linear = paddle.nn.Linear(4, 5) - - amp_attrs = core._get_amp_attrs() - amp_attrs._use_promote = True - amp_attrs._amp_level = core.AmpLevel.O1 - amp_attrs._amp_dtype = 'float16' - ( - original_white_list, - original_black_list, - ) = core._get_amp_op_list() - _white_list, _black_list = _update_list( - None, None, 'O1', 'float16' - ) - core._set_amp_op_list(_white_list, _black_list) - - out1 = linear(x) - out2 = paddle.mean(out1) + with paddle.amp.auto_cast( + level='O1', dtype='float16', use_promote=True + ): + out1 = linear(x) + out2 = paddle.mean(out1) cast_op_count = 0 for op in main.global_block().ops: @@ -72,11 +60,6 @@ def test_linear_amp_o1(self): np.testing.assert_equal(out1.dtype, core.DataType.FLOAT32) np.testing.assert_equal(out2.dtype, core.DataType.FLOAT32) np.testing.assert_equal(cast_op_count, 3) - - amp_attrs._use_promote = False - amp_attrs._amp_level = core.AmpLevel.O0 - amp_attrs._amp_dtype = 'float32' - core._set_amp_op_list(original_white_list, original_black_list) _white_list, _black_list = core._get_amp_op_list() np.testing.assert_equal(len(_white_list), 0) np.testing.assert_equal(len(_black_list), 0) From d6fc2a377da3584214454161e5ab3a4b23026a48 Mon Sep 17 00:00:00 2001 From: risemeup1 <62429225+risemeup1@users.noreply.github.com> Date: Tue, 27 Feb 2024 15:18:44 +0800 Subject: [PATCH 123/282] fix openssl-cpu compile bug (#62079) * fix openssl-cpu compile bug * fix openssl-cpu compile bug * fix --- paddle/fluid/pybind/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/pybind/CMakeLists.txt b/paddle/fluid/pybind/CMakeLists.txt index 27b47485f29df..48f16f87f9aeb 100755 --- a/paddle/fluid/pybind/CMakeLists.txt +++ b/paddle/fluid/pybind/CMakeLists.txt @@ -296,7 +296,7 @@ if(WITH_PYTHON) eager_legacy_op_function_generator.cc) set(GENERATOR_DEPS ${PYBIND_DEPS}) list(REMOVE_DUPLICATES GENERATOR_DEPS) - if(NOT WITH_ARM) + if(WIN32) list(REMOVE_ITEM GENERATOR_DEPS python) endif() target_link_libraries(eager_legacy_op_function_generator ${GENERATOR_DEPS}) From 8f490c57eb90428c52c0fd7f6be840e04560aafc Mon Sep 17 00:00:00 2001 From: lzydev Date: Tue, 27 Feb 2024 16:06:39 +0800 Subject: [PATCH 124/282] fix master_grad bug (#62100) --- paddle/phi/api/lib/tensor.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/paddle/phi/api/lib/tensor.cc b/paddle/phi/api/lib/tensor.cc index 49c47cbcce363..2ab68b2e846f2 100644 --- a/paddle/phi/api/lib/tensor.cc +++ b/paddle/phi/api/lib/tensor.cc @@ -153,6 +153,9 @@ bool Tensor::is_dense_tensor() const { return phi::DenseTensor::classof(impl_.get()); } bool Tensor::is_dist_tensor() const { + if (impl_.get() == nullptr) { + return false; + } return phi::distributed::DistTensor::classof(impl_.get()); } bool Tensor::is_selected_rows() const { From 365d8be7512eb15597b3e70caf72c6e7ddd34e00 Mon Sep 17 00:00:00 2001 From: lizexu123 <39205361+lizexu123@users.noreply.github.com> Date: Tue, 27 Feb 2024 16:21:11 +0800 Subject: [PATCH 125/282] add inference api:exp_disable_tensorrt_subgraph (#61967) add inference api:exp_disable_tensorrt_subgraph (#61967) --- .../fluid/framework/ir/subgraph_detector.cc | 14 ++ paddle/fluid/framework/ir/subgraph_detector.h | 3 + paddle/fluid/inference/analysis/argument.h | 3 + .../inference/analysis/ir_pass_manager.cc | 3 + .../analysis/ir_passes/dlnne_subgraph_pass.cc | 1 + .../analysis/ir_passes/lite_subgraph_pass.cc | 3 +- .../ir_passes/tensorrt_subgraph_pass.cc | 1 + paddle/fluid/inference/api/analysis_config.cc | 11 + .../fluid/inference/api/analysis_predictor.cc | 1 + .../inference/api/paddle_analysis_config.h | 4 + paddle/fluid/pybind/inference_api.cc | 3 +- .../test_trt_exp_tensorrt_subgraph.py | 190 ++++++++++++++++++ 12 files changed, 235 insertions(+), 2 deletions(-) create mode 100644 test/ir/inference/test_trt_exp_tensorrt_subgraph.py diff --git a/paddle/fluid/framework/ir/subgraph_detector.cc b/paddle/fluid/framework/ir/subgraph_detector.cc index 1cfe740fc55d2..79df75bd780d5 100644 --- a/paddle/fluid/framework/ir/subgraph_detector.cc +++ b/paddle/fluid/framework/ir/subgraph_detector.cc @@ -424,6 +424,20 @@ void SubGraphFuser::ReplaceNodesWithSubGraphs() { auto subgraphs = SubgraphDetector(graph_, node_inside_subgraph_teller_)(); for (auto &subgraph : subgraphs) { if (subgraph.size() <= static_cast(min_subgraph_size_)) continue; + + bool continue_run = true; + + for (auto *node : subgraph) { + for (const auto tmp_name : node->outputs) { + if (std::find(trt_exclude_var_names_.begin(), + trt_exclude_var_names_.end(), + tmp_name->Name()) != trt_exclude_var_names_.end()) { + continue_run = false; + } + } + } + + if (continue_run == false) continue; std::unordered_set subgraph_uniq(subgraph.begin(), subgraph.end()); // replace this sub-graph with the first node. Two steps: 1. Create a Block // Node that contains this subgraph 2. Mark the nodes inside the sub-graph diff --git a/paddle/fluid/framework/ir/subgraph_detector.h b/paddle/fluid/framework/ir/subgraph_detector.h index 3607ab8d019d3..bd5a534aaf877 100644 --- a/paddle/fluid/framework/ir/subgraph_detector.h +++ b/paddle/fluid/framework/ir/subgraph_detector.h @@ -71,10 +71,12 @@ class SubGraphFuser { SubGraphFuser(Graph *graph, const NodeInsideSubgraphTeller &teller, int min_subgraph_size, + const std::vector &trt_exclude_var_names = {}, std::string name = "tensorrt_engine") : graph_(graph), node_inside_subgraph_teller_(teller), min_subgraph_size_{min_subgraph_size}, + trt_exclude_var_names_(trt_exclude_var_names), name_{name} {} // The main method which run all the logic. @@ -88,6 +90,7 @@ class SubGraphFuser { Graph *graph_; NodeInsideSubgraphTeller node_inside_subgraph_teller_; int min_subgraph_size_; + std::vector trt_exclude_var_names_; const std::string name_; }; diff --git a/paddle/fluid/inference/analysis/argument.h b/paddle/fluid/inference/analysis/argument.h index 69b78b1ef33f3..a87c919bbe2c1 100644 --- a/paddle/fluid/inference/analysis/argument.h +++ b/paddle/fluid/inference/analysis/argument.h @@ -247,6 +247,9 @@ struct Argument { DECL_ARGUMENT_FIELD(trt_output_tensor_names, TRTOutputTensorNames, std::vector); + DECL_ARGUMENT_FIELD(trt_exclude_var_names, + TRTExcludeVarNames, + std::vector); DECL_ARGUMENT_FIELD(tensorrt_disabled_ops, TensorRtDisabledOPs, std::vector); diff --git a/paddle/fluid/inference/analysis/ir_pass_manager.cc b/paddle/fluid/inference/analysis/ir_pass_manager.cc index 31797562e5409..eca0c8fedd0a2 100644 --- a/paddle/fluid/inference/analysis/ir_pass_manager.cc +++ b/paddle/fluid/inference/analysis/ir_pass_manager.cc @@ -170,6 +170,9 @@ void IRPassManager::CreatePasses(Argument *argument, pass->Set( "output_tensor_names", new std::vector(argument->trt_output_tensor_names())); + pass->Set( + "trt_exclude_var_names", + new std::vector(argument->trt_exclude_var_names())); pass->Set("program", new framework::ProgramDesc *(&argument->main_program())); pass->Set("predictor_id", new int(argument->predictor_id())); diff --git a/paddle/fluid/inference/analysis/ir_passes/dlnne_subgraph_pass.cc b/paddle/fluid/inference/analysis/ir_passes/dlnne_subgraph_pass.cc index a9b01c9951f90..5e132cc4b6303 100644 --- a/paddle/fluid/inference/analysis/ir_passes/dlnne_subgraph_pass.cc +++ b/paddle/fluid/inference/analysis/ir_passes/dlnne_subgraph_pass.cc @@ -349,6 +349,7 @@ void analysis::DlnneSubgraphPass::ApplyImpl(framework::ir::Graph *graph) const { graph, teller, Get("min_subgraph_size") /*min subgraph size*/, + {}, "dlnne_engine"); fuser(); diff --git a/paddle/fluid/inference/analysis/ir_passes/lite_subgraph_pass.cc b/paddle/fluid/inference/analysis/ir_passes/lite_subgraph_pass.cc index ad95fe3091ce1..2d484a943cf20 100644 --- a/paddle/fluid/inference/analysis/ir_passes/lite_subgraph_pass.cc +++ b/paddle/fluid/inference/analysis/ir_passes/lite_subgraph_pass.cc @@ -429,7 +429,8 @@ void LiteSubgraphPass::ApplyImpl(framework::ir::Graph* graph) const { *node->Op()); }; - SubGraphFuser fuser(graph, teller, 0 /* min_subgraph_size */, "lite_engine"); + SubGraphFuser fuser( + graph, teller, 0 /* min_subgraph_size */, {}, "lite_engine"); fuser(); std::vector repetitive_params; diff --git a/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc b/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc index d8d8e583e8f8e..69b27b1214839 100644 --- a/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc +++ b/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc @@ -183,6 +183,7 @@ void analysis::TensorRtSubgraphPass::ApplyImpl( graph, teller, Get("min_subgraph_size") /*min subgraph size*/, + Get>("trt_exclude_var_names"), "tensorrt_engine"); fuser(); diff --git a/paddle/fluid/inference/api/analysis_config.cc b/paddle/fluid/inference/api/analysis_config.cc index 7c1dad8a0d2b3..98f031aa14719 100644 --- a/paddle/fluid/inference/api/analysis_config.cc +++ b/paddle/fluid/inference/api/analysis_config.cc @@ -484,6 +484,7 @@ AnalysisConfig::AnalysisConfig(const AnalysisConfig &other) { CP_MEMBER(trt_engine_memory_sharing_identifier_); CP_MEMBER(trt_optimization_level_); CP_MEMBER(trt_ops_run_float_); + CP_MEMBER(trt_exclude_var_names_); // Dlnne related CP_MEMBER(use_dlnne_); CP_MEMBER(dlnne_min_subgraph_size_); @@ -865,6 +866,13 @@ void AnalysisConfig::Exp_DisableTensorRtOPs( trt_disabled_ops_.insert(trt_disabled_ops_.end(), ops.begin(), ops.end()); } +void AnalysisConfig::Exp_DisableTensorRtSubgraph( + const std::vector &var_name_not_trt) { + trt_exclude_var_names_.insert(trt_exclude_var_names_.end(), + var_name_not_trt.begin(), + var_name_not_trt.end()); +} + void AnalysisConfig::EnableVarseqlen() { trt_use_varseqlen_ = true; } void AnalysisConfig::SetTensorRtOptimizationLevel(int level) { @@ -1127,6 +1135,9 @@ std::string AnalysisConfig::SerializeInfoCache() { for (auto &op : trt_disabled_ops_) ss << op.c_str(); ss << ";"; + for (auto &name : trt_exclude_var_names_) ss << name.c_str(); + ss << ";"; + ss << trt_use_dla_; ss << trt_dla_core_; diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index 299e69d628745..076d3b567fa86 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -1702,6 +1702,7 @@ void AnalysisPredictor::PrepareArgument() { argument_->SetTRTMarkOutput(config_.trt_mark_output_); argument_->SetTRTOutputTensorNames(config_.trt_output_tensor_names_); argument_->SetTensorRtDisabledOPs(config_.trt_disabled_ops_); + argument_->SetTRTExcludeVarNames(config_.trt_exclude_var_names_); argument_->SetTensorRtUseDLA(config_.trt_use_dla_); argument_->SetTensorRtDLACore(config_.trt_dla_core_); argument_->SetTensorRtUseStaticEngine(config_.trt_use_static_engine_); diff --git a/paddle/fluid/inference/api/paddle_analysis_config.h b/paddle/fluid/inference/api/paddle_analysis_config.h index 5f187e3cb7a22..473977ef35d95 100644 --- a/paddle/fluid/inference/api/paddle_analysis_config.h +++ b/paddle/fluid/inference/api/paddle_analysis_config.h @@ -810,6 +810,9 @@ struct PD_INFER_DECL AnalysisConfig { /// void Exp_DisableTensorRtOPs(const std::vector& ops); + void Exp_DisableTensorRtSubgraph( + const std::vector& var_name_not_trt); + /// /// \brief Replace some TensorRT plugins to TensorRT OSS( /// https://github.com/NVIDIA/TensorRT), with which some models's inference @@ -1269,6 +1272,7 @@ struct PD_INFER_DECL AnalysisConfig { bool trt_with_interleaved_{false}; bool trt_mark_output_{false}; std::vector trt_output_tensor_names_{}; + std::vector trt_exclude_var_names_{}; std::string tensorrt_transformer_posid_{""}; std::string tensorrt_transformer_maskid_{""}; bool trt_use_dla_{false}; diff --git a/paddle/fluid/pybind/inference_api.cc b/paddle/fluid/pybind/inference_api.cc index 2072bb3802cdd..ee0244e853258 100644 --- a/paddle/fluid/pybind/inference_api.cc +++ b/paddle/fluid/pybind/inference_api.cc @@ -20,7 +20,6 @@ #include #include -#include #include #include #include @@ -933,6 +932,8 @@ void BindAnalysisConfig(py::module *m) { .def("trt_allow_build_at_runtime", &AnalysisConfig::trt_allow_build_at_runtime) .def("exp_disable_tensorrt_ops", &AnalysisConfig::Exp_DisableTensorRtOPs) + .def("exp_disable_tensorrt_subgraph", + &AnalysisConfig::Exp_DisableTensorRtSubgraph) .def("enable_tensorrt_dla", &AnalysisConfig::EnableTensorRtDLA, py::arg("dla_core") = 0) diff --git a/test/ir/inference/test_trt_exp_tensorrt_subgraph.py b/test/ir/inference/test_trt_exp_tensorrt_subgraph.py new file mode 100644 index 0000000000000..a5f2303d48bad --- /dev/null +++ b/test/ir/inference/test_trt_exp_tensorrt_subgraph.py @@ -0,0 +1,190 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from functools import partial + +import numpy as np +from program_config import ProgramConfig, TensorConfig +from trt_layer_auto_scan_test import TrtLayerAutoScanTest + +import paddle.inference as paddle_infer + + +class TrtConvertSetValue(TrtLayerAutoScanTest): + def is_program_valid(self, program_config: ProgramConfig) -> bool: + return True + + def create_inference_config(self, use_trt=True) -> paddle_infer.Config: + config = paddle_infer.Config() + config.disable_glog_info() + config.enable_use_gpu(100, 0) + config.exp_disable_tensorrt_subgraph(["input_data"]) + config.set_optim_cache_dir(self.cache_dir) + if use_trt: + config.switch_ir_debug() + config.enable_tensorrt_engine( + max_batch_size=self.trt_param.max_batch_size, + workspace_size=self.trt_param.workspace_size, + min_subgraph_size=self.trt_param.min_subgraph_size, + precision_mode=self.trt_param.precision, + use_static=self.trt_param.use_static, + use_calib_mode=self.trt_param.use_calib_mode, + ) + if self.dynamic_shape.min_input_shape and ( + self.dynamic_shape.min_input_shape.keys() + == self.dynamic_shape.max_input_shape.keys() + == self.dynamic_shape.opt_input_shape.keys() + ): + config.set_trt_dynamic_shape_info( + self.dynamic_shape.min_input_shape, + self.dynamic_shape.max_input_shape, + self.dynamic_shape.opt_input_shape, + self.dynamic_shape.disable_trt_plugin_fp16, + ) + return config + + def sample_program_configs(self): + def generate_input1(): + return np.random.random([2, 3, 3]).astype(np.float32) + + def generate_input2(): + return np.random.random([2, 2, 3]).astype(np.float32) + + for update_scalar in [True, False]: + self.update_scalar = update_scalar + set_value_inputs = {} + if update_scalar: + set_value_inputs = { + "Input": ["input_data"], + } + else: + set_value_inputs = { + "Input": ["input_data"], + "ValueTensor": ["update_data"], + } + ops_config = [ + { + "op_type": "set_value", + "op_inputs": set_value_inputs, + "op_outputs": {"Out": ["input_data"]}, + "op_attrs": { + "axes": [1], + "starts": [0], + "ends": [2], + "steps": [1], + "decrease_axes": [], + "values": [0.0], + }, + }, + { + "op_type": "relu", + "op_inputs": { + "X": ["input_data"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": {}, + }, + ] + + ops = self.generate_op_config(ops_config) + if update_scalar: + program_config = ProgramConfig( + ops=ops, + weights={}, + inputs={ + "input_data": TensorConfig( + data_gen=partial(generate_input1) + ), + }, + outputs=["output_data"], + ) + else: + program_config = ProgramConfig( + ops=ops, + weights={}, + inputs={ + "input_data": TensorConfig( + data_gen=partial(generate_input1) + ), + "update_data": TensorConfig( + data_gen=partial(generate_input2) + ), + }, + outputs=["output_data"], + ) + + yield program_config + + def sample_predictor_configs(self, program_config): + def generate_dynamic_shape(attrs): + if self.update_scalar: + self.dynamic_shape.min_input_shape = { + "input_data": [2, 3, 3], + } + self.dynamic_shape.max_input_shape = { + "input_data": [3, 3, 4], + } + self.dynamic_shape.opt_input_shape = { + "input_data": [3, 3, 3], + } + else: + self.dynamic_shape.min_input_shape = { + "input_data": [2, 3, 3], + "update_data": [2, 2, 3], + } + self.dynamic_shape.max_input_shape = { + "input_data": [3, 3, 4], + "update_data": [3, 2, 4], + } + self.dynamic_shape.opt_input_shape = { + "input_data": [3, 3, 3], + "update_data": [3, 2, 3], + } + + def clear_dynamic_shape(): + self.dynamic_shape.max_input_shape = {} + self.dynamic_shape.min_input_shape = {} + self.dynamic_shape.opt_input_shape = {} + + def generate_trt_nodes_num(attrs, dynamic_shape): + if dynamic_shape: + ver = paddle_infer.get_trt_compile_version() + if self.update_scalar: + if ver[0] * 1000 + ver[1] * 100 + ver[2] * 10 < 8200: + return 1, 3 + return 0, 4 + else: + if ver[0] * 1000 + ver[1] * 100 + ver[2] * 10 < 8200: + return 1, 4 + return 0, 5 + + attrs = [ + program_config.ops[i].attrs for i in range(len(program_config.ops)) + ] + + generate_dynamic_shape(attrs) + self.trt_param.precision = paddle_infer.PrecisionType.Float32 + # program_config.set_input_type(np.float32) + self.trt_param.workspace_size = 2013265920 + yield self.create_inference_config(), generate_trt_nodes_num( + attrs, True + ), (1e-5, 1e-4) + + def test(self): + self.run_test() + + +if __name__ == "__main__": + unittest.main() From 44a5b0d3cfaf4ce2b1251d40a8d029abfa705d83 Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Tue, 27 Feb 2024 16:46:57 +0800 Subject: [PATCH 126/282] [PIR+CINN]Fix SplitOp Pattern problem (#62086) * [PIR+CINN]Fix SplitOp Pattern problem * open more UT * fix return * disable UT --- .../operator/transforms/pd_to_cinn_pass.cc | 175 +++++++++++------- .../pir/cinn/sub_graphs/test_sub_graph_31.py | 3 +- .../pir/cinn/sub_graphs/test_sub_graph_33.py | 6 +- .../pir/cinn/sub_graphs/test_sub_graph_58.py | 3 +- .../pir/cinn/sub_graphs/test_sub_graph_6.py | 3 +- .../pir/cinn/sub_graphs/test_sub_graph_61.py | 3 +- .../pir/cinn/sub_graphs/test_sub_graph_89.py | 3 +- .../cinn/sub_graphs/test_sub_graph_chunk.py | 3 +- .../cinn/sub_graphs/test_sub_graph_split.py | 3 +- 9 files changed, 113 insertions(+), 89 deletions(-) diff --git a/paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.cc index 56092ebfe50c6..ad6c7b9a060da 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.cc @@ -429,6 +429,28 @@ class PowOpPattern : public pir::OpRewritePattern { } }; +static void ReplaceSliceOp(const cinn::dialect::SplitOp &cinn_split, + pir::Operation *slice_op, + pir::PatternRewriter &rewriter) { // NOLINT + const int index = slice_op->dyn_cast<::pir::SliceOp>() + .attribute("index") + .dyn_cast<::pir::Int32Attribute>() + .data(); + rewriter.ReplaceAllUsesWith(slice_op->result(0), cinn_split.result(index)); + rewriter.EraseOp(slice_op); +} + +static void ReplaceSplitOp(const cinn::dialect::SplitOp &cinn_split, + pir::Operation *split_op, + pir::PatternRewriter &rewriter) { // NOLINT + const size_t num_results = cinn_split.num_results(); + CHECK(split_op->num_results() == num_results); + for (size_t i = 0; i < num_results; ++i) { + rewriter.ReplaceAllUsesWith(split_op->result(i), cinn_split.result(i)); + } + rewriter.EraseOp(split_op); +} + class SplitOpPattern : public pir::OpRewritePattern { public: using pir::OpRewritePattern::OpRewritePattern; @@ -446,49 +468,54 @@ class SplitOpPattern : public pir::OpRewritePattern { void Rewrite(paddle::dialect::SplitOp op, pir::PatternRewriter &rewriter) const override { - auto sections_gen_op = op->operand_source(1) - .defining_op() - ->dyn_cast(); - auto axis_gen_op = op->operand_source(2) - .defining_op() - ->dyn_cast(); - auto section_attr = sections_gen_op.attribute("value") - .dyn_cast() - .AsVector(); - - std::vector vec_sections; - if (section_attr.size() > 0) { - for (size_t i = 0; i < section_attr.size(); ++i) { - vec_sections.push_back( - section_attr[i].dyn_cast<::pir::Int64Attribute>().data()); + const std::vector sections = [&]() -> std::vector { + std::vector result; + auto sections_gen_op = op->operand_source(1) + .defining_op() + ->dyn_cast(); + auto section_attr = sections_gen_op.attribute("value") + .dyn_cast() + .AsVector(); + if (section_attr.size() > 0) { + for (size_t i = 0; i < section_attr.size(); ++i) { + result.push_back( + section_attr[i].dyn_cast<::pir::Int64Attribute>().data()); + } } - } - int axis = static_cast(axis_gen_op.attribute("value") - .dyn_cast<::pir::FloatAttribute>() - .data()); - - auto input_ele = op->operand_source(0) - .type() - .dyn_cast(); - if (axis < 0) { - axis += input_ele.dims().size(); - } + return result; + }(); + + const int axis = [&]() -> int { + auto axis_gen_op = op->operand_source(2) + .defining_op() + ->dyn_cast(); + int axis = static_cast(axis_gen_op.attribute("value") + .dyn_cast<::pir::FloatAttribute>() + .data()); + auto input_ele = op->operand_source(0) + .type() + .dyn_cast(); + if (axis < 0) { + axis += input_ele.dims().size(); + } + return axis; + }(); auto cinn_split = rewriter.Build( - op->operand_source(0), vec_sections, axis); + op->operand_source(0), sections, axis); auto orig_out = op.result(0); for (auto it = orig_out.use_begin(); it != orig_out.use_end();) { - auto slice_op = (it++)->owner(); - CHECK(slice_op->isa<::pir::SliceOp>()) - << "Currently only support pir::slice as downstream op"; - int index = slice_op->dyn_cast<::pir::SliceOp>() - .attribute("index") - .dyn_cast<::pir::Int32Attribute>() - .data(); - rewriter.ReplaceAllUsesWith(slice_op->result(0), - cinn_split.result(index)); - rewriter.EraseOp(slice_op); + auto downstream_op = (it++)->owner(); + if (downstream_op->isa<::pir::SliceOp>()) { + ReplaceSliceOp(cinn_split, downstream_op, rewriter); + } else if (downstream_op->isa<::pir::SplitOp>()) { + ReplaceSplitOp(cinn_split, downstream_op, rewriter); + } else { + CHECK(false) << "Currently only support pir::slice/split as downstream " + "op, but got: " + << downstream_op->name(); + } } rewriter.EraseOp(op); } @@ -509,49 +536,53 @@ class SplitWithNumOpPattern void Rewrite(paddle::dialect::SplitWithNumOp op, pir::PatternRewriter &rewriter) const override { - auto axis_gen_op = op->operand_source(1).defining_op(); - auto full_op = axis_gen_op->dyn_cast(); - int axis = static_cast( - full_op.attribute("value").dyn_cast<::pir::FloatAttribute>().data()); - - auto input_ele = op->operand_source(0) - .type() - .dyn_cast(); - if (axis < 0) { - axis += input_ele.dims().size(); - } - std::vector sections; - - auto split_dim = input_ele.dims()[axis]; - - auto split_num = - op->attribute("num").dyn_cast<::pir::Int32Attribute>().data(); - auto part_ele = (split_dim + split_num - 1) / split_num; - - int total_split_num = 0; - for (int i = 0; i < split_num - 1; ++i) { - sections.push_back(part_ele); - total_split_num += part_ele; - } + const auto input_ele = op->operand_source(0) + .type() + .dyn_cast(); + + const int axis = [&]() -> int { + auto axis_gen_op = op->operand_source(1).defining_op(); + auto full_op = axis_gen_op->dyn_cast(); + int axis = static_cast( + full_op.attribute("value").dyn_cast<::pir::FloatAttribute>().data()); + if (axis < 0) { + axis += input_ele.dims().size(); + } + return axis; + }(); + + const auto sections = [&]() -> std::vector { + std::vector result; + auto split_dim = input_ele.dims()[axis]; + auto split_num = + op->attribute("num").dyn_cast<::pir::Int32Attribute>().data(); + auto part_ele = (split_dim + split_num - 1) / split_num; + int total_split_num = 0; + for (int i = 0; i < split_num - 1; ++i) { + result.push_back(part_ele); + total_split_num += part_ele; + } - sections.push_back(split_dim - total_split_num); + result.push_back(split_dim - total_split_num); + return result; + }(); auto cinn_split = rewriter.Build( op->operand_source(0), sections, axis); auto orig_out = op.result(0); for (auto it = orig_out.use_begin(); it != orig_out.use_end();) { - auto slice_op = (it++)->owner(); - CHECK(slice_op->isa<::pir::SliceOp>()); - int index = slice_op->dyn_cast<::pir::SliceOp>() - .attribute("index") - .dyn_cast<::pir::Int32Attribute>() - .data(); - rewriter.ReplaceAllUsesWith(slice_op->result(0), - cinn_split.result(index)); - rewriter.EraseOp(slice_op); + auto downstream_op = (it++)->owner(); + if (downstream_op->isa<::pir::SliceOp>()) { + ReplaceSliceOp(cinn_split, downstream_op, rewriter); + } else if (downstream_op->isa<::pir::SplitOp>()) { + ReplaceSplitOp(cinn_split, downstream_op, rewriter); + } else { + CHECK(false) << "Currently only support pir::slice/split as downstream " + "op, but got: " + << downstream_op->name(); + } } - rewriter.EraseOp(op); } }; diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_31.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_31.py index c335a9df27a8a..ae5a5adb3d4c5 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_31.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_31.py @@ -64,11 +64,10 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): outs = net(*self.inputs) return outs - # NOTE prim + cinn lead to error def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) cinn_out = self.train( - self.net, to_static=True, with_prim=True, with_cinn=False + self.net, to_static=True, with_prim=True, with_cinn=True ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_33.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_33.py index 95b55b9bfe331..eff3e66cf20cf 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_33.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_33.py @@ -86,16 +86,16 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): outs = net(*self.inputs) return outs - # NOTE prim + cinn lead to error def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) + # NOTE(Aurelius84): atol only satisfy 1e-5 under with_cinn=True cinn_out = self.train( - self.net, to_static=True, with_prim=True, with_cinn=False + self.net, to_static=True, with_prim=True, with_cinn=True ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) ): - np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) + np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-5) if __name__ == '__main__': diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_58.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_58.py index 7c49069686803..17efb1621e403 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_58.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_58.py @@ -78,11 +78,10 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): outs = net(*self.inputs) return outs - # NOTE prim + cinn lead to error def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) cinn_out = self.train( - self.net, to_static=True, with_prim=True, with_cinn=False + self.net, to_static=True, with_prim=True, with_cinn=True ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_6.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_6.py index 30dddf1f146e3..5949d57dbd357 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_6.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_6.py @@ -57,11 +57,10 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): outs = net(*self.inputs) return outs - # NOTE prim + cinn lead to error def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) cinn_out = self.train( - self.net, to_static=True, with_prim=True, with_cinn=False + self.net, to_static=True, with_prim=True, with_cinn=True ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_61.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_61.py index d7beb397563c4..8bda88384089f 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_61.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_61.py @@ -95,7 +95,6 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): outs = net(*self.inputs) return outs - # NOTE prim + cinn lead to error def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) cinn_out = self.train( @@ -104,7 +103,7 @@ def test_ast_prim_cinn(self): for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) ): - np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) + np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-6) if __name__ == '__main__': diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_89.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_89.py index e456dd86f1f53..c5027f48d58ec 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_89.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_89.py @@ -101,7 +101,6 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): outs = net(*self.inputs) return outs - # NOTE prim + cinn lead to error def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) cinn_out = self.train( @@ -110,7 +109,7 @@ def test_ast_prim_cinn(self): for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) ): - np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) + np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-6) if __name__ == '__main__': diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_chunk.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_chunk.py index 6b9d3e3c94557..420d85ffa97be 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_chunk.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_chunk.py @@ -56,11 +56,10 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): outs = net(*self.inputs) return outs - # NOTE prim + cinn lead to error def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) cinn_out = self.train( - self.net, to_static=True, with_prim=True, with_cinn=False + self.net, to_static=True, with_prim=True, with_cinn=True ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_split.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_split.py index 97f9b4adb8eed..a70332ef939c6 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_split.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_split.py @@ -58,11 +58,10 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): outs = net(*self.inputs) return outs - # NOTE prim + cinn lead to error def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) cinn_out = self.train( - self.net, to_static=True, with_prim=True, with_cinn=False + self.net, to_static=True, with_prim=True, with_cinn=True ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) From b6225ee1deb0093da16e9d950cf7f4d816f8cc76 Mon Sep 17 00:00:00 2001 From: lanxianghit <47554610+lanxianghit@users.noreply.github.com> Date: Tue, 27 Feb 2024 17:00:41 +0800 Subject: [PATCH 127/282] [PIR][DynamicShape] Add InferSymbolicShape for buildin.constant & fetch op (#62061) * Add InferSymbolicShape for buildin.constant & fetch op --- .../same_operands_and_result.cc | 9 ++++++ .../same_operands_and_result.h | 3 ++ .../pir/dialect/operator/ir/op_dialect.cc | 31 +++++++++++++++++++ paddle/fluid/pir/dialect/operator/ir/ops.yaml | 1 + 4 files changed, 44 insertions(+) diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.cc index 571b90f7ff552..98a6d670869ca 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.cc @@ -66,6 +66,15 @@ bool Exp_OpInferSymbolicShape(pir::Operation *op, return SameOperandsAndResultShape(op, shape_analysis); } +bool FetchOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + shape_analysis->SetShapeOrDataForValue( + op->result(0), + shape_analysis->GetShapeOrDataForValue(op->operand_source(0))); + + return true; +} + bool IncrementOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h index b72111b1173d5..d96f4efe1f825 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h @@ -38,6 +38,9 @@ bool ExpOpInferSymbolicShape(pir::Operation *op, bool Exp_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool FetchOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); + bool FullWithTensorOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); diff --git a/paddle/fluid/pir/dialect/operator/ir/op_dialect.cc b/paddle/fluid/pir/dialect/operator/ir/op_dialect.cc index e7aff91b4a99a..6816d64a05467 100644 --- a/paddle/fluid/pir/dialect/operator/ir/op_dialect.cc +++ b/paddle/fluid/pir/dialect/operator/ir/op_dialect.cc @@ -48,6 +48,7 @@ static std::unordered_map kCustomTypeMap = { {"std::vector", "pir::ArrayAttribute"}, {"std::vector", "pir::ArrayAttribute"}, {"std::vector", "pir::ArrayAttribute"}}; + struct CombineOpInferSymbolicShapeInterfaceModel : public InferSymbolicShapeInterface::Concept { static inline bool InferSymbolicShape( @@ -75,6 +76,36 @@ struct CombineOpInferSymbolicShapeInterfaceModel : InferSymbolicShapeInterface::Concept(InferSymbolicShape) {} }; +struct ConstantOpInferSymbolicShapeInterfaceModel + : public InferSymbolicShapeInterface::Concept { + static inline bool InferSymbolicShape( + pir::Operation* op, pir::ShapeConstraintIRAnalysis* shape_analysis) { + IR_ENFORCE(op->result(0).type().dyn_cast(), + "Currently InferSymbolicShape of ConstantOp only support " + "DenseTensorType result."); + + const std::vector out_dims = [op] { + std::vector dims; + const std::vector result_dims = common::vectorize( + op->result(0).type().dyn_cast().dims()); + for (size_t i = 0; i < result_dims.size(); i++) { + dims.emplace_back(result_dims[i]); + } + return dims; + }(); + + shape_analysis->SetShapeOrDataForValue( + op->result(0), + symbol::ShapeOrDataDimExprs{ + symbol::TensorShapeOrDataDimExprs(out_dims)}); + + return true; + } + + ConstantOpInferSymbolicShapeInterfaceModel() + : InferSymbolicShapeInterface::Concept(InferSymbolicShape) {} +}; + struct ParameterOpInferSymbolicShapeInterfaceModel : public InferSymbolicShapeInterface::Concept { static inline bool InferSymbolicShape( diff --git a/paddle/fluid/pir/dialect/operator/ir/ops.yaml b/paddle/fluid/pir/dialect/operator/ir/ops.yaml index 594130926d569..dac35221ee83b 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops.yaml @@ -558,6 +558,7 @@ func : fetch param : [x] traits : pir::SideEffectTrait + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : floor_divide args : (Tensor x, Tensor y) From 1ca27f11f886293b7577e86f3e7a8985f396f747 Mon Sep 17 00:00:00 2001 From: Zhan Rongrui <46243324+zrr1999@users.noreply.github.com> Date: Tue, 27 Feb 2024 17:26:58 +0800 Subject: [PATCH 128/282] =?UTF-8?q?[PIR]=20D-7=E3=80=81D-10=20Adapt=20test?= =?UTF-8?q?=5Ferrors=20(#61963)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/legacy_test/test_cumsum_op.py | 5 ++- test/legacy_test/test_pad_op.py | 65 +++++++++++++++--------------- 2 files changed, 37 insertions(+), 33 deletions(-) diff --git a/test/legacy_test/test_cumsum_op.py b/test/legacy_test/test_cumsum_op.py index 1f223ba05d0d5..5cc45e0b0b117 100644 --- a/test/legacy_test/test_cumsum_op.py +++ b/test/legacy_test/test_cumsum_op.py @@ -496,9 +496,12 @@ def test_check_grad(self): class BadInputTest(unittest.TestCase): + @test_with_pir_api def test_error(self): paddle.enable_static() - with base.program_guard(base.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): def test_bad_x(): data = [1, 2, 4] diff --git a/test/legacy_test/test_pad_op.py b/test/legacy_test/test_pad_op.py index 5912e57bef649..f916cea1cf097 100644 --- a/test/legacy_test/test_pad_op.py +++ b/test/legacy_test/test_pad_op.py @@ -27,7 +27,7 @@ def pad_wrapper(x, paddings, pad_value): return paddle.nn.functional.pad( - x, pad=list(paddings), mode='constant', value=pad_value + x, pad=list(paddings), mode="constant", value=pad_value ) @@ -38,16 +38,16 @@ def setUp(self): self.op_type = "pad" self.python_api = pad_wrapper self.inputs = { - 'X': np.random.random(self.shape).astype(self.dtype), + "X": np.random.random(self.shape).astype(self.dtype), } self.attrs = {} - self.attrs['paddings'] = list(np.array(self.paddings).flatten()) - self.attrs['pad_value'] = self.pad_value + self.attrs["paddings"] = list(np.array(self.paddings).flatten()) + self.attrs["pad_value"] = self.pad_value self.outputs = { - 'Out': np.pad( - self.inputs['X'], + "Out": np.pad( + self.inputs["X"], self.paddings, - mode='constant', + mode="constant", constant_values=self.pad_value, ) } @@ -62,8 +62,8 @@ def test_check_output(self): def test_check_grad_normal(self): self.check_grad( - ['X'], - 'Out', + ["X"], + "Out", check_prim=True, check_pir=True, check_prim_pir=True, @@ -109,8 +109,8 @@ def get_dtype(self): def test_check_grad_normal(self): self.check_grad( - ['X'], - 'Out', + ["X"], + "Out", check_prim=True, check_pir=True, check_prim_pir=True, @@ -128,6 +128,7 @@ def test_check_grad_normal(self): class TestPadOpError(unittest.TestCase): + @test_with_pir_api def test_errors(self): with static_guard(): with paddle.static.program_guard( @@ -139,11 +140,11 @@ def test_Variable(): paddle.nn.functional.pad(x=input_data, pad=[1, 1, 1, 1]) self.assertRaises(TypeError, test_Variable) - - data = paddle.static.data( - name='data', shape=[4], dtype='float16' - ) - paddle.nn.functional.pad(x=data, pad=[0, 1]) + if core.is_compiled_with_cuda(): + data = paddle.static.data( + name="data", shape=[4], dtype="float16" + ) + paddle.nn.functional.pad(x=data, pad=[0, 1]) class TestPaddingValueTensor(UnittestBase): @@ -171,7 +172,7 @@ def test_static(self): exe.run(startup_prog) res = exe.run(fetch_list=[feat, out]) gt = np.pad( - res[0], [1, 1], 'constant', constant_values=[1.0, 1.0] + res[0], [1, 1], "constant", constant_values=[1.0, 1.0] ) np.testing.assert_allclose(res[1], gt) @@ -183,7 +184,7 @@ def test_static(self): gt = np.pad( infer_outs[0], [1, 1], - 'constant', + "constant", constant_values=[1.0, 1.0], ) np.testing.assert_allclose(infer_outs[1], gt) @@ -207,12 +208,12 @@ def test_pir_static(self): exe.run(startup_prog) res = exe.run(fetch_list=[feat, out]) gt = np.pad( - res[0], [1, 1], 'constant', constant_values=[1.0, 1.0] + res[0], [1, 1], "constant", constant_values=[1.0, 1.0] ) np.testing.assert_allclose(res[1], gt) def path_prefix(self): - return 'padding_value' + return "padding_value" def var_prefix(self): return "Var[" @@ -220,7 +221,7 @@ def var_prefix(self): def call_func(self, x): padding_value = paddle.assign([1.0]) out = paddle.nn.functional.pad( - x, pad=[1, 1, 1, 1], value=padding_value, mode='constant' + x, pad=[1, 1, 1, 1], value=padding_value, mode="constant" ) return out @@ -238,12 +239,12 @@ class TestPaddingValueTensor3(unittest.TestCase): @test_with_pir_api def test_static(self): with static_guard(): - np_x = np.random.random((16, 16)).astype('float32') + np_x = np.random.random((16, 16)).astype("float32") main_prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, startup_prog): - x = paddle.assign(np_x).astype('float32') - pad_value = paddle.assign([0.0]).astype('float64') + x = paddle.assign(np_x).astype("float32") + pad_value = paddle.assign([0.0]).astype("float64") y = paddle.nn.functional.pad(x, [0, 1, 2, 3], value=pad_value) loss = y.sum() optimize_ops, params_grads = paddle.optimizer.SGD( @@ -273,13 +274,13 @@ def setUp(self): self.python_api = pad_wrapper x = np.random.random(self.shape).astype(np.float32) self.attrs = {} - self.attrs['paddings'] = list(np.array(self.paddings).flatten()) - self.attrs['pad_value'] = self.pad_value + self.attrs["paddings"] = list(np.array(self.paddings).flatten()) + self.attrs["pad_value"] = self.pad_value out = np.pad( - x, self.paddings, mode='constant', constant_values=self.pad_value + x, self.paddings, mode="constant", constant_values=self.pad_value ) - self.inputs = {'X': convert_float_to_uint16(x)} - self.outputs = {'Out': convert_float_to_uint16(out)} + self.inputs = {"X": convert_float_to_uint16(x)} + self.outputs = {"Out": convert_float_to_uint16(out)} self.prim_op_type = "prim" self.public_python_api = pad_wrapper self.if_enable_cinn() @@ -300,14 +301,14 @@ def test_check_grad(self): place = core.CUDAPlace(0) self.check_grad_with_place( place, - ['X'], - 'Out', + ["X"], + "Out", check_prim=True, check_pir=True, check_prim_pir=True, ) -if __name__ == '__main__': +if __name__ == "__main__": # paddle.enable_static() unittest.main() From 0eee504c94fc3fe10df35c704300758cad92e4e7 Mon Sep 17 00:00:00 2001 From: co63oc Date: Tue, 27 Feb 2024 18:50:40 +0800 Subject: [PATCH 129/282] Fix alreadly already, etc (#62105) --- paddle/fluid/framework/feed_fetch_method.cc | 2 +- paddle/fluid/framework/infershape_utils.cc | 2 +- paddle/fluid/framework/infershape_utils.h | 2 +- paddle/fluid/framework/lod_tensor.h | 4 ++-- paddle/fluid/framework/op_desc.cc | 2 +- paddle/fluid/framework/op_version_proto.h | 2 +- paddle/fluid/framework/op_version_registry.h | 11 ++++++----- paddle/fluid/framework/operator.cc | 16 ++++++++-------- paddle/fluid/framework/parallel_executor.cc | 2 +- paddle/fluid/framework/program_utils.cc | 4 ++-- paddle/fluid/framework/proto_desc.h | 2 +- paddle/fluid/framework/prune.cc | 12 ++++++------ 12 files changed, 31 insertions(+), 30 deletions(-) diff --git a/paddle/fluid/framework/feed_fetch_method.cc b/paddle/fluid/framework/feed_fetch_method.cc index b5587b7f228bc..46543e7cba9bd 100644 --- a/paddle/fluid/framework/feed_fetch_method.cc +++ b/paddle/fluid/framework/feed_fetch_method.cc @@ -97,7 +97,7 @@ FetchType& GetFetchVariable(const Scope& scope, const std::string& var_name, size_t index) { // Since we want to fetch FetchType from a variable, the variable must - // be created alreadly. + // be created already. Variable* g_fetch_value = scope.FindVar(var_name); PADDLE_ENFORCE_NOT_NULL(g_fetch_value, platform::errors::NotFound( diff --git a/paddle/fluid/framework/infershape_utils.cc b/paddle/fluid/framework/infershape_utils.cc index 88f0b496a8e4c..bcf72be80decb 100644 --- a/paddle/fluid/framework/infershape_utils.cc +++ b/paddle/fluid/framework/infershape_utils.cc @@ -859,7 +859,7 @@ CompatInferMetaContext BuildInferMetaContext(InferShapeContext* ctx, attr_names[i])); } } else { - // do nothing, skip currnet attr + // do nothing, skip current attr } } } diff --git a/paddle/fluid/framework/infershape_utils.h b/paddle/fluid/framework/infershape_utils.h index 6e1170c6ee0fe..15d70d0ec41e0 100644 --- a/paddle/fluid/framework/infershape_utils.h +++ b/paddle/fluid/framework/infershape_utils.h @@ -85,7 +85,7 @@ class CompatMetaTensor : public phi::MetaTensor { is_runtime_, true, platform::errors::Unavailable( - "Only can get phi::DenseTensor from MetaTensor in rumtime.")); + "Only can get phi::DenseTensor from MetaTensor in runtime.")); auto* var = PADDLE_GET_CONST(Variable*, var_); PADDLE_ENFORCE_EQ( var->IsType(), diff --git a/paddle/fluid/framework/lod_tensor.h b/paddle/fluid/framework/lod_tensor.h index 9df50b69f3421..9556430787153 100644 --- a/paddle/fluid/framework/lod_tensor.h +++ b/paddle/fluid/framework/lod_tensor.h @@ -166,8 +166,8 @@ std::pair> GetSubLoDAndAbsoluteOffset( const LoD& lod, size_t start_idx, size_t end_idx, size_t start_level); /* - * Serialize/Desiralize phi::DenseTensor to std::ostream - * You can pass ofstream or ostringstream to serilize to file + * Serialize/Deserialize phi::DenseTensor to std::ostream + * You can pass ofstream or ostringstream to serialize to file * or to a in memory string. GPU tensor will be copied to CPU. */ void SerializeToStream(std::ostream& os, diff --git a/paddle/fluid/framework/op_desc.cc b/paddle/fluid/framework/op_desc.cc index 9e6619f5b7b3f..32c520711d978 100644 --- a/paddle/fluid/framework/op_desc.cc +++ b/paddle/fluid/framework/op_desc.cc @@ -322,7 +322,7 @@ class CompileTimeInferShapeContext : public InferShapeContext { PADDLE_ENFORCE_EQ(arg_names.size(), 1UL, platform::errors::InvalidArgument( - "The iutput(%s) should hold only one element, but " + "The input(%s) should hold only one element, but " "now it holds %d elements.", name, arg_names.size())); diff --git a/paddle/fluid/framework/op_version_proto.h b/paddle/fluid/framework/op_version_proto.h index 2540862038c80..0de3bc82a94d2 100644 --- a/paddle/fluid/framework/op_version_proto.h +++ b/paddle/fluid/framework/op_version_proto.h @@ -55,7 +55,7 @@ class OpVersionMap { }; // get version id for operators with version id in paddle 2.4.2, this is used -// for converting ProgramDesc in 2.4 comtabible format +// for converting ProgramDesc in 2.4 compatible format const std::unordered_map& GetLegacyOpVersions(); } // namespace pb diff --git a/paddle/fluid/framework/op_version_registry.h b/paddle/fluid/framework/op_version_registry.h index 236a0e2b86187..4e13894f562a3 100644 --- a/paddle/fluid/framework/op_version_registry.h +++ b/paddle/fluid/framework/op_version_registry.h @@ -378,11 +378,12 @@ class PassVersionCheckerRegistrar { static PassVersionCheckerRegistrar& GetInstance(); PassVersionCheckers& Register(const std::string& pass_name) { - PADDLE_ENFORCE_EQ(pass_version_checkers_map_.find(pass_name), - pass_version_checkers_map_.end(), - platform::errors::AlreadyExists( - "PassVersionCheckers(%s) has alredy been registered.", - pass_name.c_str())); + PADDLE_ENFORCE_EQ( + pass_version_checkers_map_.find(pass_name), + pass_version_checkers_map_.end(), + platform::errors::AlreadyExists( + "PassVersionCheckers(%s) has already been registered.", + pass_name.c_str())); return pass_version_checkers_map_[pass_name]; } bool IsPassCompatible(const std::string& fuse_pass_name) const { diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc index bf2badc5a82cf..99ccbbe50d241 100644 --- a/paddle/fluid/framework/operator.cc +++ b/paddle/fluid/framework/operator.cc @@ -2677,7 +2677,7 @@ Scope* OperatorWithKernel::PrepareData( // target_kernel_type. // Have a discussion with @Superjomn or the inference developers if some // changes on this logic for this macro might not tested on the other - // scenerios. + // scenarios. // If this op is not called by an Executor or ParallelExecutor, it should // called by a NaiveExecutor, the NaiveExecutor will cache the scopes and // variables, that behavior a lot different. @@ -3088,10 +3088,10 @@ static void SetDnnAttrIntoDeviceContext( phi::DeviceContext* dev_ctx, const Attribute& attr, const std::string& attr_name, - const operators::ExtraAttrPropertySet& attr_propertys) { + const operators::ExtraAttrPropertySet& attr_properties) { #ifdef PADDLE_WITH_DNNL if (phi::OneDNNContext::classof(dev_ctx) && - attr_propertys.Support(operators::ExtraAttrProperty::ONEDNN)) { + attr_properties.Support(operators::ExtraAttrProperty::ONEDNN)) { VLOG(4) << "Runtime attr `" << attr_name << "` is passed to OneDNNContext."; phi::OneDNNContext* one_dnn_ctx = static_cast(dev_ctx); switch (AttrTypeID(attr)) { @@ -3124,7 +3124,7 @@ static void SetDnnAttrIntoDeviceContext( #endif #ifdef PADDLE_WITH_CUDA if (phi::GPUContext::classof(dev_ctx) && - attr_propertys.Support(operators::ExtraAttrProperty::GPUDNN)) { + attr_properties.Support(operators::ExtraAttrProperty::GPUDNN)) { VLOG(4) << "Runtime attr `" << attr_name << "` is passed to GPUDNNContext."; phi::GPUContext* gpu_dnn_ctx = static_cast(dev_ctx); switch (AttrTypeID(attr)) { @@ -3585,8 +3585,8 @@ void OperatorWithKernel::BuildPhiKernelContext( for (const auto& attr_iter : runtime_attrs) { auto& attr_name = attr_iter.first; auto& attr = attr_iter.second; - auto attr_propertys = paddle::operators::GetExtraAttrProperties(attr_name); - SetDnnAttrIntoDeviceContext(dev_ctx, attr, attr_name, attr_propertys); + auto attr_properties = paddle::operators::GetExtraAttrProperties(attr_name); + SetDnnAttrIntoDeviceContext(dev_ctx, attr, attr_name, attr_properties); } // TODO(chenweihang): Since the pass will still `SetAttr` in the OpDesc, // we try to add these Attrs to the RuntimeAttrs, but these OpDesc will lose @@ -3600,8 +3600,8 @@ void OperatorWithKernel::BuildPhiKernelContext( for (const auto& attr_iter : attrs) { auto& attr_name = attr_iter.first; auto& attr = attr_iter.second; - auto attr_propertys = paddle::operators::GetExtraAttrProperties(attr_name); - SetDnnAttrIntoDeviceContext(dev_ctx, attr, attr_name, attr_propertys); + auto attr_properties = paddle::operators::GetExtraAttrProperties(attr_name); + SetDnnAttrIntoDeviceContext(dev_ctx, attr, attr_name, attr_properties); } VLOG(4) << "Done runtime attributes"; #endif diff --git a/paddle/fluid/framework/parallel_executor.cc b/paddle/fluid/framework/parallel_executor.cc index 516c62a70f688..897e520813809 100644 --- a/paddle/fluid/framework/parallel_executor.cc +++ b/paddle/fluid/framework/parallel_executor.cc @@ -1314,7 +1314,7 @@ void ParallelExecutor::InitExecutorPrivateMemberInfo( device_name = "XPU"; } else { PADDLE_THROW( - platform::errors::Unavailable("Only CPU/CUDA/XPU is supportted. " + platform::errors::Unavailable("Only CPU/CUDA/XPU is supported. " "please use CPU/CUDA/XPU backend.")); } diff --git a/paddle/fluid/framework/program_utils.cc b/paddle/fluid/framework/program_utils.cc index 1147046c93d29..04b6adf880fc7 100644 --- a/paddle/fluid/framework/program_utils.cc +++ b/paddle/fluid/framework/program_utils.cc @@ -158,9 +158,9 @@ void ProgramProcessor::AddDepToBlockOp(const BlockDesc &block) { } else if (op_type.compare("conditional_block") == 0) { op_input_var_vec = &((*op_inputs)["kInputs"]); } else { - // Only support while_op and conditinal_block_op now + // Only support while_op and conditional_block_op now LOG(WARNING) - << "Currently, only support while_op and conditinal_block_op.\n"; + << "Currently, only support while_op and conditional_block_op.\n"; continue; } diff --git a/paddle/fluid/framework/proto_desc.h b/paddle/fluid/framework/proto_desc.h index 08c78f6e09f73..b8118ef805bde 100644 --- a/paddle/fluid/framework/proto_desc.h +++ b/paddle/fluid/framework/proto_desc.h @@ -26,7 +26,7 @@ constexpr int kNoneBlockIndex = -1; constexpr int kNoneProcessMeshIndex = -1; // If a attribute name has a certain suffix, it means that the -// atrribute is a distributed-related attribute for auto parallel. +// attribute is a distributed-related attribute for auto parallel. // e.g., "mesh_id@AUTO_PARALLEL". constexpr char kAutoParallelSuffix[] = "@AUTO_PARALLEL"; diff --git a/paddle/fluid/framework/prune.cc b/paddle/fluid/framework/prune.cc index 93467b549d6e9..9a4be9ed1f03e 100644 --- a/paddle/fluid/framework/prune.cc +++ b/paddle/fluid/framework/prune.cc @@ -168,7 +168,7 @@ int GetOpRole(const proto::OpDesc& op_desc) { } } // If attr op_role is not found, it may be operator created in c++ test, like - // prune_test.cc. In that case, the op_role should be defaut value, which is + // prune_test.cc. In that case, the op_role should be default value, which is // kNotSpecified. return static_cast(OpRole::kNotSpecified); } @@ -261,7 +261,7 @@ void prune_impl(const proto::ProgramDesc& input, for (auto op_iter = ops.rbegin(); op_iter != ops.rend(); ++op_iter) { auto& op_desc = *op_iter; - // TODO(wanghaipeng03) reconstruct the follwing if/else block + // TODO(wanghaipeng03) reconstruct the following if/else block // to extract common code // // bool should_run_flag = false; @@ -281,7 +281,7 @@ void prune_impl(const proto::ProgramDesc& input, // // should_run.push_back(should_run_flag); // if (should_run_flag) { - // for (auto & var: op_desc.iputs()) { + // for (auto & var: op_desc.inputs()) { // for (....) { // if (.....) { // dependent_vars->insert(argu); @@ -304,7 +304,7 @@ void prune_impl(const proto::ProgramDesc& input, add_dependent_var(argu); } } - // NOTE(dev): All attibute with VarDesc type is considered as Input, + // NOTE(dev): All attribute with VarDesc type is considered as Input, // so they shall be added into dependent_vars. for (auto& attr : op_desc.attrs()) { if (attr.type() == proto::AttrType::VAR) { @@ -391,7 +391,7 @@ void prune_impl(const proto::ProgramDesc& input, std::vector sub_indices; GetSubBlocksIndices(*op, &sub_indices); for (auto& sub_index : sub_indices) { - // create a copy of dependent_vars to avoid being overwrited by the + // create a copy of dependent_vars to avoid being overwritten by the // other sub_block std::unordered_set dependent_vars_copy = sub_block_dependent_vars; @@ -438,7 +438,7 @@ void prune_impl(const proto::ProgramDesc& input, add_var_names(arg); } } - // NOTE(dev): All attibute with VarDesc type is considered as Input, + // NOTE(dev): All attribute with VarDesc type is considered as Input, // so they shall be added into dependent_vars. for (auto& attr : op.attrs()) { if (attr.type() == proto::AttrType::VAR) { From 99f6b75e2350e79a50aa4b081a856d855955308f Mon Sep 17 00:00:00 2001 From: wanghuancoder Date: Tue, 27 Feb 2024 19:03:07 +0800 Subject: [PATCH 130/282] [PIR] pir onednn support pad pool2d (#62085) * pir onednn support pad pool2d --- .../dialect/operator/ir/ops_onednn_extra.yaml | 12 +++++++++--- paddle/phi/kernels/onednn/pool_grad_kernel.cc | 16 ++++++++++++++++ paddle/phi/kernels/onednn/pool_kernel.cc | 16 ++++++++++++++++ test/legacy_test/op_test.py | 4 ++-- test/legacy_test/test_pool2d_op.py | 12 ++++++++++++ test/mkldnn/test_pool2d_bf16_mkldnn_op.py | 10 +++++++--- test/mkldnn/test_pool2d_int8_mkldnn_op.py | 6 +++++- test/mkldnn/test_pool2d_mkldnn_op.py | 4 ++++ 8 files changed, 71 insertions(+), 9 deletions(-) diff --git a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml index c63c96e28c433..f51b6d7e9a3be 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml @@ -190,16 +190,22 @@ - op : nearest_interp -# - op : pad +- op : pad - op : pad3d extra_args : data_format_tensors : x dynamic_fallback : True -# - op : pool2d +- op : pool2d + extra_args : bool use_quantizer=false, str mkldnn_data_type="float32", bool is_test=false + data_format_tensors : x + dynamic_fallback : True -# - op : pool2d_grad +- op : pool2d_grad + extra_args : bool use_quantizer=false, str mkldnn_data_type="float32", bool is_test=false + data_format_tensors : x, out, out_grad + dynamic_fallback : True - op : prelu extra_args : bool is_test=false, str mkldnn_data_type="float32" diff --git a/paddle/phi/kernels/onednn/pool_grad_kernel.cc b/paddle/phi/kernels/onednn/pool_grad_kernel.cc index f5b10186a4ebc..db62f7d51b3ad 100644 --- a/paddle/phi/kernels/onednn/pool_grad_kernel.cc +++ b/paddle/phi/kernels/onednn/pool_grad_kernel.cc @@ -18,6 +18,21 @@ #include "paddle/phi/core/kernel_registry.h" namespace phi { +bool Pool2dGradCheckIfOneDNNSupport(const KernelContext* ctx) { + if (ctx->AttrAt(8) == false) { + // adaptive + return true; + } + // oneDNN is supporting only unchangable in size pool window + auto src_tz = common::vectorize(ctx->InputAt(0).dims()); + const TensorRef& kernel_size_tmp = ctx->AttrAt(0); + IntArray kernel_size_array = IntArray(*kernel_size_tmp.Get()); + std::vector kernel_size = kernel_size_array.GetData(); + // Fast but not exhaustive check + return ((src_tz[src_tz.size() - 1] % kernel_size[1] == 0) && + (src_tz[src_tz.size() - 2] % kernel_size[0] == 0)); +} + template void Pool2dGradKernel(const Context& dev_ctx, const DenseTensor& x, @@ -100,4 +115,5 @@ PD_REGISTER_KERNEL(pool2d_grad, float, phi::dtype::bfloat16) { kernel->get_kerneltype_forvar_fn_ = phi::PoolOpGradGetKernelTypeForVar; + kernel->check_if_onednn_kernel_support_ = phi::Pool2dGradCheckIfOneDNNSupport; } diff --git a/paddle/phi/kernels/onednn/pool_kernel.cc b/paddle/phi/kernels/onednn/pool_kernel.cc index 655cd67ab52df..aa43915cd4f12 100644 --- a/paddle/phi/kernels/onednn/pool_kernel.cc +++ b/paddle/phi/kernels/onednn/pool_kernel.cc @@ -18,6 +18,21 @@ #include "paddle/phi/core/kernel_registry.h" namespace phi { +bool Pool2dCheckIfOneDNNSupport(const KernelContext* ctx) { + if (ctx->AttrAt(8) == false) { + // adaptive + return true; + } + // oneDNN is supporting only unchangable in size pool window + auto src_tz = common::vectorize(ctx->InputAt(0).dims()); + const TensorRef& kernel_size_tmp = ctx->AttrAt(0); + IntArray kernel_size_array = IntArray(*kernel_size_tmp.Get()); + std::vector kernel_size = kernel_size_array.GetData(); + // Fast but not exhaustive check + return ((src_tz[src_tz.size() - 1] % kernel_size[1] == 0) && + (src_tz[src_tz.size() - 2] % kernel_size[0] == 0)); +} + template void Pool2dKernel(const Context& dev_ctx, const DenseTensor& x, @@ -104,4 +119,5 @@ PD_REGISTER_KERNEL(pool2d, uint8_t, phi::dtype::bfloat16) { kernel->get_kerneltype_forvar_fn_ = phi::PoolOpGetKernelTypeForVar; + kernel->check_if_onednn_kernel_support_ = phi::Pool2dCheckIfOneDNNSupport; } diff --git a/test/legacy_test/op_test.py b/test/legacy_test/op_test.py index a9aa986f61120..41b9caed79480 100644 --- a/test/legacy_test/op_test.py +++ b/test/legacy_test/op_test.py @@ -2643,7 +2643,7 @@ def _is_skip_name(self, name): static_checker.check() outs, fetch_list = static_checker.outputs, static_checker.fetch_list - if check_pir_onednn: + if check_pir_onednn and place == base.CPUPlace(): with pir_executor_guard(): pir_onednn_static_checker = StaticChecker(self, self.outputs) pir_onednn_static_checker.check() @@ -3313,7 +3313,7 @@ def check_grad_with_place( atol, ) - if check_pir_onednn: + if check_pir_onednn and place == base.CPUPlace(): with pir_executor_guard(): self.check_grad_with_place_for_static( user_defined_grads, diff --git a/test/legacy_test/test_pool2d_op.py b/test/legacy_test/test_pool2d_op.py index c27ba701bab89..0a63c3f85352d 100644 --- a/test/legacy_test/test_pool2d_op.py +++ b/test/legacy_test/test_pool2d_op.py @@ -428,11 +428,13 @@ def test_check_output(self): check_dygraph=(not self.use_mkldnn), check_cinn=True, check_pir=True, + check_pir_onednn=self.check_pir_onednn, ) else: self.check_output( check_dygraph=(not self.use_mkldnn), check_pir=True, + check_pir_onednn=self.check_pir_onednn, ) def test_check_grad(self): @@ -448,6 +450,7 @@ def test_check_grad(self): check_dygraph=(not self.use_mkldnn), check_cinn=True, check_pir=True, + check_pir_onednn=self.check_pir_onednn, ) elif self.pool_type != "max": self.check_grad( @@ -456,6 +459,7 @@ def test_check_grad(self): max_relative_error=0.07, check_dygraph=(not self.use_mkldnn), check_pir=True, + check_pir_onednn=self.check_pir_onednn, ) def init_data_format(self): @@ -599,6 +603,7 @@ def test_check_output(self): place, check_dygraph=(not self.use_mkldnn), check_cinn=True, + check_pir_onednn=self.check_pir_onednn, ) def test_check_grad(self): @@ -615,6 +620,7 @@ def test_check_grad(self): 'Out', check_dygraph=(not self.use_mkldnn), check_cinn=True, + check_pir_onednn=self.check_pir_onednn, ) cls_name = "{}_{}".format(parent.__name__, "CUDNNFp16Op") @@ -640,6 +646,7 @@ def test_check_output(self): place, check_dygraph=(not self.use_mkldnn), check_cinn=True, + check_pir_onednn=self.check_pir_onednn, ) def test_check_grad(self): @@ -656,6 +663,7 @@ def test_check_grad(self): 'Out', check_dygraph=(not self.use_mkldnn), check_cinn=True, + check_pir_onednn=self.check_pir_onednn, ) cls_name = "{}_{}".format(parent.__name__, "Fp16Op") @@ -679,6 +687,7 @@ def test_check_output(self): place, check_dygraph=(not self.use_mkldnn), check_cinn=True, + check_pir_onednn=self.check_pir_onednn, ) def test_check_grad(self): @@ -690,6 +699,7 @@ def test_check_grad(self): 'Out', check_dygraph=(not self.use_mkldnn), check_cinn=True, + check_pir_onednn=self.check_pir_onednn, ) cls_name = "{}_{}".format(parent.__name__, "Bf16Op") @@ -1025,6 +1035,7 @@ def test_check_grad(self): max_relative_error=1.00, check_cinn=True, check_pir=True, + check_pir_onednn=self.check_pir_onednn, ) elif self.pool_type == "max": self.check_grad( @@ -1033,6 +1044,7 @@ def test_check_grad(self): max_relative_error=1.00, check_cinn=True, check_pir=True, + check_pir_onednn=self.check_pir_onednn, ) diff --git a/test/mkldnn/test_pool2d_bf16_mkldnn_op.py b/test/mkldnn/test_pool2d_bf16_mkldnn_op.py index 0b510e5e7fa50..1a994c588c2b6 100644 --- a/test/mkldnn/test_pool2d_bf16_mkldnn_op.py +++ b/test/mkldnn/test_pool2d_bf16_mkldnn_op.py @@ -197,7 +197,7 @@ def setUp(self): self.outputs = {'Out': convert_float_to_uint16(output)} def test_check_output(self): - self.check_output_with_place(core.CPUPlace()) + self.check_output_with_place(core.CPUPlace(), check_pir_onednn=True) def test_check_grad(self): x_grad = pool2d_backward_naive( @@ -215,7 +215,11 @@ def test_check_grad(self): ) x_grad = x_grad / np.prod(self.outputs['Out'].shape) self.check_grad_with_place( - core.CPUPlace(), {'X'}, 'Out', user_defined_grads=[x_grad] + core.CPUPlace(), + {'X'}, + 'Out', + user_defined_grads=[x_grad], + check_pir_onednn=True, ) @@ -247,7 +251,7 @@ def setUp(self): self.outputs = {'Out': convert_float_to_uint16(output)} def test_check_output(self): - self.check_output_with_place(core.CPUPlace()) + self.check_output_with_place(core.CPUPlace(), check_pir_onednn=True) def test_check_grad(self): pass diff --git a/test/mkldnn/test_pool2d_int8_mkldnn_op.py b/test/mkldnn/test_pool2d_int8_mkldnn_op.py index 5b41081821b3e..ece6031105426 100644 --- a/test/mkldnn/test_pool2d_int8_mkldnn_op.py +++ b/test/mkldnn/test_pool2d_int8_mkldnn_op.py @@ -24,6 +24,7 @@ class TestPool2DMKLDNNInt8_Op(TestPool2D_Op): def init_kernel_type(self): self.use_mkldnn = True + self.check_pir_onednn = True def init_data_type(self): self.dtype = np.int8 @@ -54,7 +55,10 @@ def setUp(self): def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode self.check_output_with_place( - core.CPUPlace(), atol=1e-5, check_dygraph=False + core.CPUPlace(), + atol=1e-5, + check_dygraph=False, + check_pir_onednn=True, ) def test_check_grad(self): diff --git a/test/mkldnn/test_pool2d_mkldnn_op.py b/test/mkldnn/test_pool2d_mkldnn_op.py index e88b20750349c..439761205ba9e 100644 --- a/test/mkldnn/test_pool2d_mkldnn_op.py +++ b/test/mkldnn/test_pool2d_mkldnn_op.py @@ -30,6 +30,7 @@ def create_test_mkldnn_use_ceil_class(parent): class TestMKLDNNPool2DUseCeilCase(parent): def init_kernel_type(self): self.use_mkldnn = True + self.check_pir_onednn = True def init_ceil_mode(self): self.ceil_mode = True @@ -51,6 +52,7 @@ def create_test_mkldnn_class(parent): class TestMKLDNNCase(parent): def init_kernel_type(self): self.use_mkldnn = True + self.check_pir_onednn = True def init_data_type(self): self.dtype = np.float32 @@ -78,6 +80,7 @@ def init_pool_type(self): def init_kernel_type(self): self.use_mkldnn = True + self.check_pir_onednn = True def init_test_case(self): self.ksize = [1, 1] @@ -128,6 +131,7 @@ def init_shape(self): def init_kernel_type(self): self.use_mkldnn = True + self.check_pir_onednn = True def init_data_type(self): self.dtype = np.float32 From de1f8ab5dafd2c1beb06fbcafe850dc1274bb2eb Mon Sep 17 00:00:00 2001 From: HongyuJia Date: Tue, 27 Feb 2024 19:50:53 +0800 Subject: [PATCH 131/282] [CINN] CINN plug-in Inference Executor (#61949) * [CINN] CINN plug-in Inference Executor * Fix headerfile * Fix compile dependency * Change cinn_pass_entry position * cinn_cc_library -> cc_library * Fix compile error * Resolve dependency * Fix sub_graph_checker * Change cmake * Fix compile error * Refine inference-related codes * Refine inference-related codes * Refine inference-related codes * Refine inference-related codes * Fix compile error * Add doc * remove preprocess macro * Remove doc --- .../operator/transforms/CMakeLists.txt | 4 + .../operator/transforms/add_cinn_pass.cc | 137 ++++++++++++++++++ .../operator/transforms/add_cinn_pass.h | 31 ++++ paddle/fluid/inference/api/CMakeLists.txt | 46 +++--- paddle/fluid/inference/api/analysis_config.cc | 16 +- .../fluid/inference/api/analysis_predictor.cc | 25 +++- .../inference/api/paddle_analysis_config.h | 6 +- paddle/fluid/pybind/CMakeLists.txt | 2 +- paddle/fluid/pybind/inference_api.cc | 1 + paddle/fluid/pybind/pir.cc | 104 ++----------- 10 files changed, 241 insertions(+), 131 deletions(-) create mode 100644 paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.cc create mode 100644 paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.h diff --git a/paddle/cinn/hlir/dialect/operator/transforms/CMakeLists.txt b/paddle/cinn/hlir/dialect/operator/transforms/CMakeLists.txt index 6c5f09c3ebe3d..00eecee4d883c 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/CMakeLists.txt +++ b/paddle/cinn/hlir/dialect/operator/transforms/CMakeLists.txt @@ -14,4 +14,8 @@ if(NOT CINN_ONLY) cinn_cc_library(cinn_transforms SRCS ${cinn_transforms_srcs} DEPS ${cinn_transforms_deps}) + cc_library( + add_cinn_pass + SRCS add_cinn_pass.cc + DEPS op_dialect pir cinn_op_dialect cinnapi pir_transforms cinn_transforms) endif() diff --git a/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.cc new file mode 100644 index 0000000000000..9b18ed609dda9 --- /dev/null +++ b/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.cc @@ -0,0 +1,137 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.h" + +#include "paddle/common/errors.h" +#include "paddle/common/flags.h" +#include "paddle/fluid/pir/dialect/operator/ir/op_dialect.h" +#include "paddle/phi/core/enforce.h" +#include "paddle/pir/include/core/ir_context.h" +#include "paddle/pir/include/core/program.h" +#include "paddle/pir/include/dialect/shape/ir/shape_dialect.h" +#include "paddle/pir/include/pass/pass_manager.h" + +#include "paddle/cinn/hlir/dialect/operator/ir/op_dialect.h" +#include "paddle/cinn/hlir/dialect/operator/transforms/add_broadcast_to_elementwise_pass.h" +#include "paddle/cinn/hlir/dialect/operator/transforms/dynamic_reshape_pass.h" +#include "paddle/cinn/hlir/dialect/operator/transforms/fuse_shape_ops_into_generate_shape_op_pass.h" +#include "paddle/cinn/hlir/dialect/operator/transforms/group_merge/check_infer_symbolic_pass.h" +#include "paddle/cinn/hlir/dialect/operator/transforms/group_merge/convert_0d_to_1d_pass.h" +#include "paddle/cinn/hlir/dialect/operator/transforms/group_merge/convert_dynamic_to_static_dim_pass.h" +#include "paddle/cinn/hlir/dialect/operator/transforms/group_merge/convert_static_dim_to_dynamic_pass.h" +#include "paddle/cinn/hlir/dialect/operator/transforms/group_merge/divide_group_op_to_fusion_op_pass.h" +#include "paddle/cinn/hlir/dialect/operator/transforms/group_merge/move_generate_shape_ops_to_prologue_pass.h" +#include "paddle/cinn/hlir/dialect/operator/transforms/group_merge/simplify_dim_expr_pass.h" +#include "paddle/cinn/hlir/dialect/operator/transforms/group_merge/substitute_dim_expr_based_on_constraints_pass.h" +#include "paddle/cinn/hlir/dialect/operator/transforms/insert_broadcast_pass.h" +#include "paddle/cinn/hlir/dialect/operator/transforms/lower_cinn_fusion_op_pass.h" +#include "paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.h" +#include "paddle/cinn/hlir/dialect/operator/transforms/remove_unchanged_reshape_pass.h" +#include "paddle/cinn/hlir/dialect/operator/transforms/replace_dynamic_expand_pass.h" +#include "paddle/cinn/hlir/dialect/operator/transforms/split_generate_shape_into_shape_ops_pass.h" +#include "paddle/fluid/pir/transforms/build_cinn_pass.h" +#include "paddle/fluid/pir/transforms/dead_code_elimination_pass.h" +#include "paddle/fluid/pir/transforms/shape_optimization_pass.h" + +COMMON_DECLARE_bool(print_ir); +COMMON_DECLARE_bool(check_infer_symbolic); + +namespace cinn::dialect::ir { + +namespace { +bool HasDynamicShape(const pir::Program &program) { + for (const auto &op : *program.block()) { + if (op.isa()) { + continue; + } + for (uint32_t i = 0; i < op.num_results(); ++i) { + if (op.result(i) && op.result(i).type()) { + auto shape_type = + op.result(i).type().dyn_cast(); + if (shape_type && shape_type.IsDynamicShape()) { + return true; + } + } + } + } + return false; +} +} // namespace + +void AddCinnPass(std::shared_ptr &pass_manager, // NOLINT + pir::Program &program) { // NOLINT + pir::IrContext *ctx = pir::IrContext::Instance(); + ctx->GetOrRegisterDialect(); + ctx->GetOrRegisterDialect(); + ctx->GetOrRegisterDialect(); + + bool has_dynamic_shape = HasDynamicShape(program); + + if (FLAGS_print_ir) { + pass_manager->EnableIRPrinting(); + } + + pass_manager->AddPass(cinn::dialect::ir::CreateConvert0DTo1DPass()); + if (!has_dynamic_shape && FLAGS_check_infer_symbolic) { + pass_manager->AddPass(pir::CreateShapeOptimizationPass()); + pass_manager->AddPass(cinn::dialect::ir::CreateCheckInferSymbolicPass()); + } + pass_manager->AddPass(cinn::dialect::ir::CreatePdOpToCinnOpPass()); + pass_manager->AddPass(cinn::dialect::ir::CreateRemoveUnchangedReshapePass()); + pass_manager->AddPass( + cinn::dialect::ir::CreateAddBroadcastToElementwisePass()); + pass_manager->AddPass(pir::CreateDeadCodeEliminationPass()); + + if (has_dynamic_shape) { + pass_manager->AddPass(pir::CreateShapeOptimizationPass()); + pass_manager->AddPass(cinn::dialect::ir::CreateSimplifyDimExprPass()); + pass_manager->AddPass( + cinn::dialect::ir::CreateSubstituteDimExprBasedOnConstraintsPass()); + pass_manager->AddPass(cinn::dialect::ir::CreateInsertBroadcastPass()); + pass_manager->AddPass(pir::CreateShapeOptimizationPass()); + pass_manager->AddPass( + cinn::dialect::ir::CreateFuseShapeOpsIntoGenerateShapeOpPass()); + pass_manager->AddPass(pir::CreateDeadCodeEliminationPass()); + pass_manager->AddPass(pir::CreateShapeOptimizationPass()); + } + + pass_manager->AddPass(pir::CreateBuildCinnPass()); + + pass_manager->AddPass( + cinn::dialect::ir::CreateMoveGenerateShapeOpsToProloguePass()); + pass_manager->AddPass(cinn::dialect::ir::CreateDivideGroupOpToFusionOpPass()); + pass_manager->AddPass(cinn::dialect::ir::CreateDynamicReshapeOpPass()); + pass_manager->AddPass(cinn::dialect::ir::CreateReplaceDynamicExpandOpPass()); + pass_manager->AddPass(pir::CreateDeadCodeEliminationPass()); + + bool force_static_shape = false; + if (auto pass = cinn::dialect::ir::CreateConvertDynamicToStaticDimPass()) { + pass_manager->AddPass(std::move(pass.value())); + force_static_shape = true; + } + if (auto pass = cinn::dialect::ir::CreateConvertStaticDimToDynamicPass()) { + pass_manager->AddPass(std::move(pass.value())); + } + + if (has_dynamic_shape && !force_static_shape) { + pass_manager->AddPass( + cinn::dialect::ir::CreateLowerCinnDyShapeFusionOpPass()); + } + pass_manager->AddPass(cinn::dialect::ir::CreateLowerCinnFusionOpPass()); + pass_manager->AddPass( + cinn::dialect::ir::CreateSplitGenerateShapeIntoShapeOpsPass()); +} + +} // namespace cinn::dialect::ir diff --git a/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.h b/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.h new file mode 100644 index 0000000000000..e66aff15ac0c8 --- /dev/null +++ b/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.h @@ -0,0 +1,31 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +namespace pir { + +class PassManager; +class Program; + +} // namespace pir + +namespace cinn::dialect::ir { + +void AddCinnPass(std::shared_ptr &pass_manager, // NOLINT + pir::Program &program); // NOLINT + +} // namespace cinn::dialect::ir diff --git a/paddle/fluid/inference/api/CMakeLists.txt b/paddle/fluid/inference/api/CMakeLists.txt index ea648d8574c94..eda204189c8a6 100755 --- a/paddle/fluid/inference/api/CMakeLists.txt +++ b/paddle/fluid/inference/api/CMakeLists.txt @@ -77,33 +77,29 @@ if(WITH_GPU AND TENSORRT_FOUND) set(inference_deps ${inference_deps} tensorrt_engine tensorrt_converter) endif() +set(ANALYSIS_PREDICTOR_SRCS analysis_predictor.cc resource_manager.cc + infer_context.cc ${mkldnn_quantizer_src}) +set(ANALYSIS_PREDICTOR_DEPS + ${inference_deps} + zero_copy_tensor + ir_pass_manager + op_compatible_info + infer_io_utils + model_utils + fleet_executor) + if(WITH_ONNXRUNTIME) - cc_library( - analysis_predictor - SRCS analysis_predictor.cc onnxruntime_predictor.cc resource_manager.cc - infer_context.cc ${mkldnn_quantizer_src} - DEPS ${inference_deps} - zero_copy_tensor - ir_pass_manager - op_compatible_info - infer_io_utils - model_utils - onnxruntime - paddle2onnx - fleet_executor) -else() - cc_library( - analysis_predictor - SRCS analysis_predictor.cc resource_manager.cc infer_context.cc - ${mkldnn_quantizer_src} - DEPS ${inference_deps} - zero_copy_tensor - ir_pass_manager - op_compatible_info - infer_io_utils - model_utils - fleet_executor) + set(ANALYSIS_PREDICTOR_SRCS ${ANALYSIS_PREDICTOR_SRCS} + onnxruntime_predictor.cc) + set(ANALYSIS_PREDICTOR_DEPS ${ANALYSIS_PREDICTOR_DEPS} onnxruntime + paddle2onnx) +elseif(WITH_CINN) + set(ANALYSIS_PREDICTOR_DEPS ${ANALYSIS_PREDICTOR_DEPS} add_cinn_pass) endif() +cc_library( + analysis_predictor + SRCS ${ANALYSIS_PREDICTOR_SRCS} + DEPS ${ANALYSIS_PREDICTOR_DEPS}) if(WITH_ONNXRUNTIME AND WIN32) # Copy onnxruntime for some c++ test in Windows, since the test will diff --git a/paddle/fluid/inference/api/analysis_config.cc b/paddle/fluid/inference/api/analysis_config.cc index 98f031aa14719..0ec5151a92bc5 100644 --- a/paddle/fluid/inference/api/analysis_config.cc +++ b/paddle/fluid/inference/api/analysis_config.cc @@ -535,7 +535,7 @@ AnalysisConfig::AnalysisConfig(const AnalysisConfig &other) { CP_MEMBER(with_profile_); // cinn compiler related. - CP_MEMBER(use_cinn_compiler_); + CP_MEMBER(use_cinn_); // glog related. CP_MEMBER(with_glog_info_); @@ -606,7 +606,7 @@ AnalysisConfig::AnalysisConfig(const AnalysisConfig &other) { #undef CP_MEMBER Update(); - if (use_tensorrt_ || use_cinn_compiler_) { + if (use_tensorrt_ || use_cinn_) { // Update() will reset all the passes, when some tensorRT pass is deleted in // other.pass_builder(), it will set again, so we just remove the // deleted_pass. @@ -987,7 +987,7 @@ void AnalysisConfig::Update() { } // TODO(wilber): An ugly method to update pass, need to be fixed. - if (use_cinn_compiler_) { + if (use_cinn_) { pass_builder()->ClearPasses(); for (const auto &pass : kCINNCompilerPasses) { pass_builder()->AppendPass(pass); @@ -1479,7 +1479,7 @@ std::string AnalysisConfig::Summary() { } // cinn compiler - os.InsertRow({"use_cinn_compiler", use_cinn_compiler_ ? "true" : "false"}); + os.InsertRow({"use_cinn_compiler", use_cinn_ ? "true" : "false"}); // ir info os.InsertRow( @@ -1603,9 +1603,9 @@ void AnalysisConfig::Exp_EnableMixedPrecisionOps( mixed_white_list_ = white_list; } -void AnalysisConfig::Exp_EnableCINNCompiler() { +void AnalysisConfig::EnableCINN() { #ifdef PADDLE_WITH_CINN - use_cinn_compiler_ = true; + use_cinn_ = true; Update(); #else PADDLE_THROW(platform::errors::Unavailable( @@ -1614,8 +1614,6 @@ void AnalysisConfig::Exp_EnableCINNCompiler() { #endif } -bool AnalysisConfig::cinn_compiler_enabled() const { - return use_cinn_compiler_; -} +bool AnalysisConfig::cinn_enabled() const { return use_cinn_; } } // namespace paddle diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index 076d3b567fa86..6ce0ce2570d4f 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -105,6 +105,10 @@ #include "paddle/fluid/platform/device/gpu/cuda/cuda_profiler.h" #endif +#ifdef PADDLE_WITH_CINN +#include "paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.h" +#endif + #include "paddle/common/flags.h" #include "paddle/fluid/ir_adaptor/translator/translate.h" #include "paddle/fluid/pir/transforms/constant_folding_pass.h" @@ -874,6 +878,21 @@ bool AnalysisPredictor::PrepareExecutor() { DecompProgram decomp_object(pir_program_.get()); decomp_object.decomp_program(); } +#ifdef PADDLE_WITH_CINN + if (config_.cinn_enabled()) { + VLOG(4) << "[CINN] Begin AddCinnPass"; + auto cinn_pm = std::make_shared<::pir::PassManager>( + ::pir::IrContext::Instance(), 2); + cinn::dialect::ir::AddCinnPass(cinn_pm, *pir_program_.get()); + if (!config_.glog_info_disabled()) { + cinn_pm->EnablePrintStatistics(); + } + if (config_.ir_debug_) { + cinn_pm->EnableIRPrinting(); + } + cinn_pm->Run(pir_program_.get()); + } +#endif if (config_.use_gpu()) { ::pir::PassManager gpu_pm(::pir::IrContext::Instance(), 2); @@ -1793,8 +1812,8 @@ void AnalysisPredictor::PrepareArgument() { argument_->SetMKLDNNEnabledOpTypes(config_.mkldnn_enabled_op_types_); } - if (config_.use_cinn_compiler_) { - argument_->SetUseCinnCompiler(config_.use_cinn_compiler_); + if (config_.cinn_enabled()) { + argument_->SetUseCinnCompiler(true); } #ifdef PADDLE_WITH_DNNL @@ -1869,7 +1888,7 @@ void AnalysisPredictor::PrepareArgument() { LOG(INFO) << "Model is mixed precision type with " << model_precision_ << ", we will use a new PassStrategy. Note that only GPU/XPU " "backend is supported for now."; - if (!config_.use_cinn_compiler_) { + if (!config_.cinn_enabled()) { const auto &deleted_passes = pass_builder->GetAllDeletedPasses(); if (config_.tensorrt_engine_enabled()) { pass_builder->ClearPasses(); diff --git a/paddle/fluid/inference/api/paddle_analysis_config.h b/paddle/fluid/inference/api/paddle_analysis_config.h index 473977ef35d95..cae544ff2c234 100644 --- a/paddle/fluid/inference/api/paddle_analysis_config.h +++ b/paddle/fluid/inference/api/paddle_analysis_config.h @@ -1196,7 +1196,7 @@ struct PD_INFER_DECL AnalysisConfig { /// /// \brief Enable use cinn compiler optimization. /// - void Exp_EnableCINNCompiler(); + void EnableCINN(); /// /// \brief A boolean state telling whether the CINN compiler optimization is @@ -1204,7 +1204,7 @@ struct PD_INFER_DECL AnalysisConfig { /// /// \return bool Whether the CINN compiler optimization is turned on. /// - bool cinn_compiler_enabled() const; + bool cinn_enabled() const; protected: // Update the config. @@ -1350,7 +1350,7 @@ struct PD_INFER_DECL AnalysisConfig { bool lite_zero_copy_; // CINN compiler related. - bool use_cinn_compiler_{false}; + bool use_cinn_{false}; // XPU related. bool use_xpu_{false}; diff --git a/paddle/fluid/pybind/CMakeLists.txt b/paddle/fluid/pybind/CMakeLists.txt index 48f16f87f9aeb..f67a74bf3f8ae 100755 --- a/paddle/fluid/pybind/CMakeLists.txt +++ b/paddle/fluid/pybind/CMakeLists.txt @@ -51,7 +51,7 @@ set(PYBIND_DEPS if(WITH_CINN) set(PYBIND_DEPS ${PYBIND_DEPS} pir_transforms cinn_transforms - sub_graph_checker) + sub_graph_checker add_cinn_pass) endif() if(WITH_PSCORE) diff --git a/paddle/fluid/pybind/inference_api.cc b/paddle/fluid/pybind/inference_api.cc index ee0244e853258..268806509031e 100644 --- a/paddle/fluid/pybind/inference_api.cc +++ b/paddle/fluid/pybind/inference_api.cc @@ -978,6 +978,7 @@ void BindAnalysisConfig(py::module *m) { .def("enable_mkldnn", &AnalysisConfig::EnableMKLDNN) .def("disable_mkldnn", &AnalysisConfig::DisableMKLDNN) .def("mkldnn_enabled", &AnalysisConfig::mkldnn_enabled) + .def("enable_cinn", &AnalysisConfig::EnableCINN) .def("set_cpu_math_library_num_threads", &AnalysisConfig::SetCpuMathLibraryNumThreads) .def("cpu_math_library_num_threads", diff --git a/paddle/fluid/pybind/pir.cc b/paddle/fluid/pybind/pir.cc index 35c19c8f00c76..de3ec976c7dea 100644 --- a/paddle/fluid/pybind/pir.cc +++ b/paddle/fluid/pybind/pir.cc @@ -28,10 +28,6 @@ #include "paddle/fluid/ir_adaptor/translator/program_translator.h" #include "paddle/fluid/ir_adaptor/translator/translate.h" #include "paddle/fluid/ir_adaptor/translator/utils.h" -#include "paddle/fluid/pybind/control_flow_api.h" -#include "paddle/fluid/pybind/pybind_variant_caster.h" - -#include "paddle/cinn/hlir/dialect/operator/transforms/group_merge/convert_0d_to_1d_pass.h" #include "paddle/fluid/pir/dialect/kernel/ir/kernel_type.h" #include "paddle/fluid/pir/dialect/operator/interface/op_yaml_info.h" #include "paddle/fluid/pir/dialect/operator/ir/api_builder.h" @@ -43,7 +39,6 @@ #include "paddle/fluid/pir/dialect/operator/trait/inplace.h" #include "paddle/fluid/pir/dialect/operator/utils/op_yaml_info_parser.h" #include "paddle/fluid/pir/dialect/operator/utils/utils.h" -#include "paddle/fluid/pir/transforms/dead_code_elimination_pass.h" #include "paddle/fluid/pir/transforms/fusion/conv2d_add_act_fuse_pass.h" #include "paddle/fluid/pir/transforms/fusion/conv2d_add_fuse_pass.h" #include "paddle/fluid/pir/transforms/fusion/conv2d_bn_fuse_pass.h" @@ -64,7 +59,9 @@ #include "paddle/fluid/pir/transforms/map_op_to_another_pass.h" #include "paddle/fluid/pir/transforms/replace_fetch_with_shadow_output_pass.h" #include "paddle/fluid/pir/transforms/shape_optimization_pass.h" +#include "paddle/fluid/pybind/control_flow_api.h" #include "paddle/fluid/pybind/eager_utils.h" +#include "paddle/fluid/pybind/pybind_variant_caster.h" #include "paddle/phi/core/enforce.h" #include "paddle/pir/include/core/attribute.h" #include "paddle/pir/include/core/block.h" @@ -86,25 +83,8 @@ #include "pybind11/stl.h" #ifdef PADDLE_WITH_CINN -#include "paddle/cinn/hlir/dialect/operator/ir/op_dialect.h" -#include "paddle/cinn/hlir/dialect/operator/transforms/add_broadcast_to_elementwise_pass.h" -#include "paddle/cinn/hlir/dialect/operator/transforms/dynamic_reshape_pass.h" -#include "paddle/cinn/hlir/dialect/operator/transforms/fuse_shape_ops_into_generate_shape_op_pass.h" -#include "paddle/cinn/hlir/dialect/operator/transforms/group_merge/check_infer_symbolic_pass.h" -#include "paddle/cinn/hlir/dialect/operator/transforms/group_merge/convert_dynamic_to_static_dim_pass.h" -#include "paddle/cinn/hlir/dialect/operator/transforms/group_merge/convert_static_dim_to_dynamic_pass.h" -#include "paddle/cinn/hlir/dialect/operator/transforms/group_merge/divide_group_op_to_fusion_op_pass.h" -#include "paddle/cinn/hlir/dialect/operator/transforms/group_merge/move_generate_shape_ops_to_prologue_pass.h" -#include "paddle/cinn/hlir/dialect/operator/transforms/group_merge/simplify_dim_expr_pass.h" -#include "paddle/cinn/hlir/dialect/operator/transforms/group_merge/substitute_dim_expr_based_on_constraints_pass.h" -#include "paddle/cinn/hlir/dialect/operator/transforms/insert_broadcast_pass.h" -#include "paddle/cinn/hlir/dialect/operator/transforms/lower_cinn_fusion_op_pass.h" -#include "paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.h" -#include "paddle/cinn/hlir/dialect/operator/transforms/remove_unchanged_reshape_pass.h" -#include "paddle/cinn/hlir/dialect/operator/transforms/replace_dynamic_expand_pass.h" -#include "paddle/cinn/hlir/dialect/operator/transforms/split_generate_shape_into_shape_ops_pass.h" +#include "paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.h" #include "paddle/cinn/hlir/framework/pir_compiler.h" -#include "paddle/fluid/pir/transforms/build_cinn_pass.h" #endif #ifdef PADDLE_WITH_DNNL @@ -162,7 +142,6 @@ USE_PIR_PASS(batch_norm_act_fuse_pass); COMMON_DECLARE_bool(print_ir); COMMON_DECLARE_bool(pir_apply_shape_optimization_pass); -COMMON_DECLARE_bool(check_infer_symbolic); namespace paddle { namespace pybind { @@ -1552,7 +1531,9 @@ void BindUtils(pybind11::module *m) { }); } -static bool HasDynamicShape(const Program &program) { +namespace { + +bool HasDynamicShape(const pir::Program &program) { for (const auto &op : *program.block()) { if (op.isa()) { continue; @@ -1570,79 +1551,22 @@ static bool HasDynamicShape(const Program &program) { return false; } -void AddCinnPass(std::shared_ptr &pass_manager, // NOLINT - Program &program) { // NOLINT +void AddCinnPass(std::shared_ptr &pass_manager, // NOLINT + pir::Program &program) { // NOLINT #ifdef PADDLE_WITH_CINN - pir::IrContext *ctx = pir::IrContext::Instance(); - ctx->GetOrRegisterDialect(); - ctx->GetOrRegisterDialect(); - ctx->GetOrRegisterDialect(); - - bool has_dynamic_shape = HasDynamicShape(program); - - if (FLAGS_print_ir) { - pass_manager->EnableIRPrinting(); - } - - pass_manager->AddPass(cinn::dialect::ir::CreateConvert0DTo1DPass()); - if (!has_dynamic_shape && FLAGS_check_infer_symbolic) { - pass_manager->AddPass(pir::CreateShapeOptimizationPass()); - pass_manager->AddPass(cinn::dialect::ir::CreateCheckInferSymbolicPass()); - } - pass_manager->AddPass(cinn::dialect::ir::CreatePdOpToCinnOpPass()); - pass_manager->AddPass(cinn::dialect::ir::CreateRemoveUnchangedReshapePass()); - pass_manager->AddPass( - cinn::dialect::ir::CreateAddBroadcastToElementwisePass()); - pass_manager->AddPass(pir::CreateDeadCodeEliminationPass()); - - if (has_dynamic_shape) { - pass_manager->AddPass(pir::CreateShapeOptimizationPass()); - pass_manager->AddPass(cinn::dialect::ir::CreateSimplifyDimExprPass()); - pass_manager->AddPass( - cinn::dialect::ir::CreateSubstituteDimExprBasedOnConstraintsPass()); - pass_manager->AddPass(cinn::dialect::ir::CreateInsertBroadcastPass()); - pass_manager->AddPass(pir::CreateShapeOptimizationPass()); - pass_manager->AddPass( - cinn::dialect::ir::CreateFuseShapeOpsIntoGenerateShapeOpPass()); - pass_manager->AddPass(pir::CreateDeadCodeEliminationPass()); - pass_manager->AddPass(pir::CreateShapeOptimizationPass()); - } - - pass_manager->AddPass(pir::CreateBuildCinnPass()); - - pass_manager->AddPass( - cinn::dialect::ir::CreateMoveGenerateShapeOpsToProloguePass()); - pass_manager->AddPass(cinn::dialect::ir::CreateDivideGroupOpToFusionOpPass()); - pass_manager->AddPass(cinn::dialect::ir::CreateDynamicReshapeOpPass()); - pass_manager->AddPass(cinn::dialect::ir::CreateReplaceDynamicExpandOpPass()); - pass_manager->AddPass(pir::CreateDeadCodeEliminationPass()); - - bool force_static_shape = false; - if (auto pass = cinn::dialect::ir::CreateConvertDynamicToStaticDimPass()) { - pass_manager->AddPass(std::move(pass.value())); - force_static_shape = true; - } - if (auto pass = cinn::dialect::ir::CreateConvertStaticDimToDynamicPass()) { - pass_manager->AddPass(std::move(pass.value())); - } - - if (has_dynamic_shape && !force_static_shape) { - pass_manager->AddPass( - cinn::dialect::ir::CreateLowerCinnDyShapeFusionOpPass()); - } - pass_manager->AddPass(cinn::dialect::ir::CreateLowerCinnFusionOpPass()); - pass_manager->AddPass( - cinn::dialect::ir::CreateSplitGenerateShapeIntoShapeOpsPass()); + cinn::dialect::ir::AddCinnPass(pass_manager, program); #else - PADDLE_THROW(platform::errors::Unimplemented( + PADDLE_THROW(common::errors::Unimplemented( "Currently we only support CINN Pass for Pir under @to_static, please " "compile PaddlePaddle with CINN")); #endif } +} // namespace + void InferSymbolicShapePass( - std::shared_ptr &pass_manager, // NOLINT - Program &program) { // NOLINT + std::shared_ptr &pass_manager, // NOLINT + pir::Program &program) { // NOLINT pir::IrContext *ctx = pir::IrContext::Instance(); ctx->GetOrRegisterDialect(); if (HasDynamicShape(program) && FLAGS_pir_apply_shape_optimization_pass) { From 74777a710fe6dbc39d4702c9a2851d04f05815fc Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Tue, 27 Feb 2024 19:53:02 +0800 Subject: [PATCH 132/282] [SOT][3.12] Fix SOT analysis used names in Python 3.12 (#62131) --- .../instruction_utils/opcode_analysis.py | 11 +++++------ test/sot/skip_files_py312 | 1 - test/sot/test_analysis_inputs.py | 6 +++++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/python/paddle/jit/sot/opcode_translator/instruction_utils/opcode_analysis.py b/python/paddle/jit/sot/opcode_translator/instruction_utils/opcode_analysis.py index eb8cb1735bddf..93722f42c9602 100644 --- a/python/paddle/jit/sot/opcode_translator/instruction_utils/opcode_analysis.py +++ b/python/paddle/jit/sot/opcode_translator/instruction_utils/opcode_analysis.py @@ -43,6 +43,7 @@ def __or__(self, other): def is_read_opcode(opname): if opname in [ "LOAD_FAST", + "LOAD_FAST_CHECK", "LOAD_DEREF", "LOAD_NAME", "LOAD_GLOBAL", @@ -76,7 +77,7 @@ def analysis_used_names( instructions: list[Instruction], current_instr_idx: int, stop_instr_idx: int | None = None, -) -> OrderedSet[str]: +) -> tuple[OrderedSet[str], OrderedSet[str]]: """ Analyze the inputs of the instructions from current_instr_idx to stop_instr_idx. @@ -87,13 +88,11 @@ def analysis_used_names( If None, the analysis will stop at the end of the instructions. Returns: - set[str]: The analysis result. + State: The analysis result. """ root_state = State(OrderedSet(), OrderedSet(), OrderedSet()) - def fork( - state: State, start: int, jump: bool, jump_target: int - ) -> OrderedSet[str]: + def fork(state: State, start: int, jump: bool, jump_target: int) -> State: new_start = start + 1 if not jump else jump_target new_state = State( OrderedSet(state.reads), @@ -102,7 +101,7 @@ def fork( ) return walk(new_state, new_start) - def walk(state: State, start: int) -> OrderedSet[str]: + def walk(state: State, start: int) -> State: end = len(instructions) if stop_instr_idx is None else stop_instr_idx for i in range(start, end): if i in state.visited: diff --git a/test/sot/skip_files_py312 b/test/sot/skip_files_py312 index 92f25948d895e..ce4dbbdf80d9c 100644 --- a/test/sot/skip_files_py312 +++ b/test/sot/skip_files_py312 @@ -1,7 +1,6 @@ ./test_11_jumps.py ./test_12_for_loop.py ./test_21_global.py -./test_analysis_inputs.py ./test_break_graph.py ./test_builtin_zip.py ./test_guard_user_defined_fn.py diff --git a/test/sot/test_analysis_inputs.py b/test/sot/test_analysis_inputs.py index 880de6060d400..b400d5f781150 100644 --- a/test/sot/test_analysis_inputs.py +++ b/test/sot/test_analysis_inputs.py @@ -144,7 +144,11 @@ def case8(x): return x -case9_offset = -9 if sys.version_info >= (3, 11) else -7 +# NOTE(SigureMo): The offset should be between index of CALL instruction of assert_inputs_equals +# and the index of the CALL instruction of breakgraph_api +case9_offset = -7 +case9_offset = -9 if sys.version_info >= (3, 11) else case9_offset +case9_offset = -6 if sys.version_info >= (3, 12) else case9_offset def case9(x): From 3d88b1cd7bb8b5d0691ee799323d2bdb9790eb7b Mon Sep 17 00:00:00 2001 From: liuzhenhai93 Date: Tue, 27 Feb 2024 20:03:07 +0800 Subject: [PATCH 133/282] migrate fused_rms_norm to paddle (#60848) * fused_rms_norm * [AutoParallel] Support fused intermediate operator auto parallel code generation. * add back * polish * polish * polish * format * format * format * Polish code generation for windows. * follow_comments * follow_comments * polish * follow _comment * polish * compile * compile * polish * polish * polish * polish * back_grad * polish * polish * polish * code tidy * add test * add test * copy right * copy right * copy right * format --------- Co-authored-by: GhostScreaming Co-authored-by: liuzhenhai93 --- paddle/phi/api/yaml/backward.yaml | 12 + paddle/phi/api/yaml/generator/api_gen.py | 8 +- paddle/phi/api/yaml/generator/dist_api_gen.py | 13 +- paddle/phi/api/yaml/ops.yaml | 4 +- paddle/phi/infermeta/multiary.cc | 28 +- paddle/phi/infermeta/multiary.h | 8 +- paddle/phi/kernels/gpu/rms_norm_funcs.h | 907 ++++++++++++++++++ .../phi/kernels/gpu/rms_norm_grad_kernel.cu | 211 ++++ paddle/phi/kernels/gpu/rms_norm_kernel.cu | 129 ++- paddle/phi/kernels/rms_norm_kernel.h | 3 +- .../incubate/nn/functional/fused_rms_norm.py | 3 + test/legacy_test/test_rms_norm_op.py | 64 +- 12 files changed, 1350 insertions(+), 40 deletions(-) create mode 100644 paddle/phi/kernels/gpu/rms_norm_funcs.h create mode 100644 paddle/phi/kernels/gpu/rms_norm_grad_kernel.cu diff --git a/paddle/phi/api/yaml/backward.yaml b/paddle/phi/api/yaml/backward.yaml index d4eac9140688d..772db08fd1a2e 100644 --- a/paddle/phi/api/yaml/backward.yaml +++ b/paddle/phi/api/yaml/backward.yaml @@ -1915,6 +1915,18 @@ output : Tensor(x_grad) invoke : reverse(out_grad, axis) +- backward_op : rms_norm_grad + forward : rms_norm (Tensor x, Tensor bias, Tensor residual, Tensor norm_weight, Tensor norm_bias, float epsilon, int begin_norm_axis, float quant_scale, int quant_round_type, float quant_max_bound, float quant_min_bound) -> Tensor(out), Tensor(residual_out), Tensor(inv_var) + args : (Tensor x, Tensor bias, Tensor residual, Tensor norm_weight, Tensor norm_bias, Tensor inv_var, Tensor out_grad,float epsilon, int begin_norm_axis, float quant_scale) + output : Tensor(x_grad), Tensor(norm_weight_grad) + infer_meta : + func: RmsNormGradInferMeta + param: [x, norm_weight] + kernel : + func : rms_norm_grad + data_type : x + optional : bias, residual, norm_bias + - backward_op : roi_align_grad forward : roi_align (Tensor x, Tensor boxes, Tensor boxes_num, int pooled_height=1, int pooled_width=1, float spatial_scale=1.0, int sampling_ratio=-1, bool aligned=false) -> Tensor(out) args : (Tensor x, Tensor boxes, Tensor boxes_num, Tensor out_grad, int pooled_height, int pooled_width, float spatial_scale, int sampling_ratio, bool aligned) diff --git a/paddle/phi/api/yaml/generator/api_gen.py b/paddle/phi/api/yaml/generator/api_gen.py index c63a8bbffbde5..3e144fa27d986 100644 --- a/paddle/phi/api/yaml/generator/api_gen.py +++ b/paddle/phi/api/yaml/generator/api_gen.py @@ -495,9 +495,15 @@ def generate_api( for api in apis: forward_api = ForwardAPI(api) - if forward_api.is_dygraph_api: + if forward_api.is_dygraph_api and not is_fused_ops_yaml: forward_api.is_dygraph_api = False + if forward_api.is_dygraph_api and is_fused_ops_yaml: + forward_api.is_dygraph_api = False + header_file.write(forward_api.gene_api_declaration()) + source_file.write(forward_api.gene_api_code()) + forward_api.is_dygraph_api = True + header_file.write(forward_api.gene_api_declaration()) source_file.write(forward_api.gene_api_code()) diff --git a/paddle/phi/api/yaml/generator/dist_api_gen.py b/paddle/phi/api/yaml/generator/dist_api_gen.py index e199c5c1a520d..d0b82f3be9f70 100644 --- a/paddle/phi/api/yaml/generator/dist_api_gen.py +++ b/paddle/phi/api/yaml/generator/dist_api_gen.py @@ -1967,14 +1967,17 @@ def generate_api( for api in apis: dist_forward_api = DistForwardAPI(api) - if dist_forward_api.is_dygraph_api: + if dist_forward_api.is_dygraph_api and not is_fused_ops_yaml: dist_forward_api.is_dygraph_api = False - header_file.write(dist_forward_api.gene_api_declaration()) - if is_fused_ops_yaml is True: - source_file.write(dist_forward_api.gene_api_code()) - else: + if dist_forward_api.is_dygraph_api and is_fused_ops_yaml: + dist_forward_api.is_dygraph_api = False + header_file.write(dist_forward_api.gene_api_declaration()) source_file.write(dist_forward_api.gene_api_code()) + dist_forward_api.is_dygraph_api = True + + header_file.write(dist_forward_api.gene_api_declaration()) + source_file.write(dist_forward_api.gene_api_code()) header_file.write(namespace[1]) source_file.write(namespace[1]) diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml index 6605c214a97d4..6f4dd77c80925 100755 --- a/paddle/phi/api/yaml/ops.yaml +++ b/paddle/phi/api/yaml/ops.yaml @@ -2272,13 +2272,15 @@ - op : rms_norm args : (Tensor x, Tensor bias, Tensor residual, Tensor norm_weight, Tensor norm_bias, float epsilon, int begin_norm_axis, float quant_scale, int quant_round_type, float quant_max_bound, float quant_min_bound) - output : Tensor(out), Tensor(residual_out) + output : Tensor(out), Tensor(residual_out), Tensor(inv_var) infer_meta : func : RmsNormInferMeta kernel : func : rms_norm data_type : x optional : bias, residual, norm_bias, residual_out + intermediate : inv_var + backward : rms_norm_grad - op : rmsprop_ args : (Tensor param, Tensor mean_square, Tensor grad, Tensor moment, Tensor learning_rate, Tensor mean_grad, Tensor master_param, float epsilon = 1.0e-10f, float decay = 0.9f, float momentum = 0.0f, bool centered = false, bool multi_precision = false) diff --git a/paddle/phi/infermeta/multiary.cc b/paddle/phi/infermeta/multiary.cc index b091793ac5665..bb57e5a813aa7 100644 --- a/paddle/phi/infermeta/multiary.cc +++ b/paddle/phi/infermeta/multiary.cc @@ -3535,7 +3535,8 @@ void RmsNormInferMeta(const MetaTensor& x, const float quant_max_bound, const float quant_min_bound, MetaTensor* out, - MetaTensor* residual_out) { + MetaTensor* residual_out, + MetaTensor* inv_var) { std::vector x_dims_vec = common::vectorize(x.dims()); auto x_dims_size = x_dims_vec.size(); @@ -3544,6 +3545,10 @@ void RmsNormInferMeta(const MetaTensor& x, normalized_dims *= x_dims_vec[i]; } + std::vector inv_var_dims; + for (size_t i = size_t(0); i < static_cast(begin_norm_axis); i++) { + inv_var_dims.push_back(x_dims_vec[i]); + } PADDLE_ENFORCE_EQ(normalized_dims, norm_weight.dims()[0], phi::errors::InvalidArgument( @@ -3565,12 +3570,33 @@ void RmsNormInferMeta(const MetaTensor& x, out->set_layout(x.layout()); out->share_lod(x); + if (inv_var != nullptr) { + inv_var->set_dtype(phi::DataType::FLOAT32); + inv_var->set_dims(common::make_ddim(inv_var_dims)); + inv_var->set_layout(x.layout()); + } + residual_out->set_dims(out_dims); residual_out->set_dtype(x.dtype()); residual_out->set_layout(x.layout()); residual_out->share_lod(x); } +void RmsNormGradInferMeta(const MetaTensor& x, + const MetaTensor& norm_weight, + MetaTensor* x_grad, + MetaTensor* norm_weight_grad) { + x_grad->set_dtype(x.dtype()); + x_grad->set_layout(x.layout()); + x_grad->share_lod(x); + x_grad->set_dims(x.dims()); + + norm_weight_grad->set_dtype(norm_weight.dtype()); + norm_weight_grad->set_layout(norm_weight.layout()); + norm_weight_grad->share_lod(norm_weight); + norm_weight_grad->set_dims(norm_weight.dims()); +} + void RmspropInferMeta(const MetaTensor& param, const MetaTensor& mean_square, const MetaTensor& grad, diff --git a/paddle/phi/infermeta/multiary.h b/paddle/phi/infermeta/multiary.h index 3d9b2539267e7..e83ef2ed1825d 100644 --- a/paddle/phi/infermeta/multiary.h +++ b/paddle/phi/infermeta/multiary.h @@ -658,7 +658,13 @@ void RmsNormInferMeta(const MetaTensor& x, const float quant_max_bound, const float quant_min_bound, MetaTensor* out, - MetaTensor* residual_out); + MetaTensor* residual_out, + MetaTensor* inv_var); + +void RmsNormGradInferMeta(const MetaTensor& x, + const MetaTensor& norm_weight, + MetaTensor* x_grad, + MetaTensor* norm_weight_grad); void RmspropInferMeta(const MetaTensor& param, const MetaTensor& mean_square, diff --git a/paddle/phi/kernels/gpu/rms_norm_funcs.h b/paddle/phi/kernels/gpu/rms_norm_funcs.h new file mode 100644 index 0000000000000..a9601d7ce800e --- /dev/null +++ b/paddle/phi/kernels/gpu/rms_norm_funcs.h @@ -0,0 +1,907 @@ +/* Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include +#include "paddle/phi/backends/gpu/cuda/cudnn_helper.h" +#include "paddle/phi/backends/gpu/gpu_context.h" +#include "paddle/phi/common/amp_type_traits.h" +#include "paddle/phi/core/dense_tensor.h" +#include "paddle/phi/core/enforce.h" +#include "paddle/phi/core/kernel_registry.h" +#ifndef PADDLE_WITH_HIP +#include // NOLINT +#include // NOLINT +#include +#endif + +namespace phi { + +namespace { // NOLINT +#ifndef PADDLE_WITH_HIP + +#define DEFAULT_THROW(NAME, TYPE) \ + default: \ + do { \ + PADDLE_THROW(phi::errors::Unimplemented( \ + "(%s) is not implemented for (%s).", #NAME, TYPE)); \ + } while (0); \ + break + +#define DISPATCH_SCALE_TYPE(INPUT_TYPE, SCALE_DTYPE, NAME, ...) \ + do { \ + auto input_dtype = phi::CppTypeToDataType::Type(); \ + bool is_scale_same_dtype_with_x = input_dtype == SCALE_DTYPE; \ + using U = typename phi::backends::gpu::CudnnDataType< \ + INPUT_TYPE>::BatchNormParamType; \ + if (!is_scale_same_dtype_with_x) { \ + PADDLE_ENFORCE_EQ( \ + SCALE_DTYPE, \ + phi::CppTypeToDataType::Type(), \ + phi::errors::InvalidArgument("Unsupported data type of Scale")); \ + } \ + switch (SCALE_DTYPE) { \ + case paddle::DataType::FLOAT32: { \ + using SCALE_TYPE = float; \ + __VA_ARGS__; \ + break; \ + } \ + case paddle::DataType::FLOAT16: { \ + using SCALE_TYPE = phi::dtype::float16; \ + __VA_ARGS__; \ + break; \ + } \ + case paddle::DataType::BFLOAT16: { \ + using SCALE_TYPE = phi::dtype::bfloat16; \ + __VA_ARGS__; \ + break; \ + } \ + DEFAULT_THROW(NAME, SCALE_DTYPE); \ + } \ + } while (0) + +#define WARP_SIZE 32 + +template +__device__ __forceinline__ T WARP_SHFL_XOR(T value, + int laneMask, + int width = WARP_SIZE, + unsigned int mask = 0xffffffff) { + return __shfl_xor_sync(mask, value, laneMask, width); +} + +template +__device__ __forceinline__ T WARP_SHFL(T value, + int srcLane, + int width = WARP_SIZE, + unsigned int mask = 0xffffffff) { + return __shfl_sync(mask, value, srcLane, width); +} + +template +__device__ void cuWelfordOnlineSum(const U curr, + U& mu, // NOLINT + U& sigma2, // NOLINT + U& count) { // NOLINT + count = count + U(1); + U delta = curr - mu; + U lmean = mu + delta / count; + mu = lmean; + U delta2 = curr - lmean; + sigma2 = sigma2 + delta * delta2; +} + +template +__device__ void cuChanOnlineSum(const U muB, + const U sigma2B, + const U countB, + U& mu, // NOLINT + U& sigma2, // NOLINT + U& count) { // NOLINT + U delta = muB - mu; + U nA = count; + U nB = countB; + count = count + countB; + U nX = count; + if (nX > U(0)) { + nA = nA / nX; + nB = nB / nX; + mu = nA * mu + nB * muB; + sigma2 = sigma2 + sigma2B + delta * delta * nA * nB * nX; + } else { + mu = U(0); + sigma2 = U(0); + } +} + +template +__device__ void cuRMSOnlineSum(const U curr, U& sigma2) { // NOLINT + sigma2 = sigma2 + curr * curr; +} + +template +__device__ void cuChanRMSOnlineSum(const U sigma2B, U& sigma2) { // NOLINT + sigma2 = sigma2 + sigma2B; +} + +template +__device__ void cuWelfordMuSigma2(const T* __restrict__ vals, + const int n1, + const int n2, + const int i1, + U& mu, // NOLINT + U& sigma2, // NOLINT + U* buf, + bool rms_only) { + // Assumptions: + // 1) blockDim.x == WARP_SIZE + // 2) Tensor is contiguous + // 3) 2*blockDim.y*sizeof(U)+blockDim.y*sizeof(int) shared memory available. + // + // compute variance and mean over n2 + U count = U(0); + mu = U(0); + sigma2 = U(0); + if (i1 < n1) { + // one warp normalizes one n1 index, + // synchronization is implicit + // initialize with standard Welford algorithm + const int numx = blockDim.x * blockDim.y; + const int thrx = threadIdx.x + threadIdx.y * blockDim.x; + const T* lvals = vals + i1 * n2; + int l = 4 * thrx; + for (; l + 3 < n2; l += 4 * numx) { + for (int k = 0; k < 4; ++k) { + U curr = static_cast(lvals[l + k]); + if (!rms_only) { + cuWelfordOnlineSum(curr, mu, sigma2, count); + } else { + cuRMSOnlineSum(curr, sigma2); + } + } + } + for (; l < n2; ++l) { + U curr = static_cast(lvals[l]); + if (!rms_only) { + cuWelfordOnlineSum(curr, mu, sigma2, count); + } else { + cuRMSOnlineSum(curr, sigma2); + } + } + // intra-warp reductions + for (int l = 0; l <= 4; ++l) { + int srcLaneB = (threadIdx.x + (1 << l)) & 31; + U sigma2B = WARP_SHFL(sigma2, srcLaneB); + if (!rms_only) { + U muB = WARP_SHFL(mu, srcLaneB); + U countB = WARP_SHFL(count, srcLaneB); + cuChanOnlineSum(muB, sigma2B, countB, mu, sigma2, count); + } else { + cuChanRMSOnlineSum(sigma2B, sigma2); + } + } + // threadIdx.x == 0 has correct values for each warp + // inter-warp reductions + if (blockDim.y > 1) { + U* ubuf = (U*)buf; // NOLINT + U* ibuf = (U*)(ubuf + blockDim.y); // NOLINT + for (int offset = blockDim.y / 2; offset > 0; offset /= 2) { + // upper half of warps write to shared + if (threadIdx.x == 0 && threadIdx.y >= offset && + threadIdx.y < 2 * offset) { + const int wrt_y = threadIdx.y - offset; + if (!rms_only) { + ubuf[2 * wrt_y] = mu; + ibuf[wrt_y] = count; + } + ubuf[2 * wrt_y + 1] = sigma2; + } + __syncthreads(); + // lower half merges + if (threadIdx.x == 0 && threadIdx.y < offset) { + U sigma2B = ubuf[2 * threadIdx.y + 1]; + if (!rms_only) { + U muB = ubuf[2 * threadIdx.y]; + U countB = ibuf[threadIdx.y]; + cuChanOnlineSum(muB, sigma2B, countB, mu, sigma2, count); + } else { + cuChanRMSOnlineSum(sigma2B, sigma2); + } + } + __syncthreads(); + } + // threadIdx.x = 0 && threadIdx.y == 0 only thread that has correct + // values + if (threadIdx.x == 0 && threadIdx.y == 0) { + if (!rms_only) { + ubuf[0] = mu; + } + ubuf[1] = sigma2; + } + __syncthreads(); + if (!rms_only) { + mu = ubuf[0]; + } + sigma2 = ubuf[1] / U(n2); + // don't care about final value of count, we know count == n2 + } else { + if (!rms_only) { + mu = WARP_SHFL(mu, 0); + } + mu = WARP_SHFL(mu, 0); + sigma2 = WARP_SHFL(sigma2 / U(n2), 0); + } + } +} + +template <> +__device__ void cuWelfordMuSigma2(const phi::dtype::float16* __restrict__ vals, + const int n1, + const int n2, + const int i1, + float& mu, // NOLINT + float& sigma2, // NOLINT + float* buf, + bool rms_only) { + // Assumptions: + // 1) blockDim.x == WARP_SIZE + // 2) Tensor is contiguous + // 3) 2*blockDim.y*sizeof(U)+blockDim.y*sizeof(int) shared memory available. + // + // compute variance and mean over n2 + float count = 0.0f; + mu = float(0); // NOLINT + sigma2 = float(0); // NOLINT + if (i1 < n1) { + // one warp normalizes one n1 index, + // synchronization is implicit + // initialize with standard Welford algorithm + const int numx = blockDim.x * blockDim.y; + const int thrx = threadIdx.x + threadIdx.y * blockDim.x; + const auto* lvals = vals + i1 * n2; + int l = 8 * thrx; + if ((((size_t)lvals) & 3) != 0) { // NOLINT + // 16 bit alignment + // first thread consumes first point + if (thrx == 0) { + float curr = static_cast(lvals[0]); + if (!rms_only) { + cuWelfordOnlineSum(curr, mu, sigma2, count); + } else { + cuRMSOnlineSum(curr, sigma2); + } + } + ++l; + } + // at this point, lvals[l] are 32 bit aligned for all threads. + for (; l + 7 < n2; l += 8 * numx) { + for (int k = 0; k < 8; k += 2) { + float2 curr = __half22float2(*((__half2*)(lvals + l + k))); // NOLINT + if (!rms_only) { + cuWelfordOnlineSum(curr.x, mu, sigma2, count); + cuWelfordOnlineSum(curr.y, mu, sigma2, count); + } else { + cuRMSOnlineSum(curr.x, sigma2); + cuRMSOnlineSum(curr.y, sigma2); + } + } + } + for (; l < n2; ++l) { + float curr = static_cast(lvals[l]); + if (!rms_only) { + cuWelfordOnlineSum(curr, mu, sigma2, count); + } else { + cuRMSOnlineSum(curr, sigma2); + } + } + // intra-warp reductions + for (int l = 0; l <= 4; ++l) { + int srcLaneB = (threadIdx.x + (1 << l)) & 31; + float sigma2B = WARP_SHFL(sigma2, srcLaneB); + if (!rms_only) { + float muB = WARP_SHFL(mu, srcLaneB); + float countB = WARP_SHFL(count, srcLaneB); + cuChanOnlineSum(muB, sigma2B, countB, mu, sigma2, count); + } else { + cuChanRMSOnlineSum(sigma2B, sigma2); + } + } + // threadIdx.x == 0 has correct values for each warp + // inter-warp reductions + if (blockDim.y > 1) { + float* ubuf = (float*)buf; // NOLINT + float* ibuf = (float*)(ubuf + blockDim.y); // NOLINT + for (int offset = blockDim.y / 2; offset > 0; offset /= 2) { + // upper half of warps write to shared + if (threadIdx.x == 0 && threadIdx.y >= offset && + threadIdx.y < 2 * offset) { + const int wrt_y = threadIdx.y - offset; + ubuf[2 * wrt_y + 1] = sigma2; + if (!rms_only) { + ubuf[2 * wrt_y] = mu; + ibuf[wrt_y] = count; + } + } + __syncthreads(); + // lower half merges + if (threadIdx.x == 0 && threadIdx.y < offset) { + float sigma2B = ubuf[2 * threadIdx.y + 1]; + if (!rms_only) { + float muB = ubuf[2 * threadIdx.y]; + float countB = ibuf[threadIdx.y]; + cuChanOnlineSum(muB, sigma2B, countB, mu, sigma2, count); + } else { + cuChanRMSOnlineSum(sigma2B, sigma2); + } + } + __syncthreads(); + } + // threadIdx.x = 0 && threadIdx.y == 0 only thread that has correct + // values + if (threadIdx.x == 0 && threadIdx.y == 0) { + if (!rms_only) { + ubuf[0] = mu; + } + ubuf[1] = sigma2; + } + __syncthreads(); + if (!rms_only) { + mu = ubuf[0]; + } + sigma2 = ubuf[1] / float(n2); // NOLINT + // don't care about final value of count, we know count == n2 + } else { + if (!rms_only) { + mu = WARP_SHFL(mu, 0); + } + sigma2 = WARP_SHFL(sigma2 / float(n2), 0); // NOLINT + } + } +} + +template +__device__ U rsqrt(U v) { + return U(1) / sqrt(v); +} +template <> +__device__ float rsqrt(float v) { + return rsqrtf(v); +} +template <> +__device__ double rsqrt(double v) { + return rsqrt(v); +} + +namespace { // NOLINT +// This is the un-specialized struct. Note that we prevent instantiation of +// this struct by putting an undefined symbol in the function body so it won't +// compile. +// template +// struct SharedMemory +// { +// // Ensure that we won't compile any un-specialized types +// __device__ T *getPointer() +// { +// extern __device__ void error(void); +// error(); +// return NULL; +// } +// }; +// https://github.com/NVIDIA/apex/issues/246 +template +struct SharedMemory; + +template <> +struct SharedMemory { + __device__ float* getPointer() { + extern __shared__ float s_float[]; + return s_float; + } +}; + +} // namespace + +template +__device__ void cuApplyLayerNorm_(T* __restrict__ output_vals, + U* __restrict__ mean, + U* __restrict__ invvar, + const T* __restrict__ vals, + const int n1, + const int n2, + const U epsilon, + const V* __restrict__ gamma, + const V* __restrict__ beta, + bool rms_only) { + // Assumptions: + // 1) blockDim.x == WARP_SIZE + // 2) Tensors are contiguous + // + for (auto i1 = blockIdx.y; i1 < n1; i1 += gridDim.y) { + SharedMemory shared; + U* buf = shared.getPointer(); + U mu, sigma2; + cuWelfordMuSigma2(vals, n1, n2, i1, mu, sigma2, buf, rms_only); + const T* lvals = vals + i1 * n2; + T* ovals = output_vals + i1 * n2; + U c_invvar = rsqrt(sigma2 + epsilon); + const int numx = blockDim.x * blockDim.y; + const int thrx = threadIdx.x + threadIdx.y * blockDim.x; + if (gamma != NULL && (beta != NULL || rms_only)) { + for (int i = thrx; i < n2; i += numx) { + U curr = static_cast(lvals[i]); + if (!rms_only) { + ovals[i] = static_cast( + gamma[i] * static_cast(c_invvar * (curr - mu)) + beta[i]); + } else { + ovals[i] = static_cast(gamma[i] * static_cast(c_invvar * curr)); + } + } + } else { + for (int i = thrx; i < n2; i += numx) { + U curr = static_cast(lvals[i]); + if (!rms_only) { + ovals[i] = static_cast(c_invvar * (curr - mu)); + } else { + ovals[i] = static_cast(c_invvar * curr); + } + } + } + if (threadIdx.x == 0 && threadIdx.y == 0) { + if (!rms_only) { + mean[i1] = mu; + } + invvar[i1] = c_invvar; + } + __syncthreads(); + } +} + +template +__global__ void cuApplyRMSNorm(T* __restrict__ output_vals, + U* __restrict__ invvar, + const T* __restrict__ vals, + const int n1, + const int n2, + const U epsilon, + const V* __restrict__ gamma) { + cuApplyLayerNorm_( + output_vals, NULL, invvar, vals, n1, n2, epsilon, gamma, NULL, true); +} + +template +__device__ void cuLoadWriteStridedInputs(const int i1_block, + const int thr_load_row_off, + const int thr_load_col_off, + const int i2_off, + const int row_stride, + U* warp_buf1, + U* warp_buf2, + const T* input, + const T* dout, + const int i1_end, + const int n2, + const U* __restrict__ mean, + const U* __restrict__ invvar, + bool rms_only) { + int i1 = i1_block + thr_load_row_off; + if (i1 < i1_end) { + U curr_mean; + if (!rms_only) { + curr_mean = mean[i1]; + } + U curr_invvar = invvar[i1]; + for (int k = 0; k < blockDim.y; ++k) { + int i2 = i2_off + k; + int load_idx = i1 * n2 + i2; + int write_idx = thr_load_row_off * row_stride + thr_load_col_off + k; + if (i2 < n2) { + U curr_input = static_cast(input[load_idx]); + U curr_dout = static_cast(dout[load_idx]); + if (!rms_only) { + warp_buf1[write_idx] = curr_dout; + warp_buf2[write_idx] = + curr_dout * (curr_input - curr_mean) * curr_invvar; + } else { + warp_buf2[write_idx] = curr_dout * (curr_input)*curr_invvar; + } + } else { + if (!rms_only) { + warp_buf1[write_idx] = U(0); + } + warp_buf2[write_idx] = U(0); + } + } + } else { + for (int k = 0; k < blockDim.y; ++k) { + int write_idx = thr_load_row_off * row_stride + thr_load_col_off + k; + if (!rms_only) { + warp_buf1[write_idx] = U(0); + } + warp_buf2[write_idx] = U(0); + } + } +} + +template +__device__ void cuLoadAddStridedInputs(const int i1_block, + const int thr_load_row_off, + const int thr_load_col_off, + const int i2_off, + const int row_stride, + U* warp_buf1, + U* warp_buf2, + const T* input, + const T* dout, + const int i1_end, + const int n2, + const U* __restrict__ mean, + const U* __restrict__ invvar, + bool rms_only) { + int i1 = i1_block + thr_load_row_off; + if (i1 < i1_end) { + U curr_mean; + if (!rms_only) { + curr_mean = mean[i1]; + } + U curr_invvar = invvar[i1]; + for (int k = 0; k < blockDim.y; ++k) { + int i2 = i2_off + k; + int load_idx = i1 * n2 + i2; + int write_idx = thr_load_row_off * row_stride + thr_load_col_off + k; + if (i2 < n2) { + U curr_input = static_cast(input[load_idx]); + U curr_dout = static_cast(dout[load_idx]); + if (!rms_only) { + warp_buf1[write_idx] += curr_dout; + warp_buf2[write_idx] += + curr_dout * (curr_input - curr_mean) * curr_invvar; + } else { + warp_buf2[write_idx] += curr_dout * (curr_input)*curr_invvar; + } + } + } + } +} + +template +__global__ void cuComputePartGradGammaBeta(const T* __restrict__ dout, + const T* __restrict__ input, + const int n1, + const int n2, + const U* __restrict__ mean, + const U* __restrict__ invvar, + U epsilon, + U* part_grad_gamma, + U* part_grad_beta, + bool rms_only) { + const int numsegs_n1 = + (n1 + blockDim.y * blockDim.y - 1) / (blockDim.y * blockDim.y); + const int segs_per_block = (numsegs_n1 + gridDim.y - 1) / gridDim.y; + const int i1_beg = blockIdx.y * segs_per_block * blockDim.y * blockDim.y; + const int i1_beg_plus_one = + (blockIdx.y + 1) * segs_per_block * blockDim.y * blockDim.y; + const int i1_end = i1_beg_plus_one < n1 ? i1_beg_plus_one : n1; + const int row_stride = blockDim.x + 1; + const int thr_load_col_off = (threadIdx.x * blockDim.y) & (blockDim.x - 1); + const int thr_load_row_off = + (threadIdx.x * blockDim.y) / blockDim.x + threadIdx.y * blockDim.y; + const int i2_off = blockIdx.x * blockDim.x + thr_load_col_off; + SharedMemory shared; + U* buf = shared.getPointer(); // buf has at least blockDim.x * blockDim.y * + // blockDim.y + (blockDim.y - + // 1)*(blockDim.x/blockDim.y) elements + U* warp_buf1 = (U*)buf; // NOLINT + U* warp_buf2 = warp_buf1 + blockDim.y * blockDim.y * row_stride; + // compute partial sums from strided inputs + // do this to increase number of loads in flight + cuLoadWriteStridedInputs(i1_beg, + thr_load_row_off, + thr_load_col_off, + i2_off, + row_stride, + warp_buf1, + warp_buf2, + input, + dout, + i1_end, + n2, + mean, + invvar, + rms_only); + for (int i1_block = i1_beg + blockDim.y * blockDim.y; i1_block < i1_end; + i1_block += blockDim.y * blockDim.y) { + cuLoadAddStridedInputs(i1_block, + thr_load_row_off, + thr_load_col_off, + i2_off, + row_stride, + warp_buf1, + warp_buf2, + input, + dout, + i1_end, + n2, + mean, + invvar, + rms_only); + } + __syncthreads(); + // inter-warp reductions + // sum within each warp + U acc1 = U(0); + U acc2 = U(0); + for (int k = 0; k < blockDim.y; ++k) { + int row1 = threadIdx.y + k * blockDim.y; + int idx1 = row1 * row_stride + threadIdx.x; + if (!rms_only) { + acc1 += warp_buf1[idx1]; + } + acc2 += warp_buf2[idx1]; + } + + if (!rms_only) { + warp_buf1[threadIdx.y * row_stride + threadIdx.x] = acc1; + } + warp_buf2[threadIdx.y * row_stride + threadIdx.x] = acc2; + __syncthreads(); + // sum all warps + for (int offset = blockDim.y / 2; offset > 1; offset /= 2) { + if (threadIdx.y < offset) { + int row1 = threadIdx.y; + int row2 = threadIdx.y + offset; + int idx1 = row1 * row_stride + threadIdx.x; + int idx2 = row2 * row_stride + threadIdx.x; + if (!rms_only) { + warp_buf1[idx1] += warp_buf1[idx2]; + } + warp_buf2[idx1] += warp_buf2[idx2]; + } + __syncthreads(); + } + int i2 = blockIdx.x * blockDim.x + threadIdx.x; + if (threadIdx.y == 0 && i2 < n2) { + int row1 = threadIdx.y; + int row2 = threadIdx.y + 1; + int idx1 = row1 * row_stride + threadIdx.x; + int idx2 = row2 * row_stride + threadIdx.x; + if (!rms_only) { + part_grad_beta[blockIdx.y * n2 + i2] = warp_buf1[idx1] + warp_buf1[idx2]; + } + part_grad_gamma[blockIdx.y * n2 + i2] = warp_buf2[idx1] + warp_buf2[idx2]; + } +} + +template +__global__ void cuComputeGradGammaBeta(const U* part_grad_gamma, + const U* part_grad_beta, + const int part_size, + const int n1, + const int n2, + V* grad_gamma, + V* grad_beta, + bool rms_only) { + // sum partial gradients for gamma and beta + SharedMemory shared; + U* buf = shared.getPointer(); + int i2 = blockIdx.x * blockDim.x + threadIdx.x; + if (i2 < n2) { + // each warp does sequential reductions until reduced part_size is + // num_warps + int num_warp_reductions = part_size / blockDim.y; + U sum_gamma = U(0); + U sum_beta = U(0); + const U* part_grad_gamma_ptr = + part_grad_gamma + threadIdx.y * num_warp_reductions * n2 + i2; + const U* part_grad_beta_ptr = + part_grad_beta + threadIdx.y * num_warp_reductions * n2 + i2; + for (int warp_offset = 0; warp_offset < num_warp_reductions; + ++warp_offset) { + sum_gamma += part_grad_gamma_ptr[warp_offset * n2]; + if (!rms_only) { + sum_beta += part_grad_beta_ptr[warp_offset * n2]; + } + } + // inter-warp reductions + const int nbsize3 = blockDim.x * blockDim.y / 2; + for (int offset = blockDim.y / 2; offset >= 1; offset /= 2) { + // top half write to shared memory + if (threadIdx.y >= offset && threadIdx.y < 2 * offset) { + const int write_idx = (threadIdx.y - offset) * blockDim.x + threadIdx.x; + buf[write_idx] = sum_gamma; + if (!rms_only) { + buf[write_idx + nbsize3] = sum_beta; + } + } + __syncthreads(); + // bottom half sums + if (threadIdx.y < offset) { + const int read_idx = threadIdx.y * blockDim.x + threadIdx.x; + sum_gamma += buf[read_idx]; + if (!rms_only) { + sum_beta += buf[read_idx + nbsize3]; + } + } + __syncthreads(); + } + // write out fully summed gradients + if (threadIdx.y == 0) { + grad_gamma[i2] = sum_gamma; + if (!rms_only) { + grad_beta[i2] = sum_beta; + } + } + } +} + +template +__global__ void cuComputeGradInput(const T* __restrict__ dout, + const T* __restrict__ input, + const int n1, + const int n2, + const U* __restrict__ mean, + const U* __restrict__ invvar, + U epsilon, + const V* gamma, + T* grad_input, + bool rms_only) { + for (auto i1 = blockIdx.y; i1 < n1; i1 += gridDim.y) { + U sum_loss1 = U(0); + U sum_loss2 = U(0); + U c_mean; + if (!rms_only) { + c_mean = mean[i1]; + } + const U c_invvar = invvar[i1]; + const T* k_input = input + i1 * n2; + const T* k_dout = dout + i1 * n2; + const int numx = blockDim.x * blockDim.y; + const int thrx = threadIdx.x + threadIdx.y * blockDim.x; + if (gamma != NULL) { + int l = 4 * thrx; + for (; l + 3 < n2; l += 4 * numx) { + for (int k = 0; k < 4; ++k) { + const U c_h = static_cast(k_input[l + k]); + const U c_loss = static_cast(k_dout[l + k]); + const U gamma_tmp = static_cast(gamma[l + k]); + if (!rms_only) { + sum_loss1 += c_loss * gamma_tmp; + sum_loss2 += c_loss * gamma_tmp * (c_h - c_mean) * c_invvar; + } else { + sum_loss2 += c_loss * gamma_tmp * (c_h)*c_invvar; + } + } + } + for (; l < n2; ++l) { + const U c_h = static_cast(k_input[l]); + const U c_loss = static_cast(k_dout[l]); + const U gamma_tmp = static_cast(gamma[l]); + if (!rms_only) { + sum_loss1 += c_loss * gamma_tmp; + sum_loss2 += c_loss * gamma_tmp * (c_h - c_mean) * c_invvar; + } else { + sum_loss2 += c_loss * gamma_tmp * (c_h)*c_invvar; + } + } + } else { + int l = 4 * thrx; + for (; l + 3 < n2; l += 4 * numx) { + for (int k = 0; k < 4; ++k) { + const U c_h = static_cast(k_input[l + k]); + const U c_loss = static_cast(k_dout[l + k]); + if (!rms_only) { + sum_loss1 += c_loss; + sum_loss2 += c_loss * (c_h - c_mean) * c_invvar; + } else { + sum_loss2 += c_loss * (c_h)*c_invvar; + } + } + } + for (; l < n2; ++l) { + const U c_h = static_cast(k_input[l]); + const U c_loss = static_cast(k_dout[l]); + if (!rms_only) { + sum_loss1 += c_loss; + sum_loss2 += c_loss * (c_h - c_mean) * c_invvar; + } else { + sum_loss2 += c_loss * (c_h)*c_invvar; + } + } + } + // intra-warp reductions + for (int mask = blockDim.x / 2; mask > 0; mask /= 2) { + if (!rms_only) { + sum_loss1 += WARP_SHFL_XOR(sum_loss1, mask); + } + sum_loss2 += WARP_SHFL_XOR(sum_loss2, mask); + } + // inter-warp reductions + if (blockDim.y > 1) { + SharedMemory shared; + U* buf = shared.getPointer(); + for (int offset = blockDim.y / 2; offset > 0; offset /= 2) { + // upper half of warps write to shared + if (threadIdx.y >= offset && threadIdx.y < 2 * offset) { + const int wrt_i = (threadIdx.y - offset) * blockDim.x + threadIdx.x; + if (!rms_only) { + buf[2 * wrt_i] = sum_loss1; + } + buf[2 * wrt_i + 1] = sum_loss2; + } + __syncthreads(); + // lower half merges + if (threadIdx.y < offset) { + const int read_i = threadIdx.y * blockDim.x + threadIdx.x; + if (!rms_only) { + sum_loss1 += buf[2 * read_i]; + } + sum_loss2 += buf[2 * read_i + 1]; + } + __syncthreads(); + } + if (threadIdx.y == 0) { + if (!rms_only) { + buf[2 * threadIdx.x] = sum_loss1; + } + buf[2 * threadIdx.x + 1] = sum_loss2; + } + __syncthreads(); + if (threadIdx.y != 0) { + if (!rms_only) { + sum_loss1 = buf[2 * threadIdx.x]; + } + sum_loss2 = buf[2 * threadIdx.x + 1]; + } + } + // all threads now have the two sums over l + U fH = (U)n2; + U term1 = (U(1) / fH) * c_invvar; + T* k_grad_input = grad_input + i1 * n2; + if (gamma != NULL) { + for (int l = thrx; l < n2; l += numx) { + const U c_h = static_cast(k_input[l]); + const U c_loss = static_cast(k_dout[l]); + U f_grad_input = fH * c_loss * static_cast(gamma[l]); + if (!rms_only) { + f_grad_input -= sum_loss1; + f_grad_input -= (c_h - c_mean) * c_invvar * sum_loss2; + } else { + f_grad_input -= (c_h)*c_invvar * sum_loss2; + } + f_grad_input *= term1; + k_grad_input[l] = static_cast(f_grad_input); + } + } else { + for (int l = thrx; l < n2; l += numx) { + const U c_h = static_cast(k_input[l]); + const U c_loss = static_cast(k_dout[l]); + U f_grad_input = fH * c_loss; + if (!rms_only) { + f_grad_input -= sum_loss1; + f_grad_input -= (c_h - c_mean) * c_invvar * sum_loss2; + } else { + f_grad_input -= (c_h)*c_invvar * sum_loss2; + } + f_grad_input *= term1; + k_grad_input[l] = static_cast(f_grad_input); + } + } + // prevent race where buf is written again before reads are done + __syncthreads(); + } +} +#endif +} // namespace + +} // namespace phi diff --git a/paddle/phi/kernels/gpu/rms_norm_grad_kernel.cu b/paddle/phi/kernels/gpu/rms_norm_grad_kernel.cu new file mode 100644 index 0000000000000..bfc73faf21b9b --- /dev/null +++ b/paddle/phi/kernels/gpu/rms_norm_grad_kernel.cu @@ -0,0 +1,211 @@ +/* Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include "paddle/phi/backends/gpu/cuda/cudnn_helper.h" +#include "paddle/phi/backends/gpu/gpu_context.h" +#include "paddle/phi/common/amp_type_traits.h" +#include "paddle/phi/core/dense_tensor.h" +#include "paddle/phi/core/kernel_registry.h" +#ifndef PADDLE_WITH_HIP +#include // NOLINT +#include // NOLINT +#include +#include "paddle/phi/kernels/gpu/rms_norm_funcs.h" +#endif + +namespace phi { + +namespace { +#ifndef PADDLE_WITH_HIP + +template +void HostRMSNormGradient(const Context& dev_ctx, + const T* dout, + const U* invvar, + const DenseTensor& input, + int n1, + int n2, + const V* gamma, + double epsilon, + T* grad_input, + V* grad_gamma) { + cudaStream_t stream = dev_ctx.stream(); + if (gamma != NULL) { + const int part_size = 16; + const dim3 threads2(32, 4, 1); + const dim3 blocks2((n2 + threads2.x - 1) / threads2.x, part_size, 1); + const int nshared2_a = + 2 * sizeof(U) * threads2.y * threads2.y * (threads2.x + 1); + const int nshared2_b = threads2.x * threads2.y * sizeof(U); + const int nshared2 = nshared2_a > nshared2_b ? nshared2_a : nshared2_b; + std::vector shape = {part_size, n2}; + DenseTensor part_grad_gamma( + std::shared_ptr(nullptr), + phi::DenseTensorMeta(phi::DataType::FLOAT32, + common::make_ddim({shape}))); + dev_ctx.template Alloc(&part_grad_gamma); + + cuComputePartGradGammaBeta<<>>( + dout, + input.data(), + n1, + n2, + invvar, // unused + invvar, + U(epsilon), + part_grad_gamma.data(), + part_grad_gamma.data(), /* unused */ + true); + + const dim3 threads3(32, 8, 1); + const dim3 blocks3((n2 + threads2.x - 1) / threads2.x, 1, 1); + const int nshared3 = threads3.x * threads3.y * sizeof(U); + cuComputeGradGammaBeta<<>>( + part_grad_gamma.data(), + part_grad_gamma.data(), /* unused */ + part_size, + n1, + n2, + grad_gamma, + grad_gamma, /* unused */ + true); + } + + // compute grad_input + const uint64_t maxGridY = dev_ctx.GetCUDAMaxGridDimSize()[1]; + const dim3 blocks1(1, std::min((uint64_t)n1, maxGridY), 1); + const dim3 threads1(32, 4, 1); + int nshared = threads1.y > 1 ? threads1.y * threads1.x * sizeof(U) : 0; + + const V* gamma_tmp = gamma; + cuComputeGradInput<<>>( + dout, + input.data(), + n1, + n2, + invvar, /* unused */ + invvar, + U(epsilon), + gamma_tmp, + grad_input, + true); +} + +template +void cuda_rms_norm_gradient(const Context& dev_ctx, + const DenseTensor& x, + const DenseTensor& scale, + const DenseTensor& invvar, + const DenseTensor& dy, + const float epsilon, + DenseTensor* grad_x, + DenseTensor* grad_scale, + const int begin_norm_axis) { + const auto x_dims = x.dims(); + auto matrix_dim = phi::flatten_to_2d(x_dims, begin_norm_axis); + int rows = static_cast(matrix_dim[0]); + int cols = static_cast(matrix_dim[1]); + dev_ctx.template Alloc(grad_x); + + DISPATCH_SCALE_TYPE(T, + scale.type(), + "scale grad allocate", + dev_ctx.template Alloc(grad_scale)); + + DISPATCH_SCALE_TYPE(T, + scale.type(), + "cuda_rms_norm_gradient_kernel", + HostRMSNormGradient( + dev_ctx, + dy.data(), + invvar.data(), + x, + rows, + cols, + scale.data(), + epsilon, + grad_x->data(), + grad_scale->data())); +} +#endif +} // namespace + +template +void RmsNormGradKernel(const Context& dev_ctx, + const DenseTensor& x, + const paddle::optional& bias, + const paddle::optional& residual, + const DenseTensor& norm_weight, + const paddle::optional& norm_bias, + const DenseTensor& inv_var, + const DenseTensor& dy, + const float epsilon, + const int begin_norm_axis, + const float quant_scale, + DenseTensor* grad_x, + DenseTensor* grad_norm_weight) { +#if defined(PADDLE_WITH_HIP) + PADDLE_THROW(phi::errors::Unimplemented( + "Please compile with CUDA, ROCM platform isn't support it.")); +#else + if (bias || residual || norm_bias) { + PADDLE_THROW(phi::errors::Unimplemented( + "bias or residual or norm_bias is not supported yet")); + } + if (quant_scale > 0.0f) { + PADDLE_THROW(phi::errors::Unimplemented("quant is not supported yet")); + } + cuda_rms_norm_gradient(dev_ctx, + x, + norm_weight, + inv_var, + dy, + epsilon, + grad_x, + grad_norm_weight, + begin_norm_axis); +#endif +} +} // namespace phi + +#ifdef PADDLE_WITH_HIP +// MIOPEN do not support double + +PD_REGISTER_KERNEL(rms_norm_grad, + GPU, + ALL_LAYOUT, + phi::RmsNormGradKernel, + float, + phi::dtype::float16) {} + +#elif CUDNN_VERSION_MIN(8, 1, 0) + +PD_REGISTER_KERNEL(rms_norm_grad, + GPU, + ALL_LAYOUT, + phi::RmsNormGradKernel, + float, + phi::dtype::float16, + phi::dtype::bfloat16) {} + +#else + +PD_REGISTER_KERNEL(rms_norm_grad, + GPU, + ALL_LAYOUT, + phi::RmsNormGradKernel, + float, + phi::dtype::float16) {} +#endif diff --git a/paddle/phi/kernels/gpu/rms_norm_kernel.cu b/paddle/phi/kernels/gpu/rms_norm_kernel.cu index da8bce8996b9e..ec138271f4387 100644 --- a/paddle/phi/kernels/gpu/rms_norm_kernel.cu +++ b/paddle/phi/kernels/gpu/rms_norm_kernel.cu @@ -392,7 +392,8 @@ __global__ void __launch_bounds__(block_size) const int32_t rows, const int32_t cols, const float epsilon, - ComputeType col_divisor) { + ComputeType col_divisor, + float* inv_var_data) { using LoadType = typename LOAD::LoadType; extern __shared__ __align__(sizeof(double)) unsigned char shared_buf[]; auto* buf = reinterpret_cast(shared_buf); @@ -419,6 +420,10 @@ __global__ void __launch_bounds__(block_size) ComputeType row_rms = row_sum_square * col_divisor; ComputeType row_inv_rms = Rsqrt(row_rms + static_cast(epsilon)); + // save for backward + if (inv_var_data != nullptr) { + inv_var_data[row] = row_inv_rms; + } for (int pack_id = tid; pack_id < num_packs; pack_id += block_size) { ComputeType pack[kPackSize]; #pragma unroll @@ -443,7 +448,8 @@ inline cudaError_t LaunchRmsNormBlockSMemImpl(cudaStream_t stream, const int32_t rows, const int32_t cols, const float epsilon, - ComputeType col_divisor) { + ComputeType col_divisor, + float* inv_var_data) { constexpr int waves = 32; int grid_dim_x; { @@ -460,7 +466,7 @@ inline cudaError_t LaunchRmsNormBlockSMemImpl(cudaStream_t stream, } RmsNormBlockSMemImpl <<>>( - load, store, rows, cols, epsilon, col_divisor); + load, store, rows, cols, epsilon, col_divisor, inv_var_data); return cudaPeekAtLastError(); } @@ -488,7 +494,8 @@ inline cudaError_t TryDispatchRmsNormBlockSMemImplBlockSize( const int32_t cols, const float epsilon, ComputeType col_divisor, - bool* success) { + bool* success, + float* inv_var_data) { constexpr int block_size_conf_1 = 128; constexpr int block_size_conf_2 = 256; constexpr int block_size_conf_3 = 512; @@ -608,8 +615,15 @@ inline cudaError_t TryDispatchRmsNormBlockSMemImplBlockSize( STORE, ComputeType, kPackSize, - block_size_conf_4>( - stream, load, store, smem, rows, cols, epsilon, col_divisor); + block_size_conf_4>(stream, + load, + store, + smem, + rows, + cols, + epsilon, + col_divisor, + inv_var_data); } int max_active_blocks_conf_3; @@ -634,8 +648,15 @@ inline cudaError_t TryDispatchRmsNormBlockSMemImplBlockSize( STORE, ComputeType, kPackSize, - block_size_conf_3>( - stream, load, store, smem, rows, cols, epsilon, col_divisor); + block_size_conf_3>(stream, + load, + store, + smem, + rows, + cols, + epsilon, + col_divisor, + inv_var_data); } int max_active_blocks_conf_2; @@ -660,8 +681,15 @@ inline cudaError_t TryDispatchRmsNormBlockSMemImplBlockSize( STORE, ComputeType, kPackSize, - block_size_conf_2>( - stream, load, store, smem, rows, cols, epsilon, col_divisor); + block_size_conf_2>(stream, + load, + store, + smem, + rows, + cols, + epsilon, + col_divisor, + inv_var_data); } *success = true; @@ -669,8 +697,15 @@ inline cudaError_t TryDispatchRmsNormBlockSMemImplBlockSize( STORE, ComputeType, kPackSize, - block_size_conf_1>( - stream, load, store, smem, rows, cols, epsilon, col_divisor); + block_size_conf_1>(stream, + load, + store, + smem, + rows, + cols, + epsilon, + col_divisor, + inv_var_data); } template @@ -682,27 +717,49 @@ struct TryDispatchRmsNormBlockSMemImplPackSize { const int32_t cols, const float epsilon, ComputeType col_divisor, - bool* success) { + bool* success, + float* inv_var_data) { if (cols % 4 == 0 && CanPackAs(load, 4) && CanPackAs(store, 4)) { return TryDispatchRmsNormBlockSMemImplBlockSize( - stream, load, store, rows, cols, epsilon, col_divisor, success); + 4>(stream, + load, + store, + rows, + cols, + epsilon, + col_divisor, + success, + inv_var_data); } else if (cols % 2 == 0 && CanPackAs(load, 2) && CanPackAs(store, 2)) { return TryDispatchRmsNormBlockSMemImplBlockSize( - stream, load, store, rows, cols, epsilon, col_divisor, success); + 2>(stream, + load, + store, + rows, + cols, + epsilon, + col_divisor, + success, + inv_var_data); } else { return TryDispatchRmsNormBlockSMemImplBlockSize( - stream, load, store, rows, cols, epsilon, col_divisor, success); + 1>(stream, + load, + store, + rows, + cols, + epsilon, + col_divisor, + success, + inv_var_data); } } }; @@ -715,9 +772,18 @@ inline cudaError_t TryDispatchRmsNormBlockSMemImpl(cudaStream_t stream, const int32_t cols, const float epsilon, ComputeType col_divisor, - bool* success) { + bool* success, + float* inv_var_data) { return TryDispatchRmsNormBlockSMemImplPackSize()( - stream, load, store, rows, cols, epsilon, col_divisor, success); + stream, + load, + store, + rows, + cols, + epsilon, + col_divisor, + success, + inv_var_data); } template @@ -728,7 +794,8 @@ DispatchRmsNorm(cudaStream_t stream, STORE store, const int32_t rows, const int32_t cols, - const float epsilon) { + const float epsilon, + float* inv_var_data) { const ComputeType col_divisor = 1.0f / cols; bool dispatch_smem_impl_success; { @@ -740,7 +807,8 @@ DispatchRmsNorm(cudaStream_t stream, cols, epsilon, col_divisor, - &dispatch_smem_impl_success); + &dispatch_smem_impl_success, + inv_var_data); if (err != cudaSuccess) { return err; } @@ -948,7 +1016,8 @@ void RmsNormKernel(const Context& dev_ctx, const float quant_max_bound, const float quant_min_bound, DenseTensor* out, - DenseTensor* residual_out) { + DenseTensor* residual_out, + DenseTensor* inv_var) { #if defined(PADDLE_WITH_HIP) LOG(ERROR) << "Please compile with CUDA, ROCM platform isn't support it"; #else @@ -957,6 +1026,10 @@ void RmsNormKernel(const Context& dev_ctx, const T* x_data = x.data(); const T* norm_weight_data = norm_weight.data(); const T* norm_bias_data = norm_bias ? norm_bias.get().data() : nullptr; + float* inv_var_data = nullptr; + if (inv_var != nullptr) { + inv_var_data = dev_ctx.template Alloc(inv_var); + } int32_t rows = 1; int32_t cols = 1; @@ -981,7 +1054,7 @@ void RmsNormKernel(const Context& dev_ctx, AffineStore store( out_data, cols, norm_weight_data, norm_bias_data); DispatchRmsNorm( - dev_ctx.stream(), load, store, rows, cols, epsilon); + dev_ctx.stream(), load, store, rows, cols, epsilon, inv_var_data); } else { // Quantize and output int8. int8_t* out_data = dev_ctx.template Alloc(out); @@ -996,7 +1069,7 @@ void RmsNormKernel(const Context& dev_ctx, quant_min_bound); DispatchRmsNorm( - dev_ctx.stream(), load, store, rows, cols, epsilon); + dev_ctx.stream(), load, store, rows, cols, epsilon, inv_var_data); } } else { DirectLoad load(x_data, cols); @@ -1006,7 +1079,7 @@ void RmsNormKernel(const Context& dev_ctx, AffineStore store( out_data, cols, norm_weight_data, norm_bias_data); DispatchRmsNorm( - dev_ctx.stream(), load, store, rows, cols, epsilon); + dev_ctx.stream(), load, store, rows, cols, epsilon, inv_var_data); } else { // Quantize and output int8. int8_t* out_data = dev_ctx.template Alloc(out); @@ -1020,7 +1093,7 @@ void RmsNormKernel(const Context& dev_ctx, quant_max_bound, quant_min_bound); DispatchRmsNorm( - dev_ctx.stream(), load, store, rows, cols, epsilon); + dev_ctx.stream(), load, store, rows, cols, epsilon, inv_var_data); } } #endif diff --git a/paddle/phi/kernels/rms_norm_kernel.h b/paddle/phi/kernels/rms_norm_kernel.h index a0abaf7fd35e4..d725178b41e44 100644 --- a/paddle/phi/kernels/rms_norm_kernel.h +++ b/paddle/phi/kernels/rms_norm_kernel.h @@ -33,6 +33,7 @@ void RmsNormKernel(const Context& dev_ctx, const float quant_max_bound, const float quant_min_bound, DenseTensor* out, - DenseTensor* residual_out); + DenseTensor* residual_out, + DenseTensor* inv_var); } // namespace phi diff --git a/python/paddle/incubate/nn/functional/fused_rms_norm.py b/python/paddle/incubate/nn/functional/fused_rms_norm.py index 925c8005490f4..c5942135ad136 100644 --- a/python/paddle/incubate/nn/functional/fused_rms_norm.py +++ b/python/paddle/incubate/nn/functional/fused_rms_norm.py @@ -105,6 +105,9 @@ def fused_rms_norm( residual_out = helper.create_variable_for_type_inference(dtype=x.dtype) outputs_dict['residual_out'] = residual_out + inv_var = helper.create_variable_for_type_inference(dtype=paddle.float32) + outputs_dict['inv_var'] = inv_var + inputs = {'x': x, 'norm_weight': norm_weight} if norm_bias is not None: inputs['norm_bias'] = norm_bias diff --git a/test/legacy_test/test_rms_norm_op.py b/test/legacy_test/test_rms_norm_op.py index dc9061ad95924..f8ae5769cfaaf 100644 --- a/test/legacy_test/test_rms_norm_op.py +++ b/test/legacy_test/test_rms_norm_op.py @@ -35,13 +35,22 @@ def quant_helper( ) -def naive_rms_norm(x, gamma, beta, epsilon): +def naive_rms_norm(x, gamma, beta=None, epsilon=1e-5): variance = x.pow(2).mean(-1, keepdim=True) out = paddle.rsqrt(variance + epsilon) * x - out = out * gamma + beta + out = out * gamma + if beta is not None: + out = out + beta return out +def fused_rms_norm(x, gamma, beta=None, epsilon=1e-5, begin_norm_axis=1): + out = paddle.incubate.nn.functional.fused_rms_norm( + x, gamma, beta, epsilon, begin_norm_axis=begin_norm_axis + ) + return out[0] + + def naive_rms_norm_int8( x, gamma, @@ -285,6 +294,57 @@ def test_residual_bias_add_rmsnorm_int8(self): atol=2, ) + def test_rms_norm_backward(self): + def get_paddle_tensor(shape, dtype, bound=0.5): + tmp = paddle.uniform(shape, dtype=dtype, min=-bound, max=bound) + tmp.stop_gradient = False + return tmp + + def get_forward_backward(func, seed, dtype): + paddle.disable_static() + paddle.seed(seed) + x = get_paddle_tensor([2, 256], dtype) + scale = get_paddle_tensor([256], dtype) + out_g = paddle.randn([2, 256], dtype) + out = func(x, scale) + paddle.autograd.backward([out], [out_g], True) + return out, (x.grad, scale.grad) + + dtypes = [paddle.float32] + if paddle.amp.is_bfloat16_supported('gpu'): + dtypes.append(paddle.bfloat16) + if paddle.amp.is_float16_supported('gpu'): + dtypes.append(paddle.float16) + for dtype in dtypes: + raw_out, raw_grads = get_forward_backward( + naive_rms_norm, seed=2024, dtype=dtype + ) + fused_out, fused_grads = get_forward_backward( + fused_rms_norm, seed=2024, dtype=dtype + ) + # forward rtol + rtol = 1e-5 if dtype == paddle.float32 else 1e-2 + np.testing.assert_allclose( + raw_out.astype(paddle.float32).numpy(), + fused_out.astype(paddle.float32).numpy(), + rtol=rtol, + ) + # backward rtol, only check float32 grad + rtol = 1e-3 + if dtype == paddle.float32: + raw_x_grad, raw_scale_grad = raw_grads + fused_x_grad, fused_scale_grad = fused_grads + np.testing.assert_allclose( + raw_x_grad.astype(paddle.float32).numpy(), + fused_x_grad.astype(paddle.float32).numpy(), + rtol=rtol, + ) + np.testing.assert_allclose( + raw_scale_grad.astype(paddle.float32).numpy(), + fused_scale_grad.astype(paddle.float32).numpy(), + rtol=rtol, + ) + @unittest.skipIf( not core.is_compiled_with_cuda(), "core is not compiled with CUDA " From 93102f927450b06fa64a64cb64dcc4ab8085627c Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Tue, 27 Feb 2024 21:18:30 +0800 Subject: [PATCH 134/282] [PIR] Align reshape infermeta to legacy IR infershape (#62136) --- paddle/phi/api/yaml/ops.yaml | 2 +- paddle/phi/infermeta/unary.cc | 21 +++++++++++++++++++-- paddle/phi/infermeta/unary.h | 2 +- test/legacy_test/test_reshape_op.py | 14 ++++++++++---- 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml index 6f4dd77c80925..6cf76d4c25c06 100755 --- a/paddle/phi/api/yaml/ops.yaml +++ b/paddle/phi/api/yaml/ops.yaml @@ -1696,7 +1696,7 @@ args : (Tensor x) output : Tensor(out) infer_meta : - func : LogicalNotInfermeta + func : LogicalNotInferMeta kernel : func : logical_not data_type : x diff --git a/paddle/phi/infermeta/unary.cc b/paddle/phi/infermeta/unary.cc index 42eaa2670a0b5..5648ff0d469a3 100644 --- a/paddle/phi/infermeta/unary.cc +++ b/paddle/phi/infermeta/unary.cc @@ -2188,7 +2188,7 @@ void KthvalueInferMeta(const MetaTensor& x, indices->set_dtype(x.dtype()); } -void LogicalNotInfermeta(const MetaTensor& x, MetaTensor* out) { +void LogicalNotInferMeta(const MetaTensor& x, MetaTensor* out) { UnchangedInferMeta(x, out); if (!(out->is_same_tensor(x))) { out->set_dtype(DataType::BOOL); @@ -3588,11 +3588,28 @@ void ReshapeInferMeta(const MetaTensor& x, const IntArray& shape, MetaTensor* out, MetaConfig config) { - auto& shape_data = shape.GetData(); + auto shape_data = shape.GetData(); PADDLE_ENFORCE_NOT_NULL(out, phi::errors::InvalidArgument( "Output(Out) of ReshapeOp should not be null.")); if (!config.is_runtime && shape.FromTensor()) { + const int64_t copy_dim_flag = 0; + const auto& in_dims = x.dims(); + for (size_t i = 0; i < shape_data.size(); ++i) { + if (shape_data[i] == copy_dim_flag) { + PADDLE_ENFORCE_LT( + static_cast(i), + in_dims.size(), + phi::errors::InvalidArgument( + "The index of 0 in `shape` must be less than " + "the input tensor X's dimensions. But received shape[%d] " + "= 0, X's dimensions = %d, X's shape = [%s].", + i, + in_dims.size(), + in_dims)); + shape_data[i] = static_cast(in_dims[static_cast(i)]); + } + } out->set_dims(common::make_ddim(shape_data)); out->share_lod(x); out->set_dtype(x.dtype()); diff --git a/paddle/phi/infermeta/unary.h b/paddle/phi/infermeta/unary.h index a9f5f2eb1a13c..d62789bd5183c 100644 --- a/paddle/phi/infermeta/unary.h +++ b/paddle/phi/infermeta/unary.h @@ -328,7 +328,7 @@ void KthvalueInferMeta(const MetaTensor& x, MetaTensor* indices, MetaConfig = MetaConfig()); -void LogicalNotInfermeta(const MetaTensor& x, MetaTensor* out); +void LogicalNotInferMeta(const MetaTensor& x, MetaTensor* out); void LogsumexpInferMeta(const MetaTensor& input, const std::vector& axis, diff --git a/test/legacy_test/test_reshape_op.py b/test/legacy_test/test_reshape_op.py index c786562062554..964f487a4ed9d 100755 --- a/test/legacy_test/test_reshape_op.py +++ b/test/legacy_test/test_reshape_op.py @@ -737,15 +737,21 @@ def test_static(self): class TestReshapePirValueListShape(unittest.TestCase): def test_value_list_shape(self): with paddle.pir_utils.IrGuard(): - x = paddle.static.data( - 'x', - [3], - ) + x = paddle.static.data('x', [3]) shape = [1, paddle.full([], 3)] out = paddle.reshape(x, shape) self.assertEqual(out.shape, [1, -1]) +class TestReshapePirTensorWithZeroShape(unittest.TestCase): + def test_tensor_with_zero_shape(self): + with paddle.pir_utils.IrGuard(): + x = paddle.static.data('x', [10, -1]) + shape = [0, paddle.shape(x)[1]] + out = paddle.reshape(x, shape) + self.assertEqual(out.shape, [10, -1]) + + if __name__ == "__main__": paddle.enable_static() unittest.main() From aa86d86ab9ed2801fae7dda19dec9a9ca2079253 Mon Sep 17 00:00:00 2001 From: wanghuancoder Date: Wed, 28 Feb 2024 09:47:10 +0800 Subject: [PATCH 135/282] [PIR] pir onednn support fused elementwise (#62126) * pir onednn support fused eselemtwise --- .../fluid/pir/dialect/operator/ir/onednn.yaml | 36 +++++++++++++++++++ .../dialect/operator/ir/ops_onednn_extra.yaml | 8 ++--- paddle/phi/api/yaml/op_compat.yaml | 24 +++++++++++++ 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/pir/dialect/operator/ir/onednn.yaml b/paddle/fluid/pir/dialect/operator/ir/onednn.yaml index cfc52121febe5..1ef15ccb9c3a3 100644 --- a/paddle/fluid/pir/dialect/operator/ir/onednn.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/onednn.yaml @@ -28,6 +28,42 @@ data_type : input optional : bias, residual_param +- op : fused_elementwise_add + args : (Tensor x, Tensor y, int axis=-1, str fuse_activation="", float fuse_alpha=0.0, float fuse_beta=0.0, float fused_output_scale=1.0, int[] fused_unsqueeze2_axes={}, float scale_x=1.0, float scale_y=1.0, float scale_out=1.0) + output : Tensor(out) + infer_meta : + func : ElementwiseInferMeta + param : [x, y] + kernel : + func : fused_elementwise_add + +- op : fused_elementwise_div + args : (Tensor x, Tensor y, int axis=-1, str fuse_activation="", float fuse_alpha=0.0, float fuse_beta=0.0, float fused_output_scale=1.0, int[] fused_unsqueeze2_axes={}, float scale_x=1.0, float scale_y=1.0, float scale_out=1.0) + output : Tensor(out) + infer_meta : + func : ElementwiseInferMeta + param : [x, y] + kernel : + func : fused_elementwise_div + +- op : fused_elementwise_mul + args : (Tensor x, Tensor y, int axis=-1, str fuse_activation="", float fuse_alpha=0.0, float fuse_beta=0.0, float fused_output_scale=1.0, int[] fused_unsqueeze2_axes={}, float scale_x=1.0, float scale_y=1.0, float scale_out=1.0) + output : Tensor(out) + infer_meta : + func : ElementwiseInferMeta + param : [x, y] + kernel : + func : fused_elementwise_mul + +- op : fused_elementwise_sub + args : (Tensor x, Tensor y, int axis=-1, str fuse_activation="", float fuse_alpha=0.0, float fuse_beta=0.0, float fused_output_scale=1.0, int[] fused_unsqueeze2_axes={}, float scale_x=1.0, float scale_y=1.0, float scale_out=1.0) + output : Tensor(out) + infer_meta : + func : ElementwiseInferMeta + param : [x, y] + kernel : + func : fused_elementwise_sub + - op: multi_gru args: (Tensor x, Tensor[] weight_x, Tensor[] weight_h, Tensor[] bias, Tensor[] scale_weights, str activation="tanh", str gate_activation="sigmoid", int layers=1, bool origin_mode=false, str mkldnn_data_type="float32", float scale_data=1.0, float shift_data=1.0, bool force_fp32_output=false) output: Tensor(hidden) diff --git a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml index f51b6d7e9a3be..c4523396b3673 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml @@ -104,13 +104,13 @@ extra_args : float fuse_alpha = 0.0, float fuse_beta = 0.0, float scale_in=1.0, float scale_out=1.0, float scale_in_eltwise=1.0, float[] scale_weights={1.0f} data_format_tensors : input -# - op : fused_elementwise_add +- op : fused_elementwise_add -# - op : fused_elementwise_div +- op : fused_elementwise_div -# - op : fused_elementwise_mul +- op : fused_elementwise_mul -# - op : fused_elementwise_sub +- op : fused_elementwise_sub # - op : fused_matmul diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml index 1a3f86753fa7e..74263a1dd522d 100755 --- a/paddle/phi/api/yaml/op_compat.yaml +++ b/paddle/phi/api/yaml/op_compat.yaml @@ -1354,6 +1354,30 @@ attrs : [bool use_cudnn = false, float fuse_alpha = 0.0f, float fuse_beta = 0.0f, float Scale_in = 1.0f, float Scale_out = 1.0f, float Scale_in_eltwise = 1.0f, 'float[] Scale_weights = {1.0f}', bool use_mkldnn = true, str mkldnn_data_type = "float32"] +- op : fused_elementwise_add + inputs : + {x : X, y : Y} + outputs : + {out : Out} + +- op : fused_elementwise_div + inputs : + {x : X, y : Y} + outputs : + {out : Out} + +- op : fused_elementwise_mul + inputs : + {x : X, y : Y} + outputs : + {out : Out} + +- op : fused_elementwise_sub + inputs : + {x : X, y : Y} + outputs : + {out : Out} + - op : fused_embedding_eltwise_layernorm inputs : ids : Ids From dbb12a5207565c343164c9d8a01153fbe8b783b7 Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Wed, 28 Feb 2024 10:07:40 +0800 Subject: [PATCH 136/282] [PIR+CINN]Fix DRR OpPattern not take effect for BlockArgument as OperandSource (#62137) * [PIR+CINN]Fix DRR OpPattern not take effect for BlockArgument as OperandSource * add const * open more UT * fix comment * fix typi * fix logic --- paddle/fluid/pir/drr/src/rewrite_pattern.cc | 35 +++++++++++-------- .../pir/transforms/sub_graph_detector.cc | 11 +++--- .../pir/cinn/sub_graphs/test_sub_graph_61.py | 2 +- .../pir/cinn/sub_graphs/test_sub_graph_71.py | 2 +- .../pir/cinn/sub_graphs/test_sub_graph_76.py | 2 +- .../pir/cinn/sub_graphs/test_sub_graph_86.py | 15 ++++---- .../pir/cinn/sub_graphs/test_sub_graph_89.py | 2 +- 7 files changed, 37 insertions(+), 32 deletions(-) diff --git a/paddle/fluid/pir/drr/src/rewrite_pattern.cc b/paddle/fluid/pir/drr/src/rewrite_pattern.cc index ee1742c2c9b5f..2a2e654166522 100644 --- a/paddle/fluid/pir/drr/src/rewrite_pattern.cc +++ b/paddle/fluid/pir/drr/src/rewrite_pattern.cc @@ -301,10 +301,8 @@ bool DrrRewritePattern::MatchFromOutputToInput( source_pattern_match_ctx->BindIrValue(drr_input_tensors[i]->name(), ir_node->operand(i).source()); if (ir_node->operand_source(i).isa()) { - matched = false; - VLOG(8) << drr_node->name() - << " Match failed: it's input value is a block argument."; - break; + VLOG(8) << "Match Attention! Found BlockArgument as input of " + << drr_node->name(); } auto* drr_producer_op = drr_input_tensors[i]->producer(); @@ -520,27 +518,34 @@ void DrrRewritePattern::DeleteSourcePatternOp( GraphTopo graph_topo_visit(&source_pattern_graph); graph_topo_visit.WalkGraphNodesTopoOrder([&](const OpCall& op_call) { pir::Operation* op = src_match_ctx.IrOperation(&op_call); - VLOG(5) << "DRR delete op: " << op->name() << " pointer: " << op; + VLOG(6) << "DRR delete op: " << op->name() << " pointer: " << op; if (delete_ops_set.count(op) == 0 && op->use_empty()) { delete_ops_que.push(op); delete_ops_set.insert(op); } }); + const auto& DeleteOpAndUpdateQueue = + [&](pir::Operation* op, + std::queue* delete_ops_que) -> void { + const std::vector inputs = op->operands_source(); + rewriter.EraseOp(op); + for (const auto& input : inputs) { + const bool use_empty = + (input && input.defining_op() && input.defining_op()->use_empty()); + auto* defining_op = input.defining_op(); + if (use_empty && delete_ops_set.count(defining_op) == 0U) { + delete_ops_set.insert(defining_op); + delete_ops_que->push(defining_op); + } + } + }; while (!delete_ops_que.empty()) { pir::Operation* op = delete_ops_que.front(); delete_ops_que.pop(); - std::vector inputs = op->operands_source(); - VLOG(5) << "Delete (" << op->name() << " @" << op + VLOG(6) << "Delete (" << op->name() << " @" << op << ") in source_pattern_graph."; - rewriter.EraseOp(op); - for (const auto& input : inputs) { - if (input && input.defining_op()->use_empty() && - delete_ops_set.count(input.defining_op()) == 0) { - delete_ops_set.insert(input.defining_op()); - delete_ops_que.push(input.defining_op()); - } - } + DeleteOpAndUpdateQueue(op, &delete_ops_que); } } diff --git a/paddle/fluid/pir/transforms/sub_graph_detector.cc b/paddle/fluid/pir/transforms/sub_graph_detector.cc index e0cd23467cb04..0690bc1c8399c 100644 --- a/paddle/fluid/pir/transforms/sub_graph_detector.cc +++ b/paddle/fluid/pir/transforms/sub_graph_detector.cc @@ -528,17 +528,16 @@ void ReplaceWithGroupOp(pir::Block* block, VLOG(6) << "Insert GroupOp after " << insert_point->name(); // step 2: Replace the old op with GroupOp. - const auto& CreateGroupOp = [&]() -> cinn::dialect::GroupOp { + auto new_group_op = [&]() -> cinn::dialect::GroupOp { std::vector output_types; for (auto& value : outputs) output_types.emplace_back(value.type()); - auto new_group_op = builder.Build(output_types); + auto group_op = builder.Build(output_types); for (auto op : group_ops) { - op->MoveTo(new_group_op.block(), new_group_op.block()->end()); + op->MoveTo(group_op.block(), group_op.block()->end()); } - return new_group_op; - }; - auto new_group_op = CreateGroupOp(); + return group_op; + }(); // step 3: Replace outputs of inner ops const std::vector group_outs = new_group_op->results(); diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_61.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_61.py index 8bda88384089f..99eb7cf39bed5 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_61.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_61.py @@ -98,7 +98,7 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) cinn_out = self.train( - self.net, to_static=True, with_prim=True, with_cinn=False + self.net, to_static=True, with_prim=True, with_cinn=True ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_71.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_71.py index fd62209dc96c4..5fac613db9ade 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_71.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_71.py @@ -247,7 +247,7 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) cinn_out = self.train( - self.net, to_static=True, with_prim=True, with_cinn=False + self.net, to_static=True, with_prim=True, with_cinn=True ) # TODO(Aurelius84): atol only satisfy 1e-5 under with_cinn=True for st, cinn in zip( diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_76.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_76.py index 0b3d9fd560042..211111ae65066 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_76.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_76.py @@ -128,7 +128,7 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) cinn_out = self.train( - self.net, to_static=True, with_prim=True, with_cinn=False + self.net, to_static=True, with_prim=True, with_cinn=True ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_86.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_86.py index d117ee86a0aa8..2ad74f73a66a9 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_86.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_86.py @@ -221,10 +221,12 @@ def forward( class TestLayer(unittest.TestCase): def setUp(self): + # TODO(Aurelius84): atol only satisfy 1e-4 if shape is [1, 512, 128, 128], + # [1, 1024, 64, 64], [1, 2048, 32, 32]. self.inputs = ( - paddle.rand(shape=[1, 512, 128, 128], dtype=paddle.float32), - paddle.rand(shape=[1, 1024, 64, 64], dtype=paddle.float32), - paddle.rand(shape=[1, 2048, 32, 32], dtype=paddle.float32), + paddle.rand(shape=[1, 512, 4, 4], dtype=paddle.float32), + paddle.rand(shape=[1, 1024, 2, 2], dtype=paddle.float32), + paddle.rand(shape=[1, 2048, 1, 1], dtype=paddle.float32), ) self.net = LayerCase() @@ -239,20 +241,19 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): ) else: net = paddle.jit.to_static(net, full_graph=True) - paddle.seed(123) + paddle.seed(2024) outs = net(*self.inputs) return outs def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) - # NOTE(Aurelius84): atol only satisfy 1e-5 under with_cinn=True cinn_out = self.train( - self.net, to_static=True, with_prim=True, with_cinn=False + self.net, to_static=True, with_prim=True, with_cinn=True ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) ): - np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-5) + np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-6) if __name__ == '__main__': diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_89.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_89.py index c5027f48d58ec..77049437185d8 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_89.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_89.py @@ -104,7 +104,7 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) cinn_out = self.train( - self.net, to_static=True, with_prim=True, with_cinn=False + self.net, to_static=True, with_prim=True, with_cinn=True ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) From ab485e4e17443b42dc954a7d9aa15d2b93f76227 Mon Sep 17 00:00:00 2001 From: zyfncg Date: Wed, 28 Feb 2024 10:19:24 +0800 Subject: [PATCH 137/282] [CINN] Change add_cinn_pass to apply_cinn_pass (#62071) * refector lower cinn pass * fix bug * fix complie bug * fix complie bug * fix bug * refine * fix bug --- .../operator/transforms/add_cinn_pass.cc | 43 +++++++++++++------ .../operator/transforms/add_cinn_pass.h | 6 ++- .../divide_group_op_to_fusion_op_pass.cc | 2 +- .../transforms/lower_cinn_fusion_op_pass.cc | 2 +- .../fluid/inference/api/analysis_predictor.cc | 28 +++++++----- paddle/fluid/pybind/pir.cc | 18 ++++++-- .../jit/dy2static/pir_partial_program.py | 21 ++++----- test/ir/pir/my_task.py | 4 +- 8 files changed, 75 insertions(+), 49 deletions(-) diff --git a/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.cc index 9b18ed609dda9..6ded2f5a85c93 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.cc @@ -51,8 +51,8 @@ COMMON_DECLARE_bool(check_infer_symbolic); namespace cinn::dialect::ir { namespace { -bool HasDynamicShape(const pir::Program &program) { - for (const auto &op : *program.block()) { +bool HasDynamicShape(const pir::Program& program) { + for (const auto& op : *program.block()) { if (op.isa()) { continue; } @@ -70,18 +70,12 @@ bool HasDynamicShape(const pir::Program &program) { } } // namespace -void AddCinnPass(std::shared_ptr &pass_manager, // NOLINT - pir::Program &program) { // NOLINT - pir::IrContext *ctx = pir::IrContext::Instance(); - ctx->GetOrRegisterDialect(); - ctx->GetOrRegisterDialect(); - ctx->GetOrRegisterDialect(); - - bool has_dynamic_shape = HasDynamicShape(program); - - if (FLAGS_print_ir) { - pass_manager->EnableIRPrinting(); - } +void ApplyCinnPreprocessPass( + ::pir::Program* program, + const std::function()>& + CreatePassManager) { + std::shared_ptr pass_manager = CreatePassManager(); + bool has_dynamic_shape = HasDynamicShape(*program); pass_manager->AddPass(cinn::dialect::ir::CreateConvert0DTo1DPass()); if (!has_dynamic_shape && FLAGS_check_infer_symbolic) { @@ -116,6 +110,17 @@ void AddCinnPass(std::shared_ptr &pass_manager, // NOLINT pass_manager->AddPass(cinn::dialect::ir::CreateReplaceDynamicExpandOpPass()); pass_manager->AddPass(pir::CreateDeadCodeEliminationPass()); + pass_manager->Run(program); +} + +void ApplyCinnLowerPass( + ::pir::Program* program, + const std::function()>& + CreatePassManager) { + std::shared_ptr pass_manager = CreatePassManager(); + + bool has_dynamic_shape = HasDynamicShape(*program); + bool force_static_shape = false; if (auto pass = cinn::dialect::ir::CreateConvertDynamicToStaticDimPass()) { pass_manager->AddPass(std::move(pass.value())); @@ -129,9 +134,19 @@ void AddCinnPass(std::shared_ptr &pass_manager, // NOLINT pass_manager->AddPass( cinn::dialect::ir::CreateLowerCinnDyShapeFusionOpPass()); } + pass_manager->AddPass(cinn::dialect::ir::CreateLowerCinnFusionOpPass()); pass_manager->AddPass( cinn::dialect::ir::CreateSplitGenerateShapeIntoShapeOpsPass()); + + pass_manager->Run(program); +} + +void ApplyCinnPass(::pir::Program* program, + const std::function()>& + CreatePassManager) { + ApplyCinnPreprocessPass(program, CreatePassManager); + ApplyCinnLowerPass(program, CreatePassManager); } } // namespace cinn::dialect::ir diff --git a/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.h b/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.h index e66aff15ac0c8..4a71cbc5ee310 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.h +++ b/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.h @@ -14,6 +14,7 @@ #pragma once +#include #include namespace pir { @@ -25,7 +26,8 @@ class Program; namespace cinn::dialect::ir { -void AddCinnPass(std::shared_ptr &pass_manager, // NOLINT - pir::Program &program); // NOLINT +void ApplyCinnPass(::pir::Program* program, + const std::function()>& + CreatePassManager); } // namespace cinn::dialect::ir diff --git a/paddle/cinn/hlir/dialect/operator/transforms/group_merge/divide_group_op_to_fusion_op_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/group_merge/divide_group_op_to_fusion_op_pass.cc index aabc60652b970..886cc29efa5b1 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/group_merge/divide_group_op_to_fusion_op_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/group_merge/divide_group_op_to_fusion_op_pass.cc @@ -199,7 +199,7 @@ class DivideGroupOpToFusionOpPass : public pir::PatternRewritePass { } bool CanApplyOn(pir::Operation* op) const override { - return op->isa() && op->num_regions() > 0; + return op->num_regions() > 0; } }; diff --git a/paddle/cinn/hlir/dialect/operator/transforms/lower_cinn_fusion_op_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/lower_cinn_fusion_op_pass.cc index cf8e8edbce557..a2393a09fae21 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/lower_cinn_fusion_op_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/lower_cinn_fusion_op_pass.cc @@ -751,7 +751,7 @@ class LowerCinnFusionOpPass : public pir::PatternRewritePass { } bool CanApplyOn(pir::Operation* op) const override { - return op->isa() && op->num_regions() > 0; + return op->num_regions() > 0; } }; diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index 6ce0ce2570d4f..b61e8eaa0577d 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -106,7 +106,9 @@ #endif #ifdef PADDLE_WITH_CINN +#include "paddle/cinn/hlir/dialect/operator/ir/op_dialect.h" #include "paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.h" +#include "paddle/pir/include/dialect/shape/ir/shape_dialect.h" #endif #include "paddle/common/flags.h" @@ -880,17 +882,21 @@ bool AnalysisPredictor::PrepareExecutor() { } #ifdef PADDLE_WITH_CINN if (config_.cinn_enabled()) { - VLOG(4) << "[CINN] Begin AddCinnPass"; - auto cinn_pm = std::make_shared<::pir::PassManager>( - ::pir::IrContext::Instance(), 2); - cinn::dialect::ir::AddCinnPass(cinn_pm, *pir_program_.get()); - if (!config_.glog_info_disabled()) { - cinn_pm->EnablePrintStatistics(); - } - if (config_.ir_debug_) { - cinn_pm->EnableIRPrinting(); - } - cinn_pm->Run(pir_program_.get()); + VLOG(4) << "[CINN] Begin ApplyCinnPass"; + cinn::dialect::ir::ApplyCinnPass(pir_program_.get(), [&] { + pir::IrContext *ctx = pir::IrContext::Instance(); + ctx->GetOrRegisterDialect(); + ctx->GetOrRegisterDialect(); + auto pass_manager = std::make_shared<::pir::PassManager>( + ::pir::IrContext::Instance(), 2); + if (!config_.glog_info_disabled()) { + pass_manager->EnablePrintStatistics(); + } + if (config_.ir_debug_) { + pass_manager->EnableIRPrinting(); + } + return pass_manager; + }); } #endif diff --git a/paddle/fluid/pybind/pir.cc b/paddle/fluid/pybind/pir.cc index de3ec976c7dea..54fa9bf54f057 100644 --- a/paddle/fluid/pybind/pir.cc +++ b/paddle/fluid/pybind/pir.cc @@ -83,6 +83,7 @@ #include "pybind11/stl.h" #ifdef PADDLE_WITH_CINN +#include "paddle/cinn/hlir/dialect/operator/ir/op_dialect.h" #include "paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.h" #include "paddle/cinn/hlir/framework/pir_compiler.h" #endif @@ -1551,10 +1552,19 @@ bool HasDynamicShape(const pir::Program &program) { return false; } -void AddCinnPass(std::shared_ptr &pass_manager, // NOLINT - pir::Program &program) { // NOLINT +void ApplyCinnPass(Program &program) { // NOLINT #ifdef PADDLE_WITH_CINN - cinn::dialect::ir::AddCinnPass(pass_manager, program); + cinn::dialect::ir::ApplyCinnPass(&program, [] { + pir::IrContext *ctx = pir::IrContext::Instance(); + ctx->GetOrRegisterDialect(); + ctx->GetOrRegisterDialect(); + ctx->GetOrRegisterDialect(); + auto pass_manager = std::make_shared(ctx); + if (FLAGS_print_ir) { + pass_manager->EnableIRPrinting(); + } + return pass_manager; + }); #else PADDLE_THROW(common::errors::Unimplemented( "Currently we only support CINN Pass for Pir under @to_static, please " @@ -1575,7 +1585,7 @@ void InferSymbolicShapePass( } void BindIrPass(pybind11::module *m) { - m->def("add_cinn_pass", AddCinnPass); + m->def("apply_cinn_pass", ApplyCinnPass); m->def("infer_symbolic_shape_pass", InferSymbolicShapePass); py::class_> pass(*m, diff --git a/python/paddle/jit/dy2static/pir_partial_program.py b/python/paddle/jit/dy2static/pir_partial_program.py index 10d16bb215741..e3e20e79beb65 100644 --- a/python/paddle/jit/dy2static/pir_partial_program.py +++ b/python/paddle/jit/dy2static/pir_partial_program.py @@ -552,13 +552,17 @@ def _create_program(self, is_infer_mode=False): if is_infer_mode: def pass_fn(forward_program, backward_program): + # common pass pm = paddle.base.libpaddle.pir.PassManager() paddle.base.libpaddle.pir.infer_symbolic_shape_pass( pm, forward_program ) - if self._build_strategy.build_cinn_pass: - paddle.base.libpaddle.pir.add_cinn_pass(pm, forward_program) pm.run(forward_program) + + # if-else pass + if self._build_strategy.build_cinn_pass: + paddle.base.libpaddle.pir.apply_cinn_pass(forward_program) + return forward_program, backward_program # TODO(xiongkun) who to transfer the pruning program? @@ -574,18 +578,9 @@ def pass_fn(forward_program, backward_program): self._set_grad_type(self._params, train_program) def pass_fn(forward_program, backward_program): - fwd_pm = paddle.base.libpaddle.pir.PassManager() - bwd_pm = paddle.base.libpaddle.pir.PassManager() - if self._build_strategy.build_cinn_pass: - paddle.base.libpaddle.pir.add_cinn_pass( - fwd_pm, forward_program - ) - paddle.base.libpaddle.pir.add_cinn_pass( - bwd_pm, backward_program - ) - fwd_pm.run(forward_program) - bwd_pm.run(backward_program) + paddle.base.libpaddle.pir.apply_cinn_pass(forward_program) + paddle.base.libpaddle.pir.apply_cinn_pass(backward_program) return forward_program, backward_program train_program.apply_pir_program_pass(pass_fn) diff --git a/test/ir/pir/my_task.py b/test/ir/pir/my_task.py index b862ea07d1a9f..c3501594d0792 100644 --- a/test/ir/pir/my_task.py +++ b/test/ir/pir/my_task.py @@ -183,9 +183,7 @@ def check_infer(self, enable_cinn): def run_program(self, program, feed, fetch_list, enable_cinn): if enable_cinn: paddle.decomposition.decomp.decompose(program, []) - fwd_pm = paddle.base.libpaddle.pir.PassManager() - paddle.base.libpaddle.pir.add_cinn_pass(fwd_pm, program) - fwd_pm.run(program) + paddle.base.libpaddle.pir.apply_cinn_pass(program) exe = paddle.static.Executor(paddle.CUDAPlace(0)) outs = exe._run_pir_impl( From d356dabf64679bc5b91caf6870057dd06fa2f809 Mon Sep 17 00:00:00 2001 From: feifei-111 <2364819892@qq.com> Date: Wed, 28 Feb 2024 10:19:36 +0800 Subject: [PATCH 138/282] [CINN] add chatglm2 sub graphs (#62081) * add chatglm2 sub graphs * update * update --- .../symbolic/test_sub_graph_chatglm2_0_st.py | 249 ++++++++++++++++++ .../symbolic/test_sub_graph_chatglm2_1_st.py | 119 +++++++++ .../symbolic/test_sub_graph_chatglm2_2_st.py | 90 +++++++ .../symbolic/test_sub_graph_chatglm2_3_st.py | 96 +++++++ .../symbolic/test_sub_graph_chatglm2_4_st.py | 113 ++++++++ .../symbolic/test_sub_graph_chatglm2_5_st.py | 120 +++++++++ 6 files changed, 787 insertions(+) create mode 100644 test/ir/pir/cinn/symbolic/test_sub_graph_chatglm2_0_st.py create mode 100644 test/ir/pir/cinn/symbolic/test_sub_graph_chatglm2_1_st.py create mode 100644 test/ir/pir/cinn/symbolic/test_sub_graph_chatglm2_2_st.py create mode 100644 test/ir/pir/cinn/symbolic/test_sub_graph_chatglm2_3_st.py create mode 100644 test/ir/pir/cinn/symbolic/test_sub_graph_chatglm2_4_st.py create mode 100644 test/ir/pir/cinn/symbolic/test_sub_graph_chatglm2_5_st.py diff --git a/test/ir/pir/cinn/symbolic/test_sub_graph_chatglm2_0_st.py b/test/ir/pir/cinn/symbolic/test_sub_graph_chatglm2_0_st.py new file mode 100644 index 0000000000000..b12ba4d18c385 --- /dev/null +++ b/test/ir/pir/cinn/symbolic/test_sub_graph_chatglm2_0_st.py @@ -0,0 +1,249 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# repo: llm_sub_graphs +# model: chatglm2 +# method:astype||method:pow||method:mean||method:__add__||api:paddle.tensor.ops.rsqrt||method:__mul__||method:__mul__||method:astype||api:paddle.nn.functional.common.linear||method:split||method:reshape||method:reshape||method:reshape||method:__getitem__||method:__getitem__||method:__getitem__||method:reshape||method:reshape||method:__getitem__||method:__getitem__||method:__mul__||method:__getitem__||method:__getitem__||method:__mul__||method:__sub__||method:__getitem__||method:__getitem__||method:__mul__||method:__getitem__||method:__getitem__||method:__mul__||method:__add__||api:paddle.tensor.manipulation.stack||method:flatten||api:paddle.tensor.manipulation.concat||method:__getitem__||method:__getitem__||method:__getitem__||method:reshape||method:reshape||method:__getitem__||method:__getitem__||method:__mul__||method:__getitem__||method:__getitem__||method:__mul__||method:__sub__||method:__getitem__||method:__getitem__||method:__mul__||method:__getitem__||method:__getitem__||method:__mul__||method:__add__||api:paddle.tensor.manipulation.stack||method:flatten||api:paddle.tensor.manipulation.concat||method:unsqueeze||method:tile||method:reshape||method:unsqueeze||method:tile||method:reshape||method:reshape||method:reshape||method:transpose||method:transpose||api:paddle.tensor.linalg.bmm||method:__mul__||method:reshape||method:astype||method:__mul__||method:__add__||method:astype||api:paddle.nn.functional.activation.softmax||method:astype||api:paddle.nn.functional.common.dropout||method:reshape||method:reshape||method:transpose||api:paddle.tensor.linalg.bmm||method:reshape||method:transpose||method:reshape||api:paddle.nn.functional.common.linear||api:paddle.nn.functional.common.dropout||method:__add__||method:astype||method:pow||method:mean||method:__add__||api:paddle.tensor.ops.rsqrt||method:__mul__||method:__mul__||method:astype||api:paddle.nn.functional.common.linear||method:__getitem__||method:__getitem__||api:paddle.nn.functional.activation.silu||method:__mul__||api:paddle.nn.functional.common.linear||api:paddle.nn.functional.common.dropout||method:__add__ +import unittest + +import numpy as np + +import paddle + + +class LayerCase(paddle.nn.Layer): + def __init__(self): + super().__init__() + self.parameter_0 = self.create_parameter( + shape=[32], + dtype=paddle.float32, + ) + self.parameter_1 = self.create_parameter( + shape=[1024, 32], + dtype=paddle.float32, + ) + self.parameter_2 = self.create_parameter( + shape=[32, 32], + dtype=paddle.float32, + ) + self.parameter_3 = self.create_parameter( + shape=[32, 2048], + dtype=paddle.float32, + ) + self.parameter_4 = self.create_parameter( + shape=[32], + dtype=paddle.float32, + ) + self.parameter_5 = self.create_parameter( + shape=[32, 64], + dtype=paddle.float32, + ) + self.parameter_6 = self.create_parameter( + shape=[64], + dtype=paddle.float32, + ) + + def forward( + self, + var_0, # (shape: [1024, 4, 32], dtype: paddle.float32, stop_gradient: False) + var_1, # (shape: [4, 1, 1024, 1024], dtype: paddle.float32, stop_gradient: True) + var_2, # (shape: [1024, 1, 2, 2], dtype: paddle.float32, stop_gradient: True) + ): + var_3 = var_0.astype('float32') + var_4 = var_3.pow(2) + var_5 = var_4.mean(-1, keepdim=True) + var_6 = var_5 + 1e-05 + var_7 = paddle.tensor.ops.rsqrt(var_6) + var_8 = var_7 * var_0 + var_9 = var_8 * self.parameter_0 + var_10 = var_9.astype('float32') + var_11 = paddle.nn.functional.common.linear( + x=var_10, weight=self.parameter_5, bias=self.parameter_6, name=None + ) + var_12, var_13, var_14 = var_11.split([32, 16, 16], axis=-1) + var_15 = var_12.reshape([1024, 4, 4, 8]) + var_16 = var_13.reshape([1024, 4, -1, 8]) + var_17 = var_14.reshape([1024, 4, -1, 8]) + var_18 = var_15[(..., slice(None, 4, None))] + var_19 = var_15[(..., slice(4, None, None))] + var_20 = var_2[slice(None, 1024, None)] + var_21 = var_18.reshape([1024, -1, 4, 2, 2]) + var_22 = var_20.reshape([1024, -1, 1, 2, 2]) + var_23 = var_21[(..., 0)] + var_24 = var_22[(..., 0)] + var_25 = var_23 * var_24 + var_26 = var_21[(..., 1)] + var_27 = var_22[(..., 1)] + var_28 = var_26 * var_27 + var_29 = var_25 - var_28 + var_30 = var_21[(..., 1)] + var_31 = var_22[(..., 0)] + var_32 = var_30 * var_31 + var_33 = var_21[(..., 0)] + var_34 = var_22[(..., 1)] + var_35 = var_33 * var_34 + var_36 = var_32 + var_35 + var_37 = paddle.tensor.manipulation.stack([var_29, var_36], -1) + var_38 = var_37.flatten(3) + var_39 = paddle.tensor.manipulation.concat( + (var_38, var_19), + axis=-1, + ) + var_40 = var_16[(..., slice(None, 4, None))] + var_41 = var_16[(..., slice(4, None, None))] + var_42 = var_2[slice(None, 1024, None)] + var_43 = var_40.reshape([1024, -1, 2, 2, 2]) + var_44 = var_42.reshape([1024, -1, 1, 2, 2]) + var_45 = var_43[(..., 0)] + var_46 = var_44[(..., 0)] + var_47 = var_45 * var_46 + var_48 = var_43[(..., 1)] + var_49 = var_44[(..., 1)] + var_50 = var_48 * var_49 + var_51 = var_47 - var_50 + var_52 = var_43[ + ( + ..., + 1, + ) + ] + var_53 = var_44[(..., 0)] + var_54 = var_52 * var_53 + var_55 = var_43[(..., 0)] + var_56 = var_44[(..., 1)] + var_57 = var_55 * var_56 + var_58 = var_54 + var_57 + var_59 = paddle.tensor.manipulation.stack([var_51, var_58], -1) + var_60 = var_59.flatten(3) + var_61 = paddle.tensor.manipulation.concat( + (var_60, var_41), + axis=-1, + ) + var_62 = var_61.unsqueeze(-2) + var_63 = var_62.tile([1, 1, 1, 2, 1]) + var_64 = var_63.reshape([1024, 4, 4, 8]) + var_65 = var_17.unsqueeze(-2) + var_66 = var_65.tile([1, 1, 1, 2, 1]) + var_67 = var_66.reshape([1024, 4, 4, 8]) + var_68 = var_39.reshape([1024, 16, -1]) + var_69 = var_64.reshape([1024, 16, -1]) + var_70 = var_68.transpose([1, 0, 2]) + var_71 = var_69.transpose([1, 2, 0]) + var_72 = paddle.tensor.linalg.bmm(var_70, var_71) + var_73 = var_72 * 0.01860807318911967 + var_74 = var_73.reshape((4, 4, 1024, 1024)) + var_75 = var_74.astype('float32') + var_76 = var_75 * 19 + var_77 = var_76 + var_1 + var_78 = var_77.astype('float32') + var_79 = paddle.nn.functional.activation.softmax(var_78, axis=-1) + var_80 = var_79.astype('float32') + var_81 = paddle.nn.functional.common.dropout( + var_80, + p=0.0, + axis=None, + training=True, + mode='upscale_in_train', + name=None, + ) + var_82 = var_67.reshape([1024, 16, -1]) + var_83 = var_81.reshape([16, 1024, -1]) + var_84 = var_82.transpose([1, 0, 2]) + var_85 = paddle.tensor.linalg.bmm(var_83, var_84) + var_86 = var_85.reshape((4, 4, 1024, 8)) + var_87 = var_86.transpose([2, 0, 1, 3]) + var_88 = var_87.reshape([1024, 4, 32]) + var_89 = paddle.nn.functional.common.linear( + x=var_88, weight=self.parameter_2, bias=None, name=None + ) + var_90 = paddle.nn.functional.common.dropout( + var_89, p=0.0, training=True + ) + var_91 = var_0 + var_90 + var_92 = var_91.astype('float32') + var_93 = var_92.pow(2) + var_94 = var_93.mean(-1, keepdim=True) + var_95 = var_94 + 1e-05 + var_96 = paddle.tensor.ops.rsqrt(var_95) + var_97 = var_96 * var_91 + var_98 = var_97 * self.parameter_4 + var_99 = var_98.astype('float32') + var_100 = paddle.nn.functional.common.linear( + x=var_99, weight=self.parameter_3, bias=None, name=None + ) + var_101 = var_100[(..., slice(None, 1024, None))] + var_102 = var_100[(..., slice(1024, None, None))] + var_103 = paddle.nn.functional.activation.silu(var_101) + var_104 = var_103 * var_102 + var_105 = paddle.nn.functional.common.linear( + x=var_104, weight=self.parameter_1, bias=None, name=None + ) + var_106 = paddle.nn.functional.common.dropout( + var_105, p=0.0, training=True + ) + var_107 = var_91 + var_106 + return var_107, var_61, var_17 + + +def create_paddle_inputs(): + inputs = ( + paddle.rand(shape=[1024, 4, 32], dtype=paddle.float32), + paddle.rand(shape=[4, 1, 1024, 1024], dtype=paddle.float32), + paddle.rand(shape=[1024, 1, 2, 2], dtype=paddle.float32), + ) + return inputs + + +def create_numpy_inputs(): + inputs = ( + np.random.random(size=[1024, 4, 32]).astype('float32'), + np.random.random(size=[4, 1, 1024, 1024]).astype('float32'), + np.random.random(size=[1024, 1, 2, 2]).astype('float32'), + ) + return inputs + + +class TestLayer(unittest.TestCase): + def setUp(self): + self.inputs = create_paddle_inputs() + self.net = LayerCase() + + def train(self, net, to_static, with_prim=False, with_cinn=False): + if to_static: + paddle.set_flags({'FLAGS_prim_all': with_prim}) + if with_cinn: + build_strategy = paddle.static.BuildStrategy() + build_strategy.build_cinn_pass = True + net = paddle.jit.to_static( + net, build_strategy=build_strategy, full_graph=True + ) + else: + net = paddle.jit.to_static(net, full_graph=True) + paddle.seed(123) + outs = net(*self.inputs) + return outs + + def test_ast_prim_cinn(self): + st_out = self.train(self.net, to_static=True) + cinn_out = self.train( + self.net, to_static=True, with_prim=False, with_cinn=False + ) + for st, cinn in zip( + paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) + ): + np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-6) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/ir/pir/cinn/symbolic/test_sub_graph_chatglm2_1_st.py b/test/ir/pir/cinn/symbolic/test_sub_graph_chatglm2_1_st.py new file mode 100644 index 0000000000000..9e160198c6936 --- /dev/null +++ b/test/ir/pir/cinn/symbolic/test_sub_graph_chatglm2_1_st.py @@ -0,0 +1,119 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# repo: llm_sub_graphs +# model: chatglm2 +# method:reshape||method:reshape||method:transpose||method:transpose||api:paddle.tensor.linalg.bmm||method:__mul__||method:reshape||method:astype||method:__mul__||method:__add__||method:astype||api:paddle.nn.functional.activation.softmax||method:astype||api:paddle.nn.functional.common.dropout||method:reshape||method:reshape||method:transpose||api:paddle.tensor.linalg.bmm||method:reshape||method:transpose||method:reshape +import unittest + +import numpy as np + +import paddle + + +class LayerCase(paddle.nn.Layer): + def __init__(self): + super().__init__() + + def forward( + self, + var_0, # (shape: [1024, 4, 4, 8], dtype: paddle.float32, stop_gradient: False) + var_1, # (shape: [1024, 4, 4, 8], dtype: paddle.float32, stop_gradient: False) + var_2, # (shape: [1024, 4, 4, 8], dtype: paddle.float32, stop_gradient: False) + var_3, # (shape: [4, 1, 1024, 1024], dtype: paddle.float32, stop_gradient: True) + ): + var_4 = var_0.reshape([1024, 16, -1]) + var_5 = var_1.reshape([1024, 16, -1]) + var_6 = var_4.transpose([1, 0, 2]) + var_7 = var_5.transpose([1, 2, 0]) + var_8 = paddle.tensor.linalg.bmm(var_6, var_7) + var_9 = var_8 * 0.027196414661021056 + var_10 = var_9.reshape((4, 4, 1024, 1024)) + var_11 = var_10.astype('float32') + var_12 = var_11 * 13 + var_13 = var_12 + var_3 + var_14 = var_13.astype('float32') + var_15 = paddle.nn.functional.activation.softmax(var_14, axis=-1) + var_16 = var_15.astype('float32') + var_17 = paddle.nn.functional.common.dropout( + var_16, + p=0.0, + axis=None, + training=True, + mode='upscale_in_train', + name=None, + ) + var_18 = var_2.reshape([1024, 16, -1]) + var_19 = var_17.reshape([16, 1024, -1]) + var_20 = var_18.transpose([1, 0, 2]) + var_21 = paddle.tensor.linalg.bmm(var_19, var_20) + var_22 = var_21.reshape((4, 4, 1024, 8)) + var_23 = var_22.transpose([2, 0, 1, 3]) + var_24 = var_23.reshape([1024, 4, 32]) + return var_24 + + +def create_paddle_inputs(): + inputs = ( + paddle.rand(shape=[1024, 4, 4, 8], dtype=paddle.float32), + paddle.rand(shape=[1024, 4, 4, 8], dtype=paddle.float32), + paddle.rand(shape=[1024, 4, 4, 8], dtype=paddle.float32), + paddle.rand(shape=[4, 1, 1024, 1024], dtype=paddle.float32), + ) + return inputs + + +def create_numpy_inputs(): + inputs = ( + np.random.random(size=[1024, 4, 4, 8]).astype('float32'), + np.random.random(size=[1024, 4, 4, 8]).astype('float32'), + np.random.random(size=[1024, 4, 4, 8]).astype('float32'), + np.random.random(size=[4, 1, 1024, 1024]).astype('float32'), + ) + return inputs + + +class TestLayer(unittest.TestCase): + def setUp(self): + self.inputs = create_paddle_inputs() + self.net = LayerCase() + + def train(self, net, to_static, with_prim=False, with_cinn=False): + if to_static: + paddle.set_flags({'FLAGS_prim_all': with_prim}) + if with_cinn: + build_strategy = paddle.static.BuildStrategy() + build_strategy.build_cinn_pass = True + net = paddle.jit.to_static( + net, build_strategy=build_strategy, full_graph=True + ) + else: + net = paddle.jit.to_static(net, full_graph=True) + paddle.seed(123) + outs = net(*self.inputs) + return outs + + def test_ast_prim_cinn(self): + st_out = self.train(self.net, to_static=True) + cinn_out = self.train( + self.net, to_static=True, with_prim=True, with_cinn=False + ) + for st, cinn in zip( + paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) + ): + np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-6) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/ir/pir/cinn/symbolic/test_sub_graph_chatglm2_2_st.py b/test/ir/pir/cinn/symbolic/test_sub_graph_chatglm2_2_st.py new file mode 100644 index 0000000000000..a7583daa03e86 --- /dev/null +++ b/test/ir/pir/cinn/symbolic/test_sub_graph_chatglm2_2_st.py @@ -0,0 +1,90 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# repo: llm_sub_graphs +# model: chatglm2 +# method:astype||method:pow||method:mean||method:__add__||api:paddle.tensor.ops.rsqrt||method:__mul__||method:__mul__||method:astype +import unittest + +import numpy as np + +import paddle + + +class LayerCase(paddle.nn.Layer): + def __init__(self): + super().__init__() + self.parameter_0 = self.create_parameter( + shape=[32], + dtype=paddle.float32, + ) + + def forward( + self, + var_0, # (shape: [1024, 4, 32], dtype: paddle.float32, stop_gradient: False) + ): + var_1 = var_0.astype('float32') + var_2 = var_1.pow(2) + var_3 = var_2.mean(-1, keepdim=True) + var_4 = var_3 + 1e-05 + var_5 = paddle.tensor.ops.rsqrt(var_4) + var_6 = var_5 * var_0 + var_7 = var_6 * self.parameter_0 + var_8 = var_7.astype('float32') + return var_8 + + +def create_paddle_inputs(): + inputs = (paddle.rand(shape=[1024, 4, 32], dtype=paddle.float32),) + return inputs + + +def create_numpy_inputs(): + inputs = (np.random.random(size=[1024, 4, 32]).astype('float32'),) + return inputs + + +class TestLayer(unittest.TestCase): + def setUp(self): + self.inputs = create_paddle_inputs() + self.net = LayerCase() + + def train(self, net, to_static, with_prim=False, with_cinn=False): + if to_static: + paddle.set_flags({'FLAGS_prim_all': with_prim}) + if with_cinn: + build_strategy = paddle.static.BuildStrategy() + build_strategy.build_cinn_pass = True + net = paddle.jit.to_static( + net, build_strategy=build_strategy, full_graph=True + ) + else: + net = paddle.jit.to_static(net, full_graph=True) + paddle.seed(123) + outs = net(*self.inputs) + return outs + + def test_ast_prim_cinn(self): + st_out = self.train(self.net, to_static=True) + cinn_out = self.train( + self.net, to_static=True, with_prim=True, with_cinn=False + ) + for st, cinn in zip( + paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) + ): + np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-6) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/ir/pir/cinn/symbolic/test_sub_graph_chatglm2_3_st.py b/test/ir/pir/cinn/symbolic/test_sub_graph_chatglm2_3_st.py new file mode 100644 index 0000000000000..48f212e53ebad --- /dev/null +++ b/test/ir/pir/cinn/symbolic/test_sub_graph_chatglm2_3_st.py @@ -0,0 +1,96 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# repo: llm_sub_graphs +# model: chatglm2 +# api:paddle.nn.functional.common.linear||method:__getitem__||method:__getitem__||api:paddle.nn.functional.activation.silu||method:__mul__||api:paddle.nn.functional.common.linear +import unittest + +import numpy as np + +import paddle + + +class LayerCase(paddle.nn.Layer): + def __init__(self): + super().__init__() + self.parameter_0 = self.create_parameter( + shape=[1024, 32], + dtype=paddle.float32, + ) + self.parameter_1 = self.create_parameter( + shape=[32, 2048], + dtype=paddle.float32, + ) + + def forward( + self, + var_0, # (shape: [1024, 4, 32], dtype: paddle.float32, stop_gradient: False) + ): + var_1 = paddle.nn.functional.common.linear( + x=var_0, weight=self.parameter_1, bias=None, name=None + ) + var_2 = var_1[(..., slice(None, 1024, None))] + var_3 = var_1[(..., slice(1024, None, None))] + var_4 = paddle.nn.functional.activation.silu(var_2) + var_5 = var_4 * var_3 + var_6 = paddle.nn.functional.common.linear( + x=var_5, weight=self.parameter_0, bias=None, name=None + ) + return var_6 + + +def create_paddle_inputs(): + inputs = (paddle.rand(shape=[1024, 4, 32], dtype=paddle.float32),) + return inputs + + +def create_numpy_inputs(): + inputs = (np.random.random(size=[1024, 4, 32]).astype('float32'),) + return inputs + + +class TestLayer(unittest.TestCase): + def setUp(self): + self.inputs = create_paddle_inputs() + self.net = LayerCase() + + def train(self, net, to_static, with_prim=False, with_cinn=False): + if to_static: + paddle.set_flags({'FLAGS_prim_all': with_prim}) + if with_cinn: + build_strategy = paddle.static.BuildStrategy() + build_strategy.build_cinn_pass = True + net = paddle.jit.to_static( + net, build_strategy=build_strategy, full_graph=True + ) + else: + net = paddle.jit.to_static(net, full_graph=True) + paddle.seed(123) + outs = net(*self.inputs) + return outs + + def test_ast_prim_cinn(self): + st_out = self.train(self.net, to_static=True) + cinn_out = self.train( + self.net, to_static=True, with_prim=True, with_cinn=False + ) + for st, cinn in zip( + paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) + ): + np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/ir/pir/cinn/symbolic/test_sub_graph_chatglm2_4_st.py b/test/ir/pir/cinn/symbolic/test_sub_graph_chatglm2_4_st.py new file mode 100644 index 0000000000000..2e266168892cf --- /dev/null +++ b/test/ir/pir/cinn/symbolic/test_sub_graph_chatglm2_4_st.py @@ -0,0 +1,113 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# repo: llm_sub_graphs +# model: chatglm2 +# api:paddle.nn.functional.input.embedding||method:transpose||api:paddle.tensor.creation.ones||api:paddle.tensor.creation.tril||method:astype||api:paddle.tensor.creation.ones||method:astype||method:__and__||api:paddle.tensor.creation.arange||method:__truediv__||method:__rpow__||method:__rtruediv__||api:paddle.tensor.creation.arange||api:paddle.tensor.math.outer||method:astype||api:paddle.tensor.ops.cos||api:paddle.tensor.ops.sin||api:paddle.tensor.manipulation.stack||method:__getitem__||method:transpose +import unittest + +import numpy as np + +import paddle + + +class LayerCase(paddle.nn.Layer): + def __init__(self): + super().__init__() + self.parameter_0 = self.create_parameter( + shape=[64896, 32], + dtype=paddle.float32, + ) + + def forward( + self, + var_0, # (shape: [4, 1024], dtype: paddle.int64, stop_gradient: True) + ): + var_1 = paddle.nn.functional.input.embedding( + var_0, + weight=self.parameter_0, + padding_idx=None, + sparse=False, + name=None, + ) + var_2 = var_1.transpose([1, 0, 2]) + var_3 = paddle.tensor.creation.ones([4, 1, 1024, 1024]) + var_4 = paddle.tensor.creation.tril(var_3) + var_5 = var_4.astype('bool') + var_6 = paddle.tensor.creation.ones( + (4, 1, 1024, 1024), + dtype='bool', + ) + var_7 = var_6.astype('bool') + var_8 = var_5 and var_7 + var_9 = paddle.tensor.creation.arange(0, 4, 2, dtype='float32') + var_10 = var_9 / 4 + var_11 = 10000**var_10 + var_12 = 1.0 / var_11 + var_13 = paddle.tensor.creation.arange(0, 1024, dtype='float32') + var_14 = paddle.tensor.math.outer(var_13, var_12) + var_15 = var_14.astype('float32') + var_16 = paddle.tensor.ops.cos(var_15) + var_17 = paddle.tensor.ops.sin(var_15) + var_18 = paddle.tensor.manipulation.stack([var_16, var_17], axis=-1) + var_19 = var_18[(None, slice(None, 1024, None))] + var_20 = var_19.transpose([1, 0, 2, 3]) + return var_2, var_8, var_20 + + +def create_paddle_inputs(): + inputs = ( + paddle.randint(low=0, high=10, shape=[4, 1024], dtype=paddle.int64), + ) + return inputs + + +def create_numpy_inputs(): + inputs = (np.random.randint(low=0, high=10, size=[4, 1024], dtype='int64'),) + return inputs + + +class TestLayer(unittest.TestCase): + def setUp(self): + self.inputs = create_paddle_inputs() + self.net = LayerCase() + + def train(self, net, to_static, with_prim=False, with_cinn=False): + if to_static: + paddle.set_flags({'FLAGS_prim_all': with_prim}) + if with_cinn: + build_strategy = paddle.static.BuildStrategy() + build_strategy.build_cinn_pass = True + net = paddle.jit.to_static( + net, build_strategy=build_strategy, full_graph=True + ) + else: + net = paddle.jit.to_static(net, full_graph=True) + paddle.seed(123) + outs = net(*self.inputs) + return outs + + def test_ast_prim_cinn(self): + st_out = self.train(self.net, to_static=True) + cinn_out = self.train( + self.net, to_static=True, with_prim=False, with_cinn=False + ) + for st, cinn in zip( + paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) + ): + np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/ir/pir/cinn/symbolic/test_sub_graph_chatglm2_5_st.py b/test/ir/pir/cinn/symbolic/test_sub_graph_chatglm2_5_st.py new file mode 100644 index 0000000000000..5971d9260d760 --- /dev/null +++ b/test/ir/pir/cinn/symbolic/test_sub_graph_chatglm2_5_st.py @@ -0,0 +1,120 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# repo: llm_sub_graphs +# model: chatglm2 +# api:paddle.nn.functional.common.linear||method:transpose||method:reshape||method:astype||method:reshape||method:__ne__||method:astype||api:paddle.nn.functional.loss.cross_entropy||method:reshape||method:cast||method:reshape||method:cast||method:__mul__||api:paddle.tensor.math.sum||method:sum||method:__truediv__||method:astype||method:astype +import unittest + +import numpy as np + +import paddle + + +class LayerCase(paddle.nn.Layer): + def __init__(self): + super().__init__() + self.parameter_0 = self.create_parameter( + shape=[32, 64896], + dtype=paddle.float32, + ) + + def forward( + self, + var_0, # (shape: [1024, 4, 32], dtype: paddle.float32, stop_gradient: False) + var_1, # (shape: [4, 1024], dtype: paddle.int64, stop_gradient: True) + ): + var_2 = paddle.nn.functional.common.linear( + x=var_0, weight=self.parameter_0, bias=None, name=None + ) + var_3 = var_2.transpose([1, 0, 2]) + var_4 = var_3.reshape([-1, 64896]) + var_5 = var_4.astype('float32') + var_6 = var_1.reshape([-1]) + var_7 = var_1 != -100 + var_8 = var_7.astype('float32') + var_9 = paddle.nn.functional.loss.cross_entropy( + var_5, + var_6, + weight=None, + ignore_index=-100, + reduction='none', + soft_label=False, + axis=-1, + use_softmax=True, + label_smoothing=0.0, + name=None, + ) + var_10 = var_9.reshape([-1]) + var_11 = var_10.cast('float32') + var_12 = var_8.reshape([-1]) + var_13 = var_12.cast('float32') + var_14 = var_11 * var_13 + var_15 = paddle.tensor.math.sum(var_14) + var_16 = var_8.sum() + var_17 = var_15 / var_16 + var_18 = var_3.astype('float32') + var_19 = var_17.astype('float32') + return var_19, var_18 + + +def create_paddle_inputs(): + inputs = ( + paddle.rand(shape=[1024, 4, 32], dtype=paddle.float32), + paddle.randint(low=0, high=10, shape=[4, 1024], dtype=paddle.int64), + ) + return inputs + + +def create_numpy_inputs(): + inputs = ( + np.random.random(size=[1024, 4, 32]).astype('float32'), + np.random.randint(low=0, high=10, size=[4, 1024], dtype='int64'), + ) + return inputs + + +class TestLayer(unittest.TestCase): + def setUp(self): + self.inputs = create_paddle_inputs() + self.net = LayerCase() + + def train(self, net, to_static, with_prim=False, with_cinn=False): + if to_static: + paddle.set_flags({'FLAGS_prim_all': with_prim}) + if with_cinn: + build_strategy = paddle.static.BuildStrategy() + build_strategy.build_cinn_pass = True + net = paddle.jit.to_static( + net, build_strategy=build_strategy, full_graph=True + ) + else: + net = paddle.jit.to_static(net, full_graph=True) + paddle.seed(123) + outs = net(*self.inputs) + return outs + + def test_ast_prim_cinn(self): + st_out = self.train(self.net, to_static=True) + cinn_out = self.train( + self.net, to_static=True, with_prim=True, with_cinn=False + ) + for st, cinn in zip( + paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) + ): + np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) + + +if __name__ == '__main__': + unittest.main() From 760ae650a1a24b4613902238c68210a193457908 Mon Sep 17 00:00:00 2001 From: NeroLoh <745827440@qq.com> Date: Wed, 28 Feb 2024 10:31:55 +0800 Subject: [PATCH 139/282] [xpu]fix subgraph bug in constant folding pass (#62148) --- paddle/fluid/framework/ir/constant_folding_pass.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/framework/ir/constant_folding_pass.cc b/paddle/fluid/framework/ir/constant_folding_pass.cc index 7cfbbadfa17a5..4375043544dc8 100644 --- a/paddle/fluid/framework/ir/constant_folding_pass.cc +++ b/paddle/fluid/framework/ir/constant_folding_pass.cc @@ -85,12 +85,16 @@ void ConstantFoldingPass::ApplyImpl(ir::Graph *graph) const { std::unordered_map map; for (auto in_node : op_node->inputs) { map[in_node->Name()] = 0; - if (!in_node->Var()->Persistable() || !in_node->inputs.empty()) { + if (in_node->Var() == nullptr || !in_node->Var()->Persistable() || + !in_node->inputs.empty()) { input_persis = false; } } for (auto out_node : op_node->outputs) { map[out_node->Name()] = 0; + if (out_node->Var() == nullptr) { + input_persis = false; + } } // Forbid other node in graph having the same name with nodes in map for (auto const &iter : map) { From 5904cc89b8576662e5a57af287889deac79085db Mon Sep 17 00:00:00 2001 From: ShenLiang <1422485404@qq.com> Date: Wed, 28 Feb 2024 10:35:37 +0800 Subject: [PATCH 140/282] add check for pp/vpp (#59405) (#62156) --- .../pp_utils/p2p_communication.py | 52 ++++++++++++++----- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py b/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py index 6d470d541f66b..667040fc94443 100644 --- a/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py +++ b/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py @@ -156,21 +156,43 @@ def send_meta(self, tensor, group): ) self._send_dims_shape_dtype(d, group=group) - def set_send_message(self, tensor): + def _obtain_send_message(self, tensor): if isinstance(tensor, (paddle.Tensor, framework.core.eager.Tensor)): - self.send_shape_message = tensor.shape - self.send_dtype_message = paddle_2_number(tensor.dtype) - elif isinstance(tensor, tuple): - self.send_shape_message = tuple( - [d.shape for d in tensor if not d.stop_gradient] - ) - self.send_dtype_message = tuple( - [ - paddle_2_number(d.dtype) - for d in tensor - if not d.stop_gradient - ] - ) + return tensor.shape, paddle_2_number(tensor.dtype) + else: + shapes = [] + dtypes = [] + for d in tensor: + assert isinstance( + d, (paddle.Tensor, framework.core.eager.Tensor) + ) + if d.stop_gradient: + continue + shape, dtype = self._obtain_send_message(d) + shapes.append(shape) + dtypes.append(dtype) + return tuple(shapes), tuple(dtypes) + + def set_send_message(self, tensor): + ( + self.send_shape_message, + self.send_dtype_message, + ) = self._obtain_send_message(tensor) + + def check_send_message(self, tensor): + if self.send_shape_message is None or self.send_dtype_message is None: + return + actual_shape, actual_dtype = self._obtain_send_message(tensor) + assert ( + self.send_shape_message == actual_shape + ), "send_shape_message: {}, actual_shape: {}".format( + self.send_shape_message, actual_shape + ) + assert ( + self.send_dtype_message == actual_dtype + ), "send_dtype_message: {}, actual_dtype: {}".format( + self.send_dtype_message, actual_dtype + ) def __repr__(self): return "send_shape_message: {}, send_dtype_message: {}, recv_shape_message: {}, recv_dtype_message: {}, recv_stop_gradient: {}".format( @@ -631,6 +653,8 @@ def _send_meta(self, output_tensor): output_tensor, _hcg.get_pipe_parallel_group() ) self._send_recv_meta.has_send_meta = self._use_cache + else: + self._send_recv_meta.check_send_message(output_tensor) def _recv_meta(self): if not self._send_recv_meta.has_recv_meta: From 2a4325ff87078908b3a3be529838a04c969bc771 Mon Sep 17 00:00:00 2001 From: wanghuancoder Date: Wed, 28 Feb 2024 10:43:45 +0800 Subject: [PATCH 141/282] [PIR] pir onednn support reshape, flatten, squeeze (#62080) * pir onednn support reshape --- .../ir_adaptor/translator/op_compat_gen.py | 2 +- .../operators/mkldnn/reshape_mkldnn_op.cc | 42 --------- .../dialect/operator/ir/ops_onednn_extra.yaml | 28 +++--- paddle/phi/kernels/onednn/flatten_kernel.cc | 85 +++++++++++++++++++ .../phi/kernels/onednn/flatten_kernel_grad.cc | 61 +++++++++++++ .../phi/kernels/onednn/reshape_grad_kernel.cc | 56 ++++++++++++ test/mkldnn/test_flatten_mkldnn_op.py | 21 ++++- test/mkldnn/test_reshape_bf16_op.py | 6 +- test/mkldnn/test_reshape_mkldnn_op.py | 26 +++++- test/mkldnn/test_squeeze2_mkldnn_op.py | 18 +++- 10 files changed, 274 insertions(+), 71 deletions(-) create mode 100644 paddle/phi/kernels/onednn/flatten_kernel.cc create mode 100644 paddle/phi/kernels/onednn/flatten_kernel_grad.cc create mode 100644 paddle/phi/kernels/onednn/reshape_grad_kernel.cc diff --git a/paddle/fluid/ir_adaptor/translator/op_compat_gen.py b/paddle/fluid/ir_adaptor/translator/op_compat_gen.py index 1035674cdb3d9..1cb0ab7a3b01a 100644 --- a/paddle/fluid/ir_adaptor/translator/op_compat_gen.py +++ b/paddle/fluid/ir_adaptor/translator/op_compat_gen.py @@ -84,7 +84,7 @@ def insert_new_mutable_attributes( if k == 'tensor_name' or k == 'tensors_name': op_mutable_attribute_infos[op_name][ attribute_name - ].append(v) + ].insert(0, v) _, legacy_name = insert_new_mappings(op_compat_item["op"]) legacy_backward_op_names = [] diff --git a/paddle/fluid/operators/mkldnn/reshape_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/reshape_mkldnn_op.cc index 5e5e2f8c19abe..1e3b29da11e5b 100644 --- a/paddle/fluid/operators/mkldnn/reshape_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/reshape_mkldnn_op.cc @@ -20,10 +20,8 @@ limitations under the License. */ namespace { enum class ReshapeKernelOpName { reshape, - reshape2, squeeze, flatten, - flatten2, }; } // anonymous namespace @@ -105,9 +103,6 @@ class ReshapeMKLDNNKernel : public framework::OpKernel { InferShapeSqueezeOp(ctx, x_dims, out_dims); break; case ReshapeKernelOpName::flatten: - case ReshapeKernelOpName::flatten2: - InferShapeFlattenOp(ctx, x_dims, out_dims); - break; default: PADDLE_THROW(paddle::platform::errors::OutOfRange( "Reshape kernel doesn not support that operator name")); @@ -317,18 +312,12 @@ class ReshapeGradMKLDNNKernel : public ReshapeMKLDNNKernel { case ReshapeKernelOpName::reshape: InferShapeReshapeSqueezeGradOp(ctx, x_dims); break; - case ReshapeKernelOpName::reshape2: - InferShapeReshape2Flatten2GradOp(ctx, x_dims); - break; case ReshapeKernelOpName::squeeze: InferShapeReshapeSqueezeGradOp(ctx, x_dims); break; case ReshapeKernelOpName::flatten: InferShapeFlattenGradOp(ctx, x_dims); break; - case ReshapeKernelOpName::flatten2: - InferShapeReshape2Flatten2GradOp(ctx, x_dims); - break; default: PADDLE_THROW(paddle::platform::errors::OutOfRange( "Reshape grad kernel doesn not support that operator name")); @@ -342,13 +331,6 @@ class ReshapeGradMKLDNNKernel : public ReshapeMKLDNNKernel { dx_dims = dx->dims(); } - void InferShapeReshape2Flatten2GradOp( - const framework::ExecutionContext& ctx, - framework::DDim& dx_dims) const { // NOLINT - auto xshape_dims = ctx.Input("XShape")->dims(); - dx_dims = common::slice_ddim(xshape_dims, 1, xshape_dims.size()); - } - void InferShapeFlattenGradOp(const framework::ExecutionContext& ctx, framework::DDim& dx_dims) const { // NOLINT dx_dims = ctx.Input("X")->dims(); @@ -390,14 +372,6 @@ REGISTER_OP_KERNEL( ops::ReshapeGradMKLDNNKernel); -REGISTER_OP_KERNEL( - reshape2_grad, - MKLDNN, - phi::CPUPlace, - ops::ReshapeGradMKLDNNKernel, - ops::ReshapeGradMKLDNNKernel); - REGISTER_OP_KERNEL( flatten, MKLDNN, @@ -413,19 +387,3 @@ REGISTER_OP_KERNEL( ops::ReshapeGradMKLDNNKernel, ops::ReshapeGradMKLDNNKernel); - -REGISTER_OP_KERNEL( - flatten2, - MKLDNN, - phi::CPUPlace, - ops::ReshapeMKLDNNKernel, - ops::ReshapeMKLDNNKernel); - -REGISTER_OP_KERNEL( - flatten2_grad, - MKLDNN, - phi::CPUPlace, - ops::ReshapeGradMKLDNNKernel, - ops::ReshapeGradMKLDNNKernel); diff --git a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml index c4523396b3673..5af2b7e13d0d8 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml @@ -86,13 +86,11 @@ - op : fc extra_args : bool ALL_KERNELS_MUST_COMPUTE_RUNTIME_SHAPE=true, bool use_quantizer=false, str mkldnn_data_type="float32", float scale_in=1.0, float[] scale_weights={1.0f}, float scale_out=1.0, bool force_fp32_output=false -# - op : flatten - -# - op : flatten_grad - -# - op : flatten2 +- op : flatten + extra_args : str mkldnn_data_type="float32" -# - op : flatten2_grad +- op : flatten_grad + extra_args : str mkldnn_data_type="float32" - op : full @@ -226,13 +224,11 @@ - op : relu6_grad extra_args : float threshold=6.0 -# - op : reshape - -# - op : reshape_grad +- op : reshape + extra_args : str mkldnn_data_type="float32", bool use_quantizer=false -# - op : reshape_infer - -# - op : reshape2_grad +- op : reshape_grad + extra_args : str mkldnn_data_type="float32", bool use_quantizer=false - op : round @@ -273,11 +269,11 @@ - op : sqrt_grad -# - op : squeeze - -# - op : squeeze_grad +- op : squeeze + extra_args : str mkldnn_data_type="float32" -# - op : squeeze_infer +- op : squeeze_grad + extra_args : str mkldnn_data_type="float32" # - op : stack diff --git a/paddle/phi/kernels/onednn/flatten_kernel.cc b/paddle/phi/kernels/onednn/flatten_kernel.cc new file mode 100644 index 0000000000000..63fdf8f426505 --- /dev/null +++ b/paddle/phi/kernels/onednn/flatten_kernel.cc @@ -0,0 +1,85 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/kernels/flatten_kernel.h" + +#include "paddle/phi/backends/onednn/onednn_reuse.h" +#include "paddle/phi/core/kernel_registry.h" + +namespace phi { + +template +void ExecuteFlatten(const Context& dev_ctx, + const DenseTensor& x, + const DDim& x_dims, + const DDim& out_dims, + DenseTensor* out) { + auto x_vec_dims = common::vectorize(x_dims); + + funcs::ReorderOneDNNHandler reorder_handler( + x_vec_dims, + x.dtype(), + funcs::ToOneDNNDataType(x.dtype()), + dev_ctx.GetEngine()); + + auto reorder_src_memory_p = reorder_handler.AcquireSrcMemory( + x.mem_desc(), funcs::to_void_cast(x.data())); + out->Resize(x_dims); // to match x numel, format is changed later + // reorder is done into a plain tag to allow usage with blocked formats + auto reorder_dst_memory_p = reorder_handler.AcquireDstMemory( + out, funcs::GetPlainOneDNNFormat(x_dims.size()), dev_ctx.GetPlace()); + auto reorder_p = reorder_handler.AcquireReorder(reorder_dst_memory_p, + reorder_src_memory_p); + auto& astream = OneDNNContext::tls().get_stream(); + reorder_p->execute(astream, *reorder_src_memory_p, *reorder_dst_memory_p); + astream.wait(); + + out->Resize(out_dims); + + auto reshape_dims = out_dims.size() != 0 ? common::vectorize(out_dims) + : std::vector{1}; + out->set_mem_desc(reorder_dst_memory_p->get_desc().reshape(reshape_dims)); +} + +template +void FlattenInferKernel(const Context& dev_ctx, + const DenseTensor& x, + int start_axis, + int stop_axis, + DenseTensor* out) { + auto x_dims = x.dims(); + auto out_dims = out->dims(); + ExecuteFlatten(dev_ctx, x, x_dims, out_dims, out); +} + +template +void FlattenKernel(const Context& dev_ctx, + const DenseTensor& x, + int start_axis, + int stop_axis, + DenseTensor* out, + DenseTensor* xshape UNUSED) { + FlattenInferKernel(dev_ctx, x, start_axis, stop_axis, out); +} + +} // namespace phi +PD_REGISTER_KERNEL(flatten_infer, + OneDNN, + ONEDNN, + phi::FlattenInferKernel, + float, + phi::dtype::bfloat16) {} + +PD_REGISTER_KERNEL( + flatten, OneDNN, ONEDNN, phi::FlattenKernel, float, phi::dtype::bfloat16) {} diff --git a/paddle/phi/kernels/onednn/flatten_kernel_grad.cc b/paddle/phi/kernels/onednn/flatten_kernel_grad.cc new file mode 100644 index 0000000000000..c886c8c12850d --- /dev/null +++ b/paddle/phi/kernels/onednn/flatten_kernel_grad.cc @@ -0,0 +1,61 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/kernels/flatten_grad_kernel.h" + +#include "paddle/phi/backends/onednn/onednn_reuse.h" +#include "paddle/phi/core/kernel_registry.h" + +namespace phi { + +template +void FlattenGradKernel(const Context& dev_ctx, + const DenseTensor& xshape, + const DenseTensor& out_grad, + DenseTensor* x_grad) { + auto out_grad_vec_dims = out_grad.dims().size() != 0 + ? common::vectorize(out_grad.dims()) + : std::vector{1}; + + auto out_grad_type = funcs::ToOneDNNDataType(out_grad.dtype()); + + funcs::ReorderOneDNNHandler reorder_handler( + out_grad_vec_dims, out_grad.dtype(), out_grad_type, dev_ctx.GetEngine()); + + auto reorder_src_memory_p = reorder_handler.AcquireSrcMemory( + out_grad.mem_desc(), funcs::to_void_cast(out_grad.data())); + auto reorder_dst_memory_p = reorder_handler.AcquireDstMemory( + x_grad, + funcs::GetPlainOneDNNFormat(out_grad_vec_dims.size()), + dev_ctx.GetPlace()); + auto reorder_p = reorder_handler.AcquireReorder(reorder_dst_memory_p, + reorder_src_memory_p); + + auto& astream = OneDNNContext::tls().get_stream(); + reorder_p->execute(astream, *reorder_src_memory_p, *reorder_dst_memory_p); + astream.wait(); + + auto x_grad_dims = slice_ddim(xshape.dims(), 1, xshape.dims().size()); + x_grad->Resize(x_grad_dims); + reorder_dst_memory_p->get_desc().reshape(common::vectorize(x_grad_dims)); +} + +} // namespace phi + +PD_REGISTER_KERNEL(flatten_grad, + OneDNN, + ONEDNN, + phi::FlattenGradKernel, + float, + phi::dtype::bfloat16) {} diff --git a/paddle/phi/kernels/onednn/reshape_grad_kernel.cc b/paddle/phi/kernels/onednn/reshape_grad_kernel.cc new file mode 100644 index 0000000000000..4ab7bef53aed9 --- /dev/null +++ b/paddle/phi/kernels/onednn/reshape_grad_kernel.cc @@ -0,0 +1,56 @@ +/* Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/phi/backends/onednn/onednn_reuse.h" +#include "paddle/phi/core/kernel_registry.h" + +namespace phi { + +template +void ReshapeGradKernel(const Context& dev_ctx, + const DenseTensor& out_grad, + DenseTensor* x_grad) { + auto out_grad_vec_dims = out_grad.dims().size() != 0 + ? common::vectorize(out_grad.dims()) + : std::vector{1}; + + auto out_grad_type = funcs::ToOneDNNDataType(out_grad.dtype()); + + funcs::ReorderOneDNNHandler reorder_handler( + out_grad_vec_dims, out_grad.dtype(), out_grad_type, dev_ctx.GetEngine()); + + auto reorder_src_memory_p = reorder_handler.AcquireSrcMemory( + out_grad.mem_desc(), funcs::to_void_cast(out_grad.data())); + auto reorder_dst_memory_p = reorder_handler.AcquireDstMemory( + x_grad, + funcs::GetPlainOneDNNFormat(out_grad_vec_dims.size()), + dev_ctx.GetPlace()); + auto reorder_p = reorder_handler.AcquireReorder(reorder_dst_memory_p, + reorder_src_memory_p); + + auto& astream = OneDNNContext::tls().get_stream(); + reorder_p->execute(astream, *reorder_src_memory_p, *reorder_dst_memory_p); + astream.wait(); + + auto grad_shape = x_grad->dims().size() == 0 + ? std::vector{1} + : phi::vectorize(x_grad->dims()); + reorder_dst_memory_p->get_desc().reshape(grad_shape); +} + +} // namespace phi + +PD_REGISTER_KERNEL(reshape_grad, + OneDNN, + ONEDNN, + phi::ReshapeGradKernel, + float, + phi::dtype::bfloat16) {} diff --git a/test/mkldnn/test_flatten_mkldnn_op.py b/test/mkldnn/test_flatten_mkldnn_op.py index 483262dee59dc..7bd90724082a1 100644 --- a/test/mkldnn/test_flatten_mkldnn_op.py +++ b/test/mkldnn/test_flatten_mkldnn_op.py @@ -38,10 +38,17 @@ def set_op_type(self): self.op_type = "flatten" def test_check_output(self): - self.check_output_with_place(core.CPUPlace()) + self.check_output_with_place( + core.CPUPlace(), check_pir_onednn=(self.op_type == "flatten2") + ) def test_check_grad(self): - self.check_grad_with_place(core.CPUPlace(), ["X"], "Out") + self.check_grad_with_place( + core.CPUPlace(), + ["X"], + "Out", + check_pir_onednn=(self.op_type == "flatten2"), + ) def init_test_case(self): self.in_shape = (3, 2, 2, 10) @@ -93,7 +100,9 @@ def calculate_grads(self): def test_check_output(self): self.check_output_with_place( - core.CPUPlace(), no_check_set=["XShape"] + core.CPUPlace(), + no_check_set=["XShape"], + check_pir_onednn=(self.op_type == "flatten2"), ) def test_check_grad(self): @@ -104,6 +113,7 @@ def test_check_grad(self): "Out", user_defined_grads=[self.dx], user_defined_grad_outputs=[self.dout], + check_pir_onednn=(self.op_type == "flatten2"), ) cls_name = "{}_{}".format(parent.__name__, "Flatten2_BF16") @@ -129,7 +139,9 @@ def calculate_grads(self): self.dx = np.reshape(self.dout, self.ori_shape) def test_check_output(self): - self.check_output_with_place(core.CPUPlace()) + self.check_output_with_place( + core.CPUPlace(), check_pir_onednn=(self.op_type == "flatten2") + ) def test_check_grad(self): self.calculate_grads() @@ -139,6 +151,7 @@ def test_check_grad(self): "Out", user_defined_grads=[self.dx], user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + check_pir_onednn=(self.op_type == "flatten2"), ) cls_name = "{}_{}".format(parent.__name__, "Flatten_BF16") diff --git a/test/mkldnn/test_reshape_bf16_op.py b/test/mkldnn/test_reshape_bf16_op.py index 245c47a327db7..8780bdcecaea0 100644 --- a/test/mkldnn/test_reshape_bf16_op.py +++ b/test/mkldnn/test_reshape_bf16_op.py @@ -56,7 +56,10 @@ def init_input_data(self): def test_check_output(self): self.check_output_with_place( - core.CPUPlace(), no_check_set=['XShape'], check_dygraph=False + core.CPUPlace(), + no_check_set=['XShape'], + check_dygraph=False, + check_pir_onednn=(self.op_type == "reshape2"), ) def test_check_grad(self): @@ -69,6 +72,7 @@ def test_check_grad(self): user_defined_grad_outputs=[ self.inputs["X"].reshape(self.infered_shape) ], + check_pir_onednn=(self.op_type == "reshape2"), ) diff --git a/test/mkldnn/test_reshape_mkldnn_op.py b/test/mkldnn/test_reshape_mkldnn_op.py index f30fd939f3aae..5f7b95a6172d3 100644 --- a/test/mkldnn/test_reshape_mkldnn_op.py +++ b/test/mkldnn/test_reshape_mkldnn_op.py @@ -55,10 +55,19 @@ def set_outputs(self): pass def test_check_output(self): - self.check_output(no_check_set=['XShape'], check_dygraph=False) + self.check_output( + no_check_set=['XShape'], + check_dygraph=False, + check_pir_onednn=(self.op_type == "reshape2"), + ) def test_check_grad(self): - self.check_grad(["X"], "Out", check_dygraph=False) + self.check_grad( + ["X"], + "Out", + check_dygraph=False, + check_pir_onednn=(self.op_type == "reshape2"), + ) class TestReshape2OneDNNOpZeroDim(TestReshape2OneDNNOp): @@ -212,7 +221,10 @@ def calculate_grads(self): def test_check_output(self): self.check_output_with_place( - core.CPUPlace(), no_check_set=["XShape"], check_dygraph=False + core.CPUPlace(), + no_check_set=["XShape"], + check_dygraph=False, + check_pir_onednn=(self.op_type == "reshape2"), ) def test_check_grad(self): @@ -224,6 +236,7 @@ def test_check_grad(self): user_defined_grads=[self.dx], user_defined_grad_outputs=[self.dout], check_dygraph=False, + check_pir_onednn=(self.op_type == "reshape2"), ) cls_name = "{}_{}".format(parent.__name__, "Reshape2_BF16") @@ -239,7 +252,11 @@ def set_outputs(self): self.outputs = {"Out": self.x.reshape(self.new_shape)} def test_check_output(self): - self.check_output_with_place(core.CPUPlace(), check_dygraph=False) + self.check_output_with_place( + core.CPUPlace(), + check_dygraph=False, + check_pir_onednn=(self.op_type == "reshape2"), + ) def test_check_grad(self): self.calculate_grads() @@ -250,6 +267,7 @@ def test_check_grad(self): user_defined_grads=[self.dx], user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], check_dygraph=False, + check_pir_onednn=(self.op_type == "reshape2"), ) cls_name = "{}_{}".format(parent.__name__, "Reshape_BF16") diff --git a/test/mkldnn/test_squeeze2_mkldnn_op.py b/test/mkldnn/test_squeeze2_mkldnn_op.py index 61521ecf8bc80..fc0f731f35b68 100644 --- a/test/mkldnn/test_squeeze2_mkldnn_op.py +++ b/test/mkldnn/test_squeeze2_mkldnn_op.py @@ -55,10 +55,19 @@ def setUp(self): self.set_outputs() def test_check_output(self): - self.check_output_with_place(core.CPUPlace(), no_check_set=['XShape']) + self.check_output_with_place( + core.CPUPlace(), + no_check_set=['XShape'], + check_pir_onednn=(self.op_type == "squeeze2"), + ) def test_check_grad(self): - self.check_grad_with_place(core.CPUPlace(), ["X"], "Out") + self.check_grad_with_place( + core.CPUPlace(), + ["X"], + "Out", + check_pir_onednn=(self.op_type == "squeeze2"), + ) class TestSqueezeOneDNNOp(TestSqueeze2OneDNNOp): @@ -158,6 +167,7 @@ def test_check_grad(self): "Out", user_defined_grads=[self.dx], user_defined_grad_outputs=[self.dout], + check_pir_onednn=(self.op_type == "squeeze2"), ) cls_name = "{}_{}".format(parent.__name__, "Squeeze2_BF16") @@ -173,7 +183,9 @@ def set_outputs(self): self.outputs = {"Out": self.x.reshape(self.new_shape)} def test_check_output(self): - self.check_output_with_place(core.CPUPlace()) + self.check_output_with_place( + core.CPUPlace(), check_pir_onednn=(self.op_type == "squeeze2") + ) cls_name = "{}_{}".format(parent.__name__, "Squeeze_BF16") TestSqueezeBF16OneDNNOp.__name__ = cls_name From 81fb3f0cc722e0a8665bdb04d2502ec735b757ca Mon Sep 17 00:00:00 2001 From: bukejiyu <52310069+bukejiyu@users.noreply.github.com> Date: Wed, 28 Feb 2024 10:44:41 +0800 Subject: [PATCH 142/282] [DRR] drr match bug fix (#62158) --- paddle/fluid/pir/drr/src/rewrite_pattern.cc | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/pir/drr/src/rewrite_pattern.cc b/paddle/fluid/pir/drr/src/rewrite_pattern.cc index 2a2e654166522..68a7b14f81a3e 100644 --- a/paddle/fluid/pir/drr/src/rewrite_pattern.cc +++ b/paddle/fluid/pir/drr/src/rewrite_pattern.cc @@ -298,8 +298,20 @@ bool DrrRewritePattern::MatchFromOutputToInput( source_pattern_match_ctx->BindIrOperation(drr_node, ir_node); // binding input_tensor of current_op for (size_t i = 0; i < drr_input_tensors.size(); ++i) { - source_pattern_match_ctx->BindIrValue(drr_input_tensors[i]->name(), - ir_node->operand(i).source()); + if (source_pattern_match_ctx->tensor_map().count( + drr_input_tensors[i]->name()) != 0 && + ir_node->operand(i).source() != + source_pattern_match_ctx->tensor_map().at( + drr_input_tensors[i]->name())) { + matched = false; + VLOG(8) << " tensor_map key[" << drr_input_tensors[i]->name() + << "] already exists,but value is different!"; + break; + } else { + source_pattern_match_ctx->BindIrValue(drr_input_tensors[i]->name(), + ir_node->operand(i).source()); + } + if (ir_node->operand_source(i).isa()) { VLOG(8) << "Match Attention! Found BlockArgument as input of " << drr_node->name(); From 18acedffd672b3e24679449ae6580e6d3d707dc3 Mon Sep 17 00:00:00 2001 From: zyfncg Date: Wed, 28 Feb 2024 11:00:21 +0800 Subject: [PATCH 143/282] fix bug of test mlp (#62150) --- test/ir/pir/cinn/symbolic/test_llama_mlp_dy.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/test/ir/pir/cinn/symbolic/test_llama_mlp_dy.py b/test/ir/pir/cinn/symbolic/test_llama_mlp_dy.py index fd63b726f3f24..96cbbd8076702 100644 --- a/test/ir/pir/cinn/symbolic/test_llama_mlp_dy.py +++ b/test/ir/pir/cinn/symbolic/test_llama_mlp_dy.py @@ -36,25 +36,16 @@ def __init__(self): self.gate_proj = nn.Linear( self.hidden_size, self.intermediate_size, - weight_attr=paddle.ParamAttr( - initializer=nn.initializer.Constant(value=0.5) - ), bias_attr=False, ) self.up_proj = nn.Linear( self.hidden_size, self.intermediate_size, - weight_attr=paddle.ParamAttr( - initializer=nn.initializer.Constant(value=0.5) - ), bias_attr=False, ) self.down_proj = nn.Linear( self.intermediate_size, self.hidden_size, - weight_attr=paddle.ParamAttr( - initializer=nn.initializer.Constant(value=0.5) - ), bias_attr=False, ) From 5b7727850a7a4850fb1827a48b499f82f91ca6c6 Mon Sep 17 00:00:00 2001 From: lzydev Date: Wed, 28 Feb 2024 11:04:43 +0800 Subject: [PATCH 144/282] =?UTF-8?q?=E3=80=90AutoParallel=E3=80=91Enable=20?= =?UTF-8?q?amp=20strategy=20in=20`dist.to=5Fstatic`=20(#62015)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * enable amp strategy in dist.to_static * support parameter init in to_static * polish * polish --- .../paddle/distributed/auto_parallel/api.py | 1 + .../auto_parallel/static/helper.py | 53 +++++++++++++++++++ .../distributed/passes/auto_parallel_fp16.py | 23 ++++++-- 3 files changed, 73 insertions(+), 4 deletions(-) diff --git a/python/paddle/distributed/auto_parallel/api.py b/python/paddle/distributed/auto_parallel/api.py index d5de3545b2ea6..28f15011190f2 100644 --- a/python/paddle/distributed/auto_parallel/api.py +++ b/python/paddle/distributed/auto_parallel/api.py @@ -1630,6 +1630,7 @@ def __convert_strategy(self, strategy): "fused_dropout_add_pass" ) + inner_strategy.amp = copy.deepcopy(strategy.amp) inner_strategy.sharding = copy.deepcopy(strategy.sharding) inner_strategy.gradient_merge = copy.deepcopy(strategy.gradient_merge) inner_strategy.pipeline = copy.deepcopy(strategy.pipeline) diff --git a/python/paddle/distributed/auto_parallel/static/helper.py b/python/paddle/distributed/auto_parallel/static/helper.py index e7bd7553d5094..50b67e0cbb946 100644 --- a/python/paddle/distributed/auto_parallel/static/helper.py +++ b/python/paddle/distributed/auto_parallel/static/helper.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy import inspect import logging from collections import defaultdict +import numpy as np + import paddle from paddle.jit import not_to_static, to_static from paddle.jit.dy2static.program_translator import ( @@ -27,6 +30,7 @@ from paddle.static import Parameter, global_scope, program_guard from paddle.static.amp.fp16_utils import ( DEFAULT_AMP_OPTIONS, + _convert_float_to_bfloat16, prepare_op_amp_options, ) @@ -385,6 +389,55 @@ def init(self, main_program, place, dist_context): dense_tensor = global_scope().var(param.name).get_tensor() dense_tensor._share_data_with(param.get_tensor().get_tensor()) + # transform the parameter in eager mode for amp. + amp_stragety = dist_context.strategy.amp + amp_config = copy.deepcopy(amp_stragety.to_dict()) + if amp_stragety.enable and amp_config["level"] in ["o2", "o3"]: + for param in self.concrete_program.parameters: + amp_dtype = amp_config["dtype"] + scope_var = global_scope().find_var(param.name) + scope_tensor = global_scope().var(param.name).get_tensor() + # The parameter is not in this rank. + if not scope_var: + continue + # The parameter do not need to transform + if param.dtype in [paddle.float16, paddle.bfloat16]: + continue + assert ( + scope_var and scope_tensor._is_initialized() + ), f"Parameter: {param.name} is not put into global_scope or not initialized." + var = main_program.global_block().vars[param.name] + var_dist_attr = dist_context.get_tensor_dist_attr_for_program( + var + ) + dist_attr = { + "dims_mapping": var_dist_attr.dims_mapping, + "process_shape": var_dist_attr.process_mesh.shape, + "process_group": var_dist_attr.process_mesh.process_ids, + } + if amp_dtype == "float16": + if param.is_dist(): + sliced_param = np.float16(param._local_value().numpy()) + else: + sliced_param = Converter.slice_with_dist_attr( + np.float16(param.numpy()), dist_attr + ) + scope_tensor.set(sliced_param, place) + elif amp_dtype == "bfloat16": + if param.is_dist(): + sliced_param = _convert_float_to_bfloat16( + place, param._local_value().numpy() + ) + else: + sliced_param = Converter.slice_with_dist_attr( + _convert_float_to_bfloat16(place, param.numpy()), + dist_attr, + ) + scope_tensor.set( + sliced_param, + place, + ) + world_group = get_world_process_group() if ( is_comm diff --git a/python/paddle/distributed/passes/auto_parallel_fp16.py b/python/paddle/distributed/passes/auto_parallel_fp16.py index c1d8c54c6b4b2..73cad3e3e928c 100644 --- a/python/paddle/distributed/passes/auto_parallel_fp16.py +++ b/python/paddle/distributed/passes/auto_parallel_fp16.py @@ -308,10 +308,25 @@ def resolute_cast_op(self, block): if op.type == "cast": in_name = op.input('X')[0] out_name = op.output('Out')[0] - in_var = block._find_var_recursive(in_name) - out_var = block._find_var_recursive(out_name) - op._set_attr("in_dtype", in_var.dtype) - op._set_attr("out_dtype", out_var.dtype) + if "@GRAD" in in_name: + in_var_fw = block._find_var_recursive( + in_name[: in_name.find("@")] + ) + out_var_fw = block._find_var_recursive( + out_name[: out_name.find("@")] + ) + op._set_attr('in_dtype', in_var_fw.dtype) + op._set_attr('out_dtype', out_var_fw.dtype) + + in_var = block._find_var_recursive(in_name) + out_var = block._find_var_recursive(out_name) + in_var.desc.set_dtype(in_var_fw.dtype) + out_var.desc.set_dtype(out_var_fw.dtype) + else: + in_var = block._find_var_recursive(in_name) + out_var = block._find_var_recursive(out_name) + op._set_attr("in_dtype", in_var.dtype) + op._set_attr("out_dtype", out_var.dtype) def resolute_tensor_dtype(self, block): for op in block.ops: From 629641d62df03b307b4e36983292a18d05bdc280 Mon Sep 17 00:00:00 2001 From: WangZhen <23097963+0x45f@users.noreply.github.com> Date: Wed, 28 Feb 2024 11:14:22 +0800 Subject: [PATCH 145/282] [PIR]Fix expand infershape error when shape contains value (#62069) --- .../pir/dialect/operator/ir/manual_op.cc | 88 ++++++++++++++----- python/paddle/nn/functional/loss.py | 2 +- test/legacy_test/test_expand_v2_op.py | 24 +++-- 3 files changed, 84 insertions(+), 30 deletions(-) diff --git a/paddle/fluid/pir/dialect/operator/ir/manual_op.cc b/paddle/fluid/pir/dialect/operator/ir/manual_op.cc index 2468ae05ee1e5..1f645b0a29d66 100644 --- a/paddle/fluid/pir/dialect/operator/ir/manual_op.cc +++ b/paddle/fluid/pir/dialect/operator/ir/manual_op.cc @@ -3573,6 +3573,7 @@ std::vector ExpandOp::InferMeta( pir::Value shape_ = input_values[1]; VLOG(4) << "Builder construction outputs"; + bool is_from_tensor = false; paddle::dialect::DenseTensorType x; if (x_.type().isa()) { x = x_.type().dyn_cast(); @@ -3591,29 +3592,70 @@ std::vector ExpandOp::InferMeta( "paddle::dialect::AllocatedDenseTensorType")); } - phi::IntArray shape; - if (shape_.defining_op()->isa()) { - shape = std::move(phi::IntArray(paddle::dialect::GetInt64Vector( - shape_.defining_op() - ->dyn_cast() - .attribute("value")))); - } else if (shape_.type().isa()) { - size_t shape_size = shape_.type().dyn_cast().size(); - // In ExpandInferMeta use -2 to represent the element in expand_shape is a - // var. - shape = std::move(phi::IntArray(std::vector(shape_size, -2))); - shape.SetFromTensor(true); - } else if (shape_.type().isa()) { - size_t shape_size = common::product( - shape_.type().dyn_cast().dims()); - // In ExpandInferMeta use -2 to represent the element in expand_shape is a - // var. - shape = std::move(phi::IntArray(std::vector(shape_size, -2))); - shape.SetFromTensor(true); - } else { - PADDLE_THROW(phi::errors::Unimplemented( - "Only support VectorType or DenseTensorType")); - } + std::function(const pir::Value &, bool *)> + ParseValueShape = [&](const pir::Value &shape, + bool *is_from_tensor) -> std::vector { + std::vector vec_shape; + if (shape.isa() && + shape.defining_op()->isa()) { + vec_shape = paddle::dialect::GetInt64Vector( + shape.defining_op() + ->dyn_cast() + .attribute("value")); + } else if (shape.isa() && + shape.defining_op()->isa()) { + auto shape_item = shape.defining_op() + ->dyn_cast() + .attribute("value") + .dyn_cast() + .data(); + vec_shape = {static_cast(shape_item)}; + } else if (shape.isa() && + shape.defining_op()->isa()) { + std::vector inputs = shape.defining_op() + ->operand_source(0) + .defining_op() + ->operands_source(); + for (auto item : inputs) { + auto tmp = ParseValueShape(item, is_from_tensor); + vec_shape.insert(vec_shape.end(), tmp.begin(), tmp.end()); + } + } else if (shape.type().isa()) { + size_t shape_size = shape.type().dyn_cast().size(); + vec_shape = std::vector(shape_size, -2); + *is_from_tensor = true; + } else if (shape.type().isa()) { + common::DDim shape_dim = + shape.type().dyn_cast().dims(); + size_t shape_size = common::product(shape_dim); + if (common::contain_unknown_dim(shape_dim)) { + shape_size = 1; + } + vec_shape = std::vector(shape_size, -2); + *is_from_tensor = true; + } else if (shape.type().isa()) { + common::DDim shape_dim = + shape.type() + .dyn_cast() + .dims(); + size_t shape_size = common::product(shape_dim); + if (common::contain_unknown_dim(shape_dim)) { + shape_size = 1; + } + vec_shape = std::vector(shape_size, -2); + *is_from_tensor = true; + } else { + PADDLE_THROW(phi::errors::Unimplemented( + "Only support VectorType or DenseTensorType " + "or AllocatedDenseTensorType")); + } + return vec_shape; + }; + + is_from_tensor = false; + phi::IntArray shape = + std::move(phi::IntArray(ParseValueShape(shape_, &is_from_tensor))); + if (is_from_tensor) shape.SetFromTensor(true); VLOG(4) << "Builder construction dense_x"; paddle::dialect::IrTensor ir_meta_tensor_x( diff --git a/python/paddle/nn/functional/loss.py b/python/paddle/nn/functional/loss.py index d678e4213317d..446eb7d62a2f5 100644 --- a/python/paddle/nn/functional/loss.py +++ b/python/paddle/nn/functional/loss.py @@ -3192,7 +3192,7 @@ def sigmoid_focal_loss( if in_dynamic_or_pir_mode(): place = _current_expected_place() - one = _C_ops.full(logit.shape, 1.0, logit.dtype, place) + one = _C_ops.full(paddle.shape(logit), 1.0, logit.dtype, place) loss = _C_ops.sigmoid_cross_entropy_with_logits( logit, label, None, False, -100 diff --git a/test/legacy_test/test_expand_v2_op.py b/test/legacy_test/test_expand_v2_op.py index 317862623a9dd..d31cceddb1bba 100644 --- a/test/legacy_test/test_expand_v2_op.py +++ b/test/legacy_test/test_expand_v2_op.py @@ -18,6 +18,7 @@ import numpy as np from decorator_helper import prog_scope from op_test import OpTest, convert_float_to_uint16 +from utils import static_guard import paddle from paddle import base @@ -542,12 +543,23 @@ def test_check_output(self): class TestExpandPirValueListShape(unittest.TestCase): - def test_value_list_shape(self): - with paddle.pir_utils.IrGuard(): - x = paddle.static.data('x', [1, 3]) - shape = [2, paddle.full([], 4)] - out = paddle.expand(x, shape) - np.testing.assert_array_equal(tuple(out.shape), (-1, -1)) + @test_with_pir_api + def test_value_list_shape1(self): + with static_guard(): + with paddle.static.program_guard(paddle.static.Program()): + x = paddle.static.data('x', [1, 3]) + shape = [2, paddle.full([], 4)] + out = paddle.expand(x, shape) + np.testing.assert_array_equal(tuple(out.shape), (2, -1)) + + @test_with_pir_api + def test_value_list_shape2(self): + with static_guard(): + with paddle.static.program_guard(paddle.static.Program()): + x = paddle.static.data('x', [1, 1, -1, -1], 'float32') + shape1 = paddle.static.data('shape1', [], 'int32') + x = paddle.expand(x, shape=[shape1, 1, -1, -1]) + np.testing.assert_equal(tuple(x.shape), (-1, 1, -1, -1)) if __name__ == "__main__": From 52b4ee28c01c60718aae7c54c6dcc81bc3be6bcd Mon Sep 17 00:00:00 2001 From: Bo Zhang <105368690+zhangbopd@users.noreply.github.com> Date: Wed, 28 Feb 2024 11:21:18 +0800 Subject: [PATCH 146/282] add all InferSymShape (#62151) --- .../paddle_op_infer_sym.cc | 735 ++++++++++++++++++ .../paddle_op_infer_sym.h | 247 ++++++ paddle/fluid/pir/dialect/operator/ir/ops.yaml | 22 + paddle/phi/api/yaml/ops.yaml | 65 ++ 4 files changed, 1069 insertions(+) diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc index 1bbdf90d67fa4..65e9770350c80 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc @@ -1113,4 +1113,739 @@ bool SplitOpInferSymbolicShape(pir::Operation *op, return true; } +// Not Impelmented Ops. +bool AcosOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Acos_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool AcoshOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Acosh_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool AngleOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool ArgmaxOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool ArgminOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool ArgsortOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool AsComplexOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool AsRealOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool AsStridedOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool AsinOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Asin_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool AsinhOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Asinh_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool AtanOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Atan_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool AtanhOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Atanh_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool BernoulliOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool BitwiseNotOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool BitwiseNot_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool BitwiseXorOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool BitwiseXor_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool CeilOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Ceil_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool ComplexOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool ConjOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool CosOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Cos_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool CoshOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Cosh_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool CummaxOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool CumminOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool CumprodOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Cumprod_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool CumsumOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Cumsum_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool DiagEmbedOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool DiagonalOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool DirichletOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool ErfOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Erf_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool ErfinvOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Erfinv_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Expm1OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Expm1_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool FlipOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool FloorOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Floor_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool FmaxOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool FminOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool GatherOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool ImagOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool IsinfOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool IsinfSrOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool IsnanOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool IsnanSrOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool KronOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool KthvalueOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool LgammaOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Lgamma_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Log1pOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Log1p_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool LogcumsumexpOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool LogicalOrOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool LogicalOr_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool LogicalXorOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool LogicalXor_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool LogitOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Logit_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool MaskedSelectOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool PoissonOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool PutAlongAxisOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool PutAlongAxis_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool RealOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool RollOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool RoundOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Round_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool ScatterNdAddOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool ScatterOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Scatter_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool SearchsortedOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool SignOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool SinOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Sin_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool SinhOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Sinh_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool TakeAlongAxisOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool TanOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Tan_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool TanhOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Tanh_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool TopkOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool UnbindOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool UniqueConsecutiveOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} + +bool EinsumOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool EmptyOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool EqualOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Equal_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Exponential_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool GaussianOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool GreaterEqualOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool GreaterEqual_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool LessEqualOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool LessEqual_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool LinspaceOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool LogspaceOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool LogsumexpOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool MaximumOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool MinOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool MinimumOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool PadOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool RandintOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool RemainderOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool Remainder_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool RepeatInterleaveOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool SplitWithNumOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool TrilIndicesOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool TriuIndicesOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool UniformOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} +bool UniqueOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + PADDLE_THROW(phi::errors::Unimplemented( + op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + return true; +} + } // namespace paddle::dialect diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h index 7c61075247ce0..ee5bcacf63a1f 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h @@ -112,4 +112,251 @@ bool ExpandAsOpInferSymbolicShape( bool SplitOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +// Not Impelmented Ops. +bool AcosOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Acos_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool AcoshOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Acosh_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool AngleOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool ArgmaxOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool ArgminOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool ArgsortOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool AsComplexOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool AsRealOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool AsStridedOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool AsinOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Asin_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool AsinhOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Asinh_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool AtanOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Atan_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool AtanhOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Atanh_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool BernoulliOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool BitwiseNotOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool BitwiseNot_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool BitwiseXorOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool BitwiseXor_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool CeilOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Ceil_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool ComplexOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool ConjOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool CosOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Cos_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool CoshOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Cosh_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool CummaxOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool CumminOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool CumprodOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Cumprod_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool CumsumOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Cumsum_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool DiagEmbedOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool DiagonalOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool DirichletOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool ErfOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Erf_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool ErfinvOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Erfinv_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Expm1OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Expm1_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool FlipOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool FloorOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Floor_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool FmaxOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool FminOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool GatherOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool ImagOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool IsinfOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool IsinfSrOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool IsnanOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool IsnanSrOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool KronOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool KthvalueOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool LgammaOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Lgamma_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Log1pOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Log1p_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool LogcumsumexpOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool LogicalOrOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool LogicalOr_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool LogicalXorOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool LogicalXor_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool LogitOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Logit_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool MaskedSelectOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool PoissonOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool PutAlongAxisOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool PutAlongAxis_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool RealOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool RollOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool RoundOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Round_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool ScatterNdAddOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool ScatterOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Scatter_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool SearchsortedOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool SignOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool SinOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Sin_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool SinhOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Sinh_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool TakeAlongAxisOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool TanOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Tan_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool TanhOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Tanh_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool TopkOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool UnbindOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool UniqueConsecutiveOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool EinsumOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool EmptyOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool EqualOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Equal_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Exponential_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool GaussianOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool GreaterEqualOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool GreaterEqual_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool LessEqualOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool LessEqual_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool LinspaceOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool LogspaceOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool LogsumexpOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool MaximumOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool MinOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool MinimumOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool PadOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool RandintOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool RemainderOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Remainder_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool RepeatInterleaveOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool SplitWithNumOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool TrilIndicesOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool TriuIndicesOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool UniformOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool UniqueOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); + } // namespace paddle::dialect diff --git a/paddle/fluid/pir/dialect/operator/ir/ops.yaml b/paddle/fluid/pir/dialect/operator/ir/ops.yaml index dac35221ee83b..5c163637450c3 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops.yaml @@ -445,6 +445,7 @@ func : einsum optional : inner_cache, xshape backward : einsum_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : elementwise_pow args : (Tensor x, Tensor y) @@ -484,6 +485,7 @@ param : [shape, dtype] data_type : dtype backend : place + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : empty_like args : (Tensor x, DataType dtype = DataType::UNDEFINED, Place place = {}) @@ -519,6 +521,7 @@ data_transform : support_trans_dtype : x, y inplace: (x -> out) + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : exponential_ args : (Tensor x, float lam) @@ -530,6 +533,7 @@ func : exponential inplace : (x -> out) backward : exponential__grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : eye args : (Scalar num_rows, Scalar num_columns, DataType dtype=DataType::FLOAT32, Place place={}) @@ -708,6 +712,7 @@ param : [shape, mean, std, seed, dtype] data_type : dtype backend : place + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : get_tensor_from_selected_rows args : (Tensor x) @@ -727,6 +732,7 @@ data_transform : support_trans_dtype : x, y inplace: (x -> out) + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : greater_than args : (Tensor x, Tensor y) @@ -781,6 +787,7 @@ data_transform : support_trans_dtype : x, y inplace: (x -> out) + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : less_than args : (Tensor x, Tensor y) @@ -805,6 +812,7 @@ param: [start, stop, number, dtype] data_type : dtype backend : place + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : load_combine args : (str file_path, bool load_as_fp16, bool model_from_memory) @@ -829,6 +837,7 @@ param : [start, stop, num, base, dtype] data_type : dtype backend : place + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : logsumexp args : (Tensor x, int64_t[] axis, bool keepdim, bool reduce_all) @@ -838,6 +847,7 @@ kernel : func : logsumexp backward : logsumexp_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : lrn args : (Tensor x, int n=5, float k=2.0, float alpha=0.0001, float beta=0.75, str data_format="NCHW") @@ -911,6 +921,7 @@ data_transform : support_trans_dtype : x, y backward : maximum_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : mean args : (Tensor x, IntArray axis={}, bool keepdim=false) @@ -958,6 +969,7 @@ kernel : func : min backward : min_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : minimum args : (Tensor x, Tensor y) @@ -969,6 +981,7 @@ data_transform : support_trans_dtype : x, y backward : minimum_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : mish args : (Tensor x, float lambda) @@ -1043,6 +1056,7 @@ kernel : func : pad backward : pad_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : pool2d args : (Tensor x, IntArray kernel_size, int[] strides, int[] paddings, bool ceil_mode, bool exclusive, str data_format, str pooling_type, bool global_pooling, bool adaptive, str padding_algorithm) @@ -1121,6 +1135,7 @@ param : [low, high, shape, dtype] data_type : dtype backend : place + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : randperm args : (int n, DataType dtype, Place place={}) @@ -1167,6 +1182,7 @@ data_transform : support_trans_dtype : x, y inplace : (x -> out) + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : repeat_interleave args : (Tensor x, int repeats, int axis) @@ -1177,6 +1193,7 @@ func : repeat_interleave data_type : x backward: repeat_interleave_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : repeat_interleave_with_tensor_index args : (Tensor x, Tensor repeats, int axis) @@ -1365,6 +1382,7 @@ kernel : func : split_with_num backward : split_with_num_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : strided_slice args : (Tensor x, int[] axes, IntArray starts, IntArray ends, IntArray strides) @@ -1487,6 +1505,7 @@ param : [rows, cols, offset, dtype] data_type : dtype backend : place + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : triu args : (Tensor x, int diagonal) @@ -1510,6 +1529,7 @@ param : [row, col, offset, dtype] data_type : dtype backend : place + interfaces : paddle::dialect::InferSymbolicShapeInterface # python API: paddle.nn.initializer.TruncatedNormal - op : truncated_gaussian_random @@ -1535,6 +1555,7 @@ param: [shape, dtype, min, max, seed] data_type : dtype backend : place + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : uniform_random_batch_size_like args : (Tensor input, int[] shape, int input_dim_idx=0, int output_dim_idx=0, float min=-1.0f, float max=1.0f, int seed=0, int diag_num=0, int diag_step=0, float diag_val=1.0f, DataType dtype=DataType::FLOAT32) @@ -1556,6 +1577,7 @@ func : unique data_type : x interfaces : paddle::dialect::ParseKernelKeyInterface + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : unpool args: (Tensor x, Tensor indices, int[] ksize, int[] strides, int[] padding, IntArray output_size, str data_format) diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml index 6cf76d4c25c06..cf3986cae89e0 100755 --- a/paddle/phi/api/yaml/ops.yaml +++ b/paddle/phi/api/yaml/ops.yaml @@ -35,6 +35,7 @@ func : acos inplace: (x -> out) backward : acos_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : acosh args : (Tensor x) @@ -45,6 +46,7 @@ func : acosh inplace: (x -> out) backward : acosh_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : adagrad_ args : (Tensor param, Tensor grad, Tensor moment, Tensor learning_rate, Tensor master_param, float epsilon = 1.0e-6f, bool multi_precision = false) @@ -134,6 +136,7 @@ kernel : func : angle backward : angle_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : apply_per_channel_scale args: (Tensor x, Tensor scales) @@ -152,6 +155,7 @@ kernel : func : argmax data_type : x + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : argmin args : (Tensor x, Scalar(int64_t) axis, bool keepdims = false, bool flatten = false, DataType dtype = DataType::INT64) @@ -161,6 +165,7 @@ kernel : func : argmin data_type : x + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : argsort args : (Tensor x, int axis=-1, bool descending=false) @@ -170,6 +175,7 @@ kernel : func : argsort backward : argsort_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : as_complex args : (Tensor x) @@ -179,6 +185,7 @@ kernel : func : as_complex backward : as_complex_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : as_real args : (Tensor x) @@ -188,6 +195,7 @@ kernel : func : as_real backward : as_real_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : as_strided args : (Tensor input, int64_t[] dims = {}, int64_t[] stride = {}, int64_t offset = 0) @@ -199,6 +207,7 @@ func : as_strided backward : as_strided_grad no_need_buffer : input + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : asgd_ args : (Tensor param, Tensor grad, Tensor learning_rate, Tensor d, Tensor y, Tensor n, Tensor master_param, bool multi_precision=false) @@ -222,6 +231,7 @@ func : asin inplace: (x -> out) backward : asin_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : asinh args : (Tensor x) @@ -232,6 +242,7 @@ func : asinh inplace: (x -> out) backward : asinh_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : atan args : (Tensor x) @@ -242,6 +253,7 @@ func : atan inplace: (x -> out) backward : atan_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : atan2 args : (Tensor x, Tensor y) @@ -261,6 +273,7 @@ func : atanh inplace: (x -> out) backward : atanh_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : auc args : (Tensor x, Tensor label, Tensor stat_pos, Tensor stat_neg, Tensor ins_tag_weight, str curve = "ROC", int num_thresholds = (2 << 12) - 1, int slide_steps = 1) @@ -300,6 +313,7 @@ func : UnchangedInferMeta kernel : func : bernoulli + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : bicubic_interp args : (Tensor x, Tensor out_size, Tensor[] size_tensor, Tensor scale_tensor, str data_format="NCHW", int out_d=0, int out_h=0, int out_w=0, float[] scale={}, str interp_method="bilinear", bool align_corners=true, int align_mode=1) @@ -386,6 +400,7 @@ func : bitwise_not backend : x inplace: (x -> out) + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : bitwise_or args : (Tensor x, Tensor y) @@ -416,6 +431,7 @@ func : bitwise_xor backend : x inplace: (x -> out) + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : bmm args : (Tensor x, Tensor y) @@ -454,6 +470,7 @@ func : ceil inplace : (x -> out) backward : ceil_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : celu args : (Tensor x, float alpha = 1.0) @@ -551,6 +568,7 @@ func : complex data_type : real backward : complex_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : concat args : (Tensor[] x, Scalar axis=0) @@ -573,6 +591,7 @@ kernel : func : conj backward : conj_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : conv2d args : (Tensor input, Tensor filter, int[] strides={1, 1}, int[] paddings={0, 0}, str padding_algorithm="EXPLICIT", int[] dilations={1, 1}, int groups=1, str data_format="NCHW") @@ -624,6 +643,7 @@ func : cos inplace: (x -> out) backward : cos_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : cosh args : (Tensor x) @@ -634,6 +654,7 @@ func : cosh inplace: (x -> out) backward : cosh_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : crop args : (Tensor x, IntArray shape = {}, IntArray offsets = {}) @@ -677,6 +698,7 @@ func : cummax data_type : x backward : cummax_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : cummin args : (Tensor x, int axis=-1, DataType dtype = DataType::INT64) @@ -687,6 +709,7 @@ func : cummin data_type : x backward : cummin_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : cumprod args : (Tensor x, int dim) @@ -697,6 +720,7 @@ func : cumprod inplace: (x -> out) backward : cumprod_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : cumsum args : (Tensor x, Scalar axis=-1, bool flatten=false, bool exclusive=false, bool reverse=false) @@ -708,6 +732,7 @@ data_type : x inplace: (x -> out) backward : cumsum_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : data args : (str name, IntArray shape, DataType dtype, Place place) @@ -757,6 +782,7 @@ func : DiagEmbedInferMeta kernel : func : diag_embed + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : diagonal args : (Tensor x, int offset = 0, int axis1 = 0, int axis2 = 1) @@ -766,6 +792,7 @@ kernel : func : diagonal backward : diagonal_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : digamma args : (Tensor x) @@ -784,6 +811,7 @@ func: DirichletInferMeta kernel: func: dirichlet + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : dist args : (Tensor x, Tensor y, float p = 2.0) @@ -878,6 +906,7 @@ func : erf inplace : (x -> out) backward : erf_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : erfinv args : (Tensor x) @@ -888,6 +917,7 @@ func : erfinv inplace : (x -> out) backward : erfinv_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : exp args : (Tensor x) @@ -934,6 +964,7 @@ func : expm1 inplace: (x -> out) backward : expm1_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : fft_c2c args : (Tensor x, int64_t[] axes, str normalization, bool forward) @@ -1041,6 +1072,7 @@ kernel : func : flip backward : flip_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : floor args : (Tensor x) @@ -1051,6 +1083,7 @@ func : floor inplace : (x -> out) backward : floor_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : fmax args : (Tensor x, Tensor y) @@ -1061,6 +1094,7 @@ kernel : func : fmax backward : fmax_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : fmin args : (Tensor x, Tensor y) @@ -1071,6 +1105,7 @@ kernel : func : fmin backward : fmin_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : fold args: (Tensor x, int[] output_sizes, int[] kernel_sizes, int[] strides, int[] paddings, int[] dilations) @@ -1151,6 +1186,7 @@ func : gather data_type: x backward : gather_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : gather_nd args : (Tensor x, Tensor index) @@ -1369,6 +1405,7 @@ kernel : func : imag backward : imag_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : index_add args : (Tensor x, Tensor index, Tensor add_value, int axis = 0) @@ -1482,6 +1519,7 @@ kernel : func : isinf {dense -> dense}, isinf_sr {selected_rows -> selected_rows} + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : isnan args : (Tensor x) @@ -1491,6 +1529,7 @@ kernel : func : isnan {dense -> dense}, isnan_sr {selected_rows -> selected_rows} + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : kldiv_loss args : (Tensor x, Tensor label, str reduction = "mean") @@ -1510,6 +1549,7 @@ kernel : func : kron backward : kron_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : kthvalue args : (Tensor x, int k = 1, int axis = -1, bool keepdim = false) @@ -1519,6 +1559,7 @@ kernel : func : kthvalue backward : kthvalue_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : label_smooth args : (Tensor label, Tensor prior_dist, float epsilon = 0.0f) @@ -1587,6 +1628,7 @@ func : lgamma inplace: (x -> out) backward : lgamma_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : linear_interp args : (Tensor x, Tensor out_size, Tensor[] size_tensor, Tensor scale_tensor, str data_format="NCHW", int out_d=0, int out_h=0, int out_w=0, float[] scale={}, str interp_method="bilinear", bool align_corners=true, int align_mode=1) @@ -1641,6 +1683,7 @@ func : log1p inplace: (x -> out) backward: log1p_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : log2 args : (Tensor x) @@ -1679,6 +1722,7 @@ kernel : func : logcumsumexp backward : logcumsumexp_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : logical_and args : (Tensor x, Tensor y) @@ -1714,6 +1758,7 @@ data_type : x backend : x inplace: (x -> out) + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : logical_xor args : (Tensor x, Tensor y) @@ -1725,6 +1770,7 @@ data_type : x backend : x inplace: (x -> out) + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : logit args : (Tensor x, float eps = 1e-6f) @@ -1736,6 +1782,7 @@ func : logit inplace: (x -> out) backward : logit_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : logsigmoid args : (Tensor x) @@ -1807,6 +1854,7 @@ func : masked_select data_type : x backward : masked_select_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : matrix_nms args : (Tensor bboxes, Tensor scores, float score_threshold, int nms_top_k, int keep_top_k, float post_threshold=0., bool use_gaussian = false, float gaussian_sigma = 2., int background_label = 0, bool normalized = true) @@ -2121,6 +2169,7 @@ kernel : func : poisson backward : poisson_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : polygamma args : (Tensor x, int n) @@ -2188,6 +2237,7 @@ data_type : arr inplace : (arr -> out) backward : put_along_axis_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : qr args : (Tensor x, str mode = "reduced") @@ -2206,6 +2256,7 @@ kernel : func : real backward : real_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : reciprocal args : (Tensor x) @@ -2326,6 +2377,7 @@ func : roll data_type : x backward : roll_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : round args : (Tensor x) @@ -2336,6 +2388,7 @@ func : round inplace : (x -> out) backward : round_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : rprop_ args : (Tensor param, Tensor grad, Tensor prev, Tensor learning_rate, Tensor master_param, Tensor learning_rate_range, Tensor etas, bool multi_precision=false) @@ -2387,6 +2440,7 @@ data_type : x inplace : (x -> out) backward : scatter_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : scatter_nd_add args : (Tensor x, Tensor index, Tensor updates) @@ -2397,6 +2451,7 @@ func : scatter_nd_add data_type : x backward : scatter_nd_add_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : searchsorted args : (Tensor sorted_sequence, Tensor values, bool out_int32 = false, bool right = false) @@ -2406,6 +2461,7 @@ kernel : func : searchsorted data_type : sorted_sequence + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : segment_pool args : (Tensor x, Tensor segment_ids, str pooltype="SUM") @@ -2525,6 +2581,7 @@ kernel : func : sign backward : sign_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : silu args : (Tensor x) @@ -2546,6 +2603,7 @@ func : sin inplace: (x -> out) backward : sin_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : sinh args : (Tensor x) @@ -2556,6 +2614,7 @@ func : sinh inplace: (x -> out) backward : sinh_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : slogdet args : (Tensor x) @@ -2720,6 +2779,7 @@ func : take_along_axis data_type : arr backward : take_along_axis_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : tan args : (Tensor x) @@ -2730,6 +2790,7 @@ func : tan inplace: (x -> out) backward : tan_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : tanh args : (Tensor x) @@ -2740,6 +2801,7 @@ func : tanh inplace : (x -> out) backward : tanh_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : tanh_shrink args : (Tensor x) @@ -2802,6 +2864,7 @@ func : topk data_type : x backward : topk_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : trace args : (Tensor x, int offset = 0, int axis1 = 0, int axis2 = 1) @@ -2853,6 +2916,7 @@ kernel : func : unbind backward : unbind_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : unfold args : (Tensor x, int[] kernel_sizes, int[] strides, int[] paddings, int[] dilations) @@ -2883,6 +2947,7 @@ func : unique_consecutive data_type : x optional : index, counts + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : unpool3d args: (Tensor x, Tensor indices, int[] ksize, int[] strides={1,1,1}, int[] paddings={0,0,0}, int[] output_size={0,0,0}, str data_format="NCDHW") From 8d2a1bc5868fbde6178e9c970f6d2453e2f8d090 Mon Sep 17 00:00:00 2001 From: Jianbang Yang Date: Wed, 28 Feb 2024 11:27:32 +0800 Subject: [PATCH 147/282] [XPU] remove unnecessary xpu_wait in TensorSetConstantXPU (#61818) --- paddle/phi/kernels/funcs/math_function.h | 1 - 1 file changed, 1 deletion(-) diff --git a/paddle/phi/kernels/funcs/math_function.h b/paddle/phi/kernels/funcs/math_function.h index 2e1013ec7fc1b..509f3f0d6b9b5 100644 --- a/paddle/phi/kernels/funcs/math_function.h +++ b/paddle/phi/kernels/funcs/math_function.h @@ -153,7 +153,6 @@ struct TensorSetConstantXPU { numel, static_cast(value_)); PADDLE_ENFORCE_XDNN_SUCCESS(r, "constant"); - dev_ctx->Wait(); } else { std::unique_ptr data_cpu(new T[numel]); std::fill(data_cpu.get(), data_cpu.get() + numel, static_cast(value_)); From a4387839a29f42836c4c22406e1d2d974c1ae869 Mon Sep 17 00:00:00 2001 From: Yuanle Liu Date: Wed, 28 Feb 2024 13:17:54 +0800 Subject: [PATCH 148/282] [PIR] Fix ValueIsPersistable (#62152) --- paddle/fluid/pir/transforms/transform_general_functions.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/paddle/fluid/pir/transforms/transform_general_functions.cc b/paddle/fluid/pir/transforms/transform_general_functions.cc index 55a1dc463dc6d..2ef3d6d5b81dc 100644 --- a/paddle/fluid/pir/transforms/transform_general_functions.cc +++ b/paddle/fluid/pir/transforms/transform_general_functions.cc @@ -140,6 +140,9 @@ std::vector GetUsedExternalValue(const pir::Block& block) { } bool ValueIsPersitable(pir::Value value) { + if (!value.defining_op()) { + return false; + } if (value.defining_op()->num_operands() > 0) { for (const auto& source_value : value.defining_op()->operands_source()) { if (!ValueIsPersitable(source_value)) { From 60906438e2464e2d47f67897d515f0ad2da8ba8d Mon Sep 17 00:00:00 2001 From: QingshuChen Date: Wed, 28 Feb 2024 13:41:00 +0800 Subject: [PATCH 149/282] [XPU] fix xpu fused rope embedding (#62143) --- paddle/phi/kernels/fusion/xpu/fused_rope_grad_kernel.cc | 1 + paddle/phi/kernels/fusion/xpu/fused_rope_kernel.cc | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/paddle/phi/kernels/fusion/xpu/fused_rope_grad_kernel.cc b/paddle/phi/kernels/fusion/xpu/fused_rope_grad_kernel.cc index ee76e25539f16..1e988ca9ea03e 100644 --- a/paddle/phi/kernels/fusion/xpu/fused_rope_grad_kernel.cc +++ b/paddle/phi/kernels/fusion/xpu/fused_rope_grad_kernel.cc @@ -28,6 +28,7 @@ void FusedRopeGradKernel(const Context& dev_ctx, const paddle::optional& dout_k, const paddle::optional& dout_v, bool use_neox_rotary_style, + bool time_major, DenseTensor* dq, DenseTensor* dk, DenseTensor* dv) { diff --git a/paddle/phi/kernels/fusion/xpu/fused_rope_kernel.cc b/paddle/phi/kernels/fusion/xpu/fused_rope_kernel.cc index de8cd0b87a302..c8980310fb0f9 100644 --- a/paddle/phi/kernels/fusion/xpu/fused_rope_kernel.cc +++ b/paddle/phi/kernels/fusion/xpu/fused_rope_kernel.cc @@ -29,6 +29,7 @@ void FusedRopeKernel(const Context& dev_ctx, const paddle::optional& cos, const paddle::optional& position_ids, bool use_neox_rotary_style, + bool time_major, DenseTensor* out_q, DenseTensor* out_k, DenseTensor* out_v) { @@ -37,6 +38,10 @@ void FusedRopeKernel(const Context& dev_ctx, return; } + PADDLE_ENFORCE_EQ( + time_major, + false, + phi::errors::InvalidArgument("time_major is not supported in xpu")); int64_t batch_size = q.dims()[0]; int64_t seq_len = q.dims()[1]; int64_t num_heads = q.dims()[2]; @@ -84,7 +89,7 @@ void FusedRopeKernel(const Context& dev_ctx, num_heads, head_dim); - if (k.get_ptr()) { + if (k) { auto* outk_data = reinterpret_cast(dev_ctx.template Alloc(out_k)); XPUFusedRotaryHalf( @@ -99,7 +104,7 @@ void FusedRopeKernel(const Context& dev_ctx, head_dim); } - if (v.get_ptr()) { + if (v) { auto* outv_data = reinterpret_cast(dev_ctx.template Alloc(out_v)); XPUFusedRotaryHalf( From 8d2f57591ac2beec070c2ee9995cb88bd5884766 Mon Sep 17 00:00:00 2001 From: risemeup1 <62429225+risemeup1@users.noreply.github.com> Date: Wed, 28 Feb 2024 13:52:55 +0800 Subject: [PATCH 150/282] fix inference-proxy bug (#62142) * fix inference-proxy bug * fix --- test/cpp/inference/infer_ut/external-cmake/gtest-cpp.cmake | 7 +++---- test/cpp/inference/infer_ut/run.sh | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/cpp/inference/infer_ut/external-cmake/gtest-cpp.cmake b/test/cpp/inference/infer_ut/external-cmake/gtest-cpp.cmake index b141b76c6f33b..5a70355ef535c 100644 --- a/test/cpp/inference/infer_ut/external-cmake/gtest-cpp.cmake +++ b/test/cpp/inference/infer_ut/external-cmake/gtest-cpp.cmake @@ -1,12 +1,12 @@ find_package(Git REQUIRED) message("${CMAKE_BUILD_TYPE}") set(GTEST_PREFIX_DIR ${CMAKE_CURRENT_BINARY_DIR}/gtest) -set(GTEST_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/gtest/src/extern_gtest) +set(PADDLE_SOURCE_DIR $ENV{PADDLE_SOURCE_DIR}) +set(GTEST_SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/gtest) set(GTEST_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/install/gtest) set(GTEST_INCLUDE_DIR "${GTEST_INSTALL_DIR}/include" CACHE PATH "gtest include directory." FORCE) -set(GTEST_REPOSITORY https://github.com/google/googletest.git) set(GTEST_TAG release-1.8.1) include_directories(${GTEST_INCLUDE_DIR}) if(WIN32) @@ -30,8 +30,7 @@ endif() ExternalProject_Add( extern_gtest PREFIX gtest - GIT_REPOSITORY ${GTEST_REPOSITORY} - GIT_TAG ${GTEST_TAG} + SOURCE_DIR ${GTEST_SOURCE_DIR} DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GTEST_INSTALL_DIR} diff --git a/test/cpp/inference/infer_ut/run.sh b/test/cpp/inference/infer_ut/run.sh index 6f967eb0aa6c7..f55f7811606aa 100755 --- a/test/cpp/inference/infer_ut/run.sh +++ b/test/cpp/inference/infer_ut/run.sh @@ -16,6 +16,7 @@ set -x PADDLE_ROOT=$1 +export PADDLE_SOURCE_DIR=$1 TURN_ON_MKL=$2 # use MKL or Openblas TEST_GPU_CPU=$3 # test both GPU/CPU mode or only CPU mode DATA_DIR=$4 # dataset From 0651cba270349786e1dfff00e061cac6cca43ef7 Mon Sep 17 00:00:00 2001 From: diadestiny <44188454+diadestiny@users.noreply.github.com> Date: Wed, 28 Feb 2024 14:24:51 +0800 Subject: [PATCH 151/282] [SOT][3.12] Support `POP_JUMP_IF_NONE` and `POP_JUMP_IF_NOT_NONE` opcode in Python 3.12 (#62120) --- .../jit/sot/opcode_translator/executor/opcode_executor.py | 2 ++ test/sot/skip_files_py312 | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py index d5635c94d159c..7d58a78a9322d 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py +++ b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py @@ -1396,11 +1396,13 @@ def JUMP_IF_TRUE_OR_POP(self, instr: Instruction): POP_JUMP_FORWARD_IF_NONE = pop_jump_if_op_wrapper([operator_is_none]) POP_JUMP_BACKWARD_IF_NONE = POP_JUMP_FORWARD_IF_NONE + POP_JUMP_IF_NONE = POP_JUMP_FORWARD_IF_NONE POP_JUMP_FORWARD_IF_NOT_NONE = pop_jump_if_op_wrapper( [operator_is_not_none] ) POP_JUMP_BACKWARD_IF_NOT_NONE = POP_JUMP_FORWARD_IF_NOT_NONE + POP_JUMP_IF_NOT_NONE = POP_JUMP_FORWARD_IF_NOT_NONE @call_break_graph_decorator(push_n=lambda arg: arg) def UNPACK_SEQUENCE(self, instr: Instruction): diff --git a/test/sot/skip_files_py312 b/test/sot/skip_files_py312 index ce4dbbdf80d9c..796fdb62e5001 100644 --- a/test/sot/skip_files_py312 +++ b/test/sot/skip_files_py312 @@ -1,9 +1,7 @@ ./test_11_jumps.py ./test_12_for_loop.py ./test_21_global.py -./test_break_graph.py ./test_builtin_zip.py -./test_guard_user_defined_fn.py ./test_inplace_api.py ./test_min_graph_size.py ./test_side_effects.py From a8691f87b4f6bbfccaa55c591cfb8a739ca81ff7 Mon Sep 17 00:00:00 2001 From: Tian <121000916+SylarTiaNII@users.noreply.github.com> Date: Wed, 28 Feb 2024 14:30:06 +0800 Subject: [PATCH 152/282] [CustomDevice] adapt tensor fusion helper to npu device (#61942) --- .../fleet/utils/tensor_fusion_helper.py | 38 ++++++++++++++++--- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/python/paddle/distributed/fleet/utils/tensor_fusion_helper.py b/python/paddle/distributed/fleet/utils/tensor_fusion_helper.py index dff62c1a22db1..0ea2d12b292a9 100644 --- a/python/paddle/distributed/fleet/utils/tensor_fusion_helper.py +++ b/python/paddle/distributed/fleet/utils/tensor_fusion_helper.py @@ -20,6 +20,7 @@ import paddle from paddle.framework import ( + _current_expected_place_, base as imperative_base, core, ) @@ -33,6 +34,7 @@ class HOOK_ACTION: alignment = { "gpu": 256, + "npu": 256, } align = { @@ -42,6 +44,28 @@ class HOOK_ACTION: } +__current_device_type__ = None + + +def get_current_device_type(): + global __current_device_type__ + if __current_device_type__ is None: + if paddle.is_compiled_with_cuda(): + device_type = "gpu" + elif paddle.is_compiled_with_xpu(): + device_type = "xpu" + elif paddle.is_compiled_with_custom_device(): + current_device = _current_expected_place_() + device_type = current_device.get_device_type() + else: + device_type = "unknown" + assert ( + device_type in alignment.keys() + ), f"tensor fusion helper now only support {alignment.keys()}, but got device {device_type} instead." + __current_device_type__ = device_type + return __current_device_type__ + + def assign_group_by_size(parameters, group_size=128 * 1024 * 1024): is_sparse_gradient = [False] * len(parameters) @@ -76,8 +100,12 @@ def flatten_dense_tensors( for param in parameters: assert param.trainable, "param must be trainable..." size = np.prod(param.shape) * align[dtype] - remaining = size % alignment["gpu"] - ali = 0 if remaining == 0 else alignment["gpu"] - remaining + remaining = size % alignment[get_current_device_type()] + ali = ( + 0 + if remaining == 0 + else alignment[get_current_device_type()] - remaining + ) align_ = ali // align[dtype] _param2offset[param.name] = _buffer_size _buffer_size += np.prod(param.shape) + align_ @@ -88,7 +116,7 @@ def flatten_dense_tensors( if fuse_param: param_storage = ParamStorage( - size=_buffer_size, dtype=dtype, device="gpu" + size=_buffer_size, dtype=dtype, device=get_current_device_type() ) param_storage.add_rank_params(parameters, _param2align) @@ -97,7 +125,7 @@ def flatten_dense_tensors( grad_storage = GradStorage( size=_buffer_size, dtype=grad_dtype, - device="gpu", + device=get_current_device_type(), destination="0", parm2align=_param2align, ) @@ -261,7 +289,7 @@ def build_reduce_scatter_buffer( def get_padded_size(param): size = np.prod(param.shape) - align_size = alignment["gpu"] // align[dtype] + align_size = alignment[get_current_device_type()] // align[dtype] align_size = align_size * sharding_degree padded_size = ((size + align_size - 1) // align_size) * align_size return padded_size From 3dc84d6cab580c7dc97ee3c394aea80de2f155ea Mon Sep 17 00:00:00 2001 From: zhengzhonghui Date: Wed, 28 Feb 2024 15:07:50 +0800 Subject: [PATCH 153/282] [AutoParallel] add global_input unittest (#62115) * [AutoParallel] add global_input unittest * polish * compare with single card result --- .../hybrid_strategy/CMakeLists.txt | 8 + .../semi_auto_parallel_global_input.py | 222 ++++++++++++++++++ .../test_semi_auto_parallel_global_input.py | 57 +++++ .../hybrid_strategy/testslist.csv | 1 + 4 files changed, 288 insertions(+) create mode 100644 test/auto_parallel/hybrid_strategy/semi_auto_parallel_global_input.py create mode 100644 test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_global_input.py diff --git a/test/auto_parallel/hybrid_strategy/CMakeLists.txt b/test/auto_parallel/hybrid_strategy/CMakeLists.txt index 9d19c4e08b64d..08a9f42c02a1f 100644 --- a/test/auto_parallel/hybrid_strategy/CMakeLists.txt +++ b/test/auto_parallel/hybrid_strategy/CMakeLists.txt @@ -65,3 +65,11 @@ if((WITH_GPU) AND (LINUX)) set_tests_properties(test_global_mesh_reshard PROPERTIES TIMEOUT "120" LABELS "RUN_TYPE=HYBRID") endif() +if((WITH_GPU) AND (LINUX)) + py_test_modules( + test_semi_auto_parallel_global_input MODULES + test_semi_auto_parallel_global_input ENVS + "http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python") + set_tests_properties(test_semi_auto_parallel_global_input + PROPERTIES TIMEOUT "120" LABELS "RUN_TYPE=HYBRID") +endif() diff --git a/test/auto_parallel/hybrid_strategy/semi_auto_parallel_global_input.py b/test/auto_parallel/hybrid_strategy/semi_auto_parallel_global_input.py new file mode 100644 index 0000000000000..768b78163fedc --- /dev/null +++ b/test/auto_parallel/hybrid_strategy/semi_auto_parallel_global_input.py @@ -0,0 +1,222 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import numpy as np + +import paddle +import paddle.distributed as dist +from paddle.io import BatchSampler, DataLoader, Dataset + +SEQ_LEN = 4 +HIDDLE_SIZE = 8 +global_mesh = dist.ProcessMesh( + [[[0, 1], [2, 3]], [[4, 5], [6, 7]]], dim_names=['pp', 'dp', 'mp'] +) +mesh0 = dist.ProcessMesh([[0, 1], [2, 3]], dim_names=['dp', 'mp']) +mesh1 = dist.ProcessMesh([[4, 5], [6, 7]], dim_names=['dp', 'mp']) + + +class MlpModel(paddle.nn.Layer): + def __init__(self, variable_initial_values, run_single_process=False): + super().__init__() + self.w0 = self.create_parameter( + shape=[HIDDLE_SIZE, HIDDLE_SIZE], + default_initializer=paddle.nn.initializer.Assign( + variable_initial_values[0] + ), + ) + self.w1 = self.create_parameter( + shape=[HIDDLE_SIZE, HIDDLE_SIZE], + default_initializer=paddle.nn.initializer.Assign( + variable_initial_values[1] + ), + ) + self.global_input = paddle.uniform( + shape=[SEQ_LEN, HIDDLE_SIZE], + dtype=paddle.float32, + min=-0.0001, + max=0.0001, + ) + if run_single_process is False: + self.w0 = dist.shard_tensor( + self.w0, + mesh0, + [dist.Replicate(), dist.Shard(1)], + ) + self.w1 = dist.shard_tensor( + self.w1, + mesh1, + [dist.Replicate(), dist.Shard(0)], + ) + self.global_input = dist.shard_tensor( + self.global_input, + global_mesh, + [dist.Replicate(), dist.Replicate(), dist.Replicate()], + ) + self.run_single_process = run_single_process + + def process_global_input(self, input): + return input + 0.0001 + + def forward(self, x): + # x: [bs, seq_len, hidden] + # forward on mesh0 + global_input = self.process_global_input(self.global_input) + if self.run_single_process is False: + global_input1 = dist.reshard( + global_input, mesh0, [dist.Replicate(), dist.Replicate()] + ) + else: + global_input1 = global_input + x = x + global_input1 + y = paddle.matmul(x, self.w0) + # forward on mesh1 + if self.run_single_process is False: + y = dist.reshard(y, mesh1, [dist.Shard(0), dist.Shard(2)]) + global_input2 = dist.reshard( + global_input, mesh1, [dist.Replicate(), dist.Replicate()] + ) + else: + global_input2 = global_input + + y = y + global_input2 + z = paddle.matmul(y, self.w1) + return z + + +class RandomDataset(Dataset): + def __init__(self, seq_len, hidden, num_samples=8): + super().__init__() + self.seq_len = seq_len + self.hidden = hidden + self.num_samples = num_samples + self.inputs = [ + np.random.uniform(size=[self.seq_len, self.hidden]).astype( + "float32" + ) + for _ in range(num_samples) + ] + self.labels = [ + np.array(index, dtype="float32") for index in range(num_samples) + ] + + def __getitem__(self, index): + return self.inputs[index], self.labels[index] + + def __len__(self): + return self.num_samples + + +def create_dataloader(): + dataset = RandomDataset(SEQ_LEN, HIDDLE_SIZE) + sampler = BatchSampler( + dataset, + batch_size=2, + ) + dataloader = DataLoader( + dataset, + batch_sampler=sampler, + ) + return dataloader + + +def get_variable_initial_value(var_num=2): + res = [] + for i in range(var_num): + res.append( + paddle.uniform( + shape=[HIDDLE_SIZE, HIDDLE_SIZE], + dtype=paddle.float32, + min=-0.0001, + max=0.0001, + ) + ) + return res + + +def loss_fn(logits, label): + # logits: [bs, seq_len, hidden], label: [bs] + loss = paddle.nn.MSELoss(reduction="sum") + logits = paddle.sum(logits, axis=[1, 2]) + return loss(logits, label) + + +class TestSemiAutoParallelGlobalInput: + def __init__(self): + self._backend = os.getenv("backend") + self._seed = eval(os.getenv("seed")) + self._run_static = eval(os.getenv("run_static")) + paddle.seed(self._seed) + np.random.seed(self._seed) + paddle.set_device(self._backend) + self.dataloader = create_dataloader() + self.variable_initial_values = get_variable_initial_value() + self.single_process_loss = self.get_single_process_loss() + + def get_single_process_loss(self): + model = MlpModel( + variable_initial_values=self.variable_initial_values, + run_single_process=True, + ) + opt = paddle.optimizer.AdamW( + learning_rate=0.001, parameters=model.parameters() + ) + for step, (input, label) in enumerate(self.dataloader()): + logits = model(input) + loss = loss_fn(logits, label) + loss.backward() + opt.step() + opt.clear_grad() + return loss.numpy() + + def test_basic(self): + model = MlpModel(variable_initial_values=self.variable_initial_values) + opt = paddle.optimizer.AdamW( + learning_rate=0.001, parameters=model.parameters() + ) + dist_dataloader = dist.shard_dataloader( + dataloader=self.dataloader, meshes=[mesh0, mesh1], shard_dims="dp" + ) + cur_rank = paddle.distributed.get_rank() + if self._run_static: + dist_model = dist.to_static(model, dist_dataloader, loss_fn, opt) + + for step, (input, label) in enumerate(dist_dataloader()): + loss = dist_model(input, label) + + if cur_rank in [5, 7]: + loss = paddle.to_tensor(loss) + group = paddle.distributed.new_group([5, 7]) + dist.all_reduce(loss, group=group) + else: + dist_opt = dist.shard_optimizer(opt) + for step, (input, label) in enumerate(dist_dataloader()): + logits = model(input) + loss = loss_fn(logits, label) + loss.backward() + dist_opt.step() + dist_opt.clear_grad() + if cur_rank in [5, 7]: + np.testing.assert_allclose( + loss.numpy(), self.single_process_loss, rtol=1e-06, verbose=True + ) + + def run_test_case(self): + self.test_basic() + + +if __name__ == '__main__': + TestSemiAutoParallelGlobalInput().run_test_case() diff --git a/test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_global_input.py b/test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_global_input.py new file mode 100644 index 0000000000000..6f1a303dc513d --- /dev/null +++ b/test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_global_input.py @@ -0,0 +1,57 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import collective.test_communication_api_base as test_base + + +class TestSemiAutoParallelGlobalInput(test_base.CommunicationTestDistBase): + def setUp(self): + super().setUp( + num_of_devices=8, + timeout=120, + nnode=1, + ) + self._default_envs = { + "dtype": "float32", + "seed": "1024", + } + self._changeable_envs = {"backend": ["gpu"]} + + def test_dynamic(self): + self._default_envs.update({"run_static": "0"}) + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) + for envs in envs_list: + self.run_test_case( + "semi_auto_parallel_global_input.py", + user_defined_envs=envs, + ) + + def test_static(self): + self._default_envs.update({"run_static": "1"}) + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) + for envs in envs_list: + self.run_test_case( + "semi_auto_parallel_global_input.py", + user_defined_envs=envs, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/auto_parallel/hybrid_strategy/testslist.csv b/test/auto_parallel/hybrid_strategy/testslist.csv index 9c1105615890b..5791b71d0d5ff 100644 --- a/test/auto_parallel/hybrid_strategy/testslist.csv +++ b/test/auto_parallel/hybrid_strategy/testslist.csv @@ -7,3 +7,4 @@ test_cross_mesh_reshard,LINUX,GPU,120,HYBRID,test_runner.py,,,http_proxy=;https_ test_semi_auto_parallel_llama_model_amp,LINUX,GPU,180,HYBRID,test_runner.py,,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_semi_auto_parallel_hybrid_sharding_strategy,LINUX,GPU,120,HYBRID,test_runner.py,,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_global_mesh_reshard,LINUX,GPU,120,HYBRID,test_runner.py,,,http_proxy=;https_proxy=;PYTHONPATH=../.., +test_semi_auto_parallel_global_input,LINUX,GPU,120,HYBRID,test_runner.py,,,http_proxy=;https_proxy=;PYTHONPATH=../.., From 380c37ad6b6c4bead924f3ddd6ed75988747f643 Mon Sep 17 00:00:00 2001 From: risemeup1 <62429225+risemeup1@users.noreply.github.com> Date: Wed, 28 Feb 2024 16:22:55 +0800 Subject: [PATCH 154/282] fix mac-m1-arm bug (#62144) --- python/CMakeLists.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index b42b1e65c552a..fcd93656b30b3 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -190,9 +190,8 @@ endif() add_custom_target(paddle_python ALL DEPENDS ${PADDLE_PYTHON_BUILD_DIR}/.timestamp) if(BUILD_WHL_PACKAGE AND NOT WITH_SETUP_INSTALL) - add_custom_target( - paddle_copy ALL DEPENDS paddle_python - ${PADDLE_PYTHON_BUILD_DIR}/.timestamp_wheel) + add_custom_target(paddle_copy ALL + DEPENDS ${PADDLE_PYTHON_BUILD_DIR}/.timestamp_wheel) endif() set(PADDLE_PYTHON_PACKAGE_DIR ${CMAKE_CURRENT_BINARY_DIR}/dist/) From d9aaf16dee5f024a3d2ce91d8465f2b2d7fbb1d2 Mon Sep 17 00:00:00 2001 From: HongyuJia Date: Wed, 28 Feb 2024 17:31:18 +0800 Subject: [PATCH 155/282] [Dynamic Shape] Convert0DTo1DPass supports more case (#62027) * [Dynamic Shape] Convert0DTo1DPass supports more case * Pass while unittest * Adjust LOG priority * Fix dtype * Change function name * Polish codes --- .../operator/transforms/add_cinn_pass.cc | 2 + .../group_merge/convert_0d_to_1d_pass.cc | 163 ++++++++++++++++-- paddle/cinn/hlir/op/broadcast.cc | 3 + 3 files changed, 151 insertions(+), 17 deletions(-) diff --git a/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.cc index 6ded2f5a85c93..496370ee7bfcd 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.cc @@ -89,11 +89,13 @@ void ApplyCinnPreprocessPass( pass_manager->AddPass(pir::CreateDeadCodeEliminationPass()); if (has_dynamic_shape) { + pass_manager->AddPass(cinn::dialect::ir::CreateConvert0DTo1DPass()); pass_manager->AddPass(pir::CreateShapeOptimizationPass()); pass_manager->AddPass(cinn::dialect::ir::CreateSimplifyDimExprPass()); pass_manager->AddPass( cinn::dialect::ir::CreateSubstituteDimExprBasedOnConstraintsPass()); pass_manager->AddPass(cinn::dialect::ir::CreateInsertBroadcastPass()); + pass_manager->AddPass(cinn::dialect::ir::CreateConvert0DTo1DPass()); pass_manager->AddPass(pir::CreateShapeOptimizationPass()); pass_manager->AddPass( cinn::dialect::ir::CreateFuseShapeOpsIntoGenerateShapeOpPass()); diff --git a/paddle/cinn/hlir/dialect/operator/transforms/group_merge/convert_0d_to_1d_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/group_merge/convert_0d_to_1d_pass.cc index 325421d92abe6..549cdf8ae7b07 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/group_merge/convert_0d_to_1d_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/group_merge/convert_0d_to_1d_pass.cc @@ -19,9 +19,11 @@ #include "paddle/cinn/hlir/dialect/operator/ir/op_dialect.h" #include "paddle/cinn/hlir/dialect/runtime/ir/runtime_dialect.h" #include "paddle/fluid/pir/dialect/kernel/ir/kernel_dialect.h" +#include "paddle/fluid/pir/dialect/operator/ir/control_flow_op.h" #include "paddle/fluid/pir/dialect/operator/ir/op_attribute.h" #include "paddle/fluid/pir/dialect/operator/ir/pd_op.h" #include "paddle/pir/include/core/builtin_type.h" +#include "paddle/pir/include/pattern_rewrite/pattern_rewrite_driver.h" namespace cinn { namespace dialect { @@ -41,7 +43,7 @@ class FullOpPattern : public pir::OpRewritePattern { } void Rewrite(paddle::dialect::FullOp op, - pir::PatternRewriter &rewriter) const override { + pir::PatternRewriter& rewriter) const override { float factor = op->attribute("value").dyn_cast<::pir::FloatAttribute>().data(); phi::DataType dtype = op->attribute("dtype") @@ -58,20 +60,110 @@ class FullOpPattern : public pir::OpRewritePattern { } }; +class SumOpPattern : public pir::OpRewritePattern { + public: + using pir::OpRewritePattern::OpRewritePattern; + + bool Match(paddle::dialect::SumOp op) const override { + const auto& tensor_type = + op.result(0).type().dyn_cast(); + return tensor_type.dims().size() == 0; + } + + void Rewrite(paddle::dialect::SumOp op, + pir::PatternRewriter& rewriter) const override { + std::vector axis{}; + const auto& dtype = op->attribute("dtype") + .dyn_cast() + .data(); + auto new_reduce_op = rewriter.Build( + op.operand_source(0), axis, dtype, /*keepdim=*/true); + auto reshape_op = rewriter.Build( + new_reduce_op.result(0), /*shape=*/std::vector({1})); + rewriter.ReplaceAllUsesWith(op.result(0), reshape_op.result(0)); + rewriter.EraseOp(op); + } +}; + +pir::DenseTensorType Make1DTensorType(const pir::DenseTensorType& tensor_type) { + return pir::DenseTensorType::get(pir::IrContext::Instance(), + tensor_type.dtype(), + {1}, + tensor_type.data_layout(), + tensor_type.lod(), + tensor_type.offset()); +} + +void ConvertValue0DTo1D(pir::Value operand) { + auto ConvertVectorType0DTo1D = + [](const pir::VectorType& vector_tensor_type) -> std::vector { + std::vector types; + for (std::size_t i = 0; i < vector_tensor_type.size(); ++i) { + CHECK(vector_tensor_type[i].isa()); + const auto& dense_type = + vector_tensor_type[i].dyn_cast(); + types.push_back(dense_type.dims().size() == 0 + ? Make1DTensorType(dense_type) + : vector_tensor_type[i]); + } + return types; + }; + + if (const auto& tensor_type = + operand.type().dyn_cast()) { + if (tensor_type.dims().size() == 0) { + operand.set_type(Make1DTensorType(tensor_type)); + } + } else if (const auto& vector_tensor_type = + operand.type().dyn_cast()) { + pir::Builder builder(pir::IrContext::Instance()); + std::vector inputs_type = + ConvertVectorType0DTo1D(vector_tensor_type); + operand.set_type(builder.vec_type(inputs_type)); + } else { + VLOG(4) << "Unsupported operand type: " << operand.type(); + } +} + +class WhileOpPattern : public pir::OpRewritePattern { + public: + using pir::OpRewritePattern::OpRewritePattern; + + bool Match(paddle::dialect::WhileOp op) const override { + for (const auto& value : op.block_args()) { + if (const auto& tensor_type = + value.type().template dyn_cast()) { + if (tensor_type.dims().size() == 0) { + return true; + } + } + } + return false; + } + + void Rewrite(paddle::dialect::WhileOp op, + pir::PatternRewriter& rewriter) const override { + for (pir::Value value : op.block_args()) { + ConvertValue0DTo1D(value); + } + } +}; + class CombineOpPattern : public pir::OpRewritePattern { public: using pir::OpRewritePattern::OpRewritePattern; bool Match(pir::CombineOp op) const override { - auto out_type = op.result(0).type().dyn_cast(); - for (auto type : out_type.data()) { - if (HasZeroDim(type)) return true; + for (std::size_t i = 1; i < op->operands().size(); ++i) { + if (op.operand_source(i).type() != op.operand_source(0).type()) { + return true; + } } return false; } void Rewrite(pir::CombineOp op, - pir::PatternRewriter &rewriter) const override { + pir::PatternRewriter& rewriter) const override { pir::Builder builder(rewriter.ir_context()); const std::vector inputs_type = [&]() { @@ -83,30 +175,67 @@ class CombineOpPattern : public pir::OpRewritePattern { }(); op.result(0).set_type(builder.vec_type(inputs_type)); } - - private: - bool HasZeroDim(pir::Type type) const { - if (!type) return false; - const auto dense_tensor_type = type.dyn_cast(); - return dense_tensor_type && (dense_tensor_type.dims().size() == 0U); - } }; -class Convert0DTo1DPass : public pir::PatternRewritePass { +class Convert0DTo1DPass : public pir::Pass { public: - Convert0DTo1DPass() : pir::PatternRewritePass("convert_0D_to_1D", 1) {} + Convert0DTo1DPass() : pir::Pass("convert_0D_to_1D", 1) {} - pir::RewritePatternSet InitializePatterns(pir::IrContext *context) override { + bool Initialize(pir::IrContext* context) override { pir::RewritePatternSet ps(context); ps.Add(context); ps.Add(context); + ps.Add(context); + ps.Add(context); + patterns_ = pir::FrozenRewritePatternSet(std::move(ps)); + return true; + } + + void Run(pir::Operation* op) override { + for (uint32_t i = 0; i < op->num_regions(); ++i) { + ApplyPatternOnOperation(op->region(i)); + for (const auto& block : op->region(i)) { + ConvertBlock0DTo1D(block); + } + } + } - return ps; + void ApplyPatternOnOperation(pir::Region& region) { // NOLINT + pir::GreedyRewriteConfig cfg; + cfg.use_top_down_traversal = true; + cfg.max_iterations = 10; + const auto& [_, num_rewrites] = + pir::ApplyPatternsGreedily(region, patterns_, cfg); + AddStatistics(num_rewrites); } - bool CanApplyOn(pir::Operation *op) const override { + bool CanApplyOn(pir::Operation* op) const override { return op->isa() && op->num_regions() > 0; } + + void ConvertOperation0DTo1D(const pir::Operation& op) { // NOLINT + for (std::size_t i = 0; i < op.num_operands(); ++i) { + ConvertValue0DTo1D(op.operand_source(i)); + } + for (std::size_t i = 0; i < op.num_results(); ++i) { + ConvertValue0DTo1D(op.result(i)); + } + } + + void ConvertBlock0DTo1D(const pir::Block& block) { + for (auto& op : block) { + ConvertOperation0DTo1D(op); + for (std::size_t i = 0; i < op.num_regions(); ++i) { + ApplyPatternOnOperation(op.region(i)); + for (auto& inner_block : op.region(i)) { + ConvertBlock0DTo1D(inner_block); + } + } + } + } + + private: + pir::FrozenRewritePatternSet patterns_; }; } // namespace diff --git a/paddle/cinn/hlir/op/broadcast.cc b/paddle/cinn/hlir/op/broadcast.cc index bf71267b2c618..d6df20f1a60eb 100644 --- a/paddle/cinn/hlir/op/broadcast.cc +++ b/paddle/cinn/hlir/op/broadcast.cc @@ -574,6 +574,9 @@ CINN_REGISTER_HELPER(broadcast_ops) { .set_num_outputs(1) \ .set_attr( \ "CINNStrategy", cinn::hlir::op::StrategyFor##op_stragegy__) \ + .set_attr( \ + "CINNStrategySymbolic", \ + cinn::hlir::op::StrategyFor##op_stragegy__##Symbolic) \ .set_attr("infershape", \ MakeOpFunction(cinn::hlir::op::InferShapeForBroadcast)) \ .set_attr("inferdtype", \ From b0ae0c2bc81f2199830572e5b364af34bddb2d53 Mon Sep 17 00:00:00 2001 From: xiaoguoguo626807 <100397923+xiaoguoguo626807@users.noreply.github.com> Date: Wed, 28 Feb 2024 18:15:50 +0800 Subject: [PATCH 156/282] =?UTF-8?q?=E3=80=90pir=E3=80=91modify=20Paddle=20?= =?UTF-8?q?detection=20bug=20(#62165)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * modify if nest pop_to_push_map * modify paddledectation * modify utf-8 bug --- .../pir/dialect/operator/ir/manual_op.cc | 6 +-- python/paddle/autograd/backward_utils.py | 20 +++++++-- python/paddle/autograd/ir_backward.py | 44 +++++++++++-------- 3 files changed, 44 insertions(+), 26 deletions(-) diff --git a/paddle/fluid/pir/dialect/operator/ir/manual_op.cc b/paddle/fluid/pir/dialect/operator/ir/manual_op.cc index 1f645b0a29d66..0863737842ba2 100644 --- a/paddle/fluid/pir/dialect/operator/ir/manual_op.cc +++ b/paddle/fluid/pir/dialect/operator/ir/manual_op.cc @@ -196,7 +196,7 @@ std::vector AddNOp::InferMeta( paddle::dialect::IrTensor dense_out; paddle::dialect::IrMetaTensor meta_out(&dense_out); - phi::AddNInferMeta(meta_x, &meta_out); + phi::AddNInferMeta(meta_x, &meta_out, phi::MetaConfig(false, false)); std::vector argument_outputs; pir::Type out_dense_tensor_type = paddle::dialect::DenseTensorType::get( @@ -358,7 +358,7 @@ std::vector AddN_Op::InferMeta( paddle::dialect::IrTensor dense_out; paddle::dialect::IrMetaTensor meta_out(&dense_out); - phi::AddNInferMeta(meta_inputs, &meta_out); + phi::AddNInferMeta(meta_inputs, &meta_out, phi::MetaConfig(false, false)); std::vector argument_outputs; pir::Type out_dense_tensor_type = paddle::dialect::DenseTensorType::get( @@ -548,7 +548,7 @@ std::vector AddNWithKernelOp::InferMeta( paddle::dialect::IrTensor dense_out; paddle::dialect::IrMetaTensor meta_out(&dense_out); - phi::AddNInferMeta(meta_inputs, &meta_out); + phi::AddNInferMeta(meta_inputs, &meta_out, phi::MetaConfig(false, false)); std::vector argument_outputs; pir::Type out_dense_tensor_type = paddle::dialect::DenseTensorType::get( diff --git a/python/paddle/autograd/backward_utils.py b/python/paddle/autograd/backward_utils.py index f0d90d08426d3..1627c565be01a 100644 --- a/python/paddle/autograd/backward_utils.py +++ b/python/paddle/autograd/backward_utils.py @@ -419,17 +419,22 @@ def remove_useless_full_like_ops(block, ops, state): remove ops which are not in use recursively, ''' + remove_ops = [] + inverse_ops = inverse_sort_op(list(ops)) # from output to input - for op in inverse_sort_op(list(ops)): - if op.name() == 'pd_op.full_like': + for op in inverse_ops: + if op.name() == "pd_op.full_like": if op.result(0).use_empty(): full_op = op.operand_source(1).get_defining_op() - remove_op(block, op, state) - remove_op(block, full_op, state) + remove_ops.append(op) + remove_ops.append(full_op) elif is_control_flow(op): for sub_block in op.blocks(): remove_useless_full_like_ops(sub_block, sub_block.ops, state) + for op in remove_ops: + remove_op(block, op, state) + def all_stop_gradient_true(block): for op in block.ops: @@ -518,3 +523,10 @@ def get_grad_semantic_info(op): else: grad_semantic_info = op.get_input_grad_semantics() return grad_semantic_info + + +def get_split_op(value): + for op in value.all_used_ops(): + if op.name() == "builtin.split": + return op + return None diff --git a/python/paddle/autograd/ir_backward.py b/python/paddle/autograd/ir_backward.py index 18f5054921ab7..a023a4c659e82 100644 --- a/python/paddle/autograd/ir_backward.py +++ b/python/paddle/autograd/ir_backward.py @@ -29,6 +29,7 @@ dynamic_shape_prim_vjp_guard, get_grad_semantic_info, get_real_op_inputs, + get_split_op, inverse_sort_op, is_control_flow, is_inplace_net, @@ -90,24 +91,30 @@ def append_add_n( # need add sum op to accumulate gradient add_n_list = [] for item in state.value_to_valuegrad[value]: - add_n_list.append( - return_map_value(item[0], bwd_value_to_block_argument_map) - ) + if item[0] is not None: + add_n_list.append( + return_map_value(item[0], bwd_value_to_block_argument_map) + ) - if value.is_dense_tensor_array_type(): - add_n_value = paddle._pir_ops.add_n_array(add_n_list) + if len(add_n_list) == 0: + for tmp in state.value_to_valuegrad[value]: + state.value_to_sumvaluegrad[value].append(tmp) + state.value_to_valuegrad[value] = [] else: - add_n_value = paddle.add_n(add_n_list) + if value.is_dense_tensor_array_type(): + add_n_value = paddle._pir_ops.add_n_array(add_n_list) + else: + add_n_value = paddle.add_n(add_n_list) - add_n_op = add_n_value.get_defining_op() - combine_op = add_n_op.operand_source(0).get_defining_op() - update_bwdop_structure( - backward_ops, state.op_to_opgrad[op], [combine_op, add_n_op] - ) + add_n_op = add_n_value.get_defining_op() + combine_op = add_n_op.operand_source(0).get_defining_op() + update_bwdop_structure( + backward_ops, state.op_to_opgrad[op], [combine_op, add_n_op] + ) - for tmp in state.value_to_valuegrad[value]: - state.value_to_sumvaluegrad[value].append(tmp) - state.value_to_valuegrad[value] = [[add_n_value]] + for tmp in state.value_to_valuegrad[value]: + state.value_to_sumvaluegrad[value].append(tmp) + state.value_to_valuegrad[value] = [[add_n_value]] def update_bwdop_structure(backward_ops, op_to_opgrad_list, grad_op_list): @@ -342,10 +349,7 @@ def make_output_with_output_grad(op): value not in state.value_to_valuegrad or state.value_to_valuegrad[value] == [] ): - if ( - not value.use_empty() - and value.first_use().owner().name() == "builtin.split" - ): + if not value.use_empty() and get_split_op(value) is not None: # pattern case: # this fwd_op's output is vectorType, it will split to # Type by builtin_split op, so need get from split op's outputs. @@ -353,7 +357,7 @@ def make_output_with_output_grad(op): split_zero_flag, split_outputs, split_output_grad, - ) = make_output_with_output_grad(value.first_use().owner()) + ) = make_output_with_output_grad(get_split_op(value)) zero_flag[i] = all(split_zero_flag) grad_values = [value[0] for value in split_output_grad] state.value_to_valuegrad[value] = [grad_values] @@ -374,6 +378,8 @@ def make_output_with_output_grad(op): outputs.append(new_value) grad_value = state.value_to_valuegrad[value][0] + if grad_value[0] is None: + zero_flag[i] = True output_grads.append( return_map_value_list( grad_value, bwd_value_to_block_argument_map From 1b38a067d2ea851c8e84b0c129941f54a02c073e Mon Sep 17 00:00:00 2001 From: Lu Qi <61354321+MarioLulab@users.noreply.github.com> Date: Wed, 28 Feb 2024 19:17:05 +0800 Subject: [PATCH 157/282] Fix fused_rope dist op by adding time_major attr (#62180) * fix * fix --- paddle/phi/infermeta/spmd_rules/fused_rope.h | 12 ++++++------ .../static/operators/dist_fused_rope.py | 3 +++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/paddle/phi/infermeta/spmd_rules/fused_rope.h b/paddle/phi/infermeta/spmd_rules/fused_rope.h index fdd9ae27500b0..3a5c331098ad1 100644 --- a/paddle/phi/infermeta/spmd_rules/fused_rope.h +++ b/paddle/phi/infermeta/spmd_rules/fused_rope.h @@ -29,8 +29,8 @@ SpmdInfo FusedRopeInferSpmd(const DistMetaTensor& q, const DistMetaTensor& sin, const DistMetaTensor& cos, const DistMetaTensor& position_ids, - bool use_neox_rotary_style, - bool time_major); + bool use_neox_rotary_style = true, + bool time_major = false); SpmdInfo FusedRopeInferSpmdReverse(const DistMetaTensor& q, const DistMetaTensor& k, @@ -41,8 +41,8 @@ SpmdInfo FusedRopeInferSpmdReverse(const DistMetaTensor& q, const DistMetaTensor& out_q, const DistMetaTensor& out_k, const DistMetaTensor& out_v, - bool use_neox_rotary_style, - bool time_major); + bool use_neox_rotary_style = true, + bool time_major = false); SpmdInfo FusedRopeGradInferSpmd(const DistMetaTensor& sin, const DistMetaTensor& cos, @@ -50,8 +50,8 @@ SpmdInfo FusedRopeGradInferSpmd(const DistMetaTensor& sin, const DistMetaTensor& out_q_grad, const DistMetaTensor& out_k_grad, const DistMetaTensor& out_v_grad, - bool use_neox_rotary_style, - bool time_major); + bool use_neox_rotary_style = true, + bool time_major = false); } // namespace distributed } // namespace phi diff --git a/python/paddle/distributed/auto_parallel/static/operators/dist_fused_rope.py b/python/paddle/distributed/auto_parallel/static/operators/dist_fused_rope.py index 24e1392843dd2..db54199ac248d 100644 --- a/python/paddle/distributed/auto_parallel/static/operators/dist_fused_rope.py +++ b/python/paddle/distributed/auto_parallel/static/operators/dist_fused_rope.py @@ -100,6 +100,7 @@ def update_dims_mapping(dist_op): ) use_neox_rotary_style = op_desc.attr("use_neox_rotary_style") + time_major = op_desc.attr("time_major") # step2: infer spmd rule = get_phi_spmd_rule("fused_rotary_position_embedding") @@ -112,6 +113,7 @@ def update_dims_mapping(dist_op): cos_spec, position_ids_spec, use_neox_rotary_style, + time_major, ) bw_results = rule.infer_backward( q_spec, @@ -124,6 +126,7 @@ def update_dims_mapping(dist_op): out_k_spec, out_v_spec, use_neox_rotary_style, + time_major, ) # remove optional args in spmd results From ffedd986c99b3e714b25bfe08cb39c3249f57084 Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Wed, 28 Feb 2024 20:22:21 +0800 Subject: [PATCH 158/282] [PIR+CINN]Fix FullOpInferSymbolicShape BUG (#62141) * [PIR+CINN]Fix FullOpInferSymbolicShape BUG * add more UT * fix UT * fix typi --- .../paddle_op_infer_sym.cc | 45 ++++++++++++------- .../pir/cinn/sub_graphs/test_sub_graph_19.py | 11 +++-- .../pir/cinn/sub_graphs/test_sub_graph_39.py | 10 ++--- .../pir/cinn/sub_graphs/test_sub_graph_80.py | 3 +- .../pir/cinn/sub_graphs/test_sub_graph_88.py | 17 ++++--- 5 files changed, 51 insertions(+), 35 deletions(-) diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc index 65e9770350c80..cb14bad351274 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc @@ -409,30 +409,45 @@ bool FullOpInferSymbolicShape(pir::Operation *op, const auto &attributes = op->attributes(); const std::vector shape = [&] { - std::vector shape; pir::Attribute attr_shape = attributes.at("shape"); const auto &shape_vec = attr_shape.dyn_cast() .data() .GetData(); - - for (auto &dim : shape_vec) { - shape.push_back(symbol::DimExpr(dim)); - } + std::vector shape(shape_vec.begin(), shape_vec.end()); return shape; }(); - // Keep shape info always with `int64_t` type. - int64_t value = attributes.at("value") - .dyn_cast() - .data() - .to(); - std::vector data{symbol::DimExpr(value)}; - - symbol::ShapeOrDataDimExprs shape_data{ - symbol::TensorShapeOrDataDimExprs(shape, data)}; + const auto shape_data = [&]() -> symbol::TensorShapeOrDataDimExprs { + // NOTE(Aurelius84): to is a risky operation when Scalar's dtype is + // not int32/int64. However, we found Full's Value could be like '3.0' but + // used as int. + const int64_t value = attributes.at("value") + .dyn_cast() + .data() + .to(); + const size_t shape_size = shape.size(); + // NOTE(Aurelius84): When shape.size()==1, a new std::vector with + // length = shape[0] will be constructed, but not all cases are used for + // ShapeAnalysis. Considering MAX_RANK < 9 in Paddle, we limit it below + // DATA_MAX_LENGTH = 128 and will not create this vector once length > + // DATA_MAX_LENGTH. + constexpr int64_t DATA_MAX_LENGTH = 128; + if (shape_size == 0U) { + std::vector data{value}; + return symbol::TensorShapeOrDataDimExprs(shape, data); + } else if (shape_size == 1U && + shape[0].template Get() <= DATA_MAX_LENGTH) { + std::vector data(shape[0].template Get(), + symbol::DimExpr(value)); + return symbol::TensorShapeOrDataDimExprs(shape, data); + } else { + return symbol::TensorShapeOrDataDimExprs(shape); + } + }(); - shape_analysis->SetShapeOrDataForValue(op->result(0), shape_data); + shape_analysis->SetShapeOrDataForValue( + op->result(0), symbol::ShapeOrDataDimExprs(shape_data)); return true; } diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_19.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_19.py index 07c05e44f41f6..c99906880760d 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_19.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_19.py @@ -17,8 +17,6 @@ # api:paddle.nn.functional.common.dropout||api:paddle.nn.functional.conv._conv_nd||method:squeeze||method:squeeze import unittest -import numpy as np - import paddle @@ -87,17 +85,18 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): outs = net(*self.inputs) return outs - # NOTE prim + cinn lead to error - # NOTE output mismatch with prim def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) cinn_out = self.train( - self.net, to_static=True, with_prim=False, with_cinn=False + self.net, to_static=True, with_prim=True, with_cinn=True ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) ): - np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) + # TODO(Aurelius84): dropout will decompose into uniform_random, which implementation + # is different from CINN. So it's not easy to compare the result. + pass + # np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) if __name__ == '__main__': diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_39.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_39.py index c2cfa2786670d..ba66c88ee23df 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_39.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_39.py @@ -30,10 +30,9 @@ def forward( self, var_0, # (shape: [12, 288, 192], dtype: paddle.float32, stop_gradient: False) ): - var_1 = paddle.tensor.creation.to_tensor(6, 'int32') - var_2 = var_0.reshape([var_1, 2, 1, 12, 24, 192]) + var_2 = var_0.reshape([6, 2, 1, 12, 24, 192]) var_3 = var_2.transpose([0, 1, 3, 2, 4, 5]) - var_4 = var_3.reshape([var_1, 24, 24, 192]) + var_4 = var_3.reshape([6, 24, 24, 192]) return var_4 @@ -57,16 +56,15 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): outs = net(*self.inputs) return outs - # NOTE prim + cinn lead to error def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) cinn_out = self.train( - self.net, to_static=True, with_prim=True, with_cinn=False + self.net, to_static=True, with_prim=True, with_cinn=True ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) ): - np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) + np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-6) if __name__ == '__main__': diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_80.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_80.py index 9ce0cb50db21d..1741a17ac0c62 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_80.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_80.py @@ -98,10 +98,11 @@ def test_ast_prim_cinn(self): cinn_out = self.train( self.net, to_static=True, with_prim=True, with_cinn=True ) + # NOTE(Aurelous84): atol only satisfy 1e-5 under with_cinn=True for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) ): - np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-6) + np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-5) if __name__ == '__main__': diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_88.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_88.py index f83e1aed2eb5e..32a9ece2de252 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_88.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_88.py @@ -38,15 +38,19 @@ def forward( var_6 = paddle.tensor.creation.full( shape=[1, 500, 1], fill_value=0, dtype='int64' ) - var_7 = paddle.tensor.manipulation.concat([var_6], axis=0) + # TODO(Aurelius84): CINN doesn't support concat single element. + # var_7 = paddle.tensor.manipulation.concat([var_6], axis=0) + var_7 = var_6 var_8 = paddle.tensor.manipulation.concat(x=[var_7, var_5], axis=2) var_9 = paddle.tensor.manipulation.gather_nd(var_4, index=var_8) var_10 = paddle.tensor.manipulation.unsqueeze(var_2, axis=2) var_11 = paddle.tensor.manipulation.expand_as(var_10, var_9) var_12 = var_11 > 0 - var_13 = paddle.tensor.search.masked_select(var_9, var_12) - var_14 = paddle.tensor.manipulation.reshape(var_13, shape=[-1, 128]) - return var_8, var_14 + # TODO(Aurelius84): masked_select will introduce dynamtic shape, skip it for now. + # var_13 = paddle.tensor.search.masked_select(var_9, var_12) + # var_14 = paddle.tensor.manipulation.reshape(var_13, shape=[-1, 128]) + # return var_8, var_14 + return var_9 + var_12 class TestLayer(unittest.TestCase): @@ -73,16 +77,15 @@ def train(self, net, to_static, with_prim=False, with_cinn=False): outs = net(*self.inputs) return outs - # NOTE prim + cinn lead to error def test_ast_prim_cinn(self): st_out = self.train(self.net, to_static=True) cinn_out = self.train( - self.net, to_static=True, with_prim=True, with_cinn=False + self.net, to_static=True, with_prim=True, with_cinn=True ) for st, cinn in zip( paddle.utils.flatten(st_out), paddle.utils.flatten(cinn_out) ): - np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) + np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-6) if __name__ == '__main__': From 1928ce83b41e9572dae97202e467c986a3f6a352 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Wed, 28 Feb 2024 20:40:32 +0800 Subject: [PATCH 159/282] clean legacy code of spmd (#62171) --- .../distributed/auto_parallel/CMakeLists.txt | 2 +- .../auto_parallel/spmd_rules/CMakeLists.txt | 6 +- .../auto_parallel/spmd_rules/common.cc | 297 ------------------ .../auto_parallel/spmd_rules/common.h | 191 ----------- .../spmd_rules/matmul_spmd_rule.h | 54 ---- .../spmd_rules/replicated_spmd_rule.cc | 49 --- .../spmd_rules/replicated_spmd_rule.h | 41 --- .../auto_parallel/spmd_rules/rules.h | 30 -- .../auto_parallel/test/CMakeLists.txt | 9 - paddle/fluid/pybind/auto_parallel_py.cc | 43 +-- .../auto_parallel/static/completion.py | 1 - test/cpp/auto_parallel/CMakeLists.txt | 36 +-- test/cpp/auto_parallel/spmd_rule_test.cc | 5 +- test/cpp/auto_parallel/spmd_rule_test_util.h | 5 +- test/cpp/auto_parallel/tile_spmd_rule_test.cc | 1 + 15 files changed, 30 insertions(+), 740 deletions(-) delete mode 100644 paddle/fluid/distributed/auto_parallel/spmd_rules/common.cc delete mode 100644 paddle/fluid/distributed/auto_parallel/spmd_rules/common.h delete mode 100644 paddle/fluid/distributed/auto_parallel/spmd_rules/matmul_spmd_rule.h delete mode 100644 paddle/fluid/distributed/auto_parallel/spmd_rules/replicated_spmd_rule.cc delete mode 100644 paddle/fluid/distributed/auto_parallel/spmd_rules/replicated_spmd_rule.h delete mode 100644 paddle/fluid/distributed/auto_parallel/spmd_rules/rules.h delete mode 100644 paddle/fluid/distributed/auto_parallel/test/CMakeLists.txt diff --git a/paddle/fluid/distributed/auto_parallel/CMakeLists.txt b/paddle/fluid/distributed/auto_parallel/CMakeLists.txt index d1eae7f599549..0fd2d6e884d1e 100644 --- a/paddle/fluid/distributed/auto_parallel/CMakeLists.txt +++ b/paddle/fluid/distributed/auto_parallel/CMakeLists.txt @@ -5,4 +5,4 @@ cc_library( SRCS dist_attr.cc DEPS phi common auto_parallel_proto proto_desc) -cc_library(auto_parallel DEPS op_dist_attr spmd_rules) +cc_library(auto_parallel DEPS op_dist_attr dist_tensor_spec) diff --git a/paddle/fluid/distributed/auto_parallel/spmd_rules/CMakeLists.txt b/paddle/fluid/distributed/auto_parallel/spmd_rules/CMakeLists.txt index f16c155890579..38aecc5b39b3b 100644 --- a/paddle/fluid/distributed/auto_parallel/spmd_rules/CMakeLists.txt +++ b/paddle/fluid/distributed/auto_parallel/spmd_rules/CMakeLists.txt @@ -1,6 +1,6 @@ -file(GLOB spmd_srcs *.cc) +file(GLOB dist_tensor_spec_srcs *.cc) cc_library( - spmd_rules - SRCS ${spmd_srcs} + dist_tensor_spec + SRCS ${dist_tensor_spec_srcs} DEPS phi common) diff --git a/paddle/fluid/distributed/auto_parallel/spmd_rules/common.cc b/paddle/fluid/distributed/auto_parallel/spmd_rules/common.cc deleted file mode 100644 index d38de8d90e2e4..0000000000000 --- a/paddle/fluid/distributed/auto_parallel/spmd_rules/common.cc +++ /dev/null @@ -1,297 +0,0 @@ -/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#include "paddle/fluid/distributed/auto_parallel/spmd_rules/common.h" - -#include - -#include "paddle/fluid/distributed/auto_parallel/spmd_rules/rules.h" -#include "paddle/phi/core/distributed/auto_parallel/utils.h" - -namespace paddle { -namespace distributed { -namespace auto_parallel { - -using phi::distributed::auto_parallel::str_join; - -std::pair, std::vector> -SPMDRuleBase::InferForward(const std::vector& input_specs, - const paddle::framework::AttributeMap& attrs) { - PADDLE_THROW( - phi::errors::Unimplemented("InferForward should be called from a " - "derived class of SPMDRuleBase !")); -} - -std::pair, std::vector> -SPMDRuleBase::InferBackward(const std::vector& input_specs, - const std::vector& output_specs, - const paddle::framework::AttributeMap& attrs) { - PADDLE_THROW( - phi::errors::Unimplemented("InferBackward should be called from a " - "derived class of SPMDRuleBase !")); -} - -// deprecated -std::pair, std::vector> -SPMDRuleBase::InferBackward(const std::vector& output_specs, - const paddle::framework::AttributeMap& attrs) { - PADDLE_THROW( - phi::errors::Unimplemented("InferBackward should be called from a " - "derived class of SPMDRuleBase !")); -} - -std::unordered_map ShardingMergeForTensors( - const std::vector>>& - tensor_axes_to_dim_pairs, - const bool merge_conflicts) { - std::unordered_map axis_to_dim_map; - std::unordered_map dim_to_axis_map; - int64_t merge_dim = 0; - - for (auto& pair : tensor_axes_to_dim_pairs) { - for (size_t i = 0; i < pair.second.size(); ++i) { - auto tensor_axis = pair.first.substr(i, 1); - auto mesh_dim = pair.second[i]; - - if (axis_to_dim_map.count(tensor_axis) == 0) { - merge_dim = mesh_dim; - } else { - merge_dim = ShardingMergeForAxis( - tensor_axis, mesh_dim, axis_to_dim_map[tensor_axis]); - } - axis_to_dim_map[tensor_axis] = merge_dim; - if (merge_dim != -1) { - if (dim_to_axis_map.count(merge_dim) == 0) { - dim_to_axis_map.insert({merge_dim, tensor_axis}); - } else if (dim_to_axis_map[merge_dim].find(tensor_axis) == - std::string::npos) { - dim_to_axis_map[merge_dim] += tensor_axis; - } - } - } - } - - // Resolute "mesh_dim shard by more than one axis" conflict. - // Now we just naive pick the first axis naively. - // (TODO) use local cost model to pick the axis with lowest cost(in concern of - // memory or communication or computation). - for (auto& it : dim_to_axis_map) { - if (it.second.size() > 1) { - if (merge_conflicts) { - VLOG(4) << "Sharding Conflict: Mesh_Dim [" << it.first - << "] are Sharding Multiple Tensor Axis: [" << it.second - << "]. The Axis: [" << it.second[0] << "] is Picked."; - for (size_t i = 1; i < it.second.size(); ++i) { - axis_to_dim_map[it.second.substr(i, 1)] = -1; - } - } else { - PADDLE_THROW(phi::errors::PreconditionNotMet( - "Multiple Tensor Axes [%s] is sharded by same mesh dimension [%d].", - str_join(it.second), - it.first)); - } - } - } - - return axis_to_dim_map; -} - -// Rule1: A replicated dimension could be merged by any sharded dimension. -// Rule2: A tensor axis could at most be sharded by one mesh dimension. -// (TODO trigger heuristics cost model and reshard to handle axis sharded by -// multiple dimension case.) -int64_t ShardingMergeForAxis(const std::string& axis, - const int64_t& mesh_dim1, - const int64_t& mesh_dim2) { - if (mesh_dim1 != mesh_dim2) { - if (mesh_dim1 == -1) { - return mesh_dim2; - } else if (mesh_dim2 == -1) { - return mesh_dim1; - } else { - // (TODO) local cost model here. - PADDLE_THROW( - phi::errors::Unimplemented("Tensor Axis[%s] is Sharded by two " - "different mesh dimension [%d] and [%d].", - axis, - mesh_dim1, - mesh_dim2)); - } - - } else { - return mesh_dim1; - } -} - -TensorDistAttr CopyTensorDistAttrForOutput( - const TensorDistAttr& src_dist_attr) { - TensorDistAttr new_dist_attr = TensorDistAttr(); - new_dist_attr.set_process_mesh(src_dist_attr.process_mesh()); - new_dist_attr.set_batch_dim(src_dist_attr.batch_dim()); - new_dist_attr.set_dynamic_dims(src_dist_attr.dynamic_dims()); - // new_dist_attr.set_annotated(false); TODO unset field is false by default. - return new_dist_attr; -} - -std::vector ResoluteOutputPartialDimension( - const std::unordered_map& axis_to_dim_map, - const std::string& tensor_axes) { - std::vector partial_on_dims; - - for (auto& it : axis_to_dim_map) { - if (tensor_axes.find(it.first) == std::string::npos) { - if (it.second > -1) { - partial_on_dims.push_back(it.second); - } - } - } - return partial_on_dims; -} - -std::string GetBroadcastAxes(const int64_t& tensor_ndim, - const int64_t& broadcast_ndim, - const std::string& alphabet) { - PADDLE_ENFORCE_GE( - alphabet.size(), - broadcast_ndim, - phi::errors::InvalidArgument( - "size of alphabet [%d] is less than broadcast ndim [%d]", - alphabet.size(), - broadcast_ndim)); - PADDLE_ENFORCE_GE(broadcast_ndim, - tensor_ndim, - phi::errors::InvalidArgument( - "broadcast ndim [%d] is less than tensor ndim [%d]", - broadcast_ndim, - tensor_ndim)); - if (tensor_ndim <= 0) { - return std::string(); - } - return alphabet.substr(broadcast_ndim - tensor_ndim, tensor_ndim); -} - -TensorDistAttr ReplicatedOnMesh(const TensorDistAttr& src_dist_attr) { - TensorDistAttr replicated_dist_attr = src_dist_attr; - replicated_dist_attr.clear_annotated(); - size_t tensor_ndim = replicated_dist_attr.dims_mapping().size(); - replicated_dist_attr.set_dims_mapping(std::vector(tensor_ndim, -1)); - return replicated_dist_attr; -} - -void VerifySpecs(const std::vector& specs, - const std::string& op_name) { - for (size_t i = 0, n = specs.size(); i < n; ++i) { - const std::vector& shape = specs[i].shape(); - const std::vector& dims_mapping = specs[i].dims_mapping(); - PADDLE_ENFORCE_EQ(shape.size(), - dims_mapping.size(), - phi::errors::InvalidArgument( - "Mismatch in %s, spec[%d]'s tensor size: [%d] and " - "spec[%d]'s dims_mapping size [%d].", - op_name, - i, - shape.size(), - i, - dims_mapping.size())); - } -} - -std::vector>> -GetAxesDimsMappingPair(const std::vector& tensor_axes, - const std::vector& specs) { - std::vector>> res; - size_t ntensor = specs.size(); - for (size_t i = 0; i < ntensor; ++i) { - res.emplace_back(tensor_axes[i], specs[i].dims_mapping()); - } - return res; -} - -std::vector GetDimsMappingForAxes( - const std::string& axes, - const std::unordered_map& axis_to_dim_map, - const bool unsharded_miss_axis) { - std::vector dims_mapping; - for (int64_t i = 0, n = static_cast(axes.size()); i < n; i++) { - std::string axis = axes.substr(i, 1); - if (axis == "1") { - dims_mapping.emplace_back(-1); - } else { - auto iter = axis_to_dim_map.find(axis); - if (iter == axis_to_dim_map.end()) { - if (unsharded_miss_axis) { - dims_mapping.emplace_back(-1); - } else { - phi::errors::InvalidArgument( - "Tensor axis [%s] of not in axis_to_dim_map.", axis); - } - } else { - dims_mapping.emplace_back(iter->second); - } - } - } - return dims_mapping; -} - -// SPMDRuleMap -SPMDRuleMap& SPMDRuleMap::Instance() { - static SPMDRuleMap g_spmd_rule_map; - return g_spmd_rule_map; -} - -// To enable default replicated spmd rule for op that are NOT registered -// which all tensors of inputs and outputs will be replicated in all ranks of -// the mesh. -SPMDRuleBase* SPMDRuleMap::Get(const std::string& op_type) const { - auto rule_ptr = GetNullable(op_type); - if (rule_ptr == nullptr) { - std::string str; - for (const auto& item : map_) { - str += item.first + ", "; - } - VLOG(4) << "Size of current map [" << map_.size() << "]"; - VLOG(4) << "Keys are [" << str << "]"; - } - PADDLE_ENFORCE_NOT_NULL( - rule_ptr, - platform::errors::NotFound( - "NO SPMD Rule has been registered for Operator [%s].", op_type)); - return rule_ptr; -} - -SPMDRuleBase* SPMDRuleMap::GetNullable(const std::string& op_type) const { - auto it = map_.find(op_type); - if (it == map_.end()) { - return nullptr; - } else { - return it->second.get(); - } -} - -int SPMDRuleMap::Insert(const std::string& op_type, - std::unique_ptr rule) { - VLOG(4) << "Call SPMDRuleMap::Insert!"; - PADDLE_ENFORCE_NE( - Has(op_type), - true, - platform::errors::AlreadyExists( - "SPMD Rule for Operator [%s] has been registered.", op_type)); - map_.insert({op_type, std::move(rule)}); - - return 1; -} - -} // namespace auto_parallel -} // namespace distributed -} // namespace paddle diff --git a/paddle/fluid/distributed/auto_parallel/spmd_rules/common.h b/paddle/fluid/distributed/auto_parallel/spmd_rules/common.h deleted file mode 100644 index 9f6a52750580b..0000000000000 --- a/paddle/fluid/distributed/auto_parallel/spmd_rules/common.h +++ /dev/null @@ -1,191 +0,0 @@ -/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#pragma once - -#include -#include -#include -#include -#include - -#include "paddle/fluid/distributed/auto_parallel/spmd_rules/dist_tensor_spec.h" -#include "paddle/fluid/framework/attribute.h" -#include "paddle/fluid/framework/type_defs.h" -#include "paddle/fluid/platform/enforce.h" -#include "paddle/phi/core/distributed/auto_parallel/dist_attr.h" -#include "paddle/utils/flat_hash_map.h" - -namespace paddle { -namespace distributed { -namespace auto_parallel { - -using paddle::framework::Attribute; - -class SPMDRuleBase { - public: - virtual ~SPMDRuleBase() {} - - // Based on the information of Input Tensors and Op Attribute: - // 1. Merge the Sharding (dims_mapping) among Input Tensors. - // 2. Infer the Sharding (dims_mapping) for Output Tensors. - // The Info of input tensors (Shape and DistAttr) are wrapped as - // DistTensorSpec, and op attribute should be given as AttributeMap. The - // Output is a pair consist of two vectors: - // 1. The first vector: the merged DistAttr of input tensors. - // 2. The inferred DistAttr of output tensors. - // The Merged DistAttr might be different from the original Intput DistAttrs, - // which means that the corresponding input tensor need to be reshard. - virtual std::pair, std::vector> - InferForward(const std::vector& input_specs, - const paddle::framework::AttributeMap& attrs); - - // Based on the information of Input & Output Tensors and Op Attribute: - // 1. Merge the Sharding (dims_mapping) among Output Tensors. - // 2. Infer the Sharding (dims_mapping) for Input Tensors. - // The Info of output tensors (Shape and DistAttr) are wrapped as - // DistTensorSpec, and op attribute should be given as AttributeMap. The - // Output is a pair consist of two vectors: - // 1. The first vector: the merged DistAttr of output tensors. - // 2. The inferred DistAttr of Input tensors. - virtual std::pair, std::vector> - InferBackward(const std::vector& input_specs, - const std::vector& output_specs, - const paddle::framework::AttributeMap& attrs); - - // deprecated, to be remove in future - virtual std::pair, std::vector> - InferBackward(const std::vector& output_specs, - const paddle::framework::AttributeMap& attrs); - - template - inline const T ExtractAttr( - const std::string& name, - const paddle::framework::AttributeMap& attrs) const { - auto attr = GetAttr(name, attrs); - return *paddle::framework::ExtractAttribute(name)(attr); - } - - Attribute GetAttr(const std::string& name, - const paddle::framework::AttributeMap& attrs) const { - auto iter = attrs.find(name); - PADDLE_ENFORCE_NE(iter, - attrs.end(), - paddle::platform::errors::NotFound( - "(%s) is not found in AttributeMap.", name)); - return iter->second; - } -}; - -// Merge sharding specification (dims mapping) of given tensors. -// The same axes of different tensors will be merged. -std::unordered_map ShardingMergeForTensors( - const std::vector>>& - tensor_axes_to_dim_pairs, - const bool merge_conflicts = true); - -// Merge the sharding specification (dims mapping) for one tensor Axis. -// Rule1: A replicated dimension could be merged by any sharded dimension. -// Rule2: A tensor axis could at most be sharded by one mesh dimension. -// (TODO trigger heuristics cost model and reshard to handle axis sharded by -// multiple dimension case.) -int64_t ShardingMergeForAxis(const std::string& axis, - const int64_t& mesh_dim1, - const int64_t& mesh_dim2); - -// Intend to use for generating the TensorDistAttr of output based on the input -// activation TensorDistAttr. The process_mesh, batch_dim, dynamic_dim are -// copied with annotated is forced to False, and dims_mapping is leave to be -// null. -TensorDistAttr CopyTensorDistAttrForOutput(const TensorDistAttr& src_dist_attr); - -// Resolute the partial mesh dimension of a output tensor, giving the -// merged sharding specification of input tensors and the axis names of output -// tensor. Input are -std::vector ResoluteOutputPartialDimension( - const std::unordered_map& axis_to_dim_map, - const std::string& tensor_axes); - -// Generate the axis notation of tensor for the einsum notation of a broadcast -// operation(alignment star from the rightmost axis). tensor_ndim: the size of -// the tensor. broadcast_ndim: the maximum size of tensors in this broadcast -// operation. alphabet: the characters used to represent the axes of tensor. -// length of alphabet should >= broadcast_ndim. -std::string GetBroadcastAxes(const int64_t& tensor_ndim, - const int64_t& broadcast_ndim, - const std::string& alphabet); - -// Return a NEW TensorDistAttr whose dims mapping is consist of "-1" -// (unsharded). -TensorDistAttr ReplicatedOnMesh(const TensorDistAttr& src_dist_attr); - -// Check whether the given DistTensorSpec objects are valid. For each -// DistTensorSpec, the rank of its dims mapping must be equal to the rank of its -// corresponding tensor shape. the parameter op_name is used for logging error -// message. -void VerifySpecs(const std::vector& specs, - const std::string& op_name); - -// Get dims mapping for the given tensors. Return the pair of each -// tensor's einsum notation and the corresponding dims mapping. -std::vector>> -GetAxesDimsMappingPair(const std::vector& tensor_axes, - const std::vector& specs); - -// Get dims mapping for the given axes according to sharding information of -// the annotated axes after inferring forward or backward. The parameter axis -// stores the axes of the tensor. "1" is a special axis, for the axis "1", set -// its dims mapping to -1. -// if unsharded_miss_axis, "-1" is assigned to axes that has no key in -// axis_to_dim_map. -std::vector GetDimsMappingForAxes( - const std::string& axes, - const std::unordered_map& axis_to_dim_map, - const bool unsharded_miss_axis = false); - -// The static map that stores and initializes all the registered SPMD rules. -class SPMDRuleMap { - public: - ~SPMDRuleMap() = default; - - // A singleton - static SPMDRuleMap& Instance(); - - // Returns the spmd rule for the given op_type - SPMDRuleBase* Get(const std::string& op_type) const; - - // Returns the spmd by name or nullptr if not registered - SPMDRuleBase* GetNullable(const std::string& op_type) const; - - // Register a spmd for an op_type. - int Insert(const std::string& op_type, std::unique_ptr rule); - - bool Has(const std::string& op_type) const { - return map_.find(op_type) != map_.end(); - } - - private: - SPMDRuleMap() = default; - paddle::flat_hash_map> map_; - DISABLE_COPY_AND_ASSIGN(SPMDRuleMap); -}; - -#define REGISTER_SPMD_RULE(op_type, rule_class, ...) \ - UNUSED static int __spmd_rule_holder_##op_type = \ - ::paddle::distributed::auto_parallel::SPMDRuleMap::Instance().Insert( \ - #op_type, std::make_unique(__VA_ARGS__)) - -} // namespace auto_parallel -} // namespace distributed -} // namespace paddle diff --git a/paddle/fluid/distributed/auto_parallel/spmd_rules/matmul_spmd_rule.h b/paddle/fluid/distributed/auto_parallel/spmd_rules/matmul_spmd_rule.h deleted file mode 100644 index 70d603e509c43..0000000000000 --- a/paddle/fluid/distributed/auto_parallel/spmd_rules/matmul_spmd_rule.h +++ /dev/null @@ -1,54 +0,0 @@ -/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#pragma once - -#include -#include -#include -#include - -#include "paddle/fluid/distributed/auto_parallel/spmd_rules/common.h" - -namespace paddle { -namespace distributed { -namespace auto_parallel { - -TensorDistAttr GetInferedDistAttr( - const TensorDistAttr& origin_dist_attr, - const std::vector& shape, - const std::string& tensor_axes, - const std::unordered_map& axis_to_dim_map, - const bool trans_axis); - -void FillMatmulOperandNotation(const int x_ndim, - const int y_ndim, - std::string* x_axes, - std::string* y_axes, - std::string* out_axes); - -class MatmulSPMDRule : public SPMDRuleBase { - public: - std::pair, std::vector> - InferForward(const std::vector& input_specs, - const paddle::framework::AttributeMap& attrs) override; - - std::pair, std::vector> - InferBackward(const std::vector& input_specs, - const std::vector& output_specs, - const paddle::framework::AttributeMap& attrs) override; -}; -} // namespace auto_parallel -} // namespace distributed -} // namespace paddle diff --git a/paddle/fluid/distributed/auto_parallel/spmd_rules/replicated_spmd_rule.cc b/paddle/fluid/distributed/auto_parallel/spmd_rules/replicated_spmd_rule.cc deleted file mode 100644 index 5227a82a4b8b5..0000000000000 --- a/paddle/fluid/distributed/auto_parallel/spmd_rules/replicated_spmd_rule.cc +++ /dev/null @@ -1,49 +0,0 @@ -/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#include "paddle/fluid/distributed/auto_parallel/spmd_rules/replicated_spmd_rule.h" - -namespace paddle { -namespace distributed { -namespace auto_parallel { - -std::pair, std::vector> -ReplicatedSPMDRule::InferForward(const std::vector& input_specs, - const paddle::framework::AttributeMap& attrs) { - std::vector intput_dist_attrs; - std::vector output_dist_attrs; - intput_dist_attrs.reserve(input_specs.size()); - - for (auto& input_spec : input_specs) { - intput_dist_attrs.push_back(ReplicatedOnMesh(input_spec.dist_attr())); - } - - // TODO(ljz): we need to know num of output and size of each output before - // generate the exact replicated dist tensor attr for the current op. - // here we just assume that only one output tensor and has the same size as - // the first input tensor. - return {intput_dist_attrs, {ReplicatedOnMesh(input_specs[0].dist_attr())}}; -} - -std::pair, std::vector> -ReplicatedSPMDRule::InferBackward( - const std::vector& input_specs, - const paddle::framework::AttributeMap& attrs) { - PADDLE_THROW(phi::errors::Unimplemented( - "InferBackward of ReplicatedSPMDRule is NOT implemented yet.")); -} - -} // namespace auto_parallel -} // namespace distributed -} // namespace paddle diff --git a/paddle/fluid/distributed/auto_parallel/spmd_rules/replicated_spmd_rule.h b/paddle/fluid/distributed/auto_parallel/spmd_rules/replicated_spmd_rule.h deleted file mode 100644 index bcca646d351d5..0000000000000 --- a/paddle/fluid/distributed/auto_parallel/spmd_rules/replicated_spmd_rule.h +++ /dev/null @@ -1,41 +0,0 @@ -/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#pragma once - -#include "paddle/fluid/distributed/auto_parallel/spmd_rules/common.h" - -namespace paddle { -namespace distributed { -namespace auto_parallel { - -// A Bottom Line Rule that enforces input(s) and output(s) of the Op to be -// replicated among the given mesh. -class ReplicatedSPMDRule : public SPMDRuleBase { - public: - // The dims_mapping of ALL TensorDistAttrs would be repeat of "-1" - // (unsharded). - std::pair, std::vector> - InferForward(const std::vector& input_specs, - const paddle::framework::AttributeMap& attrs) override; - - // The dims_mapping of ALL TensorDistAttrs would be repeat of "-1" - // (unsharded). - std::pair, std::vector> - InferBackward(const std::vector& output_specs, - const paddle::framework::AttributeMap& attrs) override; -}; -} // namespace auto_parallel -} // namespace distributed -} // namespace paddle diff --git a/paddle/fluid/distributed/auto_parallel/spmd_rules/rules.h b/paddle/fluid/distributed/auto_parallel/spmd_rules/rules.h deleted file mode 100644 index e63d58886d46f..0000000000000 --- a/paddle/fluid/distributed/auto_parallel/spmd_rules/rules.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include "paddle/fluid/distributed/auto_parallel/spmd_rules/common.h" -#include "paddle/fluid/distributed/auto_parallel/spmd_rules/replicated_spmd_rule.h" - -// TODO(ljz) Automatic this process in cmake file. -namespace paddle { -namespace distributed { -namespace auto_parallel { - -// replicated rule -REGISTER_SPMD_RULE(replicated, ReplicatedSPMDRule); - -} // namespace auto_parallel -} // namespace distributed -} // namespace paddle diff --git a/paddle/fluid/distributed/auto_parallel/test/CMakeLists.txt b/paddle/fluid/distributed/auto_parallel/test/CMakeLists.txt deleted file mode 100644 index 449ee65ccc751..0000000000000 --- a/paddle/fluid/distributed/auto_parallel/test/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -paddle_test(device_mesh_test SRCS device_mesh_test.cc) - -paddle_test(process_mesh_test SRCS process_mesh_test.cc) - -paddle_test(dist_attr_test SRCS dist_attr_test.cc) - -paddle_test(dist_mapper_test SRCS dist_mapper_test.cc) - -paddle_test(spmd_rule_test SRCS spmd_rule_test.cc) diff --git a/paddle/fluid/pybind/auto_parallel_py.cc b/paddle/fluid/pybind/auto_parallel_py.cc index 8a044b678d79b..87895d6b4df31 100644 --- a/paddle/fluid/pybind/auto_parallel_py.cc +++ b/paddle/fluid/pybind/auto_parallel_py.cc @@ -17,6 +17,8 @@ #include #include +#include "paddle/fluid/distributed/auto_parallel/spmd_rules/dist_tensor_spec.h" +#include "paddle/fluid/eager/api/manual/eager_manual/dygraph_forward_api.h" #include "paddle/fluid/framework/block_desc.h" #include "paddle/fluid/framework/op_desc.h" #include "paddle/fluid/framework/var_desc.h" @@ -24,24 +26,18 @@ #include "paddle/fluid/pybind/eager_utils.h" #include "paddle/fluid/pybind/op_function_common.h" #include "paddle/fluid/pybind/pybind_variant_caster.h" +#include "paddle/phi/api/lib/data_transform.h" +#include "paddle/phi/backends/context_pool.h" +#include "paddle/phi/common/reduce_type.h" +#include "paddle/phi/core/dense_tensor.h" #include "paddle/phi/core/device_context.h" #include "paddle/phi/core/distributed/auto_parallel/device_mesh.h" #include "paddle/phi/core/distributed/auto_parallel/dist_attr.h" #include "paddle/phi/core/distributed/auto_parallel/dist_mapper.h" +#include "paddle/phi/core/distributed/auto_parallel/dist_tensor.h" #include "paddle/phi/core/distributed/auto_parallel/inferspmd_utils.h" #include "paddle/phi/core/distributed/auto_parallel/placement_types.h" #include "paddle/phi/core/distributed/auto_parallel/process_mesh.h" -#include "paddle/utils/optional.h" -#include "paddle/utils/pybind.h" - -#include "paddle/fluid/distributed/auto_parallel/spmd_rules/common.h" -#include "paddle/fluid/distributed/auto_parallel/spmd_rules/dist_tensor_spec.h" -#include "paddle/fluid/eager/api/manual/eager_manual/dygraph_forward_api.h" -#include "paddle/phi/api/lib/data_transform.h" -#include "paddle/phi/backends/context_pool.h" -#include "paddle/phi/common/reduce_type.h" -#include "paddle/phi/core/dense_tensor.h" -#include "paddle/phi/core/distributed/auto_parallel/dist_tensor.h" #include "paddle/phi/core/distributed/auto_parallel/reshard/nd_mesh_reshard_function.h" #include "paddle/phi/core/distributed/auto_parallel/reshard/p_to_r_reshard_function.h" #include "paddle/phi/core/distributed/auto_parallel/reshard/p_to_s_reshard_function.h" @@ -53,6 +49,8 @@ #include "paddle/phi/core/distributed/auto_parallel/reshard/same_status_reshard_function.h" #include "paddle/phi/core/distributed/auto_parallel/reshard/x_to_r_reshard_function.h" #include "paddle/phi/core/enforce.h" +#include "paddle/utils/optional.h" +#include "paddle/utils/pybind.h" #ifdef PADDLE_WITH_DISTRIBUTE #include "paddle/phi/infermeta/spmd_rules/rules.h" @@ -74,8 +72,6 @@ static bool PyCheckInteger(PyObject *obj) { using paddle::distributed::auto_parallel::DistTensorSpec; using paddle::distributed::auto_parallel::kDefault; using paddle::distributed::auto_parallel::OperatorDistAttr; -using paddle::distributed::auto_parallel::SPMDRuleBase; -using paddle::distributed::auto_parallel::SPMDRuleMap; using paddle::framework::BlockDesc; using paddle::framework::OpDesc; using paddle::framework::VarDesc; @@ -590,17 +586,6 @@ void BindAutoParallel(py::module *m) { }) .def("_clean_partial_status", &TensorDistAttr::clean_partial_status); - py::class_(*m, "SPMDRuleBase") - .def("infer_forward", &SPMDRuleBase::InferForward) - .def("infer_backward", - static_cast, - std::vector> (SPMDRuleBase::*)( - const std::vector &, - const std::vector &, - const paddle::framework::AttributeMap &)>( - &SPMDRuleBase::InferBackward)); - // .def("infer_backward", &SPMDRuleBase::InferBackward) [revert in future] - py::class_(*m, "SpmdRule") .def("infer_forward", &infer_forward) .def("infer_backward", &infer_backward); @@ -750,15 +735,7 @@ void BindAutoParallel(py::module *m) { "contains_spmd_rule", [](const std::string op_type) { return phi::distributed::SpmdRuleFactory::Instance().ContainsSpmdRule( - op_type) || - SPMDRuleMap::Instance().Has(op_type); // TODO(ljz): unify here - }, - py::return_value_policy::reference); - - m->def( - "get_spmd_rule", - [](const std::string op_type) { - return SPMDRuleMap::Instance().Get(op_type); + op_type); }, py::return_value_policy::reference); diff --git a/python/paddle/distributed/auto_parallel/static/completion.py b/python/paddle/distributed/auto_parallel/static/completion.py index 900b90a0f6496..01db8beacb7e4 100644 --- a/python/paddle/distributed/auto_parallel/static/completion.py +++ b/python/paddle/distributed/auto_parallel/static/completion.py @@ -22,7 +22,6 @@ from paddle.base.core import ( # noqa: F401 contains_spmd_rule, get_phi_spmd_rule, - get_spmd_rule, ) from paddle.base.framework import Operator from paddle.base.log_helper import get_logger diff --git a/test/cpp/auto_parallel/CMakeLists.txt b/test/cpp/auto_parallel/CMakeLists.txt index e48b634d68db2..2985dffa7da18 100644 --- a/test/cpp/auto_parallel/CMakeLists.txt +++ b/test/cpp/auto_parallel/CMakeLists.txt @@ -9,47 +9,31 @@ cc_test( if(WITH_DISTRIBUTE) cc_library(spmd_rule_test_util SRCS spmd_rule_test_util.cc) - add_dependencies(spmd_rule_test_util spmd_rules) cc_test( dist_tensor_test SRCS dist_tensor_test.cc DEPS phi common) - paddle_test(spmd_rule_test SRCS spmd_rule_test.cc DEPS spmd_rule_test_util - spmd_rules) + paddle_test(spmd_rule_test SRCS spmd_rule_test.cc DEPS spmd_rule_test_util) paddle_test(softmax_grad_spmd_rule_test SRCS softmax_grad_spmd_rule_test.cc - DEPS spmd_rule_test_util spmd_rules) + DEPS spmd_rule_test_util) paddle_test(tile_spmd_rule_test SRCS tile_spmd_rule_test.cc DEPS - spmd_rule_test_util spmd_rules) + spmd_rule_test_util) paddle_test( fused_linear_param_grad_add_spmd_rule_test SRCS - fused_linear_param_grad_add_spmd_rule_test.cc DEPS spmd_rule_test_util - spmd_rules) + fused_linear_param_grad_add_spmd_rule_test.cc DEPS spmd_rule_test_util) - paddle_test( - cross_entropy_softmax_spmd_rule_test SRCS - cross_entropy_softmax_spmd_rule_test.cc DEPS spmd_rule_test_util spmd_rules) + paddle_test(cross_entropy_softmax_spmd_rule_test SRCS + cross_entropy_softmax_spmd_rule_test.cc DEPS spmd_rule_test_util) - paddle_test( - custom_op_spmd_rule_test - SRCS - custom_op_spmd_rule_test.cc - DEPS - spmd_rule_test_util - spmd_rules - phi) + paddle_test(custom_op_spmd_rule_test SRCS custom_op_spmd_rule_test.cc DEPS + spmd_rule_test_util phi) - paddle_test( - fused_rms_norm_spmd_rule_test - SRCS - fused_rms_norm_spmd_rule_test.cc - DEPS - spmd_rule_test_util - spmd_rules - phi) + paddle_test(fused_rms_norm_spmd_rule_test SRCS + fused_rms_norm_spmd_rule_test.cc DEPS spmd_rule_test_util phi) endif() diff --git a/test/cpp/auto_parallel/spmd_rule_test.cc b/test/cpp/auto_parallel/spmd_rule_test.cc index 25e99fb52575b..49544cb508c7c 100644 --- a/test/cpp/auto_parallel/spmd_rule_test.cc +++ b/test/cpp/auto_parallel/spmd_rule_test.cc @@ -256,7 +256,6 @@ TEST(LayerNormSPMDRule, Ctor) { bias_dist_attr.set_dims_mapping(std::vector({-1})); bias_dist_attr.set_dynamic_dims(std::vector({false})); - paddle::framework::AttributeMap attrs; float epsilon = 1e-5; int begin_norm_axis = 2; @@ -912,7 +911,7 @@ TEST(ReduceMaxRule, Ctor) { t_dist_attr.set_dynamic_dims({false, false, false}); phi::distributed::DistMetaTensor x = phi::distributed::DistMetaTensor( common::make_ddim({4, 6, 8}), t_dist_attr); - IntArray axis = {1}; + phi::IntArray axis = {1}; bool keep_dim = false; phi::distributed::SpmdInfo forward_info = phi::distributed::ReductionMaxInferSpmdDynamic(x, axis, keep_dim); @@ -944,7 +943,7 @@ TEST(ReduceAllRule, Ctor) { t_dist_attr.set_dynamic_dims({false, false, false}); phi::distributed::DistMetaTensor x = phi::distributed::DistMetaTensor(phi::make_ddim({4, 6, 8}), t_dist_attr); - IntArray axis = {1}; + phi::IntArray axis = {1}; bool keep_dim = false; phi::distributed::SpmdInfo forward_info = phi::distributed::ReductionAllInferSpmdDynamic(x, axis, keep_dim); diff --git a/test/cpp/auto_parallel/spmd_rule_test_util.h b/test/cpp/auto_parallel/spmd_rule_test_util.h index a36564aa51c01..fdf0af96768bb 100644 --- a/test/cpp/auto_parallel/spmd_rule_test_util.h +++ b/test/cpp/auto_parallel/spmd_rule_test_util.h @@ -20,8 +20,6 @@ limitations under the License. */ #include "glog/logging.h" #include "gtest/gtest.h" -#include "paddle/fluid/distributed/auto_parallel/spmd_rules/common.h" -#include "paddle/fluid/distributed/auto_parallel/spmd_rules/dist_tensor_spec.h" #include "paddle/phi/core/distributed/auto_parallel/dist_attr.h" #include "paddle/phi/core/distributed/auto_parallel/inferspmd_utils.h" #include "paddle/phi/core/distributed/auto_parallel/process_mesh.h" @@ -33,6 +31,9 @@ namespace paddle { namespace distributed { namespace auto_parallel { +using phi::distributed::ProcessMesh; +using phi::distributed::TensorDistAttr; + const std::vector& get_dims_mapping( const phi::distributed::ArgDistAttr& dist_attr); diff --git a/test/cpp/auto_parallel/tile_spmd_rule_test.cc b/test/cpp/auto_parallel/tile_spmd_rule_test.cc index df1df74bd91c0..11acbba71b91f 100644 --- a/test/cpp/auto_parallel/tile_spmd_rule_test.cc +++ b/test/cpp/auto_parallel/tile_spmd_rule_test.cc @@ -17,6 +17,7 @@ limitations under the License. */ namespace paddle { namespace distributed { namespace auto_parallel { + TEST(Tile, Ctor) { std::vector mesh_shape = {2, 2}; std::vector process_ids = {0, 1, 2, 3}; From d7e22f64dfb4266c00513cd333369d9c475a7041 Mon Sep 17 00:00:00 2001 From: YuanRisheng Date: Wed, 28 Feb 2024 20:42:17 +0800 Subject: [PATCH 160/282] adapt top_p_sampling (#62169) --- python/paddle/tensor/search.py | 2 +- test/legacy_test/test_top_p_sampling.py | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/python/paddle/tensor/search.py b/python/paddle/tensor/search.py index 9e5d070268e3f..7d619ca5e2e8a 100755 --- a/python/paddle/tensor/search.py +++ b/python/paddle/tensor/search.py @@ -1281,7 +1281,7 @@ def top_p_sampling(x, ps, threshold=None, seed=None, name=None): if seed is None: seed = -1 - if in_dynamic_mode(): + if in_dynamic_or_pir_mode(): return _C_ops.top_p_sampling(x, ps, threshold, seed) inputs = {"x": x, "ps": ps, "threshold": threshold} diff --git a/test/legacy_test/test_top_p_sampling.py b/test/legacy_test/test_top_p_sampling.py index 8b7b9aeabf186..f4e736696dbec 100644 --- a/test/legacy_test/test_top_p_sampling.py +++ b/test/legacy_test/test_top_p_sampling.py @@ -18,6 +18,7 @@ import paddle from paddle.base import core +from paddle.pir_utils import test_with_pir_api def TopPProcess(probs, top_p): @@ -138,11 +139,17 @@ def run_static(self, place): paddle_result[1], paddle_result[3], rtol=1e-05 ) - def test_cases(self): + def test_dygraph(self): if core.is_compiled_with_cuda(): places = [core.CUDAPlace(0)] for place in places: self.run_dygraph(place) + + @test_with_pir_api + def test_static(self): + if core.is_compiled_with_cuda(): + places = [core.CUDAPlace(0)] + for place in places: self.run_static(place) From 6ce8f9ec6217bb53ec5635df8f08f62c0210edec Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Wed, 28 Feb 2024 22:14:39 +0800 Subject: [PATCH 161/282] [Dy2St][PIR] Handle `OutletType` in middle values (#62199) --- .../eager/to_static/run_program_op_func.h | 18 ++++++++++---- .../eager/to_static/run_program_op_node.h | 24 ++++++++++++++++++- paddle/fluid/pybind/pir.cc | 13 ++++++---- test/dygraph_to_static/test_ifelse.py | 3 +-- 4 files changed, 46 insertions(+), 12 deletions(-) diff --git a/paddle/fluid/eager/to_static/run_program_op_func.h b/paddle/fluid/eager/to_static/run_program_op_func.h index f6b8e21cd8b17..c767ad0b6106c 100644 --- a/paddle/fluid/eager/to_static/run_program_op_func.h +++ b/paddle/fluid/eager/to_static/run_program_op_func.h @@ -20,9 +20,12 @@ #include "paddle/fluid/eager/eager_tensor.h" #include "paddle/fluid/eager/to_static/run_program_op_node.h" #include "paddle/fluid/eager/utils.h" +#include "paddle/fluid/framework/tensor_ref_array.h" #include "paddle/fluid/memory/allocation/allocator.h" #include "paddle/pir/include/core/block.h" +#include "paddle/pir/include/core/builtin_type.h" #include "paddle/pir/include/core/value.h" +#include "paddle/pir/include/dialect/control_flow/ir/cf_type.h" // Filter params without grads in global block. In this case, we will // tag its AutogradMeta with stop_gradient = True to avoid fault from @@ -244,8 +247,9 @@ inline void pir_run_program_ad_func( trace_backward, &p_autograd_x, &p_autograd_params); // Create Middle Output for GradNode. - auto middle_size = - PADDLE_GET_CONST(std::vector<::pir::Value>, attrs.at("fm")).size(); + auto middle_values = + PADDLE_GET_CONST(std::vector<::pir::Value>, attrs.at("fm")); + auto middle_size = middle_values.size(); auto output_size = PADDLE_GET_CONST(std::vector<::pir::Value>, attrs.at("fo")).size(); auto middles = std::vector(); @@ -264,8 +268,14 @@ inline void pir_run_program_ad_func( grad_node->GetMiddle().resize(middle_size); grad_node->GetOutputs().resize(output_size); for (size_t i = 0; i < middle_size; ++i) { - grad_node->GetMiddle()[i] = - paddle::Tensor(std::make_shared()); + auto middle_value = middle_values[i]; + if (middle_value.type().isa()) { + grad_node->GetMiddle()[i] = + paddle::Tensor(std::make_shared()); + } else if (middle_value.type().isa()) { + grad_node->GetMiddle()[i] = paddle::Tensor( + std::make_shared()); + } middles.push_back(&grad_node->GetMiddle()[i]); } diff --git a/paddle/fluid/eager/to_static/run_program_op_node.h b/paddle/fluid/eager/to_static/run_program_op_node.h index fdebfbb1e3771..da04f129c01aa 100644 --- a/paddle/fluid/eager/to_static/run_program_op_node.h +++ b/paddle/fluid/eager/to_static/run_program_op_node.h @@ -19,6 +19,7 @@ #include "paddle/fluid/eager/tensor_wrapper.h" #include "paddle/fluid/framework/executor_cache.h" #include "paddle/fluid/framework/new_executor/interpretercore.h" +#include "paddle/fluid/framework/tensor_ref_array.h" #include "paddle/fluid/framework/variable_helper.h" #include "paddle/fluid/ir_adaptor/translator/program_translator.h" #include "paddle/fluid/operators/run_program_op.h" @@ -120,10 +121,20 @@ static void CheckOutputVarStatus(const paddle::framework::Variable &src_var, "RunProgram(Grad)Op's internal scope holds " "wrong type. Expect type is SelectedRows", name)); + } else if (paddle::framework::VariableRefArray::classof( + dst_tensor.impl().get())) { + auto &src_tensor = src_var.Get(); + PADDLE_ENFORCE_EQ(paddle::framework::VariableRefArray::classof(&src_tensor), + true, + paddle::platform::errors::InvalidArgument( + "The output tensor %s get from " + "RunProgram(Grad)Op's internal scope holds " + "wrong type. Expect type is VariableRefArray", + name)); } else { PADDLE_THROW(paddle::platform::errors::InvalidArgument( "The RunProgram(Grad)Op only support output " - "variable of type LoDTensor or SelectedRows", + "variable of type DenseTensor, SelectedRows or VariableRefArray", name)); } } @@ -320,6 +331,17 @@ static void ShareTensorsFromScopeByValue( auto *dst_tensor = const_cast( dynamic_cast(tensors[i]->impl().get())); *dst_tensor = src_tensor; + } else if (var->IsType()) { + auto &src_tensor = var->Get(); + auto *dst_tensor = const_cast( + dynamic_cast( + tensors[i]->impl().get())); + *dst_tensor = src_tensor; + } else { + PADDLE_THROW(paddle::platform::errors::InvalidArgument( + "The RunProgram(Grad)Op only support output " + "variable of type DenseTensor, SelectedRows or VariableRefArray", + name)); } } } diff --git a/paddle/fluid/pybind/pir.cc b/paddle/fluid/pybind/pir.cc index 54fa9bf54f057..bd603e326a9ad 100644 --- a/paddle/fluid/pybind/pir.cc +++ b/paddle/fluid/pybind/pir.cc @@ -950,11 +950,14 @@ AnalysisMiddleVariable(const Program &program, program.block(), forward_range, [&middle_values, &backward_inputs, &x_or_param](Operation *op) { - for (auto &t : op->results()) { - auto v = Value(t.Value::impl()); - if (backward_inputs.count(v) && !x_or_param.count(v)) - middle_values.push_back(v); - } + pir::Walk(op, [&](Operation *inner_op) { + for (auto &t : inner_op->results()) { + auto v = Value(t.Value::impl()); + if (backward_inputs.count(v) && !x_or_param.count(v)) { + middle_values.push_back(v); + } + } + }); }); return std::make_pair(middle_values, backward_inputs); } diff --git a/test/dygraph_to_static/test_ifelse.py b/test/dygraph_to_static/test_ifelse.py index a05f3d07510e9..fef4c48d49512 100644 --- a/test/dygraph_to_static/test_ifelse.py +++ b/test/dygraph_to_static/test_ifelse.py @@ -23,7 +23,6 @@ enable_to_static_guard, test_ast_only, test_legacy_and_pt_and_pir, - test_legacy_only, test_pir_only, ) from ifelse_simple_func import ( @@ -338,7 +337,7 @@ def _run(self, to_static=False): ret = net(x_v) return ret.numpy() - @test_legacy_only + @test_legacy_and_pt_and_pir def test_ast_to_func(self): self.assertTrue((self._run_dygraph() == self._run_static()).all()) From b09e0d72cdbaefa295f0d072e02817afe2a84c47 Mon Sep 17 00:00:00 2001 From: Tian <121000916+SylarTiaNII@users.noreply.github.com> Date: Thu, 29 Feb 2024 08:44:17 +0800 Subject: [PATCH 162/282] [CustomDevice] register bf16 empty kernel for custom devices (#62140) --- paddle/phi/kernels/empty_kernel.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/paddle/phi/kernels/empty_kernel.cc b/paddle/phi/kernels/empty_kernel.cc index 0250fdd3b1f69..eb818ae120f66 100644 --- a/paddle/phi/kernels/empty_kernel.cc +++ b/paddle/phi/kernels/empty_kernel.cc @@ -158,7 +158,8 @@ PD_REGISTER_KERNEL(empty, int, int64_t, bool, - phi::dtype::float16) {} + phi::dtype::float16, + phi::dtype::bfloat16) {} PD_REGISTER_KERNEL(empty_like, Custom, ALL_LAYOUT, @@ -171,7 +172,8 @@ PD_REGISTER_KERNEL(empty_like, int, int64_t, bool, - phi::dtype::float16) { + phi::dtype::float16, + phi::dtype::bfloat16) { kernel->InputAt(0).SetBackend(phi::Backend::ALL_BACKEND); } #endif From dc982b43d15b6bc012725bebc66b10376453090f Mon Sep 17 00:00:00 2001 From: lanxianghit <47554610+lanxianghit@users.noreply.github.com> Date: Thu, 29 Feb 2024 09:51:12 +0800 Subject: [PATCH 163/282] Remove unused codes (#62134) Remove unused codes --- .../interface/infer_symbolic_shape/paddle_op_infer_sym.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc index cb14bad351274..5663733a26121 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc @@ -1089,9 +1089,6 @@ bool FeedOpInferSymbolicShape(pir::Operation *op, bool TopPSamplingOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - const auto &x_dims = [op, shape_analysis] { const auto &shape_or_data = shape_analysis->GetShapeOrDataForValue(op->operand_source(0)); From cd21bc89afb2a9524a7eef23e5e780ffa2c1b0c3 Mon Sep 17 00:00:00 2001 From: Bo Zhang <105368690+zhangbopd@users.noreply.github.com> Date: Thu, 29 Feb 2024 10:25:38 +0800 Subject: [PATCH 164/282] add all same_operands_and_res ops (#62192) --- .../paddle_op_infer_sym.cc | 384 +----------------- .../paddle_op_infer_sym.h | 136 +------ .../same_operands_and_result.cc | 311 ++++++++++++-- .../same_operands_and_result.h | 155 ++++++- paddle/phi/api/yaml/ops.yaml | 2 + 5 files changed, 433 insertions(+), 555 deletions(-) diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc index 5663733a26121..6f4a4dacd7ba2 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc @@ -1126,36 +1126,7 @@ bool SplitOpInferSymbolicShape(pir::Operation *op, } // Not Impelmented Ops. -bool AcosOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Acos_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool AcoshOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Acosh_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool AngleOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} + bool ArgmaxOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { PADDLE_THROW(phi::errors::Unimplemented( @@ -1168,12 +1139,7 @@ bool ArgminOpInferSymbolicShape( op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); return true; } -bool ArgsortOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} + bool AsComplexOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { PADDLE_THROW(phi::errors::Unimplemented( @@ -1192,72 +1158,7 @@ bool AsStridedOpInferSymbolicShape( op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); return true; } -bool AsinOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Asin_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool AsinhOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Asinh_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool AtanOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Atan_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool AtanhOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Atanh_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool BernoulliOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool BitwiseNotOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool BitwiseNot_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} + bool BitwiseXorOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { PADDLE_THROW(phi::errors::Unimplemented( @@ -1270,54 +1171,14 @@ bool BitwiseXor_OpInferSymbolicShape( op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); return true; } -bool CeilOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Ceil_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} + bool ComplexOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { PADDLE_THROW(phi::errors::Unimplemented( op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); return true; } -bool ConjOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool CosOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Cos_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool CoshOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Cosh_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} + bool CummaxOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { PADDLE_THROW(phi::errors::Unimplemented( @@ -1372,60 +1233,7 @@ bool DirichletOpInferSymbolicShape( op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); return true; } -bool ErfOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Erf_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool ErfinvOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Erfinv_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Expm1OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Expm1_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool FlipOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool FloorOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Floor_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} + bool FmaxOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { PADDLE_THROW(phi::errors::Unimplemented( @@ -1444,36 +1252,7 @@ bool GatherOpInferSymbolicShape( op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); return true; } -bool ImagOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool IsinfOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool IsinfSrOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool IsnanOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool IsnanSrOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} + bool KronOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { PADDLE_THROW(phi::errors::Unimplemented( @@ -1486,30 +1265,7 @@ bool KthvalueOpInferSymbolicShape( op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); return true; } -bool LgammaOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Lgamma_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Log1pOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Log1p_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} + bool LogcumsumexpOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { PADDLE_THROW(phi::errors::Unimplemented( @@ -1540,18 +1296,7 @@ bool LogicalXor_OpInferSymbolicShape( op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); return true; } -bool LogitOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Logit_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} + bool MaskedSelectOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { PADDLE_THROW(phi::errors::Unimplemented( @@ -1576,114 +1321,21 @@ bool PutAlongAxis_OpInferSymbolicShape( op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); return true; } -bool RealOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool RollOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool RoundOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Round_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool ScatterNdAddOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool ScatterOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Scatter_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} + bool SearchsortedOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { PADDLE_THROW(phi::errors::Unimplemented( op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); return true; } -bool SignOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool SinOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Sin_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool SinhOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Sinh_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} + bool TakeAlongAxisOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { PADDLE_THROW(phi::errors::Unimplemented( op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); return true; } -bool TanOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Tan_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool TanhOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Tanh_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} + bool TopkOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { PADDLE_THROW(phi::errors::Unimplemented( @@ -1715,18 +1367,6 @@ bool EmptyOpInferSymbolicShape(pir::Operation *op, op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); return true; } -bool EqualOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Equal_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} bool Exponential_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { PADDLE_THROW(phi::errors::Unimplemented( diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h index ee5bcacf63a1f..a13d93486b140 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h @@ -113,70 +113,26 @@ bool SplitOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); // Not Impelmented Ops. -bool AcosOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Acos_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool AcoshOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Acosh_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool AngleOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); + bool ArgmaxOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool ArgminOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool ArgsortOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + bool AsComplexOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool AsRealOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool AsStridedOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool AsinOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Asin_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool AsinhOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Asinh_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool AtanOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Atan_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool AtanhOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Atanh_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool BernoulliOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool BitwiseNotOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool BitwiseNot_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + bool BitwiseXorOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool BitwiseXor_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool CeilOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Ceil_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); bool ComplexOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool ConjOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool CosOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Cos_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool CoshOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Cosh_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); + bool CummaxOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool CumminOpInferSymbolicShape(pir::Operation *op, @@ -189,58 +145,26 @@ bool CumsumOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool Cumsum_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + bool DiagEmbedOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool DiagonalOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool DirichletOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool ErfOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Erf_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool ErfinvOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Erfinv_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Expm1OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Expm1_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool FlipOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool FloorOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Floor_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); + bool FmaxOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool FminOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool GatherOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool ImagOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool IsinfOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool IsinfSrOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool IsnanOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool IsnanSrOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + bool KronOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool KthvalueOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool LgammaOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Lgamma_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Log1pOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Log1p_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); + bool LogcumsumexpOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool LogicalOrOpInferSymbolicShape( @@ -251,10 +175,7 @@ bool LogicalXorOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool LogicalXor_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool LogitOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Logit_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); + bool MaskedSelectOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool PoissonOpInferSymbolicShape( @@ -263,42 +184,13 @@ bool PutAlongAxisOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool PutAlongAxis_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool RealOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool RollOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool RoundOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Round_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool ScatterNdAddOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool ScatterOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Scatter_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + bool SearchsortedOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool SignOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool SinOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Sin_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool SinhOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Sinh_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); + bool TakeAlongAxisOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool TanOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Tan_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool TanhOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Tanh_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); + bool TopkOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool UnbindOpInferSymbolicShape(pir::Operation *op, @@ -310,10 +202,6 @@ bool EinsumOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool EmptyOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool EqualOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Equal_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); bool Exponential_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool GaussianOpInferSymbolicShape( diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.cc index 98a6d670869ca..31fe14209cc61 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.cc @@ -30,86 +30,258 @@ bool AbsOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); } - bool Abs_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); } - +bool AcosOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Acos_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool AcoshOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Acosh_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool AngleOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool ArgsortOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool AsinOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Asin_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool AsinhOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Asinh_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} bool AssignOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); } - bool Assign_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - return AssignOpInferSymbolicShape(op, shape_analysis); + return SameOperandsAndResultShape(op, shape_analysis); +} +bool AtanOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Atan_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool AtanhOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Atanh_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool BernoulliOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool BitwiseNotOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool BitwiseNot_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); } - bool CastOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); } - bool Cast_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); } - +bool CeilOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Ceil_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool ConjOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool CosOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Cos_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool CoshOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Cosh_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool DigammaOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Digamma_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool EqualOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Equal_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool ErfOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Erf_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool ErfinvOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Erfinv_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} bool ExpOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); } - bool Exp_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); } - +bool Expm1OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Expm1_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} bool FetchOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - shape_analysis->SetShapeOrDataForValue( - op->result(0), - shape_analysis->GetShapeOrDataForValue(op->operand_source(0))); - - return true; + return SameOperandsAndResultShape(op, shape_analysis); +} +bool FlipOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool FloorOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Floor_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool FullWithTensorOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool ImagOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); } - bool IncrementOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); } - bool Increment_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - return IncrementOpInferSymbolicShape(op, shape_analysis); + return SameOperandsAndResultShape(op, shape_analysis); +} +bool IsinfOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool IsinfSrOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool IsnanOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool IsnanSrOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool LgammaOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Lgamma_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Log1pOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Log1p_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); } - bool LogOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); } - bool Log_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - return LogOpInferSymbolicShape(op, shape_analysis); + return SameOperandsAndResultShape(op, shape_analysis); } - bool LogicalNotOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); } - bool LogicalNot_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - return LogicalNotOpInferSymbolicShape(op, shape_analysis); + return SameOperandsAndResultShape(op, shape_analysis); } - -bool FullWithTensorOpInferSymbolicShape( +bool LogitOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Logit_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); } - bool PowOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); @@ -118,17 +290,30 @@ bool Pow_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); } - +bool RealOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} bool ReluOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); } - bool Relu_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); } - +bool RollOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool RoundOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Round_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} bool RsqrtOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); @@ -137,42 +322,92 @@ bool Rsqrt_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); } - bool ScaleOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); } +bool ScaleSrOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool ScaleSr_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} bool Scale_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); } -bool ScaleSrOpInferSymbolicShape( +bool ScatterNdAddOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); } -bool ScaleSr_OpInferSymbolicShape( +bool ScatterOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); } - +bool Scatter_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool SignOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool SinOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Sin_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool SinhOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Sinh_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} bool SubtractOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); } - bool Subtract_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); } - +bool TanOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Tan_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool TanhOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Tanh_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} bool TrilOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); } - bool Tril_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - return TrilOpInferSymbolicShape(op, shape_analysis); + return SameOperandsAndResultShape(op, shape_analysis); +} +bool TruncOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} +bool Trunc_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); } - } // namespace paddle::dialect diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h index d96f4efe1f825..32941dd0c6f78 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h @@ -21,81 +21,194 @@ bool AbsOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool Abs_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - +bool AcosOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Acos_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool AcoshOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Acosh_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool AngleOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool ArgsortOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool AsinOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Asin_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool AsinhOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Asinh_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); bool AssignOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - bool Assign_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - +bool AtanOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Atan_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool AtanhOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Atanh_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool BernoulliOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool BitwiseNotOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool BitwiseNot_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool CastOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool Cast_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - +bool CeilOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Ceil_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool ConjOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool CosOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Cos_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool CoshOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Cosh_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool DigammaOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Digamma_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool EqualOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Equal_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool ErfOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Erf_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool ErfinvOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Erfinv_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool ExpOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool Exp_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - +bool Expm1OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Expm1_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); bool FetchOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - +bool FlipOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool FloorOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Floor_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); bool FullWithTensorOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - +bool ImagOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); bool IncrementOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - bool Increment_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - +bool IsinfOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool IsinfSrOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool IsnanOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool IsnanSrOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool LgammaOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Lgamma_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Log1pOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Log1p_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); bool LogOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - bool Log_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - bool LogicalNotOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - bool LogicalNot_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - +bool LogitOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Logit_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); bool PowOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool Pow_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - +bool RealOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); bool ReluOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool Relu_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - +bool RollOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool RoundOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Round_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); bool RsqrtOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool Rsqrt_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - bool ScaleOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Scale_OpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); bool ScaleSrOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool ScaleSr_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - +bool Scale_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool ScatterNdAddOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool ScatterOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Scatter_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool SignOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool SinOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Sin_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool SinhOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Sinh_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); bool SubtractOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool Subtract_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - +bool TanOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Tan_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool TanhOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Tanh_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); bool TrilOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - bool Tril_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool TruncOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Trunc_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); } // namespace paddle::dialect diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml index cf3986cae89e0..5b8d2132c519d 100755 --- a/paddle/phi/api/yaml/ops.yaml +++ b/paddle/phi/api/yaml/ops.yaml @@ -803,6 +803,7 @@ func : digamma inplace: (x -> out) backward : digamma_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : dirichlet args: (Tensor alpha) @@ -2907,6 +2908,7 @@ func : trunc inplace: (input -> out) backward : trunc_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : unbind args : (Tensor input, int axis = 0) From ee2e49a95365732442df8c7de37436166bad102f Mon Sep 17 00:00:00 2001 From: Galaxy1458 <55453380+Galaxy1458@users.noreply.github.com> Date: Thu, 29 Feb 2024 10:28:01 +0800 Subject: [PATCH 165/282] cinn (#62177) * cinn * fix * update * Update paddle_coverage.sh --- paddle/scripts/paddle_build.sh | 3 +++ tools/coverage/paddle_coverage.sh | 31 +++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 71ee30a115ef7..19e9cf3803a84 100644 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -4235,6 +4235,9 @@ function main() { ;; test) parallel_test + if [ "${WITH_CINN}" == "ON" ] ; then + check_coverage + fi ;; single_test) single_test $2 diff --git a/tools/coverage/paddle_coverage.sh b/tools/coverage/paddle_coverage.sh index ee2a38f5da851..90e02715876ca 100644 --- a/tools/coverage/paddle_coverage.sh +++ b/tools/coverage/paddle_coverage.sh @@ -39,6 +39,28 @@ lcov --capture -d ./ -o coverage.info --rc lcov_branch_coverage=0 # full html report +function gen_full_html_report_cinn(){ + lcov --extract coverage.info \ + '/paddle/paddle/cinn/adt/*' \ + '/paddle/paddle/cinn/api/*' \ + '/paddle/paddle/cinn/ast_gen_ius/*' \ + '/paddle/paddle/cinn/auto_schedule/*' \ + '/paddle/paddle/cinn/backends/*' \ + '/paddle/paddle/cinn/common/*' \ + '/paddle/paddle/cinn/frontend/*' \ + '/paddle/paddle/cinn/hlir/*' \ + '/paddle/paddle/cinn/ir/*' \ + '/paddle/paddle/cinn/lang/*' \ + '/paddle/paddle/cinn/optim/*' \ + '/paddle/paddle/cinn/poly/*' \ + '/paddle/paddle/cinn/pybind/*' \ + '/paddle/paddle/cinn/runtime/*' \ + '/paddle/paddle/cinn/utils/*' \ + -o coverage-full.tmp \ + --rc lcov_branch_coverage=0 +} + + function gen_full_html_report() { lcov --extract coverage.info \ '/paddle/paddle/fluid/framework/*' \ @@ -120,6 +142,12 @@ else gen_full_html_report || true fi +if [ ${WITH_CINN:-OFF} == "ON" ]; then + gen_full_html_report_cinn || true +else + gen_full_html_report || true +fi + # diff html report function gen_diff_html_report() { @@ -222,5 +250,8 @@ fi if [ "$COVERAGE_LINES_ASSERT" = "1" ] || [ "$PYTHON_COVERAGE_LINES_ASSERT" = "1" ]; then echo "exit 9" > /tmp/paddle_coverage.result + if [ "${WITH_CINN}" == "ON" ]; then + echo "You must one RD(liuhongyu or lanxiang or zhenghuihuang or tianchao zhangliujie)to approval this PR." + fi exit 9 fi From 5845c3a615210deb61f22bc2fa240113bdc9b8d5 Mon Sep 17 00:00:00 2001 From: Yichen Zhang <32740647+pkuzyc@users.noreply.github.com> Date: Thu, 29 Feb 2024 10:37:11 +0800 Subject: [PATCH 166/282] add scatter forward spmd rule (#62096) --- paddle/phi/infermeta/spmd_rules/rules.cc | 5 + paddle/phi/infermeta/spmd_rules/rules.h | 1 + paddle/phi/infermeta/spmd_rules/scatter.cc | 169 ++++++++++++++ paddle/phi/infermeta/spmd_rules/scatter.h | 37 ++++ .../spmd_rules/spmd_rule_macro_define.h | 2 +- .../spmd_rules/test_scatter_rule.py | 208 ++++++++++++++++++ 6 files changed, 421 insertions(+), 1 deletion(-) create mode 100644 paddle/phi/infermeta/spmd_rules/scatter.cc create mode 100644 paddle/phi/infermeta/spmd_rules/scatter.h create mode 100644 test/auto_parallel/spmd_rules/test_scatter_rule.py diff --git a/paddle/phi/infermeta/spmd_rules/rules.cc b/paddle/phi/infermeta/spmd_rules/rules.cc index 0921763df1229..aff1633ee2cba 100644 --- a/paddle/phi/infermeta/spmd_rules/rules.cc +++ b/paddle/phi/infermeta/spmd_rules/rules.cc @@ -605,5 +605,10 @@ PD_REGISTER_SPMD_RULE( PD_INFER_SPMD( phi::distributed::FusedLinearParamGradAddInferSpmdFakeReverse)); +// scatter +PD_REGISTER_SPMD_RULE(scatter, + PD_INFER_SPMD(phi::distributed::ScatterInferSpmd), + PD_INFER_SPMD(phi::distributed::ScatterInferSpmdReverse)); + } // namespace distributed } // namespace phi diff --git a/paddle/phi/infermeta/spmd_rules/rules.h b/paddle/phi/infermeta/spmd_rules/rules.h index 03446ca5d2789..ed6a6cbb9641c 100644 --- a/paddle/phi/infermeta/spmd_rules/rules.h +++ b/paddle/phi/infermeta/spmd_rules/rules.h @@ -35,6 +35,7 @@ limitations under the License. */ #include "paddle/phi/infermeta/spmd_rules/reshape.h" #include "paddle/phi/infermeta/spmd_rules/rms_norm.h" #include "paddle/phi/infermeta/spmd_rules/scale.h" +#include "paddle/phi/infermeta/spmd_rules/scatter.h" #include "paddle/phi/infermeta/spmd_rules/slice.h" #include "paddle/phi/infermeta/spmd_rules/softmax.h" #include "paddle/phi/infermeta/spmd_rules/split.h" diff --git a/paddle/phi/infermeta/spmd_rules/scatter.cc b/paddle/phi/infermeta/spmd_rules/scatter.cc new file mode 100644 index 0000000000000..98040cebfa741 --- /dev/null +++ b/paddle/phi/infermeta/spmd_rules/scatter.cc @@ -0,0 +1,169 @@ +/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/phi/infermeta/spmd_rules/scatter.h" + +#include "glog/logging.h" + +#include "paddle/phi/core/distributed/auto_parallel/dist_attr.h" +#include "paddle/phi/core/distributed/auto_parallel/inferspmd_utils.h" +#include "paddle/phi/core/distributed/auto_parallel/utils.h" +#include "paddle/phi/infermeta/spmd_rules/spmd_rule_macro_define.h" +#include "paddle/phi/infermeta/spmd_rules/utils.h" + +namespace phi { +namespace distributed { + +using phi::distributed::auto_parallel::str_join; + +////////////////// Utils Functions ////////////////// + +SpmdInfo ScatterInferSpmd(const DistMetaTensor& x, + const DistMetaTensor& index, + const DistMetaTensor& updates, + bool overwrite) { + // Step0: Verify Input Args Based on Scatter Logic + // extract and check x_ndim, x_shape, x_dist_attr_src and + // x_dims_mapping_src with the macro + EXTRACT_SHAPE_AND_DIST_ATTR(x); + EXTRACT_SHAPE_AND_DIST_ATTR(index); + EXTRACT_SHAPE_AND_DIST_ATTR(updates); + PADDLE_ENFORCE_LE( + index_ndim, + updates_ndim, + phi::errors::InvalidArgument( + "%s (%d): The Index's rank [%d] should be less or equal " + "to Updates' rank [%d].", + __FILE__, + __LINE__, + index_ndim, + updates_ndim)); + + // Step1: Build Einsum Notation + std::string alphabet = "abcdefghijklmnopqrstuvwxyz"; + // x should be replicated on 0th axis + std::string index_axes = GetBroadcastAxes(index_ndim, index_ndim, alphabet); + std::string updates_axes = + GetBroadcastAxes(updates_ndim, updates_ndim, alphabet); + std::string out_axes = GetBroadcastAxes(x_ndim, x_ndim, alphabet); + out_axes[0] = '1'; + + // Step2: Sharding Propogation + // Step2.1: Merge input shardings + std::unordered_map axis_to_dim_map = + ShardingMergeForTensors({{index_axes, index_dims_mapping_src}, + {updates_axes, updates_dims_mapping_src}}); + + std::vector index_dims_mapping = + GetDimsMappingForAxes(index_axes, axis_to_dim_map); + TensorDistAttr index_dist_attr_dst = + CopyTensorDistAttrForOutput(index_dist_attr_src); + index_dist_attr_dst.set_dims_mapping(index_dims_mapping); + + std::vector updates_dims_mapping = + GetDimsMappingForAxes(updates_axes, axis_to_dim_map); + TensorDistAttr updates_dist_attr_dst = + CopyTensorDistAttrForOutput(updates_dist_attr_src); + updates_dist_attr_dst.set_dims_mapping(updates_dims_mapping); + + // Step2.2: Infer output dims mapping + std::vector out_dims_mapping = + GetDimsMappingForAxes(out_axes, axis_to_dim_map); + // the batch axis of output must be replicated + out_dims_mapping[0] = -1; + TensorDistAttr out_dist_attr = CopyTensorDistAttrForOutput(x_dist_attr_src); + out_dist_attr.set_dims_mapping(out_dims_mapping); + + // the dims mapping of x should be the same as output + TensorDistAttr x_dist_attr_dst = CopyTensorDistAttrForOutput(x_dist_attr_src); + x_dist_attr_dst.set_dims_mapping(out_dims_mapping); + + // Step3: Handle partial + // output partial status + // output is partialed if the batch axis of index and updates are sharded + if (updates_dims_mapping[0] != -1) { + std::vector partial_dims(1, updates_dims_mapping[0]); + out_dist_attr.set_partial_status(partial_dims); + } + + VLOG(4) << "index_axes: " << index_axes << " updates_axes: " << updates_axes + << " out_axes: " << out_axes; + LOG_SPMD_INPUT(x); + LOG_SPMD_INPUT(index); + LOG_SPMD_INPUT(updates); + VLOG(4) << "Out dist_attr: [" << out_dist_attr.to_string() << "]"; + return {{x_dist_attr_dst, index_dist_attr_dst, updates_dist_attr_dst}, + {out_dist_attr}}; +} + +SpmdInfo ScatterInferSpmdReverse(const DistMetaTensor& x, + const DistMetaTensor& index, + const DistMetaTensor& updates, + const DistMetaTensor& out, + bool overwrite) { + // Step0: Verify Input Args Based on Scatter Logic + // extract and check out_ndim, out_shape, out_dist_attr_src and + // out_dims_mapping_src with the macro + EXTRACT_SHAPE_AND_DIST_ATTR(x); + EXTRACT_SHAPE_AND_DIST_ATTR(index); + EXTRACT_SHAPE_AND_DIST_ATTR(updates); + EXTRACT_SHAPE_AND_DIST_ATTR(out); + + // Step1: Build Einsum Notation + std::string alphabet = "abcdefghijklmnopqrstuvwxyz"; + // x should be replicated on 0th axis + std::string x_axes = GetBroadcastAxes(x_ndim, x_ndim, alphabet); + std::string index_axes = GetBroadcastAxes(index_ndim, index_ndim, alphabet); + std::string updates_axes = + GetBroadcastAxes(updates_ndim, updates_ndim, alphabet); + std::string out_axes = GetBroadcastAxes(out_ndim, out_ndim, alphabet); + + // Step2: Sharding Propogation + // Step2.1: Merge output shardings + // the batch axis of output must be replicated + // TODO(zhangyichen): consider the case when the output is partial + std::vector out_dims_mapping(out_dims_mapping_src); + out_dims_mapping[0] = -1; + TensorDistAttr out_dist_attr_dst = + CopyTensorDistAttrForOutput(out_dist_attr_src); + out_dist_attr_dst.set_dims_mapping(out_dims_mapping); + std::unordered_map axis_to_dim_map = + ShardingMergeForTensors({{out_axes, out_dims_mapping}}); + + // Step2.2: Infer input dims mapping + std::vector x_dims_mapping = + GetDimsMappingForAxes(x_axes, axis_to_dim_map); + std::vector index_dims_mapping = + GetDimsMappingForAxes(index_axes, axis_to_dim_map); + std::vector updates_dims_mapping = + GetDimsMappingForAxes(updates_axes, axis_to_dim_map); + TensorDistAttr x_dist_attr_dst = CopyTensorDistAttrForOutput(x_dist_attr_src); + x_dist_attr_dst.set_dims_mapping(x_dims_mapping); + TensorDistAttr index_dist_attr_dst = + CopyTensorDistAttrForOutput(index_dist_attr_src); + index_dist_attr_dst.set_dims_mapping(index_dims_mapping); + TensorDistAttr updates_dist_attr_dst = + CopyTensorDistAttrForOutput(updates_dist_attr_src); + updates_dist_attr_dst.set_dims_mapping(updates_dims_mapping); + + LOG_SPMD_INPUT(out); + LOG_SPMD_INPUT(x); + LOG_SPMD_INPUT(index); + LOG_SPMD_INPUT(updates); + return {{x_dist_attr_dst, index_dist_attr_dst, updates_dist_attr_dst}, + {out_dist_attr_dst}}; +} + +} // namespace distributed +} // namespace phi diff --git a/paddle/phi/infermeta/spmd_rules/scatter.h b/paddle/phi/infermeta/spmd_rules/scatter.h new file mode 100644 index 0000000000000..f19bc78261fc7 --- /dev/null +++ b/paddle/phi/infermeta/spmd_rules/scatter.h @@ -0,0 +1,37 @@ +/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include + +#include "paddle/phi/core/distributed/auto_parallel/dist_meta_tensor.h" +#include "paddle/phi/core/distributed/type_defs.h" + +namespace phi { +namespace distributed { + +SpmdInfo ScatterInferSpmd(const DistMetaTensor& x, + const DistMetaTensor& index, + const DistMetaTensor& updates, + bool overwrite); + +SpmdInfo ScatterInferSpmdReverse(const DistMetaTensor& x, + const DistMetaTensor& index, + const DistMetaTensor& updates, + const DistMetaTensor& out, + bool overwrite); + +} // namespace distributed +} // namespace phi diff --git a/paddle/phi/infermeta/spmd_rules/spmd_rule_macro_define.h b/paddle/phi/infermeta/spmd_rules/spmd_rule_macro_define.h index a9d49f3718171..65e90a5850614 100644 --- a/paddle/phi/infermeta/spmd_rules/spmd_rule_macro_define.h +++ b/paddle/phi/infermeta/spmd_rules/spmd_rule_macro_define.h @@ -50,7 +50,7 @@ using phi::distributed::auto_parallel::str_join; VLOG(4) << #name; \ VLOG(4) << "shape: [" << str_join(name##_shape) << "] " \ << "src_dist_attr: [" << name##_dist_attr_src.to_string() << "] " \ - << "src_dist_attr: [" << name##_dist_attr_dst.to_string() << "]"; \ + << "dst_dist_attr: [" << name##_dist_attr_dst.to_string() << "]"; \ } while (0) #define LOG_SPMD_OUTPUT(name) \ diff --git a/test/auto_parallel/spmd_rules/test_scatter_rule.py b/test/auto_parallel/spmd_rules/test_scatter_rule.py new file mode 100644 index 0000000000000..30d1bd444bfff --- /dev/null +++ b/test/auto_parallel/spmd_rules/test_scatter_rule.py @@ -0,0 +1,208 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from collections import OrderedDict + +from paddle.distributed.auto_parallel.static.dist_attribute import ( + DistTensorSpec, + TensorDistAttr, +) +from paddle.distributed.fleet import auto +from paddle.framework import core + + +class TestScatterSPMDRule(unittest.TestCase): + """ + Unit tests for scatter spmd rule. + """ + + def setUp(self): + x_shape = [64, 32, 48] + index_shape = [16] + updates_shape = [32, 32, 48] + process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3]) + self.attrs = OrderedDict() + self.attrs['overwrite'] = True + self.rule = core.get_phi_spmd_rule("scatter") + + x_dist_attr = TensorDistAttr() + x_dist_attr.dims_mapping = [-1, -1, -1] + x_dist_attr.process_mesh = process_mesh + self.x_spec = DistTensorSpec(x_shape, x_dist_attr) + + index_dist_attr = TensorDistAttr() + index_dist_attr.dims_mapping = [-1] + index_dist_attr.process_mesh = process_mesh + self.index_spec = DistTensorSpec(index_shape, index_dist_attr) + + updates_dist_attr = TensorDistAttr() + updates_dist_attr.dims_mapping = [-1, -1, -1] + updates_dist_attr.process_mesh = process_mesh + self.updates_spec = DistTensorSpec(updates_shape, updates_dist_attr) + + def test_single_mesh_dim(self): + # [-1, -1, -1], [-1], [-1, 0, -1] --> [-1, 0, -1], [-1], [-1, 0, -1], [-1, 0, -1] + self.x_spec.set_dims_mapping([-1, -1, -1]) + self.index_spec.set_dims_mapping([-1]) + self.updates_spec.set_dims_mapping([-1, 0, -1]) + + result_dist_attrs = self.rule.infer_forward( + self.x_spec, + self.index_spec, + self.updates_spec, + self.attrs['overwrite'], + ) + infered_input_dist_attrs = result_dist_attrs[0] + infered_output_dist_attrs = result_dist_attrs[1] + self.assertEqual(len(result_dist_attrs), 2) + self.assertEqual(len(infered_input_dist_attrs), 3) + self.assertEqual(len(infered_output_dist_attrs), 1) + + self.assertEqual(infered_input_dist_attrs[0].dims_mapping, [-1, 0, -1]) + self.assertEqual(infered_input_dist_attrs[1].dims_mapping, [-1]) + self.assertEqual(infered_input_dist_attrs[2].dims_mapping, [-1, 0, -1]) + self.assertEqual(infered_output_dist_attrs[0].dims_mapping, [-1, 0, -1]) + self.assertFalse(infered_output_dist_attrs[0]._is_partial()) + + # [0, -1, -1], [-1], [0, -1, -1] --> [-1, -1, -1], [0], [0, -1, -1], [-1, -1, -1] + self.x_spec.set_dims_mapping([0, -1, -1]) + self.index_spec.set_dims_mapping([-1]) + self.updates_spec.set_dims_mapping([0, -1, -1]) + result_dist_attrs = self.rule.infer_forward( + self.x_spec, + self.index_spec, + self.updates_spec, + self.attrs['overwrite'], + ) + infered_input_dist_attrs = result_dist_attrs[0] + infered_output_dist_attrs = result_dist_attrs[1] + self.assertEqual(len(result_dist_attrs), 2) + self.assertEqual(len(infered_input_dist_attrs), 3) + self.assertEqual(len(infered_output_dist_attrs), 1) + + self.assertEqual(infered_input_dist_attrs[0].dims_mapping, [-1, -1, -1]) + self.assertEqual(infered_input_dist_attrs[1].dims_mapping, [0]) + self.assertEqual(infered_input_dist_attrs[2].dims_mapping, [0, -1, -1]) + self.assertEqual( + infered_output_dist_attrs[0].dims_mapping, [-1, -1, -1] + ) + self.assertTrue(infered_output_dist_attrs[0]._is_partial()) + self.assertEqual(infered_output_dist_attrs[0]._partial_dims(), {0}) + + # [-1, 0, -1], [-1], [-1, -1, -1] --> [-1, -1, -1], [-1], [-1, -1, -1], [-1, -1, -1] + self.x_spec.set_dims_mapping([-1, 0, -1]) + self.index_spec.set_dims_mapping([-1]) + self.updates_spec.set_dims_mapping([-1, -1, -1]) + + result_dist_attrs = self.rule.infer_forward( + self.x_spec, + self.index_spec, + self.updates_spec, + self.attrs['overwrite'], + ) + infered_input_dist_attrs = result_dist_attrs[0] + infered_output_dist_attrs = result_dist_attrs[1] + self.assertEqual(len(result_dist_attrs), 2) + self.assertEqual(len(infered_input_dist_attrs), 3) + self.assertEqual(len(infered_output_dist_attrs), 1) + + self.assertEqual(infered_input_dist_attrs[0].dims_mapping, [-1, -1, -1]) + self.assertEqual(infered_input_dist_attrs[1].dims_mapping, [-1]) + self.assertEqual(infered_input_dist_attrs[2].dims_mapping, [-1, -1, -1]) + self.assertEqual( + infered_output_dist_attrs[0].dims_mapping, [-1, -1, -1] + ) + self.assertFalse(infered_output_dist_attrs[0]._is_partial()) + + def test_multi_mesh_dim(self): + process_mesh = auto.ProcessMesh(mesh=[[0, 1, 2], [3, 4, 5]]) + self.x_spec.set_process_mesh(process_mesh) + self.index_spec.set_process_mesh(process_mesh) + self.updates_spec.set_process_mesh(process_mesh) + + # [1, -1, 0], [-1], [-1, 0, -1] --> [-1, 0, -1], [-1], [-1, 0, -1], [-1, 0, -1] + self.x_spec.set_dims_mapping([1, -1, 0]) + self.index_spec.set_dims_mapping([-1]) + self.updates_spec.set_dims_mapping([-1, 0, -1]) + result_dist_attrs = self.rule.infer_forward( + self.x_spec, + self.index_spec, + self.updates_spec, + self.attrs['overwrite'], + ) + infered_input_dist_attrs = result_dist_attrs[0] + infered_output_dist_attrs = result_dist_attrs[1] + self.assertEqual(len(result_dist_attrs), 2) + self.assertEqual(len(infered_input_dist_attrs), 3) + self.assertEqual(len(infered_output_dist_attrs), 1) + + self.assertEqual(infered_input_dist_attrs[0].dims_mapping, [-1, 0, -1]) + self.assertEqual(infered_input_dist_attrs[1].dims_mapping, [-1]) + self.assertEqual(infered_input_dist_attrs[2].dims_mapping, [-1, 0, -1]) + self.assertEqual(infered_output_dist_attrs[0].dims_mapping, [-1, 0, -1]) + + # [-1, -1, -1], [0], [-1, 1, -1] --> [-1, 1, -1], [0], [0, 1, -1], [-1, 0, -1] + self.x_spec.set_dims_mapping([-1, -1, -1]) + self.index_spec.set_dims_mapping([0]) + self.updates_spec.set_dims_mapping([-1, 1, -1]) + result_dist_attrs = self.rule.infer_forward( + self.x_spec, + self.index_spec, + self.updates_spec, + self.attrs['overwrite'], + ) + infered_input_dist_attrs = result_dist_attrs[0] + infered_output_dist_attrs = result_dist_attrs[1] + self.assertEqual(len(result_dist_attrs), 2) + self.assertEqual(len(infered_input_dist_attrs), 3) + self.assertEqual(len(infered_output_dist_attrs), 1) + + self.assertEqual(infered_input_dist_attrs[0].dims_mapping, [-1, 1, -1]) + self.assertEqual(infered_input_dist_attrs[1].dims_mapping, [0]) + self.assertEqual(infered_input_dist_attrs[2].dims_mapping, [0, 1, -1]) + self.assertEqual(infered_output_dist_attrs[0].dims_mapping, [-1, 1, -1]) + self.assertTrue(infered_output_dist_attrs[0]._is_partial()) + self.assertEqual(infered_output_dist_attrs[0]._partial_dims(), {0}) + + def test_reverse_multi_mesh_dim(self): + process_mesh = auto.ProcessMesh(mesh=[[0, 1, 2], [3, 4, 5]]) + self.x_spec.set_process_mesh(process_mesh) + self.index_spec.set_process_mesh(process_mesh) + self.updates_spec.set_process_mesh(process_mesh) + self.out_spec = DistTensorSpec(self.x_spec) + + # [1, 0, -1] --> [-1, 0, -1], [-1], [-1, 0, -1], [-1, 0, -1] + self.out_spec.set_dims_mapping([1, 0, -1]) + result_dist_attrs = self.rule.infer_backward( + self.x_spec, + self.index_spec, + self.updates_spec, + self.out_spec, + self.attrs['overwrite'], + ) + infered_input_dist_attrs = result_dist_attrs[0] + infered_output_dist_attrs = result_dist_attrs[1] + self.assertEqual(len(result_dist_attrs), 2) + self.assertEqual(len(infered_input_dist_attrs), 3) + self.assertEqual(len(infered_output_dist_attrs), 1) + + self.assertEqual(infered_input_dist_attrs[0].dims_mapping, [-1, 0, -1]) + self.assertEqual(infered_input_dist_attrs[1].dims_mapping, [-1]) + self.assertEqual(infered_input_dist_attrs[2].dims_mapping, [-1, 0, -1]) + self.assertEqual(infered_output_dist_attrs[0].dims_mapping, [-1, 0, -1]) + + +if __name__ == "__main__": + unittest.main() From 4ee98e71845c3ae1f3266afd1ab03f071bec9e1f Mon Sep 17 00:00:00 2001 From: NeroLoh <745827440@qq.com> Date: Thu, 29 Feb 2024 10:45:13 +0800 Subject: [PATCH 167/282] [XPU] add roformer relative embedding pass & kernel and spport in multi_encoder_xpu (#62089) --- paddle/fluid/framework/ir/CMakeLists.txt | 2 + ...i_encoder_xpu_adaptive_seqlen_fuse_pass.cc | 48 +-- ...ti_encoder_xpu_adaptive_seqlen_fuse_pass.h | 6 +- .../ir/xpu/multi_encoder_xpu_fuse_pass.cc | 300 +++++++++++++++-- .../ir/xpu/multi_encoder_xpu_fuse_pass.h | 4 +- .../ir/xpu/roformer_relative_pos_fuse_pass.cc | 301 ++++++++++++++++++ .../inference/api/paddle_pass_builder.cc | 1 + paddle/phi/api/yaml/fused_ops.yaml | 11 +- paddle/phi/backends/xpu/xpu2_op_list.cc | 2 + paddle/phi/infermeta/fusion.cc | 54 ++++ paddle/phi/infermeta/fusion.h | 7 + .../fusion/xpu/multi_encoder_xpu_kernel.cc | 35 +- .../xpu/roformer_relative_embedding_kernel.cc | 78 +++++ .../test_xpu_roformer_relative_pos_pass.py | 167 ++++++++++ 14 files changed, 969 insertions(+), 47 deletions(-) create mode 100644 paddle/fluid/framework/ir/xpu/roformer_relative_pos_fuse_pass.cc create mode 100644 paddle/phi/kernels/fusion/xpu/roformer_relative_embedding_kernel.cc create mode 100644 test/ir/inference/test_xpu_roformer_relative_pos_pass.py diff --git a/paddle/fluid/framework/ir/CMakeLists.txt b/paddle/fluid/framework/ir/CMakeLists.txt index 765fa1779b0e5..cb8093298d9bb 100755 --- a/paddle/fluid/framework/ir/CMakeLists.txt +++ b/paddle/fluid/framework/ir/CMakeLists.txt @@ -322,6 +322,8 @@ if(WITH_XPU) ${XPU_PASS_DEPS}) pass_library(sine_pos_fuse_pass inference DIR xpu DEPS ${XPU_PASS_DEPS}) pass_library(quant_dequant_xpu_pass inference DIR xpu DEPS ${XPU_PASS_DEPS}) + pass_library(roformer_relative_pos_fuse_pass inference DIR xpu DEPS + ${XPU_PASS_DEPS}) endif() cc_library( diff --git a/paddle/fluid/framework/ir/xpu/multi_encoder_xpu_adaptive_seqlen_fuse_pass.cc b/paddle/fluid/framework/ir/xpu/multi_encoder_xpu_adaptive_seqlen_fuse_pass.cc index e20320e29a959..fa75f29ae9187 100644 --- a/paddle/fluid/framework/ir/xpu/multi_encoder_xpu_adaptive_seqlen_fuse_pass.cc +++ b/paddle/fluid/framework/ir/xpu/multi_encoder_xpu_adaptive_seqlen_fuse_pass.cc @@ -25,7 +25,9 @@ namespace ir { namespace patterns { struct AdaptiveSeqlenPatternV1 : public PatternBase { - AdaptiveSeqlenPatternV1(PDPattern* pattern, const std::string& name_scope); + AdaptiveSeqlenPatternV1(PDPattern* pattern, + const std::string& name_scope, + const std::string& matmul_type); // declare operator node's name PATTERN_DECL_NODE(embedding_xpu); @@ -44,7 +46,8 @@ struct AdaptiveSeqlenPatternV1 : public PatternBase { }; AdaptiveSeqlenPatternV1::AdaptiveSeqlenPatternV1(PDPattern* pattern, - const std::string& name_scope) + const std::string& name_scope, + const std::string& matmul_type) : PatternBase(pattern, name_scope, name_scope) { auto* embedding_xpu = pattern->NewNode(embedding_xpu_repr()) ->assert_is_op("embedding_with_eltwise_add_xpu"); @@ -59,11 +62,11 @@ AdaptiveSeqlenPatternV1::AdaptiveSeqlenPatternV1(PDPattern* pattern, ->assert_is_op_input("multi_encoder_xpu", "x"); auto* mask = pattern->NewNode(mask_repr()) - ->assert_is_op_input("matmul", "X") - ->assert_is_op_input("matmul", "Y"); - auto* matmul = pattern->NewNode(matmul_repr())->assert_is_op("matmul"); + ->assert_is_op_input(matmul_type, "X") + ->assert_is_op_input(matmul_type, "Y"); + auto* matmul = pattern->NewNode(matmul_repr())->assert_is_op(matmul_type); auto* matmul_out = pattern->NewNode(matmul_out_repr()) - ->assert_is_op_output("matmul", "Out") + ->assert_is_op_output(matmul_type, "Out") ->assert_is_op_input("scale", "X"); auto* scale = pattern->NewNode(scale_repr())->assert_is_op("scale"); auto* scale_out = pattern->NewNode(scale_out_repr()) @@ -88,9 +91,10 @@ AdaptiveSeqlenPatternV1::AdaptiveSeqlenPatternV1(PDPattern* pattern, } // namespace patterns int MultiEncoderXPUAdaptiveSeqlenFusePass::ApplyAdaptiveSeqlenPassV1( - ir::Graph* graph) const { + ir::Graph* graph, const std::string& matmul_type) const { GraphPatternDetector gpd; - patterns::AdaptiveSeqlenPatternV1 pattern(gpd.mutable_pattern(), name_scope_); + patterns::AdaptiveSeqlenPatternV1 pattern( + gpd.mutable_pattern(), name_scope_, matmul_type); int found_subgraph_count = 0; auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph, @@ -143,7 +147,9 @@ int MultiEncoderXPUAdaptiveSeqlenFusePass::ApplyAdaptiveSeqlenPassV1( namespace patterns { struct AdaptiveSeqlenPatternV2 : public PatternBase { - AdaptiveSeqlenPatternV2(PDPattern* pattern, const std::string& name_scope); + AdaptiveSeqlenPatternV2(PDPattern* pattern, + const std::string& name_scope, + const std::string& matmul_type); // declare operator node's name PATTERN_DECL_NODE(embedding_xpu); @@ -172,7 +178,8 @@ struct AdaptiveSeqlenPatternV2 : public PatternBase { }; AdaptiveSeqlenPatternV2::AdaptiveSeqlenPatternV2(PDPattern* pattern, - const std::string& name_scope) + const std::string& name_scope, + const std::string& matmul_type) : PatternBase(pattern, name_scope, name_scope) { auto* embedding_xpu = pattern->NewNode(embedding_xpu_repr()) ->assert_is_op("embedding_with_eltwise_add_xpu"); @@ -201,11 +208,11 @@ AdaptiveSeqlenPatternV2::AdaptiveSeqlenPatternV2(PDPattern* pattern, pattern->NewNode(unsqueeze_0_repr())->assert_is_op("unsqueeze2"); auto* unsqueeze_0_out = pattern->NewNode(unsqueeze_0_out_repr()) ->assert_is_op_output("unsqueeze2", "Out") - ->assert_is_op_input("matmul_v2", "X") - ->assert_is_op_input("matmul_v2", "Y"); - auto* matmul = pattern->NewNode(matmul_repr())->assert_is_op("matmul_v2"); + ->assert_is_op_input(matmul_type, "X") + ->assert_is_op_input(matmul_type, "Y"); + auto* matmul = pattern->NewNode(matmul_repr())->assert_is_op(matmul_type); auto* matmul_out = pattern->NewNode(matmul_out_repr()) - ->assert_is_op_output("matmul_v2", "Out") + ->assert_is_op_output(matmul_type, "Out") ->assert_is_op_input("scale", "X"); auto* scale_0 = pattern->NewNode(scale_0_repr())->assert_is_op("scale"); auto* scale_0_out = pattern->NewNode(scale_0_out_repr()) @@ -244,9 +251,10 @@ AdaptiveSeqlenPatternV2::AdaptiveSeqlenPatternV2(PDPattern* pattern, } // namespace patterns int MultiEncoderXPUAdaptiveSeqlenFusePass::ApplyAdaptiveSeqlenPassV2( - ir::Graph* graph) const { + ir::Graph* graph, const std::string& matmul_type) const { GraphPatternDetector gpd; - patterns::AdaptiveSeqlenPatternV2 pattern(gpd.mutable_pattern(), name_scope_); + patterns::AdaptiveSeqlenPatternV2 pattern( + gpd.mutable_pattern(), name_scope_, matmul_type); int found_subgraph_count = 0; auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph, @@ -324,9 +332,13 @@ void MultiEncoderXPUAdaptiveSeqlenFusePass::ApplyImpl(ir::Graph* graph) const { PADDLE_ENFORCE_NOT_NULL( graph, platform::errors::PreconditionNotMet("graph should not be null.")); Init(name_scope_, graph); + std::vector matmul_types{"matmul", "matmul_v2"}; + int found_subgraph_count = 0; + for (auto& matmul_type : matmul_types) { + found_subgraph_count += ApplyAdaptiveSeqlenPassV1(graph, matmul_type); + found_subgraph_count += ApplyAdaptiveSeqlenPassV2(graph, matmul_type); + } - int found_subgraph_count = ApplyAdaptiveSeqlenPassV1(graph); - found_subgraph_count += ApplyAdaptiveSeqlenPassV2(graph); AddStatis(found_subgraph_count); } diff --git a/paddle/fluid/framework/ir/xpu/multi_encoder_xpu_adaptive_seqlen_fuse_pass.h b/paddle/fluid/framework/ir/xpu/multi_encoder_xpu_adaptive_seqlen_fuse_pass.h index 22910c2120530..ea3b52bf35a24 100644 --- a/paddle/fluid/framework/ir/xpu/multi_encoder_xpu_adaptive_seqlen_fuse_pass.h +++ b/paddle/fluid/framework/ir/xpu/multi_encoder_xpu_adaptive_seqlen_fuse_pass.h @@ -76,7 +76,8 @@ class MultiEncoderXPUAdaptiveSeqlenFusePass : public FusePassBase { | out_var* */ - int ApplyAdaptiveSeqlenPassV1(ir::Graph* graph) const; + int ApplyAdaptiveSeqlenPassV1(ir::Graph* graph, + const std::string& matmul_type) const; /* adaptive seqlen V2, before: @@ -132,7 +133,8 @@ class MultiEncoderXPUAdaptiveSeqlenFusePass : public FusePassBase { | out_var* */ - int ApplyAdaptiveSeqlenPassV2(ir::Graph* graph) const; + int ApplyAdaptiveSeqlenPassV2(ir::Graph* graph, + const std::string& matmul_type) const; private: const std::string name_scope_{"multi_encoder_xpu_adaptive_seqlen_fuse_pass"}; diff --git a/paddle/fluid/framework/ir/xpu/multi_encoder_xpu_fuse_pass.cc b/paddle/fluid/framework/ir/xpu/multi_encoder_xpu_fuse_pass.cc index 8e126df64ad41..e7a5acac2bae2 100644 --- a/paddle/fluid/framework/ir/xpu/multi_encoder_xpu_fuse_pass.cc +++ b/paddle/fluid/framework/ir/xpu/multi_encoder_xpu_fuse_pass.cc @@ -38,7 +38,8 @@ struct SingleEncoderXPUPattern : public PatternBase { bool norm_before, bool with_q_scale, bool with_mask, - bool is_smooth_quant); + bool is_smooth_quant, + const std::string& relative_type); // declare operator node's name // If norm_before, use ln_0 & ln_1. @@ -141,6 +142,16 @@ struct SingleEncoderXPUPattern : public PatternBase { PATTERN_DECL_NODE(smooth_scale_1_out); PATTERN_DECL_NODE(smooth_scale_2_out); + // roformer_relative_embedding_xpu + PATTERN_DECL_NODE(q_relative_emb); + PATTERN_DECL_NODE(q_cos_embedding); + PATTERN_DECL_NODE(q_sin_embedding); + PATTERN_DECL_NODE(q_relative_emb_out); + PATTERN_DECL_NODE(k_relative_emb); + PATTERN_DECL_NODE(k_cos_embedding); + PATTERN_DECL_NODE(k_sin_embedding); + PATTERN_DECL_NODE(k_relative_emb_out); + private: std::string act_type_; std::string matmul_type_0_; @@ -150,6 +161,7 @@ struct SingleEncoderXPUPattern : public PatternBase { bool with_q_scale_{false}; bool with_mask_{true}; bool is_smooth_quant_{false}; + std::string relative_type_ = ""; }; SingleEncoderXPUPattern::SingleEncoderXPUPattern( @@ -162,7 +174,8 @@ SingleEncoderXPUPattern::SingleEncoderXPUPattern( bool norm_before, bool with_q_scale, bool with_mask, - bool is_smooth_quant) + bool is_smooth_quant, + const std::string& relative_type) : PatternBase(pattern, name_scope, name_scope), act_type_(act_type), matmul_type_0_(matmul_type_0), @@ -171,7 +184,8 @@ SingleEncoderXPUPattern::SingleEncoderXPUPattern( norm_before_(norm_before), with_q_scale_(with_q_scale), with_mask_(with_mask), - is_smooth_quant_(is_smooth_quant) { + is_smooth_quant_(is_smooth_quant), + relative_type_(relative_type) { // layer_norm 0 PDNode* ln_0_x = pattern->NewNode(ln_0_x_repr()); PDNode* ln_0_bias = nullptr; @@ -244,14 +258,38 @@ SingleEncoderXPUPattern::SingleEncoderXPUPattern( ->assert_var_not_persistable(); PDNode* q_scale = nullptr; PDNode* q_scale_out = nullptr; + std::string target_op_type = matmul_type_1_; if (with_q_scale_) { q_scale = pattern->NewNode(q_scale_repr())->assert_is_op("scale"); q_scale_out = pattern->NewNode(q_scale_out_repr()) ->assert_is_op_output("scale", "Out") ->assert_is_op_input(matmul_type_1_, "X") ->assert_var_not_persistable(); + target_op_type = "scale"; } else { - q_transpose_out->assert_is_op_input(matmul_type_1_, "X"); + if (relative_type_.empty()) { + q_transpose_out->assert_is_op_input(target_op_type, "X"); + } else { + q_transpose_out->assert_is_op_input(relative_type_, "x"); + } + } + PDNode* q_relative_emb = nullptr; + PDNode* q_cos_embedding = nullptr; + PDNode* q_sin_embedding = nullptr; + PDNode* q_relative_emb_out = nullptr; + if (relative_type_ == "roformer_relative_embedding_xpu") { + VLOG(3) << "build q_relative_emb"; + q_relative_emb = + pattern->NewNode(q_relative_emb_repr())->assert_is_op(relative_type_); + q_sin_embedding = pattern->NewNode(q_sin_embedding_repr()) + ->assert_is_op_input(relative_type_, "sin_emb") + ->AsInput(); + q_cos_embedding = pattern->NewNode(q_cos_embedding_repr()) + ->assert_is_op_input(relative_type_, "cos_emb") + ->AsInput(); + q_relative_emb_out = pattern->NewNode(q_relative_emb_out_repr()) + ->assert_is_op_output(relative_type_, "out") + ->assert_is_op_input(target_op_type, "X"); } // k: matmul + add + reshape + transpose @@ -279,9 +317,23 @@ SingleEncoderXPUPattern::SingleEncoderXPUPattern( pattern->NewNode(k_transpose_repr())->assert_is_op("transpose2"); auto* k_transpose_out = pattern->NewNode(k_transpose_out_repr()) ->assert_is_op_output("transpose2", "Out") - ->assert_is_op_input(matmul_type_1_, "Y") ->assert_var_not_persistable(); + PDNode* k_relative_emb = nullptr; + PDNode* k_sin_embedding = q_sin_embedding; + PDNode* k_cos_embedding = q_cos_embedding; + PDNode* k_relative_emb_out = nullptr; + if (relative_type_.empty()) { + k_transpose_out->assert_is_op_input(matmul_type_1_, "Y"); + } else if (relative_type_ == "roformer_relative_embedding_xpu") { + VLOG(3) << "build k_relative_emb"; + k_transpose_out->assert_is_op_input(relative_type_, "x"); + k_relative_emb = + pattern->NewNode(k_relative_emb_repr())->assert_is_op(relative_type_); + k_relative_emb_out = pattern->NewNode(k_relative_emb_out_repr()) + ->assert_is_op_output(relative_type_, "out") + ->assert_is_op_input(matmul_type_1_, "Y"); + } // qk: matmul + add + softmax auto* qk_matmul = pattern->NewNode(qk_matmul_repr())->assert_is_op(matmul_type_1_); @@ -482,18 +534,31 @@ SingleEncoderXPUPattern::SingleEncoderXPUPattern( q_add->LinksFrom({q_matmul_out, q_add_bias}).LinksTo({q_add_out}); q_reshape->LinksFrom({q_add_out}).LinksTo({q_reshape_out}); q_transpose->LinksFrom({q_reshape_out}).LinksTo({q_transpose_out}); - PDNode* qk_matmul_x = q_transpose_out; + PDNode* last_node = q_transpose_out; + if (relative_type_ == "roformer_relative_embedding_xpu") { + VLOG(3) << "build q_relative_emb link"; + q_relative_emb->LinksFrom({last_node, q_sin_embedding, q_cos_embedding}) + .LinksTo({q_relative_emb_out}); + last_node = q_relative_emb_out; + } if (with_q_scale_) { - q_scale->LinksFrom({q_transpose_out}).LinksTo({q_scale_out}); - qk_matmul_x = q_scale_out; + q_scale->LinksFrom({last_node}).LinksTo({q_scale_out}); + last_node = q_scale_out; } + PDNode* qk_matmul_x = last_node; k_matmul->LinksFrom({q_matmul_x, k_matmul_w}).LinksTo({k_matmul_out}); k_add->LinksFrom({k_matmul_out, k_add_bias}).LinksTo({k_add_out}); k_reshape->LinksFrom({k_add_out}).LinksTo({k_reshape_out}); k_transpose->LinksFrom({k_reshape_out}).LinksTo({k_transpose_out}); - - qk_matmul->LinksFrom({qk_matmul_x, k_transpose_out}).LinksTo({qk_matmul_out}); + last_node = k_transpose_out; + if (relative_type_ == "roformer_relative_embedding_xpu") { + VLOG(3) << "build k_relative_emb link"; + k_relative_emb->LinksFrom({last_node, k_sin_embedding, k_cos_embedding}) + .LinksTo({k_relative_emb_out}); + last_node = k_relative_emb_out; + } + qk_matmul->LinksFrom({qk_matmul_x, last_node}).LinksTo({qk_matmul_out}); PDNode* qk_softmax_x = qk_matmul_out; if (with_mask_) { qk_add->LinksFrom({qk_matmul_out, qk_add_mask}).LinksTo({qk_add_out}); @@ -571,7 +636,8 @@ void MultiEncoderXPUFusePass::ApplyImpl(ir::Graph* graph) const { pattern_param.norm_before, pattern_param.with_q_scale, pattern_param.with_mask, - pattern_param.is_smooth_quant); + pattern_param.is_smooth_quant, + pattern_param.relative_type); while (ApplyMultiEncoderXPUFuse(graph)) { multi_encoder_fused_counts++; } @@ -950,7 +1016,8 @@ int MultiEncoderXPUFusePass::ApplySingleEncoderXPUFuse( bool norm_before, bool with_q_scale, bool with_mask, - bool is_smooth_quant) const { + bool is_smooth_quant, + const std::string& relative_type) const { bool local_quant = false; if (std::getenv("XPU_LOCAL_QUANT")) { local_quant = atoi(std::getenv("XPU_LOCAL_QUANT")); @@ -965,7 +1032,8 @@ int MultiEncoderXPUFusePass::ApplySingleEncoderXPUFuse( norm_before, with_q_scale, with_mask, - is_smooth_quant); + is_smooth_quant, + relative_type); int found_subgraph_count = 0; auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph, @@ -1068,6 +1136,16 @@ int MultiEncoderXPUFusePass::ApplySingleEncoderXPUFuse( GET_IR_NODE(smooth_scale_1_out); GET_IR_NODE(smooth_scale_2_out); + // roformer_relative_embedding_xpu + GET_IR_NODE(q_relative_emb); + GET_IR_NODE(q_cos_embedding); + GET_IR_NODE(q_sin_embedding); + GET_IR_NODE(q_relative_emb_out); + GET_IR_NODE(k_relative_emb); + GET_IR_NODE(k_cos_embedding); + GET_IR_NODE(k_sin_embedding); + GET_IR_NODE(k_relative_emb_out); + auto* block = q_matmul->Op()->Block(); auto* scope = param_scope(); auto weight_dtype = @@ -1275,6 +1353,24 @@ int MultiEncoderXPUFusePass::ApplySingleEncoderXPUFuse( op_desc.SetAttr("relative_type", static_cast(0)); op_desc.SetAttr("use_precision", use_precision); op_desc.SetAttr("is_per_channel", is_per_channel); + if (relative_type == "roformer_relative_embedding_xpu") { + // q/k share the rotary embedding + op_desc.SetInput("roformer_embedding", + {q_cos_embedding->Name(), q_sin_embedding->Name()}); + op_desc.SetAttr("relative_type", 1); + auto q_cos_emb_shape = q_cos_embedding->Var()->GetShape(); + CHECK_GE(static_cast(q_cos_emb_shape.size()), 2) + << q_cos_emb_shape.size(); + auto size_per_head = q_reshape_out->Var()->GetShape()[3]; + CHECK_EQ(size_per_head, q_cos_emb_shape[q_cos_emb_shape.size() - 1]); + int max_pos_len = q_cos_emb_shape[q_cos_emb_shape.size() - 2]; + VLOG(3) << "relative embedding max sequence len: " << max_pos_len; + op_desc.SetAttr("max_pos_len", max_pos_len); + } else { + op_desc.SetInput("roformer_embedding", {}); + op_desc.SetAttr("max_pos_len", 0); + } + // if quant,skip softmax,and use qk_matmul out_threshold as softmax_max auto softmax_max_name = qk_matmul->Op()->Output("Out")[0]; if (var_quant_scales.find(softmax_max_name) != var_quant_scales.end()) { @@ -1320,6 +1416,10 @@ int MultiEncoderXPUFusePass::ApplySingleEncoderXPUFuse( IR_NODE_LINK_TO(smooth_scale_1_weight, single_encoder_xpu); IR_NODE_LINK_TO(smooth_scale_2_weight, single_encoder_xpu); } + if (relative_type == "roformer_relative_embedding_xpu") { + IR_NODE_LINK_TO(q_cos_embedding, single_encoder_xpu); + IR_NODE_LINK_TO(q_sin_embedding, single_encoder_xpu); + } // Delete nodes std::unordered_set delete_nodes{ln_1, @@ -1405,6 +1505,12 @@ int MultiEncoderXPUFusePass::ApplySingleEncoderXPUFuse( delete_nodes.insert(smooth_scale_1_out); delete_nodes.insert(smooth_scale_2_out); } + if (relative_type == "roformer_relative_embedding_xpu") { + delete_nodes.insert(q_relative_emb); + delete_nodes.insert(q_relative_emb_out); + delete_nodes.insert(k_relative_emb); + delete_nodes.insert(k_relative_emb_out); + } GraphSafeRemoveNodes(graph, delete_nodes); found_subgraph_count++; }; @@ -1453,7 +1559,8 @@ bool MultiEncoderXPUFusePass::ApplyMultiEncoderXPUFuse(ir::Graph* graph) const { "fc_bias", "ln_scale", "ln_bias", - "smooth_scale_weight"}; + "smooth_scale_weight", + "roformer_embedding"}; std::map> arg_names_map; std::string mask_name = single_encoders[0]->Op()->Inputs().count("mask") > 0 ? single_encoders[0]->Op()->Inputs().at("mask")[0] @@ -1556,6 +1663,11 @@ bool MultiEncoderXPUFusePass::ApplyMultiEncoderXPUFuse(ir::Graph* graph) const { quant_types.end(), per_quant_types.begin(), per_quant_types.end()); } op_desc.SetAttr("quant_types", quant_types); + if (single_encoders[0]->Op()->HasAttr("max_pos_len")) { + op_desc.SetAttr("max_pos_len", + PADDLE_GET_CONST( + int, single_encoders[0]->Op()->GetAttr("max_pos_len"))); + } op_desc.SetOutput("out", {out_name}); op_desc.SetOutput("x_fp16", {x_fp16_name}); op_desc.SetOutput("out_fp16", {out_fp16_name}); @@ -1642,15 +1754,157 @@ std::vector MultiEncoderXPUFusePass::GeneratePatternParams() const { return std::vector{ // Params are arranged in alphabetic order - {"gelu", "matmul_v2", "matmul", "matmul_v2", false, false, true, false}, - {"gelu", "matmul_v2", "matmul_v2", "matmul_v2", false, true, true, false}, - {"gelu", "mul", "matmul", "matmul", false, true, true, false}, - {"relu", "mul", "matmul", "matmul", false, true, true, false}, - - {"gelu", "matmul_v2", "matmul", "matmul_v2", false, false, true, true}, - {"gelu", "matmul_v2", "matmul_v2", "matmul_v2", false, true, true, true}, - {"gelu", "mul", "matmul", "matmul", false, true, true, true}, - {"relu", "mul", "matmul", "matmul", false, true, true, true}, + {"gelu", + "matmul_v2", + "matmul", + "matmul_v2", + false, + false, + true, + false, + ""}, + {"gelu", + "matmul_v2", + "matmul_v2", + "matmul_v2", + false, + true, + true, + false, + ""}, + {"gelu", "mul", "matmul", "matmul", false, true, true, false, ""}, + {"relu", "mul", "matmul", "matmul", false, true, true, false, ""}, + {"relu", + "matmul_v2", + "matmul_v2", + "matmul_v2", + false, + true, + true, + false, + ""}, + + {"gelu", + "matmul_v2", + "matmul", + "matmul_v2", + false, + false, + true, + true, + ""}, + {"gelu", + "matmul_v2", + "matmul_v2", + "matmul_v2", + false, + true, + true, + true, + ""}, + {"gelu", "mul", "matmul", "matmul", false, true, true, true, ""}, + {"relu", "mul", "matmul", "matmul", false, true, true, true, ""}, + {"relu", + "matmul_v2", + "matmul_v2", + "matmul_v2", + false, + true, + true, + true, + ""}, + + {"gelu", + "matmul_v2", + "matmul", + "matmul_v2", + false, + false, + true, + false, + "roformer_relative_embedding_xpu"}, + {"gelu", + "matmul_v2", + "matmul_v2", + "matmul_v2", + false, + true, + true, + false, + "roformer_relative_embedding_xpu"}, + {"gelu", + "mul", + "matmul", + "matmul", + false, + true, + true, + false, + "roformer_relative_embedding_xpu"}, + {"relu", + "mul", + "matmul", + "matmul", + false, + true, + true, + false, + "roformer_relative_embedding_xpu"}, + {"relu", + "matmul_v2", + "matmul_v2", + "matmul_v2", + false, + true, + true, + false, + "roformer_relative_embedding_xpu"}, + + {"gelu", + "matmul_v2", + "matmul", + "matmul_v2", + false, + false, + true, + true, + "roformer_relative_embedding_xpu"}, + {"gelu", + "matmul_v2", + "matmul_v2", + "matmul_v2", + false, + true, + true, + true, + "roformer_relative_embedding_xpu"}, + {"gelu", + "mul", + "matmul", + "matmul", + false, + true, + true, + true, + "roformer_relative_embedding_xpu"}, + {"relu", + "mul", + "matmul", + "matmul", + false, + true, + true, + true, + "roformer_relative_embedding_xpu"}, + {"relu", + "matmul_v2", + "matmul_v2", + "matmul_v2", + false, + true, + true, + true, + "roformer_relative_embedding_xpu"}, }; } diff --git a/paddle/fluid/framework/ir/xpu/multi_encoder_xpu_fuse_pass.h b/paddle/fluid/framework/ir/xpu/multi_encoder_xpu_fuse_pass.h index 6c45838073af6..238f7d8d419c5 100644 --- a/paddle/fluid/framework/ir/xpu/multi_encoder_xpu_fuse_pass.h +++ b/paddle/fluid/framework/ir/xpu/multi_encoder_xpu_fuse_pass.h @@ -129,6 +129,7 @@ struct PatternParam { bool with_q_scale; bool with_mask; bool is_smooth_quant; + std::string relative_type; }; class MultiEncoderXPUFusePass : public FusePassBase { @@ -144,7 +145,8 @@ class MultiEncoderXPUFusePass : public FusePassBase { bool norm_before, bool with_q_scale, bool with_mask, - bool is_smooth_quant) const; + bool is_smooth_qunat, + const std::string& relative_type) const; bool ApplyMultiEncoderXPUFuse(ir::Graph* graph) const; diff --git a/paddle/fluid/framework/ir/xpu/roformer_relative_pos_fuse_pass.cc b/paddle/fluid/framework/ir/xpu/roformer_relative_pos_fuse_pass.cc new file mode 100644 index 0000000000000..2c50c77cad8d7 --- /dev/null +++ b/paddle/fluid/framework/ir/xpu/roformer_relative_pos_fuse_pass.cc @@ -0,0 +1,301 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include "glog/logging.h" + +#include "paddle/fluid/framework/ir/fuse_pass_base.h" +#include "paddle/fluid/framework/ir/graph_pattern_detector.h" +#include "paddle/fluid/framework/ir/pass.h" +#include "paddle/fluid/framework/ir/quantize_helper.h" +#include "paddle/fluid/framework/ir/xpu/pass_utils.h" +#include "paddle/fluid/framework/ir/xpu/quant_utils.h" +#include "paddle/fluid/framework/op_version_registry.h" +#include "paddle/fluid/platform/enforce.h" + +namespace phi { +class DenseTensor; +} // namespace phi + +namespace paddle { +namespace framework { +class Scope; +} // namespace framework +} // namespace paddle + +namespace paddle { +namespace framework { +namespace ir { +namespace patterns { +/* +fuse block in vis model to reformer_relative_pos_xpu op +------------------------------------------------------ */ +/* support xpu roformer relative pos */ +/* x --------------- */ +/* | \ | */ +/* | \ | */ +/* split shape | */ +/* / | \ | */ +/* / | \ | */ +/* | scale slice | */ +/* \ | / \ | */ +/* \ | / \ | */ +/* concat slice slice | */ +/* | / \ | */ +/* | / \ | */ +/* elementwise_mul elementwise_mul */ +/* | / */ +/* | / */ +/* elementwise_add */ +/* | */ +/* | */ +/* out */ +/*-------------------------------------------*/ +/* After the pass apply: */ +/* x */ +/* cos_emb | sin_emb */ +/* \ | / */ +/* xpu_roformer_relative */ +/* | */ +/* | */ +/* out */ +/*-------------------------------------------*/ + +struct RoformerRelativePosXPUPattern : public PatternBase { + RoformerRelativePosXPUPattern(PDPattern* pattern, + const std::string& name_scope); + // declare operator node's name + PATTERN_DECL_NODE(split); + PATTERN_DECL_NODE(scale); + PATTERN_DECL_NODE(concat); + PATTERN_DECL_NODE(mul1); + + PATTERN_DECL_NODE(shape); + PATTERN_DECL_NODE(slice1); + PATTERN_DECL_NODE(slice_sin); + PATTERN_DECL_NODE(slice_cos); + + PATTERN_DECL_NODE(mul2); + PATTERN_DECL_NODE(add); + // declare variable node's name + PATTERN_DECL_NODE(x); + PATTERN_DECL_NODE(sin_emb); + PATTERN_DECL_NODE(cos_emb); + PATTERN_DECL_NODE(split_out1); + PATTERN_DECL_NODE(split_out2); + PATTERN_DECL_NODE(scale_out); + PATTERN_DECL_NODE(concat_out); + PATTERN_DECL_NODE(mul1_out); + PATTERN_DECL_NODE(shape_out); + PATTERN_DECL_NODE(slice1_out); + PATTERN_DECL_NODE(slice_sin_out); + PATTERN_DECL_NODE(slice_cos_out); + PATTERN_DECL_NODE(mul2_out); + PATTERN_DECL_NODE(add_out); +}; + +RoformerRelativePosXPUPattern::RoformerRelativePosXPUPattern( + PDPattern* pattern, const std::string& name_scope) + : PatternBase(pattern, name_scope, name_scope) { + auto* x = pattern->NewNode(x_repr()) + ->assert_is_op_input("split", "X") + ->assert_is_op_input("elementwise_mul", "X") + ->assert_is_op_input("shape", "Input") + ->AsInput(); + + auto* split = pattern->NewNode(split_repr()) + ->assert_is_op("split") + ->assert_op_attr("axis", 3) + ->assert_op_attr("num", 2); // do we really need it + + auto* split_out1 = pattern->NewNode(split_out1_repr()) + ->assert_is_op_input("scale", "X") + ->assert_is_op_nth_output("split", "Out", 1); + auto* split_out2 = pattern->NewNode(split_out2_repr()) + ->assert_is_op_nth_input("concat", "X", 1) + ->assert_is_op_nth_output("split", "Out", 0); + split->LinksFrom({x}).LinksTo({split_out1, split_out2}); + + auto* scale = pattern->NewNode(scale_repr()) + ->assert_is_op("scale") + ->assert_more([&](Node* node) { + auto* op_desc = node->Op(); + auto scale = op_desc->GetAttrIfExists("scale"); + return (std::fabs(scale + 1.0) < 1e-5); + }); + auto* scale_out = pattern->NewNode(scale_out_repr()) + ->assert_is_op_input("concat", "X") + ->assert_is_op_output("scale", "Out"); + scale->LinksFrom({split_out1}).LinksTo({scale_out}); + auto* concat = pattern->NewNode(concat_repr())->assert_is_op("concat"); + auto* concat_out = pattern->NewNode(concat_out_repr()) + ->assert_is_op_input("elementwise_mul", "X") + ->assert_is_op_output("concat", "Out"); + concat->LinksFrom({scale_out, split_out2}).LinksTo({concat_out}); + auto* shape = pattern->NewNode(shape_repr())->assert_is_op("shape"); + auto* shape_out = pattern->NewNode(shape_out_repr()) + ->assert_is_op_input("slice", "Input") + ->assert_is_op_output("shape", "Out"); + shape->LinksFrom({x}).LinksTo({shape_out}); + auto* slice1 = pattern->NewNode(slice1_repr())->assert_is_op("slice"); + auto* slice1_out = pattern->NewNode(slice1_out_repr()) + ->assert_is_op_input("slice", "EndsTensorList") + ->assert_is_op_output("slice", "Out"); + slice1->LinksFrom({shape_out}).LinksTo({slice1_out}); + auto* sin_emb = pattern->NewNode(sin_emb_repr()) + ->assert_is_op_input("slice", "Input") + ->AsInput(); + auto* cos_emb = pattern->NewNode(cos_emb_repr()) + ->assert_is_op_input("slice", "Input") + ->AsInput(); + auto* slice_sin = pattern->NewNode(slice_sin_repr())->assert_is_op("slice"); + auto* slice_sin_out = pattern->NewNode(slice_sin_out_repr()) + ->assert_is_op_input("elementwise_mul", "Y") + ->assert_is_op_output("slice", "Out"); + slice_sin->LinksFrom({sin_emb, slice1_out}).LinksTo({slice_sin_out}); + auto* mul1 = pattern->NewNode(mul1_repr())->assert_is_op("elementwise_mul"); + auto* mul1_out = pattern->NewNode(mul1_out_repr()) + ->assert_is_op_input("elementwise_add", "Y") + ->assert_is_op_output("elementwise_mul", "Out"); + mul1->LinksFrom({concat_out, slice_sin_out}).LinksTo({mul1_out}); + auto* add = pattern->NewNode(add_repr())->assert_is_op("elementwise_add"); + auto* add_out = pattern->NewNode(add_out_repr()) + ->assert_is_op_output("elementwise_add", "Out") + ->AsOutput(); + auto* slice_cos = pattern->NewNode(slice_cos_repr())->assert_is_op("slice"); + auto* slice_cos_out = pattern->NewNode(slice_cos_out_repr()) + ->assert_is_op_input("elementwise_mul", "Y") + ->assert_is_op_output("slice", "Out"); + slice_cos->LinksFrom({cos_emb, slice1_out}).LinksTo({slice_cos_out}); + auto* mul2 = pattern->NewNode(mul2_repr())->assert_is_op("elementwise_mul"); + auto* mul2_out = pattern->NewNode(mul2_out_repr()) + ->assert_is_op_input("elementwise_add", "X") + ->assert_is_op_output("elementwise_mul", "Out"); + mul2->LinksFrom({x, slice_cos_out}).LinksTo({mul2_out}); + add->LinksFrom({mul2_out, mul1_out}).LinksTo({add_out}); +} + +} // namespace patterns + +class RoformerRelativePosFusePass : public FusePassBase { + protected: + void ApplyImpl(ir::Graph* graph) const override; + + private: + const std::string name_scope_{"roformer_relative_pos_fuse_pass"}; +}; + +void RoformerRelativePosFusePass::ApplyImpl(ir::Graph* graph) const { + PADDLE_ENFORCE_NOT_NULL( + graph, platform::errors::PreconditionNotMet("graph should not be null.")); + Init(name_scope_, graph); + + GraphPatternDetector gpd; + patterns::RoformerRelativePosXPUPattern pattern(gpd.mutable_pattern(), + name_scope_); + int found_subgraph_count = 0; + auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph, + Graph* graph) { + VLOG(4) << "handle RoformerRelativePosFusePass fuse"; + /* declare operator node's name */ + // declare variable node's name + GET_IR_NODE(split); + GET_IR_NODE(scale); + GET_IR_NODE(concat); + GET_IR_NODE(mul1); + GET_IR_NODE(shape); + GET_IR_NODE(slice1); + GET_IR_NODE(slice_sin); + GET_IR_NODE(slice_cos); + GET_IR_NODE(mul2); + GET_IR_NODE(add); + // declare variable node's name + GET_IR_NODE(x); + GET_IR_NODE(sin_emb); + GET_IR_NODE(cos_emb); + GET_IR_NODE(split_out1); + GET_IR_NODE(split_out2); + GET_IR_NODE(scale_out); + GET_IR_NODE(concat_out); + GET_IR_NODE(mul1_out); + GET_IR_NODE(shape_out); + GET_IR_NODE(slice1_out); + GET_IR_NODE(slice_sin_out); + GET_IR_NODE(slice_cos_out); + GET_IR_NODE(mul2_out); + GET_IR_NODE(add_out); + auto* block = add->Op()->Block(); + auto* scope = param_scope(); + PADDLE_ENFORCE_NOT_NULL( + scope, platform::errors::InvalidArgument("Scope cannot be nullptr.")); + // Generate roformer_relative_embedding_xpu fused op + framework::OpDesc fused_op_desc(block); + fused_op_desc.SetType("roformer_relative_embedding_xpu"); + // set attrs for fused op + fused_op_desc.SetInput("x", {x->Name()}); + fused_op_desc.SetInput("sin_emb", {sin_emb->Name()}); + fused_op_desc.SetInput("cos_emb", {cos_emb->Name()}); + + fused_op_desc.SetOutput("out", {add_out->Name()}); + fused_op_desc.SetAttr("max_pos_len", + static_cast(cos_emb->Var()->GetShape()[2])); + + // relink fused op + auto* fused_op = graph->CreateOpNode(&fused_op_desc); + IR_NODE_LINK_TO(x, fused_op); + IR_NODE_LINK_TO(sin_emb, fused_op); + IR_NODE_LINK_TO(cos_emb, fused_op); + IR_NODE_LINK_TO(fused_op, add_out); + // delete useless node + std::unordered_set delete_nodes = {split, + scale, + concat, + mul1, + shape, + slice1, + slice_sin, + slice_cos, + mul2, + add, + split_out1, + split_out2, + scale_out, + concat_out, + shape_out, + slice1_out, + slice_sin_out, + slice_cos_out, + mul2_out}; + GraphSafeRemoveNodes(graph, delete_nodes); + found_subgraph_count++; + }; + + gpd(graph, handler); + + AddStatis(found_subgraph_count); +} + +} // namespace ir +} // namespace framework +} // namespace paddle + +REGISTER_PASS(roformer_relative_pos_fuse_pass, + paddle::framework::ir::RoformerRelativePosFusePass); + +REGISTER_PASS_CAPABILITY(roformer_relative_pos_fuse_pass) + .AddCombination( + paddle::framework::compatible::OpVersionComparatorCombination().EQ( + "roformer_relative_embedding_xpu", 0)); diff --git a/paddle/fluid/inference/api/paddle_pass_builder.cc b/paddle/fluid/inference/api/paddle_pass_builder.cc index 0684064df81e8..508381dc3a310 100644 --- a/paddle/fluid/inference/api/paddle_pass_builder.cc +++ b/paddle/fluid/inference/api/paddle_pass_builder.cc @@ -528,6 +528,7 @@ XpuPassStrategy::XpuPassStrategy() : PassStrategy({}) { "delete_dropout_op_pass", "delete_concat_op_pass", "gather_squeeze_pass", + "roformer_relative_pos_fuse_pass", "delete_repeated_ops_pass", "identity_op_clean_pass", "fused_continuous_same_ops_pass", diff --git a/paddle/phi/api/yaml/fused_ops.yaml b/paddle/phi/api/yaml/fused_ops.yaml index 2ca0a32be59f5..c7b0b14606b98 100644 --- a/paddle/phi/api/yaml/fused_ops.yaml +++ b/paddle/phi/api/yaml/fused_ops.yaml @@ -399,7 +399,7 @@ backward : max_pool2d_v2_grad - op : multi_encoder_xpu - args : (Tensor x, Tensor[] fc_input_max, Tensor[] fc_weight, Tensor[] fc_weight_max, Tensor[] fc_bias, Tensor[] ln_scale, Tensor[] ln_bias, Tensor[] smooth_scale_weight, Tensor mask, Tensor seq_lod, Tensor max_seq_len, int layer_num, bool norm_before, int hidden_dim, int head_num, int size_per_head, int ffn_hidden_dim_scale, int act_type, int relative_type, int slice_idx, bool is_per_channel, float[] softmax_max_value, str[] quant_types) + args : (Tensor x, Tensor[] fc_input_max, Tensor[] fc_weight, Tensor[] fc_weight_max, Tensor[] fc_bias, Tensor[] ln_scale, Tensor[] ln_bias, Tensor[] smooth_scale_weight, Tensor[] roformer_embedding, Tensor mask, Tensor seq_lod, Tensor max_seq_len, int layer_num, bool norm_before, int hidden_dim, int head_num, int size_per_head, int ffn_hidden_dim_scale, int act_type, int relative_type, int slice_idx, bool is_per_channel, int max_pos_len, float[] softmax_max_value, str[] quant_types) output : Tensor(out), Tensor(x_fp16), Tensor(out_fp16) infer_meta : func : MultiEncoderXPUInferMeta @@ -437,6 +437,15 @@ func : quantize_xpu data_type : x +- op : roformer_relative_embedding_xpu + args : (Tensor x, Tensor sin_emb, Tensor cos_emb, int max_pos_len) + output : Tensor(out) + infer_meta : + func : RoformerRelativePosXPUInferMeta + kernel : + func : roformer_relative_embedding_xpu + data_type : x + - op : self_dp_attention args : (Tensor x, float alpha = 1.0f, int head_number = 1) output : Tensor(out) diff --git a/paddle/phi/backends/xpu/xpu2_op_list.cc b/paddle/phi/backends/xpu/xpu2_op_list.cc index 55aae9f24c1a6..14d761a1f1479 100644 --- a/paddle/phi/backends/xpu/xpu2_op_list.cc +++ b/paddle/phi/backends/xpu/xpu2_op_list.cc @@ -1196,6 +1196,8 @@ XPUOpMap& get_kl2_ops() { phi::DataType::INT32})}, {"sine_pos_xpu", XPUKernelSet({phi::DataType::FLOAT16, phi::DataType::FLOAT32})}, + {"roformer_relative_embedding_xpu", + XPUKernelSet({phi::DataType::FLOAT16, phi::DataType::FLOAT32})}, }; return s_xpu2_kernels; diff --git a/paddle/phi/infermeta/fusion.cc b/paddle/phi/infermeta/fusion.cc index 6e85754335ce9..af280b44d6501 100644 --- a/paddle/phi/infermeta/fusion.cc +++ b/paddle/phi/infermeta/fusion.cc @@ -1447,6 +1447,7 @@ void MultiEncoderXPUInferMeta( const std::vector& ln_scale, const std::vector& ln_bias, const std::vector& smooth_scale_weight, + const std::vector& roformer_embedding, const MetaTensor& mask, const MetaTensor& seq_lod, const MetaTensor& max_seq_len, @@ -1460,6 +1461,7 @@ void MultiEncoderXPUInferMeta( int relative_type, int slice_idx, bool is_per_channel, + int max_pos_len, const std::vector& softmax_max_value, const std::vector& quant_types, MetaTensor* out, @@ -3829,4 +3831,56 @@ void MultiGruInferMeta( hidden->set_dims(out_dims); hidden->share_lod(x); } + +void RoformerRelativePosXPUInferMeta(const MetaTensor& x, + const MetaTensor& sin_emb, + const MetaTensor& cos_emb, + int max_pos_len, + MetaTensor* out) { + auto x_dims = x.dims(); + auto x_dims_size = x_dims.size(); + auto sin_emb_dims = sin_emb.dims(); + auto sin_emb_dims_size = sin_emb_dims.size(); + auto cos_emb_dims = cos_emb.dims(); + auto cos_emb_dims_size = cos_emb_dims.size(); + PADDLE_ENFORCE_EQ( + x_dims_size, + 4, + phi::errors::InvalidArgument( + "x_dims_size should be 4, but received x_dims_size is %d", + x_dims_size)); + PADDLE_ENFORCE_EQ( + sin_emb_dims_size, + 4, + phi::errors::InvalidArgument( + "sin_emb_dims_size should be 4, but received sin_emb_dims_size is %d", + sin_emb_dims_size)); + PADDLE_ENFORCE_EQ( + cos_emb_dims_size, + 4, + phi::errors::InvalidArgument( + "cos_emb_dims_size should be 4, but received cos_emb_dims_size is %d", + cos_emb_dims_size)); + for (int i = 0; i < sin_emb_dims_size; i++) { + PADDLE_ENFORCE_EQ( + sin_emb_dims[i], + cos_emb_dims[i], + phi::errors::InvalidArgument( + "sin_emb_dims[i] should be equal to cos_emb_dims[i], index i is " + "%d, sin_emb_dims[i] is %d, cos_emb_dims[i] is %d", + i, + sin_emb_dims[i], + cos_emb_dims[i])); + } + PADDLE_ENFORCE_EQ( + x_dims[3], + cos_emb_dims[3], + phi::errors::InvalidArgument("x_dims[3] should be equal to cos_dims[3], " + "but sin_dims[3] is %d, cos_dims[3] is %d", + x_dims[3], + cos_emb_dims[3])); + out->set_dims(x_dims); + out->set_dtype(x.dtype()); +} + } // namespace phi diff --git a/paddle/phi/infermeta/fusion.h b/paddle/phi/infermeta/fusion.h index 767f22fd245f4..87999ab2b4564 100644 --- a/paddle/phi/infermeta/fusion.h +++ b/paddle/phi/infermeta/fusion.h @@ -151,6 +151,7 @@ void MultiEncoderXPUInferMeta( const std::vector& ln_scale, const std::vector& ln_bias, const std::vector& smooth_scale_weight, + const std::vector& roformer_embedding, const MetaTensor& mask, const MetaTensor& seq_lod, const MetaTensor& max_seq_len, @@ -164,6 +165,7 @@ void MultiEncoderXPUInferMeta( int relative_type, int slice_idx, bool is_per_channel, + int max_pos_len, const std::vector& softmax_max_value, const std::vector& quant_types, MetaTensor* out, @@ -838,6 +840,11 @@ void QKVAttentionXPUInferMeta(const MetaTensor& q, void SinePosXPUInferMeta(const MetaTensor& x, const MetaTensor& y, MetaTensor* out); +void RoformerRelativePosXPUInferMeta(const MetaTensor& x, + const MetaTensor& sin_emb, + const MetaTensor& cos_emb, + int max_pos_len, + MetaTensor* out); void MultiGruInferMeta( const MetaTensor& x, diff --git a/paddle/phi/kernels/fusion/xpu/multi_encoder_xpu_kernel.cc b/paddle/phi/kernels/fusion/xpu/multi_encoder_xpu_kernel.cc index 1f76fc3ef02d8..0b311eb0e65f7 100644 --- a/paddle/phi/kernels/fusion/xpu/multi_encoder_xpu_kernel.cc +++ b/paddle/phi/kernels/fusion/xpu/multi_encoder_xpu_kernel.cc @@ -47,6 +47,7 @@ void MultiEncoderXPUKernel( const std::vector& ln_scale, const std::vector& ln_bias, const std::vector& smooth_scale_weight, + const std::vector& roformer_embedding, const paddle::optional& mask, const paddle::optional& seq_lod, const paddle::optional& max_seq_len, @@ -60,6 +61,7 @@ void MultiEncoderXPUKernel( int relative_type, int slice_idx, bool is_per_channel, + int max_pos_len, const std::vector& softmax_max_value, const std::vector& quant_types, DenseTensor* out, @@ -150,7 +152,6 @@ void MultiEncoderXPUKernel( } } - std::vector test_data(6, 0); for (size_t i = 0; i < fc_input_max.size(); i++) { fc_input_max_data.push_back(fc_input_max[i]->data()); } @@ -199,6 +200,16 @@ void MultiEncoderXPUKernel( qkv_attn_param.quant_type_.assign(set_quant_types.begin(), set_quant_types.end()); qkv_attn_param.scale_of_hidden_units = ffn_hidden_dim_scale; + if (!roformer_embedding.empty()) { + std::vector roformer_embedding_data; + for (size_t i = 0; i < roformer_embedding.size(); i++) { + roformer_embedding_data.push_back(roformer_embedding[i]->data()); + } + qkv_attn_param.relative_type = relative_type; + qkv_attn_param.max_pos_len = max_pos_len; + qkv_attn_param.relative_pos.assign(roformer_embedding_data.begin(), + roformer_embedding_data.end()); + } if (!enable_int8) { if (local_quant) { TRANSFORMER_ENCODER_KERNEL_IMPL(XPUTypeFP16, XPUTypeFP16, float) @@ -242,6 +253,16 @@ void MultiEncoderXPUKernel( qkv_attn_param.quant_type_.assign(set_quant_types.begin(), set_quant_types.end()); qkv_attn_param.scale_of_hidden_units = ffn_hidden_dim_scale; + if (!roformer_embedding.empty()) { + std::vector roformer_embedding_data; + for (size_t i = 0; i < roformer_embedding.size(); i++) { + roformer_embedding_data.push_back(roformer_embedding[i]->data()); + } + qkv_attn_param.relative_type = relative_type; + qkv_attn_param.max_pos_len = max_pos_len; + qkv_attn_param.relative_pos.assign(roformer_embedding_data.begin(), + roformer_embedding_data.end()); + } if (!enable_int8) { if (local_quant) { TRANSFORMER_ENCODER_KERNEL_IMPL(XPUTypeFP16, XPUTypeFP16, float) @@ -288,6 +309,16 @@ void MultiEncoderXPUKernel( qkv_attn_param.quant_type_.assign(set_quant_types.begin(), set_quant_types.end()); qkv_attn_param.scale_of_hidden_units = ffn_hidden_dim_scale; + if (!roformer_embedding.empty()) { + std::vector roformer_embedding_data; + for (size_t i = 0; i < roformer_embedding.size(); i++) { + roformer_embedding_data.push_back(roformer_embedding[i]->data()); + } + qkv_attn_param.relative_type = relative_type; + qkv_attn_param.max_pos_len = max_pos_len; + qkv_attn_param.relative_pos.assign(roformer_embedding_data.begin(), + roformer_embedding_data.end()); + } if (!enable_int8) { if (local_quant) { TRANSFORMER_ENCODER_KERNEL_IMPL(XPUTypeFP16, XPUTypeFP16, float) @@ -319,6 +350,6 @@ PD_REGISTER_KERNEL(multi_encoder_xpu, phi::fusion::MultiEncoderXPUKernel, float, phi::dtype::float16) { - kernel->InputAt(9).SetBackend(phi::Backend::CPU); kernel->InputAt(10).SetBackend(phi::Backend::CPU); + kernel->InputAt(11).SetBackend(phi::Backend::CPU); } diff --git a/paddle/phi/kernels/fusion/xpu/roformer_relative_embedding_kernel.cc b/paddle/phi/kernels/fusion/xpu/roformer_relative_embedding_kernel.cc new file mode 100644 index 0000000000000..ae42b0eabc614 --- /dev/null +++ b/paddle/phi/kernels/fusion/xpu/roformer_relative_embedding_kernel.cc @@ -0,0 +1,78 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/backends/xpu/enforce_xpu.h" +#include "paddle/phi/core/kernel_registry.h" + +namespace phi { +namespace fusion { + +template +void RoformerRelativePosXPUKernel(const Context& ctx, + const DenseTensor& x, + const DenseTensor& sin_emb, + const DenseTensor& cos_emb, + int max_pos_len, + DenseTensor* out) { + using XPUType = typename XPUTypeTrait::Type; + + auto* x_data = reinterpret_cast(x.data()); + auto* sin_emb_data = sin_emb.data(); + auto* cos_emb_data = cos_emb.data(); + auto* out_data = reinterpret_cast(ctx.template Alloc(out)); + xpu::ctx_guard RAII_GUARD(ctx.x_context()); + auto x_dims = x.dims(); + int batch = x_dims[0]; + int head_num = x_dims[1]; + int seqlen = x_dims[2]; + int head_dim = x_dims[3]; + if (seqlen > max_pos_len) { + PADDLE_THROW(phi::errors::InvalidArgument( + "The input sequence length should be less than or equal to the " + "maximum position length. But received seqlen: %d, max_pos_len: %d", + seqlen, + max_pos_len)); + } + std::vector lod; + lod.resize(batch + 1); + for (int i = 0; i < batch + 1; i++) { + lod[i] = i * seqlen; + } + int r = + xpu::rope(ctx.x_context(), + x_data, + out_data, + cos_emb_data, + sin_emb_data, + batch, + head_num, + head_dim, + head_num * head_dim, + lod, + max_pos_len, + false, // no vsl + true); // transpose to [n, seql, head_num, head_dim] + + PADDLE_ENFORCE_XDNN_SUCCESS(r, "roformer_relative_embedding_xpu"); +} + +} // namespace fusion +} // namespace phi + +PD_REGISTER_KERNEL(roformer_relative_embedding_xpu, + XPU, + ALL_LAYOUT, + phi::fusion::RoformerRelativePosXPUKernel, + float, + phi::dtype::float16) {} diff --git a/test/ir/inference/test_xpu_roformer_relative_pos_pass.py b/test/ir/inference/test_xpu_roformer_relative_pos_pass.py new file mode 100644 index 0000000000000..93c448463af9c --- /dev/null +++ b/test/ir/inference/test_xpu_roformer_relative_pos_pass.py @@ -0,0 +1,167 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from functools import partial + +import hypothesis.strategies as st +import numpy as np +from auto_scan_test import PassAutoScanTest +from program_config import OpConfig, ProgramConfig, TensorConfig + + +class TestRoformerRelativePosXPUPass(PassAutoScanTest): + def sample_predictor_configs(self, program_config): + config = self.create_inference_config(use_xpu=True) + # config.switch_ir_optim(True) + # config.switch_ir_debug(True) + yield config, ["roformer_relative_embedding_xpu"], (1e-3, 1e-3) + + def sample_program_config(self, draw): + x_shape = draw( + st.lists( + st.integers(min_value=1, max_value=10), min_size=4, max_size=4 + ) + ) + x_shape[1] = draw(st.integers(min_value=12, max_value=12)) + x_shape[2] = draw(st.integers(min_value=512, max_value=512)) + x_shape[3] = draw(st.integers(min_value=32, max_value=32)) + sin_emb_shape = draw( + st.lists( + st.integers(min_value=1, max_value=1), + min_size=4, + max_size=4, + ) + ) + sin_emb_shape[1] = draw(st.integers(min_value=1, max_value=1)) + sin_emb_shape[2] = draw(st.integers(min_value=512, max_value=512)) + sin_emb_shape[3] = draw(st.integers(min_value=32, max_value=32)) + cos_emb_shape = sin_emb_shape + + def generate_data(shape): + return np.random.random(shape).astype(np.float32) + + # Here we will compose a program + # Still has some risks that the program is invalid or cause bug while running + # Use function `is_program_valid` to filter the invalid programs before running + # Use function `add_skip_pass_case` to ignore the programs even if they cause bug while runing + split_op = OpConfig( + "split", + inputs={"X": ["x"]}, + outputs={"Out": ["split_out1", "split_out2"]}, + axis=3, + num=2, + ) + scale_op = OpConfig( + "scale", + inputs={"X": ["split_out2"]}, + outputs={"Out": ["scale_out"]}, + scale=-1, + ) + concat_op = OpConfig( + "concat", + inputs={"X": ["scale_out", "split_out1"]}, + outputs={"Out": ["concat_out"]}, + axis=-1, + ) + shape_op = OpConfig( + "shape", + inputs={"Input": ["x"]}, + outputs={"Out": ["shape_out"]}, + ) + slice1_op = OpConfig( + "slice", + inputs={"Input": ["shape_out"]}, + outputs={"Out": ["slice1_out"]}, + axes=[0], + starts=[-2], + ends=[-1], + infer_flags=[1], + decrease_axis=[0], + ) + slice_sin_op = OpConfig( + "slice", + inputs={"Input": ["sin_emb"], "EndsTensorList": ["slice1_out"]}, + outputs={"Out": ["slice_sin_out"]}, + axes=[2], + starts=[0], + ends=[-1], + infer_flags=[-1], + decrease_axis=[], + ) + slice_cos_op = OpConfig( + "slice", + inputs={"Input": ["cos_emb"], "EndsTensorList": ["slice1_out"]}, + outputs={"Out": ["slice_cos_out"]}, + axes=[2], + starts=[0], + ends=[-1], + infer_flags=[-1], + decrease_axis=[], + ) + mul1_op = OpConfig( + "elementwise_mul", + inputs={"X": ["concat_out"], "Y": ["slice_sin_out"]}, + outputs={"Out": ["mul1_out"]}, + ) + mul2_op = OpConfig( + "elementwise_mul", + inputs={"X": ["x"], "Y": ["slice_cos_out"]}, + outputs={"Out": ["mul2_out"]}, + ) + add_op = OpConfig( + "elementwise_add", + inputs={"X": ["mul2_out"], "Y": ["mul1_out"]}, + outputs={"Out": ["add_out"]}, + ) + + ops = [ + split_op, + scale_op, + concat_op, + shape_op, + slice1_op, + slice_sin_op, + slice_cos_op, + mul1_op, + mul2_op, + add_op, + ] + + program_config = ProgramConfig( + ops=ops, + inputs={ + "x": TensorConfig(data_gen=partial(generate_data, x_shape)), + "sin_emb": TensorConfig( + data_gen=partial(generate_data, sin_emb_shape) + ), + "cos_emb": TensorConfig( + data_gen=partial(generate_data, cos_emb_shape) + ), + }, + weights={}, + outputs=ops[-1].outputs["Out"], + ) + return program_config + + def test(self): + self.run_and_statis( + quant=False, + max_examples=25, + passes=["roformer_relative_pos_fuse_pass"], + ) + + +if __name__ == "__main__": + unittest.main() From 08d2b797128a5197385b42ed584d7c05535b2471 Mon Sep 17 00:00:00 2001 From: Ruibiao Chen Date: Thu, 29 Feb 2024 11:14:21 +0800 Subject: [PATCH 168/282] Add 'index' parameter for ProcessMesh.get_mesh_with_dim (#62125) * Add 'index' parameter for ProcessMesh.get_mesh_with_dim * Add UT --- python/paddle/distributed/auto_parallel/process_mesh.py | 5 ++++- test/auto_parallel/test_interface.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/python/paddle/distributed/auto_parallel/process_mesh.py b/python/paddle/distributed/auto_parallel/process_mesh.py index f321ba3ffdf5c..c0dbd3a9d2790 100644 --- a/python/paddle/distributed/auto_parallel/process_mesh.py +++ b/python/paddle/distributed/auto_parallel/process_mesh.py @@ -239,7 +239,7 @@ def get_dim_size(self, dim: Union[str, int]) -> int: assert dim_name in self._dim_names return self._shape[self._dim_names.index(dim_name)] - def get_mesh_with_dim(self, dim_name): + def get_mesh_with_dim(self, dim_name, index=None): assert ( dim_name in self._dim_names ), f'{dim_name} is not a valid dim name.' @@ -251,6 +251,9 @@ def get_mesh_with_dim(self, dim_name): dim for dim in self._dim_names if dim != dim_name ] new_mesh = self._mesh.transpose(new_order) + + if index is not None: + return ProcessMesh(new_mesh[index], new_dim_names[1:]) return ProcessMesh(new_mesh, new_dim_names) def __enter__(self): diff --git a/test/auto_parallel/test_interface.py b/test/auto_parallel/test_interface.py index 989cc8eed2797..c5c4584bfcdcb 100644 --- a/test/auto_parallel/test_interface.py +++ b/test/auto_parallel/test_interface.py @@ -269,7 +269,8 @@ def test_create_mesh(self): first_pp_mesh.process_ids, list(arr.transpose([1, 0, 2]).flatten()) ) - pp_stage_0_mesh = first_pp_mesh[0] + pp_stage_0_mesh = auto.get_mesh().get_mesh_with_dim("pp", 0) + self.assertEqual(pp_stage_0_mesh, first_pp_mesh[0]) self.assertEqual(pp_stage_0_mesh.shape, [2, 4]) self.assertEqual( pp_stage_0_mesh.process_ids, [0, 1, 2, 3, 16, 17, 18, 19] From 7d84d55e831ebfb6e1c8cdc0af2a0e9a596e7788 Mon Sep 17 00:00:00 2001 From: Yuanle Liu Date: Thu, 29 Feb 2024 11:32:58 +0800 Subject: [PATCH 169/282] Forbid control flow related ops to constant folding (#62206) * forbid control flow ops to constant folding * refine --- .../framework/ir/constant_folding_pass.cc | 42 +++++++++++++++++-- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/paddle/fluid/framework/ir/constant_folding_pass.cc b/paddle/fluid/framework/ir/constant_folding_pass.cc index 4375043544dc8..099209db48840 100644 --- a/paddle/fluid/framework/ir/constant_folding_pass.cc +++ b/paddle/fluid/framework/ir/constant_folding_pass.cc @@ -13,9 +13,13 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/fluid/framework/ir/constant_folding_pass.h" + #include #include #include "glog/logging.h" + +#include "paddle/fluid/framework/convert_utils.h" +#include "paddle/fluid/framework/ir/graph.h" #include "paddle/fluid/framework/ir/graph_helper.h" #include "paddle/fluid/framework/ir/graph_pattern_detector.h" #include "paddle/fluid/framework/ir/pass.h" @@ -23,8 +27,6 @@ limitations under the License. */ #include "paddle/fluid/framework/op_version_registry.h" #include "paddle/fluid/platform/enforce.h" -#include "paddle/fluid/framework/convert_utils.h" - namespace paddle { namespace framework { namespace ir { @@ -51,6 +53,37 @@ struct ConstantFolding : public PatternBase { }; } // namespace patterns +namespace { +std::unordered_set GetControlFlowVarNames(ir::Graph *graph) { + std::unordered_set control_flow_ops{"while", + "conditional_block"}; + std::unordered_set control_flow_var_names; + for (auto *node : graph->Nodes()) { + if (!node->IsOp() || control_flow_ops.count(node->Op()->Type()) == 0) + continue; + for (auto const &in_names : node->Op()->Inputs()) { + auto var_names = in_names.second; + control_flow_var_names.insert(var_names.begin(), var_names.end()); + } + for (auto const &out_names : node->Op()->Outputs()) { + auto var_names = out_names.second; + control_flow_var_names.insert(var_names.begin(), var_names.end()); + } + } + return control_flow_var_names; +} + +bool OutputUsedByControlFlow(ir::Node *node, + const std::unordered_set &cf_vars) { + for (auto out_node : node->outputs) { + if (cf_vars.count(out_node->Name())) { + return true; + } + } + return false; +} +} // namespace + ConstantFoldingPass::ConstantFoldingPass() = default; void ConstantFoldingPass::ApplyImpl(ir::Graph *graph) const { @@ -69,6 +102,7 @@ void ConstantFoldingPass::ApplyImpl(ir::Graph *graph) const { "save", "quantize_linear", "dequantize_linear"}; + const auto cf_vars = GetControlFlowVarNames(graph); int folded_op_num = 0; auto op_node_sorted = framework::ir::TopologyVariantSort( @@ -78,7 +112,9 @@ void ConstantFoldingPass::ApplyImpl(ir::Graph *graph) const { if (std::find(blacklist.begin(), blacklist.end(), op_node->Name()) != blacklist.end()) continue; - + if (OutputUsedByControlFlow(op_node, cf_vars)) { + continue; + } bool input_persis = true; // map is used to record how many time a name string occurs in the whole // graph's nodes From 239b830f9939ca706d8b0e38a502d81ede3572cf Mon Sep 17 00:00:00 2001 From: Zhan Rongrui <46243324+zrr1999@users.noreply.github.com> Date: Thu, 29 Feb 2024 14:11:03 +0800 Subject: [PATCH 170/282] =?UTF-8?q?[PIR]=20A-20=E3=80=81B-9=E3=80=81B-10?= =?UTF-8?q?=20Adapt=20test=5Ferrors=20(#62118)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/legacy_test/test_activation_op.py | 39 ++++++++++++++++---------- test/legacy_test/test_full_like_op.py | 6 ++-- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/test/legacy_test/test_activation_op.py b/test/legacy_test/test_activation_op.py index deecf7fd09a9e..45c79e6aba5c9 100644 --- a/test/legacy_test/test_activation_op.py +++ b/test/legacy_test/test_activation_op.py @@ -40,9 +40,12 @@ def dynamic_guard(): class TestSqrtOpError(unittest.TestCase): + @test_with_pir_api def test_errors(self): with static_guard(): - with program_guard(Program(), Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): # The input type of sqrt op must be Variable or numpy.ndarray. in1 = 1 self.assertRaises(TypeError, paddle.sqrt, in1) @@ -643,6 +646,7 @@ def test_dygraph_api(self): np.testing.assert_allclose(out_ref, r.numpy(), rtol=1e-05) paddle.enable_static() + @test_with_pir_api def test_errors(self): with static_guard(): with paddle.static.program_guard(paddle.static.Program()): @@ -890,6 +894,7 @@ def test_dygraph_api(self): for r in [out1, out2, out3]: np.testing.assert_allclose(out_ref, r.numpy(), rtol=1e-05) + @test_with_pir_api def test_errors(self): with static_guard(): with paddle.static.program_guard(paddle.static.Program()): @@ -2702,22 +2707,24 @@ def test_dygraph_api(self): for r in [out1, out2]: np.testing.assert_allclose(out_ref, r.numpy(), rtol=1e-05) + @test_with_pir_api def test_errors(self): with static_guard(): - with static_guard(): - with paddle.static.program_guard(paddle.static.Program()): - # The input type must be Variable. - self.assertRaises(TypeError, self.relu, 1) - # The input dtype must be float16, float32, float64. - x_int32 = paddle.static.data( - name='x_int32', shape=[10, 12], dtype='int32' - ) - self.assertRaises(TypeError, self.relu, x_int32) - # support the input dtype is float16 - x_fp16 = paddle.static.data( - name='x_fp16', shape=[10, 12], dtype='float16' - ) - self.relu(x_fp16) + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + # The input type must be Variable. + self.assertRaises(TypeError, self.relu, 1) + # The input dtype must be float16, float32, float64. + x_int32 = paddle.static.data( + name='x_int32', shape=[10, 12], dtype='int32' + ) + self.assertRaises(TypeError, self.relu, x_int32) + # support the input dtype is float16 + x_fp16 = paddle.static.data( + name='x_fp16', shape=[10, 12], dtype='float16' + ) + self.relu(x_fp16) class TestReluInplaceAPI(TestReluAPI): @@ -2846,6 +2853,7 @@ def test_dygraph_api(self): for r in [out1, out2]: np.testing.assert_allclose(out_ref, r.numpy(), rtol=1e-05) + @test_with_pir_api def test_errors(self): with static_guard(): with paddle.static.program_guard(paddle.static.Program()): @@ -3029,6 +3037,7 @@ def test_dygraph_api(self): for r in [out1, out2]: np.testing.assert_allclose(out_ref, r.numpy(), rtol=1e-05) + @test_with_pir_api def test_errors(self): with static_guard(): with paddle.static.program_guard(paddle.static.Program()): diff --git a/test/legacy_test/test_full_like_op.py b/test/legacy_test/test_full_like_op.py index 9f327b0b0107a..81322bd431c31 100644 --- a/test/legacy_test/test_full_like_op.py +++ b/test/legacy_test/test_full_like_op.py @@ -23,7 +23,6 @@ from paddle.base.framework import convert_np_dtype_to_dtype_ from paddle.framework import in_pir_mode from paddle.pir_utils import test_with_pir_api -from paddle.static import Program, program_guard def fill_any_like_wrapper(x, value, out_dtype=None, name=None): @@ -98,8 +97,11 @@ def test_full_like_fill_inf(self): class TestFullOpError(unittest.TestCase): + @test_with_pir_api def test_errors(self): - with program_guard(Program(), Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): # for ci coverage input_data = paddle.static.data( From 73f9671b168fc8f01480e7886bd5dbc98f54cff2 Mon Sep 17 00:00:00 2001 From: Yuanle Liu Date: Thu, 29 Feb 2024 14:23:57 +0800 Subject: [PATCH 171/282] [Inference] Export pir&pass headers for inference lib (#61863) * export pir&pass headers in inference * fix * final --- cmake/cuda.cmake | 2 +- ...eader.cmake => export_paddle_header.cmake} | 46 +++++++++++++----- cmake/inference_lib.cmake | 48 +++++++++++++++++-- paddle/cinn/hlir/framework/pir/op_mapper.h | 3 ++ paddle/extension.h | 23 +++++++++ .../inference/api/demo_ci/CMakeLists.txt | 2 +- .../fluid/pir/dialect/kernel/ir/kernel_op.cc | 4 +- paddle/fluid/pir/drr/src/pattern_graph.cc | 4 +- paddle/fluid/pir/drr/src/pattern_graph.h | 2 +- paddle/fluid/pir/drr/src/rewrite_pattern.cc | 7 +-- paddle/phi/api/all.h | 5 -- paddle/pir/include/core/block_argument.h | 1 + .../pir/include/core/builtin_type_storage.h | 2 + paddle/pir/include/core/interface_support.h | 3 +- paddle/pir/include/core/interface_value.h | 2 + paddle/pir/include/core/ir_context.h | 1 + paddle/pir/include/core/ir_mapping.h | 2 + paddle/pir/include/core/iterator.h | 3 ++ paddle/pir/include/core/op_base.h | 1 + paddle/pir/include/core/op_info.h | 1 + paddle/pir/include/core/op_operand.h | 1 + paddle/pir/include/core/op_result.h | 1 + paddle/pir/include/core/operation_utils.h | 1 + paddle/pir/include/core/parameter.h | 2 + .../include/core/storage_manager_support.h | 1 + paddle/pir/include/core/type.h | 1 + paddle/pir/include/core/type_id.h | 1 - paddle/pir/include/core/visitors.h | 1 + .../include/dialect/control_flow/ir/cf_op.h | 2 + .../pir/include/dialect/shape/ir/shape_op.h | 1 + paddle/pir/include/pass/pass.h | 8 +--- paddle/pir/src/core/block.cc | 1 + paddle/pir/src/core/block_argument.cc | 2 + paddle/pir/src/core/builder.cc | 2 + paddle/pir/src/core/builtin_op.cc | 4 +- paddle/pir/src/core/dialect.cc | 2 + paddle/pir/src/core/ir_context.cc | 1 + paddle/pir/src/core/op_info_impl.cc | 4 +- paddle/pir/src/core/op_result_impl.cc | 4 +- paddle/pir/src/core/op_trait.cc | 4 +- paddle/pir/src/core/operation.cc | 1 + paddle/pir/src/core/storage_manager.cc | 1 + paddle/pir/src/core/value_impl.cc | 2 + .../pir/src/dialect/control_flow/ir/cf_op.cc | 4 +- paddle/pir/src/pass/print_statistics.cc | 2 + .../pattern_rewrite/pattern_rewrite_driver.cc | 1 + .../utils/cpp_extension/cpp_extension.py | 2 +- .../utils/cpp_extension/extension_utils.py | 6 +-- python/setup.py.in | 8 +++- setup.py | 21 ++++++++ test/cpp/pir/tools/test_op.h | 2 + 51 files changed, 208 insertions(+), 48 deletions(-) rename cmake/{phi_header.cmake => export_paddle_header.cmake} (52%) diff --git a/cmake/cuda.cmake b/cmake/cuda.cmake index 81a7228629d25..e0a2a7eb34739 100644 --- a/cmake/cuda.cmake +++ b/cmake/cuda.cmake @@ -294,7 +294,7 @@ select_nvcc_arch_flags(NVCC_FLAGS_EXTRA NVCC_ARCH_BIN) set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${NVCC_FLAGS_EXTRA}") message(STATUS "NVCC_FLAGS_EXTRA: ${NVCC_FLAGS_EXTRA}") -# Set C++14 support +# Set C++17 support set(CUDA_PROPAGATE_HOST_FLAGS OFF) # Release/Debug flags set by cmake. Such as -O3 -g -DNDEBUG etc. # So, don't set these flags here. diff --git a/cmake/phi_header.cmake b/cmake/export_paddle_header.cmake similarity index 52% rename from cmake/phi_header.cmake rename to cmake/export_paddle_header.cmake index ac633b747bcef..9b139da98ad2d 100644 --- a/cmake/phi_header.cmake +++ b/cmake/export_paddle_header.cmake @@ -15,33 +15,57 @@ set(PADDLE_INFERENCE_INSTALL_DIR "${CMAKE_BINARY_DIR}/paddle_inference_install_dir") -function(phi_header_path_compat TARGET_PATH) - message(STATUS "phi header path compat processing: ${TARGET_PATH}") +function(header_path_compat TARGET_PATH) + message(STATUS "header path compat processing: ${TARGET_PATH}") file(GLOB HEADERS "${TARGET_PATH}/*" "*.h") foreach(header ${HEADERS}) if(${header} MATCHES ".*.h$") file(READ ${header} HEADER_CONTENT) string(REPLACE "paddle/fluid/platform/" "paddle/phi/" HEADER_CONTENT "${HEADER_CONTENT}") + string(REPLACE "paddle/pir/include/" "paddle/pir/" HEADER_CONTENT + "${HEADER_CONTENT}") + string(REPLACE "paddle/fluid/pir/drr/include/" "paddle/pir/drr/" + HEADER_CONTENT "${HEADER_CONTENT}") + string(REPLACE "paddle/fluid/pir/transforms/" "paddle/pir/transforms/" + HEADER_CONTENT "${HEADER_CONTENT}") file(WRITE ${header} "${HEADER_CONTENT}") - message(STATUS "phi header path compat processing complete: ${header}") + message(STATUS "header path compat processing complete: ${header}") endif() endforeach() endfunction() -phi_header_path_compat(${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle) -phi_header_path_compat( - ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/phi) -phi_header_path_compat( +header_path_compat(${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle) +header_path_compat(${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/phi) +header_path_compat( ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/phi/api) -phi_header_path_compat( +header_path_compat( ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/phi/api/ext) -phi_header_path_compat( +header_path_compat( ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/phi/api/include) -phi_header_path_compat( +header_path_compat( ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/phi/common) -phi_header_path_compat( +header_path_compat( ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/phi/core) +header_path_compat( + ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/pir/core) +header_path_compat( + ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/pir/core/parser) +header_path_compat( + ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/pir/dialect/control_flow/ir +) +header_path_compat( + ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/pir/dialect/shape/ir) +header_path_compat( + ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/pir/dialect/shape/utils) +header_path_compat( + ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/pir/drr) +header_path_compat( + ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/pir/pass) +header_path_compat( + ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/pir/pattern_rewrite) +header_path_compat( + ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/pir/transforms) # NOTE(liuyuanle): In inference lib, no need include paddle/utils/pybind.h, so we delete this. file(READ ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/extension.h diff --git a/cmake/inference_lib.cmake b/cmake/inference_lib.cmake index f4a8286985094..7db3a7de046fd 100755 --- a/cmake/inference_lib.cmake +++ b/cmake/inference_lib.cmake @@ -354,12 +354,54 @@ copy( SRCS ${PADDLE_SOURCE_DIR}/paddle/extension.h DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/) -# the include path of phi needs to be changed to adapt to inference api path +copy( + inference_lib_dist + SRCS ${PADDLE_SOURCE_DIR}/paddle/pir/include/core/parser/*.h + DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/pir/core/parser/) +copy( + inference_lib_dist + SRCS ${PADDLE_SOURCE_DIR}/paddle/pir/include/core/*.h + DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/pir/core/) +copy( + inference_lib_dist + SRCS ${PADDLE_SOURCE_DIR}/paddle/pir/include/dialect/control_flow/ir/*.h + DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/pir/dialect/control_flow/ir/ +) +copy( + inference_lib_dist + SRCS ${PADDLE_SOURCE_DIR}/paddle/pir/include/dialect/shape/ir/*.h + DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/pir/dialect/shape/ir/ +) +copy( + inference_lib_dist + SRCS ${PADDLE_SOURCE_DIR}/paddle/pir/include/dialect/shape/utils/*.h + DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/pir/dialect/shape/utils/ +) +copy( + inference_lib_dist + SRCS ${PADDLE_SOURCE_DIR}/paddle/pir/include/pass/*.h + DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/pir/pass/) +copy( + inference_lib_dist + SRCS ${PADDLE_SOURCE_DIR}/paddle/pir/include/pattern_rewrite/*.h + DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/pir/pattern_rewrite/ +) +copy( + inference_lib_dist + SRCS ${PADDLE_SOURCE_DIR}/paddle/fluid/pir/drr/include/*.h + DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/pir/drr/) +copy( + inference_lib_dist + SRCS ${PADDLE_SOURCE_DIR}/paddle/fluid/pir/transforms/transform_general_functions.h + DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/pir/transforms/) + +# the include path of paddle needs to be changed to adapt to inference api path add_custom_command( TARGET inference_lib_dist POST_BUILD - COMMAND ${CMAKE_COMMAND} -P "${PADDLE_SOURCE_DIR}/cmake/phi_header.cmake" - COMMENT "Change phi header include path to adapt to inference api path") + COMMAND ${CMAKE_COMMAND} -P + "${PADDLE_SOURCE_DIR}/cmake/export_paddle_header.cmake" + COMMENT "Change paddle header include path to adapt to inference api path") # CAPI inference library for only inference set(PADDLE_INFERENCE_C_INSTALL_DIR diff --git a/paddle/cinn/hlir/framework/pir/op_mapper.h b/paddle/cinn/hlir/framework/pir/op_mapper.h index 73e8d9581e4b0..87053a8c02d53 100644 --- a/paddle/cinn/hlir/framework/pir/op_mapper.h +++ b/paddle/cinn/hlir/framework/pir/op_mapper.h @@ -13,9 +13,12 @@ // limitations under the License. #pragma once + +#include #include #include #include + #include "paddle/cinn/utils/type_defs.h" #include "paddle/pir/include/core/operation.h" diff --git a/paddle/extension.h b/paddle/extension.h index 3c79adcde5d69..f3c6e0a1b15f9 100644 --- a/paddle/extension.h +++ b/paddle/extension.h @@ -14,12 +14,35 @@ limitations under the License. */ #pragma once +#if defined(__clang__) || defined(__GNUC__) +#define CPP_STANDARD __cplusplus +#elif defined(_MSC_VER) +#define CPP_STANDARD _MSVC_LANG +#endif + #ifndef CUSTOM_OP_WITH_SPMD #define CUSTOM_OP_WITH_SPMD #endif // All paddle apis in C++ frontend +// phi headers #include "paddle/phi/api/all.h" +// common headers +#include "paddle/common/ddim.h" +#include "paddle/common/exception.h" +#include "paddle/common/layout.h" + +#if CPP_STANDARD >= 201703L && !defined(__clang__) +// pir&pass headers +#include "paddle/fluid/pir/drr/include/drr_pattern_base.h" +#include "paddle/pir/include/core/operation.h" +#include "paddle/pir/include/core/type.h" +#include "paddle/pir/include/core/value.h" +#include "paddle/pir/include/pass/pass.h" +#include "paddle/pir/include/pass/pass_manager.h" +#include "paddle/pir/include/pattern_rewrite/pattern_match.h" +#endif + #if !defined(PADDLE_ON_INFERENCE) && !defined(PADDLE_NO_PYTHON) // Python bindings for the C++ frontend (includes Python.h) #include "paddle/utils/pybind.h" diff --git a/paddle/fluid/inference/api/demo_ci/CMakeLists.txt b/paddle/fluid/inference/api/demo_ci/CMakeLists.txt index 727af4e00605e..1206ac1fd6859 100644 --- a/paddle/fluid/inference/api/demo_ci/CMakeLists.txt +++ b/paddle/fluid/inference/api/demo_ci/CMakeLists.txt @@ -85,7 +85,7 @@ else() if(WITH_MKL) set(FLAG_OPENMP "-fopenmp") endif() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 ${FLAG_OPENMP}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 ${FLAG_OPENMP}") endif() if(WITH_GPU) diff --git a/paddle/fluid/pir/dialect/kernel/ir/kernel_op.cc b/paddle/fluid/pir/dialect/kernel/ir/kernel_op.cc index 0c8f007a51a9d..c3e44d4e3ef35 100644 --- a/paddle/fluid/pir/dialect/kernel/ir/kernel_op.cc +++ b/paddle/fluid/pir/dialect/kernel/ir/kernel_op.cc @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "paddle/fluid/pir/dialect/kernel/ir/kernel_op.h" +#include + #include "paddle/fluid/pir/dialect/kernel/ir/kernel_attribute.h" +#include "paddle/fluid/pir/dialect/kernel/ir/kernel_op.h" #include "paddle/phi/core/enforce.h" #include "paddle/pir/include/core/builtin_attribute.h" diff --git a/paddle/fluid/pir/drr/src/pattern_graph.cc b/paddle/fluid/pir/drr/src/pattern_graph.cc index a8c72a064d0b8..eccbb30dea890 100644 --- a/paddle/fluid/pir/drr/src/pattern_graph.cc +++ b/paddle/fluid/pir/drr/src/pattern_graph.cc @@ -147,7 +147,7 @@ void GraphTopo::WalkGraphNodesTopoOrder( const std::unordered_set &inputs_tensor = graph_->input_tensors(); const std::unordered_map> - &id2owned_tensor = graph_->id2owend_tensor(); + &id2owned_tensor = graph_->id2owned_tensor(); const std::vector> &owend_opcall = graph_->owned_op_call(); @@ -202,7 +202,7 @@ void GraphTopo::WalkGraphNodesTopoOrder( std::ostream &operator<<(std::ostream &os, const PatternGraph &pattern_graph) { os << "\nAll Tensors:\n"; - for (const auto &kv : pattern_graph.id2owend_tensor()) { + for (const auto &kv : pattern_graph.id2owned_tensor()) { os << " " << kv.first; } os << "\n\n"; diff --git a/paddle/fluid/pir/drr/src/pattern_graph.h b/paddle/fluid/pir/drr/src/pattern_graph.h index e5cd74b2fa217..7243c99bfc853 100644 --- a/paddle/fluid/pir/drr/src/pattern_graph.h +++ b/paddle/fluid/pir/drr/src/pattern_graph.h @@ -57,7 +57,7 @@ class PatternGraph { } const std::unordered_map>& - id2owend_tensor() const { + id2owned_tensor() const { return id2owned_tensor_; } diff --git a/paddle/fluid/pir/drr/src/rewrite_pattern.cc b/paddle/fluid/pir/drr/src/rewrite_pattern.cc index 68a7b14f81a3e..04390126ddddf 100644 --- a/paddle/fluid/pir/drr/src/rewrite_pattern.cc +++ b/paddle/fluid/pir/drr/src/rewrite_pattern.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include "paddle/fluid/pir/drr/include/drr_pattern_base.h" @@ -414,13 +415,13 @@ MatchContextImpl DrrRewritePattern::CreateOperations( // add input tensors info for res_match_ctx for (const auto& in_tensor : result_pattern_graph.input_tensors()) { PADDLE_ENFORCE_NE( - result_pattern_graph.id2owend_tensor().count(in_tensor), + result_pattern_graph.id2owned_tensor().count(in_tensor), 0, phi::errors::NotFound("Not found the input tensor." "Drr input tensor [%s] must exist in the result " "pattern graph to be obtained.", in_tensor)); - if (!result_pattern_graph.id2owend_tensor().at(in_tensor)->is_none()) { + if (!result_pattern_graph.id2owned_tensor().at(in_tensor)->is_none()) { res_match_ctx.BindIrValue(in_tensor, src_match_ctx.GetIrValue(in_tensor)); } } @@ -508,7 +509,7 @@ void DrrRewritePattern::ReplaceOutputTensor( const MatchContextImpl& res_match_ctx, pir::PatternRewriter& rewriter) const { // NOLINT for (const auto& output_name : result_pattern_graph_->output_tensors()) { - if (source_pattern_graph_->id2owend_tensor().count(output_name)) { + if (source_pattern_graph_->id2owned_tensor().count(output_name)) { const auto& src_ir_tensor = src_match_ctx.GetIrValue(output_name); const auto& res_ir_tensor = res_match_ctx.GetIrValue(output_name); rewriter.ReplaceAllUsesWith(src_ir_tensor, res_ir_tensor); diff --git a/paddle/phi/api/all.h b/paddle/phi/api/all.h index 93c97605f9f3f..aaafec306401a 100644 --- a/paddle/phi/api/all.h +++ b/paddle/phi/api/all.h @@ -38,8 +38,3 @@ limitations under the License. */ #include "paddle/phi/api/ext/dispatch.h" #include "paddle/phi/api/ext/op_meta_info.h" #include "paddle/phi/api/ext/tensor_compat.h" - -// common headers -#include "paddle/common/ddim.h" -#include "paddle/common/exception.h" -#include "paddle/common/layout.h" diff --git a/paddle/pir/include/core/block_argument.h b/paddle/pir/include/core/block_argument.h index 3ddf7847fd8a2..b3b8c78660c34 100644 --- a/paddle/pir/include/core/block_argument.h +++ b/paddle/pir/include/core/block_argument.h @@ -16,6 +16,7 @@ #include "paddle/pir/include/core/operation_utils.h" #include "paddle/pir/include/core/value.h" + namespace pir { class Block; diff --git a/paddle/pir/include/core/builtin_type_storage.h b/paddle/pir/include/core/builtin_type_storage.h index 03f06279a0dfd..f706e0c66277e 100644 --- a/paddle/pir/include/core/builtin_type_storage.h +++ b/paddle/pir/include/core/builtin_type_storage.h @@ -14,6 +14,8 @@ #pragma once +#include + #include "paddle/common/ddim.h" #include "paddle/common/dim.h" #include "paddle/common/hash_funcs.h" diff --git a/paddle/pir/include/core/interface_support.h b/paddle/pir/include/core/interface_support.h index a035114e44bf2..12d419b3291c6 100644 --- a/paddle/pir/include/core/interface_support.h +++ b/paddle/pir/include/core/interface_support.h @@ -19,6 +19,7 @@ namespace pir { namespace detail { + template class ConstructInterfacesOrTraits { public: @@ -45,14 +46,12 @@ class ConstructInterfacesOrTraits { IR_ENFORCE(suceess, "Interface: id[%u] is already registered. inset failed", TypeId::get()); - VLOG(10) << "New a interface: id[" << TypeId::get() << "]."; } /// Placement new trait. template static void PlacementConstrctTrait(pir::TypeId *&p_trait) { // NOLINT *p_trait = TypeId::get(); - VLOG(10) << "New a trait: id[" << *p_trait << "]."; ++p_trait; } }; diff --git a/paddle/pir/include/core/interface_value.h b/paddle/pir/include/core/interface_value.h index 00f8cc289143f..64619a0e0f591 100644 --- a/paddle/pir/include/core/interface_value.h +++ b/paddle/pir/include/core/interface_value.h @@ -13,8 +13,10 @@ // limitations under the License. #pragma once + #include #include + #include "paddle/pir/include/core/type_id.h" #include "paddle/pir/include/core/utils.h" diff --git a/paddle/pir/include/core/ir_context.h b/paddle/pir/include/core/ir_context.h index dbf7ff4cdd73e..914fecc60a056 100644 --- a/paddle/pir/include/core/ir_context.h +++ b/paddle/pir/include/core/ir_context.h @@ -13,6 +13,7 @@ // limitations under the License. #pragma once + #include #include #include diff --git a/paddle/pir/include/core/ir_mapping.h b/paddle/pir/include/core/ir_mapping.h index 83994ea284570..e67c507059b17 100644 --- a/paddle/pir/include/core/ir_mapping.h +++ b/paddle/pir/include/core/ir_mapping.h @@ -12,7 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. #pragma once + #include + #include "paddle/common/enforce.h" #include "paddle/pir/include/core/value.h" diff --git a/paddle/pir/include/core/iterator.h b/paddle/pir/include/core/iterator.h index 8fbfae8cb4b2d..fc88d981c3661 100644 --- a/paddle/pir/include/core/iterator.h +++ b/paddle/pir/include/core/iterator.h @@ -13,9 +13,12 @@ // limitations under the License. #pragma once + #include #include + #include "paddle/common/macros.h" + namespace pir { class Operation; diff --git a/paddle/pir/include/core/op_base.h b/paddle/pir/include/core/op_base.h index 93e6939be8adf..698f65c791dbe 100644 --- a/paddle/pir/include/core/op_base.h +++ b/paddle/pir/include/core/op_base.h @@ -13,6 +13,7 @@ // limitations under the License. #pragma once + #include #include "paddle/common/enforce.h" diff --git a/paddle/pir/include/core/op_info.h b/paddle/pir/include/core/op_info.h index fbeb679463a4d..124ed660db0f4 100644 --- a/paddle/pir/include/core/op_info.h +++ b/paddle/pir/include/core/op_info.h @@ -13,6 +13,7 @@ // limitations under the License. #pragma once + #include #include diff --git a/paddle/pir/include/core/op_operand.h b/paddle/pir/include/core/op_operand.h index 5366ab390ffa0..4944c31fdb283 100644 --- a/paddle/pir/include/core/op_operand.h +++ b/paddle/pir/include/core/op_operand.h @@ -13,6 +13,7 @@ // limitations under the License. #pragma once + #include #include "paddle/pir/include/core/dll_decl.h" diff --git a/paddle/pir/include/core/op_result.h b/paddle/pir/include/core/op_result.h index 04ae0e848e511..58af7c1a81e97 100644 --- a/paddle/pir/include/core/op_result.h +++ b/paddle/pir/include/core/op_result.h @@ -15,6 +15,7 @@ #pragma once #include "paddle/pir/include/core/value.h" + namespace pir { namespace detail { diff --git a/paddle/pir/include/core/operation_utils.h b/paddle/pir/include/core/operation_utils.h index 4360af17e08a4..891f109eaa8a2 100644 --- a/paddle/pir/include/core/operation_utils.h +++ b/paddle/pir/include/core/operation_utils.h @@ -16,6 +16,7 @@ #include #include + #include "paddle/pir/include/core/attribute.h" #include "paddle/pir/include/core/dll_decl.h" #include "paddle/pir/include/core/op_info.h" diff --git a/paddle/pir/include/core/parameter.h b/paddle/pir/include/core/parameter.h index cad6839ea8851..bfcbe17b3289c 100644 --- a/paddle/pir/include/core/parameter.h +++ b/paddle/pir/include/core/parameter.h @@ -14,6 +14,8 @@ #pragma once +#include + #include "paddle/pir/include/core/type.h" namespace pir { diff --git a/paddle/pir/include/core/storage_manager_support.h b/paddle/pir/include/core/storage_manager_support.h index 9952d2d144d66..7d4d540382dcd 100644 --- a/paddle/pir/include/core/storage_manager_support.h +++ b/paddle/pir/include/core/storage_manager_support.h @@ -15,6 +15,7 @@ #pragma once #include + #include "paddle/pir/include/core/interface_support.h" #include "paddle/pir/include/core/ir_context.h" #include "paddle/pir/include/core/type.h" diff --git a/paddle/pir/include/core/type.h b/paddle/pir/include/core/type.h index 98ef867bef49b..569b356135b18 100644 --- a/paddle/pir/include/core/type.h +++ b/paddle/pir/include/core/type.h @@ -19,6 +19,7 @@ #include "paddle/pir/include/core/cast_utils.h" #include "paddle/pir/include/core/storage_manager_support.h" #include "paddle/pir/include/core/type_id.h" + namespace pir { class TypeStorage; class AbstractType; diff --git a/paddle/pir/include/core/type_id.h b/paddle/pir/include/core/type_id.h index b6e107c777559..2bce5d92752d2 100644 --- a/paddle/pir/include/core/type_id.h +++ b/paddle/pir/include/core/type_id.h @@ -14,7 +14,6 @@ #pragma once -#include #include #include "paddle/pir/include/core/dll_decl.h" diff --git a/paddle/pir/include/core/visitors.h b/paddle/pir/include/core/visitors.h index c2cf137e44624..31f0262865127 100644 --- a/paddle/pir/include/core/visitors.h +++ b/paddle/pir/include/core/visitors.h @@ -14,6 +14,7 @@ #pragma once #include + #include "paddle/pir/include/core/dll_decl.h" namespace pir { diff --git a/paddle/pir/include/dialect/control_flow/ir/cf_op.h b/paddle/pir/include/dialect/control_flow/ir/cf_op.h index 0d6e60a017ab3..e01dec38ce73c 100644 --- a/paddle/pir/include/dialect/control_flow/ir/cf_op.h +++ b/paddle/pir/include/dialect/control_flow/ir/cf_op.h @@ -13,7 +13,9 @@ // limitations under the License. #pragma once + #include + #include "paddle/pir/include/core/builder.h" #include "paddle/pir/include/core/op_base.h" #include "paddle/pir/include/core/op_trait.h" diff --git a/paddle/pir/include/dialect/shape/ir/shape_op.h b/paddle/pir/include/dialect/shape/ir/shape_op.h index 84440d64abc43..3bc7562eaf0e4 100644 --- a/paddle/pir/include/dialect/shape/ir/shape_op.h +++ b/paddle/pir/include/dialect/shape/ir/shape_op.h @@ -15,6 +15,7 @@ #pragma once #include + #include "paddle/pir/include/core/builder.h" #include "paddle/pir/include/core/builtin_type_interfaces.h" #include "paddle/pir/include/core/ir_printer.h" diff --git a/paddle/pir/include/pass/pass.h b/paddle/pir/include/pass/pass.h index 3be04b71051f7..bdd530782c034 100644 --- a/paddle/pir/include/pass/pass.h +++ b/paddle/pir/include/pass/pass.h @@ -136,23 +136,17 @@ class IR_API Pass { // Set a pointer to the attribute. Pass takes ownership of the attribute. template void Set(const std::string& attr_name, AttrType* attr) { - VLOG(3) << "Setting the attribute " << attr_name << " for the pass " - << name(); if (Has(attr_name)) { Erase(attr_name); } attrs_[attr_name] = attr; - attr_dels_[attr_name] = [attr, attr_name]() { - VLOG(8) << "deleting " << attr_name; - delete attr; - }; + attr_dels_[attr_name] = [attr, attr_name]() { delete attr; }; } // Set a pointer to the attribute. Pass doesn't take ownership. Caller // should delete the attribute. template void SetNotOwned(const std::string& attr_name, AttrType* attr) { - VLOG(3) << "Setting the attribute " << attr_name << " for the " << name(); IR_ENFORCE( !Has(attr_name), "Attribute %s already set in the pass.", attr_name); attrs_[attr_name] = attr; diff --git a/paddle/pir/src/core/block.cc b/paddle/pir/src/core/block.cc index 258f681b303cb..39b347dfe81b4 100644 --- a/paddle/pir/src/core/block.cc +++ b/paddle/pir/src/core/block.cc @@ -14,6 +14,7 @@ #include "paddle/pir/include/core/block.h" +#include #include #include "paddle/common/enforce.h" diff --git a/paddle/pir/src/core/block_argument.cc b/paddle/pir/src/core/block_argument.cc index 99a799e9f592e..1966aa191476a 100644 --- a/paddle/pir/src/core/block_argument.cc +++ b/paddle/pir/src/core/block_argument.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + #include "paddle/pir/include/core/block_argument.h" #include "paddle/pir/include/core/builtin_attribute.h" #include "paddle/pir/include/core/operation_utils.h" diff --git a/paddle/pir/src/core/builder.cc b/paddle/pir/src/core/builder.cc index 80147428922ba..2b6d000b8639e 100644 --- a/paddle/pir/src/core/builder.cc +++ b/paddle/pir/src/core/builder.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + #include "paddle/pir/include/core/builder.h" #include "paddle/pir/include/core/builtin_attribute.h" #include "paddle/pir/include/core/builtin_type.h" diff --git a/paddle/pir/src/core/builtin_op.cc b/paddle/pir/src/core/builtin_op.cc index 24b7624dafc63..fca2ebe63eea5 100644 --- a/paddle/pir/src/core/builtin_op.cc +++ b/paddle/pir/src/core/builtin_op.cc @@ -12,9 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "paddle/pir/include/core/builtin_op.h" +#include + #include "paddle/common/enforce.h" #include "paddle/pir/include/core/builtin_attribute.h" +#include "paddle/pir/include/core/builtin_op.h" #include "paddle/pir/include/core/builtin_type.h" namespace pir { diff --git a/paddle/pir/src/core/dialect.cc b/paddle/pir/src/core/dialect.cc index b09709da6b0db..668c56111d0ac 100644 --- a/paddle/pir/src/core/dialect.cc +++ b/paddle/pir/src/core/dialect.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + #include "paddle/pir/include/core/dialect.h" namespace pir { diff --git a/paddle/pir/src/core/ir_context.cc b/paddle/pir/src/core/ir_context.cc index a4839bb2d4a34..90393fe4370b9 100644 --- a/paddle/pir/src/core/ir_context.cc +++ b/paddle/pir/src/core/ir_context.cc @@ -14,6 +14,7 @@ #include "paddle/pir/include/core/ir_context.h" +#include #include #include "paddle/pir/include/core/attribute_base.h" diff --git a/paddle/pir/src/core/op_info_impl.cc b/paddle/pir/src/core/op_info_impl.cc index efbcedf42cc0f..f9d5295671113 100644 --- a/paddle/pir/src/core/op_info_impl.cc +++ b/paddle/pir/src/core/op_info_impl.cc @@ -12,9 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "paddle/pir/src/core/op_info_impl.h" +#include + #include "paddle/pir/include/core/dialect.h" #include "paddle/pir/include/core/interface_support.h" +#include "paddle/pir/src/core/op_info_impl.h" namespace pir { diff --git a/paddle/pir/src/core/op_result_impl.cc b/paddle/pir/src/core/op_result_impl.cc index 3bc9e5023b3b2..dd895cc04d10d 100644 --- a/paddle/pir/src/core/op_result_impl.cc +++ b/paddle/pir/src/core/op_result_impl.cc @@ -12,9 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "paddle/pir/src/core/op_result_impl.h" +#include + #include "paddle/pir/include/core/builtin_attribute.h" #include "paddle/pir/include/core/operation.h" +#include "paddle/pir/src/core/op_result_impl.h" namespace pir { namespace detail { diff --git a/paddle/pir/src/core/op_trait.cc b/paddle/pir/src/core/op_trait.cc index 4261dbcc8a457..39a0f6001da18 100644 --- a/paddle/pir/src/core/op_trait.cc +++ b/paddle/pir/src/core/op_trait.cc @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "paddle/pir/include/core/op_trait.h" +#include + #include "paddle/common/enforce.h" +#include "paddle/pir/include/core/op_trait.h" #include "paddle/pir/include/core/type_utils.h" namespace { diff --git a/paddle/pir/src/core/operation.cc b/paddle/pir/src/core/operation.cc index e7dce069ebd81..923316c765245 100644 --- a/paddle/pir/src/core/operation.cc +++ b/paddle/pir/src/core/operation.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include diff --git a/paddle/pir/src/core/storage_manager.cc b/paddle/pir/src/core/storage_manager.cc index 6018917062d43..a6fb1621292a6 100644 --- a/paddle/pir/src/core/storage_manager.cc +++ b/paddle/pir/src/core/storage_manager.cc @@ -14,6 +14,7 @@ #include "paddle/pir/include/core/storage_manager.h" +#include #include #include diff --git a/paddle/pir/src/core/value_impl.cc b/paddle/pir/src/core/value_impl.cc index 37dcb48370b6e..5b37e24e8240d 100644 --- a/paddle/pir/src/core/value_impl.cc +++ b/paddle/pir/src/core/value_impl.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + #include "paddle/pir/src/core/value_impl.h" namespace { diff --git a/paddle/pir/src/dialect/control_flow/ir/cf_op.cc b/paddle/pir/src/dialect/control_flow/ir/cf_op.cc index 3ead6991b272a..8b4cf4727df5b 100644 --- a/paddle/pir/src/dialect/control_flow/ir/cf_op.cc +++ b/paddle/pir/src/dialect/control_flow/ir/cf_op.cc @@ -12,9 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "paddle/pir/include/dialect/control_flow/ir/cf_op.h" +#include + #include "paddle/pir/include/core/builtin_type.h" #include "paddle/pir/include/core/ir_printer.h" +#include "paddle/pir/include/dialect/control_flow/ir/cf_op.h" #include "paddle/pir/include/dialect/control_flow/ir/cf_type.h" namespace pir { diff --git a/paddle/pir/src/pass/print_statistics.cc b/paddle/pir/src/pass/print_statistics.cc index 2b92c9e4cc9f6..21d4d67945ce8 100644 --- a/paddle/pir/src/pass/print_statistics.cc +++ b/paddle/pir/src/pass/print_statistics.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + #include "paddle/common/macros.h" #include "paddle/pir/include/core/operation.h" #include "paddle/pir/include/pass/pass.h" diff --git a/paddle/pir/src/pattern_rewrite/pattern_rewrite_driver.cc b/paddle/pir/src/pattern_rewrite/pattern_rewrite_driver.cc index 474e395c10b6c..7bb086014c8f4 100644 --- a/paddle/pir/src/pattern_rewrite/pattern_rewrite_driver.cc +++ b/paddle/pir/src/pattern_rewrite/pattern_rewrite_driver.cc @@ -14,6 +14,7 @@ #include "paddle/pir/include/pattern_rewrite/pattern_rewrite_driver.h" +#include #include #include #include diff --git a/python/paddle/utils/cpp_extension/cpp_extension.py b/python/paddle/utils/cpp_extension/cpp_extension.py index 0ea8bb96566ab..35bda07cab67b 100644 --- a/python/paddle/utils/cpp_extension/cpp_extension.py +++ b/python/paddle/utils/cpp_extension/cpp_extension.py @@ -488,7 +488,7 @@ def unix_custom_single_compiler( cflags.append('-DPADDLE_WITH_CUDA') add_std_without_repeat( - cflags, self.compiler.compiler_type, use_std14=True + cflags, self.compiler.compiler_type, use_std17=True ) original_compile(obj, src, ext, cc_args, cflags, pp_opts) finally: diff --git a/python/paddle/utils/cpp_extension/extension_utils.py b/python/paddle/utils/cpp_extension/extension_utils.py index 55a9a2e993f31..009176f61fe80 100644 --- a/python/paddle/utils/cpp_extension/extension_utils.py +++ b/python/paddle/utils/cpp_extension/extension_utils.py @@ -418,13 +418,13 @@ def prepare_win_cudaflags(cflags): return cflags -def add_std_without_repeat(cflags, compiler_type, use_std14=False): +def add_std_without_repeat(cflags, compiler_type, use_std17=False): """ - Append -std=c++11/14 in cflags if without specific it before. + Append -std=c++14/17 in cflags if without specific it before. """ cpp_flag_prefix = '/std:' if compiler_type == 'msvc' else '-std=' if not any(cpp_flag_prefix in flag for flag in cflags): - suffix = 'c++14' if use_std14 else 'c++11' + suffix = 'c++17' if use_std17 else 'c++14' cpp_flag = cpp_flag_prefix + suffix cflags.append(cpp_flag) diff --git a/python/setup.py.in b/python/setup.py.in index f140b66bd1c44..9fd352ddd26be 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -874,7 +874,13 @@ headers = ( # utils api headers list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/utils', recursive=True)) + # paddle utils headers # init headers - list(find_files('init_phi.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/platform'))) # phi init headers + list(find_files('init_phi.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/platform')) + # phi init headers + # init headers + list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/pir/include')) + # pir init headers + # init headers + list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/pir/drr/include')) + # drr init headers + # init headers + list(find_files('transform_general_functions.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/pir/transforms'))) # pass utils init headers jit_layer_headers = ['layer.h', 'serializer.h', 'serializer_utils.h', 'all.h', 'function.h'] for f in jit_layer_headers: diff --git a/setup.py b/setup.py index 215f767b73d53..2601cfe7b11b3 100644 --- a/setup.py +++ b/setup.py @@ -1370,6 +1370,27 @@ def get_headers(): recursive=True, ) ) + + list( # pir init headers + find_files( + '*.h', + paddle_source_dir + '/paddle/pir/include', + recursive=True, + ) + ) + + list( # drr init headers + find_files( + '*.h', + paddle_source_dir + '/paddle/fluid/pir/drr/include', + recursive=True, + ) + ) + + list( # pass utils init headers + find_files( + 'transform_general_functions.h', + paddle_source_dir + '/paddle/fluid/pir/transforms', + recursive=True, + ) + ) ) jit_layer_headers = [ diff --git a/test/cpp/pir/tools/test_op.h b/test/cpp/pir/tools/test_op.h index 1f61f0ff001ba..31fc4445c36ee 100644 --- a/test/cpp/pir/tools/test_op.h +++ b/test/cpp/pir/tools/test_op.h @@ -14,6 +14,8 @@ #pragma once +#include + #include "paddle/pir/include/core/builder.h" #include "paddle/pir/include/core/builtin_type.h" #include "paddle/pir/include/core/op_base.h" From 4ee55da3426a40e607a1f9615a0f10040c48e4e0 Mon Sep 17 00:00:00 2001 From: risemeup1 <62429225+risemeup1@users.noreply.github.com> Date: Thu, 29 Feb 2024 14:37:37 +0800 Subject: [PATCH 172/282] Revert "cinn (#62177)" (#62221) This reverts commit ee2e49a95365732442df8c7de37436166bad102f. --- paddle/scripts/paddle_build.sh | 3 --- tools/coverage/paddle_coverage.sh | 31 ------------------------------- 2 files changed, 34 deletions(-) diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 19e9cf3803a84..71ee30a115ef7 100644 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -4235,9 +4235,6 @@ function main() { ;; test) parallel_test - if [ "${WITH_CINN}" == "ON" ] ; then - check_coverage - fi ;; single_test) single_test $2 diff --git a/tools/coverage/paddle_coverage.sh b/tools/coverage/paddle_coverage.sh index 90e02715876ca..ee2a38f5da851 100644 --- a/tools/coverage/paddle_coverage.sh +++ b/tools/coverage/paddle_coverage.sh @@ -39,28 +39,6 @@ lcov --capture -d ./ -o coverage.info --rc lcov_branch_coverage=0 # full html report -function gen_full_html_report_cinn(){ - lcov --extract coverage.info \ - '/paddle/paddle/cinn/adt/*' \ - '/paddle/paddle/cinn/api/*' \ - '/paddle/paddle/cinn/ast_gen_ius/*' \ - '/paddle/paddle/cinn/auto_schedule/*' \ - '/paddle/paddle/cinn/backends/*' \ - '/paddle/paddle/cinn/common/*' \ - '/paddle/paddle/cinn/frontend/*' \ - '/paddle/paddle/cinn/hlir/*' \ - '/paddle/paddle/cinn/ir/*' \ - '/paddle/paddle/cinn/lang/*' \ - '/paddle/paddle/cinn/optim/*' \ - '/paddle/paddle/cinn/poly/*' \ - '/paddle/paddle/cinn/pybind/*' \ - '/paddle/paddle/cinn/runtime/*' \ - '/paddle/paddle/cinn/utils/*' \ - -o coverage-full.tmp \ - --rc lcov_branch_coverage=0 -} - - function gen_full_html_report() { lcov --extract coverage.info \ '/paddle/paddle/fluid/framework/*' \ @@ -142,12 +120,6 @@ else gen_full_html_report || true fi -if [ ${WITH_CINN:-OFF} == "ON" ]; then - gen_full_html_report_cinn || true -else - gen_full_html_report || true -fi - # diff html report function gen_diff_html_report() { @@ -250,8 +222,5 @@ fi if [ "$COVERAGE_LINES_ASSERT" = "1" ] || [ "$PYTHON_COVERAGE_LINES_ASSERT" = "1" ]; then echo "exit 9" > /tmp/paddle_coverage.result - if [ "${WITH_CINN}" == "ON" ]; then - echo "You must one RD(liuhongyu or lanxiang or zhenghuihuang or tianchao zhangliujie)to approval this PR." - fi exit 9 fi From f1e3179b95b7de66baf09765c97ceaa7dc590547 Mon Sep 17 00:00:00 2001 From: wanghuancoder Date: Thu, 29 Feb 2024 14:45:52 +0800 Subject: [PATCH 173/282] [PIR] refine pir add_n and pir onednn support add_n (#62024) * pir onednn support add_n --- .../ir_adaptor/translator/op_translator.cc | 20 +- .../fluid/pir/dialect/op_generator/op_gen.py | 1 - .../pir/dialect/op_generator/ops_api_gen.py | 1 - .../pir/dialect/operator/ir/manual_op.cc | 194 +----------------- .../fluid/pir/dialect/operator/ir/manual_op.h | 24 --- .../fluid/pir/dialect/operator/ir/onednn.yaml | 10 + paddle/fluid/pir/dialect/operator/ir/ops.yaml | 21 +- .../dialect/operator/ir/ops_onednn_extra.yaml | 3 +- .../pir/transforms/pd_op_to_kernel_pass.cc | 2 +- test/mkldnn/test_sum_bf16_mkldnn_op.py | 2 +- test/mkldnn/test_sum_mkldnn_op.py | 6 +- 11 files changed, 34 insertions(+), 250 deletions(-) diff --git a/paddle/fluid/ir_adaptor/translator/op_translator.cc b/paddle/fluid/ir_adaptor/translator/op_translator.cc index 6e1ec454b6bab..1c75d198ef07d 100644 --- a/paddle/fluid/ir_adaptor/translator/op_translator.cc +++ b/paddle/fluid/ir_adaptor/translator/op_translator.cc @@ -1355,13 +1355,21 @@ struct ShadowOutputOpTranscriber : public OpTranscriber { struct AddNOpTranscriber : public OpTranscriber { pir::OpInfo LookUpOpInfo(pir::IrContext* ctx, const OpDesc& op_desc) override { - std::string target_op_name = - GetPrefix(ctx, op_desc) + OpNameCompatibleMapping(op_desc.Type()); - if (IsInplace(op_desc)) { - target_op_name += "_"; - } else { - target_op_name += "_with_kernel"; + auto prefix = GetPrefix(ctx, op_desc); + std::string target_op_name; +#ifdef PADDLE_WITH_DNNL + if (prefix == kOneDNNTargetDialectPrefix) { + target_op_name = std::string(kOneDNNTargetDialectPrefix) + "add_n_onednn"; + } else // NOLINT +#endif + { + target_op_name = + GetPrefix(ctx, op_desc) + OpNameCompatibleMapping(op_desc.Type()); + if (IsInplace(op_desc)) { + target_op_name += "_"; + } } + const auto& op_info = ctx->GetRegisteredOpInfo(target_op_name); if (!op_info) { IR_THROW("Op add_n should have corresponding OpInfo %s", target_op_name); diff --git a/paddle/fluid/pir/dialect/op_generator/op_gen.py b/paddle/fluid/pir/dialect/op_generator/op_gen.py index 67462983fbf0a..5513bbb3f5552 100644 --- a/paddle/fluid/pir/dialect/op_generator/op_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/op_gen.py @@ -312,7 +312,6 @@ class {TEST_API} {op_name} : public pir::Op<{op_name}{interfaces}{traits}> {{ PD_MANUAL_OP_LIST = { 'add_n', 'add_n_', - 'add_n_with_kernel', 'split_grad', 'expand', 'increment', diff --git a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py index 54b56a2e3c887..534ea49a61f45 100644 --- a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py @@ -118,7 +118,6 @@ NO_NEED_GEN_STATIC_ONLY_APIS = [ 'add_n_', - 'add_n_with_kernel', 'c_allgather', 'c_allreduce_max', 'c_allreduce_min', diff --git a/paddle/fluid/pir/dialect/operator/ir/manual_op.cc b/paddle/fluid/pir/dialect/operator/ir/manual_op.cc index 0863737842ba2..ec61f6c7dd88d 100644 --- a/paddle/fluid/pir/dialect/operator/ir/manual_op.cc +++ b/paddle/fluid/pir/dialect/operator/ir/manual_op.cc @@ -13,8 +13,7 @@ // limitations under the License. #ifdef GET_OP_LIST #undef GET_OP_LIST -paddle::dialect::AddNOp, paddle::dialect::AddN_Op, - paddle::dialect::AddNWithKernelOp, paddle::dialect::AddNArrayOp, +paddle::dialect::AddNOp, paddle::dialect::AddN_Op, paddle::dialect::AddNArrayOp, paddle::dialect::FusedGemmEpilogueOp, paddle::dialect::AssignOut_Op, paddle::dialect::FusedGemmEpilogueGradOp, paddle::dialect::SplitGradOp, paddle::dialect::ExpandOp, paddle::dialect::CreateArrayOp, @@ -372,196 +371,6 @@ std::vector AddN_Op::InferMeta( return argument_outputs; } -OpInfoTuple AddNWithKernelOp::GetOpInfo() { - std::vector inputs = { - paddle::dialect::OpInputInfo( - "inputs", - "pir::VectorType", - false, - false, - false, - true)}; - std::vector attributes = {}; - std::vector outputs = { - paddle::dialect::OpOutputInfo( - "out", "paddle::dialect::DenseTensorType", false, false)}; - paddle::dialect::OpRunTimeInfo run_time_info = paddle::dialect::OpRunTimeInfo( - "AddNInferMeta", {"inputs"}, "add_n", {"inputs"}, {}, {}, {}, {}); - return std::make_tuple( - inputs, attributes, outputs, run_time_info, "add_n_with_kernel"); -} - -void AddNWithKernelOp::Build(pir::Builder &builder, - pir::OperationArgument &argument, - pir::Value inputs_) { - VLOG(4) << "Start build AddNWithKernelOp"; - - VLOG(4) << "Builder construction inputs"; - std::vector argument_inputs = {inputs_}; - argument.AddInput(inputs_); - - VLOG(4) << "Builder construction attributes"; - pir::AttributeMap argument_attributes = {}; - std::vector argument_outputs = - AddNWithKernelOp::InferMeta(argument_inputs, argument_attributes); - - argument.AddOutputs(argument_outputs.begin(), argument_outputs.end()); -} - -void AddNWithKernelOp::VerifySig() { - VLOG(4) << "Start Verifying inputs, outputs and attributes for: " - "AddNWithKernelOp."; - VLOG(4) << "Verifying inputs:"; - { - auto input_size = num_operands(); - PADDLE_ENFORCE_EQ( - input_size, - 1u, - phi::errors::PreconditionNotMet( - "The size %d of inputs must be equal to 1.", input_size)); - if (auto vec_type = - (*this)->operand_source(0).type().dyn_cast()) { - for (size_t i = 0; i < vec_type.size(); ++i) { - PADDLE_ENFORCE(vec_type[i].isa() || - vec_type[i].isa(), - phi::errors::PreconditionNotMet( - "Type validation failed for the 0th input.")); - } - } else { - PADDLE_ENFORCE((*this)->operand_source(0) - .type() - .isa() || - (*this) - ->operand_source(0) - .type() - .isa(), - phi::errors::PreconditionNotMet( - "Type validation failed for the 0th input.")); - } - } - VLOG(4) << "Verifying attributes:"; - { - // Attributes num is 0, not need to check attributes type. - } - VLOG(4) << "Verifying outputs:"; - { - auto output_size = num_results(); - PADDLE_ENFORCE_EQ( - output_size, - 1u, - phi::errors::PreconditionNotMet( - "The size %d of outputs must be equal to 1.", output_size)); - PADDLE_ENFORCE( - (*this)->result(0).type().isa() || - (*this)->result(0).type().isa(), - phi::errors::PreconditionNotMet( - "Type validation failed for the 0th output.")); - } - VLOG(4) << "End Verifying for: AddNWithKernelOp."; -} - -void AddNWithKernelOp::InferMeta(phi::InferMetaContext *infer_meta) { - auto fn = PD_INFER_META(phi::AddNInferMeta); - fn(infer_meta); -} - -std::vector AddNWithKernelOp::InferMeta( - const std::vector &input_values, - const pir::AttributeMap &attributes) { - VLOG(4) << "Start infermeta AddNWithKernelOp"; - IR_ENFORCE(input_values.size() == 1, - "Num of inputs is expected to be 1 but got %d.", - input_values.size()); - pir::Value inputs_ = input_values[0]; - - VLOG(4) << "Builder construction outputs"; - pir::VectorType inputs = inputs_.type().dyn_cast(); - std::vector vec_dense_inputs; - for (size_t i = 0; i < static_cast(inputs.size()); i++) { - if (inputs[i].isa()) { - vec_dense_inputs.push_back(paddle::dialect::IrTensor( - paddle::dialect::TransToPhiDataType( - inputs[i].dyn_cast().dtype()), - inputs[i].dyn_cast().dims(), - inputs[i].dyn_cast().data_layout(), - inputs[i].dyn_cast().lod(), - inputs[i].dyn_cast().offset())); - } else if (inputs[i].isa()) { - vec_dense_inputs.push_back(paddle::dialect::IrTensor( - TransToPhiDataType( - inputs[i] - .dyn_cast() - .dtype()), - inputs[i] - .dyn_cast() - .dims(), - inputs[i] - .dyn_cast() - .data_layout(), - inputs[i].dyn_cast().lod(), - inputs[i] - .dyn_cast() - .offset())); - } else if (inputs[i].isa()) { - vec_dense_inputs.push_back(paddle::dialect::IrTensor( - paddle::dialect::TransToPhiDataType( - inputs[i].dyn_cast().dtype()), - inputs[i].dyn_cast().dims(), - inputs[i].dyn_cast().data_layout(), - inputs[i].dyn_cast().lod(), - inputs[i].dyn_cast().offset())); - } else if (inputs[i].isa()) { - vec_dense_inputs.push_back(paddle::dialect::IrTensor( - TransToPhiDataType( - inputs[i] - .dyn_cast() - .dtype()), - inputs[i] - .dyn_cast() - .dims(), - inputs[i] - .dyn_cast() - .data_layout(), - inputs[i] - .dyn_cast() - .lod(), - inputs[i] - .dyn_cast() - .offset())); - } else { - PADDLE_THROW(phi::errors::Unimplemented( - "Only support DenseTensorType or AllocatedDenseTensorType or " - "SelectedRowsType or AllocatedSelectedRowsType")); - } - } - - std::vector vec_meta_inputs; - for (size_t i = 0; i < vec_dense_inputs.size(); i++) { - vec_meta_inputs.push_back( - paddle::dialect::IrMetaTensor(&vec_dense_inputs[i])); - } - - std::vector meta_inputs; - for (size_t i = 0; i < static_cast(vec_meta_inputs.size()); i++) { - meta_inputs.push_back(&vec_meta_inputs[i]); - } - paddle::dialect::IrTensor dense_out; - paddle::dialect::IrMetaTensor meta_out(&dense_out); - - phi::AddNInferMeta(meta_inputs, &meta_out, phi::MetaConfig(false, false)); - - std::vector argument_outputs; - pir::Type out_dense_tensor_type = paddle::dialect::DenseTensorType::get( - pir::IrContext::Instance(), - paddle::dialect::TransToIrDataType(dense_out.dtype()), - dense_out.dims(), - dense_out.layout(), - dense_out.lod(), - dense_out.offset()); - argument_outputs.push_back(out_dense_tensor_type); - return argument_outputs; -} - OpInfoTuple AddNArrayOp::GetOpInfo() { std::vector inputs = { OpInputInfo("inputs", @@ -4701,7 +4510,6 @@ phi::DataType ArrayPopOp::GetKernelTypeForVar( IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::AddNOp) IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::SplitGradOp) IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::AddN_Op) -IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::AddNWithKernelOp) IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::AddNArrayOp) IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::AssignOut_Op) IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::FusedGemmEpilogueOp) diff --git a/paddle/fluid/pir/dialect/operator/ir/manual_op.h b/paddle/fluid/pir/dialect/operator/ir/manual_op.h index ea836f68a4959..1f8be853ddcf5 100644 --- a/paddle/fluid/pir/dialect/operator/ir/manual_op.h +++ b/paddle/fluid/pir/dialect/operator/ir/manual_op.h @@ -90,29 +90,6 @@ class AddN_Op : public pir::Op { - public: - using Op::Op; - static const char *name() { return "pd_op.add_n_with_kernel"; } - static constexpr const char **attributes_name = nullptr; - static constexpr uint32_t attributes_num = 0; - static OpInfoTuple GetOpInfo(); - static void Build(pir::Builder &builder, // NOLINT - pir::OperationArgument &argument, // NOLINT - pir::Value inputs_); - - void VerifySig(); - pir::Value inputs() { return operand_source(0); } - pir::Value out() { return result(0); } - - static void InferMeta(phi::InferMetaContext *infer_meta); - static std::vector InferMeta( - const std::vector &input_values, - const pir::AttributeMap &attributes); -}; - class AddNArrayOp : public pir::Op { @@ -818,7 +795,6 @@ class ArrayPopOp : public pir::OpOpRuntimeInfo().kernel_func; } - if (op_item->isa() || op_item->isa()) { + if (op_item->isa() || op_item->isa()) { if (op_item->result(0).type().isa()) { kernel_fn_str = "add_n_sr"; } diff --git a/test/mkldnn/test_sum_bf16_mkldnn_op.py b/test/mkldnn/test_sum_bf16_mkldnn_op.py index 8fbef74e38d2d..c59fa0d7b8359 100644 --- a/test/mkldnn/test_sum_bf16_mkldnn_op.py +++ b/test/mkldnn/test_sum_bf16_mkldnn_op.py @@ -48,7 +48,7 @@ def setUp(self): self.attrs = {'use_mkldnn': self.use_mkldnn} def test_check_output(self): - self.check_output_with_place(core.CPUPlace()) + self.check_output_with_place(core.CPUPlace(), check_pir_onednn=True) def test_check_grad(self): pass diff --git a/test/mkldnn/test_sum_mkldnn_op.py b/test/mkldnn/test_sum_mkldnn_op.py index 6750f1a79c7ce..fc86c6834b940 100644 --- a/test/mkldnn/test_sum_mkldnn_op.py +++ b/test/mkldnn/test_sum_mkldnn_op.py @@ -39,11 +39,13 @@ def init_data_type(self): def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_output(check_dygraph=False) + self.check_output(check_dygraph=False, check_pir_onednn=True) def test_check_grad(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_grad(['x0'], 'Out', check_dygraph=False) + self.check_grad( + ['x0'], 'Out', check_dygraph=False, check_pir_onednn=True + ) class TestMKLDNNSumInplaceOp(unittest.TestCase): From ba71b838d694912576e3d3512ff15b737fa4c73c Mon Sep 17 00:00:00 2001 From: zhangbo9674 <82555433+zhangbo9674@users.noreply.github.com> Date: Thu, 29 Feb 2024 15:28:45 +0800 Subject: [PATCH 174/282] fix (#62216) --- paddle/fluid/ir_adaptor/translator/program_translator.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/ir_adaptor/translator/program_translator.cc b/paddle/fluid/ir_adaptor/translator/program_translator.cc index 608d24a60b577..e40da8a7b8fb6 100644 --- a/paddle/fluid/ir_adaptor/translator/program_translator.cc +++ b/paddle/fluid/ir_adaptor/translator/program_translator.cc @@ -309,7 +309,7 @@ void ProgramTranslator::TranslateIfOperation( TranslationContext* translation_ctx, pir::Block* dst_block, bool for_bwd) { - VLOG(8) << "=============>Start to translate if op:" << op; + LOG_FIRST_N(INFO, 1) << "Translate ConditionalBlockOp"; auto& type_translator = TypeTranslator::instance(); auto cond_op_cond = op->Input("Cond")[0]; @@ -479,7 +479,7 @@ void ProgramTranslator::TranslateWhileOperation( const OpDesc* op, TranslationContext* translation_ctx, pir::Block* dst_block) { - VLOG(8) << "=============>Start to translate while op:" << op; + LOG_FIRST_N(INFO, 1) << "Translate WhileOp"; auto& sub_block = legacy_program_->Block(op->GetBlockAttrId("sub_block")); auto& inputs = op->Output("Out"); auto& cond_var = op->Input("Condition")[0]; From 4865fed1cd3f56dfffd5388bc4152bc64dc7dba3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=98=A5=E4=B9=94?= <83450930+Liyulingyue@users.noreply.github.com> Date: Thu, 29 Feb 2024 16:50:24 +0800 Subject: [PATCH 175/282] Delete useless test files (#62209) * Update CMakeLists.txt * mv cc file * add TEST_API * delete use_op_itself * Update test_reference_count_pass_last_lived_ops.cc * Update CMakeLists.txt * Delete paddle/fluid/framework/ir/memory_optimize_pass/share_varinfo_into_cinn_pass_test.cc * Delete paddle/fluid/framework/ir/memory_optimize_pass/test_reference_count_pass_last_lived_ops.cc --- .../share_varinfo_into_cinn_pass_test.cc | 154 ------------ ...est_reference_count_pass_last_lived_ops.cc | 228 ------------------ 2 files changed, 382 deletions(-) delete mode 100644 paddle/fluid/framework/ir/memory_optimize_pass/share_varinfo_into_cinn_pass_test.cc delete mode 100644 paddle/fluid/framework/ir/memory_optimize_pass/test_reference_count_pass_last_lived_ops.cc diff --git a/paddle/fluid/framework/ir/memory_optimize_pass/share_varinfo_into_cinn_pass_test.cc b/paddle/fluid/framework/ir/memory_optimize_pass/share_varinfo_into_cinn_pass_test.cc deleted file mode 100644 index 1f78e293a21a3..0000000000000 --- a/paddle/fluid/framework/ir/memory_optimize_pass/share_varinfo_into_cinn_pass_test.cc +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "gtest/gtest.h" -#include "paddle/fluid/framework/details/computation_op_handle.h" -#include "paddle/fluid/framework/details/eager_deletion_op_handle.h" -#include "paddle/fluid/framework/ir/graph.h" -#include "paddle/fluid/framework/ir/memory_optimize_pass/memory_optimization_var_info.h" -#include "paddle/fluid/framework/ir/pass.h" -#include "paddle/fluid/framework/paddle2cinn/build_cinn_pass.h" -#include "paddle/fluid/framework/paddle2cinn/cinn_compiler.h" -#include "paddle/fluid/framework/parallel_executor.h" -#include "paddle/fluid/framework/program_desc.h" - -USE_OP_ITSELF(mul); -USE_OP_ITSELF(elementwise_add); - -USE_OP_ITSELF(cinn_launch); -PD_DECLARE_KERNEL(cinn_launch, CPU, ALL_LAYOUT); -#ifdef PADDLE_WITH_CUDA -PD_DECLARE_KERNEL(cinn_launch, GPU, ALL_LAYOUT); -#endif - -namespace paddle::framework { - -using Name2VarInfoMap = - std::unordered_map>; - -static ProgramDesc BuildProgramInsideCinnLaunchOp() { - ProgramDesc program; - auto* block = program.MutableBlock(0); - block->Var("var1"); - block->Var("var2"); - block->Var("var3"); - block->Var("var4"); - block->Var("var5"); - - auto add_op = - std::unique_ptr(new OpDesc("elementwise_add", - {{"X", {"var1"}}, {"Y", {"var2"}}}, - {{"Out", {"var3"}}}, - {})); - block->AppendAllocatedOp(std::move(add_op)); - auto mul_op = std::unique_ptr(new OpDesc( - "mul", {{"X", {"var3"}}, {"Y", {"var4"}}}, {{"Out", {"var5"}}}, {})); - block->AppendAllocatedOp(std::move(mul_op)); - return program; -} - -static ProgramDesc BuildProgramWithCinnLaunchOp(int64_t compilation_key) { - // create a cinn_launch op - ProgramDesc program; - auto* block = program.MutableBlock(0); - block->Var("var1"); - block->Var("var2"); - block->Var("var4"); - block->Var("var5"); - - auto cinn_launch_op = std::unique_ptr( - new OpDesc("cinn_launch", - {{"X", {"var1", "var2", "var4"}}}, - {{"Out", {"var5"}}}, - {{"compilation_key", compilation_key}})); - block->AppendAllocatedOp(std::move(cinn_launch_op)); - return program; -} - -struct TestPassContext { - explicit TestPassContext(const ProgramDesc& program) { - graph = std::make_unique(program); - details::BuildStrategy build_strategy; - details::ExecutionStrategy exec_strategy; - exec_strategy.use_device_ = paddle::platform::kCUDA; - executor.reset(new ParallelExecutor(platform::CUDAPlace(0), - &scope, - exec_strategy, - build_strategy, - graph.get())); - } - - Scope scope; - std::unique_ptr graph; - std::unique_ptr executor; -}; - -TEST(ShareMemInfoToSubGraphPassTest, test_main_graph_share_varinfo) { - // add a subgraph to CinnCompiler - auto subgraph = std::make_unique(BuildProgramInsideCinnLaunchOp()); - subgraph->GetOrInit( - paddle2cinn::kMemOptVarInfoFromMainGraph); - auto compilation_key = - paddle2cinn::CinnCompiler::GetInstance()->AddGraph(std::move(subgraph)); - - // build test data and apply pass - auto context = std::make_unique( - BuildProgramWithCinnLaunchOp(compilation_key)); - - // check result - const ir::Graph& result_subgraph = - paddle2cinn::CinnCompiler::GetInstance()->FindGraph(compilation_key); - const auto& dst_varinfo_map = result_subgraph.Get( - paddle2cinn::kMemOptVarInfoFromMainGraph); - ASSERT_EQ(dst_varinfo_map.size(), 4); - EXPECT_EQ(dst_varinfo_map.count("var1"), 1); - EXPECT_EQ(dst_varinfo_map.count("var5"), 1); - EXPECT_EQ(dst_varinfo_map.at("var1").use_count(), 2); - EXPECT_EQ(dst_varinfo_map.at("var5").use_count(), 2); -} - -TEST(ShareMemInfoToSubGraphPassTest, test_subgraph_take_varinfo) { - // build test data and apply pass - auto context = - std::make_unique(BuildProgramInsideCinnLaunchOp()); - auto& varinfo_map_shared = context->graph->GetOrInit( - paddle2cinn::kMemOptVarInfoFromMainGraph); - varinfo_map_shared = { - {"var1", std::make_shared("var1", 1)}, - {"var2", std::make_shared("var2", 2)}, - }; - - ir::MemOptVarInfoMapList varinfo_maps(1); - auto& dst_varinfo_map = varinfo_maps.front(); - dst_varinfo_map = {{"var1", std::make_shared("var1", 1)}, - {"var2", std::make_shared("var2", 1)}, - {"var3", std::make_shared("var3", 1)}, - {"var4", std::make_shared("var4", 1)}, - {"var5", std::make_shared("var5", 1)}}; - auto share_pass = - ir::PassRegistry::Instance().Get("share_varinfo_into_cinn_pass"); - share_pass->SetNotOwned(ir::kMemOptVarInfoMapList, &varinfo_maps); - share_pass->Apply(context->graph.get()); - - // check result - ASSERT_NE(dst_varinfo_map.at("var1")->ParentHolder(), nullptr); - ASSERT_NE(dst_varinfo_map.at("var2")->ParentHolder(), nullptr); - ASSERT_EQ(dst_varinfo_map.at("var3")->ParentHolder(), nullptr); - ASSERT_EQ(dst_varinfo_map.at("var4")->ParentHolder(), nullptr); - ASSERT_EQ(dst_varinfo_map.at("var5")->ParentHolder(), nullptr); -} - -} // namespace paddle::framework diff --git a/paddle/fluid/framework/ir/memory_optimize_pass/test_reference_count_pass_last_lived_ops.cc b/paddle/fluid/framework/ir/memory_optimize_pass/test_reference_count_pass_last_lived_ops.cc deleted file mode 100644 index eeec6fd8788d4..0000000000000 --- a/paddle/fluid/framework/ir/memory_optimize_pass/test_reference_count_pass_last_lived_ops.cc +++ /dev/null @@ -1,228 +0,0 @@ -// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "gtest/gtest.h" -#include "paddle/common/flags.h" -#include "paddle/fluid/framework/details/multi_devices_helper.h" -#include "paddle/fluid/framework/ir/graph.h" -#include "paddle/fluid/framework/ir/graph_helper.h" -#include "paddle/fluid/framework/ir/memory_optimize_pass/memory_optimization_var_info.h" -#include "paddle/fluid/framework/ir/memory_optimize_pass/reference_count_pass_helper.h" -#include "paddle/fluid/framework/parallel_executor.h" -#include "paddle/fluid/framework/program_desc.h" -#include "paddle/phi/core/kernel_registry.h" - -COMMON_DECLARE_double(eager_delete_tensor_gb); - -namespace paddle { -namespace framework { -namespace p = paddle::platform; - -static std::vector CreatePlaces(size_t num, bool use_cuda) { - std::vector result; - result.reserve(num); - for (size_t i = 0; i < num; ++i) { - if (use_cuda) { - result.emplace_back(platform::CUDAPlace(static_cast(i))); - } else { - result.emplace_back(platform::CPUPlace()); - } - } - return result; -} - -static void NewVar(BlockDesc *block, - const std::string &name, - const std::vector &shape) { - auto *var_desc = block->Var(name); - var_desc->SetShape(shape); -} - -static void AppendOp(BlockDesc *block, - const std::string &type, - VariableNameMap inputs, - VariableNameMap outputs, - AttributeMap attrs) { - auto &op_info = OpInfoMap::Instance().Get(type); - if (op_info.Checker()) { - op_info.Checker()->Check(&attrs); - } - - auto *op = block->AppendOp(); - op->SetType(type); - for (auto &pair : inputs) { - op->SetInput(pair.first, pair.second); - } - - for (auto &pair : outputs) { - op->SetOutput(pair.first, pair.second); - for (auto &var_name : pair.second) { - if (!block->FindVarRecursive(var_name)) { - NewVar(block, var_name, {}); - } - } - } - - op->SetAttrMap(attrs); - op->InferVarType(block); - op->InferShape(*block); -} - -class ReferenceCountPassTestHelper { - public: - ReferenceCountPassTestHelper(const ProgramDesc &program, bool use_cuda) - : graph_(program) { - details::BuildStrategy build_strategy; - build_strategy.enable_inplace_ = false; - build_strategy.memory_optimize_ = false; - FLAGS_eager_delete_tensor_gb = -1; - - details::ExecutionStrategy exec_strategy; - exec_strategy.use_device_ = use_cuda ? p::kCUDA : p::kCPU; - - executor_ = std::make_unique(CreatePlaces(1, use_cuda), - std::vector(), - "", - &scope_, - std::vector(), - exec_strategy, - build_strategy, - &graph_); - - auto ref_cnt_pass = - ir::PassRegistry::Instance().Get("reference_count_pass"); - ref_cnt_pass->SetNotOwned(ir::kMemOptVarInfoMapList, &mem_opt_var_infos_); - ref_cnt_pass->SetNotOwned(ir::kLastLiveOpsOfVars, &last_live_ops_of_vars_); - ref_cnt_pass->Apply(&const_cast(executor_->Graph())); - } - - bool IsLastLivedOps(const std::string &name, - std::vector ops) const { - std::sort(ops.begin(), ops.end()); - return LastLivedOpTypes(name) == ops; - } - - std::vector LastLivedOps(const std::string &name) const { - auto &ops = last_live_ops_of_vars_[0].at(name).ops(); - std::vector ret; - ret.reserve(ops.size()); - for (auto *op : ops) { - ret.emplace_back(op->GetOp()); - } - return ret; - } - - private: - std::vector LastLivedOpTypes(const std::string &name) const { - auto iter = last_live_ops_of_vars_[0].find(name); - std::vector ret; - if (iter != last_live_ops_of_vars_[0].end()) { - for (auto *op : iter->second.ops()) { - ret.emplace_back(op->GetOp()->Type()); - } - } - std::sort(ret.begin(), ret.end()); - return ret; - } - - private: - ir::Graph graph_; - Scope scope_; - std::unique_ptr executor_; - - ir::MemOptVarInfoMapList mem_opt_var_infos_; - std::vector last_live_ops_of_vars_; -}; - -TEST(test_reference_count_pass, test_no_need_buffer_var_shrink) { - ProgramDesc program; - auto *block = program.MutableBlock(0); - std::vector shape{{3, 4, 5}}; - - /** - * The network is: - * - * x0 = fluid.layer.data(...) - * x1 = scale(x0, scale=1) - * x2 = scale(x1, scale=2) - * x3 = elementwise_mul(x1, x2) - * scale(x3, out=x1, scale=3) # produce a new version of x1 - * x4, x5 = elementwise_add_grad(dout=x3, x=x2, y=x1) - * x6 = elementwise_mul(x4, x5) - * x7 = elementwise_add(x5, x5) - */ - std::string x0 = "x0"; - std::string x1 = "x1"; - std::string x2 = "x2"; - std::string x3 = "x3"; - std::string x4 = "x4"; - std::string x5 = "x5"; - std::string x6 = "x6"; - std::string x7 = "x7"; - - NewVar(block, x0, shape); - AppendOp(block, "scale", {{"X", {x0}}}, {{"Out", {x1}}}, {{"scale", 1.0f}}); - AppendOp(block, "scale", {{"X", {x1}}}, {{"Out", {x2}}}, {{"scale", 2.0f}}); - AppendOp(block, - "elementwise_mul", - {{"X", {x1}}, {"Y", {x2}}}, - {{"Out", {x3}}}, - {}); - AppendOp(block, "scale", {{"X", {x3}}}, {{"Out", {x1}}}, {{"scale", 3.0f}}); - AppendOp(block, - "elementwise_add_grad", - {{GradVarName("Out"), {x3}}, {"X", {x2}}, {"Y", {x1}}}, - {{GradVarName("X"), {x4}}, {GradVarName("Y"), {x5}}}, - {}); - AppendOp(block, - "elementwise_mul", - {{"X", {x4}}, {"Y", {x5}}}, - {{"Out", {x6}}}, - {}); - AppendOp(block, - "elementwise_add", - {{"X", {x5}}, {"Y", {x5}}}, - {{"Out", {x7}}}, - {}); - - std::vector use_cuda_list{false}; -#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) - use_cuda_list.push_back(true); -#endif - for (auto use_cuda : use_cuda_list) { - ReferenceCountPassTestHelper helper(program, use_cuda); - ASSERT_TRUE(helper.IsLastLivedOps(x0, {"scale"})); - ASSERT_EQ(PADDLE_GET_CONST(float, - helper.LastLivedOps(x0)[0]->Attrs().at("scale")), - 1.0f); - - ASSERT_TRUE(helper.IsLastLivedOps(x1, {"scale"})); - ASSERT_EQ(PADDLE_GET_CONST(float, - helper.LastLivedOps(x1)[0]->Attrs().at("scale")), - 3.0f); - - ASSERT_TRUE(helper.IsLastLivedOps(x2, {"elementwise_mul"})); - ASSERT_TRUE(helper.IsLastLivedOps(x3, {"elementwise_add_grad"})); - - ASSERT_TRUE(helper.IsLastLivedOps(x4, {"elementwise_mul"})); - ASSERT_TRUE( - helper.IsLastLivedOps(x5, {"elementwise_mul", "elementwise_add"})); - - ASSERT_TRUE(helper.IsLastLivedOps(x6, {"elementwise_mul"})); - ASSERT_TRUE(helper.IsLastLivedOps(x7, {"elementwise_add"})); - } -} - -} // namespace framework -} // namespace paddle From 4448d45cafa17d085368550f836a1e0396d2b4d0 Mon Sep 17 00:00:00 2001 From: hong <43953930+phlrain@users.noreply.github.com> Date: Thu, 29 Feb 2024 16:55:24 +0800 Subject: [PATCH 176/282] [CINN]update dyshape workflow (#62101) * update dyshape workflow * update * polish code * poslish code * fix compiler bug --- .../operator/transforms/add_cinn_pass.cc | 2 +- .../transforms/dynamic_reshape_pass.cc | 2 +- .../transforms/replace_dynamic_expand_pass.cc | 25 +++++++++++++++++-- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.cc index 496370ee7bfcd..24c05b6b006c3 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.cc @@ -107,9 +107,9 @@ void ApplyCinnPreprocessPass( pass_manager->AddPass( cinn::dialect::ir::CreateMoveGenerateShapeOpsToProloguePass()); - pass_manager->AddPass(cinn::dialect::ir::CreateDivideGroupOpToFusionOpPass()); pass_manager->AddPass(cinn::dialect::ir::CreateDynamicReshapeOpPass()); pass_manager->AddPass(cinn::dialect::ir::CreateReplaceDynamicExpandOpPass()); + pass_manager->AddPass(cinn::dialect::ir::CreateDivideGroupOpToFusionOpPass()); pass_manager->AddPass(pir::CreateDeadCodeEliminationPass()); pass_manager->Run(program); diff --git a/paddle/cinn/hlir/dialect/operator/transforms/dynamic_reshape_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/dynamic_reshape_pass.cc index cab96a8bd27f9..60c9edca4fb3c 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/dynamic_reshape_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/dynamic_reshape_pass.cc @@ -118,7 +118,7 @@ class DynamicReshapeOpPass : public pir::Pass { for (uint32_t i = 0; i < op->num_regions(); ++i) { for (auto& block : op->region(i)) { for (auto& op : block) { - if (op.isa()) { + if (op.isa()) { auto [_, num_rewrites] = pir::ApplyPatternsGreedily(&op, patterns_, cfg); AddStatistics(num_rewrites); diff --git a/paddle/cinn/hlir/dialect/operator/transforms/replace_dynamic_expand_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/replace_dynamic_expand_pass.cc index b37ab970da882..85bdf3985c8a5 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/replace_dynamic_expand_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/replace_dynamic_expand_pass.cc @@ -52,7 +52,28 @@ class DynamicExpandOpPattern for (size_t i = 0; i < x_rank; ++i) { broadcast_axes[i] = i + index_gap; } - std::vector out_shape(out_rank, -1); + + pir::ShapeConstraintIRAnalysis& shape_analysis = + pir::ShapeAnalysisManager::Instance().Get(op->GetParentProgram()); + + const auto& UpdateOutputShapeByDimExpr = [&]() -> std::vector { + std::vector out_shape(out_rank, -1); + if (shape_analysis.HasShapeOrDataForValue(op->result(0))) { + VLOG(3) << "found shape dialect"; + auto shape_info = + shape_analysis.GetShapeOrDataForValue(op->result(0)).shape(); + + for (size_t i = 0; i < shape_info.size(); ++i) { + if (shape_info[i].isa()) { + out_shape[i] = shape_info[i].Get(); + } + } + } + return out_shape; + }; + + auto out_shape = UpdateOutputShapeByDimExpr(); + return rewriter.Build( op->operand_source(0), broadcast_axes, out_shape); }(); @@ -91,7 +112,7 @@ class ReplaceDynamicExpandOpPass : public pir::Pass { for (uint32_t i = 0; i < op->num_regions(); ++i) { for (auto& block : op->region(i)) { for (auto& op : block) { - if (op.isa()) { + if (op.isa()) { const auto& [_, num_rewrites] = pir::ApplyPatternsGreedily(&op, patterns_, cfg); AddStatistics(num_rewrites); From 473f7ba0a218df3691f261005447a9139b649e70 Mon Sep 17 00:00:00 2001 From: diadestiny <44188454+diadestiny@users.noreply.github.com> Date: Thu, 29 Feb 2024 17:18:09 +0800 Subject: [PATCH 177/282] [SOT][3.12] fix codegen out of range about generating `LOAD_ATTR` in Python 3.12 (#62176) --- .../jit/sot/opcode_translator/executor/pycode_generator.py | 6 +++++- test/sot/skip_files_py312 | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/python/paddle/jit/sot/opcode_translator/executor/pycode_generator.py b/python/paddle/jit/sot/opcode_translator/executor/pycode_generator.py index 2ada3f7228f11..ce25cabd6f2d4 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/pycode_generator.py +++ b/python/paddle/jit/sot/opcode_translator/executor/pycode_generator.py @@ -742,12 +742,14 @@ def gen_load_deref(self, name): idx = self.cell_free_storage.index(name) return self.add_instr("LOAD_DEREF", arg=idx, argval=name) - def gen_load_attr(self, name: str): + def gen_load_attr(self, name: str, is_method=False): if name not in self._code_options["co_names"]: self._code_options["co_names"].append(name) idx = self._code_options["co_names"].index(name) if sys.version_info >= (3, 12): idx <<= 1 + if is_method: + idx |= 1 return self.add_instr("LOAD_ATTR", arg=idx, argval=name) def gen_store_attr(self, name: str): @@ -763,6 +765,8 @@ def gen_delete_attr(self, name: str): return self.add_instr("DELETE_ATTR", arg=idx, argval=name) def gen_load_method(self, name: str): + if sys.version_info >= (3, 12): + return self.gen_load_attr(name, True) if name not in self._code_options["co_names"]: self._code_options["co_names"].append(name) idx = self._code_options["co_names"].index(name) diff --git a/test/sot/skip_files_py312 b/test/sot/skip_files_py312 index 796fdb62e5001..4d3ee9050ad6c 100644 --- a/test/sot/skip_files_py312 +++ b/test/sot/skip_files_py312 @@ -1,6 +1,5 @@ ./test_11_jumps.py ./test_12_for_loop.py -./test_21_global.py ./test_builtin_zip.py ./test_inplace_api.py ./test_min_graph_size.py From 18ea0edb5b1f1a5048efdfe9047e218f02bf5b53 Mon Sep 17 00:00:00 2001 From: wanghuancoder Date: Thu, 29 Feb 2024 18:56:45 +0800 Subject: [PATCH 178/282] pir onednn support slice,stack (#62220) --- .../fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml | 8 +++++--- test/mkldnn/test_slice_mkldnn_op.py | 7 ++++--- test/mkldnn/test_stack_mkldnn_op.py | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml index e85e39621ee9d..b2e5cc7000f87 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml @@ -248,9 +248,11 @@ - op : sigmoid_grad -# - op : slice +- op : slice + extra_args : str mkldnn_data_type="float32" -# - op : slice_grad +- op : slice_grad + extra_args : str mkldnn_data_type="float32" - op : softmax extra_args : str data_format="AnyLayout", str mkldnn_data_type="float32", bool is_test=false @@ -276,7 +278,7 @@ - op : squeeze_grad extra_args : str mkldnn_data_type="float32" -# - op : stack +- op : stack - op : subtract diff --git a/test/mkldnn/test_slice_mkldnn_op.py b/test/mkldnn/test_slice_mkldnn_op.py index 66161dbad4908..1a71278a9f216 100644 --- a/test/mkldnn/test_slice_mkldnn_op.py +++ b/test/mkldnn/test_slice_mkldnn_op.py @@ -55,10 +55,10 @@ def config(self): self.out = self.input[1:3, 0:3, 2:4, :] def test_check_output(self): - self.check_output() + self.check_output(check_pir_onednn=True) def test_check_grad(self): - self.check_grad(['Input'], 'Out') + self.check_grad(['Input'], 'Out', check_pir_onednn=True) class TestSliceOneDNNOp1(TestSliceOneDNNOp): @@ -217,7 +217,7 @@ def calculate_grads(self): ] = self.dout def test_check_output(self): - self.check_output_with_place(core.CPUPlace()) + self.check_output_with_place(core.CPUPlace(), check_pir_onednn=True) def test_check_grad(self): self.calculate_grads() @@ -227,6 +227,7 @@ def test_check_grad(self): "Out", user_defined_grads=[self.dx], user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + check_pir_onednn=True, ) cls_name = "{}_{}".format(parent.__name__, "BF16") diff --git a/test/mkldnn/test_stack_mkldnn_op.py b/test/mkldnn/test_stack_mkldnn_op.py index 82acf285ce16d..8b91c246d6e6b 100644 --- a/test/mkldnn/test_stack_mkldnn_op.py +++ b/test/mkldnn/test_stack_mkldnn_op.py @@ -59,7 +59,7 @@ def setUp(self): self.attrs = {'axis': self.axis, 'use_mkldnn': True} def test_check_output(self): - self.check_output_with_place(core.CPUPlace()) + self.check_output_with_place(core.CPUPlace(), check_pir_onednn=True) # JUST FOR CI TO PASS, GRAD IS NOT IMPLEMENTED YET def test_check_grad(self): From e0027d222284c148b50a7bde5f915676acdc7585 Mon Sep 17 00:00:00 2001 From: wanghuancoder Date: Thu, 29 Feb 2024 19:05:52 +0800 Subject: [PATCH 179/282] [PIR] pir onednn support some fused ops (#62187) * onednn support some fused ops --- .../pir_adaptor/pir_adaptor_util.cc | 8 +- .../fused/mkldnn/fusion_lstm_mkldnn_op.cc | 16 +- .../fluid/pir/dialect/operator/ir/onednn.yaml | 38 +++++ .../dialect/operator/ir/ops_onednn_extra.yaml | 11 +- .../fluid/pir/dialect/operator/utils/utils.cc | 1 + paddle/phi/api/yaml/op_compat.yaml | 38 +++++ paddle/phi/infermeta/fusion.cc | 160 ++++++++++++++++++ paddle/phi/infermeta/fusion.h | 27 +++ test/legacy_test/op_test.py | 8 +- test/legacy_test/test_fusion_lstm_op.py | 4 +- .../mkldnn/test_fusion_lstm_bf16_mkldnn_op.py | 5 +- .../mkldnn/test_fusion_lstm_int8_mkldnn_op.py | 1 + test/mkldnn/test_fusion_lstm_mkldnn_op.py | 7 +- test/white_list/op_accuracy_white_list.py | 1 + 14 files changed, 305 insertions(+), 20 deletions(-) diff --git a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc index 1e2fa3269bb41..11b263f540500 100644 --- a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc +++ b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc @@ -951,27 +951,27 @@ std::shared_ptr BuildOperatorBase( } attr_map[legacy_arg_name] = vec_int; } else if (array_list[0].isa()) { - std::vector vec_int64; + std::vector vec_int64; for (auto attribute : array_list) { vec_int64.push_back( attribute.dyn_cast().data()); // NOLINT } attr_map[legacy_arg_name] = vec_int64; } else if (array_list[0].isa()) { - std::vector vec_bool; + std::vector vec_bool; for (auto attribute : array_list) { vec_bool.push_back(attribute.dyn_cast().data()); } attr_map[legacy_arg_name] = vec_bool; } else if (array_list[0].isa()) { - std::vector vec_float; + std::vector vec_float; for (auto attribute : array_list) { vec_float.push_back( attribute.dyn_cast().data()); // NOLINT } attr_map[legacy_arg_name] = vec_float; } else if (array_list[0].isa()) { - std::vector vec_double; + std::vector vec_double; for (auto attribute : array_list) { vec_double.push_back( attribute.dyn_cast().data()); // NOLINT diff --git a/paddle/fluid/operators/fused/mkldnn/fusion_lstm_mkldnn_op.cc b/paddle/fluid/operators/fused/mkldnn/fusion_lstm_mkldnn_op.cc index ada14e280a0f3..e004b35d0c3ec 100644 --- a/paddle/fluid/operators/fused/mkldnn/fusion_lstm_mkldnn_op.cc +++ b/paddle/fluid/operators/fused/mkldnn/fusion_lstm_mkldnn_op.cc @@ -321,7 +321,7 @@ class LSTMMKLDNNHandler } }; -template +template class FusionLSTMMKLDNNKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -473,9 +473,11 @@ class FusionLSTMMKLDNNKernel : public framework::OpKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_KERNEL(fusion_lstm, - MKLDNN, - phi::CPUPlace, - ops::FusionLSTMMKLDNNKernel, - ops::FusionLSTMMKLDNNKernel, - ops::FusionLSTMMKLDNNKernel); + +PD_REGISTER_STRUCT_KERNEL(fusion_lstm, + OneDNN, + ONEDNN, + ops::FusionLSTMMKLDNNKernel, + float, + uint8_t, + paddle::platform::bfloat16) {} diff --git a/paddle/fluid/pir/dialect/operator/ir/onednn.yaml b/paddle/fluid/pir/dialect/operator/ir/onednn.yaml index a786f395db1af..18a799dfb28a9 100644 --- a/paddle/fluid/pir/dialect/operator/ir/onednn.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/onednn.yaml @@ -74,6 +74,44 @@ kernel : func : fused_elementwise_sub +- op : fused_matmul + args : (Tensor x, Tensor y, Tensor residual_data, bool trans_x=false, bool trans_y=false, float matmul_alpha=1.0, str fuse_activation="", float fuse_alpha=0.0, float fuse_beta=0.0, float fused_output_scale=1.0, int[] fused_reshape_x={}, int[] fused_transpose_x={}, int[] fused_reshape_y={}, int[] fused_transpose_y={}, int[] fused_reshape_out={}, int[] fused_transpose_out={}, str mkldnn_data_type="float32", float scale_x=1.0, float scale_y=1.0, float scale_in_eltwise=0.0, float scale_out=1.0,bool force_fp32_output=false) + output : Tensor(out) + infer_meta : + func : FusedMatmulInferMeta + kernel : + func : fused_matmul + optional : residual_data + +- op : fused_softplus + args : (Tensor x, float beta=1.0, float threshold=20.0, str fuse_activation="", float fuse_alpha=0.0, float fuse_beta=0.0) + output : Tensor(out) + infer_meta : + func : UnchangedExceptDtypeInferMeta + param : [x] + kernel : + func : fused_softplus + +- op : fused_transpose + args : (Tensor x, int[] axis={}, int[] fused_squeeze2_axes={}, int[] fused_unsqueeze2_axes={}, int[] fused_reshape2_shape={}, float scale=1.0, float shift=0.0, str output_data_type="") + output : Tensor(out) + infer_meta : + func : TransposeInferMeta + param : [x, axis] + kernel : + func : fused_transpose + +- op : fusion_lstm + args : (Tensor x, Tensor weight_x, Tensor weight_h, Tensor bias, Tensor h0, Tensor c0, bool use_peepholes=true, bool is_reverse=false, bool use_seq=true, str gate_activation="sigmoid", str cell_activation="tanh", str candidate_activation="tanh", float scale_data=1.0, float shift_data=0.0, float[] scale_weights={1.0}, bool force_fp32_output=false) + output : Tensor(hidden), Tensor(cell), Tensor(xx), Tensor(batched_input), Tensor(batched_hidden), Tensor(batched_cell), Tensor(reordered_h0), Tensor(reordered_c0), Tensor(checked_cell) + infer_meta : + func : FusionLstmInferMeta + kernel : + func : fusion_lstm + data_type : x + optional : h0, c0 + intermediate : xx, batched_input, batched_hidden, batched_cell, reordered_h0, reordered_c0, checked_cell + - op: multi_gru args: (Tensor x, Tensor[] weight_x, Tensor[] weight_h, Tensor[] bias, Tensor[] scale_weights, str activation="tanh", str gate_activation="sigmoid", int layers=1, bool origin_mode=false, str mkldnn_data_type="float32", float scale_data=1.0, float shift_data=1.0, bool force_fp32_output=false) output: Tensor(hidden) diff --git a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml index b2e5cc7000f87..fd8c3a409a573 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml @@ -111,16 +111,19 @@ - op : fused_elementwise_sub -# - op : fused_matmul +- op : fused_matmul -# - op : fused_softplus +- op : fused_softplus -# - op : fused_transpose +- op : fused_transpose + extra_args : str data_format="AnyLayout", str mkldnn_data_type="float32" + data_format_tensors : x - op : fusion_gru extra_args : str mkldnn_data_type="float32", float scale_data=1.0, float shift_data=0.0, float[] scale_weights={1.0f} -# - op : fusion_lstm +- op : fusion_lstm + extra_args : str mkldnn_data_type="float32" - op : gaussian diff --git a/paddle/fluid/pir/dialect/operator/utils/utils.cc b/paddle/fluid/pir/dialect/operator/utils/utils.cc index 9b450977814b6..931c7d4b33624 100644 --- a/paddle/fluid/pir/dialect/operator/utils/utils.cc +++ b/paddle/fluid/pir/dialect/operator/utils/utils.cc @@ -84,6 +84,7 @@ const std::unordered_set LegacyOpList = { paddle::onednn::dialect::QuantizeOp::name(), paddle::onednn::dialect::RequantizeOp::name(), paddle::onednn::dialect::MultiGruOp::name(), + paddle::onednn::dialect::FusionLstmOp::name(), #endif CReduceMinOp::name(), PushSparseV2Op::name(), diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml index 74263a1dd522d..840ce5ef29de3 100755 --- a/paddle/phi/api/yaml/op_compat.yaml +++ b/paddle/phi/api/yaml/op_compat.yaml @@ -1445,6 +1445,10 @@ {x_grad : DX, y_grad : DY, bias_grad : DBias} - op : fused_transpose + inputs: + {x : X} + outputs : + {out : Out} extra : attrs : [str data_format = "AnyLayout"] @@ -1467,6 +1471,26 @@ attrs : [bool use_mkldnn = false, str mkldnn_data_type = "float32", float Scale_data = 1.0f, float Shift_data = 0.0f, 'float[] Scale_weights = {1.0f}'] - op : fusion_lstm + inputs : + x : X + h0 : H0 + weight_x : WeightX + weight_h : WeightH + bias : Bias + c0 : C0 + outputs : + out : Out + hidden : Hidden + cell : Cell + xx : XX + batched_input : BatchedInput + batched_hidden : BatchedHidden + batched_cell : BatchedCell + reordered_h0 : ReorderedH0 + reordered_c0 : ReorderedC0 + checked_cell : CheckedCell + attrs : + {scale_data : Scale_data, shift_data : Shift_data, scale_weights : Scale_weights} extra : attrs : [bool use_mkldnn = true, str mkldnn_data_type = "float32"] @@ -3610,6 +3634,20 @@ outputs : {out : Out, intermediate_out : IntermediateOut} +- op: fused_matmul + inputs : + {x: X, y: Y, residual_data: ResidualData} + outputs : + {out : Out} + attrs : + {scale_x : Scale_x, scale_y : Scale_y, scale_out : Scale_out, scale_in_eltwise : Scale_in_eltwise, fused_reshape_x : fused_reshape_X, fused_transpose_x : fused_transpose_X, fused_reshape_y : fused_reshape_Y, fused_transpose_y : fused_transpose_Y, fused_reshape_out : fused_reshape_Out, fused_transpose_out : fused_transpose_Out} + +- op: fused_softplus + inputs : + {x: X} + outputs : + {out : Out} + - op: fusion_squared_mat_sub inputs : x : X diff --git a/paddle/phi/infermeta/fusion.cc b/paddle/phi/infermeta/fusion.cc index af280b44d6501..4af21b36b34da 100644 --- a/paddle/phi/infermeta/fusion.cc +++ b/paddle/phi/infermeta/fusion.cc @@ -3832,6 +3832,166 @@ void MultiGruInferMeta( hidden->share_lod(x); } +void FusionLstmInferMeta(const MetaTensor& x, + const MetaTensor& weight_x, + const MetaTensor& weight_h, + const MetaTensor& bias, + const MetaTensor& h0, + const MetaTensor& c0, + const bool use_peepholes, + const bool is_reverse, + const bool use_seq, + const std::string& gate_activation, + const std::string& cell_activation, + const std::string& candidate_activation, + const float scale_data, + const float shift_data, + const std::vector& scale_weights, + const bool force_fp32_output, + MetaTensor* hidden, + MetaTensor* cell, + MetaTensor* xx, + MetaTensor* batched_input, + MetaTensor* batched_hidden, + MetaTensor* batched_cell, + MetaTensor* reordered_h0, + MetaTensor* reordered_c0, + MetaTensor* checked_cell) { + auto x_dims = x.dims(); + PADDLE_ENFORCE_EQ(x_dims.size(), + 2, + phi::errors::InvalidArgument( + "Input(X)'s rank must be 2, but received x's rank " + "is:%d, x dim is:[%s]", + x_dims.size(), + x_dims)); + + if (h0.initialized()) { + PADDLE_ENFORCE_EQ( + c0.initialized(), + true, + phi::errors::InvalidArgument( + "fusion_lstm must has h0 and c0 input at the same time.")); + auto h_dims = h0.dims(); + auto c_dims = c0.dims(); + PADDLE_ENFORCE_EQ(h_dims, + c_dims, + phi::errors::InvalidArgument( + "The dimension of Input(H0) and Input(C0) should be " + "same, but received h0 dims is:[%s], c0 dims is:[%s]", + h_dims, + c_dims)); + } + + auto wx_dims = weight_x.dims(); + PADDLE_ENFORCE_EQ(wx_dims.size(), + 2, + phi::errors::InvalidArgument( + "The rank of Input(WeightX) should be 2, but received " + "WeightX's rank is:%d, WeightX dim is:[%s]", + wx_dims.size(), + wx_dims)); + PADDLE_ENFORCE_EQ(wx_dims[0], + x_dims[1], + phi::errors::InvalidArgument( + "The first dimension of Input(WeightX) " + "should equal to second dimension of Input(X), but " + "received WeightX first dim is:%d, X second dim is:%d", + wx_dims[0], + x_dims[1])); + + int frame_size = static_cast(wx_dims[1] / 4); + auto wh_dims = weight_h.dims(); + + PADDLE_ENFORCE_EQ(wh_dims.size(), + 2, + phi::errors::InvalidArgument( + "The rank of Input(WeightH) should be 2, but received " + "WeightH rank is:%d, WeightH dim is:[%s]", + wh_dims.size(), + wh_dims)); + PADDLE_ENFORCE_EQ(wh_dims[0], + frame_size, + phi::errors::InvalidArgument( + "The first dimension of Input(WeightH) " + "should equal to frame size, but received WeightH " + "first dim is:%d, frame size is:%d.", + wh_dims[0], + frame_size)); + + PADDLE_ENFORCE_EQ(wh_dims[1], + 4 * frame_size, + phi::errors::InvalidArgument( + "The second dimension of Input(WeightH) " + "should equal to 4 * frame_size, but received WeightH " + "second dimension is:%d, frame size is:%d.", + wh_dims[1], + frame_size)); + + auto b_dims = bias.dims(); + PADDLE_ENFORCE_EQ(b_dims.size(), + 2, + phi::errors::InvalidArgument( + "The rank of Input(Bias) should be 2, but received " + "Bias rank is:%d, Bias dim is:[%s]", + b_dims.size(), + b_dims)); + PADDLE_ENFORCE_EQ(b_dims[0], + 1, + phi::errors::InvalidArgument( + "The first dimension of Input(Bias) should be 1, but " + "received Bias's dimension is:[%s]", + b_dims)); + + if (use_peepholes) { + PADDLE_ENFORCE_EQ(b_dims[1], + 7 * frame_size, + phi::errors::InvalidArgument( + "The second dimension of Input(Bias) should be " + "7 * %d if enable peepholes connection, but received " + "Bias dim is:[%s]", + frame_size, + b_dims)); + checked_cell->set_dims(phi::make_ddim({2, frame_size})); + checked_cell->set_dtype(x.dtype()); + } else { + PADDLE_ENFORCE_EQ( + b_dims[1], + 4 * frame_size, + phi::errors::InvalidArgument( + "The second dimension of Input(Bias) should be " + "4 * %d if disable peepholes, but received Bias dim is:[%s]", + frame_size, + b_dims)); + } + + auto out_dims = phi::make_ddim({x_dims[0], frame_size}); + hidden->set_dims(out_dims); + cell->set_dims(out_dims); + hidden->share_lod(x); + cell->share_lod(x); + hidden->set_dtype(x.dtype()); + cell->set_dtype(x.dtype()); + + int xx_width = 0; + if (use_seq) { + xx_width = static_cast(wx_dims[1]); + } else { + xx_width = + static_cast(x_dims[1] > wx_dims[1] ? wx_dims[1] : x_dims[1]); + + batched_input->set_dims(phi::make_ddim({x_dims[0], wx_dims[1]})); + batched_hidden->set_dims(out_dims); + batched_cell->set_dims(out_dims); + batched_input->set_dtype(x.dtype()); + batched_hidden->set_dtype(x.dtype()); + batched_cell->set_dtype(x.dtype()); + } + xx->set_dims(phi::make_ddim({x_dims[0], xx_width})); + xx->set_dtype(x.dtype()); + xx->share_lod(x); +} + void RoformerRelativePosXPUInferMeta(const MetaTensor& x, const MetaTensor& sin_emb, const MetaTensor& cos_emb, diff --git a/paddle/phi/infermeta/fusion.h b/paddle/phi/infermeta/fusion.h index 87999ab2b4564..a724000bab9f0 100644 --- a/paddle/phi/infermeta/fusion.h +++ b/paddle/phi/infermeta/fusion.h @@ -861,4 +861,31 @@ void MultiGruInferMeta( float shift_data, bool force_fp32_output, MetaTensor* hidden); + +void FusionLstmInferMeta(const MetaTensor& x, + const MetaTensor& weight_x, + const MetaTensor& weight_h, + const MetaTensor& bias, + const MetaTensor& h0, + const MetaTensor& c0, + const bool use_peepholes, + const bool is_reverse, + const bool use_seq, + const std::string& gate_activation, + const std::string& cell_activation, + const std::string& candidate_activation, + const float scale_data, + const float shift_data, + const std::vector& scale_weights, + const bool force_fp32_output, + MetaTensor* hidden, + MetaTensor* cell, + MetaTensor* xx, + MetaTensor* batched_input, + MetaTensor* batched_hidden, + MetaTensor* batched_cell, + MetaTensor* reordered_h0, + MetaTensor* reordered_c0, + MetaTensor* checked_cell); + } // namespace phi diff --git a/test/legacy_test/op_test.py b/test/legacy_test/op_test.py index 41b9caed79480..c18a142a1ec9d 100644 --- a/test/legacy_test/op_test.py +++ b/test/legacy_test/op_test.py @@ -2643,7 +2643,9 @@ def _is_skip_name(self, name): static_checker.check() outs, fetch_list = static_checker.outputs, static_checker.fetch_list - if check_pir_onednn and place == base.CPUPlace(): + if check_pir_onednn and isinstance( + place, paddle.base.libpaddle.CPUPlace + ): with pir_executor_guard(): pir_onednn_static_checker = StaticChecker(self, self.outputs) pir_onednn_static_checker.check() @@ -3313,7 +3315,9 @@ def check_grad_with_place( atol, ) - if check_pir_onednn and place == base.CPUPlace(): + if check_pir_onednn and isinstance( + place, paddle.base.libpaddle.CPUPlace + ): with pir_executor_guard(): self.check_grad_with_place_for_static( user_defined_grads, diff --git a/test/legacy_test/test_fusion_lstm_op.py b/test/legacy_test/test_fusion_lstm_op.py index bbcb5e8a8396c..e733d047daf26 100644 --- a/test/legacy_test/test_fusion_lstm_op.py +++ b/test/legacy_test/test_fusion_lstm_op.py @@ -140,7 +140,9 @@ def setUp(self): def test_check_output(self): for use_seq in {True, False}: self.attrs['use_seq'] = use_seq - self.check_output(check_dygraph=False) + self.check_output( + check_dygraph=False, check_pir_onednn=self.check_pir_onednn + ) class TestFusionLSTMOpInit(TestFusionLSTMOp): diff --git a/test/mkldnn/test_fusion_lstm_bf16_mkldnn_op.py b/test/mkldnn/test_fusion_lstm_bf16_mkldnn_op.py index 9b8f1f684e2a4..c893238e758ec 100644 --- a/test/mkldnn/test_fusion_lstm_bf16_mkldnn_op.py +++ b/test/mkldnn/test_fusion_lstm_bf16_mkldnn_op.py @@ -32,7 +32,10 @@ def test_check_output(self): for use_seq in {True, False}: self.attrs['use_seq'] = use_seq self.check_output( - check_dygraph=False, no_check_set=["Cell"], atol=2e-2 + check_dygraph=False, + no_check_set=["Cell"], + atol=2e-2, + check_pir_onednn=True, ) def setUp(self): diff --git a/test/mkldnn/test_fusion_lstm_int8_mkldnn_op.py b/test/mkldnn/test_fusion_lstm_int8_mkldnn_op.py index 96bee8d9927bf..c876eb74ff626 100644 --- a/test/mkldnn/test_fusion_lstm_int8_mkldnn_op.py +++ b/test/mkldnn/test_fusion_lstm_int8_mkldnn_op.py @@ -145,6 +145,7 @@ def test_check_output(self): check_dygraph=False, no_check_set=["Cell"], atol=self.error_margin, + check_pir_onednn=True, ) diff --git a/test/mkldnn/test_fusion_lstm_mkldnn_op.py b/test/mkldnn/test_fusion_lstm_mkldnn_op.py index f9fdfa116acab..7be690aacf42f 100644 --- a/test/mkldnn/test_fusion_lstm_mkldnn_op.py +++ b/test/mkldnn/test_fusion_lstm_mkldnn_op.py @@ -20,11 +20,16 @@ class TestFusionLSTMONEDNNOp(TestFusionLSTMOp): def set_conf(self): self.use_mkldnn = True + self.check_pir_onednn = True def test_check_output(self): for use_seq in {True, False}: self.attrs['use_seq'] = use_seq - self.check_output(check_dygraph=False, no_check_set=["Cell"]) + self.check_output( + check_dygraph=False, + no_check_set=["Cell"], + check_pir_onednn=True, + ) class TestFusionLSTMONEDNNOpReverse(TestFusionLSTMONEDNNOp): diff --git a/test/white_list/op_accuracy_white_list.py b/test/white_list/op_accuracy_white_list.py index 98429a013f829..00d0ffccbac02 100644 --- a/test/white_list/op_accuracy_white_list.py +++ b/test/white_list/op_accuracy_white_list.py @@ -97,4 +97,5 @@ NO_BF16_COMPARED_WITH_FP32_OP_LIST = [ 'dequantize', + 'fusion_lstm', ] From 4c0243489e3c8f3e6bcfa924ad7ae720338eef0c Mon Sep 17 00:00:00 2001 From: wanghuancoder Date: Thu, 29 Feb 2024 19:06:24 +0800 Subject: [PATCH 180/282] pir onednn support transpose (#62219) --- .../fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml | 8 ++++++-- test/mkldnn/test_transpose_bf16_mkldnn_op.py | 4 +++- test/mkldnn/test_transpose_int8_mkldnn_op.py | 6 +++++- test/mkldnn/test_transpose_mkldnn_op.py | 8 ++++++-- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml index fd8c3a409a573..283761ec09903 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml @@ -303,6 +303,10 @@ - op : tanh_grad -# - op : transpose +- op : transpose + extra_args : str data_format="AnyLayout", str mkldnn_data_type="float32" + data_format_tensors : x -# - op : transpose_grad +- op : transpose_grad + extra_args : str data_format="AnyLayout", str mkldnn_data_type="float32" + data_format_tensors : out_grad diff --git a/test/mkldnn/test_transpose_bf16_mkldnn_op.py b/test/mkldnn/test_transpose_bf16_mkldnn_op.py index bd0f8473205d6..4eff0b96bd5d2 100644 --- a/test/mkldnn/test_transpose_bf16_mkldnn_op.py +++ b/test/mkldnn/test_transpose_bf16_mkldnn_op.py @@ -47,7 +47,9 @@ def setUp(self): } def test_check_output(self): - self.check_output_with_place(core.CPUPlace(), no_check_set=['XShape']) + self.check_output_with_place( + core.CPUPlace(), no_check_set=['XShape'], check_pir_onednn=True + ) def init_test_case(self): self.shape = (2, 3, 4, 5) diff --git a/test/mkldnn/test_transpose_int8_mkldnn_op.py b/test/mkldnn/test_transpose_int8_mkldnn_op.py index b800d6b40c504..e2a3fba8d2bc0 100644 --- a/test/mkldnn/test_transpose_int8_mkldnn_op.py +++ b/test/mkldnn/test_transpose_int8_mkldnn_op.py @@ -50,7 +50,11 @@ def init_op_type(self): def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode self.check_output_with_place( - core.CPUPlace(), 1e-5, no_check_set=['XShape'], check_dygraph=False + core.CPUPlace(), + 1e-5, + no_check_set=['XShape'], + check_dygraph=False, + check_pir_onednn=True, ) def initTestCase(self): diff --git a/test/mkldnn/test_transpose_mkldnn_op.py b/test/mkldnn/test_transpose_mkldnn_op.py index 66185f9daaf48..34a25cf2f8b1e 100644 --- a/test/mkldnn/test_transpose_mkldnn_op.py +++ b/test/mkldnn/test_transpose_mkldnn_op.py @@ -38,11 +38,15 @@ def init_op_type(self): def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_output(no_check_set=['XShape'], check_dygraph=False) + self.check_output( + no_check_set=['XShape'], check_dygraph=False, check_pir_onednn=True + ) def test_check_grad(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_grad(['X'], 'Out', check_dygraph=False) + self.check_grad( + ['X'], 'Out', check_dygraph=False, check_pir_onednn=True + ) def initTestCase(self): self.shape = (30, 4) From bd7562d54dbaf18c023746460c6102c6e9d8f058 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E5=91=A8=E5=91=A8?= <39978853+zhoutianzi666@users.noreply.github.com> Date: Thu, 29 Feb 2024 20:13:28 +0800 Subject: [PATCH 181/282] [Paddle Inference]support sm80 cutlass conv2d (#62017) modify ../test/ir/inference/test_cutlass_fused_conv2d_add_act_op.py add conv+bias+elementwise_add add some to README.md * use write_kernel_to_file * add -std=c++17 in CUDA_NVCC_FLAGS for compiling cut --- paddle/fluid/framework/ir/cutlass_teller.h | 109 ++++++++++- .../fusion/cutlass/conv2d/CMakeLists.txt | 12 +- .../kernels/fusion/cutlass/conv2d/README.md | 6 + .../kernels/fusion/cutlass/conv2d/compile.sh | 2 +- .../fusion/cutlass/conv2d/conv2d_bias_act.py | 176 ++++++++++++++++- .../cutlass/conv2d/conv2d_bias_residual.py | 185 ++++++++++++++++-- .../fusion/cutlass/conv2d/conv2d_common.py | 35 +++- .../fusion/cutlass/conv2d/conv2d_decl.h | 17 +- .../conv2d/conv2d_depthwise_bias_act.py | 1 + .../fusion/cutlass/conv2d/conv2d_util.cu | 96 +++++---- .../fusion/cutlass/conv2d/conv2d_util.h | 1 + .../cutlass/fused_conv2d_add_act_kernel.cu | 91 ++++++--- paddle/phi/kernels/fusion/cutlass/util.py | 26 +++ 13 files changed, 650 insertions(+), 107 deletions(-) diff --git a/paddle/fluid/framework/ir/cutlass_teller.h b/paddle/fluid/framework/ir/cutlass_teller.h index 3d50544ede13b..2bc829e2fc8e9 100644 --- a/paddle/fluid/framework/ir/cutlass_teller.h +++ b/paddle/fluid/framework/ir/cutlass_teller.h @@ -1,5 +1,5 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. + // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -20,8 +20,9 @@ namespace framework { namespace ir { typedef enum { - cba, - cbaa, + cba, // This servers for conv_elementwise_add_fuse_pass + cbaa, // This servers for conv_elementwise_add2_act_fuse_pass + cbaele, // This servers for conv2d_fusion_cutlass_elementwise } CutlassFusionType; class CutlassTeller { @@ -33,6 +34,7 @@ class CutlassTeller { #if defined(PADDLE_WITH_CUTLASS) // Determine this NCHW conv2d + bias can be fused with activation by cutlass? + // This servers for conv_elementwise_add_fuse_pass. // will not set or change any attribute in op_desc bool CbaCanSupport(OpDesc *op_desc, Scope *scope, @@ -85,7 +87,8 @@ class CutlassTeller { } // Determine this NCHW conv2d + bias + elewise_add + act can be fused by - // cutlass? will not set or change any attribute in op_desc + // cutlass?, this is for conv_elementwise_add_fuse_pass + // will not set or change any attribute in op_desc bool CbaaCanSupport(OpDesc *op_desc, Scope *scope, std::string act_type, @@ -136,6 +139,69 @@ class CutlassTeller { return true; } + // Determine this NCHW conv2d_fusion + elewise_op + act1 can be fused by + // cutlass? + // This servers for conv2d_fusion_cutlass_elementwise. + // will not set or change any attribute in op_desc + bool CbaeleCanSupport(OpDesc *op_desc, + Scope *scope, + std::string ele_type, + std::string act1_type, + int device_id) { + auto strides = op_desc->GetAttrIfExists>("strides"); + auto dilations = op_desc->GetAttrIfExists>("dilations"); + CHECK_EQ(strides.size() == 2UL, true); + CHECK_EQ(dilations.size() == 2UL, true); + int stride_h = strides[0]; + int stride_w = strides[1]; + int dilation_h = dilations[0]; + int dilation_w = dilations[1]; + auto act_type = op_desc->GetAttrIfExists("activation"); + + // Do not allow conv2d_fusion already have residual input. + if (op_desc->Input("ResidualData").size() >= 1) { + return false; + } + + auto filter_names = op_desc->Input("Filter"); + + for (const auto &filter_name : filter_names) { + auto *filter_var = scope->FindLocalVar(filter_name); + const auto &filter_tensor = filter_var->Get(); + CHECK_EQ(filter_tensor.dims().size() == 4UL, true); + auto groups = op_desc->GetAttrIfExists("groups"); + int oc = filter_tensor.dims()[0]; + int kc = filter_tensor.dims()[1]; + int kh = filter_tensor.dims()[2]; + int kw = filter_tensor.dims()[3]; + + // For convience, we only support EXPLICIT + auto padding_algorithm = + op_desc->GetAttrIfExists("padding_algorithm"); + if (padding_algorithm != "EXPLICIT") { + return false; + } + + if (!Conv2dCanSupport(oc, + kc, + kh, + kw, + stride_h, + stride_w, + dilation_h, + dilation_w, + groups, + act_type, + device_id, + CutlassFusionType::cbaele, + act1_type, + ele_type)) { + return false; + } + } + return true; + } + // Determine whether this conv can be fused with the activation by cutlass // backend. bool Conv2dCanSupport(int oc, @@ -149,7 +215,10 @@ class CutlassTeller { int groups, std::string activation, int device_id, - CutlassFusionType fuse_type) { + CutlassFusionType fuse_type, + // below two are used by cbaele + std::string activation1 = "identity", + std::string elemenstwise_type = "elementwise_add") { int sm_version = platform::GetGPUComputeCapability(device_id); int ic = kc * groups; if (!cutlass_sm.count(sm_version)) { @@ -173,6 +242,14 @@ class CutlassTeller { !cbaa_act_set.count(activation)) { return false; } + + // conv + bias + act + elementwise_op + if (fuse_type == CutlassFusionType::cbaele && + !cbaele_act_set.count(activation + "_" + elemenstwise_type + "_" + + activation1)) { + return false; + } + } else if (groups == ic && ic == oc) { // return false; // conv2d_depthwise not support residual input @@ -250,6 +327,14 @@ class CutlassTeller { return false; } + bool CbaeleCanSupport(OpDesc *op_desc, + Scope *scope, + std::string ele_type, + std::string act1_type, + int device_id) { + return false; + } + bool Conv2dCanSupport(int oc, int kc, int kh, @@ -261,7 +346,10 @@ class CutlassTeller { int groups, std::string activation, int device_id, - CutlassFusionType fuse_type) { + CutlassFusionType fuse_type, + // below two are used by cbaele + std::string activation1 = "identity", + std::string elemenstwise_type = "elementwise_add") { return false; } std::unordered_set CbaAct(int device_id) { return {}; } @@ -270,6 +358,9 @@ class CutlassTeller { static const int CUTLASS_NHWC_ALIGNMENT = 8; const std::unordered_set cutlass_sm = { 75, + 80, + 85, + 86, }; const std::unordered_set cba_act_set = { "relu", "swish", "identity", "leaky_relu", "sigmoid"}; @@ -278,6 +369,10 @@ class CutlassTeller { const std::unordered_set cdba_act_set = { "identity", "relu", "swish", "sigmoid"}; const std::unordered_set cbaa_act_set = {"relu"}; + const std::unordered_set cbaele_act_set = { + "identity_elementwise_add_identity", + "swish_elementwise_add_identity", + }; }; } // namespace ir diff --git a/paddle/phi/kernels/fusion/cutlass/conv2d/CMakeLists.txt b/paddle/phi/kernels/fusion/cutlass/conv2d/CMakeLists.txt index cd82bbf1dc8b7..b77a565121bee 100644 --- a/paddle/phi/kernels/fusion/cutlass/conv2d/CMakeLists.txt +++ b/paddle/phi/kernels/fusion/cutlass/conv2d/CMakeLists.txt @@ -21,15 +21,17 @@ execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_CURRENT_BINARY_DIR}/generated_tmp") execute_process( - COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/conv2d_bias_act.py" + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/conv2d_bias_act.py + --cuda_arch ${COMPUTE_CAPABILITY} + COMMAND + ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/conv2d_bias_residual.py + --cuda_arch ${COMPUTE_CAPABILITY} COMMAND ${PYTHON_EXECUTABLE} - "${CMAKE_CURRENT_SOURCE_DIR}/conv2d_bias_residual.py" - COMMAND ${PYTHON_EXECUTABLE} - "${CMAKE_CURRENT_SOURCE_DIR}/conv2d_depthwise_bias_act.py" + ${CMAKE_CURRENT_SOURCE_DIR}/conv2d_depthwise_bias_act.py WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") find_package(CUDA) - +# you can append -std=c++17 in CUDA_NVCC_FLAGS for compiling cutlass 3.0 set(CUDA_NVCC_FLAGS -gencode arch=compute_${COMPUTE_CAPABILITY},code=sm_${COMPUTE_CAPABILITY};) #set(CMAKE_CXX_FLAGS -fvisibility=hidden) diff --git a/paddle/phi/kernels/fusion/cutlass/conv2d/README.md b/paddle/phi/kernels/fusion/cutlass/conv2d/README.md index a717b3d692b91..4a2b6c6ac61aa 100644 --- a/paddle/phi/kernels/fusion/cutlass/conv2d/README.md +++ b/paddle/phi/kernels/fusion/cutlass/conv2d/README.md @@ -23,3 +23,9 @@ compile.sh 脚本中会下载cutlass，执行CMakeLists.txt脚本，编译生成 step2. step1执行后，就可以看到在 build 目录生成了 `libCutlassConv2d.so` ，并将build目录添加到LD_LIBRARY_PATH中即可使用此库。 + + +step3. + +默认情况下，在处理conv2d类算子时，Paddle Inference 会调用cuDNN实现； +基于 cutlass 开发的conv2d类算子能够融合更多的后处理算子，用户可以通过python API `exp_enable_use_cutlass()` 和 C++ API `Exp_EnableUseCutlass()`来获得一定的速度和显存收益。 diff --git a/paddle/phi/kernels/fusion/cutlass/conv2d/compile.sh b/paddle/phi/kernels/fusion/cutlass/conv2d/compile.sh index 44c0fdf3a04da..d43bda262f543 100644 --- a/paddle/phi/kernels/fusion/cutlass/conv2d/compile.sh +++ b/paddle/phi/kernels/fusion/cutlass/conv2d/compile.sh @@ -25,7 +25,7 @@ fi python_exe_path="python" cuda_root_path="/usr/local/cuda" -gpu_cc="75" +gpu_cc="80" cd $build_directory cmake .. -DPYTHON_EXECUTABLE=$python_exe_path -DCUDA_TOOLKIT_ROOT_DIR=$cuda_root_path -DCOMPUTE_CAPABILITY=$gpu_cc diff --git a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_bias_act.py b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_bias_act.py index 0cb925489f14a..2104c676c9b82 100644 --- a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_bias_act.py +++ b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_bias_act.py @@ -21,7 +21,7 @@ CommonTail, GenerateFunctionForPhi, ) -from util import SubstituteTemplate, TileDesc +from util import SubstituteTemplate, TileDesc, parse_args, write_kernel_to_file # this is a file's header part @@ -54,10 +54,10 @@ + ''' typename ImplicitGemm::Arguments arguments{ problem_size, - {(cutlass::half_t *)(input), {ic, ic * iw, ic * iw * ih}}, - {(cutlass::half_t *)(weight), {kc, kc * kw, kc * kw * kh}}, - {(cutlass::half_t *)(bias), {0, 0, 0}}, - {(cutlass::half_t *)(output), {oc, oc * ow, oc * ow * oh}}, + {input, {ic, ic * iw, ic * iw * ih}}, + {weight, {kc, kc * kw, kc * kw * kh}}, + {bias, {0, 0, 0}}, + {output, {oc, oc * ow, oc * ow * oh}}, {1.f, 1.f}}; ''' + CommonCutlassConvKernelExecute @@ -170,10 +170,11 @@ def generate_sm75_1688(): sm75_code = "" for epi_func in SupportedAct: op_dict = {} - op_dict["func_name"] = UnderScoreName[epi_func].lower() + "_sm75" + op_dict["func_name"] = UnderScoreName[epi_func].lower() + "_sm75_fp16" op_dict["enum_op_name"] = UnderScoreName[epi_func].upper() # For a function, we record all its kernels into a std::vector in C++ code all_kernel_names = "" + all_kernel_declares = "" kernel_dict["epi_func"] = ActTag[epi_func] suffix = 0 for iterator_algorithm in iterator_algorithms: @@ -203,23 +204,178 @@ def generate_sm75_1688(): cba_kernel = cba_kernel_no_alpha if epi_func in [CbaAct.LeakyRelu]: cba_kernel = cba_kernel_alpha - sm75_code += SubstituteTemplate(cba_kernel, kernel_dict) + # sm75_code += SubstituteTemplate(cba_kernel, kernel_dict) + + kernel_str = ( + cba_header + + SubstituteTemplate(cba_kernel, kernel_dict) + + CommonTail + ) + file_name = ( + "generated_tmp/" + + kernel_dict["kernel_func_name"] + + ".cu" + ) + write_kernel_to_file(kernel_str, file_name) + all_kernel_names += ( kernel_dict["kernel_func_name"] + ", \n" ) + all_kernel_declares += ( + "cutlass::Status " + + kernel_dict["kernel_func_name"] + + "(const ConvAllParams& params);" + ) # Generate op code + op_dict["kernel_func_declare"] = all_kernel_declares op_dict["all_kernel_func_name"] = all_kernel_names sm75_code += SubstituteTemplate(CommonConvFunction, op_dict) return sm75_code +def generate_sm80_16816(cutlass_dtype="cutlass::half_t"): + kernel_dict = { + "element_a": cutlass_dtype, + "layout_a": "cutlass::layout::TensorNHWC", + "element_b": cutlass_dtype, + "layout_b": "cutlass::layout::TensorNHWC", + "element_c": cutlass_dtype, + "layout_c": "cutlass::layout::TensorNHWC", + "opcode_class": "cutlass::arch::OpClassTensorOp", + "arch": "cutlass::arch::Sm80", + "swizzling_functor": "cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<4>", + # alpha is always float! + "element_epilogue": "float", + "math_operator": "cutlass::arch::OpMultiplyAdd", + } + + kernel_dict["stride_support"] = "cutlass::conv::StrideSupport::kStrided" + + # iterate over this loop + iterator_algorithms = [ + "cutlass::conv::IteratorAlgorithm::kOptimized", + ] + + math_instructions = [ + ( + "16,8,16", + cutlass_dtype, + cutlass_dtype, + "float", + ), + ] + + alignments = [8] + + kernel_dict["align_a"] = "8" + kernel_dict["align_b"] = "8" + # this should divided by oc + kernel_dict["epilogue_vector_length"] = "8" + kernel_dict["split_k_slices"] = "1" + + sm80_code = "" + for epi_func in SupportedAct: + op_dict = {} + op_dict["func_name"] = ( + UnderScoreName[epi_func].lower() + + "_sm80_" + + ("fp16" if "half" in cutlass_dtype else "bf16") + ) + op_dict["enum_op_name"] = UnderScoreName[epi_func].upper() + # For a function, we record all its kernels into a std::vector in C++ code + all_kernel_names = "" + all_kernel_declares = "" + kernel_dict["epi_func"] = ActTag[epi_func] + suffix = 0 + for iterator_algorithm in iterator_algorithms: + for alignment in alignments: + for math_inst in math_instructions: + tiles = [ + TileDesc("256, 128, 32", 3, "64, 64, 32", math_inst), + TileDesc("128, 256, 32", 3, "64, 64, 32", math_inst), + TileDesc("256, 64, 32", 3, "64, 64, 32", math_inst), + TileDesc("256, 64, 32", 4, "64, 64, 32", math_inst), + TileDesc("64, 256, 32", 4, "64, 64, 32", math_inst), + TileDesc("128, 128, 32", 3, "64, 64, 32", math_inst), + TileDesc("128, 128, 32", 4, "64, 64, 32", math_inst), + TileDesc("128, 128, 32", 5, "64, 64, 32", math_inst), + TileDesc("128, 64, 32", 6, "64, 32, 32", math_inst), + TileDesc("64, 128, 32", 6, "32, 64, 32", math_inst), + TileDesc("64, 64, 32", 10, "32, 32, 32", math_inst), + TileDesc("256, 128, 64", 3, "64, 64, 64", math_inst), + TileDesc("128, 256, 64", 3, "64, 64, 64", math_inst), + TileDesc("256, 64, 64", 4, "64, 64, 64", math_inst), + TileDesc("64, 256, 64", 4, "64, 64, 64", math_inst), + TileDesc("128, 128, 64", 4, "64, 64, 64", math_inst), + TileDesc("256, 64, 64", 3, "64, 64, 64", math_inst), + TileDesc("64, 256, 64", 3, "64, 64, 64", math_inst), + TileDesc("128, 128, 64", 3, "64, 64, 64", math_inst), + TileDesc("128, 64, 64", 3, "64, 32, 64", math_inst), + TileDesc("64, 128, 64", 3, "32, 64, 64", math_inst), + TileDesc("64, 64, 64", 5, "32, 32, 64", math_inst), + ] + for tile in tiles: + kernel_dict["iterator_algorithm"] = iterator_algorithm + kernel_dict["Tshape"] = tile.Tshape + kernel_dict["Wshape"] = tile.Wshape + kernel_dict["Ishape"] = tile.math_inst[0] + kernel_dict["stages"] = str(tile.stages) + kernel_dict["element_accum"] = tile.math_inst[3] + kernel_dict["kernel_func_name"] = op_dict[ + "func_name" + ] + str(suffix) + suffix += 1 + cba_kernel = cba_kernel_no_alpha + if epi_func in [CbaAct.LeakyRelu]: + cba_kernel = cba_kernel_alpha + # sm80_code += SubstituteTemplate(cba_kernel, kernel_dict) + + kernel_str = ( + cba_header + + SubstituteTemplate(cba_kernel, kernel_dict) + + CommonTail + ) + file_name = ( + "generated_tmp/" + + kernel_dict["kernel_func_name"] + + ".cu" + ) + write_kernel_to_file(kernel_str, file_name) + + all_kernel_names += ( + kernel_dict["kernel_func_name"] + ", \n" + ) + + all_kernel_declares += ( + "cutlass::Status " + + kernel_dict["kernel_func_name"] + + "(const ConvAllParams& params);" + ) + + # Generate op code + op_dict["kernel_func_declare"] = all_kernel_declares + op_dict["all_kernel_func_name"] = all_kernel_names + sm80_code += SubstituteTemplate(CommonConvFunction, op_dict) + return sm80_code + + if __name__ == "__main__": - sm_versions = ["75"] + sm_versions_and_types = [] + args = parse_args() + all_code = cba_header - all_code += generate_sm75_1688() + if args.cuda_arch == "75": + sm_versions_and_types.append(["75", "fp16"]) + all_code += generate_sm75_1688() + if args.cuda_arch in ["80", "86", "89"]: + sm_versions_and_types.append(["80", "fp16"]) + sm_versions_and_types.append(["80", "bf16"]) + all_code += generate_sm80_16816() + all_code += generate_sm80_16816(cutlass_dtype="cutlass::bfloat16_t") + all_code += GenerateFunctionForPhi( - sm_versions, SupportedAct, UnderScoreName, CamelName + sm_versions_and_types, SupportedAct, UnderScoreName, CamelName ) all_code += CommonTail with open("generated_tmp/conv2d_bias_act.cu", "w") as f: diff --git a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_bias_residual.py b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_bias_residual.py index 55fde0722b6b3..629ffc12415e9 100644 --- a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_bias_residual.py +++ b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_bias_residual.py @@ -21,7 +21,7 @@ CommonTail, GenerateFunctionForPhi, ) -from util import SubstituteTemplate, TileDesc +from util import SubstituteTemplate, TileDesc, parse_args, write_kernel_to_file # this is a file's header part @@ -48,13 +48,12 @@ cbr_kernel = ( SubstituteTemplate(CommonCutlassConvKernelDeclare, dict_for_declare_part) + ''' - const half *residual = params.residual; typename ImplicitGemm::Arguments arguments{ problem_size, - {(cutlass::half_t *)input, {ic, ic * iw, ic * iw * ih}}, - {(cutlass::half_t *)(weight), {kc, kc * kw, kc * kw * kh}}, - {(cutlass::half_t *)residual, {oc, oc * ow, oc * ow * oh}}, - {(cutlass::half_t *)output, {oc, oc * ow, oc * ow * oh}}, + {input, {ic, ic * iw, ic * iw * ih}}, + {weight, {kc, kc * kw, kc * kw * kh}}, + {residual, {oc, oc * ow, oc * ow * oh}}, + {output, {oc, oc * ow, oc * ow * oh}}, {1.f, 1.f}, cutlass::conv::SplitKMode::kSerial, (cutlass::half_t *)(bias), nullptr, @@ -80,16 +79,19 @@ class CbrAct(enum.Enum): SupportedEpilogue = [ (CbrAct.Silu, "cutlass::plus", CbrAct.Identity), (CbrAct.Identity, "cutlass::plus", CbrAct.Relu), + (CbrAct.Identity, "cutlass::plus", CbrAct.Identity), ] UnderScoreName = { SupportedEpilogue[0]: "conv2d_bias_silu_add", SupportedEpilogue[1]: "conv2d_bias_add_relu", + SupportedEpilogue[2]: "conv2d_bias_add", } CamelName = { SupportedEpilogue[0]: "Conv2dBiasSiluAdd", SupportedEpilogue[1]: "Conv2dBiasAddRelu", + SupportedEpilogue[2]: "Conv2dBiasAdd", } # Generate sm75 TensorOp conv code. @@ -150,10 +152,13 @@ def generate_sm75_1688(): sm75_code = "" for epi_res_block in SupportedEpilogue: op_dict = {} - op_dict["func_name"] = UnderScoreName[epi_res_block].lower() + "_sm75" + op_dict["func_name"] = ( + UnderScoreName[epi_res_block].lower() + "_sm75_fp16" + ) op_dict["enum_op_name"] = UnderScoreName[epi_res_block].upper() # for a op, we record all its kernels into a std::vector in C++ code all_kernel_names = "" + all_kernel_declares = "" suffix = 0 for iterator_algorithm in iterator_algorithms: for alignment in alignments: @@ -188,23 +193,179 @@ def generate_sm75_1688(): kernel_dict["act2"] = ActTag[epi_res_block[2]] suffix += 1 - sm75_code += SubstituteTemplate(cbr_kernel, kernel_dict) + # sm75_code += SubstituteTemplate(cbr_kernel, kernel_dict) + + kernel_str = ( + cbr_header + + SubstituteTemplate(cbr_kernel, kernel_dict) + + CommonTail + ) + file_name = ( + "generated_tmp/" + + kernel_dict["kernel_func_name"] + + ".cu" + ) + write_kernel_to_file(kernel_str, file_name) + all_kernel_names += ( kernel_dict["kernel_func_name"] + ", \n" ) + all_kernel_declares += ( + "cutlass::Status " + + kernel_dict["kernel_func_name"] + + "(const ConvAllParams& params);" + ) - # Generate op code with sm_version + # Generate op code + op_dict["kernel_func_declare"] = all_kernel_declares op_dict["all_kernel_func_name"] = all_kernel_names sm75_code += SubstituteTemplate(CommonConvFunction, op_dict) return sm75_code +def generate_sm80_16816(cutlass_dtype="cutlass::half_t"): + kernel_dict = { + "conv_kind_name": "Fprop", + "element_a": cutlass_dtype, + "layout_a": "cutlass::layout::TensorNHWC", + "element_b": cutlass_dtype, + "layout_b": "cutlass::layout::TensorNHWC", + "element_c": cutlass_dtype, + "layout_c": "cutlass::layout::TensorNHWC", + "opcode_class": "cutlass::arch::OpClassTensorOp", + "arch": "cutlass::arch::Sm80", + "swizzling_functor": "cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<4>", + # alpha is always float! + "element_epilogue": "float", + "math_operator": "cutlass::arch::OpMultiplyAdd", + "element_residul": cutlass_dtype, + } + + kernel_dict["stride_support"] = "cutlass::conv::StrideSupport::kStrided" + + # iterate over this loop + iterator_algorithms = [ + "cutlass::conv::IteratorAlgorithm::kOptimized", + ] + + math_instructions = [ + ( + "16,8,16", + cutlass_dtype, + cutlass_dtype, + "float", + ), + ] + + alignments = [8] + + kernel_dict["align_a"] = "8" + kernel_dict["align_b"] = "8" + kernel_dict["epilogue_vector_length"] = "8" + kernel_dict["split_k_slices"] = "1" + + sm80_code = "" + for epi_res_block in SupportedEpilogue: + op_dict = {} + op_dict["func_name"] = ( + UnderScoreName[epi_res_block].lower() + + "_sm80_" + + ("fp16" if "half" in cutlass_dtype else "bf16") + ) + + op_dict["enum_op_name"] = UnderScoreName[epi_res_block].upper() + # for a op, we record all its kernels into a std::vector in C++ code + all_kernel_names = "" + all_kernel_declares = "" + suffix = 0 + for iterator_algorithm in iterator_algorithms: + for alignment in alignments: + for math_inst in math_instructions: + tiles = [ + TileDesc("256, 128, 32", 3, "64, 64, 32", math_inst), + TileDesc("128, 256, 32", 3, "64, 64, 32", math_inst), + TileDesc("256, 64, 32", 3, "64, 64, 32", math_inst), + TileDesc("256, 64, 32", 4, "64, 64, 32", math_inst), + TileDesc("64, 256, 32", 4, "64, 64, 32", math_inst), + TileDesc("128, 128, 32", 3, "64, 64, 32", math_inst), + TileDesc("128, 128, 32", 4, "64, 64, 32", math_inst), + TileDesc("128, 128, 32", 5, "64, 64, 32", math_inst), + TileDesc("128, 64, 32", 6, "64, 32, 32", math_inst), + TileDesc("64, 128, 32", 6, "32, 64, 32", math_inst), + TileDesc("64, 64, 32", 10, "32, 32, 32", math_inst), + TileDesc("256, 128, 64", 3, "64, 64, 64", math_inst), + TileDesc("128, 256, 64", 3, "64, 64, 64", math_inst), + TileDesc("256, 64, 64", 4, "64, 64, 64", math_inst), + TileDesc("64, 256, 64", 4, "64, 64, 64", math_inst), + TileDesc("128, 128, 64", 4, "64, 64, 64", math_inst), + TileDesc("256, 64, 64", 3, "64, 64, 64", math_inst), + TileDesc("64, 256, 64", 3, "64, 64, 64", math_inst), + TileDesc("128, 128, 64", 3, "64, 64, 64", math_inst), + TileDesc("128, 64, 64", 3, "64, 32, 64", math_inst), + TileDesc("64, 128, 64", 3, "32, 64, 64", math_inst), + TileDesc("64, 64, 64", 5, "32, 32, 64", math_inst), + ] + + for tile in tiles: + kernel_dict["iterator_algorithm"] = iterator_algorithm + kernel_dict["Tshape"] = tile.Tshape + kernel_dict["Wshape"] = tile.Wshape + kernel_dict["Ishape"] = tile.math_inst[0] + kernel_dict["stages"] = str(tile.stages) + kernel_dict["element_accum"] = tile.math_inst[3] + kernel_dict["kernel_func_name"] = op_dict[ + "func_name" + ] + str(suffix) + kernel_dict["act1"] = ActTag[epi_res_block[0]] + kernel_dict["binary"] = epi_res_block[1] + kernel_dict["act2"] = ActTag[epi_res_block[2]] + suffix += 1 + + # sm80_code += SubstituteTemplate(cbr_kernel, kernel_dict) + kernel_str = ( + cbr_header + + SubstituteTemplate(cbr_kernel, kernel_dict) + + CommonTail + ) + file_name = ( + "generated_tmp/" + + kernel_dict["kernel_func_name"] + + ".cu" + ) + write_kernel_to_file(kernel_str, file_name) + + all_kernel_names += ( + kernel_dict["kernel_func_name"] + ", \n" + ) + all_kernel_declares += ( + "cutlass::Status " + + kernel_dict["kernel_func_name"] + + "(const ConvAllParams& params);" + ) + + # Generate op code + op_dict["kernel_func_declare"] = all_kernel_declares + op_dict["all_kernel_func_name"] = all_kernel_names + sm80_code += SubstituteTemplate(CommonConvFunction, op_dict) + return sm80_code + + if __name__ == "__main__": - sm_versions = ["75"] + sm_versions_and_types = [] + args = parse_args() + all_code = cbr_header - all_code += generate_sm75_1688() + if args.cuda_arch == "75": + sm_versions_and_types.append(["75", "fp16"]) + all_code += generate_sm75_1688() + if args.cuda_arch in ["80", "86", "89"]: + sm_versions_and_types.append(["80", "fp16"]) + sm_versions_and_types.append(["80", "bf16"]) + all_code += generate_sm80_16816() + all_code += generate_sm80_16816(cutlass_dtype="cutlass::bfloat16_t") + all_code += GenerateFunctionForPhi( - sm_versions, SupportedEpilogue, UnderScoreName, CamelName + sm_versions_and_types, SupportedEpilogue, UnderScoreName, CamelName ) all_code += CommonTail with open("generated_tmp/conv2d_bias_residual.cu", "w") as f: diff --git a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_common.py b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_common.py index 7c95892006c43..6dbf6bcbbb82a 100644 --- a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_common.py +++ b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_common.py @@ -51,10 +51,14 @@ using ImplicitGemm = cutlass::conv::device::ImplicitGemmConvolution; - const half *input = params.input; - const half *weight = params.weight; - const half *bias = params.bias; - half *output = params.output; + + ${element_a} *input = (${element_a} *)(params.input); + ${element_b} *weight = (${element_b} *)(params.weight); + ${element_c} *bias = (${element_c} *)(params.bias); + ${element_c} *output = (${element_c} *)(params.output); + // only used by conv2d_bias_residual + auto residual = (${element_c} *)(params.residual); + int batch = params.batch; int ic = params.ic; int ih = params.ih; @@ -112,6 +116,9 @@ # ${enum_op_name} is like CONV2D_BIAS_SILU CommonConvFunction = """ + +${kernel_func_declare} + std::vector> ${func_name}_all_func = {${all_kernel_func_name}}; @@ -163,8 +170,15 @@ """ +def convert_c_data_type(dtype): + if dtype == "fp16": + return "Conv2dDataType::fp16" + if dtype == "bf16": + return "Conv2dDataType::bf16" + + CommonDispatchTemp = ''' - if (params.sm_version == ${sm_code}) + if (params.sm_version == ${sm_code} && params.data_type == ${data_type}) { ${op_name_with_sm}(params); } @@ -182,16 +196,21 @@ # Wrap different sm versions into a function called by phi def GenerateFunctionForPhi( - sm_versions, support_epi_funcs, underscore_names, camel_names + sm_versions_and_types, support_epi_funcs, underscore_names, camel_names ): generated_code = "" for epi_func in support_epi_funcs: dispatch_body = "" - for sm_version in sm_versions: + for sm_version, data_type in sm_versions_and_types: sm_dicts = {} sm_dicts["sm_code"] = sm_version + sm_dicts["data_type"] = convert_c_data_type(data_type) sm_dicts["op_name_with_sm"] = ( - underscore_names[epi_func].lower() + "_sm" + sm_version + underscore_names[epi_func].lower() + + "_sm" + + sm_version + + "_" + + data_type ) dispatch_body += SubstituteTemplate(CommonDispatchTemp, sm_dicts) op_dicts = {} diff --git a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_decl.h b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_decl.h index aaad46de5cb0d..b29ce65f5230a 100644 --- a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_decl.h +++ b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_decl.h @@ -20,12 +20,18 @@ namespace phi { namespace fusion { namespace cutlass_internal { +typedef enum { + fp32, + fp16, + bf16, +} Conv2dDataType; + typedef struct { - const half *input; - const half *weight; - const half *bias; - const half *residual; - half *output; + const void *input; + const void *weight; + const void *bias; + const void *residual; + void *output; int batch; int ic; int ih; @@ -48,6 +54,7 @@ typedef struct { cudaStream_t stream; float alpha; // for leaky_relu use int sm_version = 75; + Conv2dDataType data_type; void *workspace = nullptr; } ConvAllParams; diff --git a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_depthwise_bias_act.py b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_depthwise_bias_act.py index fb2f2be096110..5114d69e97060 100644 --- a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_depthwise_bias_act.py +++ b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_depthwise_bias_act.py @@ -208,6 +208,7 @@ def generate_conv2d_depthwise(): ) # generate op code op_dict["all_kernel_func_name"] = all_kernel_names + op_dict["kernel_func_declare"] = ";" all_code += SubstituteTemplate(CommonConvFunction, op_dict) return all_code diff --git a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_util.cu b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_util.cu index 51bc71983105a..0a08cd165519d 100644 --- a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_util.cu +++ b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_util.cu @@ -26,10 +26,11 @@ struct logical_coord { int w; }; -float diff(const half *c, const float *c_baseline, int n) { +template +float diff(const T *c, const float *c_baseline, int n) { float max_diff = -1.; for (int i = 0; i < n; i++) { - float c_value = __half2float(c[i]); + float c_value = static_cast(c[i]); if (std::abs(c_baseline[i] - c_value) > max_diff) { max_diff = std::abs(c_baseline[i] - c_value); } @@ -42,10 +43,10 @@ __device__ int gpu_nhwc(struct logical_coord shape, return index.n * shape.h * shape.w * shape.c + index.h * shape.w * shape.c + index.w * shape.c + index.c; } - -__global__ void naive_conv2d_kernel(const half *input, - const half *weight, - const half *bias, +template +__global__ void naive_conv2d_kernel(const T *input, + const T *weight, + const T *bias, float *output, int batch, int ic, @@ -63,7 +64,7 @@ __global__ void naive_conv2d_kernel(const half *input, int oh, int ow, int groups, - const half *residual, + const T *residual, float alpha, // for leaky_relu OpType op_type) { int M = batch * oh * ow; @@ -100,12 +101,12 @@ __global__ void naive_conv2d_kernel(const half *input, if (iw_i < 0 || iw_i >= iw) continue; struct logical_coord input_index = {batch_i, ic_i, ih_i, iw_i}; - const half *weight_ptr = weight + gpu_nhwc(weight_shape, weight_index); - const half *in_ptr = input + gpu_nhwc(input_shape, input_index); - sum += __half2float(*in_ptr) * __half2float(*weight_ptr); + const T *weight_ptr = weight + gpu_nhwc(weight_shape, weight_index); + const T *in_ptr = input + gpu_nhwc(input_shape, input_index); + sum += static_cast(*in_ptr) * static_cast(*weight_ptr); } - sum += __half2float(*(bias + oc_i)); + sum += static_cast(*(bias + oc_i)); float x = sum; switch (op_type) { @@ -121,10 +122,19 @@ __global__ void naive_conv2d_kernel(const half *input, case CONV2D_DEPTHWISE_BIAS_SILU: *out_ptr = x * (1.f / (1 + exp(-x))); break; + case CONV2D_BIAS_SILU_ADD: + x = x * (1.f / (1 + exp(-x))); + x += static_cast(*(residual + out_offset)); + *out_ptr = x; + break; case CONV2D_BIAS_ADD_RELU: - x += __half2float(*(residual + out_offset)); + x += static_cast(*(residual + out_offset)); *out_ptr = x > 0 ? x : 0; break; + case CONV2D_BIAS_ADD: + x += static_cast(*(residual + out_offset)); + *out_ptr = x; + break; case CONV2D_BIAS_LEAKY_RELU: *out_ptr = x > 0 ? x : (x * alpha); break; @@ -136,12 +146,12 @@ __global__ void naive_conv2d_kernel(const half *input, break; } } - -float conv2d_diff_gpu(const ConvAllParams ¶ms, OpType op_type) { - const half *input = params.input; - const half *weight = params.weight; - const half *bias = params.bias; - half *output = params.output; +template +float conv2d_diff_gpu(const ConvAllParams ¶ms, OpType op_type, T a) { + const T *input = (const T *)(params.input); + const T *weight = (const T *)(params.weight); + const T *bias = (const T *)(params.bias); + T *output = static_cast(params.output); int batch = params.batch; int ic = params.ic; int ih = params.ih; @@ -155,7 +165,7 @@ float conv2d_diff_gpu(const ConvAllParams ¶ms, OpType op_type) { int stride_w = params.stride_w; int dilation_h = params.dilation_h; int dilation_w = params.dilation_w; - const half *residual = params.residual; + const T *residual = (const T *)(params.residual); int groups = params.groups; int oh = params.oh; @@ -169,11 +179,11 @@ float conv2d_diff_gpu(const ConvAllParams ¶ms, OpType op_type) { uint3 block = {blockM, blockN, 1}; int output_size = batch * oc * oh * ow; - half *output_from_cutlass = - reinterpret_cast(malloc(sizeof(half) * output_size)); + T *output_from_cutlass = + reinterpret_cast(malloc(sizeof(T) * output_size)); cudaMemcpy(output_from_cutlass, output, - output_size * sizeof(half), + output_size * sizeof(T), cudaMemcpyDeviceToHost); float *gpu_output; @@ -207,6 +217,13 @@ float conv2d_diff_gpu(const ConvAllParams ¶ms, OpType op_type) { gpu_output, output_size * sizeof(float), cudaMemcpyDeviceToHost); + + // cudaMemcpy(output, + // gpu_output, + // output_size * sizeof(T), + // cudaMemcpyDeviceToDevice); + // cudaMemset(output, 0, output_size * sizeof(T)); + float max_diff = diff(output_from_cutlass, output_from_gpu, output_size); free(output_from_cutlass); @@ -232,6 +249,12 @@ std::string OpType2String(OpType op_type) { case CONV2D_BIAS_ADD_RELU: return "conv2d_bias_add_relu"; break; + case CONV2D_BIAS_ADD: + return "conv2d_bias_add"; + break; + case CONV2D_BIAS_SILU_ADD: + return "conv2d_bias_silu_add"; + break; case CONV2D_BIAS_LEAKY_RELU: return "conv2d_bias_leaky_relu"; case CONV2D_DEPTHWISE_BIAS: @@ -253,7 +276,7 @@ int ProfileToGetBestConfig( const ConvAllParams ¶ms, OpType op_type) { constexpr int WARMUP = 10; - constexpr int REPEAT = 100; + constexpr int REPEAT = 10; float min_time = 100000.f; int min_time_index = -1; for (int i = 0; i < all_func.size(); i++) { @@ -286,11 +309,23 @@ int ProfileToGetBestConfig( if (elapsed_time < min_time && status == cutlass::Status::kSuccess) { min_time = elapsed_time; min_time_index = i; - // debug code - std::cout << OpType2String(op_type) << ": tactic " << i - << " has max diff " << conv2d_diff_gpu(params, op_type) - << " compared with baseline," - << "cost_time: " << elapsed_time << "ms." << std::endl; + + if (params.data_type == Conv2dDataType::fp16) { + // debug code + std::cout << OpType2String(op_type) << ": tactic " << i + << " has max diff " + << conv2d_diff_gpu(params, op_type, (half)(1.0)) + << " compared with baseline," + << "cost_time: " << elapsed_time << "ms." << std::endl; + } else if (params.data_type == Conv2dDataType::bf16) { + // debug code + std::cout << OpType2String(op_type) << ": tactic " << i + << " has max diff " + << conv2d_diff_gpu( + params, op_type, static_cast(1.0)) + << " compared with baseline," + << "cost_time: " << elapsed_time << "ms." << std::endl; + } } } @@ -301,11 +336,6 @@ int ProfileToGetBestConfig( return min_time_index; } -__attribute__((dllexport)) int HelloFromCutlassConv2d(int a, int b) { - std::cout << "welcom using Cutlass Conv2d" << std::endl; - return 1; -} - } // namespace cutlass_internal } // namespace fusion } // namespace phi diff --git a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_util.h b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_util.h index 80865e0e1cded..508b8a8f1ae3b 100644 --- a/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_util.h +++ b/paddle/phi/kernels/fusion/cutlass/conv2d/conv2d_util.h @@ -37,6 +37,7 @@ typedef enum { CONV2D_BIAS, CONV2D_BIAS_RELU, CONV2D_BIAS_ADD_RELU, + CONV2D_BIAS_ADD, CONV2D_BIAS_SILU, CONV2D_BIAS_LEAKY_RELU, CONV2D_BIAS_SIGMOID, diff --git a/paddle/phi/kernels/fusion/cutlass/fused_conv2d_add_act_kernel.cu b/paddle/phi/kernels/fusion/cutlass/fused_conv2d_add_act_kernel.cu index dceaafd2e7172..5c09b92fd83de 100644 --- a/paddle/phi/kernels/fusion/cutlass/fused_conv2d_add_act_kernel.cu +++ b/paddle/phi/kernels/fusion/cutlass/fused_conv2d_add_act_kernel.cu @@ -98,30 +98,66 @@ void FusedConv2dAddActKernel(const Context& ctx, const int oh = out_dims[1]; const int ow = out_dims[2]; - ConvAllParams params = {reinterpret_cast(x.data()), - reinterpret_cast(filter.data()), - reinterpret_cast(bias.data()), - nullptr, - reinterpret_cast(output->data()), - batch, - ic, - ih, - iw, - kh, - kw, - oc, - pad_h0, - pad_h1, - pad_w0, - pad_w1, - stride_h, - stride_w, - dilation_h, - dilation_w, - oh, - ow, - groups, - ctx.stream()}; + int64_t device_id = ctx.GetPlace().GetDeviceId(); + int sm_version = backends::gpu::GetGPUComputeCapability(device_id); + + auto get_conv2d_dtype = [&](decltype(x.dtype()) x_type) + -> phi::fusion::cutlass_internal::Conv2dDataType { + switch (x_type) { + case phi::DataType::FLOAT32: + return Conv2dDataType::fp32; + case phi::DataType::FLOAT16: + return Conv2dDataType::fp16; + case phi::DataType::BFLOAT16: + return Conv2dDataType::bf16; + } + }; + + auto cutlass_dispatch_sm_version = [&](int device_sm_version) -> int { + if (device_sm_version < 75) { + PADDLE_ENFORCE_GE( + device_sm_version, + 75, + phi::errors::PreconditionNotMet( + "fused_conv2d_add_act only supports sm >= 75, but got %d.", + device_sm_version)); + } else if (device_sm_version > 80) { + return 80; + } else { + return device_sm_version; + } + }; + + ConvAllParams params = { + reinterpret_cast(x.data()), + reinterpret_cast(filter.data()), + reinterpret_cast(bias.data()), + nullptr, + reinterpret_cast(output->data()), + batch, + ic, + ih, + iw, + kh, + kw, + oc, + pad_h0, + pad_h1, + pad_w0, + pad_w1, + stride_h, + stride_w, + dilation_h, + dilation_w, + oh, + ow, + groups, + ctx.stream(), + 0, // alpha + cutlass_dispatch_sm_version(sm_version), + get_conv2d_dtype(x.dtype()), + nullptr, + }; void* dlhandler = phi::dynload::GetCutlassConv2dHandle(); func conv_func = NULL; @@ -161,11 +197,13 @@ void FusedConv2dAddActKernel(const Context& ctx, CHECK_EQ(groups == 1, true); if (residual) { if (activation == "relu") { - params.residual = reinterpret_cast(residual->data()); + params.residual = reinterpret_cast(residual->data()); conv_func = (func)(dlsym(dlhandler, "Conv2dBiasAddRelu")); } else { PADDLE_THROW(phi::errors::InvalidArgument( - "Cutlass now only support relu activation in a residual block")); + "Cutlass now only support relu activation in a residual block, but " + "got %s.", + activation.c_str())); } } else if (activation == "relu") { conv_func = (func)(dlsym(dlhandler, "Conv2dBiasRelu")); @@ -194,4 +232,5 @@ PD_REGISTER_KERNEL(fused_conv2d_add_act, ALL_LAYOUT, phi::fusion::cutlass_internal::FusedConv2dAddActKernel, float, + phi::dtype::bfloat16, phi::dtype::float16) {} diff --git a/paddle/phi/kernels/fusion/cutlass/util.py b/paddle/phi/kernels/fusion/cutlass/util.py index 200960f39c56e..d3ffb648362f6 100644 --- a/paddle/phi/kernels/fusion/cutlass/util.py +++ b/paddle/phi/kernels/fusion/cutlass/util.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import argparse import re @@ -35,3 +36,28 @@ def SubstituteTemplate(template, values): changed = True text = newtext return text + + +def parse_args(): + parser = argparse.ArgumentParser( + description="The argument for generating the conv2d_bias_act kernels." + ) + + parser.add_argument( + "--cuda_arch", + type=str, + default=None, + help="The CUDA architecture to be generated.", + ) + args = parser.parse_args() + + return args + + +def write_kernel_to_file(kernel, file_name): + with open( + file_name, + "w", + ) as f: + f.write(kernel) + f.close() From becb078860c32cdeabf22083f322b7bc6480edb8 Mon Sep 17 00:00:00 2001 From: ming1753 <61511741+ming1753@users.noreply.github.com> Date: Thu, 29 Feb 2024 20:56:30 +0800 Subject: [PATCH 182/282] [Inference] Fix absolute paths bug in tensorrt_engine op (#62205) * fix absolute paths bug in tensorrt_engine op * fix bug * fix bug * fix bug --- .../ir_passes/tensorrt_subgraph_pass.cc | 4 +-- .../passes/save_optimized_model_pass.cc | 4 +-- .../fluid/inference/api/analysis_predictor.cc | 27 ++++++++++++++++--- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc b/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc index 69b27b1214839..5b2bed7745fcf 100644 --- a/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc +++ b/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc @@ -506,8 +506,8 @@ std::string TensorRtSubgraphPass::CreateTensorRTOp( &max_shape_tensor, &optim_shape_tensor); } else { - shape_range_info_path = - Get("model_opt_cache_dir") + "shape_range_info.pbtxt"; + shape_range_info_path = Get("model_opt_cache_dir") + "/" + + "shape_range_info.pbtxt"; if (open(shape_range_info_path.c_str(), O_RDONLY) != -1) { VLOG(1) << "trt dynamic_shape deserialize from " << shape_range_info_path; diff --git a/paddle/fluid/inference/analysis/passes/save_optimized_model_pass.cc b/paddle/fluid/inference/analysis/passes/save_optimized_model_pass.cc index cc463ce45f105..8d988de162100 100644 --- a/paddle/fluid/inference/analysis/passes/save_optimized_model_pass.cc +++ b/paddle/fluid/inference/analysis/passes/save_optimized_model_pass.cc @@ -74,7 +74,7 @@ void SaveOptimizedModelPass::SaveOptimizedModel(Argument* argument) { } } - std::string save_params_path = path + ".pdiparams"; + std::string save_params_path = path + "/" + "_optimized.pdiparams"; std::vector save_var_list(save_var_set.begin(), save_var_set.end()); std::sort(save_var_list.begin(), save_var_list.end()); @@ -105,7 +105,7 @@ void SaveOptimizedModelPass::SaveOptimizedModel(Argument* argument) { } } } - std::string save_model_path = path + ".pdmodel"; + std::string save_model_path = path + "/" + "_optimized.pdmodel"; auto str = optimized_program_desc.Proto()->SerializeAsString(); std::ofstream file(save_model_path.c_str(), std::ios::binary); file.write(str.c_str(), str.size()); // NOLINT diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index b61e8eaa0577d..d52f71573dc44 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -424,8 +424,10 @@ bool AnalysisPredictor::Init( // Use Optimized model to inference if (config_.use_optimized_model_) { std::string optimized_model_path = GetOptimizedModelPath(); - std::string optimized_model = optimized_model_path + ".pdmodel"; - std::string optimized_params = optimized_model_path + ".pdiparams"; + std::string optimized_model = + optimized_model_path + "/" + "_optimized.pdmodel"; + std::string optimized_params = + optimized_model_path + "/" + "_optimized.pdiparams"; if (FileExists(optimized_model) && FileExists(optimized_params)) { config_.SetModel(optimized_model, optimized_params); LOG(INFO) << "Load Optimized model from " << optimized_model_path; @@ -596,7 +598,7 @@ std::string AnalysisPredictor::GetOptimizedModelPath() { ? config_.model_dir() : inference::analysis::GetDirRoot(config_.prog_file()); } - return model_opt_cache_dir + "/" + "_optimized"; + return model_opt_cache_dir; } void AnalysisPredictor::ClearExtraParams() { @@ -608,6 +610,25 @@ void AnalysisPredictor::ClearExtraParams() { op_desc->GetAttr("parameters")); trt_repetitive_params.insert( trt_repetitive_params.end(), trt_params.begin(), trt_params.end()); + // NOTE(ming1753): This is a trick solution to the problem of possible + // absolute paths in the model_opt_cache_dir and shape_range_info_path + // attributes in tensorrt_engine op. + auto model_opt_cache_dir_from_model = PADDLE_GET_CONST( + std::string, op_desc->GetAttr("model_opt_cache_dir")); + auto model_opt_cache_dir = GetOptimizedModelPath(); + if (op_desc->HasAttr("model_opt_cache_dir")) { + op_desc->SetAttr("model_opt_cache_dir", model_opt_cache_dir); + } + if (op_desc->HasAttr("shape_range_info_path")) { + if (config_.shape_range_info_path_.empty()) { + op_desc->SetAttr( + "shape_range_info_path", + model_opt_cache_dir + "/" + "shape_range_info.pbtxt"); + } else { + op_desc->SetAttr("shape_range_info_path", + config_.shape_range_info_path_); + } + } } } From 762ae52a616764e23ea0d88b27dfa6decd57750b Mon Sep 17 00:00:00 2001 From: lzydev Date: Thu, 29 Feb 2024 21:09:28 +0800 Subject: [PATCH 183/282] fix amp pass bug (#62239) --- .../distributed/passes/auto_parallel_fp16.py | 23 ++++--------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/python/paddle/distributed/passes/auto_parallel_fp16.py b/python/paddle/distributed/passes/auto_parallel_fp16.py index 73cad3e3e928c..c1d8c54c6b4b2 100644 --- a/python/paddle/distributed/passes/auto_parallel_fp16.py +++ b/python/paddle/distributed/passes/auto_parallel_fp16.py @@ -308,25 +308,10 @@ def resolute_cast_op(self, block): if op.type == "cast": in_name = op.input('X')[0] out_name = op.output('Out')[0] - if "@GRAD" in in_name: - in_var_fw = block._find_var_recursive( - in_name[: in_name.find("@")] - ) - out_var_fw = block._find_var_recursive( - out_name[: out_name.find("@")] - ) - op._set_attr('in_dtype', in_var_fw.dtype) - op._set_attr('out_dtype', out_var_fw.dtype) - - in_var = block._find_var_recursive(in_name) - out_var = block._find_var_recursive(out_name) - in_var.desc.set_dtype(in_var_fw.dtype) - out_var.desc.set_dtype(out_var_fw.dtype) - else: - in_var = block._find_var_recursive(in_name) - out_var = block._find_var_recursive(out_name) - op._set_attr("in_dtype", in_var.dtype) - op._set_attr("out_dtype", out_var.dtype) + in_var = block._find_var_recursive(in_name) + out_var = block._find_var_recursive(out_name) + op._set_attr("in_dtype", in_var.dtype) + op._set_attr("out_dtype", out_var.dtype) def resolute_tensor_dtype(self, block): for op in block.ops: From 6470913f2e37ebfc17deefa3e0a61a3261ef36e7 Mon Sep 17 00:00:00 2001 From: liuzhenhai93 Date: Thu, 29 Feb 2024 21:36:02 +0800 Subject: [PATCH 184/282] =?UTF-8?q?=E3=80=90auto=20parallel=E3=80=91expand?= =?UTF-8?q?=20as=20infer=20spmd=20(#62159)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * expand as infer spmd * compile * add test * polish * polish --- paddle/phi/infermeta/spmd_rules/expand_as.cc | 86 +++++++++++++++++ paddle/phi/infermeta/spmd_rules/expand_as.h | 38 ++++++++ paddle/phi/infermeta/spmd_rules/rules.cc | 10 ++ paddle/phi/infermeta/spmd_rules/rules.h | 1 + .../auto_parallel/static/completion.py | 1 + .../static/operators/__init__.py | 1 + .../static/operators/dist_default.py | 18 ++-- .../static/operators/dist_expand_as.py | 80 ++++++++++++++++ test/cpp/auto_parallel/CMakeLists.txt | 3 + .../auto_parallel/expand_as_spmd_rule_test.cc | 95 +++++++++++++++++++ 10 files changed, 326 insertions(+), 7 deletions(-) create mode 100644 paddle/phi/infermeta/spmd_rules/expand_as.cc create mode 100644 paddle/phi/infermeta/spmd_rules/expand_as.h create mode 100644 python/paddle/distributed/auto_parallel/static/operators/dist_expand_as.py create mode 100644 test/cpp/auto_parallel/expand_as_spmd_rule_test.cc diff --git a/paddle/phi/infermeta/spmd_rules/expand_as.cc b/paddle/phi/infermeta/spmd_rules/expand_as.cc new file mode 100644 index 0000000000000..6bd663c826664 --- /dev/null +++ b/paddle/phi/infermeta/spmd_rules/expand_as.cc @@ -0,0 +1,86 @@ +/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/phi/infermeta/spmd_rules/expand_as.h" + +#include "glog/logging.h" +#include "paddle/phi/infermeta/spmd_rules/spmd_rule_macro_define.h" +#include "paddle/phi/infermeta/spmd_rules/utils.h" + +namespace phi { +namespace distributed { + +std::tuple AlignExpandAsDistAttrs( + const DistMetaTensor& x, const DistMetaTensor& y) { + EXTRACT_SHAPE_AND_DIST_ATTR(x); + EXTRACT_SHAPE_AND_DIST_ATTR(y); + auto x_dist_attr_dst = CopyTensorDistAttrForOutput(x_dist_attr_src); + auto y_dist_attr_dst = CopyTensorDistAttrForOutput(y_dist_attr_src); + auto x_dims_mapping_dst = x_dims_mapping_src; + auto y_dims_mapping_dst = y_dims_mapping_src; + int dims_diff = y_ndim - x_ndim; + for (int i = 0; i < y_ndim; ++i) { + if (i >= dims_diff) { + if (x_shape[i - dims_diff] == y_shape[i]) { + x_dims_mapping_dst[i - dims_diff] = y_dims_mapping_src[i]; + } else { + x_dims_mapping_dst[i - dims_diff] = -1; + } + } + } + x_dist_attr_dst.set_dims_mapping(x_dims_mapping_dst); + y_dist_attr_dst.set_dims_mapping(y_dims_mapping_dst); + LOG_SPMD_INPUT(x); + LOG_SPMD_INPUT(y); + return {x_dist_attr_dst, y_dist_attr_dst}; +} + +SpmdInfo ExpandAsInferSpmd(const DistMetaTensor& x, + const DistMetaTensor& y, + const std::vector& target_shape) { + auto [x_dist_attr, y_dist_attr] = AlignExpandAsDistAttrs(x, y); + return {{x_dist_attr, y_dist_attr}, {y_dist_attr}}; +} + +SpmdInfo ExpandAsInferSpmdReverse(const DistMetaTensor& x, + const DistMetaTensor& y, + const DistMetaTensor& output, + const std::vector& target_shape) { + auto [x_dist_attr, y_dist_attr] = AlignExpandAsDistAttrs(x, output); + return {{x_dist_attr, y_dist_attr}, {y_dist_attr}}; +} + +SpmdInfo ExpandAsGradInferSpmd(const DistMetaTensor& x, + const DistMetaTensor& out_grad, + const std::vector& target_shape) { + auto [x_dist_attr, y_dist_attr] = AlignExpandAsDistAttrs(x, out_grad); + const auto& x_dims_mapping = x_dist_attr.dims_mapping(); + const auto& y_dims_mapping = y_dist_attr.dims_mapping(); + + // handle partial grad + auto x_grad_dist_attr = x_dist_attr; + int x_ndims = x_dims_mapping.size(); + int y_ndims = y_dims_mapping.size(); + int dims_diff = y_ndims - x_ndims; + std::vector partial; + for (int i = 0; i < y_ndims; ++i) { + if (i < dims_diff || x_dims_mapping[i - dims_diff] != y_dims_mapping[i]) { + if (y_dims_mapping[i] >= 0) { + partial.push_back(y_dims_mapping[i]); + } + } + } + x_grad_dist_attr.set_partial_status(partial); + return {{x_dist_attr, y_dist_attr}, {x_grad_dist_attr}}; +} + +} // namespace distributed +} // namespace phi diff --git a/paddle/phi/infermeta/spmd_rules/expand_as.h b/paddle/phi/infermeta/spmd_rules/expand_as.h new file mode 100644 index 0000000000000..67cc6f3853dc1 --- /dev/null +++ b/paddle/phi/infermeta/spmd_rules/expand_as.h @@ -0,0 +1,38 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include "paddle/phi/core/distributed/auto_parallel/dist_meta_tensor.h" +#include "paddle/phi/core/distributed/type_defs.h" + +namespace phi { +namespace distributed { +SpmdInfo ExpandAsInferSpmd(const DistMetaTensor& x, + const DistMetaTensor& y, + const std::vector& target_shape); + +SpmdInfo ExpandAsInferSpmdReverse(const DistMetaTensor& x, + const DistMetaTensor& y, + const DistMetaTensor& output, + const std::vector& target_shape); + +SpmdInfo ExpandAsGradInferSpmd(const DistMetaTensor& x, + const DistMetaTensor& out_grad, + const std::vector& target_shape); + +} // namespace distributed +} // namespace phi diff --git a/paddle/phi/infermeta/spmd_rules/rules.cc b/paddle/phi/infermeta/spmd_rules/rules.cc index aff1633ee2cba..d8ba17971b6a9 100644 --- a/paddle/phi/infermeta/spmd_rules/rules.cc +++ b/paddle/phi/infermeta/spmd_rules/rules.cc @@ -605,6 +605,16 @@ PD_REGISTER_SPMD_RULE( PD_INFER_SPMD( phi::distributed::FusedLinearParamGradAddInferSpmdFakeReverse)); +PD_REGISTER_SPMD_RULE( + expand_as, + PD_INFER_SPMD(phi::distributed::ExpandAsInferSpmd), + PD_INFER_SPMD(phi::distributed::ExpandAsInferSpmdReverse)); + +PD_REGISTER_SPMD_RULE( + expand_as_v2, + PD_INFER_SPMD(phi::distributed::ExpandAsInferSpmd), + PD_INFER_SPMD(phi::distributed::ExpandAsInferSpmdReverse)); + // scatter PD_REGISTER_SPMD_RULE(scatter, PD_INFER_SPMD(phi::distributed::ScatterInferSpmd), diff --git a/paddle/phi/infermeta/spmd_rules/rules.h b/paddle/phi/infermeta/spmd_rules/rules.h index ed6a6cbb9641c..805d20904c8a5 100644 --- a/paddle/phi/infermeta/spmd_rules/rules.h +++ b/paddle/phi/infermeta/spmd_rules/rules.h @@ -20,6 +20,7 @@ limitations under the License. */ #include "paddle/phi/infermeta/spmd_rules/default_data_parallel.h" #include "paddle/phi/infermeta/spmd_rules/elementwise.h" #include "paddle/phi/infermeta/spmd_rules/embedding.h" +#include "paddle/phi/infermeta/spmd_rules/expand_as.h" #include "paddle/phi/infermeta/spmd_rules/flash_attention.h" #include "paddle/phi/infermeta/spmd_rules/flatten.h" #include "paddle/phi/infermeta/spmd_rules/full_like.h" diff --git a/python/paddle/distributed/auto_parallel/static/completion.py b/python/paddle/distributed/auto_parallel/static/completion.py index 01db8beacb7e4..663cd1afd94a4 100644 --- a/python/paddle/distributed/auto_parallel/static/completion.py +++ b/python/paddle/distributed/auto_parallel/static/completion.py @@ -181,6 +181,7 @@ def _can_apply_infer_spmd_rule(dist_op): "unsqueeze2", "silu", "concat", + "expand_as_v2", ] parallel_ce = os.getenv("PARALLEL_CROSS_ENTROPY") if parallel_ce == "true": diff --git a/python/paddle/distributed/auto_parallel/static/operators/__init__.py b/python/paddle/distributed/auto_parallel/static/operators/__init__.py index a0415fe4e6b00..93d2c2597e819 100644 --- a/python/paddle/distributed/auto_parallel/static/operators/__init__.py +++ b/python/paddle/distributed/auto_parallel/static/operators/__init__.py @@ -21,6 +21,7 @@ dist_dropout, dist_eltwise, dist_embedding, + dist_expand_as, dist_fill_constant_batch_size_like, dist_flash_attn, dist_fused_attention, diff --git a/python/paddle/distributed/auto_parallel/static/operators/dist_default.py b/python/paddle/distributed/auto_parallel/static/operators/dist_default.py index 472621c99cada..85163c57a3baa 100644 --- a/python/paddle/distributed/auto_parallel/static/operators/dist_default.py +++ b/python/paddle/distributed/auto_parallel/static/operators/dist_default.py @@ -49,6 +49,7 @@ "fill_constant_batch_size_like", "fill_constant", "expand_v2", + "expand_as_v2", ] @@ -534,12 +535,15 @@ def forward(ctx, *args, **kwargs): # replicate op in dist program dst_op = copy_op_without_infer_shape(src_op, main_block, ctx, kwargs) - if ( - src_op.has_attr('shape') - and src_op.attr('shape') - and src_op.type in __op_has_shape_attr__ - ): - shape_list = src_op.attr('shape') + def get_shape_attr_name(): + for name in ["shape", "target_shape"]: + if src_op.has_attr(name) and src_op.attr(name): + return name + return None + + shape_attr_name = get_shape_attr_name() + if shape_attr_name and src_op.type in __op_has_shape_attr__: + shape_list = src_op.attr(shape_attr_name) Out_var = main_block._var_recursive(kwargs['Out'][0]) op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) dim_mapping = op_dist_attr.get_output_dims_mapping(Out_var.name) @@ -552,7 +556,7 @@ def forward(ctx, *args, **kwargs): shape_list[idx] = ( shape_list[idx] // process_mesh_shape[axis] ) - dst_op.desc._set_attr('shape', shape_list) + dst_op.desc._set_attr(shape_attr_name, shape_list) # data parallel synchronization for primitive operators from paddle.incubate.autograd import prim_enabled diff --git a/python/paddle/distributed/auto_parallel/static/operators/dist_expand_as.py b/python/paddle/distributed/auto_parallel/static/operators/dist_expand_as.py new file mode 100644 index 0000000000000..db592342d6b0f --- /dev/null +++ b/python/paddle/distributed/auto_parallel/static/operators/dist_expand_as.py @@ -0,0 +1,80 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +from ..completion import get_phi_spmd_rule +from ..utils import get_dist_tensor_spec +from .common import ( + DistributedOperatorImplContainer, + get_default_distributed_operator_impl, + register_distributed_operator_impl_container, + update_op_dims_mapping, +) + + +class DistributedExpandAs(DistributedOperatorImplContainer): + def __init__(self, op_type): + super().__init__(op_type) + + @staticmethod + def update_dims_mapping(dist_op): + # step1: prepare inputs need for rule (order args as PHI definition and filter out unnecessary args) + op_desc = dist_op.serial_op.desc + + input_arg_names = op_desc.input_arg_names() + output_arg_names = op_desc.output_arg_names() + target_shape = op_desc.attr('target_shape') + + input_specs = [] + for name in input_arg_names: + input_specs.append(get_dist_tensor_spec(dist_op, name)) + + assert len(input_specs) == 2 + + output_spec = get_dist_tensor_spec(dist_op, output_arg_names[0], False) + + # step2: infer spmd + rule = get_phi_spmd_rule("expand_as") + # tensor order following order in PHI definition + fw_results = rule.infer_forward( + input_specs[0], input_specs[1], target_shape + ) + bw_results = rule.infer_backward( + input_specs[0], input_specs[1], output_spec, target_shape + ) + + # step3: update dist_attr + # tensor order following order in PHI definition + changed = update_op_dims_mapping( + dist_op, + input_arg_names, + output_arg_names, + fw_results, + bw_results, + ) + + return changed + + @staticmethod + def mapping_to_dist_operator_impl(dist_op, original_op_dist_attr): + op_dist_attr = dist_op.dist_attr + default_impl = get_default_distributed_operator_impl() + op_dist_attr.impl_type = default_impl.type + op_dist_attr.impl_idx = default_impl.idx + + return False + + +register_distributed_operator_impl_container( + DistributedExpandAs("expand_as_v2") +) diff --git a/test/cpp/auto_parallel/CMakeLists.txt b/test/cpp/auto_parallel/CMakeLists.txt index 2985dffa7da18..2db1baa4da642 100644 --- a/test/cpp/auto_parallel/CMakeLists.txt +++ b/test/cpp/auto_parallel/CMakeLists.txt @@ -29,6 +29,9 @@ if(WITH_DISTRIBUTE) paddle_test(cross_entropy_softmax_spmd_rule_test SRCS cross_entropy_softmax_spmd_rule_test.cc DEPS spmd_rule_test_util) + paddle_test(expand_as_spmd_rule_test SRCS expand_as_spmd_rule_test.cc DEPS + spmd_rule_test_util phi) + paddle_test(custom_op_spmd_rule_test SRCS custom_op_spmd_rule_test.cc DEPS spmd_rule_test_util phi) diff --git a/test/cpp/auto_parallel/expand_as_spmd_rule_test.cc b/test/cpp/auto_parallel/expand_as_spmd_rule_test.cc new file mode 100644 index 0000000000000..ca9daa84f99fd --- /dev/null +++ b/test/cpp/auto_parallel/expand_as_spmd_rule_test.cc @@ -0,0 +1,95 @@ +/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "test/cpp/auto_parallel/spmd_rule_test_util.h" + +namespace paddle { +namespace distributed { +namespace auto_parallel { + +TEST(ExpandAsInferSpmd, Ctor) { + // Sharding along axes besides softmax axis. + std::vector x_shape = {1, 48}; + std::vector y_shape = {2, 32, 48}; + + std::vector mesh_shape = {2, 3}; + std::vector process_ids = {0, 1, 2, 3, 4, 5}; + std::vector dim_names = {"x", "y"}; + ProcessMesh process_mesh(mesh_shape, process_ids, dim_names); + + TensorDistAttr x_dist_attr = TensorDistAttr(); + x_dist_attr.set_process_mesh(process_mesh); + x_dist_attr.set_dims_mapping(std::vector({-1, -1})); + x_dist_attr.set_dynamic_dims(std::vector({false, false})); + + TensorDistAttr y_dist_attr = TensorDistAttr(); + y_dist_attr.set_process_mesh(process_mesh); + y_dist_attr.set_dims_mapping(std::vector({0, 1, -1})); + y_dist_attr.set_dynamic_dims(std::vector({false, false, false})); + + phi::distributed::DistMetaTensor x(phi::make_ddim(x_shape), x_dist_attr); + phi::distributed::DistMetaTensor y(phi::make_ddim(y_shape), y_dist_attr); + + // test info forward + auto spmdinfo = ExpandAsInferSpmd(x, y, y_shape); + EXPECT_EQ(spmdinfo.first.size(), 2UL); + EXPECT_EQ(spmdinfo.second.size(), 1UL); + + EXPECT_EQ(get_dims_mapping(spmdinfo.first[0]), + std::vector({-1, -1})); + EXPECT_EQ(get_dims_mapping(spmdinfo.first[1]), + std::vector({0, 1, -1})); + EXPECT_EQ(get_dims_mapping(spmdinfo.second[0]), + std::vector({0, 1, -1})); + EXPECT_DOUBLE_EQ( + PADDLE_GET_CONST(TensorDistAttr, spmdinfo.second[0]).is_partial(), false); + VLOG(4) << "Test ExpandAsInferSpmd" << std::endl << std::endl << std::endl; + + // test info reverse + spmdinfo = ExpandAsInferSpmdReverse(x, y, y, y_shape); + EXPECT_EQ(spmdinfo.first.size(), 2UL); + EXPECT_EQ(spmdinfo.second.size(), 1UL); + + EXPECT_EQ(get_dims_mapping(spmdinfo.first[0]), + std::vector({-1, -1})); + EXPECT_EQ(get_dims_mapping(spmdinfo.first[1]), + std::vector({0, 1, -1})); + EXPECT_EQ(get_dims_mapping(spmdinfo.second[0]), + std::vector({0, 1, -1})); + EXPECT_DOUBLE_EQ( + PADDLE_GET_CONST(TensorDistAttr, spmdinfo.second[0]).is_partial(), false); + VLOG(4) << "Test ExpandAsInferSpmdReverse" << std::endl + << std::endl + << std::endl; + + // test info grad + spmdinfo = ExpandAsGradInferSpmd(x, y, y_shape); + EXPECT_EQ(spmdinfo.first.size(), 2UL); + EXPECT_EQ(spmdinfo.second.size(), 1UL); + + EXPECT_EQ(get_dims_mapping(spmdinfo.first[0]), + std::vector({-1, -1})); + EXPECT_EQ(get_dims_mapping(spmdinfo.first[1]), + std::vector({0, 1, -1})); + EXPECT_EQ(get_dims_mapping(spmdinfo.second[0]), + std::vector({-1, -1})); + check_partial_dims(spmdinfo.second[0], {0, 1}); + VLOG(4) << "Test ExpandAsGradInferSpmd" << std::endl + << std::endl + << std::endl; +} + +} // namespace auto_parallel +} // namespace distributed +} // namespace paddle From 102c515fb5dd3743e117e64b2a62a60dcc744539 Mon Sep 17 00:00:00 2001 From: Baizhou Zhang Date: Thu, 29 Feb 2024 21:51:42 +0800 Subject: [PATCH 185/282] [Dy2St] Delete legacy class TracedLayer and its related unit tests (#62227) --- python/paddle/jit/api.py | 412 +----------------- ...imperative_trace_non_persistable_inputs.py | 101 ----- .../legacy_test/test_op_function_generator.py | 8 - test/legacy_test/test_traced_layer_err_msg.py | 272 ------------ 4 files changed, 1 insertion(+), 792 deletions(-) delete mode 100644 test/legacy_test/test_imperative_trace_non_persistable_inputs.py delete mode 100644 test/legacy_test/test_traced_layer_err_msg.py diff --git a/python/paddle/jit/api.py b/python/paddle/jit/api.py index fbc562d881a20..f81cb801d14bc 100644 --- a/python/paddle/jit/api.py +++ b/python/paddle/jit/api.py @@ -30,28 +30,20 @@ from paddle.base import core, dygraph from paddle.base.compiler import ( BuildStrategy, - CompiledProgram, - ExecutionStrategy, ) -from paddle.base.data_feeder import check_type from paddle.base.dygraph.base import ( - program_desc_tracing_guard, switch_to_static_graph, ) from paddle.base.executor import Executor, scope_guard from paddle.base.framework import ( - Block, EagerParamBase, Parameter, - Program, Variable, _current_expected_place, - _dygraph_guard, - _dygraph_tracer, dygraph_only, ) from paddle.base.wrapped_decorator import wrap_decorator -from paddle.framework import in_dynamic_mode, use_pir_api +from paddle.framework import use_pir_api from paddle.nn import Layer from paddle.static.io import save_inference_model from paddle.utils.environments import ( @@ -85,34 +77,6 @@ def sot_mode_guard(value: bool): yield -def create_program_from_desc(program_desc): - program = Program() - program.desc = program_desc - program.blocks = [Block(program, 0)] - program._sync_with_cpp() - return program - - -def _extract_vars(inputs, result_list, err_tag='inputs'): - if isinstance(inputs, Variable): - result_list.append(inputs) - elif isinstance(inputs, (list, tuple)): - for var in inputs: - _extract_vars(var, result_list, err_tag) - else: - raise TypeError( - "The type of 'each element of {}' in paddle.jit.api.TracedLayer.trace must be base.Variable, but received {}.".format( - err_tag, type(inputs) - ) - ) - - -def extract_vars(inputs, err_tag='inputs'): - result_list = [] - _extract_vars(inputs, result_list, err_tag) - return result_list - - def copy_decorator_attrs(original_func, decorated_obj): """ Copies some necessary attributes from original function into decorated function. @@ -1524,380 +1488,6 @@ def load(path, **configs): return TranslatedLayer._construct(model_path, config) -@dygraph_only -def _trace( - layer, inputs, feed_prefix='feed_', fetch_prefix='fetch_', tmp_prefix='t_' -): - assert isinstance(layer, Layer) - - if not isinstance(inputs, (list, tuple)): - inputs = [inputs] - - tracer = _dygraph_tracer()._get_program_desc_tracer() - - var_list = extract_vars(inputs) - - with program_desc_tracing_guard(True): - original_outputs = layer(*inputs) - if not isinstance(original_outputs, (list, tuple)): - outputs = [original_outputs] - else: - outputs = original_outputs - out_vars = extract_vars(outputs, err_tag='outputs') - - ( - program_desc, - feed_names, - fetch_names, - parameters, - ) = tracer.create_program_desc( - var_list, feed_prefix, out_vars, fetch_prefix, tmp_prefix - ) - tracer.reset() - - with _dygraph_guard(None): - program = create_program_from_desc(program_desc) - - return original_outputs, program, feed_names, fetch_names, parameters - - -class TracedLayer: - """ - :api_attr: imperative - - TracedLayer is used to convert a forward dygraph model to a static - graph model. This is mainly used to save the dygraph model for online - inference using C++. Besides, users can also do inference in Python - using the converted static graph model, which usually has better - performance than the original dygraph model. - - TracedLayer would run the static graph model using :code:`Executor` - and :code:`CompiledProgram` . The static graph model would share - parameters with the dygraph model. - - All TracedLayer objects should not be created by constructor and should - be created by static method :code:`TracedLayer.trace(layer, inputs)` . - - The TracedLayer can only be used to convert the data-independent dygraph - model into the static graph model, which means the dygraph model should - be independent with the tensor data and shape. - """ - - def __init__(self, program, parameters, feed_names, fetch_names): - self._program = program - self._feed_names = feed_names - self._fetch_names = fetch_names - self._params = parameters - - self._place = _current_expected_place() - - self._scope = core.Scope() - for p in parameters: - src_tensor = p.value().get_tensor() - dst_tensor = self._scope.var(p.name).get_tensor() - dst_tensor._share_data_with(src_tensor) - - self._exe = Executor(self._place) - self._compiled_program = None - self._build_strategy = None - self._exec_strategy = None - - @property - def program(self): - return self._program - - def _switch(self, is_test=True): - for block_id in range(self._program.num_blocks): - block = self._program.block(block_id) - for op in block.ops: - if op.has_attr("is_test"): - op._set_attr("is_test", is_test) - - @staticmethod - @dygraph_only - def trace(layer, inputs): - """ - This method is the only allowed method to create TracedLayer object. - It would call the :code:`layer(*inputs)` method to run the dygraph - model and convert it into a static graph model. - - Args: - layer (paddle.nn.Layer): the layer object to be traced. - inputs (list(Tensor)|tuple(Tensor)|Tensor): the input tensors of - the layer object. - - Returns: - tuple: A tuple of 2 items, whose the first item is the output of - :code:`layer(*inputs)` , and the second item is the created - TracedLayer object. - - Examples: - .. code-block:: python - - >>> import paddle - - >>> class ExampleLayer(paddle.nn.Layer): - ... def __init__(self): - ... super().__init__() - ... self._fc = paddle.nn.Linear(3, 10) - ... - ... def forward(self, input): - ... return self._fc(input) - - - >>> layer = ExampleLayer() - >>> in_var = paddle.uniform(shape=[2, 3], dtype='float32') - >>> out_dygraph, static_layer = paddle.jit.api.TracedLayer.trace(layer, inputs=[in_var]) - - >>> # run the static graph model using Executor inside - >>> out_static_graph = static_layer([in_var]) - - >>> print(len(out_static_graph)) # 1 - >>> print(out_static_graph[0].shape) # (2, 10) - - >>> # save the static graph model for inference - >>> static_layer.save_inference_model('./saved_infer_model') - - """ - assert isinstance( - layer, Layer - ), "The type of 'layer' in paddle.jit.api.TracedLayer.trace must be paddle.nn.Layer, but received {}.".format( - type(layer) - ) - outs, prog, feed, fetch, parameters = _trace(layer, inputs) - traced = TracedLayer(prog, parameters, feed, fetch) - return outs, traced - - def set_strategy(self, build_strategy=None, exec_strategy=None): - """ - Set the strategies when running static graph model. - - Args: - build_strategy (BuildStrategy, optional): build strategy of - :code:`CompiledProgram` inside TracedLayer. Default None. - exec_strategy (ExecutionStrategy, optional): execution strategy of - :code:`CompiledProgram` inside TracedLayer. Default None. - - Returns: - None - - Examples: - .. code-block:: python - - >>> import paddle - - >>> class ExampleLayer(paddle.nn.Layer): - ... def __init__(self): - ... super().__init__() - ... self._fc = paddle.nn.Linear(3, 10) - ... - ... def forward(self, input): - ... return self._fc(input) - - >>> layer = ExampleLayer() - >>> in_var = paddle.uniform(shape=[2, 3], dtype='float32') - - >>> out_dygraph, static_layer = paddle.jit.api.TracedLayer.trace(layer, inputs=[in_var]) - - >>> build_strategy = paddle.static.BuildStrategy() - >>> build_strategy.enable_inplace = True - - >>> exec_strategy = paddle.static.ExecutionStrategy() - >>> exec_strategy.num_threads = 2 - - >>> static_layer.set_strategy(build_strategy=build_strategy, exec_strategy=exec_strategy) - >>> out_static_graph = static_layer([in_var]) - - """ - assert self._compiled_program is None, "Cannot set strategy after run" - assert isinstance( - build_strategy, (type(None), BuildStrategy) - ), "The type of 'build_strategy' in paddle.jit.api.TracedLayer.set_strategy must be base.BuildStrategy, but received {}.".format( - type(build_strategy) - ) - assert isinstance( - exec_strategy, (type(None), ExecutionStrategy) - ), "The type of 'exec_strategy' in paddle.jit.api.TracedLayer.set_strategy must be base.ExecutionStrategy, but received {}.".format( - type(exec_strategy) - ) - self._build_strategy = build_strategy - self._exec_strategy = exec_strategy - - @switch_to_static_graph - def _compile(self): - self._compiled_program = CompiledProgram( - self._program, - build_strategy=self._build_strategy, - ) - - def _build_feed(self, inputs): - assert isinstance( - inputs, (list, tuple) - ), "Inputs should be a list or tuple of variables" - assert len(inputs) == len(self._feed_names) - feed_dict = {} - if in_dynamic_mode(): - for x, name in zip(inputs, self._feed_names): - feed_dict[name] = x.value().get_tensor() - else: - for x, name in zip(inputs, self._feed_names): - feed_dict[name] = x - - return feed_dict - - @switch_to_static_graph - def _run(self, feed): - return self._exe.run( - self._compiled_program, feed=feed, fetch_list=self._fetch_names - ) - - def __call__(self, inputs): - with scope_guard(self._scope): - if self._compiled_program is None: - self._compile() - - return self._run(self._build_feed(inputs)) - - @switch_to_static_graph - def save_inference_model(self, path, feed=None, fetch=None, **kwargs): - """ - Save the TracedLayer to a model for inference. The saved - inference model can be loaded by C++ inference APIs. - - ``path`` is the prefix of saved objects, and the saved translated program file - suffix is ``.pdmodel`` , the saved persistable variables file suffix is ``.pdiparams`` . - - Args: - path(str): The path prefix to save model. The format is ``dirname/file_prefix`` or ``file_prefix``. - feed (list[int], optional): the input variable indices of the saved - inference model. If None, all input variables of the - TracedLayer object would be the inputs of the saved inference - model. Default None. - fetch (list[int], optional): the output variable indices of the - saved inference model. If None, all output variables of the - TracedLayer object would be the outputs of the saved inference - model. Default None. - kwargs: Supported keys including - - clip_extra(bool): whether to clip extra information for every operator. Defaults to True. - - legacy_format(bool): whether to save program in legacy format. Default to False. - - Returns: - None - - Examples: - .. code-block:: python - - >>> import numpy as np - >>> import paddle - - >>> class ExampleLayer(paddle.nn.Layer): - ... def __init__(self): - ... super().__init__() - ... self._fc = paddle.nn.Linear(3, 10) - ... - ... def forward(self, input): - ... return self._fc(input) - - >>> save_dirname = './saved_infer_model' - >>> in_np = np.random.random([2, 3]).astype('float32') - >>> in_var = paddle.to_tensor(in_np) - >>> layer = ExampleLayer() - - >>> out_dygraph, static_layer = paddle.jit.api.TracedLayer.trace(layer, inputs=[in_var]) - >>> static_layer.save_inference_model(save_dirname, feed=[0], fetch=[0]) - - >>> paddle.enable_static() - >>> place = paddle.CPUPlace() - >>> exe = paddle.static.Executor(place) - >>> program, feed_vars, fetch_vars = paddle.static.load_inference_model( - ... save_dirname, - ... exe - ... ) - - >>> fetch, = exe.run(program, feed={feed_vars[0]: in_np}, fetch_list=fetch_vars) - >>> print(fetch.shape) - [2, 10] - """ - check_type( - path, - "path", - str, - "paddle.jit.api.TracedLayer.save_inference_model", - ) - check_type( - feed, - "feed", - (type(None), list), - "paddle.jit.api.TracedLayer.save_inference_model", - ) - if isinstance(feed, list): - for f in feed: - check_type( - f, - "each element of feed", - int, - "paddle.jit.api.TracedLayer.save_inference_model", - ) - check_type( - fetch, - "fetch", - (type(None), list), - "paddle.jit.api.TracedLayer.save_inference_model", - ) - if isinstance(fetch, list): - for f in fetch: - check_type( - f, - "each element of fetch", - int, - "paddle.jit.api.TracedLayer.save_inference_model", - ) - clip_extra = kwargs.get('clip_extra', True) - # path check - file_prefix = os.path.basename(path) - if file_prefix == "": - raise ValueError( - "The input path MUST be format of dirname/file_prefix " - "[dirname\\file_prefix in Windows system], but received " - "file_prefix is empty string." - ) - - dirname = os.path.dirname(path) - if dirname and not os.path.exists(dirname): - os.makedirs(dirname) - - def get_feed_fetch(all_vars, partial_vars): - if partial_vars is None: - return all_vars - - return [all_vars[idx] for idx in partial_vars] - - with scope_guard(self._scope): - feeded_var_names = get_feed_fetch(self._feed_names, feed) - target_var_names = get_feed_fetch(self._fetch_names, fetch) - feed_vars = [] - for name in feeded_var_names: - feed_var = self._program.global_block().vars.get(name, None) - assert feed_var is not None, f"{name} cannot be found" - feed_vars.append(feed_var) - target_vars = [] - for name in target_var_names: - target_var = self._program.global_block().vars.get(name, None) - assert target_var is not None, f"{name} cannot be found" - target_vars.append(target_var) - legacy_format = kwargs.get('legacy_format', False) - file_prefix = os.path.join(dirname, file_prefix) - save_inference_model( - path_prefix=file_prefix, - feed_vars=feed_vars, - fetch_vars=target_vars, - executor=self._exe, - program=self._program.clone(), - clip_extra=clip_extra, - legacy_format=legacy_format, - ) - - def set_dynamic_shape(variable, shape_list): if paddle.base.dygraph.base.in_to_static_mode(): if isinstance(variable, paddle.base.framework.Variable): diff --git a/test/legacy_test/test_imperative_trace_non_persistable_inputs.py b/test/legacy_test/test_imperative_trace_non_persistable_inputs.py deleted file mode 100644 index 5238e37df5a5a..0000000000000 --- a/test/legacy_test/test_imperative_trace_non_persistable_inputs.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import unittest - -import numpy as np - -import paddle -from paddle import base - - -class SimpleFCLayer(paddle.nn.Layer): - def __init__(self, feature_size, batch_size, fc_size): - super().__init__() - self._linear = paddle.nn.Linear(feature_size, fc_size) - self._offset = paddle.to_tensor( - np.random.random((batch_size, fc_size)).astype('float32') - ) - - def forward(self, x): - fc = self._linear(x) - return fc + self._offset - - -class TestTracedLayerRecordNonPersistableInput(unittest.TestCase): - def test_main(self): - if base.framework.in_dygraph_mode(): - return - traced_layer = None - with base.dygraph.guard(): - feature_size = 3 - batch_size = 4 - fc_size = 2 - layer = SimpleFCLayer(feature_size, batch_size, fc_size) - optimizer = paddle.optimizer.SGD( - learning_rate=1e-3, parameters=layer.parameters() - ) - - expected_persistable_vars = { - layer._linear.weight.name, - layer._linear.bias.name, - layer._offset.name, - } - - for _ in range(10): - in_x = paddle.to_tensor( - np.random.random((batch_size, feature_size)).astype( - 'float32' - ) - ) - if traced_layer is None: - dygraph_out, traced_layer = base.dygraph.TracedLayer.trace( - layer, [in_x] - ) - else: - dygraph_out = layer(in_x) - dygraph_out_numpy = dygraph_out.numpy() - static_out = traced_layer([in_x])[0] - np.testing.assert_array_equal(dygraph_out_numpy, static_out) - - loss = paddle.mean(dygraph_out) - loss.backward() - - optimizer.minimize(loss) - - del layer - - program = traced_layer.program - actual_persistable_vars = set() - for var in program.list_vars(): - if var.persistable: - actual_persistable_vars.add(var.name) - - self.assertEqual(actual_persistable_vars, expected_persistable_vars) - - traced_layer.save_inference_model( - path='./traced_layer_test_non_persistable_vars' - ) - self.assertTrue( - 'traced_layer_test_non_persistable_vars.pdmodel' in os.listdir('./') - ) - self.assertTrue( - 'traced_layer_test_non_persistable_vars.pdiparams' - in os.listdir('./') - ) - - -if __name__ == '__main__': - unittest.main() diff --git a/test/legacy_test/test_op_function_generator.py b/test/legacy_test/test_op_function_generator.py index c37dd56c6a98a..d34d0aff45edd 100644 --- a/test/legacy_test/test_op_function_generator.py +++ b/test/legacy_test/test_op_function_generator.py @@ -21,14 +21,6 @@ from paddle import _legacy_C_ops, base -class TestTracedLayer(paddle.nn.Layer): - def __init__(self, name_scope): - super().__init__(name_scope) - - def forward(self, input): - return _legacy_C_ops.relu(input) - - class TestVariable(unittest.TestCase): def setUp(self): self.shape = [512, 768] diff --git a/test/legacy_test/test_traced_layer_err_msg.py b/test/legacy_test/test_traced_layer_err_msg.py deleted file mode 100644 index 4927fdea82a54..0000000000000 --- a/test/legacy_test/test_traced_layer_err_msg.py +++ /dev/null @@ -1,272 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import tempfile -import unittest - -import numpy as np - -import paddle -from paddle import base, nn - - -class SimpleFCLayer(nn.Layer): - def __init__(self, feature_size, batch_size, fc_size): - super().__init__() - self._linear = nn.Linear(feature_size, fc_size) - self._offset = paddle.to_tensor( - np.random.random((batch_size, fc_size)).astype('float32') - ) - - def forward(self, x): - fc = self._linear(x) - return fc + self._offset - - -class LinearNetWithNone(nn.Layer): - def __init__(self, feature_size, fc_size): - super().__init__() - self._linear = nn.Linear(feature_size, fc_size) - - def forward(self, x): - fc = self._linear(x) - - return [fc, [None, 2]] - - -class TestTracedLayerErrMsg(unittest.TestCase): - def setUp(self): - self.batch_size = 4 - self.feature_size = 3 - self.fc_size = 2 - self.layer = self._train_simple_net() - self.type_str = 'class' - self.temp_dir = tempfile.TemporaryDirectory() - - def tearDown(self): - self.temp_dir.cleanup() - - def test_trace_err(self): - if base.framework.in_dygraph_mode(): - return - with base.dygraph.guard(): - in_x = paddle.to_tensor( - np.random.random((self.batch_size, self.feature_size)).astype( - 'float32' - ) - ) - - with self.assertRaises(AssertionError) as e: - dygraph_out, traced_layer = base.dygraph.TracedLayer.trace( - None, [in_x] - ) - self.assertEqual( - "The type of 'layer' in paddle.jit.TracedLayer.trace must be paddle.nn.Layer, but received <{} 'NoneType'>.".format( - self.type_str - ), - str(e.exception), - ) - with self.assertRaises(TypeError) as e: - dygraph_out, traced_layer = base.dygraph.TracedLayer.trace( - self.layer, 3 - ) - self.assertEqual( - "The type of 'each element of inputs' in paddle.jit.TracedLayer.trace must be base.Variable, but received <{} 'int'>.".format( - self.type_str - ), - str(e.exception), - ) - with self.assertRaises(TypeError) as e: - dygraph_out, traced_layer = base.dygraph.TracedLayer.trace( - self.layer, [True, 1] - ) - self.assertEqual( - "The type of 'each element of inputs' in paddle.jit.TracedLayer.trace must be base.Variable, but received <{} 'bool'>.".format( - self.type_str - ), - str(e.exception), - ) - - dygraph_out, traced_layer = base.dygraph.TracedLayer.trace( - self.layer, [in_x] - ) - - def test_set_strategy_err(self): - if base.framework.in_dygraph_mode(): - return - with base.dygraph.guard(): - in_x = paddle.to_tensor( - np.random.random((self.batch_size, self.feature_size)).astype( - 'float32' - ) - ) - dygraph_out, traced_layer = base.dygraph.TracedLayer.trace( - self.layer, [in_x] - ) - - with self.assertRaises(AssertionError) as e: - traced_layer.set_strategy(1, base.ExecutionStrategy()) - self.assertEqual( - "The type of 'build_strategy' in paddle.jit.TracedLayer.set_strategy must be base.BuildStrategy, but received <{} 'int'>.".format( - self.type_str - ), - str(e.exception), - ) - - with self.assertRaises(AssertionError) as e: - traced_layer.set_strategy(base.BuildStrategy(), False) - self.assertEqual( - "The type of 'exec_strategy' in paddle.jit.TracedLayer.set_strategy must be base.ExecutionStrategy, but received <{} 'bool'>.".format( - self.type_str - ), - str(e.exception), - ) - - traced_layer.set_strategy(build_strategy=base.BuildStrategy()) - traced_layer.set_strategy(exec_strategy=base.ExecutionStrategy()) - traced_layer.set_strategy( - base.BuildStrategy(), base.ExecutionStrategy() - ) - - def test_save_inference_model_err(self): - if base.framework.in_dygraph_mode(): - return - with base.dygraph.guard(): - in_x = paddle.to_tensor( - np.random.random((self.batch_size, self.feature_size)).astype( - 'float32' - ) - ) - dygraph_out, traced_layer = base.dygraph.TracedLayer.trace( - self.layer, [in_x] - ) - - path = os.path.join(self.temp_dir.name, './traced_layer_err_msg') - with self.assertRaises(TypeError) as e: - traced_layer.save_inference_model([0]) - self.assertEqual( - "The type of 'path' in paddle.jit.TracedLayer.save_inference_model must be <{} 'str'>, but received <{} 'list'>. ".format( - self.type_str, self.type_str - ), - str(e.exception), - ) - with self.assertRaises(TypeError) as e: - traced_layer.save_inference_model(path, [0], [None]) - self.assertEqual( - "The type of 'each element of fetch' in paddle.jit.TracedLayer.save_inference_model must be <{} 'int'>, but received <{} 'NoneType'>. ".format( - self.type_str, self.type_str - ), - str(e.exception), - ) - with self.assertRaises(TypeError) as e: - traced_layer.save_inference_model(path, [0], False) - self.assertEqual( - "The type of 'fetch' in paddle.jit.TracedLayer.save_inference_model must be (<{} 'NoneType'>, <{} 'list'>), but received <{} 'bool'>. ".format( - self.type_str, self.type_str, self.type_str - ), - str(e.exception), - ) - with self.assertRaises(TypeError) as e: - traced_layer.save_inference_model(path, [None], [0]) - self.assertEqual( - "The type of 'each element of feed' in paddle.jit.TracedLayer.save_inference_model must be <{} 'int'>, but received <{} 'NoneType'>. ".format( - self.type_str, self.type_str - ), - str(e.exception), - ) - with self.assertRaises(TypeError) as e: - traced_layer.save_inference_model(path, True, [0]) - self.assertEqual( - "The type of 'feed' in paddle.jit.TracedLayer.save_inference_model must be (<{} 'NoneType'>, <{} 'list'>), but received <{} 'bool'>. ".format( - self.type_str, self.type_str, self.type_str - ), - str(e.exception), - ) - with self.assertRaises(ValueError) as e: - traced_layer.save_inference_model("") - self.assertEqual( - "The input path MUST be format of dirname/file_prefix [dirname\\file_prefix in Windows system], " - "but received file_prefix is empty string.", - str(e.exception), - ) - - traced_layer.save_inference_model(path) - - def _train_simple_net(self): - layer = None - with base.dygraph.guard(): - layer = SimpleFCLayer( - self.feature_size, self.batch_size, self.fc_size - ) - optimizer = paddle.optimizer.SGD( - learning_rate=1e-3, parameters=layer.parameters() - ) - - for i in range(5): - in_x = paddle.to_tensor( - np.random.random( - (self.batch_size, self.feature_size) - ).astype('float32') - ) - dygraph_out = layer(in_x) - loss = paddle.mean(dygraph_out) - loss.backward() - optimizer.minimize(loss) - return layer - - -class TestOutVarWithNoneErrMsg(unittest.TestCase): - def test_linear_net_with_none(self): - if base.framework.in_dygraph_mode(): - return - model = LinearNetWithNone(100, 16) - in_x = paddle.to_tensor(np.random.random((4, 100)).astype('float32')) - with self.assertRaises(TypeError): - dygraph_out, traced_layer = base.dygraph.TracedLayer.trace( - model, [in_x] - ) - - -class TestTracedLayerSaveInferenceModel(unittest.TestCase): - """test save_inference_model will automatically create non-exist dir""" - - def setUp(self): - self.temp_dir = tempfile.TemporaryDirectory() - self.save_path = os.path.join(self.temp_dir.name, "./nonexist_dir/fc") - import shutil - - if os.path.exists(os.path.dirname(self.save_path)): - shutil.rmtree(os.path.dirname(self.save_path)) - - def tearDown(self): - self.temp_dir.cleanup() - - def test_mkdir_when_input_path_non_exist(self): - if base.framework.in_dygraph_mode(): - return - fc_layer = SimpleFCLayer(3, 4, 2) - input_var = paddle.to_tensor(np.random.random([4, 3]).astype('float32')) - with base.dygraph.guard(): - dygraph_out, traced_layer = base.dygraph.TracedLayer.trace( - fc_layer, inputs=[input_var] - ) - self.assertFalse(os.path.exists(os.path.dirname(self.save_path))) - traced_layer.save_inference_model(self.save_path) - self.assertTrue(os.path.exists(os.path.dirname(self.save_path))) - - -if __name__ == '__main__': - unittest.main() From c6be4727b1747f204455b919a77ac3ac9e8ec880 Mon Sep 17 00:00:00 2001 From: Yuanle Liu Date: Thu, 29 Feb 2024 22:44:16 +0800 Subject: [PATCH 186/282] [PIR] Fix dce pass for not eliminated completely (#62242) --- paddle/fluid/pir/transforms/dead_code_elimination_pass.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/pir/transforms/dead_code_elimination_pass.cc b/paddle/fluid/pir/transforms/dead_code_elimination_pass.cc index 442aec918e08f..d802a470e86f1 100644 --- a/paddle/fluid/pir/transforms/dead_code_elimination_pass.cc +++ b/paddle/fluid/pir/transforms/dead_code_elimination_pass.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "paddle/fluid/pir/transforms/dead_code_elimination_pass.h" +#include #include "paddle/fluid/pir/dialect/operator/ir/op_dialect.h" #include "paddle/fluid/pir/dialect/operator/ir/pd_op.h" @@ -31,7 +32,12 @@ class DeadCodeEliminationPass : public pir::Pass { void Run(pir::Operation* op) override { VLOG(6) << "apply dead_code_elimination_pass"; int64_t num_erasers{0}; - EraseOp(*op->GetParentProgram()->block(), &num_erasers); + bool updated{true}; + while (updated) { + int64_t pre_num_erasers = num_erasers; + EraseOp(*op->GetParentProgram()->block(), &num_erasers); + updated = pre_num_erasers != num_erasers; + } AddStatistics(num_erasers); } From 4e0779cbfc025e0b46068e291bbcee42371dd771 Mon Sep 17 00:00:00 2001 From: co63oc Date: Thu, 29 Feb 2024 23:24:07 +0800 Subject: [PATCH 187/282] Fix CPUAPlace CPUPlace, etc (#62214) --- paddle/fluid/platform/collective_helper.cc | 4 ++-- paddle/fluid/platform/device_event_base.cc | 6 ++--- paddle/fluid/platform/device_event_cpu.h | 2 +- paddle/fluid/platform/device_event_test.cc | 4 ++-- .../platform/profiler/chrometracing_logger.cc | 2 +- .../platform/profiler/chrometracing_logger.h | 2 +- .../profiler/dump/deserialization_reader.cc | 12 +++++----- .../profiler/dump/serialization_logger.h | 2 +- .../fluid/platform/profiler/event_tracing.h | 2 +- paddle/fluid/platform/profiler/profiler.cc | 24 +++++++++---------- paddle/fluid/platform/profiler/utils.cc | 8 +++---- paddle/fluid/platform/profiler_helper.h | 2 +- 12 files changed, 35 insertions(+), 35 deletions(-) diff --git a/paddle/fluid/platform/collective_helper.cc b/paddle/fluid/platform/collective_helper.cc index 4ffcf53b1a574..3444f71639b46 100644 --- a/paddle/fluid/platform/collective_helper.cc +++ b/paddle/fluid/platform/collective_helper.cc @@ -183,7 +183,7 @@ void NCCLCommContext::CreateNCCLCommMultiTrainer( VLOG(1) << "ncclCommInitRank: " << i; } PADDLE_ENFORCE_GPU_SUCCESS(dynload::ncclGroupEnd()); - VLOG(1) << "nccl group end seccessss"; + VLOG(1) << "nccl group end success"; } PADDLE_ENFORCE_EQ(comm_map_.count(ring_id), 0, @@ -261,7 +261,7 @@ NCCLComm* NCCLCommContext::AssignNCCLComm( platform::CUDAPlace(dev_id))); dev_ctx->set_nccl_comm(comm); } - VLOG(4) << "add mccl comm: " << comm_map_[ring_id][dev_id].get() + VLOG(4) << "add nccl comm: " << comm_map_[ring_id][dev_id].get() << ", ring_id:" << ring_id << ", dev_id:" << dev_id; return comm_map_[ring_id][dev_id].get(); } diff --git a/paddle/fluid/platform/device_event_base.cc b/paddle/fluid/platform/device_event_base.cc index cd2d31f1fbefb..6079691fe873c 100644 --- a/paddle/fluid/platform/device_event_base.cc +++ b/paddle/fluid/platform/device_event_base.cc @@ -66,9 +66,9 @@ void DeviceEventRecordCPU(DeviceEvent* event, const DeviceContext* context) { auto* wrapper = static_cast(event->GetEvent().get()); std::unique_lock lock(wrapper->mutex_); - // NOTE: As for CudaEvent_t, it can be used to Record() repeatly. CudaEvent_t - // internally reset its status from finished into initialized. - // So we simulate the process here. + // NOTE: As for CudaEvent_t, it can be used to Record() repeatedly. + // CudaEvent_t internally reset its status from finished into initialized. So + // we simulate the process here. if (wrapper->status_.load() == EventStatus::SUCCESS) { VLOG(3) << "Found EventStatus is SUCCESS before RecordCPU. Reset it into " "INITIALIZED."; diff --git a/paddle/fluid/platform/device_event_cpu.h b/paddle/fluid/platform/device_event_cpu.h index 9490d5f3ceec8..e6faeb5fd01a4 100644 --- a/paddle/fluid/platform/device_event_cpu.h +++ b/paddle/fluid/platform/device_event_cpu.h @@ -30,7 +30,7 @@ struct CPUDeviceEventWrapper { platform::is_cpu_place(place), true, platform::errors::PreconditionNotMet( - "Required device shall be CPUAPlace, but received %d. ", place)); + "Required device shall be CPUPlace, but received %d. ", place)); } std::mutex mutex_; std::condition_variable cv_completed_; diff --git a/paddle/fluid/platform/device_event_test.cc b/paddle/fluid/platform/device_event_test.cc index b2e3d3242d219..4eb0da7740f3a 100644 --- a/paddle/fluid/platform/device_event_test.cc +++ b/paddle/fluid/platform/device_event_test.cc @@ -63,7 +63,7 @@ TEST(DeviceEvent, CUDA) { status = event.Query(); ASSERT_EQ(status, false); // async - event.Wait(kCPU, context); // step 3. EventSynchornize + event.Wait(kCPU, context); // step 3. EventSynchronize status = event.Query(); ASSERT_EQ(status, true); // sync @@ -114,7 +114,7 @@ TEST(DeviceEvent, CUDA) { status = event.Query(); ASSERT_EQ(status, false); // async - event.Wait(kCPU, context); // step 3. EventSynchornize + event.Wait(kCPU, context); // step 3. EventSynchronize status = event.Query(); ASSERT_EQ(status, true); // sync diff --git a/paddle/fluid/platform/profiler/chrometracing_logger.cc b/paddle/fluid/platform/profiler/chrometracing_logger.cc index de8fd01a1e59d..87fbe61979876 100644 --- a/paddle/fluid/platform/profiler/chrometracing_logger.cc +++ b/paddle/fluid/platform/profiler/chrometracing_logger.cc @@ -788,7 +788,7 @@ void ChromeTracingLogger::RefineDisplayName( "name": "process_name", "pid": %lld, "tid": %lld, "ph": "M", "args": { - "name": "Deivce %lld (%s)" + "name": "Device %lld (%s)" } }, { diff --git a/paddle/fluid/platform/profiler/chrometracing_logger.h b/paddle/fluid/platform/profiler/chrometracing_logger.h index 37323d1450bf2..89808bee842df 100644 --- a/paddle/fluid/platform/profiler/chrometracing_logger.h +++ b/paddle/fluid/platform/profiler/chrometracing_logger.h @@ -57,7 +57,7 @@ class ChromeTracingLogger : public BaseLogger { void RefineDisplayName(std::unordered_map); std::string filename_; std::ofstream output_file_stream_; - static const char* categary_name_[]; + static const char* category_name_[]; std::set> pid_tid_set_; std::set> deviceid_streamid_set_; uint64_t start_time_; diff --git a/paddle/fluid/platform/profiler/dump/deserialization_reader.cc b/paddle/fluid/platform/profiler/dump/deserialization_reader.cc index 329c9f6871461..f02496ed5d082 100644 --- a/paddle/fluid/platform/profiler/dump/deserialization_reader.cc +++ b/paddle/fluid/platform/profiler/dump/deserialization_reader.cc @@ -44,12 +44,12 @@ std::unique_ptr DeserializationReader::Parse() { return nullptr; } // restore extra info - ExtraInfo extrainfo; + ExtraInfo extra_info; for (auto indx = 0; indx < node_trees_proto_->extra_info_size(); indx++) { ExtraInfoMap extra_info_map = node_trees_proto_->extra_info(indx); - extrainfo.AddExtraInfo(extra_info_map.key(), - std::string("%s"), - extra_info_map.value().c_str()); + extra_info.AddExtraInfo(extra_info_map.key(), + std::string("%s"), + extra_info_map.value().c_str()); } // restore NodeTrees @@ -139,10 +139,10 @@ std::unique_ptr DeserializationReader::Parse() { RestoreDeviceProperty(device_property_proto); } ProfilerResult* profiler_result_ptr = - new ProfilerResult(std::move(tree), extrainfo, device_property_map); + new ProfilerResult(std::move(tree), extra_info, device_property_map); #else ProfilerResult* profiler_result_ptr = - new ProfilerResult(std::move(tree), extrainfo); + new ProfilerResult(std::move(tree), extra_info); #endif // restore version and span indx profiler_result_ptr->SetVersion(node_trees_proto_->version()); diff --git a/paddle/fluid/platform/profiler/dump/serialization_logger.h b/paddle/fluid/platform/profiler/dump/serialization_logger.h index 80d5413106ded..e61ed701cd798 100644 --- a/paddle/fluid/platform/profiler/dump/serialization_logger.h +++ b/paddle/fluid/platform/profiler/dump/serialization_logger.h @@ -21,7 +21,7 @@ limitations under the License. */ namespace paddle { namespace platform { -// Dump a NodeTrees into a profobuf file. +// Dump a NodeTrees into a protobuf file. // A SerializationLogger object can only dump a NodeTrees object, // creates a file in the constructor and closes the file in the destructor. // Should only call LogNodeTrees and LogMetaInfo. diff --git a/paddle/fluid/platform/profiler/event_tracing.h b/paddle/fluid/platform/profiler/event_tracing.h index 08890f1369733..b427a9ba55210 100644 --- a/paddle/fluid/platform/profiler/event_tracing.h +++ b/paddle/fluid/platform/profiler/event_tracing.h @@ -28,7 +28,7 @@ namespace platform { // Chrome Trace Viewer Format: Instant Event struct RecordInstantEvent { /** - * @param name: It is the caller's reponsibility to manage the underlying + * @param name: It is the caller's responsibility to manage the underlying * storage. RecordInstantEvent stores the pointer. * @param type: Classification which is used to instruct the profiling * data statistics. diff --git a/paddle/fluid/platform/profiler/profiler.cc b/paddle/fluid/platform/profiler/profiler.cc index bcb35f5b7bd35..c9d458b1d250a 100644 --- a/paddle/fluid/platform/profiler/profiler.cc +++ b/paddle/fluid/platform/profiler/profiler.cc @@ -148,19 +148,19 @@ std::unique_ptr Profiler::Stop() { collector.MemEvents(), collector.OperatorSupplementEvents())); cpu_utilization_.RecordEndTimeInfo(); - ExtraInfo extrainfo; - extrainfo.AddExtraInfo(std::string("System Cpu Utilization"), - std::string("%f"), - cpu_utilization_.GetCpuUtilization()); - extrainfo.AddExtraInfo(std::string("Process Cpu Utilization"), - std::string("%f"), - cpu_utilization_.GetCpuCurProcessUtilization()); + ExtraInfo extra_info; + extra_info.AddExtraInfo(std::string("System Cpu Utilization"), + std::string("%f"), + cpu_utilization_.GetCpuUtilization()); + extra_info.AddExtraInfo(std::string("Process Cpu Utilization"), + std::string("%f"), + cpu_utilization_.GetCpuCurProcessUtilization()); const std::unordered_map thread_names = collector.ThreadNames(); for (const auto& kv : thread_names) { - extrainfo.AddExtraInfo(string_format(std::string("%llu"), kv.first), - std::string("%s"), - kv.second.c_str()); + extra_info.AddExtraInfo(string_format(std::string("%llu"), kv.first), + std::string("%s"), + kv.second.c_str()); } #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) std::map device_property_map; @@ -170,10 +170,10 @@ std::unique_ptr Profiler::Stop() { device_property_map[device_id] = device_property; } ProfilerResult* profiler_result_ptr = new platform::ProfilerResult( - std::move(tree), extrainfo, device_property_map); + std::move(tree), extra_info, device_property_map); #else ProfilerResult* profiler_result_ptr = - new platform::ProfilerResult(std::move(tree), extrainfo); + new platform::ProfilerResult(std::move(tree), extra_info); #endif profiler_result_ptr->SetVersion(std::string(version)); profiler_result_ptr->SetSpanIndx(span_indx); diff --git a/paddle/fluid/platform/profiler/utils.cc b/paddle/fluid/platform/profiler/utils.cc index 46a94e7fcb23c..8c12f84416579 100644 --- a/paddle/fluid/platform/profiler/utils.cc +++ b/paddle/fluid/platform/profiler/utils.cc @@ -145,16 +145,16 @@ float CalculateEstOccupancy(uint32_t DeviceId, #endif // PADDLE_WITH_CUPTI const char* StringTracerMemEventType(TracerMemEventType type) { - static const char* categary_name_[] = {// NOLINT + static const char* category_name_[] = {// NOLINT "Allocate", "Free", "ReservedAllocate", "ReservedFree"}; - return categary_name_[static_cast(type)]; + return category_name_[static_cast(type)]; } const char* StringTracerEventType(TracerEventType type) { - static const char* categary_name_[] = {"Operator", // NOLINT + static const char* category_name_[] = {"Operator", // NOLINT "Dataloader", "ProfileStep", "CudaRuntime", @@ -169,7 +169,7 @@ const char* StringTracerEventType(TracerEventType type) { "Communication", "PythonOp", "PythonUserDefined"}; - return categary_name_[static_cast(type)]; + return category_name_[static_cast(type)]; } } // namespace platform diff --git a/paddle/fluid/platform/profiler_helper.h b/paddle/fluid/platform/profiler_helper.h index 8ce6fee8a5f6e..f79b801f1a095 100644 --- a/paddle/fluid/platform/profiler_helper.h +++ b/paddle/fluid/platform/profiler_helper.h @@ -740,7 +740,7 @@ void AnalyzeEvent( size_t *max_name_width, OverHead *overhead, bool merge_thread) { - // In oreder to deal with special event in main thread + // In order to deal with special event in main thread std::set main_thread_event_name; for (size_t i = 0; i < (*analyze_events).size(); i++) { for (size_t j = 0; j < (*analyze_events)[i].size(); j++) { From 7921a77a83c51b14fa3ca2a123fcb02b77fce683 Mon Sep 17 00:00:00 2001 From: co63oc Date: Thu, 29 Feb 2024 23:25:09 +0800 Subject: [PATCH 188/282] Fix precison_mode precision_mode, etc (#62212) --- .../transforms/auto_mixed_precision_pass.cc | 4 +-- .../fusion/conv2d_add_act_fuse_pass.cc | 4 +-- .../fused_linear_param_grad_add_pass.cc | 28 +++++++++---------- .../fusion/fused_weight_only_linear_pass.cc | 6 ++-- .../pir/transforms/sub_graph_detector.cc | 10 +++---- .../fluid/pir/transforms/sub_graph_detector.h | 2 +- 6 files changed, 27 insertions(+), 27 deletions(-) diff --git a/paddle/fluid/pir/transforms/auto_mixed_precision_pass.cc b/paddle/fluid/pir/transforms/auto_mixed_precision_pass.cc index 4f5c4c0e4cd6b..dee9aad09ed1d 100644 --- a/paddle/fluid/pir/transforms/auto_mixed_precision_pass.cc +++ b/paddle/fluid/pir/transforms/auto_mixed_precision_pass.cc @@ -66,7 +66,7 @@ class AutoMixedPrecisionPass : public pir::Pass { "Use Set method to set the place attribute."); IR_ENFORCE(Has("__mixed_precision_mode__"), "Pass initialize failed." - "When using AutoMixedPrecisionPass, precison_mode attribute is " + "When using AutoMixedPrecisionPass, precision_mode attribute is " "required!" "Use Set method to set the scope attribute."); @@ -224,7 +224,7 @@ class AutoMixedPrecisionPass : public pir::Pass { precision_updated = true; } if (!OpRunLowPrecision(op)) continue; - // if the producer's output is in float VectorType, then the precsion + // if the producer's output is in float VectorType, then the precision // between two op should be the same for (size_t idx = 0; idx < op->num_operands(); ++idx) { if (!op->operand_source(idx)) continue; diff --git a/paddle/fluid/pir/transforms/fusion/conv2d_add_act_fuse_pass.cc b/paddle/fluid/pir/transforms/fusion/conv2d_add_act_fuse_pass.cc index 9e950dc2d11b9..4968ae9744248 100644 --- a/paddle/fluid/pir/transforms/fusion/conv2d_add_act_fuse_pass.cc +++ b/paddle/fluid/pir/transforms/fusion/conv2d_add_act_fuse_pass.cc @@ -207,7 +207,7 @@ class Conv2dAddActFusePass : public pir::PatternRewritePass { 1, std::vector{ paddle::dialect::FusedConv2dAddActOp::name()}); - auto conv2d_doublue_add_act_fuse_pattern = + auto conv2d_double_add_act_fuse_pattern = std::make_unique( context, 1, @@ -215,7 +215,7 @@ class Conv2dAddActFusePass : public pir::PatternRewritePass { paddle::dialect::FusedConv2dAddActOp::name()}); // conv2d+add+add+act->fused_conv2d_add_act - ps.Add(std::move(conv2d_doublue_add_act_fuse_pattern)); + ps.Add(std::move(conv2d_double_add_act_fuse_pattern)); // conv2d+add+act->fused_conv2d_add_act ps.Add(std::move(conv2d_add_act_fuse_pattern)); return ps; diff --git a/paddle/fluid/pir/transforms/fusion/fused_linear_param_grad_add_pass.cc b/paddle/fluid/pir/transforms/fusion/fused_linear_param_grad_add_pass.cc index 120b882a67194..074d2d1acb009 100644 --- a/paddle/fluid/pir/transforms/fusion/fused_linear_param_grad_add_pass.cc +++ b/paddle/fluid/pir/transforms/fusion/fused_linear_param_grad_add_pass.cc @@ -67,7 +67,7 @@ class FusedMatmulAddGradAddPattern : public paddle::drr::DrrPatternBase { }); paddle::drr::ResultPattern res = pat.ResultPattern(); - const auto &muti_precision_attr = + const auto &multi_precision_attr = res.ComputeAttr([](const paddle::drr::MatchContext &match_ctx) -> bool { return !(pir::GetDataTypeFromValue(match_ctx.Tensor("dweight")) == pir::GetDataTypeFromValue(match_ctx.Tensor("weight_grad"))); @@ -78,7 +78,7 @@ class FusedMatmulAddGradAddPattern : public paddle::drr::DrrPatternBase { {"transpose_y", res.BoolAttr(true)}}); const auto &fused_linear_param_grad_add = res.Op(paddle::dialect::FusedLinearParamGradAddOp::name(), - {{{"multi_precision", muti_precision_attr}, + {{{"multi_precision", multi_precision_attr}, {"has_bias", res.BoolAttr(true)}}}); matmul({&res.Tensor("fwd_add_out_grad"), &res.Tensor("weight")}, @@ -122,7 +122,7 @@ class FusedMatmulGradAddPattern : public paddle::drr::DrrPatternBase { paddle::drr::ResultPattern res = pat.ResultPattern(); - const auto &muti_precision_attr = + const auto &multi_precision_attr = res.ComputeAttr([](const paddle::drr::MatchContext &match_ctx) -> bool { return !(pir::GetDataTypeFromValue(match_ctx.Tensor("dweight")) == pir::GetDataTypeFromValue(match_ctx.Tensor("weight_grad"))); @@ -133,7 +133,7 @@ class FusedMatmulGradAddPattern : public paddle::drr::DrrPatternBase { {"transpose_y", res.BoolAttr(true)}}); const auto &fused_linear_param_grad_add = res.Op(paddle::dialect::FusedLinearParamGradAddOp::name(), - {{{"multi_precision", muti_precision_attr}, + {{{"multi_precision", multi_precision_attr}, {"has_bias", res.BoolAttr(false)}}}); matmul({&res.Tensor("out_grad"), &res.Tensor("weight")}, @@ -194,7 +194,7 @@ class FusedMatmulReshapeMatmulAddPattern : public paddle::drr::DrrPatternBase { }); paddle::drr::ResultPattern res = pat.ResultPattern(); - const auto &muti_precision_attr = + const auto &multi_precision_attr = res.ComputeAttr([](const paddle::drr::MatchContext &match_ctx) -> bool { return !(pir::GetDataTypeFromValue(match_ctx.Tensor("dweight")) == pir::GetDataTypeFromValue(match_ctx.Tensor("w_grad"))); @@ -202,7 +202,7 @@ class FusedMatmulReshapeMatmulAddPattern : public paddle::drr::DrrPatternBase { const auto &fused_linear_param_grad_add = res.Op(paddle::dialect::FusedLinearParamGradAddOp::name(), - {{{"multi_precision", muti_precision_attr}, + {{{"multi_precision", multi_precision_attr}, {"has_bias", res.BoolAttr(false)}}}); fused_linear_param_grad_add( @@ -239,7 +239,7 @@ class FusedMatmulAddaPattern : public paddle::drr::DrrPatternBase { }); paddle::drr::ResultPattern res = pat.ResultPattern(); - const auto &muti_precision_attr = + const auto &multi_precision_attr = res.ComputeAttr([](const paddle::drr::MatchContext &match_ctx) -> bool { return !(pir::GetDataTypeFromValue(match_ctx.Tensor("dweight")) == pir::GetDataTypeFromValue(match_ctx.Tensor("weight_grad"))); @@ -247,7 +247,7 @@ class FusedMatmulAddaPattern : public paddle::drr::DrrPatternBase { const auto &fused_linear_param_grad_add = res.Op(paddle::dialect::FusedLinearParamGradAddOp::name(), - {{{"multi_precision", muti_precision_attr}, + {{{"multi_precision", multi_precision_attr}, {"has_bias", res.BoolAttr(false)}}}); fused_linear_param_grad_add( {&res.Tensor("x"), @@ -283,7 +283,7 @@ class FusedMatmulAddbPattern : public paddle::drr::DrrPatternBase { }); paddle::drr::ResultPattern res = pat.ResultPattern(); - const auto &muti_precision_attr = + const auto &multi_precision_attr = res.ComputeAttr([](const paddle::drr::MatchContext &match_ctx) -> bool { return !(pir::GetDataTypeFromValue(match_ctx.Tensor("dweight")) == pir::GetDataTypeFromValue(match_ctx.Tensor("weight_grad"))); @@ -291,7 +291,7 @@ class FusedMatmulAddbPattern : public paddle::drr::DrrPatternBase { const auto &fused_linear_param_grad_add = res.Op(paddle::dialect::FusedLinearParamGradAddOp::name(), - {{{"multi_precision", muti_precision_attr}, + {{{"multi_precision", multi_precision_attr}, {"has_bias", res.BoolAttr(false)}}}); fused_linear_param_grad_add( {&res.Tensor("x"), @@ -341,7 +341,7 @@ class FusedMatmulAddGradAddaPattern : public paddle::drr::DrrPatternBase { }); paddle::drr::ResultPattern res = pat.ResultPattern(); - const auto &muti_precision_attr = + const auto &multi_precision_attr = res.ComputeAttr([](const paddle::drr::MatchContext &match_ctx) -> bool { return !(pir::GetDataTypeFromValue(match_ctx.Tensor("dweight")) == pir::GetDataTypeFromValue(match_ctx.Tensor("weight_grad"))); @@ -349,7 +349,7 @@ class FusedMatmulAddGradAddaPattern : public paddle::drr::DrrPatternBase { const auto &fused_linear_param_grad_add = res.Op(paddle::dialect::FusedLinearParamGradAddOp::name(), - {{{"multi_precision", muti_precision_attr}, + {{{"multi_precision", multi_precision_attr}, {"has_bias", res.BoolAttr(true)}}}); fused_linear_param_grad_add( {&res.Tensor("x"), @@ -399,14 +399,14 @@ class FusedMatmulAddGradAddbPattern : public paddle::drr::DrrPatternBase { }); paddle::drr::ResultPattern res = pat.ResultPattern(); - const auto &muti_precision_attr = + const auto &multi_precision_attr = res.ComputeAttr([](const paddle::drr::MatchContext &match_ctx) -> bool { return !(pir::GetDataTypeFromValue(match_ctx.Tensor("dweight")) == pir::GetDataTypeFromValue(match_ctx.Tensor("weight_grad"))); }); const auto &fused_linear_param_grad_add = res.Op(paddle::dialect::FusedLinearParamGradAddOp::name(), - {{{"multi_precision", muti_precision_attr}, + {{{"multi_precision", multi_precision_attr}, {"has_bias", res.BoolAttr(true)}}}); fused_linear_param_grad_add( {&res.Tensor("x"), diff --git a/paddle/fluid/pir/transforms/fusion/fused_weight_only_linear_pass.cc b/paddle/fluid/pir/transforms/fusion/fused_weight_only_linear_pass.cc index bf4ea92af67b2..fc415c3852e38 100644 --- a/paddle/fluid/pir/transforms/fusion/fused_weight_only_linear_pass.cc +++ b/paddle/fluid/pir/transforms/fusion/fused_weight_only_linear_pass.cc @@ -123,9 +123,9 @@ class FusedWeightOnlyLinearPass : public pir::PatternRewritePass { } bool CanApplyOn(pir::Operation *op) const override { - int sm_vesion = getSMVersion(); - if (sm_vesion != 70 && sm_vesion != 75 && sm_vesion != 80 && - sm_vesion != 86) { + int sm_version = getSMVersion(); + if (sm_version != 70 && sm_version != 75 && sm_version != 80 && + sm_version != 86) { return false; } return op->num_regions() > 0; diff --git a/paddle/fluid/pir/transforms/sub_graph_detector.cc b/paddle/fluid/pir/transforms/sub_graph_detector.cc index 0690bc1c8399c..0e9547f7642c7 100644 --- a/paddle/fluid/pir/transforms/sub_graph_detector.cc +++ b/paddle/fluid/pir/transforms/sub_graph_detector.cc @@ -316,11 +316,11 @@ bool SubgraphDetector::FuseSubGraph(SubGraphPtr subgraph_ptr) { if (!consumer->substitute) { continue; } - // fast depency check. + // fast dependency check. if (IsDependencySimplify(producer, consumer, consumers)) { continue; } - // global depency check. + // global dependency check. if (IsDependency(producer, consumer, consumers)) { continue; } @@ -341,7 +341,7 @@ bool SubgraphDetector::FuseSubGraph(SubGraphPtr subgraph_ptr) { producer->ops.end(), candidate->ops.begin(), candidate->ops.end()); producer->op_set.insert(candidate->op_set.begin(), candidate->op_set.end()); - // update bound for check depency + // update bound for check dependency producer->max_depth = std::max(producer->max_depth, candidate->max_depth); producer->min_depth = std::min(producer->min_depth, candidate->min_depth); @@ -364,7 +364,7 @@ bool SubgraphDetector::FuseSubGraph(SubGraphPtr subgraph_ptr) { tmp->producers.erase(candidate); } - // remove candicate in producer/consumer + // remove candidate in producer/consumer producer->producers.erase(candidate); producer->consumers.erase(candidate); @@ -387,7 +387,7 @@ bool SubgraphDetector::FuseSubGraph(SubGraphPtr subgraph_ptr) { return true; } -// check exist depency. +// check exist dependency. bool SubgraphDetector::IsDependency( const SubGraphPtr& producer_g, const SubGraphPtr& consumer, diff --git a/paddle/fluid/pir/transforms/sub_graph_detector.h b/paddle/fluid/pir/transforms/sub_graph_detector.h index 1b7ec2bc5da6a..424855b02ddcc 100644 --- a/paddle/fluid/pir/transforms/sub_graph_detector.h +++ b/paddle/fluid/pir/transforms/sub_graph_detector.h @@ -51,7 +51,7 @@ class SubgraphDetector { void DoSubGraphFusion(); bool FuseSubGraph(SubGraphPtr subgraph_ptr); - // check exist depency. + // check exist dependency. bool IsDependency(const SubGraphPtr& producer_g, const SubGraphPtr& consumer, const std::unordered_set& consumers); From 4bebcfe53bff5d6e7fd1d350db06d91814043530 Mon Sep 17 00:00:00 2001 From: co63oc Date: Thu, 29 Feb 2024 23:25:37 +0800 Subject: [PATCH 189/282] Fix transfrom transform, etc (#62183) --- paddle/fluid/operators/pull_gpups_sparse_op.h | 4 ++-- paddle/fluid/operators/py_func_op.cc | 2 +- paddle/fluid/operators/randperm_op.h | 6 +++--- paddle/fluid/operators/read_file_op.cc | 2 +- paddle/fluid/operators/repeat_interleave_op.cc | 4 ++-- paddle/fluid/operators/reshape_op.cc | 2 +- paddle/fluid/operators/split_op.cc | 2 +- paddle/fluid/operators/sum_op.cc | 2 +- paddle/fluid/operators/svd_helper.h | 8 ++++---- paddle/fluid/operators/tdm_sampler_op.h | 4 ++-- paddle/fluid/operators/teacher_student_sigmoid_loss_op.cc | 2 +- paddle/fluid/operators/tile_op.cc | 2 +- paddle/fluid/operators/top_k_op.h | 2 +- paddle/fluid/operators/top_k_op_xpu.cc | 2 +- paddle/fluid/operators/transfer_layout_op.h | 6 +++--- paddle/fluid/operators/transpose_op.cc | 2 +- .../fluid/prim/utils/static/composite_grad_desc_maker.h | 2 +- 17 files changed, 27 insertions(+), 27 deletions(-) diff --git a/paddle/fluid/operators/pull_gpups_sparse_op.h b/paddle/fluid/operators/pull_gpups_sparse_op.h index d8fdadd99cbd4..e5e08cfdde685 100644 --- a/paddle/fluid/operators/pull_gpups_sparse_op.h +++ b/paddle/fluid/operators/pull_gpups_sparse_op.h @@ -30,7 +30,7 @@ static void PullGpuPSSparseFunctor(const framework::ExecutionContext &ctx) { auto embedding_size_vec = ctx.Attr>("size"); const auto slot_size = inputs.size(); std::vector all_keys(slot_size); - // GpuPSPS only supports float now + // GpuPS only supports float now std::vector all_values(slot_size); std::vector slot_lengths(slot_size); for (size_t i = 0; i < slot_size; i++) { @@ -80,7 +80,7 @@ static void PushGpuPSSparseFunctor(const framework::ExecutionContext &ctx) { cur_batch_size, platform::errors::PreconditionNotMet( "The batch size of all input slots should be same, " - "please cheack")); + "please check")); } const float *grad_value = d_output[i]->data(); all_grad_values[i] = grad_value; diff --git a/paddle/fluid/operators/py_func_op.cc b/paddle/fluid/operators/py_func_op.cc index ecdded21bb3e6..7d9c8ceca4943 100644 --- a/paddle/fluid/operators/py_func_op.cc +++ b/paddle/fluid/operators/py_func_op.cc @@ -119,7 +119,7 @@ static void CallPythonFunc(py::object *callable, out->ShareDataWith(*py_out_tensor); } catch (py::cast_error &) { PADDLE_THROW(platform::errors::InvalidArgument( - "py::cast to phi::DenseTensor error. The %d-th output expection is " + "py::cast to phi::DenseTensor error. The %d-th output exception is " "phi::DenseTensor", i)); } diff --git a/paddle/fluid/operators/randperm_op.h b/paddle/fluid/operators/randperm_op.h index 96981a4728402..560fdeb42eaa3 100644 --- a/paddle/fluid/operators/randperm_op.h +++ b/paddle/fluid/operators/randperm_op.h @@ -29,7 +29,7 @@ namespace paddle { namespace operators { template -static inline void random_permate(T* data_ptr, int num, unsigned int seed) { +static inline void random_permute(T* data_ptr, int num, unsigned int seed) { auto engine = phi::GetCPURandomEngine(seed); for (int i = 0; i < num; ++i) { data_ptr[i] = static_cast(i); @@ -50,13 +50,13 @@ class RandpermKernel : public framework::OpKernel { if (platform::is_cpu_place(ctx.GetPlace())) { T* out_data = out_tensor->mutable_data(platform::CPUPlace()); - random_permate(out_data, n, seed); + random_permute(out_data, n, seed); } else { phi::DenseTensor tmp_tensor; tmp_tensor.Resize(common::make_ddim({n})); T* tmp_data = tmp_tensor.mutable_data(platform::CPUPlace()); - random_permate(tmp_data, n, seed); + random_permute(tmp_data, n, seed); framework::TensorCopy(tmp_tensor, ctx.GetPlace(), out_tensor); } } diff --git a/paddle/fluid/operators/read_file_op.cc b/paddle/fluid/operators/read_file_op.cc index c19d0a6344ce5..a65b51d24e245 100644 --- a/paddle/fluid/operators/read_file_op.cc +++ b/paddle/fluid/operators/read_file_op.cc @@ -46,7 +46,7 @@ class ReadFileOpMaker : public framework::OpProtoAndCheckerMaker { AddComment(R"DOC( This operator read a file. )DOC"); - AddAttr("filename", "Path of the file to be readed.") + AddAttr("filename", "Path of the file to be read.") .SetDefault({}); } }; diff --git a/paddle/fluid/operators/repeat_interleave_op.cc b/paddle/fluid/operators/repeat_interleave_op.cc index 15b4b80cb739b..d0af82510bdc4 100644 --- a/paddle/fluid/operators/repeat_interleave_op.cc +++ b/paddle/fluid/operators/repeat_interleave_op.cc @@ -77,7 +77,7 @@ class RepeatInterleaveOp : public framework::OperatorWithKernel { } else if (repeats > 0) { output_dim[dim] = input_dim[dim] * repeats; } - VLOG(3) << "infershap out " << output_dim[dim]; + VLOG(3) << "infershape out " << output_dim[dim]; ctx->SetOutputDim("Out", common::make_ddim(output_dim)); auto type = ctx->GetInputsVarType("X")[0]; if (type == framework::proto::VarType::LOD_TENSOR) { @@ -124,7 +124,7 @@ class RepeatInterleaveOpMaker : public framework::OpProtoAndCheckerMaker { void Make() override { AddInput("X", "(Tensor) the input tensor."); AddInput("RepeatsTensor", - "the 1-D tensor containing the repeats alongsize the axis.") + "the 1-D tensor containing the repeats alongside the axis.") .AsDispensable(); AddOutput("Out", "the output tensor."); AddAttr("Repeats", "the number of repetitions for each element.") diff --git a/paddle/fluid/operators/reshape_op.cc b/paddle/fluid/operators/reshape_op.cc index 822eaf514bac5..34d80604ae8b0 100644 --- a/paddle/fluid/operators/reshape_op.cc +++ b/paddle/fluid/operators/reshape_op.cc @@ -581,7 +581,7 @@ class Reshape2CompositeGradOpMaker : public prim::CompositeGradOpMakerBase { auto *dx_ptr = this->GetOutputPtr(&dx); std::string dx_name = this->GetOutputName(dx); - VLOG(6) << "Runing reshape2_grad composite func"; + VLOG(6) << "Running reshape2_grad composite func"; prim::reshape_grad(x, out_grad, dx_ptr); this->RecoverOutputName(dx, dx_name); } diff --git a/paddle/fluid/operators/split_op.cc b/paddle/fluid/operators/split_op.cc index 1842ed34a5c67..ceb087fce4cfb 100644 --- a/paddle/fluid/operators/split_op.cc +++ b/paddle/fluid/operators/split_op.cc @@ -222,7 +222,7 @@ class SplitCompositeGradOpMaker : public prim::CompositeGradOpMakerBase { "We don't support dynamic index or sections from tensor for split " "composite grad for now. ")); } else { - VLOG(6) << "Runing split_grad composite func"; + VLOG(6) << "Running split_grad composite func"; prim::split_grad(out_grad, axis, dx_ptr); this->RecoverOutputName(input_grad, dx_name); } diff --git a/paddle/fluid/operators/sum_op.cc b/paddle/fluid/operators/sum_op.cc index 718f4876406af..d8b7e35d6d3a1 100644 --- a/paddle/fluid/operators/sum_op.cc +++ b/paddle/fluid/operators/sum_op.cc @@ -127,7 +127,7 @@ class SumOpMaker : public framework::OpProtoAndCheckerMaker { void Make() override { AddInput( "X", - "A Varaible list. The shape and data type of the list elements" + "A Variable list. The shape and data type of the list elements" "should be consistent. Variable can be multi-dimensional Tensor" "or phi::DenseTensor, and data types can be: float32, float64, int32, " "int64.") diff --git a/paddle/fluid/operators/svd_helper.h b/paddle/fluid/operators/svd_helper.h index caa31565d4cf3..273e2c7b65100 100644 --- a/paddle/fluid/operators/svd_helper.h +++ b/paddle/fluid/operators/svd_helper.h @@ -271,7 +271,7 @@ struct DiagAndFillFunctor { template struct DeviceIndependenceTensorOperations { - // 1. Device indenpendence, for kernel reuse. + // 1. Device independence, for kernel reuse. // 2. Input and output is always tensor type. // 3. output phi::DenseTensor is alway allocated // 4. Basic phi::DenseTensor operator is supported @@ -315,7 +315,7 @@ struct DeviceIndependenceTensorOperations { } phi::DenseTensor Transpose(const phi::DenseTensor& x) { - // transpose the last two dimision + // transpose the last two dimension phi::DenseTensor ret; auto x_dim = x.dims(); auto x_vec = common::vectorize(x_dim); @@ -745,7 +745,7 @@ struct DeviceIndependenceTensorOperations { const framework::AttributeMap& attrs, std::vector out_shape, NameOutTensor out_str = {"Out"}) { - // varialble set dims must be phi::DenseTensor / SelectedRowTensor + // variable set dims must be phi::DenseTensor / SelectedRowTensor framework::Scope& local_scope = context.scope().NewScope(); framework::VariableNameMap op_outputs; for (auto out_name : out_str) { @@ -753,7 +753,7 @@ struct DeviceIndependenceTensorOperations { op_outputs[out_name].emplace_back("tmp_" + out_name); } auto out_var = local_scope.Var("tmp_Out"); // return the Out - // create Out phi::DenseTensor and allocat memory + // create Out phi::DenseTensor and allocate memory out_var->GetMutable()->mutable_data( common::make_ddim(out_shape), context.GetPlace()); // common::make_ddim(out_shape) diff --git a/paddle/fluid/operators/tdm_sampler_op.h b/paddle/fluid/operators/tdm_sampler_op.h index ec5587c330fc7..52f86d633307b 100644 --- a/paddle/fluid/operators/tdm_sampler_op.h +++ b/paddle/fluid/operators/tdm_sampler_op.h @@ -214,9 +214,9 @@ void TDMSamplerInner(const framework::ExecutionContext &context, label_vec[i * sample_res_length + offset] = 0; mask_vec[i * sample_res_length + offset] = 1; VLOG(3) << "TDM: node id: " << travel_data[start_offset + layer_idx] - << " Res append negitive " + << " Res append negative " << output_vec[i * sample_res_length + offset] - << " Label append negitive " + << " Label append negative " << label_vec[i * sample_res_length + offset] << " Mask append value " << mask_vec[i * sample_res_length + offset]; diff --git a/paddle/fluid/operators/teacher_student_sigmoid_loss_op.cc b/paddle/fluid/operators/teacher_student_sigmoid_loss_op.cc index ad54a49f820f9..332008894d5b9 100644 --- a/paddle/fluid/operators/teacher_student_sigmoid_loss_op.cc +++ b/paddle/fluid/operators/teacher_student_sigmoid_loss_op.cc @@ -173,7 +173,7 @@ class TeacherStudentSigmoidLossGradientOp platform::errors::InvalidArgument( "When Attr(soft_label) == false, the 2nd dimension of " "Input(Label) should be 1. But received Input(Label)'s 2nd " - "dimemsion " + "dimension " "is [%d]", label_dims[1])); } diff --git a/paddle/fluid/operators/tile_op.cc b/paddle/fluid/operators/tile_op.cc index 26657ce42f303..9d961bbd57122 100644 --- a/paddle/fluid/operators/tile_op.cc +++ b/paddle/fluid/operators/tile_op.cc @@ -185,7 +185,7 @@ class TileCompositeGradOpMaker : public prim::CompositeGradOpMakerBase { "We don't support RepeatTimes from tensor or repeat_times_tensor for " "tile composite grad for now. ")); } else { - VLOG(6) << "Runing tile_grad composite func"; + VLOG(6) << "Running tile_grad composite func"; prim::tile_grad( x, out_grad, paddle::experimental::IntArray(repeat_times), dx_ptr); this->RecoverOutputName(x_grad, dx_name); diff --git a/paddle/fluid/operators/top_k_op.h b/paddle/fluid/operators/top_k_op.h index f8fa53e2ad505..b0d30f1d22d3b 100644 --- a/paddle/fluid/operators/top_k_op.h +++ b/paddle/fluid/operators/top_k_op.h @@ -46,7 +46,7 @@ class TopkKernel : public framework::OpKernel { T* output_data = output->mutable_data(ctx.GetPlace()); int64_t* indices_data = indices->mutable_data(ctx.GetPlace()); - // reshape input to a flattern matrix(like flat_inner_dims) + // reshape input to a flatten matrix(like flat_inner_dims) framework::DDim inputdims = input->dims(); const size_t row = common::product(common::slice_ddim(inputdims, 0, inputdims.size() - 1)); diff --git a/paddle/fluid/operators/top_k_op_xpu.cc b/paddle/fluid/operators/top_k_op_xpu.cc index 55d3fa8624a8c..fff713236e9a6 100644 --- a/paddle/fluid/operators/top_k_op_xpu.cc +++ b/paddle/fluid/operators/top_k_op_xpu.cc @@ -60,7 +60,7 @@ class TopkXPUKernel : public framework::OpKernel { int* indices_int_data = RAII_GUARD.alloc_l3_or_gm(indices->numel()); PADDLE_ENFORCE_XDNN_NOT_NULL(indices_int_data); - // reshape input to a flattern matrix(like flat_inner_dims) + // reshape input to a flatten matrix(like flat_inner_dims) framework::DDim inputdims = input->dims(); const size_t row = common::product(common::slice_ddim(inputdims, 0, inputdims.size() - 1)); diff --git a/paddle/fluid/operators/transfer_layout_op.h b/paddle/fluid/operators/transfer_layout_op.h index 52633640fa95b..2736171626121 100644 --- a/paddle/fluid/operators/transfer_layout_op.h +++ b/paddle/fluid/operators/transfer_layout_op.h @@ -110,7 +110,7 @@ class TransferLayoutFunctor { } VLOG(4) << "TransDataLayoutFromOneDNN: " << in_layout << "->" << target_layout; - // Case2 - transfrom from ONEDNN OPKernel to Non-ONEDNN OPKernel + // Case2 - transform from ONEDNN OPKernel to Non-ONEDNN OPKernel // Do transform via ONEDNN lib phi::funcs::TransDataLayoutFromOneDNN(in_layout, target_layout, @@ -119,11 +119,11 @@ class TransferLayoutFunctor { dev_ctx_.GetPlace()); } } else { - // Case3 - transfrom between Non-ONEDNN OPKernels + // Case3 - transform between Non-ONEDNN OPKernels TransDataLayout(dev_ctx_, in_tensor, &out_tensor); } #else - // Case3 - transfrom between Non-ONEDNN OPKernels + // Case3 - transform between Non-ONEDNN OPKernels TransDataLayout(dev_ctx_, in_tensor, &out_tensor); #endif framework::SetTensorToVariable(*in_, out_tensor, out_); diff --git a/paddle/fluid/operators/transpose_op.cc b/paddle/fluid/operators/transpose_op.cc index 417299d24db07..340728a1b8d1e 100644 --- a/paddle/fluid/operators/transpose_op.cc +++ b/paddle/fluid/operators/transpose_op.cc @@ -202,7 +202,7 @@ class Transpose2CompositeGradOpMaker : public prim::CompositeGradOpMakerBase { std::string dx_name = this->GetOutputName(dx); std::vector axis = static_cast>(this->Attr>("axis")); - VLOG(6) << "Runing transpose2_grad composite func"; + VLOG(6) << "Running transpose2_grad composite func"; prim::transpose_grad(out_grad, axis, dx_ptr); this->RecoverOutputName(dx, dx_name); } diff --git a/paddle/fluid/prim/utils/static/composite_grad_desc_maker.h b/paddle/fluid/prim/utils/static/composite_grad_desc_maker.h index 0dd5d6fd4115c..d471b5277e029 100644 --- a/paddle/fluid/prim/utils/static/composite_grad_desc_maker.h +++ b/paddle/fluid/prim/utils/static/composite_grad_desc_maker.h @@ -72,7 +72,7 @@ class CompositeGradOpMakerBase { virtual ~CompositeGradOpMakerBase() = default; virtual std::vector> operator()() { - VLOG(3) << "Runing Composite Grad func for " << fwd_op_.Type() << "_grad "; + VLOG(3) << "Running Composite Grad func for " << fwd_op_.Type() << "_grad "; this->Apply(); std::vector> ops; // TODO(jiabin): Support multiple blocks later From 97eb5ac589bda9af1f8db548e58bf4b3f4f4e5c1 Mon Sep 17 00:00:00 2001 From: co63oc Date: Thu, 29 Feb 2024 23:26:07 +0800 Subject: [PATCH 190/282] Update random_routing_op.cc (#62182) --- paddle/fluid/operators/random_routing_op.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/operators/random_routing_op.cc b/paddle/fluid/operators/random_routing_op.cc index 9eaa3a664877c..dffcc9c361a66 100644 --- a/paddle/fluid/operators/random_routing_op.cc +++ b/paddle/fluid/operators/random_routing_op.cc @@ -22,7 +22,7 @@ class RandomRoutingOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; void InferShape(framework::InferShapeContext* ctx) const override { - OP_INOUT_CHECK(ctx->HasInput("Prob"), "Input", "Porb", "RandomRouting"); + OP_INOUT_CHECK(ctx->HasInput("Prob"), "Input", "Prob", "RandomRouting"); OP_INOUT_CHECK( ctx->HasInput("TopK_Value"), "Input", "TopKValue", "RandomRouting"); OP_INOUT_CHECK( From 108684db5854899ba67ebf3486bae44bc2fbf056 Mon Sep 17 00:00:00 2001 From: co63oc Date: Thu, 29 Feb 2024 23:26:41 +0800 Subject: [PATCH 191/282] Fix MaxSeqenceLenOp MaxSequenceLenOp, etc (#62181) --- paddle/fluid/operators/im2sequence_op.h | 16 +++++++------- paddle/fluid/operators/is_empty_op.h | 2 +- paddle/fluid/operators/isfinite_op.cc | 2 +- paddle/fluid/operators/linear_chain_crf_op.cc | 4 ++-- paddle/fluid/operators/linear_chain_crf_op.h | 8 +++---- paddle/fluid/operators/load_combine_op.h | 2 +- paddle/fluid/operators/load_op.cc | 2 +- paddle/fluid/operators/max_sequence_len_op.cc | 22 +++++++++---------- paddle/fluid/operators/nce_op.cc | 8 +++---- paddle/fluid/operators/nce_op.h | 4 ++-- paddle/fluid/operators/pad_op.cc | 2 +- .../operators/pull_box_extended_sparse_op.h | 2 +- 12 files changed, 37 insertions(+), 37 deletions(-) diff --git a/paddle/fluid/operators/im2sequence_op.h b/paddle/fluid/operators/im2sequence_op.h index 18e6d429f1b16..5fb689d5b1be0 100644 --- a/paddle/fluid/operators/im2sequence_op.h +++ b/paddle/fluid/operators/im2sequence_op.h @@ -48,13 +48,13 @@ class Im2SequenceKernel : public framework::OpKernel { auto strides = ctx.Attr>("strides"); auto paddings = ctx.Attr>("paddings"); if (ctx.HasInput("Y") && batch_size > 1) { - const phi::DenseTensor* imgrealsize = ctx.Input("Y"); + const phi::DenseTensor* img_real_size = ctx.Input("Y"); auto out_stride = ctx.Attr>("out_stride"); phi::DenseTensor cpu_shape_tensor; paddle::framework::TensorCopySync( - *imgrealsize, platform::CPUPlace(), &cpu_shape_tensor); - std::vector imgreal_h; - std::vector imgreal_w; + *img_real_size, platform::CPUPlace(), &cpu_shape_tensor); + std::vector img_real_h; + std::vector img_real_w; std::vector output_height; std::vector output_width; int result = 0; @@ -72,12 +72,12 @@ class Im2SequenceKernel : public framework::OpKernel { } else { tmp_real_w = tmp_real_w / out_stride[1] + 1; } - imgreal_h.push_back(tmp_real_h); - imgreal_w.push_back(tmp_real_w); + img_real_h.push_back(tmp_real_h); + img_real_w.push_back(tmp_real_w); output_height.push_back(Im2SeqOutputSize( - imgreal_h[i], kernels[0], paddings[0], paddings[2], strides[0])); + img_real_h[i], kernels[0], paddings[0], paddings[2], strides[0])); output_width.push_back(Im2SeqOutputSize( - imgreal_w[i], kernels[1], paddings[1], paddings[3], strides[1])); + img_real_w[i], kernels[1], paddings[1], paddings[3], strides[1])); result += output_height[i] * output_width[i]; } diff --git a/paddle/fluid/operators/is_empty_op.h b/paddle/fluid/operators/is_empty_op.h index 3c9dfbf58fae5..7c78c33621314 100644 --- a/paddle/fluid/operators/is_empty_op.h +++ b/paddle/fluid/operators/is_empty_op.h @@ -29,7 +29,7 @@ class IsEmptyOpKernel : public framework::OpKernel { auto* output_tensor = context.Output("Out"); // Note: is_empty is always executed on CPU and the output data should - // always be allocated for CPUPlace. We reigister CUDA kernel for this op to + // always be allocated for CPUPlace. We register CUDA kernel for this op to // avoid the unnecessary data transform. output_tensor->mutable_data(platform::CPUPlace())[0] = common::product(input_tensor->dims()) == 0; diff --git a/paddle/fluid/operators/isfinite_op.cc b/paddle/fluid/operators/isfinite_op.cc index 0d80a1c36b071..710cdaeb707b6 100644 --- a/paddle/fluid/operators/isfinite_op.cc +++ b/paddle/fluid/operators/isfinite_op.cc @@ -86,7 +86,7 @@ If any X contains Inf or Nan, the Out will generate a indicator. Out = Inf if any X contains Inf, Out = Nan if any X contains Nan, Out = 0 if no Inf/Nan detected. -If X contains both Inf/Nan, it will return the first indicator it meeted. +If X contains both Inf/Nan, it will return the first indicator it met. %s )DOC", diff --git a/paddle/fluid/operators/linear_chain_crf_op.cc b/paddle/fluid/operators/linear_chain_crf_op.cc index 46ff4c2e94a94..e017e43d7db2d 100644 --- a/paddle/fluid/operators/linear_chain_crf_op.cc +++ b/paddle/fluid/operators/linear_chain_crf_op.cc @@ -55,7 +55,7 @@ class LinearChainCRFOpMaker : public framework::OpProtoAndCheckerMaker { "probabilities of all possible unfinished sequences of tags that end " "at position $k$ with tag $v$. For each $k$, " "$\alpha[k, v]$ is a vector of length $D$ with a component for " - "each tag value $v$. This vector is called a forward vecotr and " + "each tag value $v$. This vector is called a forward vector and " "will also be used in backward computations.") .AsIntermediate(); AddOutput( @@ -105,7 +105,7 @@ CRF. Please refer to http://www.cs.columbia.edu/~mcollins/fb.pdf and weights, denoted as $a$ here. 3. The next D values of Input(Transition) of this operator are for ending weights, denoted as $b$ here. -4. The remaning values of Input(Transition) are for transition weights, +4. The remaining values of Input(Transition) are for transition weights, denoted as $w$ here. 5. Denote Input(Label) as $s$ here. diff --git a/paddle/fluid/operators/linear_chain_crf_op.h b/paddle/fluid/operators/linear_chain_crf_op.h index ad2fbefdfd71f..2891320506391 100644 --- a/paddle/fluid/operators/linear_chain_crf_op.h +++ b/paddle/fluid/operators/linear_chain_crf_op.h @@ -234,7 +234,7 @@ class LinearChainCRFOpKernel : public framework::OpKernel { static_cast(*std::max_element(lbl, lbl + seq_length)), tag_num, platform::errors::InvalidArgument( - "An invalid tag label that execesses the largest tag number.")); + "An invalid tag label that excesses the largest tag number.")); // Calculate the nominator part, which depends on the label sequence. ll += w[lbl[0]] /*start transition*/ + x[lbl[0]] + @@ -308,7 +308,7 @@ class LinearChainCRFGradOpKernel : public framework::OpKernel { // Now, all the inputs and outputs should be on the CPU memory. auto emission_dims = emission_exps->dims(); // Beta is the memo table used in dynamic programming to calculate the - // backwark vectors. For a backward vector i (the i-th row of beta), it + // backward vectors. For a backward vector i (the i-th row of beta), it // captures the unnormalized probabilities of partial sequences starting // at position i. phi::DenseTensor beta; @@ -372,7 +372,7 @@ class LinearChainCRFGradOpKernel : public framework::OpKernel { const size_t state_trans_base_idx = 2; // Calculate the backward vectors: beta. - // First, calculate the initialition state. + // First, calculate the initial state. for (size_t i = 0; i < tag_num; ++i) { beta_value[(seq_length - 1) * tag_num + i] = w_exps[tag_num + i]; } @@ -411,7 +411,7 @@ class LinearChainCRFGradOpKernel : public framework::OpKernel { T* trans_grad = transition_grad->data(); for (size_t k = 0; k < tag_num; ++k) { // Do not multiply by the output gradient here, because x_grad_mat has - // alrealy done this. + // already done this. trans_grad[k] += x_grad_mat(/*from start state*/ 0, k); trans_grad[tag_num + k] += x_grad_mat(/*to end state*/ seq_length - 1, k); diff --git a/paddle/fluid/operators/load_combine_op.h b/paddle/fluid/operators/load_combine_op.h index 9f15523ce0129..4641c39111fad 100644 --- a/paddle/fluid/operators/load_combine_op.h +++ b/paddle/fluid/operators/load_combine_op.h @@ -101,7 +101,7 @@ class LoadCombineOpKernel : public framework::OpKernel { framework::NFD(it->first, &tmp); if (tmp.empty()) { VLOG(0) << "The string " << it->first - << " was converted to unicode failedly! " + << " was converted to unicode unsuccessfully! " << "Then dropped to load it."; continue; } diff --git a/paddle/fluid/operators/load_op.cc b/paddle/fluid/operators/load_op.cc index dd85ccff87f2d..326746eb1e286 100644 --- a/paddle/fluid/operators/load_op.cc +++ b/paddle/fluid/operators/load_op.cc @@ -47,7 +47,7 @@ void LoadKernel(const Context& dev_ctx, PADDLE_ENFORCE_GE(seek, 0, phi::errors::InvalidArgument( - "seek witn tensor must great than or equal to 0")); + "seek with tensor must great than or equal to 0")); framework::DeserializeFromStream(fin, out, dev_ctx, seek, shape); } else { framework::DeserializeFromStream(fin, out, dev_ctx); diff --git a/paddle/fluid/operators/max_sequence_len_op.cc b/paddle/fluid/operators/max_sequence_len_op.cc index 813b1901760b9..1863787db3d3b 100644 --- a/paddle/fluid/operators/max_sequence_len_op.cc +++ b/paddle/fluid/operators/max_sequence_len_op.cc @@ -31,12 +31,12 @@ class OpBase; namespace paddle { namespace operators { -class MaxSeqenceLenOp : public framework::OperatorBase { +class MaxSequenceLenOp : public framework::OperatorBase { public: - MaxSeqenceLenOp(const std::string &type, - const framework::VariableNameMap &inputs, - const framework::VariableNameMap &outputs, - const framework::AttributeMap &attrs) + MaxSequenceLenOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) : OperatorBase(type, inputs, outputs, attrs) {} private: @@ -50,7 +50,7 @@ class MaxSeqenceLenOp : public framework::OperatorBase { } }; -class MaxSeqenceLenOpProtoMaker : public framework::OpProtoAndCheckerMaker { +class MaxSequenceLenOpProtoMaker : public framework::OpProtoAndCheckerMaker { public: void Make() override { AddInput("RankTable", "Input variable which is a LoDRankTable object"); @@ -65,11 +65,11 @@ class MaxSeqenceLenOpProtoMaker : public framework::OpProtoAndCheckerMaker { } }; -class MaxSeqenceLenInferShape : public framework::InferShapeBase { +class MaxSequenceLenInferShape : public framework::InferShapeBase { public: void operator()(framework::InferShapeContext *context) const override { OP_INOUT_CHECK( - context->HasInput("RankTable"), "Input", "RankTable", "MaxSeqenceLen"); + context->HasInput("RankTable"), "Input", "RankTable", "MaxSequenceLen"); context->SetOutputDim("Out", {1}); } }; @@ -78,8 +78,8 @@ class MaxSeqenceLenInferShape : public framework::InferShapeBase { REGISTER_OPERATOR( max_sequence_len, - paddle::operators::MaxSeqenceLenOp, - paddle::operators::MaxSeqenceLenOpProtoMaker, - paddle::operators::MaxSeqenceLenInferShape, + paddle::operators::MaxSequenceLenOp, + paddle::operators::MaxSequenceLenOpProtoMaker, + paddle::operators::MaxSequenceLenInferShape, paddle::framework::EmptyGradOpMaker, paddle::framework::EmptyGradOpMaker); diff --git a/paddle/fluid/operators/nce_op.cc b/paddle/fluid/operators/nce_op.cc index f4320cd0b6796..1b622b7571667 100644 --- a/paddle/fluid/operators/nce_op.cc +++ b/paddle/fluid/operators/nce_op.cc @@ -149,19 +149,19 @@ class NCEOpMaker : public framework::OpProtoAndCheckerMaker { AddInput( "CustomDistProbs", - "(Tensor) It is used in 'CostumDist' sampler. " + "(Tensor) It is used in 'CustomDist' sampler. " "It is a tensor with shape [num_total_classes]." "The i-th element is the probability of the i-th class being sampled.") .AsDispensable(); AddInput( "CustomDistAlias", - "(Tensor) It is used in 'CostumDist' sampler. " + "(Tensor) It is used in 'CustomDist' sampler. " "It is a tensor with shape [num_total_classes]." "The i-th element is the probability of the i-th class being sampled.") .AsDispensable(); AddInput( "CustomDistAliasProbs", - "(Tensor) It is used in 'CostumDist' sampler. " + "(Tensor) It is used in 'CustomDist' sampler. " "It is a tensor with shape [num_total_classes]." "The i-th element is the probability of the i-th class being sampled.") .AsDispensable(); @@ -194,7 +194,7 @@ class NCEOpMaker : public framework::OpProtoAndCheckerMaker { .SetDefault(10); AddAttr("sampler", "(int) Which sampler to be used to sample negative class." - "0: Uniform; 1: LogUniform; 2: CostumDist.") + "0: Uniform; 1: LogUniform; 2: CustomDist.") .SetDefault(0); AddAttr("seed", "(int) The seed used in sampler. If it is 0, " diff --git a/paddle/fluid/operators/nce_op.h b/paddle/fluid/operators/nce_op.h index a21c7c816e191..41262dca6e53c 100644 --- a/paddle/fluid/operators/nce_op.h +++ b/paddle/fluid/operators/nce_op.h @@ -146,7 +146,7 @@ class NCEKernel : public framework::OpKernel { default: { PADDLE_THROW(platform::errors::InvalidArgument( "Unsupported SamplerType. SamplerType should be 0: Uniform, " - "1: LogUniform or 2: CostumDist. Received SamplerType: %d", + "1: LogUniform or 2: CustomDist. Received SamplerType: %d", sampler_type)); } } @@ -332,7 +332,7 @@ class NCEGradKernel : public framework::OpKernel { default: { PADDLE_THROW(platform::errors::InvalidArgument( "Unsupported SamplerType. SamplerType should be 0: Uniform, " - "1: LogUniform or 2: CostumDist. Received SamplerType: %d", + "1: LogUniform or 2: CustomDist. Received SamplerType: %d", sampler_type)); } } diff --git a/paddle/fluid/operators/pad_op.cc b/paddle/fluid/operators/pad_op.cc index e2a0b3e025381..1a0f7b317d288 100644 --- a/paddle/fluid/operators/pad_op.cc +++ b/paddle/fluid/operators/pad_op.cc @@ -146,7 +146,7 @@ class PadCompositeGradOpMaker : public prim::CompositeGradOpMakerBase { std::vector paddings = static_cast>(this->Attr>("paddings")); float pad_value = static_cast(this->Attr("pad_value")); - VLOG(6) << "Runing add_grad composite func"; + VLOG(6) << "Running add_grad composite func"; prim::pad_grad(x, out_grad, paddings, pad_value, dx_ptr); this->RecoverOutputName(x_grad, dx_name); diff --git a/paddle/fluid/operators/pull_box_extended_sparse_op.h b/paddle/fluid/operators/pull_box_extended_sparse_op.h index b9508a279505e..76e570f10fb64 100644 --- a/paddle/fluid/operators/pull_box_extended_sparse_op.h +++ b/paddle/fluid/operators/pull_box_extended_sparse_op.h @@ -86,7 +86,7 @@ static void PushBoxExtendedSparseFunctor( cur_batch_size, platform::errors::PreconditionNotMet( "The batch size of all input slots should be same," - "please cheack")); + "please check")); } const float *grad_value = d_output[i]->data(); const float *grad_value_extend = d_output_extend[i]->data(); From 4fc1061358e7722c947e7e011bf5b9678899ee04 Mon Sep 17 00:00:00 2001 From: co63oc Date: Thu, 29 Feb 2024 23:27:20 +0800 Subject: [PATCH 192/282] Fix nerual neural, etc (#62179) --- .../operators/common_infer_shape_functions.cc | 4 ++-- .../fluid/operators/deformable_psroi_pooling_op.cc | 2 +- paddle/fluid/operators/dgc_op.cc | 2 +- paddle/fluid/operators/dropout_op.cc | 4 ++-- paddle/fluid/operators/expand_op.cc | 6 +++--- paddle/fluid/operators/expand_op.h | 14 +++++++------- paddle/fluid/operators/expand_v2_op.h | 10 +++++----- paddle/fluid/operators/fill_constant_op.cc | 2 +- paddle/fluid/operators/fused_token_prune_op.cc | 6 +++--- paddle/fluid/operators/gru_unit_op.h | 2 +- 10 files changed, 26 insertions(+), 26 deletions(-) diff --git a/paddle/fluid/operators/common_infer_shape_functions.cc b/paddle/fluid/operators/common_infer_shape_functions.cc index 52836ead345a1..1c13f873818f4 100644 --- a/paddle/fluid/operators/common_infer_shape_functions.cc +++ b/paddle/fluid/operators/common_infer_shape_functions.cc @@ -166,7 +166,7 @@ void BinaryOpBroadcastInferShape(framework::InferShapeContext *ctx) { "For binary broadcastable operator, if X is " "Sparse(VarType.SELECTED_ROWS" "), Y must be scalar, and the size of Y should be 1. " - "But reveived the size of Y = %s.", + "But received the size of Y = %s.", y_dims.size())); PADDLE_ENFORCE_EQ( y_dims[0], @@ -175,7 +175,7 @@ void BinaryOpBroadcastInferShape(framework::InferShapeContext *ctx) { "For binary broadcastable operator, if X is " "Sparse(VarType.SELECTED_ROWS" "), Y must be scalar, the first dimension of Y should be 1. " - "But reveived the first dimension of Y = %s.", + "But received the first dimension of Y = %s.", y_dims[0])); } else if (ctx->GetInputsVarType(x_name).front() != framework::proto::VarType::LOD_TENSOR) { diff --git a/paddle/fluid/operators/deformable_psroi_pooling_op.cc b/paddle/fluid/operators/deformable_psroi_pooling_op.cc index 1e3e52d34e41c..5b339cf96c2b1 100644 --- a/paddle/fluid/operators/deformable_psroi_pooling_op.cc +++ b/paddle/fluid/operators/deformable_psroi_pooling_op.cc @@ -101,7 +101,7 @@ class DeformablePSROIPoolOpMaker : public framework::OpProtoAndCheckerMaker { "The format is NCHW, where N is the number of ROIs, " "C is the number of output channels, " "H is the height of output, and " - "W is thewidth of output. "); + "W is the width of output. "); AddComment(R"DOC( **DeformablePSROIPooling Operator** DeformablePSROIPooling is a new method based Region of interest pooling diff --git a/paddle/fluid/operators/dgc_op.cc b/paddle/fluid/operators/dgc_op.cc index 06fb2874f2171..7325c4271f9c4 100644 --- a/paddle/fluid/operators/dgc_op.cc +++ b/paddle/fluid/operators/dgc_op.cc @@ -87,7 +87,7 @@ class DGCOpMaker : public framework::OpProtoAndCheckerMaker { .SetDefault(true); AddAttr>("sparsity", - "(vecotr, float)" + "(vector, float)" "The period sparsity of k_select."); AddAttr("rampup_begin_step", diff --git a/paddle/fluid/operators/dropout_op.cc b/paddle/fluid/operators/dropout_op.cc index 382a3f7ac920b..01df430f52161 100644 --- a/paddle/fluid/operators/dropout_op.cc +++ b/paddle/fluid/operators/dropout_op.cc @@ -108,7 +108,7 @@ class DropoutOpMaker : public framework::OpProtoAndCheckerMaker { AddComment(R"DOC( Dropout Operator. -Dropout refers to randomly dropping out units in a nerual network. It is a +Dropout refers to randomly dropping out units in a neural network. It is a regularization technique for reducing overfitting by preventing neuron co-adaption during training. The dropout operator randomly set (according to the given dropout probability) the outputs of some units to zero, while others @@ -175,7 +175,7 @@ class DropoutCompositeGradOpMaker : public prim::CompositeGradOpMakerBase { auto mode = this->Attr("dropout_implementation"); prim::dropout_grad( mask, out_grad, p, is_test, mode, x_grad_p); - VLOG(3) << "Runing dropout_grad composite func"; + VLOG(3) << "Running dropout_grad composite func"; this->RecoverOutputName(x_grad, x_grad_name); } }; diff --git a/paddle/fluid/operators/expand_op.cc b/paddle/fluid/operators/expand_op.cc index 4c2dd99265781..71295296218f0 100644 --- a/paddle/fluid/operators/expand_op.cc +++ b/paddle/fluid/operators/expand_op.cc @@ -106,7 +106,7 @@ class ExpandOpMaker : public framework::OpProtoAndCheckerMaker { "expand_times_tensor and expand_times.") .AsDispensable(); AddInput("expand_times_tensor", - "(Tensor Tensor), epxand times for X." + "(Tensor Tensor), expand times for X." "It has a higher priority than expand_times, but a lower priority " "than ExpandTimes") .AsDuplicable() @@ -165,7 +165,7 @@ class ExpandGradOp : public framework::OperatorWithKernel { out_dims[0], platform::errors::InvalidArgument( "The first dimension size (%d) of Input(Out@GRAD) should be " - "equal to the crroresponding dimension size (%d) of Input(X)", + "equal to the corresponding dimension size (%d) of Input(X)", out_dims[0], x_dims[0])); start_pos = 1u; @@ -181,7 +181,7 @@ class ExpandGradOp : public framework::OperatorWithKernel { out_dims[i], platform::errors::InvalidArgument( "The %uth dimension size (%d) of Input(Out@GRAD) should be " - "equal to the multiplication of the crroresponding dimension " + "equal to the multiplication of the corresponding dimension " "sizes of Input(X) (%d) and expand_times (%d).", i, out_dims[i], diff --git a/paddle/fluid/operators/expand_op.h b/paddle/fluid/operators/expand_op.h index 8ff69a537ff7f..ee100b3b48418 100644 --- a/paddle/fluid/operators/expand_op.h +++ b/paddle/fluid/operators/expand_op.h @@ -43,36 +43,36 @@ inline std::vector get_expand_times( expand_data = cpu_expand_tensor.data(); } #endif - auto vec_epxand_times = + auto vec_expand_times = std::vector(expand_data, expand_data + expand_tensor->numel()); - return vec_epxand_times; + return vec_expand_times; } auto list_expand_times_tensor = ctx.MultiInput("expand_times_tensor"); if (list_expand_times_tensor.size() > 0) { // get tensor from - std::vector vec_epxand_times; + std::vector vec_expand_times; for (size_t i = 0; i < list_expand_times_tensor.size(); ++i) { auto tensor = list_expand_times_tensor[i]; if (platform::is_gpu_place(tensor->place())) { phi::DenseTensor temp; paddle::framework::TensorCopySync(*tensor, platform::CPUPlace(), &temp); - vec_epxand_times.push_back(*temp.data()); + vec_expand_times.push_back(*temp.data()); } #ifdef PADDLE_WITH_XPU else if (platform::is_xpu_place(tensor->place())) { // NOLINT phi::DenseTensor temp; paddle::framework::TensorCopySync(*tensor, platform::CPUPlace(), &temp); - vec_epxand_times.push_back(*temp.data()); + vec_expand_times.push_back(*temp.data()); } #endif else { // NOLINT - vec_epxand_times.push_back(*tensor->data()); + vec_expand_times.push_back(*tensor->data()); } } - return vec_epxand_times; + return vec_expand_times; } else { return ctx.Attr>("expand_times"); } diff --git a/paddle/fluid/operators/expand_v2_op.h b/paddle/fluid/operators/expand_v2_op.h index 474ae818617fa..0a70faddb7d58 100644 --- a/paddle/fluid/operators/expand_v2_op.h +++ b/paddle/fluid/operators/expand_v2_op.h @@ -53,26 +53,26 @@ inline std::vector get_expand_shape( ctx.MultiInput("expand_shapes_tensor"); if (list_expand_shapes_tensor.size() > 0) { // get tensor from - std::vector vec_epxand_shape; + std::vector vec_expand_shape; for (size_t i = 0; i < list_expand_shapes_tensor.size(); ++i) { auto tensor = list_expand_shapes_tensor[i]; if (platform::is_gpu_place(tensor->place())) { phi::DenseTensor temp; paddle::framework::TensorCopySync(*tensor, platform::CPUPlace(), &temp); - vec_epxand_shape.push_back(*temp.data()); + vec_expand_shape.push_back(*temp.data()); } #ifdef PADDLE_WITH_XPU else if (platform::is_xpu_place(tensor->place())) { // NOLINT phi::DenseTensor temp; paddle::framework::TensorCopySync(*tensor, platform::CPUPlace(), &temp); - vec_epxand_shape.push_back(*temp.data()); + vec_expand_shape.push_back(*temp.data()); } #endif else { // NOLINT - vec_epxand_shape.push_back(*tensor->data()); + vec_expand_shape.push_back(*tensor->data()); } } - return vec_epxand_shape; + return vec_expand_shape; } else { return ctx.Attr>("shape"); } diff --git a/paddle/fluid/operators/fill_constant_op.cc b/paddle/fluid/operators/fill_constant_op.cc index 1263d156ce220..8a27649af864b 100644 --- a/paddle/fluid/operators/fill_constant_op.cc +++ b/paddle/fluid/operators/fill_constant_op.cc @@ -152,7 +152,7 @@ class FillConstantOpMaker : public framework::OpProtoAndCheckerMaker { "device") .SetDefault(false); AddAttr("place_type", - "(int, default -1) allow mamually setting place where the " + "(int, default -1) allow manually setting place where the " "variable should be hold. " "-1: not set manually, determine the place by executor. " "0: CPUPlace. " diff --git a/paddle/fluid/operators/fused_token_prune_op.cc b/paddle/fluid/operators/fused_token_prune_op.cc index 021aa95b1fe2c..9fab5c8e7c48d 100644 --- a/paddle/fluid/operators/fused_token_prune_op.cc +++ b/paddle/fluid/operators/fused_token_prune_op.cc @@ -39,7 +39,7 @@ class FusedTokenPruneOpMaker : public framework::OpProtoAndCheckerMaker { "The input of fused_token_prune op, whose shape should be [bsz, " "num_head, " "max_seq_len, max_seq_len] and dtype should be float32/float64." - "Mask is corresponding to Attn's elemnts one by one. Elements of Attn " + "Mask is corresponding to Attn's elements one by one. Elements of Attn " "will be set to zero if their corresponding mask is smaller than 0." "This process happens before sorting X by attn."); @@ -56,7 +56,7 @@ class FusedTokenPruneOpMaker : public framework::OpProtoAndCheckerMaker { "slimmed_seq_len, C]." "The tokens of X will be sorted by Attn firstly and then the " "last (max_seq_len - slimmed_seq_len)" - "tokens will be deleted. SlimmedX is the remainning part of X. " + "tokens will be deleted. SlimmedX is the remaining part of X. " ""); AddOutput( @@ -82,7 +82,7 @@ class FusedTokenPruneOpMaker : public framework::OpProtoAndCheckerMaker { 1. Elements of Attn will be set to zero if their corresponding mask is smaller than 0. 2. The second dimension of X will be sorted by Attn. 3. The last (max_seq_len - slimmed_seq_len) lines of X will be pruned. - 4. The remainning part of sorted X will output. + 4. The remaining part of sorted X will output. )DOC"); } }; diff --git a/paddle/fluid/operators/gru_unit_op.h b/paddle/fluid/operators/gru_unit_op.h index 9309ca0417f62..933176433e2d7 100644 --- a/paddle/fluid/operators/gru_unit_op.h +++ b/paddle/fluid/operators/gru_unit_op.h @@ -105,7 +105,7 @@ class GRUUnitKernel : public framework::OpKernel { gate_data, frame_size * 3); - // calculate activited gate + // calculate activated gate Eigen::array extents{{batch_size, frame_size}}; Eigen::array u_offsets{{0, 0}}; ActCompute(context.Attr("gate_activation"), From 471c8fe657c61a4f242436a1cf43a3ec608970ea Mon Sep 17 00:00:00 2001 From: co63oc Date: Thu, 29 Feb 2024 23:36:07 +0800 Subject: [PATCH 193/282] Fix StrightThroughEstimatorGradOp StraightThroughEstimatorGradOp (#62178) * Fix * Fix --- paddle/fluid/operators/fake_quantize_op.cc | 34 +++++++++++----------- paddle/fluid/operators/fake_quantize_op.cu | 4 +-- paddle/fluid/operators/fake_quantize_op.h | 4 +-- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/paddle/fluid/operators/fake_quantize_op.cc b/paddle/fluid/operators/fake_quantize_op.cc index 0515a56d41d5b..a5169892187a2 100644 --- a/paddle/fluid/operators/fake_quantize_op.cc +++ b/paddle/fluid/operators/fake_quantize_op.cc @@ -825,7 +825,7 @@ And it will not quantize the input tensor. } }; -class StrightThroughEstimatorGradOp : public framework::OperatorWithKernel { +class StraightThroughEstimatorGradOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; @@ -835,11 +835,11 @@ class StrightThroughEstimatorGradOp : public framework::OperatorWithKernel { OP_INOUT_CHECK(ctx->HasInput(out_grad_name), "Input", out_grad_name, - "StrightThroughEstimatorGradOp"); + "StraightThroughEstimatorGradOp"); OP_INOUT_CHECK(ctx->HasOutput(x_grad_name), "Output", x_grad_name, - "StrightThroughEstimatorGradOp"); + "StraightThroughEstimatorGradOp"); ctx->SetOutputDim(x_grad_name, ctx->GetInputDim(out_grad_name)); } @@ -853,13 +853,13 @@ class StrightThroughEstimatorGradOp : public framework::OperatorWithKernel { }; template -class StrightThroughEstimatorMaker : public framework::SingleGradOpMaker { +class StraightThroughEstimatorMaker : public framework::SingleGradOpMaker { public: using framework::SingleGradOpMaker::SingleGradOpMaker; protected: void Apply(GradOpPtr grad_op) const override { - grad_op->SetType("stright_throuth_estimator_grad"); + grad_op->SetType("straight_through_estimator_grad"); grad_op->SetInput(framework::GradVarName("Out"), this->OutputGrad("Out")); grad_op->SetOutput(framework::GradVarName("X"), this->InputGrad("X")); grad_op->SetAttrMap(this->Attrs()); @@ -888,8 +888,8 @@ REGISTER_OPERATOR( fake_quantize_dequantize_abs_max, ops::FakeQuantOrWithDequantAbsMaxOp, ops::FakeQuantOrWithDequantAbsMaxOpMaker, - ops::StrightThroughEstimatorMaker, - ops::StrightThroughEstimatorMaker); + ops::StraightThroughEstimatorMaker, + ops::StraightThroughEstimatorMaker); PD_REGISTER_STRUCT_KERNEL(fake_quantize_dequantize_abs_max, CPU, ALL_LAYOUT, @@ -924,8 +924,8 @@ REGISTER_OPERATOR( fake_quantize_dequantize_moving_average_abs_max, ops::FakeQuantOrWithDequantMovingAverageAbsMaxOp, ops::FakeQuantOrWithDequantMovingAverageAbsMaxOpMaker, - ops::StrightThroughEstimatorMaker, - ops::StrightThroughEstimatorMaker); + ops::StraightThroughEstimatorMaker, + ops::StraightThroughEstimatorMaker); PD_REGISTER_STRUCT_KERNEL(fake_quantize_dequantize_moving_average_abs_max, CPU, ALL_LAYOUT, @@ -948,28 +948,28 @@ REGISTER_OPERATOR( moving_average_abs_max_scale, ops::MovingAverageAbsMaxScaleOp, ops::MovingAverageAbsMaxScaleOpMaker, - ops::StrightThroughEstimatorMaker, - ops::StrightThroughEstimatorMaker); + ops::StraightThroughEstimatorMaker, + ops::StraightThroughEstimatorMaker); PD_REGISTER_STRUCT_KERNEL(moving_average_abs_max_scale, CPU, ALL_LAYOUT, ops::MovingAverageAbsMaxScaleKernel, float) {} -REGISTER_OPERATOR(stright_throuth_estimator_grad, - ops::StrightThroughEstimatorGradOp); -PD_REGISTER_STRUCT_KERNEL(stright_throuth_estimator_grad, +REGISTER_OPERATOR(straight_through_estimator_grad, + ops::StraightThroughEstimatorGradOp); +PD_REGISTER_STRUCT_KERNEL(straight_through_estimator_grad, CPU, ALL_LAYOUT, - ops::StrightThroughEstimatorGradKernel, + ops::StraightThroughEstimatorGradKernel, float) {} REGISTER_OPERATOR( fake_channel_wise_quantize_dequantize_abs_max, ops::FakeChannelWiseQuantizeDequantizeAbsMaxOp, ops::FakeChannelWiseQuantizeDequantizeAbsMaxOpMaker, - ops::StrightThroughEstimatorMaker, - ops::StrightThroughEstimatorMaker); + ops::StraightThroughEstimatorMaker, + ops::StraightThroughEstimatorMaker); PD_REGISTER_STRUCT_KERNEL(fake_channel_wise_quantize_dequantize_abs_max, CPU, ALL_LAYOUT, diff --git a/paddle/fluid/operators/fake_quantize_op.cu b/paddle/fluid/operators/fake_quantize_op.cu index bf990a451eb2d..68ceaca46d04f 100644 --- a/paddle/fluid/operators/fake_quantize_op.cu +++ b/paddle/fluid/operators/fake_quantize_op.cu @@ -60,10 +60,10 @@ PD_REGISTER_STRUCT_KERNEL(fake_quantize_dequantize_moving_average_abs_max, ops::FakeQuantizeDequantizeMovingAverageAbsMaxKernel, float, float16) {} -PD_REGISTER_STRUCT_KERNEL(stright_throuth_estimator_grad, +PD_REGISTER_STRUCT_KERNEL(straight_through_estimator_grad, GPU, ALL_LAYOUT, - ops::StrightThroughEstimatorGradKernel, + ops::StraightThroughEstimatorGradKernel, float, float16) {} PD_REGISTER_STRUCT_KERNEL(fake_channel_wise_quantize_dequantize_abs_max, diff --git a/paddle/fluid/operators/fake_quantize_op.h b/paddle/fluid/operators/fake_quantize_op.h index dd8675331fce6..6387018d1865e 100644 --- a/paddle/fluid/operators/fake_quantize_op.h +++ b/paddle/fluid/operators/fake_quantize_op.h @@ -446,7 +446,7 @@ class MovingAverageAbsMaxScaleKernel : public framework::OpKernel { }; template -class StrightThroughEstimatorGradKernel : public framework::OpKernel { +class StraightThroughEstimatorGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &context) const override { auto *d_out = @@ -455,7 +455,7 @@ class StrightThroughEstimatorGradKernel : public framework::OpKernel { auto *d_x = context.Output(x_grad_name); PADDLE_ENFORCE_NOT_NULL(d_x, platform::errors::PreconditionNotMet( - "StrightThroughEstimatorGradKernel " + "StraightThroughEstimatorGradKernel " "doesn't have the output named %s.", x_grad_name)); From cc1a2314e4754ff2f6e7303b422f3f2f1b2c28e7 Mon Sep 17 00:00:00 2001 From: co63oc Date: Thu, 29 Feb 2024 23:36:51 +0800 Subject: [PATCH 194/282] Fix summuation summation, etc(#62172) --- paddle/fluid/operators/cross_entropy_op.cc | 6 ++--- paddle/fluid/operators/cross_entropy_op.h | 6 ++--- paddle/fluid/operators/cudnn_lstm_op.cc | 2 +- .../custom_device_common_op_registry.cc | 12 +++++----- paddle/fluid/operators/data_norm_op.cc | 22 +++++++++---------- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/paddle/fluid/operators/cross_entropy_op.cc b/paddle/fluid/operators/cross_entropy_op.cc index 3a90012e1763a..cc2b4b4252835 100644 --- a/paddle/fluid/operators/cross_entropy_op.cc +++ b/paddle/fluid/operators/cross_entropy_op.cc @@ -239,7 +239,7 @@ class CrossEntropyOpMaker : public framework::OpProtoAndCheckerMaker { "represents the cross entropy loss."); AddAttr("soft_label", "(bool, default false), a flag indicating whether to " - "interpretant the given labels as soft labels.") + "interpret the given labels as soft labels.") .SetDefault(false); AddAttr("ignore_index", "(int, default -100), Specifies a target value that is" @@ -268,10 +268,10 @@ computation. $Y[i] = \sum_j{-Label[i, j] * log(X[i, j])}$ - Please make sure that in this case the summuation of each row of Label + Please make sure that in this case the summation of each row of Label equals one. -3) One-hot cross-entropy with vecterized Input(Label): +3) One-hot cross-entropy with vectorized Input(Label): As a special case of 2), when each row of Input(Label) has only one non-zero element (equals 1), soft-label cross-entropy degenerates to a one-hot cross-entropy with one-hot label representation. diff --git a/paddle/fluid/operators/cross_entropy_op.h b/paddle/fluid/operators/cross_entropy_op.h index d755cb1639572..5b76cc9a65a2b 100644 --- a/paddle/fluid/operators/cross_entropy_op.h +++ b/paddle/fluid/operators/cross_entropy_op.h @@ -62,9 +62,9 @@ class CrossEntropyOpKernel : public framework::OpKernel { }; template -class XeSoftlabelGradFunctor { +class XeSoftLabelGradFunctor { public: - XeSoftlabelGradFunctor(T* dx, + XeSoftLabelGradFunctor(T* dx, const T* dy, // NOLINT const T* x, // NOLINT const T* label, // NOLINT @@ -137,7 +137,7 @@ class CrossEntropyGradientOpKernel : public framework::OpKernel { int64_t class_num = x->dims()[rank - 1]; int64_t ignore_index = ctx.Attr("ignore_index"); if (ctx.Attr("soft_label")) { - XeSoftlabelGradFunctor functor(dx_data, + XeSoftLabelGradFunctor functor(dx_data, dy->data(), x->data(), label->data(), diff --git a/paddle/fluid/operators/cudnn_lstm_op.cc b/paddle/fluid/operators/cudnn_lstm_op.cc index e61512924f81d..a082dbbcb8bcb 100644 --- a/paddle/fluid/operators/cudnn_lstm_op.cc +++ b/paddle/fluid/operators/cudnn_lstm_op.cc @@ -152,7 +152,7 @@ the cell input ct-1 and the previous layer input xt given matrices W, R and bias which is computed based on the current input and the previous hidden state. Where sigmoid is the sigmoid operator: sigmoid(x) = 1 / (1 + e^-x), * represents a point-wise multiplication, -X represensts a matrix multiplication +X represents a matrix multiplication )DOC"); diff --git a/paddle/fluid/operators/custom_device_common_op_registry.cc b/paddle/fluid/operators/custom_device_common_op_registry.cc index 9573809d6c7cc..950b7f0663658 100644 --- a/paddle/fluid/operators/custom_device_common_op_registry.cc +++ b/paddle/fluid/operators/custom_device_common_op_registry.cc @@ -465,10 +465,10 @@ class CSoftmaxWithCrossEntropyGradCustomDeviceKernel framework::TensorCopy( *softmax, context.GetPlace(), context.device_context(), logit_grad); } - const auto sofrmax_dims = softmax->dims(); - const int axis = sofrmax_dims.size() - 1; - const int N = phi::funcs::SizeToAxis(axis, sofrmax_dims); - const int D = phi::funcs::SizeFromAxis(axis, sofrmax_dims); + const auto softmax_dims = softmax->dims(); + const int axis = softmax_dims.size() - 1; + const int N = phi::funcs::SizeToAxis(axis, softmax_dims); + const int D = phi::funcs::SizeFromAxis(axis, softmax_dims); const auto& label_type = labels->dtype(); if (label_type == phi::DataType::INT32 || @@ -514,7 +514,7 @@ class CSoftmaxWithCrossEntropyGradCustomDeviceKernel logit_grad ->ShareDataWith(*reinterpret_cast( logits_grad_out_tensor2.impl().get())) - .Resize(sofrmax_dims); + .Resize(softmax_dims); } else { PADDLE_THROW(phi::errors::Unavailable( "CustomDevice c_softmax_with_cross_entropy_grad " @@ -853,7 +853,7 @@ class AssignPosCustomDeviceKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { // assign pos decides which tokens should be fetched belong to specially - // counter orderingly. + // counter orderly. auto cum_count = context.Input( "cum_count"); // (counter number) int32 | int64 auto numbers = context.Input( diff --git a/paddle/fluid/operators/data_norm_op.cc b/paddle/fluid/operators/data_norm_op.cc index 32cc8b49cd007..cc3a224a7e862 100644 --- a/paddle/fluid/operators/data_norm_op.cc +++ b/paddle/fluid/operators/data_norm_op.cc @@ -81,28 +81,28 @@ class DataNormOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_EQ(ctx->GetInputDim("BatchSize").size(), 1UL, platform::errors::InvalidArgument( - "The input dim of BatchSize shouold be 1")); + "The input dim of BatchSize should be 1")); PADDLE_ENFORCE_EQ(ctx->GetInputDim("BatchSum").size(), 1UL, platform::errors::InvalidArgument( - "The input dim of BatchSum shouold be 1")); + "The input dim of BatchSum should be 1")); PADDLE_ENFORCE_EQ(ctx->GetInputDim("BatchSquareSum").size(), 1UL, platform::errors::InvalidArgument( - "The input dim of BatchSquareSum shouold be 1")); + "The input dim of BatchSquareSum should be 1")); if (ctx->IsRuntime()) { PADDLE_ENFORCE_EQ(ctx->GetInputDim("BatchSize")[0], C, platform::errors::InvalidArgument( - "The input dim[0] of BatchSize shouold be C")); + "The input dim[0] of BatchSize should be C")); PADDLE_ENFORCE_EQ(ctx->GetInputDim("BatchSum")[0], C, platform::errors::InvalidArgument( - "The input dim[0] of BatchSum shouold be C")); + "The input dim[0] of BatchSum should be C")); PADDLE_ENFORCE_EQ(ctx->GetInputDim("BatchSquareSum")[0], C, platform::errors::InvalidArgument( - "The input dim[0] of BatchSqureSum shouold be C")); + "The input dim[0] of BatchSquareSum should be C")); } if (enable_scale_and_shift) { @@ -112,10 +112,10 @@ class DataNormOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_EQ( scale_dim.size(), 1UL, - platform::errors::InvalidArgument("the dimensionof scale" + platform::errors::InvalidArgument("the dimension of scale" "must equal to 1. But received: " "the shape of scale is [%s], " - "the dimensionof scale is [%d]", + "the dimension of scale is [%d]", scale_dim, scale_dim.size())); PADDLE_ENFORCE_EQ( @@ -691,7 +691,7 @@ class DataNormGradKernel : public framework::OpKernel { } } } else { - // calculate data sum and squre sum + // calculate data sum and square sum Eigen::Array sample_sum(C); Eigen::Array sample_square_sum(C); // calculate data sample sum and square sum @@ -769,7 +769,7 @@ PD_REGISTER_STRUCT_KERNEL( REGISTER_OP_VERSION(data_norm).AddCheckpoint( R"ROC( - upgrad data_norm op by adding scale_w to support scale and shift.)ROC", + upgrade data_norm op by adding scale_w to support scale and shift.)ROC", paddle::framework::compatible::OpVersionDesc().NewInput( "scale_w", - "scale_w is used to do scale duirng data_norm like batchnorm ")); + "scale_w is used to do scale during data_norm like batchnorm ")); From f471aa136bdfc648707e99bb5e46c598761fe984 Mon Sep 17 00:00:00 2001 From: co63oc Date: Thu, 29 Feb 2024 23:37:56 +0800 Subject: [PATCH 195/282] Fix checkponit checkpoint, etc (#62168) --- paddle/fluid/operators/activation_op.cc | 10 +++++----- paddle/fluid/operators/activation_op.h | 2 +- paddle/fluid/operators/assign_value_op.h | 2 +- paddle/fluid/operators/attention_lstm_op.cc | 2 +- paddle/fluid/operators/batch_norm_op.cc | 6 +++--- paddle/fluid/operators/beam_search_decode_op_def.h | 2 +- paddle/fluid/operators/chunk_eval_op.h | 8 ++++---- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/paddle/fluid/operators/activation_op.cc b/paddle/fluid/operators/activation_op.cc index b848697128731..ddfbda809c1df 100644 --- a/paddle/fluid/operators/activation_op.cc +++ b/paddle/fluid/operators/activation_op.cc @@ -94,7 +94,7 @@ class ActivationGradOpMaker : public framework::SingleGradOpMaker { // paddle::Tensor dx = this->GetSingleInputGrad("X"); // auto* dx_ptr = this->GetOutputPtr(&dx); // std::string dx_name = this->GetOutputName(dx); -// VLOG(6) << "Runing hardswish_grad composite func"; +// VLOG(6) << "Running hardswish_grad composite func"; // prim::hardswish_grad(x, out_grad, dx_ptr); // this->RecoverOutputName(dx, dx_name); // } @@ -394,19 +394,19 @@ REGISTER_ACTIVATION_OP(mish, Mish, MishFunctor, MishGradFunctor); /* ========================== register checkpoint ===========================*/ REGISTER_OP_VERSION(leaky_relu) .AddCheckpoint( - R"ROC(fix leaky_relu, bahavior changed when alpha < 0 or alpha > 1)ROC", + R"ROC(fix leaky_relu, behavior changed when alpha < 0 or alpha > 1)ROC", paddle::framework::compatible::OpVersionDesc() .BugfixWithBehaviorChanged( - "leaky_relu calculate formula before checkponit: out = max(x, " + "leaky_relu calculate formula before checkpoint: out = max(x, " "alpha * x); after checkpoint: out = x if x > 0 else alpha * " "x")); REGISTER_OP_VERSION(hard_shrink) .AddCheckpoint( - R"ROC(fix hard_shrink, bahavior changed when threshold<0)ROC", + R"ROC(fix hard_shrink, behavior changed when threshold<0)ROC", paddle::framework::compatible::OpVersionDesc() .BugfixWithBehaviorChanged( - "hard_shrink calculate formula before checkponit: out = x * " + "hard_shrink calculate formula before checkpoint: out = x * " "((x < -threshold) + (x > threshold)); after checkpoint: out = " "x * (((x < -threshold) + (x > threshold)) > 0)")); diff --git a/paddle/fluid/operators/activation_op.h b/paddle/fluid/operators/activation_op.h index 8280c817b706a..38432f8768f59 100644 --- a/paddle/fluid/operators/activation_op.h +++ b/paddle/fluid/operators/activation_op.h @@ -371,7 +371,7 @@ struct AbsGradGradFunctor : public BaseActivationFunctor { // TODO(dengkaipeng): double gradient calculation for Square/Sqrt need // DOut(dy) as input(not output), tensor extraction is different from -// others. Impliment extraction kernel separately here. +// others. Implement extraction kernel separately here. inline void ExtractDoubleGradTensorWithInputDOut( const framework::ExecutionContext& ctx, const phi::DenseTensor** X, diff --git a/paddle/fluid/operators/assign_value_op.h b/paddle/fluid/operators/assign_value_op.h index 2a6a31ba03004..5ba8b9367e64e 100644 --- a/paddle/fluid/operators/assign_value_op.h +++ b/paddle/fluid/operators/assign_value_op.h @@ -29,7 +29,7 @@ typename std::enable_if::value>::type CopyVectorToTensor( const char* value_name, phi::DenseTensor* out, const framework::ExecutionContext& ctx) { - // phi::DenseTensore dtype is vector, it will be converted to + // phi::DenseTensor dtype is vector, it will be converted to // vector. // at the same time, we can not use vector to hold the value, because // the c++ use bit value to replace byte value. diff --git a/paddle/fluid/operators/attention_lstm_op.cc b/paddle/fluid/operators/attention_lstm_op.cc index 9624f752b780f..6a0775e6331a7 100644 --- a/paddle/fluid/operators/attention_lstm_op.cc +++ b/paddle/fluid/operators/attention_lstm_op.cc @@ -488,7 +488,7 @@ class AttentionLSTMKernel : public framework::OpKernel { // gate act: sigmoid act_gate(D3, lstm_out_data, lstm_out_data); - // candicate act: tanh + // candidate act: tanh act_cand(D, lstm_out_data + D3, lstm_out_data + D3); // a = forget * prev_cell diff --git a/paddle/fluid/operators/batch_norm_op.cc b/paddle/fluid/operators/batch_norm_op.cc index fd05b018bbfb6..996c6af070631 100644 --- a/paddle/fluid/operators/batch_norm_op.cc +++ b/paddle/fluid/operators/batch_norm_op.cc @@ -308,11 +308,11 @@ void BatchNormOpMaker::Make() { "to true or is_test true. the behavior is equivalent. " "In train mode, when setting use_global_stats True, the " "global mean and variance are also used during train time, " - "the BN acts as scaling and shiffting.") + "the BN acts as scaling and shifting.") .SetDefault(false); AddAttr("trainable_statistics", "(bool, default false) Whether to calculate mean and variance " - "in test mode. If setting true in test mode, mean and variace " + "in test mode. If setting true in test mode, mean and variance " "will be calculated by current batch statistics.") .SetDefault(false); AddComment(R"DOC( @@ -586,7 +586,7 @@ class BatchNormCompositeGradOpMaker : public prim::CompositeGradOpMakerBase { auto use_global_stats = this->Attr("use_global_stats"); auto trainable_statistics = this->Attr("trainable_statistics"); - VLOG(3) << "Runing batch_norm composite func"; + VLOG(3) << "Running batch_norm composite func"; prim::batch_norm_grad(x, scale, bias, diff --git a/paddle/fluid/operators/beam_search_decode_op_def.h b/paddle/fluid/operators/beam_search_decode_op_def.h index 390f728322322..d358d8255fcf3 100644 --- a/paddle/fluid/operators/beam_search_decode_op_def.h +++ b/paddle/fluid/operators/beam_search_decode_op_def.h @@ -27,7 +27,7 @@ using LoDTensorArray = framework::LoDTensorArray; // all the lod have 2 levels. // The first is source level, the second is sentence level. -// source level describe how many prefixes (branchs) for each source sentence +// source level describe how many prefixes (branches) for each source sentence // (beam). sentence level describe how these candidates belong to the prefixes. const size_t kSourceLevel = 0; const size_t kSentenceLevel = 1; diff --git a/paddle/fluid/operators/chunk_eval_op.h b/paddle/fluid/operators/chunk_eval_op.h index 22b3accba8639..baad8719db37f 100644 --- a/paddle/fluid/operators/chunk_eval_op.h +++ b/paddle/fluid/operators/chunk_eval_op.h @@ -199,7 +199,7 @@ class ChunkEvalKernel : public framework::OpKernel { const int64_t* inference_data = inference->data(); const int64_t* label_data = label->data(); T* precision_data = precision->mutable_data(place); - T* racall_data = recall->mutable_data(place); + T* recall_data = recall->mutable_data(place); T* f1_data = f1->mutable_data(place); int64_t* num_infer_chunks_data = num_infer_chunks->mutable_data(place); @@ -280,14 +280,14 @@ class ChunkEvalKernel : public framework::OpKernel { ? 0 : static_cast(*num_correct_chunks_data) / (*num_infer_chunks_data); - *racall_data = !(*num_label_chunks_data) + *recall_data = !(*num_label_chunks_data) ? 0 : static_cast(*num_correct_chunks_data) / (*num_label_chunks_data); *f1_data = !(*num_correct_chunks_data) ? 0 - : 2 * (*precision_data) * (*racall_data) / - ((*precision_data) + (*racall_data)); + : 2 * (*precision_data) * (*recall_data) / + ((*precision_data) + (*recall_data)); } void EvalOneSeq(const int64_t* output, From eee170a56f00db78c1fcc049798996fa75d5c2a7 Mon Sep 17 00:00:00 2001 From: co63oc Date: Thu, 29 Feb 2024 23:38:28 +0800 Subject: [PATCH 196/282] Fix cadidate candidate, etc (#62163) --- paddle/cinn/backends/codegen_c_test.cc | 6 +++--- paddle/cinn/ir/schedule/impl/base.cc | 2 +- .../cinn/ir/schedule/impl/compute_location.cc | 4 ++-- paddle/cinn/ir/schedule/ir_schedule_error.cc | 2 +- paddle/cinn/ir/schedule/ir_schedule_util.cc | 8 ++++---- paddle/cinn/ir/schedule/schedule_desc.cc | 12 ++++++------ paddle/cinn/ir/test/tensor_test.cc | 2 +- paddle/cinn/lang/lower_impl.h | 6 +++--- paddle/cinn/optim/insert_debug_log_callee.cc | 2 +- paddle/cinn/optim/unroll_loops.cc | 2 +- .../runtime/cuda/cuda_intrinsics_reduce.cc | 18 +++++++++--------- paddle/cinn/runtime/cuda/cuda_util.cc | 4 ++-- 12 files changed, 34 insertions(+), 34 deletions(-) diff --git a/paddle/cinn/backends/codegen_c_test.cc b/paddle/cinn/backends/codegen_c_test.cc index 91f80c190f0f8..61adad6ade461 100644 --- a/paddle/cinn/backends/codegen_c_test.cc +++ b/paddle/cinn/backends/codegen_c_test.cc @@ -61,9 +61,9 @@ TEST(CodeGenC, module) { LOG(INFO) << "C.body: " << C->get_compute_op()->body.front(); Target target; - target.arch = Target::Arch ::X86; - target.bits = Target::Bit ::k32; - target.os = Target::OS ::Linux; + target.arch = Target::Arch::X86; + target.bits = Target::Bit::k32; + target.os = Target::OS::Linux; Module::Builder builder("module1", target); ast_gen_ius::TensorGroup tensor_group({A, B, C}); diff --git a/paddle/cinn/ir/schedule/impl/base.cc b/paddle/cinn/ir/schedule/impl/base.cc index d27bcd451f508..61632dcf2452e 100644 --- a/paddle/cinn/ir/schedule/impl/base.cc +++ b/paddle/cinn/ir/schedule/impl/base.cc @@ -428,7 +428,7 @@ Expr DyScheduleImpl::SampleCategorical( std::string primitive = "SampleCategorical"; std::ostringstream os; if (candidates.size() != probs.size()) { - os << "vector params(candidates) and vector prama(probs) must " + os << "vector params(candidates) and vector params(probs) must " "have same size in SampleCategorical!\n"; throw IRScheduleErrorHandler(primitive, os.str(), module_expr_); } diff --git a/paddle/cinn/ir/schedule/impl/compute_location.cc b/paddle/cinn/ir/schedule/impl/compute_location.cc index a077039994e81..585257899968f 100644 --- a/paddle/cinn/ir/schedule/impl/compute_location.cc +++ b/paddle/cinn/ir/schedule/impl/compute_location.cc @@ -42,11 +42,11 @@ void DyScheduleImpl::ComputeAt(const Expr& block, std::string primitive = "ComputeAt"; std::ostringstream os; if (!block.As()) { - os << "Expr prama(block) should be a ScheduleBlockRealize!\n"; + os << "Expr param(block) should be a ScheduleBlockRealize!\n"; throw IRScheduleErrorHandler(primitive, os.str(), module_expr_); } if (!loop.As()) { - os << "Expr prama(loop) should be a For node!\n"; + os << "Expr param(loop) should be a For node!\n"; throw IRScheduleErrorHandler(primitive, os.str(), module_expr_); } Expr root = this->GetRootBlock(block); diff --git a/paddle/cinn/ir/schedule/ir_schedule_error.cc b/paddle/cinn/ir/schedule/ir_schedule_error.cc index 3467df28e5485..0b7a098264632 100644 --- a/paddle/cinn/ir/schedule/ir_schedule_error.cc +++ b/paddle/cinn/ir/schedule/ir_schedule_error.cc @@ -21,7 +21,7 @@ namespace ir { std::string IRScheduleErrorHandler::GeneralErrorMessage() const { std::ostringstream os; - os << "[IRScheduleError] An error occurred in the scheduel primitive < " + os << "[IRScheduleError] An error occurred in the schedule primitive < " << this->primitive_ << " >. " << std::endl; os << indent_str_ << "[Error info] " << this->err_msg_; return os.str(); diff --git a/paddle/cinn/ir/schedule/ir_schedule_util.cc b/paddle/cinn/ir/schedule/ir_schedule_util.cc index ba98382ebbf2f..739f17d06e80a 100644 --- a/paddle/cinn/ir/schedule/ir_schedule_util.cc +++ b/paddle/cinn/ir/schedule/ir_schedule_util.cc @@ -207,7 +207,7 @@ void ReplaceExpr(Expr* source, const std::vector& candidates) { CHECK_EQ(replaced.size(), candidates.size()) << "In ReplaceExpr, the size of Vars to be replaced must be equal to the " - "size of cadidate Exprs! Please check."; + "size of candidate Exprs! Please check."; if (replaced.empty()) return; std::map replacing_map; for (int i = 0; i < replaced.size(); ++i) { @@ -764,7 +764,7 @@ Expr ConstructNewLoopChain(const std::vector& chain, // } } // } } // - // We go throuph origin loop and check other body stmts, adding it as another + // We go through origin loop and check other body stmts, adding it as another // chain, such as: // // for (i, 0, 32) { @@ -1022,7 +1022,7 @@ void InsertBlock(Expr& for_loop, const Expr& insertion, int index) { // NOLINT auto dst_it = dst_block->stmts.begin() + index; if (dst_it->As()) { auto* inserted_block = dst_it->As()->true_case.As(); - CHECK(inserted_block) << "the IfThenElse node to be inserted shuold " + CHECK(inserted_block) << "the IfThenElse node to be inserted should " "contain a true_case block"; inserted_block->stmts.insert(inserted_block->stmts.begin(), insertion); } else { @@ -1060,7 +1060,7 @@ std::vector CalculateRequiredRegions( } std::vector required_buffer_range; - // deduce accessed regions of the provided tensor in block by itering each + // deduce accessed regions of the provided tensor in block by iterating each // required block for (const Expr& pro_node : provided_nodes) { std::string provided_tensor_name = diff --git a/paddle/cinn/ir/schedule/schedule_desc.cc b/paddle/cinn/ir/schedule/schedule_desc.cc index c9a26dfa1643d..b29d89fdd1dc9 100644 --- a/paddle/cinn/ir/schedule/schedule_desc.cc +++ b/paddle/cinn/ir/schedule/schedule_desc.cc @@ -27,7 +27,7 @@ namespace cinn { namespace ir { -// ------ Following codes are about `Apply` functions registry of variaous types +// ------ Following codes are about `Apply` functions registry of various types // of ScheduleDesc::Step class PackedStepContext; // uniformed function prototype of a scheduling operation in IRSchedule @@ -118,7 +118,7 @@ class PackedStepContext { return absl::get(attrs_.at(idx)); } catch (absl::bad_variant_access& ex) { LOG(FATAL) << "Attribute cast error, idx:" << idx - << ", get tpye:" << typeid(AttrType).name() + << ", get type:" << typeid(AttrType).name() << ", real index:" << attrs_.at(idx).index(); throw ex; } @@ -197,7 +197,7 @@ struct FreeFuncConverter { } }; -// used for formatting scheduling functions with variaous function signatures to +// used for formatting scheduling functions with various function signatures to // be uniformed form template struct ApplyFuncImpl; @@ -689,8 +689,8 @@ proto::ScheduleDesc ScheduleDesc::ToProto() const { } } - // each output Expr is represented by a formatted name, to be refered by - // suceeding steps + // each output Expr is represented by a formatted name, to be referred by + // succeeding steps for (auto&& expr : step.outputs) { std::string local_name = "e" + std::to_string(expr2name.size()); expr2name.emplace(expr, local_name); @@ -722,7 +722,7 @@ std::vector ScheduleDesc::ReplayWithProto( absl::flat_hash_map name2expr; std::vector last_outputs; - // resotre each scheduling step and apply to the new IRSchedule object + // restore each scheduling step and apply to the new IRSchedule object for (auto&& step_proto : desc_proto.steps()) { VLOG(4) << "Replay step:\n" << step_proto.DebugString(); ScheduleDesc::Step step; diff --git a/paddle/cinn/ir/test/tensor_test.cc b/paddle/cinn/ir/test/tensor_test.cc index cea1263f2aba3..4bf64f309735e 100644 --- a/paddle/cinn/ir/test/tensor_test.cc +++ b/paddle/cinn/ir/test/tensor_test.cc @@ -144,7 +144,7 @@ TEST(Tensor, ReshapeCopied) { stages->InsertLazily(B); - ir::Module::Builder builder("some_modue", cinn::common::DefaultHostTarget()); + ir::Module::Builder builder("some_module", cinn::common::DefaultHostTarget()); auto func = lang::Lower("fn", stages, {A, B}, {}, {}, &builder); backends::CodeGenC codegenc(cinn::common::DefaultHostTarget()); diff --git a/paddle/cinn/lang/lower_impl.h b/paddle/cinn/lang/lower_impl.h index b5f82ba7312e6..840fcfce860a0 100644 --- a/paddle/cinn/lang/lower_impl.h +++ b/paddle/cinn/lang/lower_impl.h @@ -150,8 +150,8 @@ class LowerImpl { std::vector CollectTemporaryTensors(); /** - * \brief Check both the tensor_args and sclar_args not contain duplication - * (different arguemnt with the same name). + * \brief Check both the tensor_args and scalar_args not contain duplication + * (different argument with the same name). */ void CheckArgsUnique(); @@ -304,7 +304,7 @@ struct MarkParallelMutator : public ir::IRMutator { auto it = parallels.find(tensor_n->name); if (it != parallels.end()) { for (int level : it->second) { - VLOG(1) << "Mark " << level << " Paralled"; + VLOG(1) << "Mark " << level << " Parallelled"; CHECK_LT(level, stack.size()); stack[level]->set_parallel(); } diff --git a/paddle/cinn/optim/insert_debug_log_callee.cc b/paddle/cinn/optim/insert_debug_log_callee.cc index fdab377bc88cc..1bcfd34bbaf9c 100644 --- a/paddle/cinn/optim/insert_debug_log_callee.cc +++ b/paddle/cinn/optim/insert_debug_log_callee.cc @@ -139,7 +139,7 @@ struct InsertDebugLogCalleeMutator : public ir::IRMutator<> { ir::IRMutator<>::Visit(&node->body, &node->body); auto deal_with_exprs = - [&](std::vector *exprs) { // deal with op->argument_preapre_exprs + [&](std::vector *exprs) { // deal with op->argument_prepare_exprs std::vector new_stmts; for (auto &expr : *exprs) { auto msg = diff --git a/paddle/cinn/optim/unroll_loops.cc b/paddle/cinn/optim/unroll_loops.cc index 9f2e8bf244e4c..7fa5e3a8b8222 100644 --- a/paddle/cinn/optim/unroll_loops.cc +++ b/paddle/cinn/optim/unroll_loops.cc @@ -62,7 +62,7 @@ struct UnrollMutator : public ir::IRMutator { void Visit(const ir::For* op, Expr* expr) override { IRMutator<>::Visit(op, expr); if (op->extent.As() == nullptr) { - VLOG(5) << "loop to be unrolled should have a contant extent"; + VLOG(5) << "loop to be unrolled should have a constant extent"; return; } int64_t extent = op->extent.as_int64(); diff --git a/paddle/cinn/runtime/cuda/cuda_intrinsics_reduce.cc b/paddle/cinn/runtime/cuda/cuda_intrinsics_reduce.cc index 15fcb4030e89b..685c466f7f9c9 100644 --- a/paddle/cinn/runtime/cuda/cuda_intrinsics_reduce.cc +++ b/paddle/cinn/runtime/cuda/cuda_intrinsics_reduce.cc @@ -146,22 +146,22 @@ CINN_REGISTER_HELPER(cuda_intrinsics_reduce) { #undef REGISTER_BLOCK_REDUCE_FUNC_IMPL -#define REGISTER_BLOCK_SHUFLLE_FUNC_IMPL(REDUCE_TYPE, DTYPE) \ +#define REGISTER_BLOCK_SHUFFLE_FUNC_IMPL(REDUCE_TYPE, DTYPE) \ REGISTER_FACKED_EXTERN_FUNC_HELPER(block_shuffle_##REDUCE_TYPE, target) \ .SetRetType() \ .AddInputType() \ .AddInputType() \ .End(); - EXPAND_REDUCE_INT32_REGISTER_MARCO(REGISTER_BLOCK_SHUFLLE_FUNC_IMPL) - EXPAND_REDUCE_INT64_REGISTER_MARCO(REGISTER_BLOCK_SHUFLLE_FUNC_IMPL) - EXPAND_REDUCE_BF16_REGISTER_MACRO(REGISTER_BLOCK_SHUFLLE_FUNC_IMPL) - EXPAND_REDUCE_FP16_REGISTER_MACRO(REGISTER_BLOCK_SHUFLLE_FUNC_IMPL) - EXPAND_REDUCE_FP32_REGISTER_MACRO(REGISTER_BLOCK_SHUFLLE_FUNC_IMPL) - EXPAND_REDUCE_FP64_REGISTER_MACRO(REGISTER_BLOCK_SHUFLLE_FUNC_IMPL) - EXPAND_REDUCE_BOOL_REGISTER_MACRO(REGISTER_BLOCK_SHUFLLE_FUNC_IMPL) + EXPAND_REDUCE_INT32_REGISTER_MARCO(REGISTER_BLOCK_SHUFFLE_FUNC_IMPL) + EXPAND_REDUCE_INT64_REGISTER_MARCO(REGISTER_BLOCK_SHUFFLE_FUNC_IMPL) + EXPAND_REDUCE_BF16_REGISTER_MACRO(REGISTER_BLOCK_SHUFFLE_FUNC_IMPL) + EXPAND_REDUCE_FP16_REGISTER_MACRO(REGISTER_BLOCK_SHUFFLE_FUNC_IMPL) + EXPAND_REDUCE_FP32_REGISTER_MACRO(REGISTER_BLOCK_SHUFFLE_FUNC_IMPL) + EXPAND_REDUCE_FP64_REGISTER_MACRO(REGISTER_BLOCK_SHUFFLE_FUNC_IMPL) + EXPAND_REDUCE_BOOL_REGISTER_MACRO(REGISTER_BLOCK_SHUFFLE_FUNC_IMPL) -#undef REGISTER_BLOCK_SHUFLLE_FUNC_IMPL +#undef REGISTER_BLOCK_SHUFFLE_FUNC_IMPL #undef EXPAND_REDUCE_INT32_REGISTER_MARCO #undef EXPAND_REDUCE_INT64_REGISTER_MARCO diff --git a/paddle/cinn/runtime/cuda/cuda_util.cc b/paddle/cinn/runtime/cuda/cuda_util.cc index 18c277339ddaf..074c35f1ce9f9 100644 --- a/paddle/cinn/runtime/cuda/cuda_util.cc +++ b/paddle/cinn/runtime/cuda/cuda_util.cc @@ -481,7 +481,7 @@ void cinn_call_batched_cublas(void *v_args, void *B = args[1 + g].operator cinn_buffer_t *()->memory; void *C = args[1 + num_gemm + g].operator cinn_buffer_t *()->memory; - // if opside is 1, exhange A,B. + // if opside is 1, exchange A,B. if (opside) { auto tmp = A; A = B; @@ -703,7 +703,7 @@ std::string debug_cudnn_pool_mode(cudnnPoolingMode_t pool_mode) { case CUDNN_POOLING_AVERAGE_COUNT_INCLUDE_PADDING: return "avg_include_padding"; case CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING: - return "avg_exclulude_padding"; + return "avg_exclude_padding"; default: LOG(FATAL) << "Pool only support max and avg now!"; } From 2e3ea49e96823816af152e7480cf98b662c3b708 Mon Sep 17 00:00:00 2001 From: co63oc Date: Thu, 29 Feb 2024 23:39:27 +0800 Subject: [PATCH 197/282] Fix with_mateclass with_metaclass, etc (#62162) * Fix * ci * Fix --- python/paddle/amp/auto_cast.py | 6 +-- python/paddle/amp/debugging.py | 4 +- python/paddle/autograd/py_layer.py | 4 +- .../base/dygraph/tensor_patch_methods.py | 8 ++-- .../incubate/checkpoint/auto_checkpoint.py | 4 +- python/paddle/base/layers/io.py | 4 +- .../base/layers/layer_function_generator.py | 4 +- python/paddle/base/reader.py | 4 +- python/paddle/hapi/model.py | 46 +++++++++---------- .../incubate/asp/supported_layer_list.py | 14 +++--- python/paddle/incubate/asp/utils.py | 38 +++++++-------- python/paddle/incubate/autograd/primapi.py | 8 ++-- python/paddle/incubate/autotune.py | 8 ++-- .../distribute_transpiler/__init__.py | 6 +-- .../transformers/decorator_transformer.py | 20 ++++---- .../transformers/tensorhook_transformer.py | 4 +- python/paddle/jit/dy2static/utils.py | 10 ++-- python/paddle/jit/sot/symbolic/export.py | 10 ++-- python/paddle/tensor/math.py | 2 +- .../utils/cpp_extension/cpp_extension.py | 6 +-- 20 files changed, 106 insertions(+), 104 deletions(-) diff --git a/python/paddle/amp/auto_cast.py b/python/paddle/amp/auto_cast.py index 0286a668d10f5..5a271171e09ce 100644 --- a/python/paddle/amp/auto_cast.py +++ b/python/paddle/amp/auto_cast.py @@ -53,7 +53,7 @@ def __init__(self): self.model_parameters = [] self.use_master_grad = False self.already_register_final_backward_hook = False - self.already_classify_params_meshs = False # For dist + self.already_classify_params_meshes = False # For dist self.mesh2params = {} # For dist self.amp_dtype = 'float32' @@ -471,7 +471,7 @@ def master_grad_hook(): # NOTE(lizhiyu): To support semi-auto of dygraph mode, we must # classify the params of model into different calsses according to their process_mesh. # Otherwise, fault will occur. - if not amp_global_state().already_classify_params_meshs: + if not amp_global_state().already_classify_params_meshes: for param in amp_global_state().model_parameters: if param is not None and param.process_mesh is not None: if ( @@ -485,7 +485,7 @@ def master_grad_hook(): amp_global_state().mesh2params[ param.process_mesh ].append(param) - amp_global_state().already_classify_params_meshs = True + amp_global_state().already_classify_params_meshes = True if len(amp_global_state().mesh2params): for _, params in amp_global_state().mesh2params.items(): diff --git a/python/paddle/amp/debugging.py b/python/paddle/amp/debugging.py index 0fd8fce8fe5f8..974daa0a90697 100644 --- a/python/paddle/amp/debugging.py +++ b/python/paddle/amp/debugging.py @@ -270,7 +270,7 @@ def _set_seed(self, flag): self.seed = self.initial_seed if self.seed > np.iinfo(np.uint32).max or self.seed < 0: - print("[Warnning: Seed must be between 0 and 2**32 - 1") + print("[Warning: Seed must be between 0 and 2**32 - 1") self.seed = 123 # get random seed @@ -616,7 +616,7 @@ def compare_accuracy( ... [1, 5, 2, 0], dtype="float32" ... ) ... z1 = x + y - ... out_excel = "compary_accuracy_out_excel.csv" + ... out_excel = "compare_accuracy_out_excel.csv" ... paddle.amp.debugging.compare_accuracy( ... path, path, out_excel, loss_scale=1, dump_all_tensors=False ... ) diff --git a/python/paddle/autograd/py_layer.py b/python/paddle/autograd/py_layer.py index 5ddf610bb032b..2843560f4a878 100644 --- a/python/paddle/autograd/py_layer.py +++ b/python/paddle/autograd/py_layer.py @@ -18,7 +18,7 @@ __all__ = [] -def with_mateclass(meta, *bases): +def with_metaclass(meta, *bases): class impl(meta): def __new__(cls, name, temp_bases, attrs): return meta(name, bases, attrs) @@ -267,7 +267,7 @@ def __init__(cls, name, bases, attrs): return super().__init__(name, bases, attrs) -class PyLayer(with_mateclass(PyLayerMeta, core.eager.PyLayer, PyLayerContext)): +class PyLayer(with_metaclass(PyLayerMeta, core.eager.PyLayer, PyLayerContext)): """ Paddle implements Python custom operators on the PaddlePaddle framework by creating a subclass of ``PyLayer``, which must comply with the following rules: diff --git a/python/paddle/base/dygraph/tensor_patch_methods.py b/python/paddle/base/dygraph/tensor_patch_methods.py index 7c7a3d60ebf45..275ab3a232d96 100644 --- a/python/paddle/base/dygraph/tensor_patch_methods.py +++ b/python/paddle/base/dygraph/tensor_patch_methods.py @@ -104,7 +104,7 @@ def _to_static_var(self, to_parameter=False, **kwargs): """ # Note: getattr(self, attr, None) will call x.grad=x.gradient(), but gradient() only available in dygraph. - # It will fail. So, for propery that different between dynamic and static graph, should not getattr(self, attr, None). + # It will fail. So, for property that different between dynamic and static graph, should not getattr(self, attr, None). attr_not_need_keys = [ 'grad', 'T', @@ -227,7 +227,7 @@ def set_value(self, value): # NOTE(wuweilong): self could be Tensor, the subsequent behavior are defined in different files # if self is Tensor, method value() return self that defined in this file, get_tensor() defined in eager_method.cc - # this Interface behavior will be unifed in the future. + # this Interface behavior will be unified in the future. if self.is_dist(): if isinstance(value, paddle.Tensor) and value.is_dist(): from paddle.distributed.auto_parallel.placement_type import ( @@ -702,7 +702,7 @@ def get_device_dtype_from_tensor(other): if size_args + size_kwargs > 3 or size_args + size_kwargs == 0: raise TypeError( - "to() received too mant arguments - expected one of:\n \ + "to() received too many arguments - expected one of:\n \ * (Union[str, paddle.CPUPlace(), paddle.CUDAPlace(), paddle.CUDAPinnedPlace(), paddle.XPUPlace(), paddle.CustomPlace()] \ device, Union[str, paddle.dtype, numpy.dtype] dtype, bool blocking)\n \ * (Union[str, paddle.dtype, numpy.dtype] dtype, bool blocking)\n \ @@ -976,7 +976,7 @@ def __array__(self, dtype=None): return array def pre_deal_index(self, item): - # since in pybind there is no effiency way to transfer Py_Tuple/Py_List/Py_Range to Tensor + # since in pybind there is no efficiency way to transfer Py_Tuple/Py_List/Py_Range to Tensor # we call this function in python level. item = list(item) if isinstance(item, tuple) else [item] for i, slice_item in enumerate(item): diff --git a/python/paddle/base/incubate/checkpoint/auto_checkpoint.py b/python/paddle/base/incubate/checkpoint/auto_checkpoint.py index 742289acd27f1..329cdc25ab083 100644 --- a/python/paddle/base/incubate/checkpoint/auto_checkpoint.py +++ b/python/paddle/base/incubate/checkpoint/auto_checkpoint.py @@ -419,7 +419,7 @@ def _serialize(self, pop_keys=["restored_from", "checkpoint_epoch_no"]): for k in pop_keys: d.pop(k, None) - # registerd exes + # registered exes d["exe_status"] = {} e = d["exe_status"] for k, t in self._exe_status.items(): @@ -625,7 +625,7 @@ def train_epoch_range(max_epoch_num, save_checkpoint_inter=None): global g_acp_type if not _get_checker().valid(): logger.warning( - "auto checkpoint will take effect automaticly on PaddleCloud" + "auto checkpoint will take effect automatically on PaddleCloud" ) for i in _normal_yield(max_epoch_num): yield i diff --git a/python/paddle/base/layers/io.py b/python/paddle/base/layers/io.py index 51f5b10fe0618..de9725ec28fac 100644 --- a/python/paddle/base/layers/io.py +++ b/python/paddle/base/layers/io.py @@ -74,7 +74,7 @@ def __create_shared_decorated_reader__(op_type, reader, attrs): var_name = unique_name(op_type) startup_blk = default_startup_program().current_block() startup_var = startup_blk.create_var(name=var_name) - startop_op = startup_blk.append_op( + startup_op = startup_blk.append_op( type=op_type, inputs={'UnderlyingReader': reader}, outputs={'Out': [startup_var]}, @@ -83,7 +83,7 @@ def __create_shared_decorated_reader__(op_type, reader, attrs): startup_var.persistable = True main_prog_block = default_main_program().current_block() main_prog_var = _copy_reader_var_(main_prog_block, startup_var) - _copy_reader_create_op_(main_prog_block, startop_op) + _copy_reader_create_op_(main_prog_block, startup_op) return monkey_patch_reader_methods(main_prog_var) diff --git a/python/paddle/base/layers/layer_function_generator.py b/python/paddle/base/layers/layer_function_generator.py index 009cb2ae49a6b..a8128603e05cd 100644 --- a/python/paddle/base/layers/layer_function_generator.py +++ b/python/paddle/base/layers/layer_function_generator.py @@ -86,7 +86,7 @@ def _generate_doc_string_( buf.write(" (Tensor): ") buf.write(escape_math(each_input.comment)) if each_input.duplicable: - buf.write(" Duplicatable.") + buf.write(" Duplicable.") if each_input.dispensable: buf.write(" Optional.") buf.write('\n') @@ -327,7 +327,7 @@ def func(x, name=None): and x.is_view_var ): raise ValueError( - 'Sorry about what\'s happend. In to_static mode, {}\'s output variable {} is a viewed Tensor in dygraph. This will result in inconsistent calculation behavior between dynamic and static graphs. You must find the location of the strided API be called, and call {} = {}.assign().'.format( + 'Sorry about what\'s happened. In to_static mode, {}\'s output variable {} is a viewed Tensor in dygraph. This will result in inconsistent calculation behavior between dynamic and static graphs. You must find the location of the strided API be called, and call {} = {}.assign().'.format( inplace_op_type, x.name, x.name, x.nameb ) ) diff --git a/python/paddle/base/reader.py b/python/paddle/base/reader.py index e90378249da03..d5695aec5b220 100644 --- a/python/paddle/base/reader.py +++ b/python/paddle/base/reader.py @@ -137,7 +137,7 @@ def _check_input_array(cls, item): arr = np.asarray(item) if arr.dtype == np.object_: raise TypeError( - "\n\tFaild to convert input data to a regular ndarray :\n\t* Usually " + "\n\tFailed to convert input data to a regular ndarray :\n\t* Usually " "this means the input data contains nested lists with different lengths. " "\n\t* Check the reader function passed to 'decorate_batch_generator'" " to locate the data causes this issue.\n\t* Please consider using " @@ -532,7 +532,7 @@ def __init__( # NOTE: the C++ LoDTensorBlockingQueue instance self._blocking_queue = None # NOTE: 1. In multiprocess mode, this thread is used to get next batch data from - # self._data_queue, then push it into self._blocking_queue; 2. In singleprocess + # self._data_queue, then push it into self._blocking_queue; 2. In single process # mode, this thread is used to get next batch data from self._batch_reader, then # push it into self._blocking_queue self._thread = None diff --git a/python/paddle/hapi/model.py b/python/paddle/hapi/model.py index 7618590b376b7..328f3e0078052 100644 --- a/python/paddle/hapi/model.py +++ b/python/paddle/hapi/model.py @@ -293,7 +293,7 @@ def _update_input_info(inputs): class StaticGraphAdapter: """ - Model traning/inference with a static graph. + Model training/inference with a static graph. """ @@ -633,7 +633,7 @@ def _make_program(self, mode): prog = self._orig_prog.clone() # NOTE: When defining learning rate scheduling in static-graph, ops to # increase the global step var and calculate learning rate would be - # prepended into _orig_prog. test program maked by `_orig_prog.clone` + # prepended into _orig_prog. test program marked by `_orig_prog.clone` # also would include these ops. Thus must prune these ops in test # program, otherwise the global step would be changed in test. if mode != 'train': @@ -794,16 +794,16 @@ def __init__(self, model): if self._nranks > 1: dist.init_parallel_env() - stradegy = paddle.distributed.parallel.ParallelStrategy() - stradegy.nranks = paddle.distributed.ParallelEnv().nranks - stradegy.local_rank = paddle.distributed.ParallelEnv().local_rank - stradegy.trainer_endpoints = ( + strategy = paddle.distributed.parallel.ParallelStrategy() + strategy.nranks = paddle.distributed.ParallelEnv().nranks + strategy.local_rank = paddle.distributed.ParallelEnv().local_rank + strategy.trainer_endpoints = ( paddle.distributed.ParallelEnv().trainer_endpoints ) - stradegy.current_endpoint = ( + strategy.current_endpoint = ( paddle.distributed.ParallelEnv().current_endpoint ) - self.ddp_model = paddle.DataParallel(self.model.network, stradegy) + self.ddp_model = paddle.DataParallel(self.model.network, strategy) @property def mode(self): @@ -879,7 +879,7 @@ def eval_batch(self, inputs, labels=None): outputs = self.model.network(*[paddle.to_tensor(x) for x in inputs]) - # Transfrom data to expected device + # Transform data to expected device expected_device = paddle.device.get_device() for o in to_list(outputs): o._to(device=expected_device) @@ -966,7 +966,7 @@ def load(self, param_state_pairs, optim_state, scaler_state=None): if scaler_state: self.model._scaler.load_state_dict(scaler_state) - # resotre optimizer states + # restore optimizer states if not self.model._optimizer or not optim_state: return @@ -1077,7 +1077,7 @@ class Model: or dict ({name: InputSpec}), and it couldn't be None in static graph. Default: None. labels (InputSpec|list|tuple|None, optional): `labels`, entry points of network, - could be a InputSpec instnace or list/tuple of InputSpec instances, + could be a InputSpec instance or list/tuple of InputSpec instances, or None. For static graph, if labels is required in loss, labels must be set. Otherwise, it could be None. Default: None. @@ -1676,7 +1676,7 @@ def prepare( ): """ - Configures the model before runing. + Configures the model before running. Args: optimizer (Optimizer|None, optional): Optimizer must be set in training @@ -1777,16 +1777,16 @@ def fit( Args: train_data (Dataset|DataLoader, optional): An iterable data loader is used for train. An instance of paddle paddle.io.Dataset or - paddle.io.Dataloader is recomended. Default: None. + paddle.io.Dataloader is recommended. Default: None. eval_data (Dataset|DataLoader, optional): An iterable data loader is used for evaluation at the end of epoch. If None, will not do evaluation. An instance of paddle.io.Dataset or paddle.io.Dataloader - is recomended. Default: None. + is recommended. Default: None. batch_size (int|list, optional): The batch size of train_data and eval_data. When train_data and eval_data are both the instance of Dataloader, this parameter will be ignored. Default: 1. epochs (int, optional): The number of epochs to train the model. Default: 1. - eval_freq (int, optional): The frequency, in number of epochs, an evalutation + eval_freq (int, optional): The frequency, in number of epochs, an evaluation is performed. Default: 1. log_freq (int, optional): The frequency, in number of steps, the training logs are printed. Default: 10. @@ -1800,7 +1800,7 @@ def fit( train_data when dataset size is not divisible by the batch size. When train_data is an instance of Dataloader, this parameter will be ignored. Default: False. - shuffle (bool, optional): Whther to shuffle train_data. When train_data is + shuffle (bool, optional): Whether to shuffle train_data. When train_data is an instance of Dataloader, this parameter will be ignored. Default: True. num_workers (int, optional): The number of subprocess to load data, 0 for no @@ -1810,7 +1810,7 @@ def fit( callbacks (Callback|None, optional): A list of `Callback` instances to apply during training. If None, :ref:`api_paddle_callbacks_ProgBarLogger` and :ref:`api_paddle_callbacks_ModelCheckpoint` are automatically inserted. Default: None. - accumulate_grad_batches (int, optional): The number of batches to accumulate gradident + accumulate_grad_batches (int, optional): The number of batches to accumulate gradient during training process before optimizer updates. It can mimic large batch size. Default: 1. num_iters (int|None, optional): The number of iterations to evaluate the model. @@ -2016,7 +2016,7 @@ def evaluate( Args: eval_data (Dataset|DataLoader): An iterable data loader is used for evaluation. An instance of paddle.io.Dataset or - paddle.io.Dataloader is recomended. + paddle.io.Dataloader is recommended. batch_size (int, optional): The batch size of train_data and eval_data. When eval_data is the instance of Dataloader, this argument will be ignored. Default: 1. @@ -2126,7 +2126,7 @@ def predict( Args: test_data (Dataset|DataLoader): An iterable data loader is used for predict. An instance of paddle.io.Dataset or paddle.io.Dataloader - is recomended. + is recommended. batch_size (int, optional): The batch size of test_data. When test_data is the instance of Dataloader, this argument will be ignored. Default: 1. num_workers (int, optional): The number of subprocess to load data, 0 for no subprocess @@ -2300,13 +2300,13 @@ def _run_one_epoch( # Data might come from different types of data_loader and have # different format, as following: # 1. DataLoader in static graph: - # [[input1, input2, ..., label1, lable2, ...]] + # [[input1, input2, ..., label1, label2, ...]] # 2. DataLoader in dygraph - # [input1, input2, ..., label1, lable2, ...] + # [input1, input2, ..., label1, label2, ...] # 3. custumed iterator yield concated inputs and labels: - # [input1, input2, ..., label1, lable2, ...] + # [input1, input2, ..., label1, label2, ...] # 4. custumed iterator yield separated inputs and labels: - # ([input1, input2, ...], [label1, lable2, ...]) + # ([input1, input2, ...], [label1, label2, ...]) # To handle all of these, flatten (nested) list to list. data = paddle.utils.flatten(data) # LoDTensor.shape is callable, where LoDTensor comes from diff --git a/python/paddle/incubate/asp/supported_layer_list.py b/python/paddle/incubate/asp/supported_layer_list.py index 0ebc6ea2d3128..7720a1cf7127c 100644 --- a/python/paddle/incubate/asp/supported_layer_list.py +++ b/python/paddle/incubate/asp/supported_layer_list.py @@ -35,16 +35,16 @@ def _default_pruning(weight_nparray, m, n, func_name, param_name): shape = weight_nparray.shape weight_pruned_nparray = copy.deepcopy(weight_nparray) weight_sparse_mask = np.ones_like(weight_pruned_nparray) - exlude_cond_shape2 = len(shape) == 2 and shape[0] < m - exlude_cond_shape4 = len(shape) == 4 and shape[1] < m - if exlude_cond_shape2: + exclude_cond_shape2 = len(shape) == 2 and shape[0] < m + exclude_cond_shape4 = len(shape) == 4 and shape[1] < m + if exclude_cond_shape2: _logger.warning( '{} is not pruned because the first dimension of {} is smaller than {}'.format( param_name, shape, m ) ) return weight_pruned_nparray, weight_sparse_mask - if exlude_cond_shape4: + if exclude_cond_shape4: _logger.warning( '{} is not pruned because the second dimension of {} is smaller than {}'.format( param_name, shape, m @@ -58,12 +58,12 @@ def _default_pruning(weight_nparray, m, n, func_name, param_name): # SPMMA in cuSparseLt: D = (AxB) + C, where matrix A (mxk) is sparse matrix. # cuSparseLt would prune matrix A along k dimension. # In sparse training, layer weight matrices is viewed sparse matrix A, so - # the math fomula should be 'Act(WX + b)'. However, default fomula in PaddlePaddle + # the math formula should be 'Act(WX + b)'. However, default formula in PaddlePaddle # is 'Act(XW + b)'. For enabling SPMMA, weights and inputs should be transposed # for computing, Act( (W^T X^T)^T + b). Therefore, we have to prune alog k dimension - # of W^T, which is m dimension of W. Moreove, all mask generating functions in + # of W^T, which is m dimension of W. Moreover, all mask generating functions in # asp/utils is row-major pruning. That is the reason we have to transpose weight - # matrices beforce invoking create_mask. Then we transpose the result mask to make + # matrices before invoking create_mask. Then we transpose the result mask to make # sure its shape to be the same as the input weight. weight_sparse_mask = asp.create_mask( weight_nparray.T, func_name=func_name, n=n, m=m diff --git a/python/paddle/incubate/asp/utils.py b/python/paddle/incubate/asp/utils.py index 4ed8d7e74d56e..f8918a5ed0ced 100644 --- a/python/paddle/incubate/asp/utils.py +++ b/python/paddle/incubate/asp/utils.py @@ -171,11 +171,11 @@ def check_mask_1d(mat, n, m): True """ if len(mat.shape) <= 1: - mat_flattern, shape = _reshape_1d(mat.reshape(1, mat.shape[0]), m) + mat_flatten, shape = _reshape_1d(mat.reshape(1, mat.shape[0]), m) else: - mat_flattern, shape = _reshape_1d(mat, m) + mat_flatten, shape = _reshape_1d(mat, m) - for sub_mat in mat_flattern: + for sub_mat in mat_flatten: if np.nonzero(sub_mat)[0].size > (m - n): return False return True @@ -210,12 +210,12 @@ def get_mask_1d(mat, n, m): >>> print(y) True """ - mat_flattern, shape = _reshape_1d(mat, m) + mat_flatten, shape = _reshape_1d(mat, m) - mask_flattern = np.ones_like(mat_flattern) + mask_flattern = np.ones_like(mat_flatten) mask = np.ones_like(mat) - for i in range(mat_flattern.shape[0]): - sub_mat = mat_flattern[i] + for i in range(mat_flatten.shape[0]): + sub_mat = mat_flatten[i] min_order_indices = np.argsort(np.absolute(sub_mat)) mask_flattern[i, min_order_indices[:n].tolist()] = 0 mask_flattern = mask_flattern.reshape(shape) @@ -252,7 +252,7 @@ def _reshape_2d(mat, m): mat_padded = np.zeros(new_shape) mat_padded[: mat.shape[0], : mat.shape[1]] = mat - mat_flattern = np.empty(new_shape).reshape(-1, m * m) + mat_flatten = np.empty(new_shape).reshape(-1, m * m) curr_idx = 0 for row_start in range(0, mat_padded.shape[0], m): row_end = row_start + m @@ -261,9 +261,9 @@ def _reshape_2d(mat, m): sub_mat = np.squeeze( mat_padded[row_start:row_end, col_start:col_end].reshape(-1) ) - mat_flattern[curr_idx] = sub_mat + mat_flatten[curr_idx] = sub_mat curr_idx += 1 - return mat_flattern, mat_padded.shape + return mat_flatten, mat_padded.shape def check_mask_2d(mat, n, m): @@ -400,7 +400,7 @@ def get_mask_2d_greedy(mat, n, m): def _compute_valid_2d_patterns(n, m): r""" - Compute all vaild 2D `n:m` sparse patterns. + Compute all valid 2D `n:m` sparse patterns. 2D `n:m` sparse pattern: At least :math:`n \times n` zeros in every :math:`m \times m` block under the constraint of at least :attr:`n` zeros for each row and column. @@ -409,7 +409,7 @@ def _compute_valid_2d_patterns(n, m): n (int): n of `n:m` sparse pattern. m (int): m of `n:m` sparse pattern. Returns: - dictionary: A dictionary with key: *m_n* (string) and value: all vaild 2D `n:m` sparse patterns. + dictionary: A dictionary with key: *m_n* (string) and value: all valid 2D `n:m` sparse patterns. """ global _valid_2d_patterns_lock global _valid_2d_patterns @@ -442,7 +442,7 @@ def _compute_valid_2d_patterns(n, m): def get_mask_2d_best(mat, n, m): r""" Generate 2D `n:m` sparse pattern mask of the input matrix :attr:`mat` - to form sparse matrix with maximun L1 norm .This function would pad each + to form sparse matrix with maximum L1 norm .This function would pad each dimension of :attr:`mat` by zero to be a multiples of :attr:`m` before mask generation. 2D `n:m` sparse pattern: At least :math:`n \times n` zeros in every :math:`m \times m` block @@ -475,10 +475,10 @@ def get_mask_2d_best(mat, n, m): """ patterns = _compute_valid_2d_patterns(n, m) - mat_flattern, shape = _reshape_2d(mat, m) - mask_flattern = np.ones_like(mat_flattern).reshape(-1, m, m) + mat_flatten, shape = _reshape_2d(mat, m) + mask_flattern = np.ones_like(mat_flatten).reshape(-1, m, m) pmax = np.argmax( - np.matmul(mat_flattern, patterns.reshape(patterns.shape[0], m * m).T), + np.matmul(mat_flatten, patterns.reshape(patterns.shape[0], m * m).T), axis=1, ) @@ -502,7 +502,7 @@ def create_mask(tensor, func_name=MaskAlgo.MASK_1D, n=2, m=4): Args: tensor (nparray): The input tensor. - func_name (MaskAlgo, optional): The function name to generate spase mask. Default is `MaskAlgo.MASK_1D`. All options please refer to `MaskAlgo`. + func_name (MaskAlgo, optional): The function name to generate sparse mask. Default is `MaskAlgo.MASK_1D`. All options please refer to `MaskAlgo`. n (int, optional): n of `n:m` sparse pattern. Default is 2. m (int, optional): m of `n:m` sparse pattern. Default is 4. Returns: @@ -573,7 +573,7 @@ def check_sparsity(tensor, func_name=CheckMethod.CHECK_1D, n=2, m=4): Args: tensor (nparray): The input tensor. - func_name (CheckMethod, optional): The function name to generate spase mask. Default is `CheckMethod.CHECK_1D`. All options please refer to `CheckMethod`. + func_name (CheckMethod, optional): The function name to generate sparse mask. Default is `CheckMethod.CHECK_1D`. All options please refer to `CheckMethod`. n (int, optional): n of `n:m` sparse pattern. Default is 2. m (int, optional): m of `n:m` sparse pattern. Default is 4. Returns: @@ -605,7 +605,7 @@ def check_sparsity(tensor, func_name=CheckMethod.CHECK_1D, n=2, m=4): t = tensor.astype(float) assert type(func_name) == CheckMethod, ( - "func_name argumet of check_sparsity is only accepted as type CheckMethod. " + "func_name argument of check_sparsity is only accepted as type CheckMethod. " f"But got {type(func_name)}" ) func = getattr(sys.modules[__name__], func_name.value, None) diff --git a/python/paddle/incubate/autograd/primapi.py b/python/paddle/incubate/autograd/primapi.py index 9f62d1f5835c7..d0c7d41ef194d 100644 --- a/python/paddle/incubate/autograd/primapi.py +++ b/python/paddle/incubate/autograd/primapi.py @@ -74,13 +74,13 @@ def forward_grad(outputs, inputs, grad_inputs=None): if not isinstance(outputs, (framework.Variable, typing.Sequence)): raise TypeError( - f'Expected outputs is Tensor|Sequence[Tesnor], ' + f'Expected outputs is Tensor|Sequence[Tensor], ' f'but got {type(outputs)}.' ) if not isinstance(inputs, (framework.Variable, typing.Sequence)): raise TypeError( - f'Expected inputs is Tensor|Sequence[Tesnor], ' + f'Expected inputs is Tensor|Sequence[Tensor], ' f'but got {type(inputs)}.' ) @@ -165,13 +165,13 @@ def grad(outputs, inputs, grad_outputs=None): if not isinstance(outputs, (framework.Variable, typing.Sequence)): raise TypeError( - f'Expected outputs is Tensor|Sequence[Tesnor], ' + f'Expected outputs is Tensor|Sequence[Tensor], ' f'but got {type(outputs)}.' ) if not isinstance(inputs, (framework.Variable, typing.Sequence)): raise TypeError( - f'Expected inputs is Tensor|Sequence[Tesnor], ' + f'Expected inputs is Tensor|Sequence[Tensor], ' f'but got {type(inputs)}.' ) diff --git a/python/paddle/incubate/autotune.py b/python/paddle/incubate/autotune.py index 745ac9fc69c07..c99b3498946c4 100644 --- a/python/paddle/incubate/autotune.py +++ b/python/paddle/incubate/autotune.py @@ -136,10 +136,10 @@ def set_config(config=None): ) if "dataloader" in config_dict: dataloader_config = config_dict["dataloader"] - use_autoune = False + use_autotune = False if "enable" in dataloader_config: if isinstance(dataloader_config['enable'], bool): - use_autoune = dataloader_config['enable'] + use_autotune = dataloader_config['enable'] else: warnings.warn( "The auto-tuning configuration of the dataloader is incorrect." @@ -148,11 +148,11 @@ def set_config(config=None): if "tuning_steps" in dataloader_config: if isinstance(dataloader_config['tuning_steps'], int): paddle.io.reader.set_autotune_config( - use_autoune, dataloader_config['tuning_steps'] + use_autotune, dataloader_config['tuning_steps'] ) else: warnings.warn( "The auto-tuning configuration of the dataloader is incorrect." "The `tuning_steps` should be int. Use default parameter instead." ) - paddle.io.reader.set_autotune_config(use_autoune) + paddle.io.reader.set_autotune_config(use_autotune) diff --git a/python/paddle/incubate/distributed/fleet/parameter_server/distribute_transpiler/__init__.py b/python/paddle/incubate/distributed/fleet/parameter_server/distribute_transpiler/__init__.py index f810014e93b3b..c6b6eec025107 100644 --- a/python/paddle/incubate/distributed/fleet/parameter_server/distribute_transpiler/__init__.py +++ b/python/paddle/incubate/distributed/fleet/parameter_server/distribute_transpiler/__init__.py @@ -257,14 +257,14 @@ def _init_transpiler_server(self, model_dir=None): sparse_varnames = self.compiled_config.get_sparse_varname_on_ps( True ) - distribtued_varnames = ( + distributed_varnames = ( self.compiled_config.get_sparse_varname_on_ps(False) ) remaining_vars = list( filter( FleetTranspiler.__exclude_vars( - sparse_varnames + distribtued_varnames + sparse_varnames + distributed_varnames ), self.main_program.list_vars(), ) @@ -282,7 +282,7 @@ def _init_transpiler_server(self, model_dir=None): ) # todo(tangwei12) load distributed vars - # self._load_sparse_params(dirname=model_dir, varnames=distribtued_varnames) + # self._load_sparse_params(dirname=model_dir, varnames=distributed_varnames) def init_server(self, model_dir=None, **kwargs): """ diff --git a/python/paddle/jit/dy2static/transformers/decorator_transformer.py b/python/paddle/jit/dy2static/transformers/decorator_transformer.py index 143d1fb1e14d7..c19ce1f95b587 100644 --- a/python/paddle/jit/dy2static/transformers/decorator_transformer.py +++ b/python/paddle/jit/dy2static/transformers/decorator_transformer.py @@ -56,13 +56,13 @@ def visit_FunctionDef(self, node): # every decorator will append a node decofun_nodes = [] - # func to be decoed next time + # func to be decoded next time deco_target = '_orig_' + node.name - # last decoed func - decoed_func = '' + # last decoded func + decoded_func = '' for deco in reversed(deco_list): - # skip INGNORE_NAMES + # skip IGNORE_NAMES deco_full_name = ast_to_source_code(deco).strip() if isinstance(deco, gast.Call): # match case like : @@ -90,7 +90,7 @@ def visit_FunctionDef(self, node): "Dy2Static : A context manager decorator is used, this may not work correctly after transform." ) - decoed_func = '_decoedby_' + deco_name + decoded_func = '_decoedby_' + deco_name # get function after decoration if isinstance(deco, gast.Call): @@ -104,7 +104,7 @@ def visit_FunctionDef(self, node): re_args = rematch.group(2) re_args_with_func = deco_target + ', ' + re_args decofun_str = 'try:\n\t{0} = _jst.Call({1})({2})\nexcept:\n\t{0} = _jst.Call({1})({3})({4})'.format( - decoed_func, + decoded_func, re_name, re_args_with_func, re_args, @@ -117,7 +117,7 @@ def visit_FunctionDef(self, node): re_args = rematch.group(2) re_args_with_func = deco_target + ', ' + re_args decofun_str = 'try:\n\t{0} = {1}({2})\nexcept:\n\t{0} = {1}({3})({4})'.format( - decoed_func, + decoded_func, re_name, re_args_with_func, re_args, @@ -126,11 +126,11 @@ def visit_FunctionDef(self, node): else: decofun_str = '{} = _jst.Call({})({})'.format( - decoed_func, deco_full_name, deco_target + decoded_func, deco_full_name, deco_target ) decofun_nodes.extend(gast.parse(decofun_str).body) - deco_target = decoed_func + deco_target = decoded_func if not decofun_nodes: return node @@ -146,7 +146,7 @@ def visit_FunctionDef(self, node): args = [arg.id for arg in node.args.args] arg_str = ','.join(args) - callfun_str = f'return {decoed_func}({arg_str})' + callfun_str = f'return {decoded_func}({arg_str})' callfun_node = gast.parse(callfun_str).body[0] node.body = [orig_func_node] + decofun_nodes + [callfun_node] diff --git a/python/paddle/jit/dy2static/transformers/tensorhook_transformer.py b/python/paddle/jit/dy2static/transformers/tensorhook_transformer.py index b0a5c56063ab4..04abaa34ef38b 100644 --- a/python/paddle/jit/dy2static/transformers/tensorhook_transformer.py +++ b/python/paddle/jit/dy2static/transformers/tensorhook_transformer.py @@ -38,7 +38,7 @@ def transform(self): self.visit(self.root) def reorder_block_statements(self, stmts): - regisiter_hook_nodes = [ + register_hook_nodes = [ n for n in stmts for stmt in gast.walk(n) @@ -46,7 +46,7 @@ def reorder_block_statements(self, stmts): ] # Analyze the register_hook nodes name dependency dependents = {} - for n in regisiter_hook_nodes: + for n in register_hook_nodes: if n not in stmts: continue for load_node in get_loads(n): diff --git a/python/paddle/jit/dy2static/utils.py b/python/paddle/jit/dy2static/utils.py index 582dd370aa4b4..ce1c26afcb333 100644 --- a/python/paddle/jit/dy2static/utils.py +++ b/python/paddle/jit/dy2static/utils.py @@ -309,7 +309,7 @@ def func_prefix(func): global DEL_TEMP_DIR if delete_on_exit and DEL_TEMP_DIR: - # Clear temporary files in TEMP_DIR while exitting Python process + # Clear temporary files in TEMP_DIR while exiting Python process atexit.register(remove_if_exit, dir_path=temp_dir) DEL_TEMP_DIR = False @@ -576,16 +576,16 @@ def name_judge(): @signature_safe_contextmanager def backend_guard(backend): core.check_and_set_prim_all_enabled() - orign_fwd = core._is_fwd_prim_enabled() - orign_bwd = core._is_bwd_prim_enabled() + origin_fwd = core._is_fwd_prim_enabled() + origin_bwd = core._is_bwd_prim_enabled() if backend == 'CINN': core._set_prim_all_enabled(True) try: yield finally: - core._set_prim_forward_enabled(orign_fwd) - core._set_prim_backward_enabled(orign_bwd) + core._set_prim_forward_enabled(origin_fwd) + core._set_prim_backward_enabled(origin_bwd) def construct_grad_names(grad_info_map, x_vars, param_vars, out_vars): diff --git a/python/paddle/jit/sot/symbolic/export.py b/python/paddle/jit/sot/symbolic/export.py index 720ef70730d20..39b06eca1891c 100644 --- a/python/paddle/jit/sot/symbolic/export.py +++ b/python/paddle/jit/sot/symbolic/export.py @@ -31,8 +31,8 @@ def __init__(self, *lines): def get_lines(self, prefix=""): lines = [prefix + line for line in self.lines] - for statment in self.sub_statement: - lines.extend(statment.get_lines(self.tab + prefix)) + for statement in self.sub_statement: + lines.extend(statement.get_lines(self.tab + prefix)) return lines def add_sub(self, *lines): @@ -302,7 +302,7 @@ def create_tail(self): ) def init_sub_layer(self, layer, layer_name): - # TODO @wuzhanfei need more effecient way to create a sub layer + # TODO @wuzhanfei need more efficient way to create a sub layer # now, we just close call_Layer behavior raise ExportError("Not support create sub layer now.") @@ -385,4 +385,6 @@ def export(SIR, path): with open(os.path.join(path, f"{SIR.name}.py"), "w") as f: f.write(string) - print(f"[SOT] Export {SIR.name} Sucess with size {len(SIR.statements)}") + print( + f"[SOT] Export {SIR.name} Success with size {len(SIR.statements)}" + ) diff --git a/python/paddle/tensor/math.py b/python/paddle/tensor/math.py index f057a261e9da7..a931912ae9572 100644 --- a/python/paddle/tensor/math.py +++ b/python/paddle/tensor/math.py @@ -1132,7 +1132,7 @@ def multiply_(x, y, name=None): return _C_ops.multiply_(x, y) -def _elementwise_op_with_axis(x, y, axis=-1, name=None, op_type="Undifined"): +def _elementwise_op_with_axis(x, y, axis=-1, name=None, op_type="Undefined"): assert ( in_dynamic_or_pir_mode() ), "You can only call `_elementwise_op_with_axis` function within in_dynamic_or_pir_mode" diff --git a/python/paddle/utils/cpp_extension/cpp_extension.py b/python/paddle/utils/cpp_extension/cpp_extension.py index 35bda07cab67b..b48f9fcaa2c28 100644 --- a/python/paddle/utils/cpp_extension/cpp_extension.py +++ b/python/paddle/utils/cpp_extension/cpp_extension.py @@ -589,7 +589,7 @@ def win_custom_spawn(cmd): finally: self.compiler.spawn = original_spawn - def object_filenames_with_cuda(origina_func, build_directory): + def object_filenames_with_cuda(original_func, build_directory): """ Decorated the function to add customized naming mechanism. Originally, both .cc/.cu will have .o object output that will @@ -598,7 +598,7 @@ def object_filenames_with_cuda(origina_func, build_directory): def wrapper(source_filenames, strip_dir=0, output_dir=''): try: - objects = origina_func( + objects = original_func( source_filenames, strip_dir, output_dir ) for i, source in enumerate(source_filenames): @@ -618,7 +618,7 @@ def wrapper(source_filenames, strip_dir=0, output_dir=''): # ensure to use abspath objects = [os.path.abspath(obj) for obj in objects] finally: - self.compiler.object_filenames = origina_func + self.compiler.object_filenames = original_func return objects From bb2943881ca9927ad9b08f1f460f90707ec901fc Mon Sep 17 00:00:00 2001 From: co63oc Date: Thu, 29 Feb 2024 23:39:58 +0800 Subject: [PATCH 198/282] Fix distribuions distributions, etc (#62161) --- test/distribution/test_distribution_categorical.py | 2 +- test/xpu/test_adamw_fp16_xpu.py | 2 +- test/xpu/test_argsort_op_xpu.py | 4 ++-- test/xpu/test_collective_allgather_xpu.py | 4 ++-- test/xpu/test_collective_allreduce_xpu.py | 4 ++-- test/xpu/test_collective_broadcast_xpu.py | 4 ++-- test/xpu/test_collective_process_group_xpu.py | 2 +- test/xpu/test_collective_reduce_xpu.py | 4 ++-- test/xpu/test_device_guard_xpu.py | 4 ++-- test/xpu/test_scatter_nd_add_op_xpu.py | 6 +++--- 10 files changed, 18 insertions(+), 18 deletions(-) diff --git a/test/distribution/test_distribution_categorical.py b/test/distribution/test_distribution_categorical.py index d87c72e73438c..8be8b31672a9d 100644 --- a/test/distribution/test_distribution_categorical.py +++ b/test/distribution/test_distribution_categorical.py @@ -313,7 +313,7 @@ def get_numpy_selected_probs(self, probability): class CategoricalTest7(CategoricalTest): def init_numpy_data(self, batch_size, dims): # input logtis is 3-D Tensor - # value used in probs and log_prob method has the same number of distribuions with input + # value used in probs and log_prob method has the same number of distributions with input self.logits_np = np.random.rand(3, 2, 5).astype('float32') self.other_logits_np = np.random.rand(3, 2, 5).astype('float32') self.value_np = np.array([2, 1, 3]).astype('int64') diff --git a/test/xpu/test_adamw_fp16_xpu.py b/test/xpu/test_adamw_fp16_xpu.py index ca7c799312410..e9a6b1540fa49 100644 --- a/test/xpu/test_adamw_fp16_xpu.py +++ b/test/xpu/test_adamw_fp16_xpu.py @@ -59,7 +59,7 @@ def test_state_dict(self): state_dict_1["linear_0.b_0_moment1_0.SCALE_VALUE"] = 12.3125 adam.set_state_dict(state_dict_1) - # check overwrited value + # check overwritten value state_dict_2 = adam.state_dict() self.assertTrue("linear_0.w_0_moment1_0.SCALE_VALUE" in state_dict_2) self.assertTrue("linear_0.b_0_moment1_0.SCALE_VALUE" in state_dict_2) diff --git a/test/xpu/test_argsort_op_xpu.py b/test/xpu/test_argsort_op_xpu.py index f3a8a69ee5ded..c8ddebf859ecd 100644 --- a/test/xpu/test_argsort_op_xpu.py +++ b/test/xpu/test_argsort_op_xpu.py @@ -165,7 +165,7 @@ def init_test_case(self): 2, 8732, 1, - ] # test for 8192 < n <= 10240 + nees_transpose + ] # test for 8192 < n <= 10240 + need_transpose self.axis = 1 class TestArgsortOpCase4(TestArgsortOpCase1): @@ -174,7 +174,7 @@ def init_test_case(self): 2, 10241, 1, - ] # test for 10240 < n <= 16384 + nees_transpose + ] # test for 10240 < n <= 16384 + need_transpose self.axis = 1 diff --git a/test/xpu/test_collective_allgather_xpu.py b/test/xpu/test_collective_allgather_xpu.py index ad232cba70a88..55f516337baff 100644 --- a/test/xpu/test_collective_allgather_xpu.py +++ b/test/xpu/test_collective_allgather_xpu.py @@ -29,7 +29,7 @@ def _setup_config(self): @unittest.skipIf( not core.is_compiled_with_xpu() or paddle.device.xpu.device_count() < 2, - "run test when having at leaset 2 XPUs.", + "run test when having at least 2 XPUs.", ) def test_allgather(self): support_types = get_xpu_op_support_types('c_allgather') @@ -40,7 +40,7 @@ def test_allgather(self): @unittest.skipIf( not core.is_compiled_with_xpu() or paddle.device.xpu.device_count() < 2, - "run test when having at leaset 2 XPUs.", + "run test when having at least 2 XPUs.", ) def test_allgather_dygraph(self): support_types = get_xpu_op_support_types('c_allgather') diff --git a/test/xpu/test_collective_allreduce_xpu.py b/test/xpu/test_collective_allreduce_xpu.py index 4d8797cc0972f..c52ca781f35af 100644 --- a/test/xpu/test_collective_allreduce_xpu.py +++ b/test/xpu/test_collective_allreduce_xpu.py @@ -29,7 +29,7 @@ def _setup_config(self): @unittest.skipIf( not core.is_compiled_with_xpu() or paddle.device.xpu.device_count() < 2, - "run test when having at leaset 2 XPUs.", + "run test when having at least 2 XPUs.", ) def test_allreduce(self): support_types = get_xpu_op_support_types('c_allreduce_sum') @@ -42,7 +42,7 @@ def test_allreduce(self): @unittest.skipIf( not core.is_compiled_with_xpu() or paddle.device.xpu.device_count() < 2, - "run test when having at leaset 2 XPUs.", + "run test when having at least 2 XPUs.", ) def test_allreduce_dygraph(self): support_types = get_xpu_op_support_types('c_allreduce_sum') diff --git a/test/xpu/test_collective_broadcast_xpu.py b/test/xpu/test_collective_broadcast_xpu.py index 7fa695b321781..91e3024ee3838 100644 --- a/test/xpu/test_collective_broadcast_xpu.py +++ b/test/xpu/test_collective_broadcast_xpu.py @@ -29,7 +29,7 @@ def _setup_config(self): @unittest.skipIf( not core.is_compiled_with_xpu() or paddle.device.xpu.device_count() < 2, - "run test when having at leaset 2 XPUs.", + "run test when having at least 2 XPUs.", ) def test_broadcast(self): support_types = get_xpu_op_support_types('c_broadcast') @@ -42,7 +42,7 @@ def test_broadcast(self): @unittest.skipIf( not core.is_compiled_with_xpu() or paddle.device.xpu.device_count() < 2, - "run test when having at leaset 2 XPUs.", + "run test when having at least 2 XPUs.", ) def test_broadcast_dygraph(self): support_types = get_xpu_op_support_types('c_broadcast') diff --git a/test/xpu/test_collective_process_group_xpu.py b/test/xpu/test_collective_process_group_xpu.py index ec351b857ab93..166b1e6707596 100644 --- a/test/xpu/test_collective_process_group_xpu.py +++ b/test/xpu/test_collective_process_group_xpu.py @@ -23,7 +23,7 @@ class TestProcessGroup(TestMultipleXpus): @unittest.skipIf( not core.is_compiled_with_xpu() or paddle.device.xpu.device_count() < 2, - "run test when having at leaset 2 XPUs.", + "run test when having at least 2 XPUs.", ) def test_process_group_bkcl(self): self.run_mnist_2xpu('process_group_bkcl.py') diff --git a/test/xpu/test_collective_reduce_xpu.py b/test/xpu/test_collective_reduce_xpu.py index be5eccdc9a0e8..b36e3e3be5203 100644 --- a/test/xpu/test_collective_reduce_xpu.py +++ b/test/xpu/test_collective_reduce_xpu.py @@ -29,7 +29,7 @@ def _setup_config(self): @unittest.skipIf( not core.is_compiled_with_xpu() or paddle.device.xpu.device_count() < 2, - "run test when having at leaset 2 XPUs.", + "run test when having at least 2 XPUs.", ) def test_reduce(self): support_types = get_xpu_op_support_types('c_reduce_sum') @@ -42,7 +42,7 @@ def test_reduce(self): @unittest.skipIf( not core.is_compiled_with_xpu() or paddle.device.xpu.device_count() < 2, - "run test when having at leaset 2 XPUs.", + "run test when having at least 2 XPUs.", ) def test_reduce_dygraph(self): support_types = get_xpu_op_support_types('c_reduce_sum') diff --git a/test/xpu/test_device_guard_xpu.py b/test/xpu/test_device_guard_xpu.py index ce85946aee74e..bcc9e85839bee 100644 --- a/test/xpu/test_device_guard_xpu.py +++ b/test/xpu/test_device_guard_xpu.py @@ -31,7 +31,7 @@ def execute(main_program, startup_program): exe.run(main_program) -def get_vaild_warning_num(warning, w): +def get_valid_warning_num(warning, w): num = 0 for i in range(len(w)): if warning in str(w[i].message): @@ -160,7 +160,7 @@ def test_without_kernel_op(self): paddle.assign(paddle.less_than(x=i, y=loop_len), cond) warning = "The Op(while) is not support to set device." - warning_num = get_vaild_warning_num(warning, w) + warning_num = get_valid_warning_num(warning, w) assert warning_num == 1 all_ops = main_program.global_block().ops diff --git a/test/xpu/test_scatter_nd_add_op_xpu.py b/test/xpu/test_scatter_nd_add_op_xpu.py index 6efb4fec3b0f7..d8733dd1a1e83 100644 --- a/test/xpu/test_scatter_nd_add_op_xpu.py +++ b/test/xpu/test_scatter_nd_add_op_xpu.py @@ -34,11 +34,11 @@ def numpy_scatter_nd(ref, index, updates, fun): end_size = index_shape[-1] # as type int32, flat_index or flat_updates can't reshape to int64 - remain_numl = np.prod(index_shape[:-1]).astype("int32") + remain_numel = np.prod(index_shape[:-1]).astype("int32") slice_size = np.prod(ref_shape[end_size : len(ref_shape)]).astype("int32") - flat_index = index.reshape([remain_numl] + list(index_shape[-1:])) - flat_updates = updates.reshape((remain_numl, slice_size)) + flat_index = index.reshape([remain_numel] + list(index_shape[-1:])) + flat_updates = updates.reshape((remain_numel, slice_size)) flat_output = ref.reshape(list(ref_shape[:end_size]) + [slice_size]) for i_up, i_out in enumerate(flat_index): From 16dfd859811df562480584a9b17cb589ccadcce2 Mon Sep 17 00:00:00 2001 From: co63oc Date: Thu, 29 Feb 2024 23:40:29 +0800 Subject: [PATCH 199/282] Fix precsion precision, etc (#62160) --- paddle/fluid/pir/drr/README.md | 4 +-- paddle/fluid/pir/drr/README_cn.md | 4 +-- .../transforms/auto_mixed_precision_pass.cc | 2 +- .../pir/transforms/identity_op_clean_pass.cc | 26 +++++++++---------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/paddle/fluid/pir/drr/README.md b/paddle/fluid/pir/drr/README.md index 1c5de89780c6f..d9b435160c41d 100644 --- a/paddle/fluid/pir/drr/README.md +++ b/paddle/fluid/pir/drr/README.md @@ -9,9 +9,9 @@ DRR can reduce the development cost of PASS, allowing developers to focus on pro Taking PASS to eliminate redundant CastOp as an example, the code example developed using DRR is as follows: ~~~ c++ // 1. Inherit class from DrPatternBase -class RemoveRedundentCastPattern : public paddle::drr::DrrPatternBase { +class RemoveRedundantCastPattern : public paddle::drr::DrrPatternBase { public: - std::string name() const override { return "RemoveRedundentCastPattern"; } + std::string name() const override { return "RemoveRedundantCastPattern"; } // 2. Overload operator() void operator()(paddle::drr::DrrPatternContext *ctx) const override { diff --git a/paddle/fluid/pir/drr/README_cn.md b/paddle/fluid/pir/drr/README_cn.md index e621e7112ac30..c01b21febeda3 100644 --- a/paddle/fluid/pir/drr/README_cn.md +++ b/paddle/fluid/pir/drr/README_cn.md @@ -9,9 +9,9 @@ DRR ( Declarative Rewrite Rule ) 是来处理这种 DAG-to-DAG 类型的一套 P 以消除冗余 CastOp 的 PASS 为例，使用 DRR 的代码开发示例如下： ~~~ c++ // 1. 继承 DrrPatternBase 类 -class RemoveRedundentCastPattern : public paddle::drr::DrrPatternBase { +class RemoveRedundantCastPattern : public paddle::drr::DrrPatternBase { public: - std::string name() const override { return "RemoveRedundentCastPattern"; } + std::string name() const override { return "RemoveRedundantCastPattern"; } // 2. 重载 operator() void operator()(paddle::drr::DrrPatternContext *ctx) const override { diff --git a/paddle/fluid/pir/transforms/auto_mixed_precision_pass.cc b/paddle/fluid/pir/transforms/auto_mixed_precision_pass.cc index dee9aad09ed1d..1ff6b34565ed0 100644 --- a/paddle/fluid/pir/transforms/auto_mixed_precision_pass.cc +++ b/paddle/fluid/pir/transforms/auto_mixed_precision_pass.cc @@ -230,7 +230,7 @@ class AutoMixedPrecisionPass : public pir::Pass { if (!op->operand_source(idx)) continue; auto operand = op->operand(idx); if (operand.type() && operand.type().isa()) { - // check if there are all float in the vectortype + // check if there are all float in the vector type auto vec_type = operand.type().dyn_cast(); if (IsVectorTypeFloat(vec_type)) { auto input_operation = GetDefiningOpForInput(op, idx); diff --git a/paddle/fluid/pir/transforms/identity_op_clean_pass.cc b/paddle/fluid/pir/transforms/identity_op_clean_pass.cc index cf27800512b0b..32346997cd6c9 100644 --- a/paddle/fluid/pir/transforms/identity_op_clean_pass.cc +++ b/paddle/fluid/pir/transforms/identity_op_clean_pass.cc @@ -53,9 +53,9 @@ class RemoveUselessScalePattern : public paddle::drr::DrrPatternBase { } }; -class RemoveRedundentScalePattern : public paddle::drr::DrrPatternBase { +class RemoveRedundantScalePattern : public paddle::drr::DrrPatternBase { public: - std::string name() const override { return "RemoveRedundentScalePattern"; } + std::string name() const override { return "RemoveRedundantScalePattern"; } void operator()(paddle::drr::DrrPatternContext *ctx) const override { paddle::drr::SourcePattern pat = ctx->SourcePattern(); @@ -83,7 +83,7 @@ class RemoveRedundentScalePattern : public paddle::drr::DrrPatternBase { paddle::drr::ResultPattern res = pat.ResultPattern(); - const auto &bais_attr = res.ComputeAttr( + const auto &bias_attr = res.ComputeAttr( [](const paddle::drr::MatchContext &match_ctx) -> float { float res_bias_1 = 0.f; float res_bias_2 = 0.f; @@ -115,7 +115,7 @@ class RemoveRedundentScalePattern : public paddle::drr::DrrPatternBase { {"place", pat.Attr("place_1")}}); const auto &scale_op_res = res.Op("pd_op.scale", - {{"bias", bais_attr}, {"bias_after_scale", res.BoolAttr(true)}}); + {{"bias", bias_attr}, {"bias_after_scale", res.BoolAttr(true)}}); scale_op_res({&res.Tensor("x"), &full_op_res()}, {&res.Tensor("scale_2_out")}); } @@ -154,9 +154,9 @@ class RemoveUselessConcatPattern : public paddle::drr::DrrPatternBase { } }; -class RemoveRedundentCastPattern : public paddle::drr::DrrPatternBase { +class RemoveRedundantCastPattern : public paddle::drr::DrrPatternBase { public: - std::string name() const override { return "RemoveRedundentCastPattern"; } + std::string name() const override { return "RemoveRedundantCastPattern"; } void operator()(paddle::drr::DrrPatternContext *ctx) const override { auto pat = ctx->SourcePattern(); @@ -245,10 +245,10 @@ class ReplaceDropoutWithScalePattern : public paddle::drr::DrrPatternBase { } }; -class RemoveRedundentTransposePattern : public paddle::drr::DrrPatternBase { +class RemoveRedundantTransposePattern : public paddle::drr::DrrPatternBase { public: std::string name() const override { - return "RemoveRedundentTransposePattern"; + return "RemoveRedundantTransposePattern"; } void operator()(paddle::drr::DrrPatternContext *ctx) const override { @@ -271,10 +271,10 @@ class RemoveRedundentTransposePattern : public paddle::drr::DrrPatternBase { } return new_perm; }); - const auto &tranpose_continuous = + const auto &transpose_continuous = res.Op("pd_op.transpose", {{"perm", new_perm_attr}}); - res.Tensor("ret") = tranpose_continuous(res.Tensor("arg_transpose")); + res.Tensor("ret") = transpose_continuous(res.Tensor("arg_transpose")); } }; @@ -286,13 +286,13 @@ class IdentityOpCleanPass : public pir::PatternRewritePass { pir::RewritePatternSet InitializePatterns(pir::IrContext *context) override { pir::RewritePatternSet ps(context); ps.Add(paddle::drr::Create(context)); - ps.Add(paddle::drr::Create(context)); + ps.Add(paddle::drr::Create(context)); ps.Add(paddle::drr::Create(context)); ps.Add(paddle::drr::Create(context)); - ps.Add(paddle::drr::Create(context)); + ps.Add(paddle::drr::Create(context)); ps.Add(paddle::drr::Create(context)); ps.Add(paddle::drr::Create(context)); - ps.Add(paddle::drr::Create(context)); + ps.Add(paddle::drr::Create(context)); return ps; } }; From c422cc561a6bc26151152e82ba387096ab453b01 Mon Sep 17 00:00:00 2001 From: co63oc Date: Thu, 29 Feb 2024 23:41:43 +0800 Subject: [PATCH 200/282] Fix quantdequant quant_dequant (#62046) * Fix * ci * ci * ci * ci --- .../ir/delete_quant_dequant_filter_op_pass.cc | 4 ++-- .../ir/delete_quant_dequant_linear_op_pass.cc | 2 +- .../fluid/framework/ir/delete_quant_dequant_op_pass.cc | 8 ++++---- paddle/fluid/framework/ir/graph_pattern_detector.cc | 10 +++++----- paddle/fluid/framework/ir/graph_pattern_detector.h | 6 +++--- .../ir/trt_delete_weight_dequant_linear_op_pass.cc | 2 +- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/paddle/fluid/framework/ir/delete_quant_dequant_filter_op_pass.cc b/paddle/fluid/framework/ir/delete_quant_dequant_filter_op_pass.cc index cfe644a61ea51..3bd051c597179 100644 --- a/paddle/fluid/framework/ir/delete_quant_dequant_filter_op_pass.cc +++ b/paddle/fluid/framework/ir/delete_quant_dequant_filter_op_pass.cc @@ -73,7 +73,7 @@ DeleteQuantDequantFilterOpPass::DeleteQuantDequantFilterOpPass() { } // Delete quant_dequant_op, then quantize and dequantize weight void DeleteQuantDequantFilterOpPass::ApplyImpl(ir::Graph* graph) const { - const std::string pattern_name = "delete_quantdequant_filter_op_pattern"; + const std::string pattern_name = "delete_quant_dequant_filter_op_pattern"; FusePassBase::Init(pattern_name, graph); GraphPatternDetector gpd; @@ -141,7 +141,7 @@ void DeleteQuantDequantFilterOpPass::ApplyImpl(ir::Graph* graph) const { "the received is %d", quant_axis)); - // To Do @Wangzheee: use "OutScale" to quantdequant + // To Do @Wangzheee: use "OutScale" to quant_dequant /*auto scales_name = quant_dequant_op->Op()->Output("OutScale"); PADDLE_ENFORCE_EQ(scales_name.size(), 1, platform::errors::InvalidArgument( diff --git a/paddle/fluid/framework/ir/delete_quant_dequant_linear_op_pass.cc b/paddle/fluid/framework/ir/delete_quant_dequant_linear_op_pass.cc index 7358a82c6ca3c..9d4006e6f3943 100644 --- a/paddle/fluid/framework/ir/delete_quant_dequant_linear_op_pass.cc +++ b/paddle/fluid/framework/ir/delete_quant_dequant_linear_op_pass.cc @@ -86,7 +86,7 @@ DeleteQuantDequantLinearOpPass::DeleteQuantDequantLinearOpPass() { } // Delete quantize_linear_op dequantize_linear_op, then add input_scales void DeleteQuantDequantLinearOpPass::ApplyImpl(ir::Graph* graph) const { - const std::string pattern_name = "delete_quantdequant_linear_op_pattern"; + const std::string pattern_name = "delete_quant_dequant_linear_op_pattern"; FusePassBase::Init(pattern_name, graph); GraphPatternDetector gpd; diff --git a/paddle/fluid/framework/ir/delete_quant_dequant_op_pass.cc b/paddle/fluid/framework/ir/delete_quant_dequant_op_pass.cc index ebb0ed9d00dc1..2a7071d54843d 100644 --- a/paddle/fluid/framework/ir/delete_quant_dequant_op_pass.cc +++ b/paddle/fluid/framework/ir/delete_quant_dequant_op_pass.cc @@ -32,21 +32,21 @@ namespace ir { GET_IR_NODE(quant_dequant_op_out); void DeleteQuantDequantOpPass::ApplyImpl(ir::Graph* graph) const { - const std::string pattern_name = "delete_quantdequant_op_pattern"; + const std::string pattern_name = "delete_quant_dequant_op_pattern"; FusePassBase::Init(pattern_name, graph); GraphPatternDetector gpd; - std::string quantdequant_types = + std::string quant_dequant_types = "fake_quantize_dequantize_moving_average_abs_max"; auto* input_node = gpd.mutable_pattern() ->NewNode("input_node") - ->assert_is_op_input(quantdequant_types, "X") + ->assert_is_op_input(quant_dequant_types, "X") ->AsInput(); patterns::DeleteQuantDequantOpPattern pattern(gpd.mutable_pattern(), pattern_name); - pattern(input_node, quantdequant_types); + pattern(input_node, quant_dequant_types); auto* scope = param_scope(); int found_count = 0; diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.cc b/paddle/fluid/framework/ir/graph_pattern_detector.cc index df804cf0d4f7b..034780ac0d0b8 100644 --- a/paddle/fluid/framework/ir/graph_pattern_detector.cc +++ b/paddle/fluid/framework/ir/graph_pattern_detector.cc @@ -3519,22 +3519,22 @@ void patterns::ShuffleChannelPattern::operator()(PDNode *reshape1_in) { } void patterns::DeleteQuantDequantOpPattern::operator()( - PDNode *input_node, const std::string &quantdequant_types) { + PDNode *input_node, const std::string &quant_dequant_types) { auto quant_dequant_op_inscale = pattern->NewNode(quant_dequant_op_inscale_repr()) - ->assert_is_op_input(quantdequant_types, "InScale") + ->assert_is_op_input(quant_dequant_types, "InScale") ->AsInput(); auto quant_dequant_op = pattern->NewNode(quant_dequant_op_repr()) - ->assert_is_op(quantdequant_types); + ->assert_is_op(quant_dequant_types); auto quant_dequant_op_out = pattern->NewNode(quant_dequant_op_out_repr()) - ->assert_is_op_output(quantdequant_types, "Out") + ->assert_is_op_output(quant_dequant_types, "Out") ->AsOutput(); auto quant_dequant_op_outscale = pattern->NewNode(quant_dequant_op_outscale_repr()) - ->assert_is_op_output(quantdequant_types, "OutScale") + ->assert_is_op_output(quant_dequant_types, "OutScale") ->AsOutput(); quant_dequant_op->LinksFrom({quant_dequant_op_inscale, input_node}); diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.h b/paddle/fluid/framework/ir/graph_pattern_detector.h index 22d88e96b2852..4eac3440a4514 100644 --- a/paddle/fluid/framework/ir/graph_pattern_detector.h +++ b/paddle/fluid/framework/ir/graph_pattern_detector.h @@ -1869,9 +1869,9 @@ struct DeleteDropoutOpPattern : public PatternBase { struct DeleteQuantDequantOpPattern : public PatternBase { DeleteQuantDequantOpPattern(PDPattern* pattern, const std::string& name_scope) - : PatternBase(pattern, name_scope, "delete_quantdequant_op_pattern") {} + : PatternBase(pattern, name_scope, "delete_quant_dequant_op_pattern") {} - void operator()(PDNode* input_node, const std::string& quantdequant_types); + void operator()(PDNode* input_node, const std::string& quant_dequant_types); PATTERN_DECL_NODE(quant_dequant_op_inscale); PATTERN_DECL_NODE(quant_dequant_op); @@ -1883,7 +1883,7 @@ struct DeleteQuantDequantFilterOpPattern : public PatternBase { DeleteQuantDequantFilterOpPattern(PDPattern* pattern, const std::string& name_scope) : PatternBase( - pattern, name_scope, "delete_quantdequant_filter_op_pattern") {} + pattern, name_scope, "delete_quant_dequant_filter_op_pattern") {} void operator()(); diff --git a/paddle/fluid/framework/ir/trt_delete_weight_dequant_linear_op_pass.cc b/paddle/fluid/framework/ir/trt_delete_weight_dequant_linear_op_pass.cc index 6e12933f0f4d5..b780c07fda0a6 100644 --- a/paddle/fluid/framework/ir/trt_delete_weight_dequant_linear_op_pass.cc +++ b/paddle/fluid/framework/ir/trt_delete_weight_dequant_linear_op_pass.cc @@ -201,7 +201,7 @@ TrtDeleteWeightQuantDequantLinearOpPass:: void TrtDeleteWeightQuantDequantLinearOpPass::ApplyImpl( ir::Graph* graph) const { const std::string pattern_name = - "delete_weight_quantdequant_linear_op_pattern"; + "delete_weight_quant_dequant_linear_op_pattern"; FusePassBase::Init(pattern_name, graph); GraphPatternDetector gpd; From 2fb56196c4aaf7af47b512f92f560a3df7de0f07 Mon Sep 17 00:00:00 2001 From: cyberslack_lee Date: Thu, 29 Feb 2024 23:48:10 +0800 Subject: [PATCH 201/282] [Typo error] fix typo error tesnor to tensor (#62175) --- paddle/fluid/framework/tensor_util.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/framework/tensor_util.h b/paddle/fluid/framework/tensor_util.h index 96f3d71c132af..02aa4b500ce7b 100644 --- a/paddle/fluid/framework/tensor_util.h +++ b/paddle/fluid/framework/tensor_util.h @@ -103,7 +103,7 @@ void TensorToVector(const phi::DenseTensor& src, const platform::DeviceContext& ctx, std::vector* dst); template -void TesnorToVector(const phi::DenseTensor& src, std::vector* dst); +void TensorToVector(const phi::DenseTensor& src, std::vector* dst); // convert dlpack's DLTensor to tensor From 180c596fb4978047e738767fd14727008dab3fd7 Mon Sep 17 00:00:00 2001 From: cyberslack_lee Date: Thu, 29 Feb 2024 23:49:13 +0800 Subject: [PATCH 202/282] =?UTF-8?q?[clang-tidy]=20fix=20about=2031?= =?UTF-8?q?=E3=80=8132=E3=80=8134=E3=80=8141=E3=80=8145=20(#62129)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- paddle/fluid/framework/io/crypto/aes_cipher.cc | 8 ++++---- .../fluid/memory/allocation/naive_best_fit_allocator.cc | 2 +- paddle/fluid/platform/enforce_test.cc | 2 +- paddle/phi/core/dense_tensor.cc | 2 +- paddle/phi/core/sparse_coo_tensor.cc | 2 +- paddle/phi/core/sparse_csr_tensor.cc | 2 +- paddle/phi/core/string_tensor.cc | 2 +- paddle/phi/core/utils/intrusive_ref_counter.h | 2 +- paddle/phi/infermeta/spmd_rules/unsqueeze.cc | 2 +- paddle/pir/src/core/builtin_type_interfaces.cc | 4 ++-- 10 files changed, 14 insertions(+), 14 deletions(-) diff --git a/paddle/fluid/framework/io/crypto/aes_cipher.cc b/paddle/fluid/framework/io/crypto/aes_cipher.cc index 8802dc1b12158..158d25a6957f7 100644 --- a/paddle/fluid/framework/io/crypto/aes_cipher.cc +++ b/paddle/fluid/framework/io/crypto/aes_cipher.cc @@ -65,7 +65,7 @@ std::string AESCipher::EncryptInternal(const std::string& plaintext, std::string ciphertext; m_filter->Attach(new CryptoPP::StringSink(ciphertext)); CryptoPP::Redirector* filter_redirector = new CryptoPP::Redirector(*m_filter); - CryptoPP::StringSource(plaintext, true, filter_redirector); + CryptoPP::StringSource ss(plaintext, true, filter_redirector); if (need_iv) { return iv_ + ciphertext; } @@ -96,7 +96,7 @@ std::string AESCipher::DecryptInternal(const std::string& ciphertext, std::string plaintext; m_filter->Attach(new CryptoPP::StringSink(plaintext)); CryptoPP::Redirector* filter_redirector = new CryptoPP::Redirector(*m_filter); - CryptoPP::StringSource( + CryptoPP::StringSource ss( ciphertext.substr(ciphertext_beg), true, filter_redirector); return plaintext; @@ -124,7 +124,7 @@ std::string AESCipher::AuthenticatedEncryptInternal( std::string ciphertext; m_filter->Attach(new CryptoPP::StringSink(ciphertext)); CryptoPP::Redirector* filter_redirector = new CryptoPP::Redirector(*m_filter); - CryptoPP::StringSource(plaintext, true, filter_redirector); + CryptoPP::StringSource ss(plaintext, true, filter_redirector); if (need_iv) { ciphertext = iv_.append(ciphertext); } @@ -155,7 +155,7 @@ std::string AESCipher::AuthenticatedDecryptInternal( std::string plaintext; m_filter->Attach(new CryptoPP::StringSink(plaintext)); CryptoPP::Redirector* filter_redirector = new CryptoPP::Redirector(*m_filter); - CryptoPP::StringSource( + CryptoPP::StringSource ss( ciphertext.substr(ciphertext_beg), true, filter_redirector); PADDLE_ENFORCE_EQ( m_filter->GetLastResult(), diff --git a/paddle/fluid/memory/allocation/naive_best_fit_allocator.cc b/paddle/fluid/memory/allocation/naive_best_fit_allocator.cc index 612ba0798d2c0..45cf3b44baa8a 100644 --- a/paddle/fluid/memory/allocation/naive_best_fit_allocator.cc +++ b/paddle/fluid/memory/allocation/naive_best_fit_allocator.cc @@ -298,7 +298,7 @@ void *Alloc(const platform::CUDAPlace &place, auto *buddy_allocator = GetGPUBuddyAllocator(place.device); auto *ptr = buddy_allocator->Alloc(size); if (ptr == nullptr) { - platform::CUDADeviceGuard(place.device); + platform::CUDADeviceGuard guard(place.device); size_t avail, total; platform::GpuMemoryUsage(&avail, &total); PADDLE_THROW(platform::errors::ResourceExhausted( diff --git a/paddle/fluid/platform/enforce_test.cc b/paddle/fluid/platform/enforce_test.cc index 9bad3f0bf1c41..e6838746fd6ac 100644 --- a/paddle/fluid/platform/enforce_test.cc +++ b/paddle/fluid/platform/enforce_test.cc @@ -594,7 +594,7 @@ TEST(enforce, cannot_to_string_type) { } TEST(GET_DATA_SAFELY_MACRO, SUCCESS) { - int* a = new int(10); + int* a = new int(10); // NOLINT GET_DATA_SAFELY(a, "Input", "X", "dummy"); } diff --git a/paddle/phi/core/dense_tensor.cc b/paddle/phi/core/dense_tensor.cc index d15cc4eeafda1..8340c4d69c380 100644 --- a/paddle/phi/core/dense_tensor.cc +++ b/paddle/phi/core/dense_tensor.cc @@ -53,7 +53,7 @@ DenseTensor::DenseTensor(const std::shared_ptr& holder, const DenseTensorMeta& meta) : meta_(meta), holder_(holder) {} -DenseTensor::DenseTensor(const DenseTensor& other) { +DenseTensor::DenseTensor(const DenseTensor& other) { // NOLINT this->meta_ = other.meta(); holder_ = other.holder_; storage_properties_ = diff --git a/paddle/phi/core/sparse_coo_tensor.cc b/paddle/phi/core/sparse_coo_tensor.cc index dfd519250aa37..d6f41168981aa 100644 --- a/paddle/phi/core/sparse_coo_tensor.cc +++ b/paddle/phi/core/sparse_coo_tensor.cc @@ -51,7 +51,7 @@ SparseCooTensor::SparseCooTensor(DenseTensor&& non_zero_indices, meta_.dtype = non_zero_elements.dtype(); } -SparseCooTensor::SparseCooTensor(const SparseCooTensor& other) { +SparseCooTensor::SparseCooTensor(const SparseCooTensor& other) { // NOLINT this->non_zero_indices_ = other.non_zero_indices_; this->non_zero_elements_ = other.non_zero_elements_; this->coalesced_ = other.coalesced_; diff --git a/paddle/phi/core/sparse_csr_tensor.cc b/paddle/phi/core/sparse_csr_tensor.cc index 525f38cd8263d..f4373f528d217 100644 --- a/paddle/phi/core/sparse_csr_tensor.cc +++ b/paddle/phi/core/sparse_csr_tensor.cc @@ -66,7 +66,7 @@ SparseCsrTensor::SparseCsrTensor(const DenseTensor& non_zero_crows, meta_.dtype = non_zero_elements.dtype(); } -SparseCsrTensor::SparseCsrTensor(const SparseCsrTensor& other) { +SparseCsrTensor::SparseCsrTensor(const SparseCsrTensor& other) { // NOLINT this->non_zero_crows_ = other.non_zero_crows_; this->non_zero_cols_ = other.non_zero_cols_; this->non_zero_elements_ = other.non_zero_elements_; diff --git a/paddle/phi/core/string_tensor.cc b/paddle/phi/core/string_tensor.cc index d370be21f4cac..bb7d06825fdbb 100644 --- a/paddle/phi/core/string_tensor.cc +++ b/paddle/phi/core/string_tensor.cc @@ -37,7 +37,7 @@ StringTensor::StringTensor(const std::shared_ptr& holder, const StringTensorMeta& meta) : meta_(meta), holder_(holder) {} -StringTensor::StringTensor(const StringTensor& other) { +StringTensor::StringTensor(const StringTensor& other) { // NOLINT this->meta_ = other.meta(); holder_ = other.holder_; } diff --git a/paddle/phi/core/utils/intrusive_ref_counter.h b/paddle/phi/core/utils/intrusive_ref_counter.h index 1681f88af054f..6b2a3e989a840 100644 --- a/paddle/phi/core/utils/intrusive_ref_counter.h +++ b/paddle/phi/core/utils/intrusive_ref_counter.h @@ -57,7 +57,7 @@ inline void intrusive_ptr_release( const intrusive_ref_counter* p) noexcept { if (p->ref_.load(std::memory_order_acquire) == 0 || p->ref_.fetch_sub(1) == 0) { - delete static_cast(p); + delete static_cast(p); // NOLINT } } diff --git a/paddle/phi/infermeta/spmd_rules/unsqueeze.cc b/paddle/phi/infermeta/spmd_rules/unsqueeze.cc index cbb010fe6c6bf..ef47b31341a73 100644 --- a/paddle/phi/infermeta/spmd_rules/unsqueeze.cc +++ b/paddle/phi/infermeta/spmd_rules/unsqueeze.cc @@ -74,7 +74,7 @@ std::vector> MakeUnsqueezeDimTransReverse( ret.resize(x_ndim); fill(ret.begin(), ret.end(), std::make_shared()); - for (int64_t i = 0, j = 0; i < out_ndim; i++) { + for (int64_t i = 0, j = 0; i < out_ndim; i++) { // NOLINT auto it = find(axis.begin(), axis.end(), i); if (it == axis.end()) { diff --git a/paddle/pir/src/core/builtin_type_interfaces.cc b/paddle/pir/src/core/builtin_type_interfaces.cc index de0538eacc0d9..5b8d14b74175a 100644 --- a/paddle/pir/src/core/builtin_type_interfaces.cc +++ b/paddle/pir/src/core/builtin_type_interfaces.cc @@ -18,11 +18,11 @@ namespace pir { Type ShapedTypeInterface::GetElementType() const { - return impl_->get_element_type(*this); + return impl_->get_element_type(*this); // NOLINT } pir::DDim ShapedTypeInterface::GetShape() const { - return impl_->get_shape(*this); + return impl_->get_shape(*this); // NOLINT } } // namespace pir From 23adc6a42e7f1ee0d38df689b1a12449a156c3b0 Mon Sep 17 00:00:00 2001 From: lanxianghit <47554610+lanxianghit@users.noreply.github.com> Date: Fri, 1 Mar 2024 09:46:44 +0800 Subject: [PATCH 203/282] [PIR][DynamicShape] Add shape pass to inference predictor (#62167) * [PIR][DynamicShape] Add shape pass to inference predictor * move decomp case * fix ci --- .../fluid/inference/api/analysis_predictor.cc | 10 ++++- .../pir/transforms/shape_optimization_pass.cc | 38 +++++++++++++++++++ .../pir/transforms/shape_optimization_pass.h | 10 +++++ paddle/fluid/pybind/pir.cc | 21 +--------- test/ir/pir/cinn/symbolic/CMakeLists.txt | 14 +++++++ .../test_decomp_inference_predictor_run.py | 7 ++-- 6 files changed, 77 insertions(+), 23 deletions(-) rename test/ir/{inference => pir/cinn/symbolic}/test_decomp_inference_predictor_run.py (96%) diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index d52f71573dc44..35ff7eb608b6a 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -131,6 +131,7 @@ #include "paddle/fluid/pir/transforms/params_sync_among_devices_pass.h" #include "paddle/fluid/pir/transforms/pd_op_to_kernel_pass.h" #include "paddle/fluid/pir/transforms/replace_fetch_with_shadow_output_pass.h" +#include "paddle/fluid/pir/transforms/shape_optimization_pass.h" #include "paddle/pir/include/pass/pass_manager.h" COMMON_DECLARE_bool(enable_pir_in_executor); @@ -896,12 +897,19 @@ bool AnalysisPredictor::PrepareExecutor() { pir_program_ = std::move( paddle::TranslateLegacyProgramToProgram(*inference_program_)); +#ifdef PADDLE_WITH_CINN if (paddle::prim::PrimCommonUtils::IsFwdPrimEnabled()) { VLOG(4) << "[Prim] Decomp program in predictor begin."; DecompProgram decomp_object(pir_program_.get()); decomp_object.decomp_program(); + + auto shape_pm = std::make_shared<::pir::PassManager>( + ::pir::IrContext::Instance(), 2); + ::pir::shape::AddShapeOptimizationPass(shape_pm, *pir_program_.get()); + VLOG(4) << "[ShapeDialect] Run AddShapeOptimizationPass"; + shape_pm->Run(pir_program_.get()); } -#ifdef PADDLE_WITH_CINN + if (config_.cinn_enabled()) { VLOG(4) << "[CINN] Begin ApplyCinnPass"; cinn::dialect::ir::ApplyCinnPass(pir_program_.get(), [&] { diff --git a/paddle/fluid/pir/transforms/shape_optimization_pass.cc b/paddle/fluid/pir/transforms/shape_optimization_pass.cc index 80d56f75ae12b..d9cf96f78efe9 100644 --- a/paddle/fluid/pir/transforms/shape_optimization_pass.cc +++ b/paddle/fluid/pir/transforms/shape_optimization_pass.cc @@ -13,12 +13,16 @@ // limitations under the License. #include "paddle/fluid/pir/transforms/shape_optimization_pass.h" +#include "paddle/common/flags.h" #include "paddle/fluid/pir/dialect/operator/ir/pd_op.h" #include "paddle/pir/include/core/dialect.h" #include "paddle/pir/include/dialect/shape/ir/shape_attribute.h" +#include "paddle/pir/include/dialect/shape/ir/shape_dialect.h" #include "paddle/pir/include/pass/pass_manager.h" #include "paddle/pir/include/pass/pass_registry.h" +COMMON_DECLARE_bool(pir_apply_shape_optimization_pass); + const int vlog_level = 3; namespace pir { @@ -155,4 +159,38 @@ std::unique_ptr CreateShapeOptimizationPass() { } // namespace pir +namespace pir::shape { + +bool HasDynamicShape(const pir::Program& program) { + for (const auto& op : *program.block()) { + if (op.isa()) { + continue; + } + for (uint32_t i = 0; i < op.num_results(); ++i) { + if (op.result(i) && op.result(i).type()) { + auto shape_type = + op.result(i).type().dyn_cast(); + if (shape_type && shape_type.IsDynamicShape()) { + VLOG(vlog_level) << "###### HasDynamicShape == true"; + return true; + } + } + } + } + VLOG(vlog_level) << "###### HasDynamicShape == false"; + return false; +} + +void AddShapeOptimizationPass( + std::shared_ptr& pass_manager, // NOLINT + pir::Program& program) { // NOLINT + pir::IrContext* ctx = pir::IrContext::Instance(); + ctx->GetOrRegisterDialect(); + if (HasDynamicShape(program) && FLAGS_pir_apply_shape_optimization_pass) { + pass_manager->AddPass(pir::CreateShapeOptimizationPass()); + } +} + +} // namespace pir::shape + REGISTER_IR_PASS(shape_optimization_pass, pir::ShapeOptimizationPass); diff --git a/paddle/fluid/pir/transforms/shape_optimization_pass.h b/paddle/fluid/pir/transforms/shape_optimization_pass.h index a23de56f35d6e..5050ea727e678 100644 --- a/paddle/fluid/pir/transforms/shape_optimization_pass.h +++ b/paddle/fluid/pir/transforms/shape_optimization_pass.h @@ -17,6 +17,7 @@ #include #include "paddle/pir/include/core/dll_decl.h" #include "paddle/pir/include/dialect/shape/utils/shape_analysis.h" +#include "paddle/pir/include/pass/pass_manager.h" namespace pir { @@ -28,3 +29,12 @@ void InferSymExprForBlock(const Block &block, ShapeConstraintIRAnalysis *shape_analysis); } // namespace pir + +namespace pir::shape { +bool HasDynamicShape(const pir::Program &program); + +void AddShapeOptimizationPass( + std::shared_ptr &pass_manager, // NOLINT + pir::Program &program); // NOLINT + +} // namespace pir::shape diff --git a/paddle/fluid/pybind/pir.cc b/paddle/fluid/pybind/pir.cc index bd603e326a9ad..45fe7263e692c 100644 --- a/paddle/fluid/pybind/pir.cc +++ b/paddle/fluid/pybind/pir.cc @@ -1537,24 +1537,6 @@ void BindUtils(pybind11::module *m) { namespace { -bool HasDynamicShape(const pir::Program &program) { - for (const auto &op : *program.block()) { - if (op.isa()) { - continue; - } - for (uint32_t i = 0; i < op.num_results(); ++i) { - if (op.result(i) && op.result(i).type()) { - auto shape_type = - op.result(i).type().dyn_cast(); - if (shape_type && shape_type.IsDynamicShape()) { - return true; - } - } - } - } - return false; -} - void ApplyCinnPass(Program &program) { // NOLINT #ifdef PADDLE_WITH_CINN cinn::dialect::ir::ApplyCinnPass(&program, [] { @@ -1582,7 +1564,8 @@ void InferSymbolicShapePass( pir::Program &program) { // NOLINT pir::IrContext *ctx = pir::IrContext::Instance(); ctx->GetOrRegisterDialect(); - if (HasDynamicShape(program) && FLAGS_pir_apply_shape_optimization_pass) { + if (pir::shape::HasDynamicShape(program) && + FLAGS_pir_apply_shape_optimization_pass) { pass_manager->AddPass(pir::CreateShapeOptimizationPass()); } } diff --git a/test/ir/pir/cinn/symbolic/CMakeLists.txt b/test/ir/pir/cinn/symbolic/CMakeLists.txt index 665d1a0b0461d..9f26f4dd17269 100644 --- a/test/ir/pir/cinn/symbolic/CMakeLists.txt +++ b/test/ir/pir/cinn/symbolic/CMakeLists.txt @@ -11,6 +11,7 @@ if(WITH_GPU) test_if_st.py test_if_dy.py test_llama_if_dy.py + test_decomp_inference_predictor_run.py test_sub_graph_for_backend.py test_sub_graph_for_frontend.py test_check_infer_symbolic.py @@ -70,6 +71,19 @@ if(WITH_GPU) WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_tests_properties(test_llama_if_dy PROPERTIES LABELS "RUN_TYPE=CINN") + add_test( + NAME test_decomp_inference_predictor_run + COMMAND + ${CMAKE_COMMAND} -E env + PYTHONPATH=${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/python/:$ENV{PYTHONPATH} + FLAGS_prim_all=true FLAGS_cinn_bucket_compile=false + FLAGS_pir_apply_shape_optimization_pass=true + FLAGS_prim_enable_dynamic=true ${PYTHON_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/test_decomp_inference_predictor_run.py + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + set_tests_properties(test_decomp_inference_predictor_run + PROPERTIES LABELS "RUN_TYPE=CINN") + add_test( NAME test_cinn_reduce_symbolic_demo COMMAND diff --git a/test/ir/inference/test_decomp_inference_predictor_run.py b/test/ir/pir/cinn/symbolic/test_decomp_inference_predictor_run.py similarity index 96% rename from test/ir/inference/test_decomp_inference_predictor_run.py rename to test/ir/pir/cinn/symbolic/test_decomp_inference_predictor_run.py index 687f28c1bcf15..0a9c091f05ee7 100644 --- a/test/ir/inference/test_decomp_inference_predictor_run.py +++ b/test/ir/pir/cinn/symbolic/test_decomp_inference_predictor_run.py @@ -32,8 +32,7 @@ def forward(self, x1, x2): y1 = self.fc1(x1) y2 = self.fc2(x2) y3 = y1 + y2 - y4 = paddle.nn.functional.layer_norm(y3, y3.shape[1:]) - z = paddle.nn.functional.softmax(y4) + z = paddle.nn.functional.softmax(y3) return z @@ -50,7 +49,9 @@ def setUp(self): net, input_spec=[ paddle.static.InputSpec( - shape=self.shape, dtype='float32', name='input0' + shape=[None, None, None, None], + dtype='float32', + name='input0', ), paddle.static.InputSpec( shape=self.shape, dtype='float32', name='input1' From 754079f9df70864300458e4bfb5e33c50d9cc527 Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Fri, 1 Mar 2024 09:49:35 +0800 Subject: [PATCH 204/282] [PIR] Add missing assign for divide with scalar (#62252) --- python/paddle/pir/math_op_patch.py | 2 +- test/legacy_test/test_math_op_patch_pir.py | 26 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/python/paddle/pir/math_op_patch.py b/python/paddle/pir/math_op_patch.py index a14e8e8c9b90b..925c5b805c9fa 100644 --- a/python/paddle/pir/math_op_patch.py +++ b/python/paddle/pir/math_op_patch.py @@ -338,7 +338,7 @@ def __impl__(self, other_var): python_api == paddle.divide and self.dtype in _supported_int_dtype_ ): - paddle.cast(self, DataType.FLOAT32) + self = paddle.cast(self, DataType.FLOAT32) # here use `scale` replace `elementwise` to get better performance # but only +, -, *, / can use this method if scalar_method is not None: diff --git a/test/legacy_test/test_math_op_patch_pir.py b/test/legacy_test/test_math_op_patch_pir.py index 8862882d89985..12bcebbb3b5f0 100644 --- a/test/legacy_test/test_math_op_patch_pir.py +++ b/test/legacy_test/test_math_op_patch_pir.py @@ -643,6 +643,32 @@ def test_math_exists(self): self.assertTrue(inspect.ismethod(a.asinh_)) self.assertTrue(inspect.ismethod(a.diag)) + def test_binary_op_with_scalar(self): + with paddle.pir_utils.IrGuard(): + main_program, exe, program_guard = new_program() + with program_guard: + x_np = np.array(10, dtype=np.int32) + x = paddle.static.data(name='x', shape=[], dtype="int32") + y1 = x / 2 + y2 = x / 5.0 + y3 = x // 2 + y4 = x * 8.0 + self.assertEqual(y1.dtype, paddle.pir.core.DataType.FLOAT32) + self.assertEqual(y2.dtype, paddle.pir.core.DataType.FLOAT32) + self.assertEqual(y3.dtype, paddle.pir.core.DataType.INT32) + self.assertEqual(y4.dtype, paddle.pir.core.DataType.FLOAT32) + (y1_out, y2_out, y3_out, y4_out) = exe.run( + main_program, + feed={ + "x": x_np, + }, + fetch_list=[y1, y2, y3, y4], + ) + np.testing.assert_allclose(x_np / 2, y1_out, rtol=1e-05) + np.testing.assert_allclose(x_np / 5.0, y2_out, rtol=1e-05) + np.testing.assert_allclose(x_np // 2, y3_out, atol=1e-05) + np.testing.assert_allclose(x_np * 8.0, y4_out, rtol=1e-05) + if __name__ == '__main__': unittest.main() From d7f26ef4a51175531c31007c596f5abed1327369 Mon Sep 17 00:00:00 2001 From: wanghuancoder Date: Fri, 1 Mar 2024 09:53:29 +0800 Subject: [PATCH 205/282] pir onednn sgd (#62244) --- paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml index 283761ec09903..c76336addc9dc 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml @@ -238,9 +238,7 @@ - op : scale -- op : sgd - -# - op : sgd_dense_param_sparse_grad +- op : sgd_ - op : shape extra_args : str mkldnn_data_type="float32" From ebc27f54db86b70196758c519aea5418674e691c Mon Sep 17 00:00:00 2001 From: wanghuancoder Date: Fri, 1 Mar 2024 10:10:08 +0800 Subject: [PATCH 206/282] [PIR] pir onednn support split (#62238) * pir onednn support split --- .../ir_adaptor/translator/op_translator.cc | 18 +++++++++++++++--- .../dialect/operator/ir/ops_onednn_extra.yaml | 5 +++-- test/mkldnn/test_split_bf16_mkldnn_op.py | 2 +- test/mkldnn/test_split_mkldnn_op.py | 14 +++++++++++--- 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/paddle/fluid/ir_adaptor/translator/op_translator.cc b/paddle/fluid/ir_adaptor/translator/op_translator.cc index 1c75d198ef07d..c4ad629fc3d91 100644 --- a/paddle/fluid/ir_adaptor/translator/op_translator.cc +++ b/paddle/fluid/ir_adaptor/translator/op_translator.cc @@ -1255,6 +1255,16 @@ struct SplitOpTranscriber : public OpTranscriber { return attribute_map; } +#ifdef PADDLE_WITH_DNNL + else if (op_desc.HasAttr("mkldnn_data_type")) { // NOLINT + pir::AttributeMap attribute_map = { + {"mkldnn_data_type", + pir::StrAttribute::get( + ctx, op_desc.GetAttrIfExists("mkldnn_data_type"))}, + }; + return attribute_map; + } +#endif return {}; } @@ -1262,17 +1272,19 @@ struct SplitOpTranscriber : public OpTranscriber { pir::OpInfo LookUpOpInfo(pir::IrContext* ctx, const OpDesc& op_desc) override { int num = paddle::get(op_desc.GetAttr("num")); + auto prefix = GetPrefix(ctx, op_desc); std::string target_op_name; if (num > 0) { - target_op_name = "pd_op.split_with_num"; + target_op_name = prefix + "split_with_num"; } else { - target_op_name = "pd_op.split"; + target_op_name = prefix + "split"; } const auto& op_info = ctx->GetRegisteredOpInfo(target_op_name); if (!op_info) { - IR_THROW("Op assign_value should have corresponding OpInfo pd_op.split"); + IR_THROW("Op assign_value should have corresponding OpInfo %s.", + target_op_name); } return op_info; diff --git a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml index c76336addc9dc..af136f8a518b5 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml @@ -265,9 +265,10 @@ - op : softplus -# - op : split +- op : split + extra_args : str mkldnn_data_type="float32" -# - op : split_with_num +- op : split_with_num - op : sqrt diff --git a/test/mkldnn/test_split_bf16_mkldnn_op.py b/test/mkldnn/test_split_bf16_mkldnn_op.py index 6e8b1b56ebc07..c9297de55fae5 100644 --- a/test/mkldnn/test_split_bf16_mkldnn_op.py +++ b/test/mkldnn/test_split_bf16_mkldnn_op.py @@ -64,7 +64,7 @@ def setUp(self): } def test_check_output(self): - self.check_output_with_place(core.CPUPlace()) + self.check_output_with_place(core.CPUPlace(), check_pir_onednn=True) class TestSplitNumBF16OneDNNOp(TestSplitSectionsBF16OneDNNOp): diff --git a/test/mkldnn/test_split_mkldnn_op.py b/test/mkldnn/test_split_mkldnn_op.py index 15a24c3b4861f..14e39ab0c01fd 100644 --- a/test/mkldnn/test_split_mkldnn_op.py +++ b/test/mkldnn/test_split_mkldnn_op.py @@ -68,10 +68,15 @@ def setUp(self): } def test_check_output(self): - self.check_output(check_dygraph=False) + self.check_output(check_dygraph=False, check_pir_onednn=True) def test_check_grad(self): - self.check_grad(['X'], ['out0', 'out1', 'out2'], check_dygraph=False) + self.check_grad( + ['X'], + ['out0', 'out1', 'out2'], + check_dygraph=False, + check_pir_onednn=True, + ) # test with attr(num) @@ -87,7 +92,10 @@ def init_test_case(self): def test_check_grad(self): self.check_grad( - ['X'], ['out0', 'out1', 'out2', 'out3'], check_dygraph=False + ['X'], + ['out0', 'out1', 'out2', 'out3'], + check_dygraph=False, + check_pir_onednn=True, ) From 3ce483b52ef4c696dccd9534ccc91998432101de Mon Sep 17 00:00:00 2001 From: winter-wang <78149749+winter-wang@users.noreply.github.com> Date: Fri, 1 Mar 2024 10:10:24 +0800 Subject: [PATCH 207/282] [PIR] add distributed dialect. (#61978) --- paddle/fluid/pir/dialect/CMakeLists.txt | 6 + .../distributed/ir/attribute_storage.h | 118 ++++++++++++++++ .../dialect/distributed/ir/dist_attribute.cc | 73 ++++++++++ .../dialect/distributed/ir/dist_attribute.h | 101 ++++++++++++++ .../dialect/distributed/ir/dist_dialect.cc | 62 +++++++++ .../pir/dialect/distributed/ir/dist_dialect.h | 41 ++++++ .../pir/dialect/distributed/ir/dist_type.cc | 43 ++++++ .../pir/dialect/distributed/ir/dist_type.h | 61 +++++++++ .../pir/dialect/distributed/ir/type_storage.h | 81 +++++++++++ paddle/fluid/pybind/pybind.cc | 3 + paddle/pir/include/core/attribute.h | 7 +- paddle/pir/include/core/attribute_base.h | 12 +- paddle/pir/include/core/storage_manager.h | 2 +- .../include/core/storage_manager_support.h | 8 +- paddle/pir/include/core/type.h | 8 +- test/cpp/pir/CMakeLists.txt | 1 + test/cpp/pir/distributed/CMakeLists.txt | 3 + test/cpp/pir/distributed/dist_dialect_test.cc | 127 ++++++++++++++++++ 18 files changed, 743 insertions(+), 14 deletions(-) create mode 100644 paddle/fluid/pir/dialect/distributed/ir/attribute_storage.h create mode 100644 paddle/fluid/pir/dialect/distributed/ir/dist_attribute.cc create mode 100644 paddle/fluid/pir/dialect/distributed/ir/dist_attribute.h create mode 100644 paddle/fluid/pir/dialect/distributed/ir/dist_dialect.cc create mode 100644 paddle/fluid/pir/dialect/distributed/ir/dist_dialect.h create mode 100644 paddle/fluid/pir/dialect/distributed/ir/dist_type.cc create mode 100644 paddle/fluid/pir/dialect/distributed/ir/dist_type.h create mode 100644 paddle/fluid/pir/dialect/distributed/ir/type_storage.h create mode 100644 test/cpp/pir/distributed/CMakeLists.txt create mode 100644 test/cpp/pir/distributed/dist_dialect_test.cc diff --git a/paddle/fluid/pir/dialect/CMakeLists.txt b/paddle/fluid/pir/dialect/CMakeLists.txt index 2955a6d57afb5..d5050b49ac582 100644 --- a/paddle/fluid/pir/dialect/CMakeLists.txt +++ b/paddle/fluid/pir/dialect/CMakeLists.txt @@ -255,6 +255,12 @@ if(WITH_MKLDNN) ${CMAKE_CURRENT_SOURCE_DIR}/operator/ir/manual_onednn_op.cc) endif() +file(GLOB_RECURSE dist_dialect_srcs + "${CMAKE_CURRENT_SOURCE_DIR}/distributed/ir/*.cc") + +if(WITH_DISTRIBUTE) + set(op_dialect_srcs ${op_dialect_srcs} ${dist_dialect_srcs}) +endif() set(op_dialect_deps phi common pir type_info string_helper) cc_library( diff --git a/paddle/fluid/pir/dialect/distributed/ir/attribute_storage.h b/paddle/fluid/pir/dialect/distributed/ir/attribute_storage.h new file mode 100644 index 0000000000000..f572e5dae762b --- /dev/null +++ b/paddle/fluid/pir/dialect/distributed/ir/attribute_storage.h @@ -0,0 +1,118 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/common/ddim.h" +#include "paddle/common/hash_funcs.h" +#include "paddle/common/layout.h" +#include "paddle/fluid/pir/dialect/distributed/ir/dist_attribute.h" +#include "paddle/phi/common/reduce_type.h" +#include "paddle/pir/include/core/attribute_base.h" +#include "paddle/pir/include/core/builtin_type.h" +#include "paddle/pir/include/core/utils.h" +#include "paddle/utils/flat_hash_map.h" + +namespace paddle { +namespace dialect { + +struct ProcessMeshAttrStorage : public pir::AttributeStorage { + /// + /// \brief Declare ParamKey according to parameter type. + /// + using ParamKey = phi::distributed::ProcessMesh; + + ProcessMeshAttrStorage(ParamKey&& process_mesh) // NOLINT + : process_mesh(std::move(process_mesh)) {} + + /// + /// \brief Each derived TypeStorage must define a Construct method, which + /// StorageManager uses to construct a derived TypeStorage. + /// + static ProcessMeshAttrStorage* Construct(ParamKey&& key) { + return new ProcessMeshAttrStorage(std::move(key)); + } + + /// + /// \brief Each derived TypeStorage must provide a HashValue method. + /// + static std::size_t HashValue(const ParamKey& key) { return key.hash(); } + + /// + /// \brief Each derived TypeStorage needs to overload operator==. + /// + bool operator==(const ParamKey& key) const { + return process_mesh == key && process_mesh.dim_names() == key.dim_names(); + } + + ParamKey process_mesh; +}; + +struct TensorDistAttrStorage : public pir::AttributeStorage { + /// + /// \brief Declare ParamKey according to parameter type. + /// + using ParamKey = std::tuple, + flat_hash_map>; + + TensorDistAttrStorage(ParamKey&& param) // NOLINT + : process_mesh(std::get<0>(param)), + dims_mapping(std::move(std::get<1>(param))), + partial_status(std::move(std::get<2>(param))) {} + /// + /// \brief Each derived TypeStorage must define a Construct method, which + /// StorageManager uses to construct a derived TypeStorage. + /// + static TensorDistAttrStorage* Construct(ParamKey&& key) { + return new TensorDistAttrStorage(std::move(key)); + } + + /// + /// \brief Each derived TypeStorage must provide a HashValue method. + /// + static std::size_t HashValue(const ParamKey& key) { + auto mesh_hash = std::get<0>(key).hash(); + auto dims_map_hash = std::hash>()(std::get<1>(key)); + std::string partial_status_str = "["; + for (auto& itr : std::get<2>(key)) { + partial_status_str += + "Partial(dims:" + std::to_string(itr.first) + ", " + + phi::ReduceTypeStrings[static_cast(itr.second)] + "), "; + } + partial_status_str += "]"; + auto combine_hash = pir::detail::hash_combine(mesh_hash, dims_map_hash); + return pir::detail::hash_combine( + combine_hash, std::hash()(partial_status_str)); + } + + /// + /// \brief Each derived TypeStorage needs to overload operator==. + /// + bool operator==(const ParamKey& key) const { + return process_mesh == std::get<0>(key) && + dims_mapping == std::get<1>(key) && + partial_status == std::get<2>(key); + } + + ProcessMeshAttribute process_mesh; + std::vector dims_mapping; + // partial map would less or equal than to mesh.size. + // iterate operation (copy and comparison) would more frequency than random + // element access. + flat_hash_map partial_status; +}; + +} // namespace dialect +} // namespace paddle diff --git a/paddle/fluid/pir/dialect/distributed/ir/dist_attribute.cc b/paddle/fluid/pir/dialect/distributed/ir/dist_attribute.cc new file mode 100644 index 0000000000000..372d6206c2be8 --- /dev/null +++ b/paddle/fluid/pir/dialect/distributed/ir/dist_attribute.cc @@ -0,0 +1,73 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/pir/dialect/distributed/ir/dist_attribute.h" +#include "paddle/fluid/pir/dialect/distributed/ir/attribute_storage.h" +namespace paddle { +namespace dialect { +/// +/// \brief ProcessMeshAttribute interface. +/// +const phi::distributed::ProcessMesh& ProcessMeshAttribute::process_mesh() + const { + return storage()->process_mesh; +} +ProcessMeshAttribute ProcessMeshAttribute::get( + pir::IrContext* ctx, const phi::distributed::ProcessMesh& mesh) { + return Base::get(ctx, mesh); +} +ProcessMeshAttribute ProcessMeshAttribute::get( + pir::IrContext* ctx, + const std::vector& shape, + const std::vector& process_ids, + const std::vector& dim_names) { + return Base::get(ctx, shape, process_ids, dim_names); +} + +/// +/// \brief TensorDistAttribute interface. +/// +ProcessMeshAttribute TensorDistAttribute::mesh_attr() const { + return storage()->process_mesh; +} +const std::vector& TensorDistAttribute::dims_mapping() const { + return storage()->dims_mapping; +} + +std::set TensorDistAttribute::partial_dims() const { + auto& partial = partial_status(); + std::set keys; + for (auto& kv : partial) { + keys.emplace(kv.first); + } + return keys; +} + +const flat_hash_map& +TensorDistAttribute::partial_status() const { + return storage()->partial_status; +} + +TensorDistAttribute TensorDistAttribute::get( + pir::IrContext* ctx, + ProcessMeshAttribute mesh, + const std::vector& dims_mapping, + const flat_hash_map& partial_status) { + return Base::get(ctx, mesh, dims_mapping, partial_status); +} + +} // namespace dialect +} // namespace paddle +IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::ProcessMeshAttribute) +IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::TensorDistAttribute) diff --git a/paddle/fluid/pir/dialect/distributed/ir/dist_attribute.h b/paddle/fluid/pir/dialect/distributed/ir/dist_attribute.h new file mode 100644 index 0000000000000..1ee05404a3df9 --- /dev/null +++ b/paddle/fluid/pir/dialect/distributed/ir/dist_attribute.h @@ -0,0 +1,101 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/phi/common/reduce_type.h" +#include "paddle/phi/core/distributed/auto_parallel/process_mesh.h" +#include "paddle/pir/include/core/attribute.h" +#include "paddle/pir/include/core/builtin_attribute_storage.h" +#include "paddle/pir/include/core/utils.h" +#include "paddle/utils/flat_hash_map.h" + +namespace paddle { +namespace dialect { +class ProcessMeshAttrStorage; +class TensorDistAttrStorage; + +class ProcessMeshAttribute : public pir::AttrBase { + public: + using Base::Base; + const phi::distributed::ProcessMesh& process_mesh() const; + const std::vector& shape() const { return process_mesh().shape(); } + const std::vector& process_ids() const { + return process_mesh().process_ids(); + } + const std::vector& dim_names() const { + return process_mesh().dim_names(); + } + int64_t size() const { return process_mesh().size(); } + int64_t ndim() const { return process_mesh().ndim(); } + int64_t dim_size(int64_t dim) const { return process_mesh().dim_size(dim); } + int64_t dim_size(const std::string& dim_name) const { + return process_mesh().dim_size(dim_name); + } + bool empty() const { return process_mesh().empty(); } + bool contains(int64_t process_id) const { + return process_mesh().contains(process_id); + } + size_t hash() const { return process_mesh().hash(); } + + std::string to_string() const { return process_mesh().to_string(); } + + static ProcessMeshAttribute get(pir::IrContext* ctx, + const phi::distributed::ProcessMesh& mesh); + static ProcessMeshAttribute get(pir::IrContext* ctx, + const std::vector& shape, + const std::vector& process_ids, + const std::vector& dim_names); +}; + +class TensorDistAttribute : public pir::AttrBase { + public: + using Base::Base; + ProcessMeshAttribute mesh_attr() const; + const phi::distributed::ProcessMesh& process_mesh() const { + return mesh_attr().process_mesh(); + } + const std::vector& dims_mapping() const; + + // return vector of mesh dims on which the this tensor is partial on + std::set partial_dims() const; + + const flat_hash_map& partial_status() const; + + static TensorDistAttribute get( + pir::IrContext* ctx, + ProcessMeshAttribute mesh, + const std::vector& dims_mapping, + const flat_hash_map& partial_status); + static TensorDistAttribute get( + pir::IrContext* ctx, + const phi::distributed::ProcessMesh& mesh, + const std::vector& dims_mapping, + const flat_hash_map& partial_status) { + return get(ctx, + ProcessMeshAttribute::get(ctx, mesh), + dims_mapping, + partial_status); + } +}; + +} // namespace dialect +} // namespace paddle + +IR_DECLARE_EXPLICIT_TYPE_ID(paddle::dialect::ProcessMeshAttribute) +IR_DECLARE_EXPLICIT_TYPE_ID(paddle::dialect::TensorDistAttribute) diff --git a/paddle/fluid/pir/dialect/distributed/ir/dist_dialect.cc b/paddle/fluid/pir/dialect/distributed/ir/dist_dialect.cc new file mode 100644 index 0000000000000..5329c0086d742 --- /dev/null +++ b/paddle/fluid/pir/dialect/distributed/ir/dist_dialect.cc @@ -0,0 +1,62 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/pir/dialect/distributed/ir/dist_dialect.h" +#include "paddle/fluid/pir/dialect/distributed/ir/attribute_storage.h" +#include "paddle/fluid/pir/dialect/distributed/ir/dist_attribute.h" +#include "paddle/fluid/pir/dialect/distributed/ir/dist_type.h" +#include "paddle/fluid/pir/dialect/distributed/ir/type_storage.h" + +REGISTER_FILE_SYMBOLS(dist_dialect); +namespace paddle { +namespace dialect { + +DistDialect::DistDialect(pir::IrContext *context) + : pir::Dialect(name(), context, pir::TypeId::get()) { + initialize(); +} + +void DistDialect::initialize() { + RegisterAttributes(); + RegisterTypes(); +} + +void DistDialect::PrintType(pir::Type type, std::ostream &os) const { + if (auto dist_dense_tensor_type = type.dyn_cast()) { + // Todo: Design the dist dense tensor type print format. + os << dist_dense_tensor_type.dense_tensor_type(); + } else { + os << "error_type!"; + } +} + +void DistDialect::PrintAttribute(pir::Attribute attr, std::ostream &os) const { + if (auto process_mesh_attr = attr.dyn_cast()) { + os << process_mesh_attr.process_mesh(); + } else if (auto tensor_dist_attr = attr.dyn_cast()) { + // Todo: Design the tensor dist attr print format. + os << tensor_dist_attr.process_mesh(); + } else { + os << "error_attribute_type"; + } +} + +pir::OpPrintFn DistDialect::PrintOperation(pir::Operation *op) const { + return nullptr; +} + +} // namespace dialect +} // namespace paddle + +IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::DistDialect) diff --git a/paddle/fluid/pir/dialect/distributed/ir/dist_dialect.h b/paddle/fluid/pir/dialect/distributed/ir/dist_dialect.h new file mode 100644 index 0000000000000..2a7420b0a495a --- /dev/null +++ b/paddle/fluid/pir/dialect/distributed/ir/dist_dialect.h @@ -0,0 +1,41 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/pir/include/core/dialect.h" + +namespace paddle { +namespace dialect { + +class DistDialect : public pir::Dialect { + public: + explicit DistDialect(pir::IrContext* context); + + static const char* name() { return "pd_dist"; } + + void PrintType(pir::Type type, std::ostream& os) const override; + + void PrintAttribute(pir::Attribute attr, std::ostream& os) const override; + + pir::OpPrintFn PrintOperation(pir::Operation* op) const override; // NOLINT + + private: + void initialize(); +}; + +} // namespace dialect +} // namespace paddle + +IR_DECLARE_EXPLICIT_TYPE_ID(paddle::dialect::DistDialect) diff --git a/paddle/fluid/pir/dialect/distributed/ir/dist_type.cc b/paddle/fluid/pir/dialect/distributed/ir/dist_type.cc new file mode 100644 index 0000000000000..94a2d85fbcdd7 --- /dev/null +++ b/paddle/fluid/pir/dialect/distributed/ir/dist_type.cc @@ -0,0 +1,43 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/pir/dialect/distributed/ir/dist_type.h" +#include "paddle/fluid/pir/dialect/distributed/ir/type_storage.h" + +namespace paddle { +namespace dialect { + +pir::DenseTensorType DistDenseTensorType::dense_tensor_type() const { + return storage()->dense_tensor_type; +} + +TensorDistAttribute DistDenseTensorType::tensor_dist_attr() const { + return storage()->tensor_dist_attr; +} + +const common::DDim& DistDenseTensorType::global_ddim() const { + return storage()->global_ddim; +} + +DistDenseTensorType DistDenseTensorType::get( + pir::IrContext* ctx, + pir::DenseTensorType dense_tensor_type, + TensorDistAttribute tensor_dist_attr, + const common::DDim& global_ddim) { + return Base::get(ctx, dense_tensor_type, tensor_dist_attr, global_ddim); +} +} // namespace dialect +} // namespace paddle + +IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::DistDenseTensorType) diff --git a/paddle/fluid/pir/dialect/distributed/ir/dist_type.h b/paddle/fluid/pir/dialect/distributed/ir/dist_type.h new file mode 100644 index 0000000000000..4aa08169440cc --- /dev/null +++ b/paddle/fluid/pir/dialect/distributed/ir/dist_type.h @@ -0,0 +1,61 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/fluid/pir/dialect/distributed/ir/dist_attribute.h" +#include "paddle/pir/include/core/builtin_type.h" +#include "paddle/pir/include/core/type.h" + +namespace paddle { +namespace dialect { + +class DistDenseTensorTypeStorage; + +class DistDenseTensorType + : public pir::Type:: + TypeBase { + public: + using Base::Base; + + pir::DenseTensorType dense_tensor_type() const; + TensorDistAttribute tensor_dist_attr() const; + const common::DDim& global_ddim() const; + const common::DDim& local_ddim() const { return dense_tensor_type().dims(); } + Type dtype() const { return dense_tensor_type().dtype(); } + DataLayout data_layout() const { return dense_tensor_type().data_layout(); } + + const phi::distributed::ProcessMesh& process_mesh() const { + return tensor_dist_attr().process_mesh(); + } + const std::vector& dims_mapping() const { + return tensor_dist_attr().dims_mapping(); + } + std::set partial_dims() const { + return tensor_dist_attr().partial_dims(); + } + const flat_hash_map& partial_status() const { + return tensor_dist_attr().partial_status(); + } + + static DistDenseTensorType get(pir::IrContext* ctx, + pir::DenseTensorType dense_tensor_type, + TensorDistAttribute tensor_dist_attr, + const common::DDim& global_ddim); +}; + +} // namespace dialect +} // namespace paddle + +IR_DECLARE_EXPLICIT_TYPE_ID(paddle::dialect::DistDenseTensorType) diff --git a/paddle/fluid/pir/dialect/distributed/ir/type_storage.h b/paddle/fluid/pir/dialect/distributed/ir/type_storage.h new file mode 100644 index 0000000000000..1f18573d3e162 --- /dev/null +++ b/paddle/fluid/pir/dialect/distributed/ir/type_storage.h @@ -0,0 +1,81 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include "paddle/fluid/pir/dialect/distributed/ir/dist_attribute.h" +#include "paddle/pir/include/core/builtin_type.h" + +namespace paddle { +namespace dialect { +/// +/// \brief Define Parametric TypeStorage for DistDenseTensorType. +/// +struct DistDenseTensorTypeStorage : public pir::TypeStorage { + /// + /// \brief Declare ParamKey according to parameter type. + /// + using ParamKey = + std::tuple; + + DistDenseTensorTypeStorage(pir::DenseTensorType dense_tensor_type, + TensorDistAttribute tensor_dist_attr, + const common::DDim& global_ddim) + : dense_tensor_type(dense_tensor_type), + tensor_dist_attr(tensor_dist_attr), + global_ddim(global_ddim) {} + + /// + /// \brief Each derived TypeStorage must define a Construct method, which + /// StorageManager uses to construct a derived TypeStorage. + /// + static DistDenseTensorTypeStorage* Construct(ParamKey&& key) { + return new DistDenseTensorTypeStorage( + std::get<0>(key), std::get<1>(key), std::get<2>(key)); + } + + /// + /// \brief Each derived TypeStorage must provide a HashValue method. + /// + static std::size_t HashValue(const ParamKey& key) { + auto dense_tensor_type_hash = std::hash()(std::get<0>(key)); + auto tensor_dist_attr_hash = std::hash()(std::get<1>(key)); + auto global_ddim_hash = std::hash()(std::get<2>(key)); + auto value = pir::detail::hash_combine(dense_tensor_type_hash, + tensor_dist_attr_hash); + return pir::detail::hash_combine(value, global_ddim_hash); + } + + /// + /// \brief Each derived TypeStorage needs to overload operator==. + /// + bool operator==(const ParamKey& key) const { + return dense_tensor_type == std::get<0>(key) && + tensor_dist_attr == std::get<1>(key) && + global_ddim == std::get<2>(key); + } + + /// + /// \brief DistDenseTensorTypeStorage include three parameters: + /// dense_tensor_type, tensor_dist_attr and global_ddim; + /// + pir::DenseTensorType dense_tensor_type; + TensorDistAttribute tensor_dist_attr; + common::DDim global_ddim; +}; + +} // namespace dialect +} // namespace paddle diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index f1d53f3f88750..ffaef54bb9da9 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -223,6 +223,9 @@ PYBIND11_MAKE_OPAQUE(paddle::framework::FetchType); DECLARE_FILE_SYMBOLS(init_phi); DECLARE_FILE_SYMBOLS(kernel_dialect); +#ifdef PADDLE_WITH_DISTRIBUTE +DECLARE_FILE_SYMBOLS(dist_dialect); +#endif DECLARE_FILE_SYMBOLS(buffered_allocator); DECLARE_FILE_SYMBOLS(best_fit_allocator); DECLARE_FILE_SYMBOLS(aligned_allocator); diff --git a/paddle/pir/include/core/attribute.h b/paddle/pir/include/core/attribute.h index 9571440679b8c..2c1ca17656811 100644 --- a/paddle/pir/include/core/attribute.h +++ b/paddle/pir/include/core/attribute.h @@ -15,6 +15,7 @@ #pragma once #include "paddle/pir/include/core/cast_utils.h" +#include "paddle/pir/include/core/storage_manager_support.h" #include "paddle/pir/include/core/type_id.h" constexpr char kAttrStopGradients[] = "stop_gradient"; @@ -87,6 +88,8 @@ class IR_API Attribute { return pir::dyn_cast(*this); } + std::size_t hash() const { return std::hash()(storage_); } + protected: const Storage *storage_{nullptr}; }; @@ -97,8 +100,6 @@ IR_API std::ostream &operator<<(std::ostream &os, Attribute attr); namespace std { template <> struct hash { - std::size_t operator()(const pir::Attribute &obj) const { - return std::hash()(obj); - } + std::size_t operator()(const pir::Attribute &obj) const { return obj.hash(); } }; } // namespace std diff --git a/paddle/pir/include/core/attribute_base.h b/paddle/pir/include/core/attribute_base.h index d6c75f2e5d8ce..0f459f23e9f99 100644 --- a/paddle/pir/include/core/attribute_base.h +++ b/paddle/pir/include/core/attribute_base.h @@ -16,8 +16,8 @@ #include "paddle/pir/include/core/ir_context.h" #include "paddle/pir/include/core/storage_manager.h" +#include "paddle/pir/include/core/storage_manager_support.h" #include "paddle/pir/include/core/type_id.h" - namespace pir { class Dialect; @@ -239,6 +239,16 @@ struct IR_API AttributeManager { } }; +template +using AttrBase = detail::StorageHelperBase; + /// /// \brief Add some necessary functions to the custom Attribute class. /// diff --git a/paddle/pir/include/core/storage_manager.h b/paddle/pir/include/core/storage_manager.h index 8cacc3bd38bd0..7024e580e4a1f 100644 --- a/paddle/pir/include/core/storage_manager.h +++ b/paddle/pir/include/core/storage_manager.h @@ -74,7 +74,7 @@ class IR_API StorageManager { return static_cast(*existing) == param; }; auto constructor = [&]() { - auto *storage = Storage::Construct(param); + auto *storage = Storage::Construct(std::move(param)); if (init_func) init_func(storage); return storage; }; diff --git a/paddle/pir/include/core/storage_manager_support.h b/paddle/pir/include/core/storage_manager_support.h index 7d4d540382dcd..b729a4480ac35 100644 --- a/paddle/pir/include/core/storage_manager_support.h +++ b/paddle/pir/include/core/storage_manager_support.h @@ -18,8 +18,6 @@ #include "paddle/pir/include/core/interface_support.h" #include "paddle/pir/include/core/ir_context.h" -#include "paddle/pir/include/core/type.h" -#include "paddle/pir/include/core/type_base.h" #include "paddle/pir/include/core/type_id.h" namespace pir { @@ -68,7 +66,7 @@ class StorageHelperBase : public BaseT { typename Filter>::Type; static ConcreteT dyn_cast_impl(BaseT type) { - if (type && type.abstract_type().type_id() == TypeId::get()) { + if (type && type.type_id() == TypeId::get()) { return ConcreteT(type.storage()); } return ConcreteT(nullptr); @@ -107,8 +105,8 @@ class StorageHelperBase : public BaseT { /// \brief Get or create a new ConcreteT instance within the ctx. /// template - static ConcreteT get(pir::IrContext *ctx, Args... args) { - return ManagerT::template get(ctx, args...); + static ConcreteT get(pir::IrContext *ctx, Args &&...args) { + return ManagerT::template get(ctx, std::forward(args)...); } /// diff --git a/paddle/pir/include/core/type.h b/paddle/pir/include/core/type.h index 569b356135b18..fcfe0a77a8ac5 100644 --- a/paddle/pir/include/core/type.h +++ b/paddle/pir/include/core/type.h @@ -18,6 +18,7 @@ #include "paddle/pir/include/core/cast_utils.h" #include "paddle/pir/include/core/storage_manager_support.h" +#include "paddle/pir/include/core/type_base.h" #include "paddle/pir/include/core/type_id.h" namespace pir { @@ -42,7 +43,6 @@ class IR_API Type { StorageType, TypeManager, TraitOrInterface...>; - using Storage = TypeStorage; using AbstractT = AbstractType; @@ -125,6 +125,8 @@ class IR_API Type { bool IsIntOrIndex() const; bool IsIndex() const; + std::size_t hash() const { return std::hash()(storage_); } + protected: const Storage *storage_{nullptr}; @@ -184,8 +186,6 @@ namespace std { /// template <> struct hash { - std::size_t operator()(const pir::Type &obj) const { - return std::hash()(obj); - } + std::size_t operator()(const pir::Type &obj) const { return obj.hash(); } }; } // namespace std diff --git a/test/cpp/pir/CMakeLists.txt b/test/cpp/pir/CMakeLists.txt index 420ffa8b6dc5a..e7de653656897 100644 --- a/test/cpp/pir/CMakeLists.txt +++ b/test/cpp/pir/CMakeLists.txt @@ -7,3 +7,4 @@ add_subdirectory(cinn) add_subdirectory(control_flow_dialect) add_subdirectory(shape_dialect) add_subdirectory(sub_graph) +add_subdirectory(distributed) diff --git a/test/cpp/pir/distributed/CMakeLists.txt b/test/cpp/pir/distributed/CMakeLists.txt new file mode 100644 index 0000000000000..0483dbe1fdac0 --- /dev/null +++ b/test/cpp/pir/distributed/CMakeLists.txt @@ -0,0 +1,3 @@ +if(WITH_DISTRIBUTE) + paddle_test(dist_dialect_test SRCS dist_dialect_test.cc) +endif() diff --git a/test/cpp/pir/distributed/dist_dialect_test.cc b/test/cpp/pir/distributed/dist_dialect_test.cc new file mode 100644 index 0000000000000..01dcb2f1010d5 --- /dev/null +++ b/test/cpp/pir/distributed/dist_dialect_test.cc @@ -0,0 +1,127 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +#include + +#include "paddle/fluid/pir/dialect/distributed/ir/dist_attribute.h" +#include "paddle/fluid/pir/dialect/distributed/ir/dist_dialect.h" +#include "paddle/fluid/pir/dialect/distributed/ir/dist_type.h" +#include "paddle/fluid/pir/dialect/operator/ir/op_dialect.h" +#include "paddle/pir/include/core/builtin_type.h" + +using namespace paddle::dialect; // NOLINT + +TEST(process_mesh_test, base) { + pir::IrContext* ctx = pir::IrContext::Instance(); + ctx->GetOrRegisterDialect(); + std::vector mesh_shape = {2, 2}; + std::vector process_ids = {0, 1, 2, 3}; + std::vector dim_names = {"x", "y"}; + std::vector dim_names_2 = {"x", "s"}; + phi::distributed::ProcessMesh process_mesh( + mesh_shape, process_ids, dim_names); + + // construct a ProcessMeshAttribute. + auto mesh_attr = + ProcessMeshAttribute::get(ctx, mesh_shape, process_ids, dim_names); + auto mesh_attr_1 = ProcessMeshAttribute::get(ctx, process_mesh); + auto mesh_attr_2 = + ProcessMeshAttribute::get(ctx, mesh_shape, process_ids, dim_names_2); + EXPECT_EQ(mesh_attr, mesh_attr_1); + EXPECT_NE(mesh_attr, mesh_attr_2); + + // test member function. + EXPECT_EQ(mesh_attr.process_mesh(), process_mesh); + EXPECT_EQ(mesh_attr.shape(), mesh_shape); + EXPECT_EQ(mesh_attr.process_ids(), process_ids); + EXPECT_EQ(mesh_attr.dim_names(), dim_names); + EXPECT_EQ(mesh_attr.size(), 4); + EXPECT_EQ(mesh_attr.ndim(), 2); + EXPECT_EQ(mesh_attr.dim_size(0), 2); + EXPECT_EQ(mesh_attr.dim_size("y"), 2); + EXPECT_FALSE(mesh_attr.empty()); + EXPECT_TRUE(mesh_attr.contains(3)); + EXPECT_EQ(mesh_attr.hash(), process_mesh.hash()); + EXPECT_EQ(mesh_attr.to_string(), process_mesh.to_string()); +} +TEST(tensor_dist_attr_test, base) { + pir::IrContext* ctx = pir::IrContext::Instance(); + ctx->GetOrRegisterDialect(); + + std::vector mesh_shape = {2, 3}; + std::vector process_ids = {0, 1, 2, 3, 4, 5}; + std::vector dim_names = {"x", "y"}; + phi::distributed::ProcessMesh process_mesh( + mesh_shape, process_ids, dim_names); + std::vector dims_mapping = {0, -1}; + paddle::flat_hash_map partial_status, + partial_status_1{{1, phi::ReduceType::kRedSum}}; + + auto mesh_attr = + ProcessMeshAttribute::get(ctx, mesh_shape, process_ids, dim_names); + + // construct a TensorDistAttribute. + auto tensor_dist_attr = + TensorDistAttribute::get(ctx, mesh_attr, dims_mapping, partial_status); + auto tensor_dist_attr_1 = + TensorDistAttribute::get(ctx, process_mesh, dims_mapping, partial_status); + auto tensor_dist_attr_2 = TensorDistAttribute::get( + ctx, process_mesh, dims_mapping, partial_status_1); + EXPECT_EQ(tensor_dist_attr, tensor_dist_attr_1); + EXPECT_NE(tensor_dist_attr, tensor_dist_attr_2); + + // test member function. + EXPECT_EQ(tensor_dist_attr.mesh_attr(), mesh_attr); + EXPECT_EQ(tensor_dist_attr.process_mesh(), process_mesh); + EXPECT_EQ(tensor_dist_attr.dims_mapping(), dims_mapping); + EXPECT_EQ(tensor_dist_attr.partial_status(), partial_status); +} + +TEST(dist_dense_tensor_type_test, base) { + pir::IrContext* ctx = pir::IrContext::Instance(); + ctx->GetOrRegisterDialect(); + ctx->GetOrRegisterDialect(); + std::vector mesh_shape = {2, 3}; + std::vector process_ids = {0, 1, 2, 3, 4, 5}; + std::vector dim_names = {"x", "y"}; + phi::distributed::ProcessMesh process_mesh( + mesh_shape, process_ids, dim_names); + auto mesh_attr = ProcessMeshAttribute::get(ctx, process_mesh); + + std::vector dims_mapping = {0, -1}; + paddle::flat_hash_map partial_status{ + {1, phi::ReduceType::kRedSum}}; + // construct a TensorDistAttribute. + auto tensor_dist_attr = + TensorDistAttribute::get(ctx, mesh_attr, dims_mapping, partial_status); + + pir::Type fp32_dtype = pir::Float32Type::get(ctx); + common::DDim dims = {2, 2}; + common::DataLayout data_layout = common::DataLayout::NCHW; + pir::LoD lod = {{0, 1, 2}}; + size_t offset = 0; + pir::DenseTensorType dense_tensor_type = pir::DenseTensorType::get( + ctx, fp32_dtype, dims, data_layout, lod, offset); + + auto dist_densor_type = + DistDenseTensorType::get(ctx, dense_tensor_type, tensor_dist_attr, dims); + + EXPECT_EQ(dist_densor_type.process_mesh(), process_mesh); + EXPECT_EQ(dist_densor_type.dims_mapping(), dims_mapping); + EXPECT_EQ(dist_densor_type.partial_status(), partial_status); + EXPECT_EQ(dist_densor_type.dtype().isa(), true); + EXPECT_EQ(dist_densor_type.global_ddim(), dims); + EXPECT_EQ(dist_densor_type.data_layout(), data_layout); + EXPECT_EQ(dist_densor_type.local_ddim(), dims); +} From 12d1ecbe8ba378fb4d5120fa0e7938e1e5c70edf Mon Sep 17 00:00:00 2001 From: gouzil <66515297+gouzil@users.noreply.github.com> Date: Fri, 1 Mar 2024 10:24:19 +0800 Subject: [PATCH 208/282] [SOT][3.12] add `LOAD_FAST_CHECK` OpCode (#62218) --- .../jit/sot/opcode_translator/executor/opcode_executor.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py index 7d58a78a9322d..3dfa9fb1b733b 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py +++ b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py @@ -809,6 +809,9 @@ def LOAD_FAST(self, instr: Instruction): var = self._locals[instr.argval] self.stack.push(var) + def LOAD_FAST_CHECK(self, instr: Instruction): + self.LOAD_FAST(instr) + def DELETE_FAST(self, instr: Instruction): varname = self._code.co_varnames[instr.arg] del self._locals[varname] From 7a0807f231b4e33bad8cab6af8cda85e5763f88e Mon Sep 17 00:00:00 2001 From: Bo Zhang <105368690+zhangbopd@users.noreply.github.com> Date: Fri, 1 Mar 2024 10:53:17 +0800 Subject: [PATCH 209/282] [PIR][DynamicShape] Fix Gather Op and Shape Op && Add BC_binary Ops' inferSymbolic shape (#62248) * add gather * add binary * fix pd.shape && cinn.concat --- .../infer_symbolic_shape/cinn_op_infer_sym.cc | 19 ++ .../infer_sym_element_wise_binary.cc | 97 ++++++-- .../infer_sym_element_wise_binary.h | 55 +++-- .../paddle_op_infer_sym.cc | 214 +++++++----------- .../paddle_op_infer_sym.h | 36 --- .../same_operands_and_result.cc | 4 + .../same_operands_and_result.h | 2 + paddle/fluid/pir/dialect/operator/ir/ops.yaml | 1 + 8 files changed, 218 insertions(+), 210 deletions(-) diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.cc index 0e8240434e070..f81624427207e 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.cc @@ -41,6 +41,25 @@ bool ConcatOpInferSymbolicShape( const auto input_values = op->operands_source(); const auto input_size = input_values.size(); + if (shape_analysis->GetShapeOrDataForValue(input_values[0]) + .data() + .has_value()) { + std::vector out_data; + for (const auto &value : input_values) { + const auto &shape_or_data = shape_analysis->GetShapeOrDataForValue(value); + for (size_t i = 0; i < shape_or_data.data().value().size(); ++i) { + out_data.emplace_back(shape_or_data.data().value()[i]); + } + } + const std::vector shape{std::int64_t(out_data.size())}; + symbol::ShapeOrDataDimExprs shape_data{ + symbol::TensorShapeOrDataDimExprs(shape, out_data)}; + + pir::Value res = op->result(0); + shape_analysis->SetShapeOrDataForValue(res, shape_data); + return true; + } + int axis = op->attributes().at("axis").dyn_cast().data(); const auto &GetOutDimExprs = [&]() -> std::vector { diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_element_wise_binary.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_element_wise_binary.cc index 21da5351c617d..da8b68aefe206 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_element_wise_binary.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_element_wise_binary.cc @@ -79,27 +79,34 @@ bool InferSymbolicShapeElementWiseBinary( } namespace paddle::dialect { - bool AddOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } - bool Add_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } - bool BitwiseAndOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } - bool BitwiseAnd_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - return BitwiseAndOpInferSymbolicShape(op, shape_analysis); + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); +} +bool BitwiseXorOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); +} +bool BitwiseXor_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); +} +bool ComplexOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } - bool DivideOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return InferSymbolicShapeElementWiseBinary(op, shape_analysis); @@ -108,42 +115,82 @@ bool Divide_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } - bool ElementwisePowOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } - +bool FmaxOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); +} +bool FminOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); +} +bool GreaterEqualOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); +} +bool GreaterEqual_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); +} bool GreaterThanOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } - bool GreaterThan_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - return GreaterThanOpInferSymbolicShape(op, shape_analysis); + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); +} +bool LessEqualOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); +} +bool LessEqual_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } - bool LessThanOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } - bool LessThan_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - return LessThanOpInferSymbolicShape(op, shape_analysis); + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } - bool LogicalAndOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } - bool LogicalAnd_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - return LogicalAndOpInferSymbolicShape(op, shape_analysis); + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); +} +bool LogicalOrOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); +} +bool LogicalOr_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); +} +bool LogicalXorOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); +} +bool LogicalXor_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); +} +bool MaximumOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); +} +bool MinimumOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } - bool MultiplyOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return InferSymbolicShapeElementWiseBinary(op, shape_analysis); @@ -152,23 +199,29 @@ bool MultiplySrOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } -bool Multiply_OpInferSymbolicShape( +bool MultiplySr_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } -bool MultiplySr_OpInferSymbolicShape( +bool Multiply_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } - bool NotEqualOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } - bool NotEqual_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - return NotEqualOpInferSymbolicShape(op, shape_analysis); + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); +} +bool RemainderOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); +} +bool Remainder_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeElementWiseBinary(op, shape_analysis); } } // namespace paddle::dialect diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_element_wise_binary.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_element_wise_binary.h index e15d769fc8b02..be23d3cb20d9f 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_element_wise_binary.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_element_wise_binary.h @@ -19,58 +19,75 @@ namespace paddle::dialect { bool AddOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - bool Add_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - bool BitwiseAndOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - bool BitwiseAnd_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - +bool BitwiseXorOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool BitwiseXor_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool ComplexOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool DivideOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool Divide_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - bool ElementwisePowOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - +bool FmaxOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool FminOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool GreaterEqualOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool GreaterEqual_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool GreaterThanOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - bool GreaterThan_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - +bool LessEqualOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool LessEqual_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool LessThanOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - bool LessThan_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - bool LogicalAndOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - bool LogicalAnd_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - +bool LogicalOrOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool LogicalOr_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool LogicalXorOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool LogicalXor_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool MaximumOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool MinimumOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool MultiplyOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - bool MultiplySrOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - -bool Multiply_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - bool MultiplySr_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - +bool Multiply_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool NotEqualOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - bool NotEqual_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool RemainderOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Remainder_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); } // namespace paddle::dialect diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc index 6f4a4dacd7ba2..d95f109563518 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc @@ -59,20 +59,12 @@ bool ShapeOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { const symbol::ShapeOrDataDimExprs &operand_shape_or_data = shape_analysis->GetShapeOrDataForValue(op->operand_source(0)); - - const std::vector sym_shape = [&] { - std::vector sym_shape; - symbol::DimExpr dim_expr( - op->result(0).type().dyn_cast().dims()[0]); - sym_shape.emplace_back(dim_expr); - return sym_shape; - }(); - - symbol::ShapeOrDataDimExprs shape_or_data{symbol::TensorShapeOrDataDimExprs( - sym_shape, operand_shape_or_data.shape())}; + const auto &out_data = operand_shape_or_data.shape(); + const std::vector shape{std::int64_t(out_data.size())}; + symbol::ShapeOrDataDimExprs shape_or_data{ + symbol::TensorShapeOrDataDimExprs(shape, out_data)}; shape_analysis->SetShapeOrDataForValue(op->result(0), shape_or_data); - return true; } @@ -511,25 +503,21 @@ bool ConcatOpInferSymbolicShape( bool GatherNdOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - auto x_shape_or_data = + const auto &x_shape_or_data = shape_analysis->GetShapeOrDataForValue(op->operand_source(0)); - auto index_shape_or_data = + const auto &index_shape_or_data = shape_analysis->GetShapeOrDataForValue(op->operand_source(1)); - std::vector x_sym_shape; - if (x_shape_or_data.data().has_value()) { - x_sym_shape = x_shape_or_data.data().value(); - } else { - x_sym_shape = x_shape_or_data.shape(); - } - int x_dims_size = x_sym_shape.size(); + const std::vector &x_sym_shape = + x_shape_or_data.data().has_value() ? x_shape_or_data.data().value() + : x_shape_or_data.shape(); - std::vector index_sym_shape; - if (index_shape_or_data.data().has_value()) { - index_sym_shape = index_shape_or_data.data().value(); - } else { - index_sym_shape = index_shape_or_data.shape(); - } + const std::vector &index_sym_shape = + index_shape_or_data.data().has_value() + ? index_shape_or_data.data().value() + : index_shape_or_data.shape(); + + int x_dims_size = x_sym_shape.size(); int index_dims_size = index_sym_shape.size(); std::vector result_sym_dims; @@ -1159,26 +1147,6 @@ bool AsStridedOpInferSymbolicShape( return true; } -bool BitwiseXorOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool BitwiseXor_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} - -bool ComplexOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} - bool CummaxOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { PADDLE_THROW(phi::errors::Unimplemented( @@ -1234,22 +1202,70 @@ bool DirichletOpInferSymbolicShape( return true; } -bool FmaxOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool FminOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} bool GatherOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); + const auto &input_shape_or_data = + shape_analysis->GetShapeOrDataForValue(op->operand_source(0)); + const auto &index_shape_or_data = + shape_analysis->GetShapeOrDataForValue(op->operand_source(1)); + + const auto &numel = [&] { + symbol::DimExpr numel{1}; + for (const auto &dim_expr : index_shape_or_data.shape()) { + numel = numel * dim_expr; + } + return numel; + }(); + + const auto &axis_shape_or_data = + shape_analysis->GetShapeOrDataForValue(op->operand_source(2)); + + const std::vector &input_sym_shape = + input_shape_or_data.data().has_value() + ? input_shape_or_data.data().value() + : input_shape_or_data.shape(); + + const std::vector &index_sym_shape = + index_shape_or_data.data().has_value() + ? index_shape_or_data.data().value() + : index_shape_or_data.shape(); + + int axis = + static_cast(axis_shape_or_data.data().value()[0].Get()); + if (axis < 0) axis += input_sym_shape.size(); + + const auto &out_sym_shape = [&] { + std::vector out_sym_shape; + + if (index_sym_shape.size() == 0) { + if (input_sym_shape.size() == 1) { + out_sym_shape.push_back(symbol::DimExpr{0}); + } else { + for (int i = 0; i < axis; ++i) { + out_sym_shape.push_back(input_sym_shape[i]); + } + for (size_t i = axis + 1; i < input_sym_shape.size(); ++i) { + out_sym_shape.push_back(input_sym_shape[i]); + } + } + } else { + for (int i = 0; i < axis; ++i) { + out_sym_shape.push_back(input_sym_shape[i]); + } + out_sym_shape.push_back(numel); + for (size_t i = axis + 1; i < input_sym_shape.size(); ++i) { + out_sym_shape.push_back(input_sym_shape[i]); + } + } + return out_sym_shape; + }(); + + symbol::ShapeOrDataDimExprs shape_data{ + symbol::TensorShapeOrDataDimExprs(out_sym_shape)}; + + pir::Value res = op->result(0); + shape_analysis->SetShapeOrDataForValue(res, shape_data); + return true; } @@ -1272,30 +1288,6 @@ bool LogcumsumexpOpInferSymbolicShape( op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); return true; } -bool LogicalOrOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool LogicalOr_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool LogicalXorOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool LogicalXor_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} bool MaskedSelectOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { @@ -1379,30 +1371,7 @@ bool GaussianOpInferSymbolicShape( op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); return true; } -bool GreaterEqualOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool GreaterEqual_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool LessEqualOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool LessEqual_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} + bool LinspaceOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { PADDLE_THROW(phi::errors::Unimplemented( @@ -1421,24 +1390,14 @@ bool LogsumexpOpInferSymbolicShape( op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); return true; } -bool MaximumOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} + bool MinOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { PADDLE_THROW(phi::errors::Unimplemented( op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); return true; } -bool MinimumOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} + bool PadOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { PADDLE_THROW(phi::errors::Unimplemented( @@ -1451,18 +1410,7 @@ bool RandintOpInferSymbolicShape( op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); return true; } -bool RemainderOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Remainder_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} + bool RepeatInterleaveOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { PADDLE_THROW(phi::errors::Unimplemented( diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h index a13d93486b140..cf5e650023fa9 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h @@ -126,13 +126,6 @@ bool AsRealOpInferSymbolicShape(pir::Operation *op, bool AsStridedOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool BitwiseXorOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool BitwiseXor_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool ComplexOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - bool CummaxOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool CumminOpInferSymbolicShape(pir::Operation *op, @@ -153,10 +146,6 @@ bool DiagonalOpInferSymbolicShape( bool DirichletOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool FmaxOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool FminOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); bool GatherOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); @@ -167,15 +156,6 @@ bool KthvalueOpInferSymbolicShape( bool LogcumsumexpOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool LogicalOrOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool LogicalOr_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool LogicalXorOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool LogicalXor_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - bool MaskedSelectOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool PoissonOpInferSymbolicShape( @@ -206,34 +186,18 @@ bool Exponential_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool GaussianOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool GreaterEqualOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool GreaterEqual_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool LessEqualOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool LessEqual_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool LinspaceOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool LogspaceOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool LogsumexpOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool MaximumOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool MinOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool MinimumOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool PadOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool RandintOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool RemainderOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Remainder_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool RepeatInterleaveOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool SplitWithNumOpInferSymbolicShape( diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.cc index 31fe14209cc61..68ca785e0fbb0 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.cc @@ -290,6 +290,10 @@ bool Pow_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); } +bool PrintOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return SameOperandsAndResultShape(op, shape_analysis); +} bool RealOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h index 32941dd0c6f78..c671d9da22818 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h @@ -149,6 +149,8 @@ bool PowOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool Pow_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool PrintOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); bool RealOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool ReluOpInferSymbolicShape(pir::Operation *op, diff --git a/paddle/fluid/pir/dialect/operator/ir/ops.yaml b/paddle/fluid/pir/dialect/operator/ir/ops.yaml index 22bae4a65ab9a..7e05e5b79de8d 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops.yaml @@ -1070,6 +1070,7 @@ kernel : func : print_kernel param: [in, first_n, message, summarize, print_tensor_name, print_tensor_type, print_tensor_shape, print_tensor_layout, print_tensor_lod, print_phase, is_forward] + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : prod args : (Tensor x, IntArray dims, bool keep_dim, bool reduce_all) From 600c058f92bc80bb5d9eff1512734c3b43ee6a93 Mon Sep 17 00:00:00 2001 From: cyberslack_lee Date: Fri, 1 Mar 2024 10:54:45 +0800 Subject: [PATCH 210/282] [clang-tidy] NO.17 enable cppcoreguidelines-explicit-virtual-functions,modernize-use-override (#61714) * clangtidy 17 * fix --- paddle/fluid/framework/details/graph_test_base.h | 6 +++--- paddle/fluid/framework/ir/graph_test.cc | 4 ++-- paddle/fluid/framework/ir/pass_test.cc | 4 ++-- .../fluid/ir_adaptor/translator/op_translator.cc | 2 +- test/cpp/fluid/framework/op_proto_maker_test.cc | 6 +++--- test/cpp/fluid/framework/operator_test.cc | 16 ++++++++-------- .../fluid/framework/var_type_inference_test.cc | 2 +- test/cpp/pir/core/add_dialect_parser_test.cc | 2 +- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/paddle/fluid/framework/details/graph_test_base.h b/paddle/fluid/framework/details/graph_test_base.h index 2f50556e771ee..09d7dcc863aed 100644 --- a/paddle/fluid/framework/details/graph_test_base.h +++ b/paddle/fluid/framework/details/graph_test_base.h @@ -44,7 +44,7 @@ class DummyOp : public OperatorBase { class SumOpMaker : public OpProtoAndCheckerMaker { public: - void Make() { + void Make() override { AddInput("X", "").AsDuplicable(); AddOutput("Out", ""); AddComment(""); @@ -53,7 +53,7 @@ class SumOpMaker : public OpProtoAndCheckerMaker { class AssignOpMaker : public OpProtoAndCheckerMaker { public: - void Make() { + void Make() override { AddInput("X", "").AsDuplicable(); AddOutput("Out", ""); AddComment(""); @@ -62,7 +62,7 @@ class AssignOpMaker : public OpProtoAndCheckerMaker { class SplitOpMaker : public OpProtoAndCheckerMaker { public: - void Make() { + void Make() override { AddInput("X", ""); AddOutput("Out", "").AsDuplicable(); AddComment(""); diff --git a/paddle/fluid/framework/ir/graph_test.cc b/paddle/fluid/framework/ir/graph_test.cc index b8ad98113a3a4..4654abe6eb48d 100644 --- a/paddle/fluid/framework/ir/graph_test.cc +++ b/paddle/fluid/framework/ir/graph_test.cc @@ -38,7 +38,7 @@ class NOP : public OperatorBase { class SumOpMaker : public OpProtoAndCheckerMaker { public: - void Make() { + void Make() override { AddInput("X", "").AsDuplicable(); AddOutput("Out", "").AsDuplicable(); AddComment(""); @@ -60,7 +60,7 @@ class SumOpVarTypeInference : public VarTypeInference { class DummyOpMaker : public OpProtoAndCheckerMaker { public: - void Make() { + void Make() override { AddInput("X", "").AsDuplicable(); AddOutput("Out", "").AsDuplicable(); AddComment(""); diff --git a/paddle/fluid/framework/ir/pass_test.cc b/paddle/fluid/framework/ir/pass_test.cc index 2d13a912d6cca..4c3d19f51e73f 100644 --- a/paddle/fluid/framework/ir/pass_test.cc +++ b/paddle/fluid/framework/ir/pass_test.cc @@ -43,7 +43,7 @@ void BuildCircleGraph(Graph* g) { class TestPass : public Pass { protected: - void ApplyImpl(ir::Graph* graph) const { + void ApplyImpl(ir::Graph* graph) const override { graph->Set("copy_test_pass_attr", new int); graph->Set("copy_test_graph_attr", new int); @@ -226,7 +226,7 @@ TEST(PassTest, TestPassAttrCheckConvertAllBlocks) { class TestPassWithDefault : public Pass { protected: - void ApplyImpl(ir::Graph* graph) const { + void ApplyImpl(ir::Graph* graph) const override { graph->Set("copy_default_attr", new int); int test_pass_attr = this->Get("default_attr"); diff --git a/paddle/fluid/ir_adaptor/translator/op_translator.cc b/paddle/fluid/ir_adaptor/translator/op_translator.cc index c4ad629fc3d91..b7081609f2f90 100644 --- a/paddle/fluid/ir_adaptor/translator/op_translator.cc +++ b/paddle/fluid/ir_adaptor/translator/op_translator.cc @@ -2722,7 +2722,7 @@ struct RandIntOpTranscriber : public OpTranscriber { std::tuple GenerateOperationOutput( pir::IrContext* ctx, const OpDesc& op_desc, - const OpOutputInfoList& output_infos) { + const OpOutputInfoList& output_infos) override { OpOutputMapping arg_to_idx; OpOutputTypeList op_output_types = {}; diff --git a/test/cpp/fluid/framework/op_proto_maker_test.cc b/test/cpp/fluid/framework/op_proto_maker_test.cc index bc25e34d8139a..7c2301cded0ce 100644 --- a/test/cpp/fluid/framework/op_proto_maker_test.cc +++ b/test/cpp/fluid/framework/op_proto_maker_test.cc @@ -21,7 +21,7 @@ limitations under the License. */ class TestAttrProtoMaker : public paddle::framework::OpProtoAndCheckerMaker { public: - void Make() { + void Make() override { AddAttr("scale", "scale of test op"); AddAttr("scale", "scale of test op"); } @@ -37,7 +37,7 @@ TEST(ProtoMaker, DuplicatedAttr) { class TestInOutProtoMaker : public paddle::framework::OpProtoAndCheckerMaker { public: - void Make() { + void Make() override { AddInput("input", "input of test op"); AddInput("input", "input of test op"); } @@ -54,7 +54,7 @@ TEST(ProtoMaker, DuplicatedInOut) { class OpProtoMakerWithScalar : public paddle::framework::OpProtoAndCheckerMaker { public: - void Make() { + void Make() override { AddAttr("generic_scalar", "generic_scalar of test op"); AddAttr>( diff --git a/test/cpp/fluid/framework/operator_test.cc b/test/cpp/fluid/framework/operator_test.cc index d40a45ae5172a..b83127a239dbf 100644 --- a/test/cpp/fluid/framework/operator_test.cc +++ b/test/cpp/fluid/framework/operator_test.cc @@ -51,7 +51,7 @@ class OpWithoutKernelTest : public OperatorBase { class OpWithoutKernelCheckerMaker : public OpProtoAndCheckerMaker { public: - void Make() { + void Make() override { AddInput("input", "input of test op"); AddOutput("output", "output of test op"); AddAttr("scale", "scale of cosine op"); @@ -106,7 +106,7 @@ static int special_type_value = 1; class OpKernelTestProtoAndCheckerMaker : public OpProtoAndCheckerMaker { public: - void Make() { + void Make() override { AddInput("x", "input of test op"); AddOutput("y", "output of test op"); AddAttr("scale", "scale of cosine op") @@ -161,7 +161,7 @@ class CPUKernel2Test : public OpKernel { class OpKernelTestMultiInputsProtoAndCheckerMaker : public OpProtoAndCheckerMaker { public: - void Make() { + void Make() override { AddInput("xs", "inputs of test op").AsDuplicable(); AddInput("k", "input of test op"); AddOutput("ys", "outputs of test op").AsDuplicable(); @@ -335,7 +335,7 @@ class IndicateLoDTensorDataTypeTest : public OperatorWithKernel { class IndicateLoDTensorDataTypeTestProtoMaker : public OpProtoAndCheckerMaker { public: - void Make() { + void Make() override { AddInput("phi::DenseTensor", "Input of phi::DenseTensor type Variable."); AddComment("This Op is only for IndicateVarDataType interface test."); } @@ -357,7 +357,7 @@ class IndicateSelectedRowsDataTypeTest : public OperatorWithKernel { class IndicateSelectedRowsDataTypeTestProtoMaker : public OpProtoAndCheckerMaker { public: - void Make() { + void Make() override { AddInput("SelectedRows", "Input of SelectedRows type Variable."); AddComment("This Op is only for IndicateVarDataType interface test."); } @@ -377,7 +377,7 @@ class IndicateOtherDataTypeTest : public OperatorWithKernel { }; class IndicateOtherDataTypeTestProtoMaker : public OpProtoAndCheckerMaker { public: - void Make() { + void Make() override { AddInput("Other", "Input of Other type Variable"); AddComment("This Op is only for IndicateVarDataType interface test."); } @@ -512,7 +512,7 @@ class SetLoDLevelTest : public OperatorWithKernel { class GetSetLoDLevelTestMaker : public OpProtoAndCheckerMaker { public: - void Make() { + void Make() override { AddInput("X", "(phi::DenseTensor) Input Variable."); AddOutput("Out", "(phi::DenseTensor) Output Variable."); AddComment("This Op is only for Get/SetLoDLevel interface test."); @@ -592,7 +592,7 @@ class OpUnusedVarTest : public OperatorWithKernel { class OpUnusedVarTestProtoAndCheckerMaker : public OpProtoAndCheckerMaker { public: - void Make() { + void Make() override { AddInput("X", "input of test op"); AddOutput("Y", "output of test op"); AddComment("This is test op for unused var check."); diff --git a/test/cpp/fluid/framework/var_type_inference_test.cc b/test/cpp/fluid/framework/var_type_inference_test.cc index b7f7f32348ec6..6a310843e95e5 100644 --- a/test/cpp/fluid/framework/var_type_inference_test.cc +++ b/test/cpp/fluid/framework/var_type_inference_test.cc @@ -41,7 +41,7 @@ class NOP : public OperatorBase { class SumOpMaker : public OpProtoAndCheckerMaker { public: - void Make() { + void Make() override { AddInput("X", "").AsDuplicable(); AddOutput("Out", ""); AddComment(""); diff --git a/test/cpp/pir/core/add_dialect_parser_test.cc b/test/cpp/pir/core/add_dialect_parser_test.cc index 5a64b28a5cbd6..1b6ae533ffa16 100644 --- a/test/cpp/pir/core/add_dialect_parser_test.cc +++ b/test/cpp/pir/core/add_dialect_parser_test.cc @@ -37,7 +37,7 @@ class TestParserDialect : public pir::Dialect { static const char* name() { return "tp"; } - void PrintAttribute(pir::Attribute attr, std::ostream& os) const; + void PrintAttribute(pir::Attribute attr, std::ostream& os) const; // NOLINT pir::Attribute ParseAttribute(pir::IrParser& parser); // NOLINT From 1ea6a51857fc9b3d47ab17a6eb47827c056f072d Mon Sep 17 00:00:00 2001 From: cyberslack_lee Date: Fri, 1 Mar 2024 10:56:10 +0800 Subject: [PATCH 211/282] [clang-tidy] NO.3 bugprone-narrowing-conversions,cppcoreguidelines-narrowing-conversions PART 2 (#62109) --- .../collective/process_group_nccl.cc | 4 +++- .../distributed/test/ctr_accessor_test.cc | 8 +++---- .../fluid/framework/downpour_lite_worker.cc | 3 ++- paddle/fluid/framework/downpour_worker.cc | 5 ++-- paddle/fluid/framework/fleet/gloo_wrapper.cc | 4 ++-- paddle/fluid/framework/fleet/metrics.cc | 2 +- .../ir/mkldnn/cpu_bfloat16_pass_tester.cc | 4 ++-- .../framework/ir/mkldnn/cpu_quantize_pass.cc | 16 +++++++------ ...ant_transpose2_dequant_onednn_fuse_pass.cc | 2 +- .../ir/trt_skip_layernorm_fuse_pass.cc | 3 ++- .../analysis/ir_passes/lite_subgraph_pass.cc | 2 +- paddle/fluid/inference/api/analysis_config.cc | 8 ++++--- .../allocation/cuda_managed_allocator.cc | 2 +- .../memory/allocation/system_allocator.cc | 3 ++- .../fluid/operators/fused/resnet_unit_op.cc | 2 +- .../operators/mkldnn/reshape_mkldnn_op.cc | 6 ++--- .../operator/utils/op_yaml_info_parser.cc | 2 +- paddle/fluid/platform/gen_comm_id_helper.cc | 4 ++-- paddle/fluid/platform/profiler/utils.cc | 11 +++++---- paddle/fluid/pybind/eager_utils.cc | 6 ++--- paddle/fluid/pybind/imperative.cc | 5 ++-- paddle/phi/api/profiler/device_tracer.cc | 8 +++---- paddle/phi/api/profiler/profiler.cc | 2 +- paddle/phi/backends/device_base.cc | 6 ++--- paddle/phi/backends/device_code.cc | 3 ++- paddle/phi/backends/gpu/cuda/cuda_info.cc | 2 +- paddle/phi/backends/gpu/gpu_info.cc | 2 +- paddle/phi/infermeta/binary.cc | 8 +++---- paddle/phi/infermeta/multiary.cc | 4 ++-- .../phi/infermeta/spmd_rules/elementwise.cc | 24 +++++++++---------- paddle/phi/infermeta/spmd_rules/reduction.cc | 8 +++---- paddle/phi/infermeta/spmd_rules/replicated.cc | 10 ++++---- paddle/phi/infermeta/spmd_rules/softmax.cc | 6 ++--- paddle/phi/infermeta/spmd_rules/unsqueeze.cc | 8 +++---- paddle/phi/infermeta/spmd_rules/utils.cc | 7 +++--- paddle/phi/kernels/funcs/jit/gen/blas.cc | 2 +- paddle/phi/kernels/funcs/jit/gen/gru.cc | 2 +- paddle/phi/kernels/funcs/jit/gen/lstm.cc | 2 +- .../fusion/onednn/fused_transpose_kernel.cc | 6 ++--- .../phi/kernels/onednn/concat_grad_kernel.cc | 4 ++-- .../phi/kernels/onednn/expand_grad_kernel.cc | 2 +- .../phi/kernels/onednn/matmul_grad_kernel.cc | 6 +++-- paddle/phi/kernels/onednn/matmul_kernel.cc | 4 ++-- .../phi/kernels/onednn/slice_grad_kernel.cc | 2 +- paddle/phi/kernels/onednn/slice_kernel.cc | 2 +- .../phi/kernels/onednn/squeeze_grad_kernel.cc | 2 +- .../cpp/fluid/fused/cudnn_bn_add_relu_test.cc | 2 +- test/cpp/fluid/memory/buddy_allocator_test.cc | 8 +++---- test/cpp/imperative/test_group.cc | 4 ++-- test/cpp/inference/api/analyzer_dam_tester.cc | 2 +- .../analyzer_int8_object_detection_tester.cc | 2 +- .../analyzer_lexical_analysis_gru_tester.cc | 2 +- .../cpp/phi/kernels/test_fused_adam_kernel.cc | 2 +- test/cpp/phi/kernels/test_memcpy_dev_api.cc | 2 +- 54 files changed, 138 insertions(+), 120 deletions(-) diff --git a/paddle/fluid/distributed/collective/process_group_nccl.cc b/paddle/fluid/distributed/collective/process_group_nccl.cc index 82e95204590bd..f38fe1207c199 100644 --- a/paddle/fluid/distributed/collective/process_group_nccl.cc +++ b/paddle/fluid/distributed/collective/process_group_nccl.cc @@ -528,7 +528,9 @@ std::shared_ptr ProcessGroupNCCL::Gather( size_t offset = 0; size_t numel = out_tensor->numel() / size_; for (auto i = 0; i < size_; i++) { - partial_tensors.push_back(GetPartialTensor(*out_tensor, offset, numel)); + partial_tensors.push_back(GetPartialTensor(*out_tensor, + static_cast(offset), + static_cast(numel))); offset += numel; } } diff --git a/paddle/fluid/distributed/test/ctr_accessor_test.cc b/paddle/fluid/distributed/test/ctr_accessor_test.cc index 9b71e4524625c..0288a93d71a96 100644 --- a/paddle/fluid/distributed/test/ctr_accessor_test.cc +++ b/paddle/fluid/distributed/test/ctr_accessor_test.cc @@ -79,7 +79,7 @@ TEST(downpour_feature_value_accessor_test, test_shrink) { float* value = new float[acc->GetAccessorInfo().dim]; for (auto i = 0u; i < acc->GetAccessorInfo().dim; ++i) { - value[i] = i * 1.0; + value[i] = static_cast(i) * 1.0; } ASSERT_TRUE(!acc->Shrink(value)); @@ -98,7 +98,7 @@ TEST(downpour_feature_value_accessor_test, test_save) { float* value = new float[acc->GetAccessorInfo().dim]; for (auto i = 0u; i < acc->GetAccessorInfo().dim; ++i) { - value[i] = i * 1.0; + value[i] = static_cast(i) * 1.0; } // save all feature @@ -166,7 +166,7 @@ TEST(downpour_feature_value_accessor_test, test_update) { for (auto i = 0u; i < item_size; ++i) { float* p = new float[acc->GetAccessorInfo().update_dim]; for (auto j = 0u; j < acc->GetAccessorInfo().update_dim; ++j) { - p[j] = i + 1; + p[j] = static_cast(i) + 1.0; } grad[i] = p; } @@ -288,7 +288,7 @@ TEST(downpour_feature_value_accessor_test, test_string_related) { const int field_size = 15; float* value = new float[field_size]; for (auto i = 0u; i < field_size; ++i) { - value[i] = i; + value[i] = static_cast(i); } auto str = acc->ParseToString(value, 0); diff --git a/paddle/fluid/framework/downpour_lite_worker.cc b/paddle/fluid/framework/downpour_lite_worker.cc index 3d453c018c1d5..e86856bf1b2ff 100644 --- a/paddle/fluid/framework/downpour_lite_worker.cc +++ b/paddle/fluid/framework/downpour_lite_worker.cc @@ -410,7 +410,8 @@ void DownpourLiteWorker::TrainFilesWithProfiler() { fprintf(stderr, "push dense time percent: %f\n", push_dense_time / total_time * 100); - fprintf(stderr, "%6.2f instances/s\n", total_inst / total_time); + fprintf( + stderr, "%6.2f instances/s\n", total_inst / total_time); // NOLINT } } timeline.Start(); diff --git a/paddle/fluid/framework/downpour_worker.cc b/paddle/fluid/framework/downpour_worker.cc index 6ce2967a08f1f..0d5bd66297c53 100644 --- a/paddle/fluid/framework/downpour_worker.cc +++ b/paddle/fluid/framework/downpour_worker.cc @@ -334,8 +334,9 @@ void DownpourWorker::AdjustInsWeight() { } float ins_weight = 1.0; if (nid_show >= 0 && nid_show < nid_adjw_threshold) { - ins_weight = log(M_E + (nid_adjw_threshold - nid_show) / - nid_adjw_threshold * nid_adjw_ratio); + ins_weight = static_cast( + log(M_E + (nid_adjw_threshold - nid_show) / nid_adjw_threshold * + nid_adjw_ratio)); // count nid adjw insnum and weight ++nid_adjw_num; nid_adjw_weight += ins_weight; diff --git a/paddle/fluid/framework/fleet/gloo_wrapper.cc b/paddle/fluid/framework/fleet/gloo_wrapper.cc index 277004b6dc164..421953ff8c02a 100644 --- a/paddle/fluid/framework/fleet/gloo_wrapper.cc +++ b/paddle/fluid/framework/fleet/gloo_wrapper.cc @@ -165,7 +165,7 @@ void HdfsStore::wait(const std::vector& keys, int32_t last_check_rank = -1; for (size_t i = 0; i < check_key_status.size(); ++i) { if (!check_key_status[i]) { - last_check_rank = i; + last_check_rank = static_cast(i); break; } } @@ -252,7 +252,7 @@ void ParallelConnectContext::connectFullMesh( connect_threads[i].reset(new std::thread( [&store, &transportContext, total_add_size, this]( size_t thread_idx, size_t thread_num) -> void { - for (int i = thread_idx; i < size; i += thread_num) { + for (int i = thread_idx; i < size; i += thread_num) { // NOLINT if (i == rank) { continue; } diff --git a/paddle/fluid/framework/fleet/metrics.cc b/paddle/fluid/framework/fleet/metrics.cc index 58e1e195fbab7..5801860f66566 100644 --- a/paddle/fluid/framework/fleet/metrics.cc +++ b/paddle/fluid/framework/fleet/metrics.cc @@ -301,7 +301,7 @@ void BasicAucCalculator::add_uid_unlock_data(double pred, WuaucRecord record; record.uid_ = uid; record.label_ = label; - record.pred_ = pred; + record.pred_ = static_cast(pred); wuauc_records_.emplace_back(std::move(record)); } diff --git a/paddle/fluid/framework/ir/mkldnn/cpu_bfloat16_pass_tester.cc b/paddle/fluid/framework/ir/mkldnn/cpu_bfloat16_pass_tester.cc index dfd838895aeb4..951d064364ce3 100644 --- a/paddle/fluid/framework/ir/mkldnn/cpu_bfloat16_pass_tester.cc +++ b/paddle/fluid/framework/ir/mkldnn/cpu_bfloat16_pass_tester.cc @@ -73,9 +73,9 @@ void MainTest(const ProgramDesc& prog, auto graph = std::make_unique(prog); auto pass = PassRegistry::Instance().Get("cpu_bfloat16_pass"); - int original_nodes_num = graph->Nodes().size(); + int original_nodes_num = static_cast(graph->Nodes().size()); graph.reset(pass->Apply(graph.release())); - int current_nodes_num = graph->Nodes().size(); + int current_nodes_num = static_cast(graph->Nodes().size()); int quantize_nodes_count = 0; int dequantize_nodes_count = 0; diff --git a/paddle/fluid/framework/ir/mkldnn/cpu_quantize_pass.cc b/paddle/fluid/framework/ir/mkldnn/cpu_quantize_pass.cc index 2f1e7e8a53865..0e9c452455de3 100644 --- a/paddle/fluid/framework/ir/mkldnn/cpu_quantize_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/cpu_quantize_pass.cc @@ -94,8 +94,8 @@ void CPUQuantizePass::QuantizeInput(Graph* g, "Var(%s) isn't the input of the %s operator.", input_name, op->Op()->Type())); - unsigned max = is_input_unsigned ? U8_MAX : S8_MAX; - float scale = scale_to_one * max; + unsigned max = is_input_unsigned ? U8_MAX : S8_MAX; // NOLINT + float scale = static_cast(scale_to_one) * max; // Create quantize output variable VarDesc quantize_out_desc(patterns::PDNodeName("quantize", "out")); @@ -175,12 +175,13 @@ void CPUQuantizePass::QuantizeInputs(Graph* g, double scale_out = GetScaleValueForNode(output); unsigned max = are_inputs_unsigned ? U8_MAX : S8_MAX; - float scale = scale_out * max; + float scale = static_cast(scale_out) * max; for (size_t var_id = 0; var_id < unique_var_names.size(); var_id++) { auto index = -1; for (size_t it = 0; it < inputs.size(); it++) { - if (inputs[it]->Name() == unique_var_names[var_id]) index = it; + if (inputs[it]->Name() == unique_var_names[var_id]) + index = static_cast(it); } if (index == -1) { @@ -249,7 +250,7 @@ void CPUQuantizePass::DequantizeOutput(Graph* g, output_name, op->Op()->Type())); unsigned max = is_unsigned ? U8_MAX : S8_MAX; - float scale = scale_to_one * max; + float scale = static_cast(scale_to_one) * max; // Create dequantize input variable VarDesc dequantize_in_desc(patterns::PDNodeName("dequantize", "in")); @@ -298,12 +299,13 @@ void CPUQuantizePass::DequantizeOutputs(Graph* g, std::vector dequantize_in_nodes(outputs.size()); unsigned max = is_unsigned ? U8_MAX : S8_MAX; - float scale = scale_to_one * max; + float scale = static_cast(scale_to_one) * max; for (size_t var_id = 0; var_id < var_names.size(); var_id++) { auto index = -1; for (size_t it = 0; it < outputs.size(); it++) { - if (outputs[it]->Name() == var_names[var_id]) index = it; + if (outputs[it]->Name() == var_names[var_id]) + index = static_cast(it); } if (index == -1) { diff --git a/paddle/fluid/framework/ir/mkldnn/quant_transpose2_dequant_onednn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/quant_transpose2_dequant_onednn_fuse_pass.cc index 09bebfaec99c3..b331cc996fffc 100644 --- a/paddle/fluid/framework/ir/mkldnn/quant_transpose2_dequant_onednn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/quant_transpose2_dequant_onednn_fuse_pass.cc @@ -137,7 +137,7 @@ void FuseQuantTranspose2DequantOneDNNPass::FuseTranspose2Dequantize( dequant_op->Op()->HasAttr("Scale") ? PADDLE_GET_CONST(float, dequant_op->Op()->GetAttr("Scale")) : 1; - float reorder_scale = 1.0 / scale; + float reorder_scale = static_cast(1.0) / scale; float shift = dequant_op->Op()->HasAttr("Shift") ? PADDLE_GET_CONST(float, dequant_op->Op()->GetAttr("Shift")) diff --git a/paddle/fluid/framework/ir/trt_skip_layernorm_fuse_pass.cc b/paddle/fluid/framework/ir/trt_skip_layernorm_fuse_pass.cc index 81f96f2fc33f4..0708218dbd07c 100644 --- a/paddle/fluid/framework/ir/trt_skip_layernorm_fuse_pass.cc +++ b/paddle/fluid/framework/ir/trt_skip_layernorm_fuse_pass.cc @@ -218,7 +218,8 @@ void TrtSkipLayerNormFusePass::ApplyImpl(ir::Graph *graph) const { } new_desc.SetAttr("begin_norm_axis", begin_norm_axis); } - int32_t hidden_size = layer_norm_scale->Var()->GetShape()[0]; + int32_t hidden_size = + static_cast(layer_norm_scale->Var()->GetShape()[0]); new_desc.SetAttr("hidden_size", hidden_size); auto fused_node = graph->CreateOpNode(&new_desc); // OpDesc will be copied. diff --git a/paddle/fluid/inference/analysis/ir_passes/lite_subgraph_pass.cc b/paddle/fluid/inference/analysis/ir_passes/lite_subgraph_pass.cc index 2d484a943cf20..f8a4d4d15af72 100644 --- a/paddle/fluid/inference/analysis/ir_passes/lite_subgraph_pass.cc +++ b/paddle/fluid/inference/analysis/ir_passes/lite_subgraph_pass.cc @@ -71,7 +71,7 @@ std::vector IOVarsFilter(const std::vector& nodes) { void StrToBinaryFile(const std::string& path, const std::string& str) { std::ofstream file(path.c_str(), std::ios::binary); - file.write(str.c_str(), str.size()); + file.write(str.c_str(), str.size()); // NOLINT file.close(); } diff --git a/paddle/fluid/inference/api/analysis_config.cc b/paddle/fluid/inference/api/analysis_config.cc index 0ec5151a92bc5..5987483220b8a 100644 --- a/paddle/fluid/inference/api/analysis_config.cc +++ b/paddle/fluid/inference/api/analysis_config.cc @@ -1232,11 +1232,13 @@ float AnalysisConfig::fraction_of_gpu_memory_for_pool() const { size_t gpu_total, gpu_available; platform::SetDeviceId(gpu_device_id_); platform::GpuMemoryUsage(&gpu_available, &gpu_total); - double total_gpu_memory = gpu_total / 1024. / 1024.; + double total_gpu_memory = static_cast(gpu_total) / 1024. / 1024.; float fraction_of_gpu_memory = - static_cast(memory_pool_init_size_mb()) / total_gpu_memory; + static_cast(memory_pool_init_size_mb()) / + static_cast(total_gpu_memory); VLOG(3) << "total_gpu_memory is " << total_gpu_memory - << "M, gpu_available is " << gpu_available / 1024. / 1024. + << "M, gpu_available is " + << static_cast(gpu_available) / 1024. / 1024. << "M, memory_pool_init_size is " << memory_pool_init_size_mb() << "M."; return fraction_of_gpu_memory; diff --git a/paddle/fluid/memory/allocation/cuda_managed_allocator.cc b/paddle/fluid/memory/allocation/cuda_managed_allocator.cc index 77ca495cacbc7..36659fdbadce2 100644 --- a/paddle/fluid/memory/allocation/cuda_managed_allocator.cc +++ b/paddle/fluid/memory/allocation/cuda_managed_allocator.cc @@ -65,7 +65,7 @@ phi::Allocation* CUDAManagedAllocator::AllocateImpl(size_t size) { std::string err_msg; if (UNLIKELY(is_limited)) { - int64_t limit_size_mb = limit_size >> 20; + int64_t limit_size_mb = limit_size >> 20; // NOLINT err_msg = string::Sprintf( "Or set environment variable `FLAGS_gpu_memory_limit_mb` to a larger " "value. Currently `FLAGS_gpu_memory_limit_mb` is %d, so the maximum " diff --git a/paddle/fluid/memory/allocation/system_allocator.cc b/paddle/fluid/memory/allocation/system_allocator.cc index 4ca1f21c563fc..8fd7967e9752d 100644 --- a/paddle/fluid/memory/allocation/system_allocator.cc +++ b/paddle/fluid/memory/allocation/system_allocator.cc @@ -208,7 +208,8 @@ void* CUDAPinnedAllocator::Alloc(size_t* index, size_t size) { if (size > usable) { LOG(WARNING) << "Cannot malloc " << size / 1024.0 / 1024.0 << " MB pinned memory." - << ", available " << usable / 1024.0 / 1024.0 << " MB"; + << ", available " << usable / 1024.0 / 1024.0 + << " MB"; // NOLINT return nullptr; } diff --git a/paddle/fluid/operators/fused/resnet_unit_op.cc b/paddle/fluid/operators/fused/resnet_unit_op.cc index f1f2628119c15..5827cd3427dee 100644 --- a/paddle/fluid/operators/fused/resnet_unit_op.cc +++ b/paddle/fluid/operators/fused/resnet_unit_op.cc @@ -27,7 +27,7 @@ static framework::DDim GetBitmaskDims(std::vector out_shape) { std::multiplies()) / // NOLINT c; int32_t c_int32_elems = ((c + 63) & ~63) / 32; - int32_t nhw_int32_elems = ((nhw + 31) & ~31); + int32_t nhw_int32_elems = static_cast(((nhw + 31) & ~31)); std::vector bitmask_shape = {nhw_int32_elems, c_int32_elems, 1}; return common::make_ddim(bitmask_shape); } diff --git a/paddle/fluid/operators/mkldnn/reshape_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/reshape_mkldnn_op.cc index 1e3b29da11e5b..8632160b04ae0 100644 --- a/paddle/fluid/operators/mkldnn/reshape_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/reshape_mkldnn_op.cc @@ -185,7 +185,7 @@ class ReshapeMKLDNNKernel : public framework::OpKernel { "be -1. But received shape = [%s], shape[%d] is also -1.", common::make_ddim(shape), i)); - unk_dim_idx = i; + unk_dim_idx = static_cast(i); } else if (shape[i] == copy_dim_val) { PADDLE_ENFORCE_LT( static_cast(i), @@ -212,9 +212,9 @@ class ReshapeMKLDNNKernel : public framework::OpKernel { shape[i])); } - capacity *= (shape[i] ? shape[i] : in_dims[i]); + capacity *= (shape[i] ? shape[i] : in_dims[i]); // NOLINT output_shape[i] = - (shape[i] ? static_cast(shape[i]) : in_dims[i]); + (shape[i] ? static_cast(shape[i]) : in_dims[i]); // NOLINT } if (unk_dim_idx != -1) { diff --git a/paddle/fluid/pir/dialect/operator/utils/op_yaml_info_parser.cc b/paddle/fluid/pir/dialect/operator/utils/op_yaml_info_parser.cc index 7f84eac85bdb8..41140053a22f0 100644 --- a/paddle/fluid/pir/dialect/operator/utils/op_yaml_info_parser.cc +++ b/paddle/fluid/pir/dialect/operator/utils/op_yaml_info_parser.cc @@ -232,7 +232,7 @@ int OpYamlInfoParser::GetTensorParamIndexByArgsName( kernel_fn_tensor_params_.end(), args_name); if (iter != kernel_fn_tensor_params_.end()) { - return std::distance(kernel_fn_tensor_params_.begin(), iter); + return std::distance(kernel_fn_tensor_params_.begin(), iter); // NOLINT } else { return -1; } diff --git a/paddle/fluid/platform/gen_comm_id_helper.cc b/paddle/fluid/platform/gen_comm_id_helper.cc index 40d80f8ef2cbc..ab10f799f68d1 100644 --- a/paddle/fluid/platform/gen_comm_id_helper.cc +++ b/paddle/fluid/platform/gen_comm_id_helper.cc @@ -82,7 +82,7 @@ static int SocketSend(int fd, const char* buffer, int size) { int offset = 0; int bytes = 0; while (offset < size) { - bytes = send(fd, buffer + offset, size - offset, 0); + bytes = send(fd, buffer + offset, size - offset, 0); // NOLINT if (bytes == -1) { if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN) { // send failed @@ -100,7 +100,7 @@ static int SocketRecv(int fd, char* buffer, int size) { int offset = 0; int bytes = 0; while (offset < size) { - bytes = recv(fd, buffer + offset, size - offset, 0); + bytes = recv(fd, buffer + offset, size - offset, 0); // NOLINT if (bytes == 0) { // closed by client, maybe probing alive client return 0; diff --git a/paddle/fluid/platform/profiler/utils.cc b/paddle/fluid/platform/profiler/utils.cc index 8c12f84416579..236c77cec5b22 100644 --- a/paddle/fluid/platform/profiler/utils.cc +++ b/paddle/fluid/platform/profiler/utils.cc @@ -106,7 +106,8 @@ float CalculateEstOccupancy(uint32_t DeviceId, float occupancy = 0.0; std::vector device_ids = GetSelectedDevices(); if (DeviceId < device_ids.size()) { - const gpuDeviceProp& device_property = GetDeviceProperties(DeviceId); + const gpuDeviceProp& device_property = + GetDeviceProperties(static_cast(DeviceId)); cudaOccFuncAttributes occFuncAttr; occFuncAttr.maxThreadsPerBlock = INT_MAX; occFuncAttr.numRegs = RegistersPerThread; @@ -127,11 +128,13 @@ float CalculateEstOccupancy(uint32_t DeviceId, blockSize, dynamicSmemSize); if (status == CUDA_OCC_SUCCESS) { - if (occ_result.activeBlocksPerMultiprocessor < BlocksPerSm) { - BlocksPerSm = occ_result.activeBlocksPerMultiprocessor; + if (static_cast(occ_result.activeBlocksPerMultiprocessor) < + BlocksPerSm) { + BlocksPerSm = + static_cast(occ_result.activeBlocksPerMultiprocessor); } occupancy = - BlocksPerSm * blockSize / + BlocksPerSm * static_cast(blockSize) / static_cast(device_property.maxThreadsPerMultiProcessor); } else { LOG(WARNING) << "Failed to calculate estimated occupancy, status = " diff --git a/paddle/fluid/pybind/eager_utils.cc b/paddle/fluid/pybind/eager_utils.cc index d613c008b4958..c6a2db061594b 100644 --- a/paddle/fluid/pybind/eager_utils.cc +++ b/paddle/fluid/pybind/eager_utils.cc @@ -518,7 +518,7 @@ std::vector CastPyArg2VectorOfInt64(PyObject* obj, size_t arg_pos) { } else if (obj == Py_None) { return {}; } else if (PyObject_CheckLongOrConvertToLong(&obj)) { - return {static_cast(PyLong_AsLong(obj))}; + return {static_cast(PyLong_AsLong(obj))}; // NOLINT } else { PADDLE_THROW(platform::errors::InvalidType( "argument (position %d) must be " @@ -566,7 +566,7 @@ std::vector CastPyArg2VectorOfSize_t(PyObject* obj, size_t arg_pos) { } else if (obj == Py_None) { return {}; } else if (PyObject_CheckLongOrConvertToLong(&obj)) { - return {PyLong_AsSize_t(obj)}; + return {PyLong_AsSize_t(obj)}; // NOLINT } else { PADDLE_THROW(platform::errors::InvalidType( "argument (position %d) must be " @@ -614,7 +614,7 @@ std::vector CastPyArg2VectorOfFloat(PyObject* obj, size_t arg_pos) { } else if (obj == Py_None) { return {}; } else if (PyObject_CheckFloatOrConvertToFloat(&obj)) { - return {static_cast(PyFloat_AsDouble(obj))}; + return {static_cast(PyFloat_AsDouble(obj))}; // NOLINT } else { PADDLE_THROW(platform::errors::InvalidType( "argument (position %d) must be " diff --git a/paddle/fluid/pybind/imperative.cc b/paddle/fluid/pybind/imperative.cc index c540fe0687d88..288a05d638b73 100644 --- a/paddle/fluid/pybind/imperative.cc +++ b/paddle/fluid/pybind/imperative.cc @@ -1357,8 +1357,9 @@ void BindImperative(py::module *m_ptr) { auto *index_data = index_tensor.data(); auto *buffer_data = buffer_tensor->mutable_data(buffer_tensor->place()); - const int &slice_size = src_tensor.numel() / src_tensor.dims()[0]; - const int ©_bytes = slice_size * sizeof(float); + const int &slice_size = + static_cast(src_tensor.numel()) / src_tensor.dims()[0]; + const int ©_bytes = static_cast(slice_size) * sizeof(float); int64_t c = 0; for (int64_t i = 0; i < index_tensor.numel(); i++) { std::memcpy(buffer_data + c * slice_size, diff --git a/paddle/phi/api/profiler/device_tracer.cc b/paddle/phi/api/profiler/device_tracer.cc index f15d6bbb88457..748eedff4ee6d 100644 --- a/paddle/phi/api/profiler/device_tracer.cc +++ b/paddle/phi/api/profiler/device_tracer.cc @@ -571,10 +571,10 @@ class DeviceTracerImpl : public DeviceTracer { Event *e = c->second; Event *parent = e->parent(); while (parent) { - parent->AddCudaElapsedTime(r.start_ns, r.end_ns); + parent->AddCudaElapsedTime(r.start_ns, r.end_ns); // NOLINT parent = parent->parent(); } - e->AddCudaElapsedTime(r.start_ns, r.end_ns); + e->AddCudaElapsedTime(r.start_ns, r.end_ns); // NOLINT } } for (const auto &r : mem_records_) { @@ -583,10 +583,10 @@ class DeviceTracerImpl : public DeviceTracer { Event *e = c->second; Event *parent = e->parent(); while (parent) { - parent->AddCudaElapsedTime(r.start_ns, r.end_ns); + parent->AddCudaElapsedTime(r.start_ns, r.end_ns); // NOLINT parent = parent->parent(); } - e->AddCudaElapsedTime(r.start_ns, r.end_ns); + e->AddCudaElapsedTime(r.start_ns, r.end_ns); // NOLINT } } #endif diff --git a/paddle/phi/api/profiler/profiler.cc b/paddle/phi/api/profiler/profiler.cc index 6dc419658d3c2..e9c49741a5e6b 100644 --- a/paddle/phi/api/profiler/profiler.cc +++ b/paddle/phi/api/profiler/profiler.cc @@ -77,7 +77,7 @@ double Event::CpuElapsedMs(const Event &e) const { double Event::CudaElapsedMs(const Event &e) const { #ifdef PADDLE_WITH_CUPTI - return gpu_ns_ / 1000000.0; + return static_cast(gpu_ns_) / 1000000.0; #else LOG_FIRST_N(WARNING, 1) << "CUDA CUPTI is not enabled"; return 0; diff --git a/paddle/phi/backends/device_base.cc b/paddle/phi/backends/device_base.cc index f27919bef05fe..7860d322f1faa 100644 --- a/paddle/phi/backends/device_base.cc +++ b/paddle/phi/backends/device_base.cc @@ -215,9 +215,9 @@ size_t DeviceInterface::AllocSize(size_t dev_id, bool realloc) { size_t flag_mb = realloc ? FLAGS_reallocate_gpu_memory_in_mb : FLAGS_initial_gpu_memory_in_mb; size_t alloc_bytes = - (flag_mb > 0ul - ? flag_mb << 20 - : available_to_alloc * FLAGS_fraction_of_gpu_memory_to_use); + (flag_mb > 0ul ? flag_mb << 20 + : available_to_alloc * + FLAGS_fraction_of_gpu_memory_to_use); // NOLINT PADDLE_ENFORCE_GE(available_to_alloc, alloc_bytes, phi::errors::ResourceExhausted( diff --git a/paddle/phi/backends/device_code.cc b/paddle/phi/backends/device_code.cc index 670e0e3781598..e2016ff78b7c3 100644 --- a/paddle/phi/backends/device_code.cc +++ b/paddle/phi/backends/device_code.cc @@ -186,7 +186,8 @@ static std::string FindCUDAIncludePath() { } for (std::string suffix : {"/lib", "/lib64"}) { if (EndWith(FLAGS_cuda_dir, suffix)) { - cuda_include_path.erase(cuda_include_path.end() - suffix.length()); + cuda_include_path.erase(cuda_include_path.end() - + suffix.length()); // NOLINT break; } } diff --git a/paddle/phi/backends/gpu/cuda/cuda_info.cc b/paddle/phi/backends/gpu/cuda/cuda_info.cc index 0af1beb782fcf..505fc7f3f6cd6 100644 --- a/paddle/phi/backends/gpu/cuda/cuda_info.cc +++ b/paddle/phi/backends/gpu/cuda/cuda_info.cc @@ -28,7 +28,7 @@ namespace gpu { int DnnVersion() { if (!dynload::HasCUDNN()) return -1; - return dynload::cudnnGetVersion(); + return dynload::cudnnGetVersion(); // NOLINT } static int GetGPUDeviceCountImpl() { diff --git a/paddle/phi/backends/gpu/gpu_info.cc b/paddle/phi/backends/gpu/gpu_info.cc index 96048de5c047c..32546f762c39e 100644 --- a/paddle/phi/backends/gpu/gpu_info.cc +++ b/paddle/phi/backends/gpu/gpu_info.cc @@ -66,7 +66,7 @@ size_t GpuAvailableMemToAlloc() { size_t available = 0; memory_utils::GpuMemoryUsage(&available, &total); size_t reserving = - static_cast(fraction_reserve_gpu_memory * available); + static_cast(fraction_reserve_gpu_memory * available); // NOLINT // If available size is less than minimum chunk size, no usable memory exists size_t available_to_alloc = available - reserving; size_t min_chunk_size = GpuMinChunkSize(); diff --git a/paddle/phi/infermeta/binary.cc b/paddle/phi/infermeta/binary.cc index fdef52a5fb6e1..ce47a88c420df 100644 --- a/paddle/phi/infermeta/binary.cc +++ b/paddle/phi/infermeta/binary.cc @@ -166,8 +166,8 @@ void ArrayReadInferMeta(const MetaTensor& array, out->set_dims({-1}); } else { double index = i.to(); - out->set_dims(array.dims(index)); - out->share_lod(array, index); + out->set_dims(array.dims(index)); // NOLINT + out->share_lod(array, index); // NOLINT } out->set_dtype(array.dtype()); out->set_layout(array.layout()); @@ -3557,8 +3557,8 @@ void WeightDequantizeInferMeta(const MetaTensor& x, dim_scale[0], (x.dims()[1] + (group_size - 1)) / group_size)); } - int n = x.dims()[1]; - int k = x.dims()[0]; + int n = static_cast(x.dims()[1]); + int k = static_cast(x.dims()[0]); out->set_dims(common::make_ddim({n, k})); out->set_dtype(out_dtype); } diff --git a/paddle/phi/infermeta/multiary.cc b/paddle/phi/infermeta/multiary.cc index bb57e5a813aa7..7575cc3cf1434 100644 --- a/paddle/phi/infermeta/multiary.cc +++ b/paddle/phi/infermeta/multiary.cc @@ -4706,8 +4706,8 @@ void MaskedMultiheadAttentionInferMeta(const MetaTensor& x, int v_num_head = k_num_head; int dim_head = static_cast(cache_kv.dims()[4]); // below's num_head is q's head actually. - int num_head = - x.dims()[x.dims().size() - 1] / dim_head - k_num_head - v_num_head; + int num_head = x.dims()[x.dims().size() - 1] / dim_head - k_num_head - + v_num_head; // NOLINT PADDLE_ENFORCE_EQ( num_head % k_num_head, diff --git a/paddle/phi/infermeta/spmd_rules/elementwise.cc b/paddle/phi/infermeta/spmd_rules/elementwise.cc index 3db396de8b613..d558dfa69b7b5 100644 --- a/paddle/phi/infermeta/spmd_rules/elementwise.cc +++ b/paddle/phi/infermeta/spmd_rules/elementwise.cc @@ -31,7 +31,7 @@ std::string GetInputBroadcastNotation(const std::vector& shape, const int max_ndim, const std::string& alphabet, std::vector* broadcast_axis_count) { - int ndim = shape.size(); + int ndim = static_cast(shape.size()); int start_dim = max_ndim - ndim; std::string axes_notation = GetBroadcastAxes(ndim, max_ndim, alphabet); @@ -54,8 +54,8 @@ void GetBinaryNotations(const std::vector& x_shape, std::string* x_axes, std::string* y_axes, std::string* out_axes) { - int x_ndim = x_shape.size(); - int y_ndim = y_shape.size(); + int x_ndim = static_cast(x_shape.size()); + int y_ndim = static_cast(y_shape.size()); int max_ndim = std::max(x_ndim, y_ndim); int ninputs = 2; std::string alphabet = "abcdefghijklmnopqrstuvwxyz"; @@ -82,7 +82,7 @@ void GetBinaryNotations(const std::vector& x_shape, SpmdInfo ElementwiseUnaryInferSpmd(const DistMetaTensor& x) { // Step0: Verify Input Args Based on Elementwise Logic auto x_shape = common::vectorize(x.dims()); - int x_ndim = x_shape.size(); + int x_ndim = static_cast(x_shape.size()); TensorDistAttr x_dist_attr_src = x.dist_attr(); std::vector x_dims_mapping = x_dist_attr_src.dims_mapping(); PADDLE_ENFORCE_EQ(x_ndim, @@ -129,7 +129,7 @@ SpmdInfo ElementwiseUnaryInferSpmd(const DistMetaTensor& x) { SpmdInfo ElementwiseUnaryWithPartialInferSpmd(const DistMetaTensor& x) { // Step0: Verify Input Args Based on Elementwise Logic auto x_shape = common::vectorize(x.dims()); - int x_ndim = x_shape.size(); + int x_ndim = static_cast(x_shape.size()); TensorDistAttr x_dist_attr_src = x.dist_attr(); std::vector x_dims_mapping = x_dist_attr_src.dims_mapping(); PADDLE_ENFORCE_EQ(x_ndim, @@ -177,9 +177,9 @@ SpmdInfo ElementwiseUnaryInferSpmdReverse(const DistMetaTensor& x, const DistMetaTensor& out) { // Step0: Verify Input Args Based on Elementwise Logic auto x_shape = common::vectorize(x.dims()); - int x_ndim = x_shape.size(); + int x_ndim = static_cast(x_shape.size()); auto out_shape = common::vectorize(out.dims()); - int out_ndim = out_shape.size(); + int out_ndim = static_cast(out_shape.size()); TensorDistAttr out_dist_attr_src = out.dist_attr(); std::vector out_dims_mapping = out_dist_attr_src.dims_mapping(); PADDLE_ENFORCE_EQ( @@ -233,9 +233,9 @@ SpmdInfo ElementwiseBinaryInferSpmd(const DistMetaTensor& x, const DistMetaTensor& y) { // Step0: Verify Input Args Based on Elementwise Logic auto x_shape = common::vectorize(x.dims()); - int x_ndim = x_shape.size(); + int x_ndim = static_cast(x_shape.size()); auto y_shape = common::vectorize(y.dims()); - int y_ndim = y_shape.size(); + int y_ndim = static_cast(y_shape.size()); TensorDistAttr x_dist_attr_src = x.dist_attr(); TensorDistAttr y_dist_attr_src = y.dist_attr(); std::vector x_dims_mapping = x_dist_attr_src.dims_mapping(); @@ -303,11 +303,11 @@ SpmdInfo ElementwiseBinaryInferSpmdReverse(const DistMetaTensor& x, const DistMetaTensor& out) { // Step0: Verify Input Args Based on Elementwise Logic auto x_shape = common::vectorize(x.dims()); - int x_ndim = x_shape.size(); + int x_ndim = static_cast(x_shape.size()); auto y_shape = common::vectorize(y.dims()); - int y_ndim = y_shape.size(); + int y_ndim = static_cast(y_shape.size()); auto out_shape = common::vectorize(out.dims()); - int out_ndim = out_shape.size(); + int out_ndim = static_cast(out_shape.size()); int max_ndim = std::max(x_ndim, y_ndim); TensorDistAttr out_dist_attr = out.dist_attr(); std::vector out_dims_mapping = out_dist_attr.dims_mapping(); diff --git a/paddle/phi/infermeta/spmd_rules/reduction.cc b/paddle/phi/infermeta/spmd_rules/reduction.cc index 608794d348541..ef5d93a04533e 100644 --- a/paddle/phi/infermeta/spmd_rules/reduction.cc +++ b/paddle/phi/infermeta/spmd_rules/reduction.cc @@ -71,7 +71,7 @@ SpmdInfo ReductionInferSpmdBase(const DistMetaTensor& x, int reduce_type) { // Step0: Verify input args based on reduction logic auto x_shape = common::vectorize(x.dims()); - int x_ndim = x_shape.size(); + int x_ndim = static_cast(x_shape.size()); auto x_dist_attr_src = x.dist_attr(); std::vector x_dims_mapping = x_dist_attr_src.dims_mapping(); PADDLE_ENFORCE_EQ( @@ -175,8 +175,8 @@ SpmdInfo ReductionInferSpmdReverse(const DistMetaTensor& x, // Step0: Verify input args based on reduction logic auto x_shape = common::vectorize(x.dims()); auto out_shape = common::vectorize(out.dims()); - int x_ndim = x_shape.size(); - int out_ndim = out_shape.size(); + int x_ndim = static_cast(x_shape.size()); + int out_ndim = static_cast(out_shape.size()); auto out_dist_attr_src = out.dist_attr(); std::vector out_dims_mapping = out_dist_attr_src.dims_mapping(); PADDLE_ENFORCE_EQ( @@ -240,7 +240,7 @@ SpmdInfo ReductionGradInferSpmd(const DistMetaTensor& x, for (size_t i = 0; i < axis_value.size(); ++i) { if (axis_value[i] < 0) { - axis_value[i] += x_dim.size(); + axis_value[i] += x_dim.size(); // NOLINT } } std::sort(axis_value.begin(), axis_value.end()); diff --git a/paddle/phi/infermeta/spmd_rules/replicated.cc b/paddle/phi/infermeta/spmd_rules/replicated.cc index 8d9c6d0d5be6c..390117862e04e 100644 --- a/paddle/phi/infermeta/spmd_rules/replicated.cc +++ b/paddle/phi/infermeta/spmd_rules/replicated.cc @@ -35,8 +35,8 @@ std::vector GetReplicatedDimsMapping(const int ndim) { SpmdInfo ReplicatedInferSpmd(const std::vector& ins, const std::vector& outs) { // step1: Build Einsum Notation for input tensor's batch axis - int64_t ninputs = ins.size(); - int64_t noutputs = outs.size(); + int64_t ninputs = static_cast(ins.size()); + int64_t noutputs = static_cast(outs.size()); // Step2: Unshard Output's Dims Mapping. std::vector output_dist_attrs; @@ -94,8 +94,8 @@ SpmdInfo ReplicatedInferSpmdReverse( const std::vector& ins, const std::vector& outs) { // step1: Build Einsum Notation for input tensor's batch axis - int64_t ninputs = ins.size(); - int64_t noutputs = outs.size(); + int64_t ninputs = static_cast(ins.size()); + int64_t noutputs = static_cast(outs.size()); // Step2: Unshard Output's Dims Mapping. std::vector output_dist_attrs; @@ -145,7 +145,7 @@ SpmdInfo ReplicatedInferDynamic( const std::vector*>>& inputs) { std::vector nonnull_inputs; - int64_t ninputs = inputs.size(); + int64_t ninputs = static_cast(inputs.size()); SpmdInfo spmd_info; auto build_tensor_dist_attr = diff --git a/paddle/phi/infermeta/spmd_rules/softmax.cc b/paddle/phi/infermeta/spmd_rules/softmax.cc index d86db4d41ae23..b6f886a49468a 100644 --- a/paddle/phi/infermeta/spmd_rules/softmax.cc +++ b/paddle/phi/infermeta/spmd_rules/softmax.cc @@ -31,7 +31,7 @@ using phi::distributed::auto_parallel::str_join; SpmdInfo SoftmaxInferSpmd(const DistMetaTensor& x, int axis) { // Step0: Verify input args based on softmax logic auto x_shape = common::vectorize(x.dims()); - int x_ndim = x_shape.size(); + int x_ndim = static_cast(x_shape.size()); auto x_dist_attr_src = x.dist_attr(); std::vector x_dims_mapping = x_dist_attr_src.dims_mapping(); PADDLE_ENFORCE_EQ( @@ -100,8 +100,8 @@ SpmdInfo SoftmaxInferSpmdReverse(const DistMetaTensor& x, // Step0: verify input args based on softmax logic auto x_shape = common::vectorize(x.dims()); auto out_shape = common::vectorize(out.dims()); - int x_ndim = x_shape.size(); - int out_ndim = out_shape.size(); + int x_ndim = static_cast(x_shape.size()); + int out_ndim = static_cast(out_shape.size()); auto out_dist_attr_src = out.dist_attr(); std::vector out_dims_mapping = out_dist_attr_src.dims_mapping(); PADDLE_ENFORCE_EQ( diff --git a/paddle/phi/infermeta/spmd_rules/unsqueeze.cc b/paddle/phi/infermeta/spmd_rules/unsqueeze.cc index ef47b31341a73..5521e1ba2a137 100644 --- a/paddle/phi/infermeta/spmd_rules/unsqueeze.cc +++ b/paddle/phi/infermeta/spmd_rules/unsqueeze.cc @@ -93,7 +93,7 @@ SpmdInfo UnsqueezeInferSpmd(const DistMetaTensor& x, const std::vector& axis) { // Step0: Verify input args based on unsqueeze logic auto x_shape = common::vectorize(x.dims()); - int x_ndim = x_shape.size(); + int x_ndim = static_cast(x_shape.size()); auto x_dist_attr_src = x.dist_attr(); std::vector x_dims_mapping = x_dist_attr_src.dims_mapping(); PADDLE_ENFORCE_EQ( @@ -162,9 +162,9 @@ SpmdInfo UnsqueezeInferSpmdReverse(const DistMetaTensor& x, const std::vector& axis) { // Step0: Verify input args based on unsqueeze logic auto x_shape = common::vectorize(x.dims()); - int x_ndim = x_shape.size(); + int x_ndim = static_cast(x_shape.size()); auto out_shape = common::vectorize(out.dims()); - int out_ndim = out_shape.size(); + int out_ndim = static_cast(out_shape.size()); auto out_dist_attr_src = out.dist_attr(); std::vector out_dims_mapping = out_dist_attr_src.dims_mapping(); PADDLE_ENFORCE_EQ( @@ -217,7 +217,7 @@ SpmdInfo UnsqueezeInferSpmdReverse(const DistMetaTensor& x, VLOG(4) << "UnsqueezeInferSpmdReverse: Out shape: [" << str_join(out_shape) << "] X shape: [" << str_join(x_shape) << "]"; VLOG(4) << "Transformation from output to input:"; - for (int64_t i = 0, n = trans.size(); i < n; i++) { + for (int64_t i = 0, n = static_cast(trans.size()); i < n; i++) { std::shared_ptr t = trans[i]; VLOG(4) << "\tX axis[" << i << "]: " << t->to_string(); } diff --git a/paddle/phi/infermeta/spmd_rules/utils.cc b/paddle/phi/infermeta/spmd_rules/utils.cc index b67d7bd251b1b..336924dd5e951 100644 --- a/paddle/phi/infermeta/spmd_rules/utils.cc +++ b/paddle/phi/infermeta/spmd_rules/utils.cc @@ -423,13 +423,14 @@ TensorDistAttr FromPlacements( auto& placement = placements[mesh_dim]; if (placement->is_shard()) { auto shard_placement = std::dynamic_pointer_cast(placement); - dims_mapping[shard_placement->get_axis()] = mesh_dim; + dims_mapping[shard_placement->get_axis()] = + static_cast(mesh_dim); } if (placement->is_partial()) { auto partial_placement = std::dynamic_pointer_cast(placement); auto reduce_type = partial_placement->get_reduce_type(); - partial_status[mesh_dim] = reduce_type; + partial_status[mesh_dim] = reduce_type; // NOLINT } } dst_dist_attr.set_dims_mapping(dims_mapping); @@ -470,7 +471,7 @@ std::vector GetLocalShape( for (size_t i = 0; i < n_placement; i++) { auto& placement = placements.at(i); if (placement->is_shard()) { - auto mesh_dim_size = mesh.dim_size(i); + auto mesh_dim_size = mesh.dim_size(i); // NOLINT auto shard_dim = std::dynamic_pointer_cast(placement)->get_axis(); auto split_size = diff --git a/paddle/phi/kernels/funcs/jit/gen/blas.cc b/paddle/phi/kernels/funcs/jit/gen/blas.cc index 8c287efcf5ddd..1e29b7f4953fe 100644 --- a/paddle/phi/kernels/funcs/jit/gen/blas.cc +++ b/paddle/phi/kernels/funcs/jit/gen/blas.cc @@ -104,7 +104,7 @@ void VXXJitCode::genCode() { } else { vmovss(ptr[param3 + offset], xmm_dst); } - offset += sizeof(float) * block; + offset += sizeof(float) * block; // NOLINT rest -= block; } ret(); diff --git a/paddle/phi/kernels/funcs/jit/gen/gru.cc b/paddle/phi/kernels/funcs/jit/gen/gru.cc index 599564f431497..33dfaa6cd097c 100644 --- a/paddle/phi/kernels/funcs/jit/gen/gru.cc +++ b/paddle/phi/kernels/funcs/jit/gen/gru.cc @@ -39,7 +39,7 @@ void GRUJitCode::genCode() { vmovaps(ymm_one, ptr[reg_ptr_tmp + OFFSET_EXP_ONE]); } int offset = 0; - int d = num_ * sizeof(float); + int d = num_ * sizeof(float); // NOLINT for (int i = 0; i < num_ / YMM_FLOAT_BLOCK; ++i) { ymm_t ymm_u = ymm_t(1); ymm_t ymm_r = ymm_t(2); diff --git a/paddle/phi/kernels/funcs/jit/gen/lstm.cc b/paddle/phi/kernels/funcs/jit/gen/lstm.cc index e22a5a2880dff..4943989a50c79 100644 --- a/paddle/phi/kernels/funcs/jit/gen/lstm.cc +++ b/paddle/phi/kernels/funcs/jit/gen/lstm.cc @@ -42,7 +42,7 @@ void LSTMJitCode::genCode() { } int offset = 0; - int d = num_ * sizeof(float); + int d = num_ * sizeof(float); // NOLINT for (int i = 0; i < num_ / YMM_FLOAT_BLOCK; ++i) { /* gates: W_ch, W_ih, W_fh, W_oh */ ymm_t ymm_c = ymm_t(0); diff --git a/paddle/phi/kernels/fusion/onednn/fused_transpose_kernel.cc b/paddle/phi/kernels/fusion/onednn/fused_transpose_kernel.cc index a7f9e49e32560..f8a2f4fe0201e 100644 --- a/paddle/phi/kernels/fusion/onednn/fused_transpose_kernel.cc +++ b/paddle/phi/kernels/fusion/onednn/fused_transpose_kernel.cc @@ -34,7 +34,7 @@ void SetInMemDescWithSqueeze2FuseSupport( int j = 0; for (size_t i = 0; i < x_vec_dims.size(); ++i) { if (squeeze2_axes_set.count(i) || - squeeze2_axes_set.count(i - x_vec_dims.size())) { + squeeze2_axes_set.count(i - x_vec_dims.size())) { // NOLINT PADDLE_ENFORCE_EQ( x_vec_dims[i], 1, @@ -68,7 +68,7 @@ void FusedTransposeKernel(const Context& dev_ctx, if ((x_dims.size() >= 3) && (phi::OneDNNContext::tls().get_cur_paddle_data_layout() == phi::DataLayout::kNHWC)) { - int axis_size = axis.size(); + int axis_size = static_cast(axis.size()); std::vector formated_axis = axis; std::vector count(axis_size, 0); for (int i = 0; i < axis_size; i++) { @@ -85,7 +85,7 @@ void FusedTransposeKernel(const Context& dev_ctx, phi::DDim out_dims(x_dims); for (size_t i = 0; i < axis.size(); i++) { - out_dims[i] = x_dims[formated_axis[i]]; + out_dims[i] = x_dims[formated_axis[i]]; // NOLINT } out->Resize(out_dims); } diff --git a/paddle/phi/kernels/onednn/concat_grad_kernel.cc b/paddle/phi/kernels/onednn/concat_grad_kernel.cc index fc36fa4ab0fd8..9563f73f0ba92 100644 --- a/paddle/phi/kernels/onednn/concat_grad_kernel.cc +++ b/paddle/phi/kernels/onednn/concat_grad_kernel.cc @@ -40,7 +40,7 @@ void ConcatGradKernel(const Context& dev_ctx, auto out_grad_vec_dims = common::vectorize(out_grad.dims()); - axis = funcs::ComputeAxis(axis, out_grad_vec_dims.size()); + axis = static_cast(funcs::ComputeAxis(axis, out_grad_vec_dims.size())); std::vector offset(out_grad_vec_dims.size(), 0); @@ -60,7 +60,7 @@ void ConcatGradKernel(const Context& dev_ctx, auto reorder_dst_memory_p = reorder_handler.AcquireDstMemory( grad, x_grad_vec_dims, - funcs::GetPlainOneDNNFormat(x_grad_vec_dims.size()), + funcs::GetPlainOneDNNFormat(static_cast(x_grad_vec_dims.size())), dev_ctx.GetPlace()); auto reorder_p = reorder_handler.AcquireReorder(reorder_dst_memory_p, slice_mem_p); diff --git a/paddle/phi/kernels/onednn/expand_grad_kernel.cc b/paddle/phi/kernels/onednn/expand_grad_kernel.cc index a8b1beb45832f..7de901df9561d 100644 --- a/paddle/phi/kernels/onednn/expand_grad_kernel.cc +++ b/paddle/phi/kernels/onednn/expand_grad_kernel.cc @@ -50,7 +50,7 @@ void ExpandGradKernel(const Context& dev_ctx, auto reorder_dst_memory_p = reorder_handler.AcquireDstMemory( in_grad, - funcs::GetPlainOneDNNFormat(in_grad_vec_dims.size()), + funcs::GetPlainOneDNNFormat(static_cast(in_grad_vec_dims.size())), dev_ctx.GetPlace()); auto reorder_p = reorder_handler.AcquireReorder(reorder_src_memory_p, diff --git a/paddle/phi/kernels/onednn/matmul_grad_kernel.cc b/paddle/phi/kernels/onednn/matmul_grad_kernel.cc index 3866a2d06ae45..46a2a7450d41c 100644 --- a/paddle/phi/kernels/onednn/matmul_grad_kernel.cc +++ b/paddle/phi/kernels/onednn/matmul_grad_kernel.cc @@ -51,8 +51,10 @@ void CalculateMatrixDims(const std::vector &x_dims, for (size_t i = 0; i < x_bd_dims->size() - 2; ++i) { (*out_bd_dims)[i] = std::max((*x_bd_dims)[i], (*y_bd_dims)[i]); } - int h_idx = trans_x ? x_bd_dims->size() - 1 : x_bd_dims->size() - 2; - int w_idx = trans_y ? y_bd_dims->size() - 2 : y_bd_dims->size() - 1; + int h_idx = + trans_x ? x_bd_dims->size() - 1 : x_bd_dims->size() - 2; // NOLINT + int w_idx = + trans_y ? y_bd_dims->size() - 2 : y_bd_dims->size() - 1; // NOLINT (*out_bd_dims)[x_bd_dims->size() - 2] = (*x_bd_dims)[h_idx]; (*out_bd_dims)[y_bd_dims->size() - 1] = (*y_bd_dims)[w_idx]; diff --git a/paddle/phi/kernels/onednn/matmul_kernel.cc b/paddle/phi/kernels/onednn/matmul_kernel.cc index b7b31ff479b30..342fce6f2be02 100644 --- a/paddle/phi/kernels/onednn/matmul_kernel.cc +++ b/paddle/phi/kernels/onednn/matmul_kernel.cc @@ -124,7 +124,7 @@ void MatmulKernel(const Context &dev_ctx, auto x_dims = common::vectorize(x.dims()); auto y_dims = common::vectorize(y.dims()); - int ndims = std::max(x_dims.size(), y_dims.size()); + int ndims = std::max(x_dims.size(), y_dims.size()); // NOLINT ndims = std::max(ndims, 3); std::vector x_bd_dims(ndims, 1); @@ -266,7 +266,7 @@ class MulPrimitiveFactory { auto scale_out_data = force_fp32_output ? 1.0f : scale_out; bool is_multi_channel = scale_y_data.size() > 1; - int count = is_multi_channel ? scale_y_data.size() : 1; + int count = is_multi_channel ? scale_y_data.size() : 1; // NOLINT std::vector output_shift_scale(count); for (int i = 0; i < count; i++) { if (scale_y_data[i] == 0.0) diff --git a/paddle/phi/kernels/onednn/slice_grad_kernel.cc b/paddle/phi/kernels/onednn/slice_grad_kernel.cc index 7f8f6b815b4f0..a929751433ab9 100644 --- a/paddle/phi/kernels/onednn/slice_grad_kernel.cc +++ b/paddle/phi/kernels/onednn/slice_grad_kernel.cc @@ -60,7 +60,7 @@ void SliceGradKernel(const Context& dev_ctx, auto reorder_dst_memory_p = reorder_handler.AcquireDstMemory( input_grad, dx_dims, - funcs::GetPlainOneDNNFormat(dx_dims.size()), + funcs::GetPlainOneDNNFormat(static_cast(dx_dims.size())), dev_ctx.GetPlace()); memset(input_grad->data(), 0, reorder_dst_memory_p->get_desc().get_size()); diff --git a/paddle/phi/kernels/onednn/slice_kernel.cc b/paddle/phi/kernels/onednn/slice_kernel.cc index bd59d61c17e79..aeff6168f047c 100644 --- a/paddle/phi/kernels/onednn/slice_kernel.cc +++ b/paddle/phi/kernels/onednn/slice_kernel.cc @@ -69,7 +69,7 @@ void SliceKernel(const Context& dev_ctx, auto reorder_dst_memory_p = reorder_handler.AcquireDstMemory( out, slice_dims, - funcs::GetPlainOneDNNFormat(x_vec_dims.size()), + funcs::GetPlainOneDNNFormat(static_cast(x_vec_dims.size())), dev_ctx.GetPlace()); auto reorder_p = diff --git a/paddle/phi/kernels/onednn/squeeze_grad_kernel.cc b/paddle/phi/kernels/onednn/squeeze_grad_kernel.cc index d8ff4e72c1b11..78a3c4dce6bd3 100644 --- a/paddle/phi/kernels/onednn/squeeze_grad_kernel.cc +++ b/paddle/phi/kernels/onednn/squeeze_grad_kernel.cc @@ -37,7 +37,7 @@ void SqueezeGradKernel(const Context& dev_ctx, dout.mem_desc(), funcs::to_void_cast(dout.data())); auto reorder_dst_memory_p = reorder_handler.AcquireDstMemory( dx, - funcs::GetPlainOneDNNFormat(dout_vec_dims.size()), + funcs::GetPlainOneDNNFormat(static_cast(dout_vec_dims.size())), dev_ctx.GetPlace()); auto reorder_p = reorder_handler.AcquireReorder(reorder_dst_memory_p, reorder_src_memory_p); diff --git a/test/cpp/fluid/fused/cudnn_bn_add_relu_test.cc b/test/cpp/fluid/fused/cudnn_bn_add_relu_test.cc index 770093efdacb4..cad204415174b 100644 --- a/test/cpp/fluid/fused/cudnn_bn_add_relu_test.cc +++ b/test/cpp/fluid/fused/cudnn_bn_add_relu_test.cc @@ -764,7 +764,7 @@ class CudnnBNAddReluTester { int c = channels_; int64_t nhw = ele_count_; int32_t c_int32_elems = ((c + 63) & ~63) / 32; - int32_t nhw_int32_elems = (nhw + 31) & ~31; + int32_t nhw_int32_elems = (static_cast(nhw) + 31) & ~31; bitmask.Resize(common::make_ddim({nhw_int32_elems, c_int32_elems, 1})); auto data_shape = common::vectorize(x.dims()); diff --git a/test/cpp/fluid/memory/buddy_allocator_test.cc b/test/cpp/fluid/memory/buddy_allocator_test.cc index b399e6fc2ade1..7f4f452d0ebc3 100644 --- a/test/cpp/fluid/memory/buddy_allocator_test.cc +++ b/test/cpp/fluid/memory/buddy_allocator_test.cc @@ -173,8 +173,8 @@ TEST(BuddyAllocator, FractionRefillPool) { // Max chunk size should be same during allocation EXPECT_EQ(max_chunk_size, buddy_allocator.GetMaxChunkSize()); - size_t alloc = - platform::GpuAvailableMemToAlloc() * FLAGS_fraction_of_gpu_memory_to_use; + size_t alloc = platform::GpuAvailableMemToAlloc() * + FLAGS_fraction_of_gpu_memory_to_use; // NOLINT // Exceed pool trigger refilling size of fraction of avaiable gpu, and should // be able to alloc 60% of the remaining GPU int* p1 = TestBuddyAllocator(&buddy_allocator, @@ -184,8 +184,8 @@ TEST(BuddyAllocator, FractionRefillPool) { // Max chunk size should be same during allocation EXPECT_EQ(max_chunk_size, buddy_allocator.GetMaxChunkSize()); - alloc = - platform::GpuAvailableMemToAlloc() * FLAGS_fraction_of_gpu_memory_to_use; + alloc = platform::GpuAvailableMemToAlloc() * + FLAGS_fraction_of_gpu_memory_to_use; // NOLINT // Exceed pool trigger refilling size of fraction of avaiable gpu, and should // be able to alloc 60% of the remaining GPU TestBuddyAllocator(&buddy_allocator, diff --git a/test/cpp/imperative/test_group.cc b/test/cpp/imperative/test_group.cc index 2243a24dee90d..287e67c9bcff4 100644 --- a/test/cpp/imperative/test_group.cc +++ b/test/cpp/imperative/test_group.cc @@ -73,7 +73,7 @@ void GroupConcatSplit(Place place, size_t size) { std::vector value; for (size_t j = 0; j < len; ++j) { - value.push_back(static_cast(1.0 * j)); + value.push_back(static_cast(1.0 * j)); // NOLINT } if (std::is_same::value) { @@ -89,7 +89,7 @@ void GroupConcatSplit(Place place, size_t size) { phi::DenseTensor tmp; tmp.ShareDataWith(*tensor).Resize({static_cast(len)}); group.dense_tensors_.push_back(std::move(tmp)); - group.all_length_ += len; + group.all_length_ += static_cast(len); group.dtype_ = framework::TransToProtoVarType(tensor->dtype()); } diff --git a/test/cpp/inference/api/analyzer_dam_tester.cc b/test/cpp/inference/api/analyzer_dam_tester.cc index d17f8670adcf4..ea31fe3760b53 100644 --- a/test/cpp/inference/api/analyzer_dam_tester.cc +++ b/test/cpp/inference/api/analyzer_dam_tester.cc @@ -193,7 +193,7 @@ void SetInput(std::vector> *inputs) { DataRecord data(FLAGS_infer_data, FLAGS_batch_size); std::vector input_slots; int test_batch_num = - FLAGS_test_all_data ? data.num_samples / FLAGS_batch_size : 1; + FLAGS_test_all_data ? data.num_samples / FLAGS_batch_size : 1; // NOLINT LOG(INFO) << "The number of samples to be test: " << test_batch_num * FLAGS_batch_size; for (int bid = 0; bid < test_batch_num; ++bid) { diff --git a/test/cpp/inference/api/analyzer_int8_object_detection_tester.cc b/test/cpp/inference/api/analyzer_int8_object_detection_tester.cc index 311fb0946ca00..12be843475b74 100644 --- a/test/cpp/inference/api/analyzer_int8_object_detection_tester.cc +++ b/test/cpp/inference/api/analyzer_int8_object_detection_tester.cc @@ -43,7 +43,7 @@ std::vector ReadObjectsNum(std::ifstream &file, file.clear(); file.seekg(offset); file.read(reinterpret_cast(num_objects.data()), - total_images * sizeof(size_t)); + total_images * sizeof(size_t)); // NOLINT if (file.eof()) LOG(ERROR) << "Reached end of stream"; if (file.fail()) throw std::runtime_error("Failed reading file."); diff --git a/test/cpp/inference/api/analyzer_lexical_analysis_gru_tester.cc b/test/cpp/inference/api/analyzer_lexical_analysis_gru_tester.cc index 2a79ce572dda2..2d0355d361b2d 100644 --- a/test/cpp/inference/api/analyzer_lexical_analysis_gru_tester.cc +++ b/test/cpp/inference/api/analyzer_lexical_analysis_gru_tester.cc @@ -49,7 +49,7 @@ std::vector ReadSentenceLod(std::ifstream &file, file.clear(); file.seekg(offset); file.read(reinterpret_cast(sentence_lod.data()), - total_sentences_num * sizeof(size_t)); + total_sentences_num * sizeof(size_t)); // NOLINT if (file.eof()) LOG(ERROR) << "Reached end of stream"; if (file.fail()) throw std::runtime_error("Failed reading file."); diff --git a/test/cpp/phi/kernels/test_fused_adam_kernel.cc b/test/cpp/phi/kernels/test_fused_adam_kernel.cc index 73e1b21ac3120..ec0926508c9e8 100644 --- a/test/cpp/phi/kernels/test_fused_adam_kernel.cc +++ b/test/cpp/phi/kernels/test_fused_adam_kernel.cc @@ -445,7 +445,7 @@ static auto GenerateRandomShapes(size_t n, uint64_t low, uint64_t high) { std::uniform_int_distribution dist(low, high); std::vector> shapes(n); for (size_t i = 0; i < n; ++i) { - shapes[i].push_back(dist(engine)); + shapes[i].push_back(static_cast(dist(engine))); } return shapes; } diff --git a/test/cpp/phi/kernels/test_memcpy_dev_api.cc b/test/cpp/phi/kernels/test_memcpy_dev_api.cc index 14f5fe15c301b..9a35a1ad99c3f 100644 --- a/test/cpp/phi/kernels/test_memcpy_dev_api.cc +++ b/test/cpp/phi/kernels/test_memcpy_dev_api.cc @@ -43,7 +43,7 @@ TEST(DEV_API, memcpy_d2h) { auto* x_cpu_data = cpu_ctx->template Alloc(&x_cpu); for (int i = 0; i < x_cpu.numel(); i++) { - x_cpu_data[i] = i; + x_cpu_data[i] = static_cast(i); } const auto alloc = From 9d7883a47040b284fb0c0006932d955345988adc Mon Sep 17 00:00:00 2001 From: cyberslack_lee Date: Fri, 1 Mar 2024 10:56:51 +0800 Subject: [PATCH 212/282] [clang-tidy] NO.5 cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays (#61751) --- .../distributed/test/graph_node_split_test.cc | 8 +-- .../fluid/distributed/test/graph_node_test.cc | 10 +-- .../test/graph_table_sample_test.cc | 6 +- .../distributed/test/sparse_sgd_rule_test.cc | 66 +++++++++---------- paddle/fluid/framework/fleet/metrics.cc | 2 +- .../fluid/framework/heter_section_worker.cc | 2 +- paddle/fluid/framework/io/shell.cc | 20 +++--- .../fluid/operators/controlflow/pylayer_op.cc | 11 ++-- paddle/fluid/operators/nccl/nccl_op.cc | 2 +- .../pir/dialect/operator/ir/manual_op.cc | 16 +++-- paddle/fluid/platform/collective_helper.cc | 4 +- .../platform/profiler/cpu_utilization.cc | 13 ++-- paddle/fluid/pybind/eager_method.cc | 42 ++++++------ paddle/fluid/pybind/eager_properties.cc | 30 ++++----- paddle/fluid/pybind/eval_frame_tools.cc | 2 +- .../fusion/cpu/self_dp_attention_kernel.cc | 4 +- test/cpp/fluid/framework/tensor_util_test.cc | 4 +- test/cpp/fluid/math/im2col_test.cc | 10 +-- test/cpp/fluid/math/vol2col_test.cc | 9 +-- .../api/analysis_predictor_tester.cc | 12 ++-- .../api/analyzer_capi_exp_gpu_tester.cc | 16 ++--- .../api/analyzer_capi_exp_int_tester.cc | 16 ++--- .../api/analyzer_capi_exp_ner_tester.cc | 23 +++---- .../api/analyzer_capi_exp_pd_tensor_tester.cc | 22 +++---- .../analyzer_capi_exp_pd_threads_tester.cc | 4 +- .../inference/api/analyzer_capi_exp_tester.cc | 4 +- test/cpp/inference/api/analyzer_dam_tester.cc | 4 +- test/cpp/inference/api/analyzer_lac_tester.cc | 2 +- test/cpp/inference/api/analyzer_ner_tester.cc | 2 +- .../cpp/inference/api/analyzer_rnn1_tester.cc | 8 ++- .../api/trt_dynamic_shape_ernie_test.cc | 14 ++-- ...rt_dynamic_shape_transformer_prune_test.cc | 28 ++++---- .../inference/api/trt_rebind_stream_test.cc | 4 +- .../new_executor/standalone_executor_test.cc | 8 +-- test/cpp/phi/api/test_from_blob.cc | 16 ++--- test/cpp/phi/core/test_custom_kernel.cc | 2 +- test/cpp/phi/kernels/strided_memcpy_test.cc | 22 ++++--- test/cpp/pir/tools/test_op.cc | 3 +- 38 files changed, 244 insertions(+), 227 deletions(-) diff --git a/paddle/fluid/distributed/test/graph_node_split_test.cc b/paddle/fluid/distributed/test/graph_node_split_test.cc index cb47f3103883f..cbb7741a0a2d3 100644 --- a/paddle/fluid/distributed/test/graph_node_split_test.cc +++ b/paddle/fluid/distributed/test/graph_node_split_test.cc @@ -55,7 +55,7 @@ std::vector edges = {std::string("37\t45\t0.34"), std::string("97\t48\t0.34"), std::string("97\t247\t0.31"), std::string("97\t111\t0.21")}; -char edge_file_name[] = "edges.txt"; +char edge_file_name[] = "edges.txt"; // NOLINT std::vector nodes = { std::string("user\t37\ta 0.34\tb 13 14\tc hello\td abc"), @@ -74,12 +74,12 @@ std::vector nodes = { std::string("item\t49\ta 0.21"), std::string("item\t248\ta 0.21"), std::string("item\t113\ta 0.21")}; -char node_file_name[] = "nodes.txt"; +char node_file_name[] = "nodes.txt"; // NOLINT std::vector graph_split = {std::string("0\t97")}; -char graph_split_file_name[] = "graph_split.txt"; +char graph_split_file_name[] = "graph_split.txt"; // NOLINT -void prepare_file(char file_name[], std::vector data) { +void prepare_file(char file_name[], std::vector data) { // NOLINT std::ofstream ofile; ofile.open(file_name); for (auto x : data) { diff --git a/paddle/fluid/distributed/test/graph_node_test.cc b/paddle/fluid/distributed/test/graph_node_test.cc index 8c29c2bf1df3f..9cc16cb2580f5 100644 --- a/paddle/fluid/distributed/test/graph_node_test.cc +++ b/paddle/fluid/distributed/test/graph_node_test.cc @@ -236,8 +236,8 @@ const char* edges[] = {"37\t45\t0.34", "59\t122\t0.21", "97\t48\t0.34", "97\t247\t0.31", - "97\t111\t0.21"}; -char edge_file_name[] = "edges.txt"; + "97\t111\t0.21"}; // NOLINT +char edge_file_name[] = "edges.txt"; // NOLINT const char* nodes[] = {"user\t37\ta 0.34\tb 13 14\tc hello\td abc", "user\t96\ta 0.31\tb 15 10\tc 96hello\td abcd", @@ -254,10 +254,10 @@ const char* nodes[] = {"user\t37\ta 0.34\tb 13 14\tc hello\td abc", "item\t122\ta 0.21", "item\t49\ta 0.21", "item\t248\ta 0.21", - "item\t113\ta 0.21"}; -char node_file_name[] = "nodes.txt"; + "item\t113\ta 0.21"}; // NOLINT +char node_file_name[] = "nodes.txt"; // NOLINT -void prepare_file(char file_name[], bool load_edge) { +void prepare_file(char file_name[], bool load_edge) { // NOLINT std::ofstream ofile; ofile.open(file_name); if (load_edge) { diff --git a/paddle/fluid/distributed/test/graph_table_sample_test.cc b/paddle/fluid/distributed/test/graph_table_sample_test.cc index 5489129a070dd..286b19b7070ac 100644 --- a/paddle/fluid/distributed/test/graph_table_sample_test.cc +++ b/paddle/fluid/distributed/test/graph_table_sample_test.cc @@ -43,7 +43,7 @@ std::vector edges = {std::string("37\t45\t0.34"), std::string("97\t247\t0.31"), std::string("97\t111\t0.21")}; // odd id:96 48 122 112 -char edge_file_name[] = "edges.txt"; +char edge_file_name[] = "edges.txt"; // NOLINT std::vector nodes = { std::string("user\t37\ta 0.34\tb 13 14\tc hello\td abc"), @@ -62,9 +62,9 @@ std::vector nodes = { std::string("item\t49\ta 0.21"), std::string("item\t248\ta 0.21"), std::string("item\t113\ta 0.21")}; -char node_file_name[] = "nodes.txt"; +char node_file_name[] = "nodes.txt"; // NOLINT -void prepare_file(char file_name[], std::vector data) { +void prepare_file(char file_name[], std::vector data) { // NOLINT std::ofstream ofile; ofile.open(file_name); for (auto x : data) { diff --git a/paddle/fluid/distributed/test/sparse_sgd_rule_test.cc b/paddle/fluid/distributed/test/sparse_sgd_rule_test.cc index 120d8de56f793..a7029d1e8b127 100644 --- a/paddle/fluid/distributed/test/sparse_sgd_rule_test.cc +++ b/paddle/fluid/distributed/test/sparse_sgd_rule_test.cc @@ -37,8 +37,8 @@ TEST(sparse_value_naive_sgd_test, init_and_update) { // check init_value for zero const int kItemSize = 10; - float w[kItemSize]; - float grad[kItemSize]; + float w[kItemSize]; // NOLINT + float grad[kItemSize]; // NOLINT rule.InitValue(w, w + 9, true); for (float item : w) { @@ -58,16 +58,16 @@ TEST(sparse_value_naive_sgd_test, init_and_update) { for (auto i = 0u; i < kItemSize; ++i) { grad[i] = static_cast(i + 1) * 1.0; } - float label[] = {-0.100000, - -0.200000, - -0.300000, - -0.400000, - -0.500000, - -0.600000, - -0.700000, - -0.800000, - -0.900000, - -1.000000}; + std::array label = {-0.100000, + -0.200000, + -0.300000, + -0.400000, + -0.500000, + -0.600000, + -0.700000, + -0.800000, + -0.900000, + -1.000000}; const float* ptr_grad = grad; rule.UpdateValue(w, w + 9, ptr_grad); @@ -93,7 +93,7 @@ TEST(downpour_sparse_adagrad_test, test_init_and_update) { // check init_value for zero const int kValueSize = 11; int kEmbSize = 10; - float w[kValueSize]; + float w[kValueSize]; // NOLINT rule.InitValue(w, w + 10, true); @@ -114,24 +114,24 @@ TEST(downpour_sparse_adagrad_test, test_init_and_update) { w[i] = 0; } w[kEmbSize] = 0; - float grad[kEmbSize]; + float grad[kEmbSize]; // NOLINT for (int i = 0; i < kEmbSize; ++i) { grad[i] = static_cast(i + 1) * 1.0; } const float* ptr_grad = grad; rule.UpdateValue(w, w + 10, ptr_grad); - float label[] = {-0.100000, - -0.200000, - -0.300000, - -0.400000, - -0.500000, - -0.600000, - -0.700000, - -0.800000, - -0.900000, - -1.000000, - 38.500000}; + std::array label = {-0.100000, + -0.200000, + -0.300000, + -0.400000, + -0.500000, + -0.600000, + -0.700000, + -0.800000, + -0.900000, + -1.000000, + 38.500000}; for (auto i = 0u; i < kValueSize; ++i) { ASSERT_FLOAT_EQ(w[i], label[i]); } @@ -190,14 +190,14 @@ TEST(downpour_sparse_adam_test, test_init_and_update) { grad[i] = static_cast(i + 1) * 1.0; } - float label[] = {-0.0999999642, -0.099999994, -0.099999994, -0.099999994, - -0.099999994, -0.099999994, -0.099999994, -0.100000001, - -0.100000009, -0.100000001, 0.100000024, 0.200000048, - 0.300000072, 0.400000095, 0.500000119, 0.600000143, - 0.700000167, 0.800000191, 0.900000215, 1.00000024, - 0.000999987125, 0.0039999485, 0.00899988413, 0.015999794, - 0.0249996781, 0.0359995365, 0.0489993691, 0.063999176, - 0.0809989572, 0.0999987125, 0.809999943, 0.998001039}; + std::array label = { + -0.0999999642, -0.099999994, -0.099999994, -0.099999994, -0.099999994, + -0.099999994, -0.099999994, -0.100000001, -0.100000009, -0.100000001, + 0.100000024, 0.200000048, 0.300000072, 0.400000095, 0.500000119, + 0.600000143, 0.700000167, 0.800000191, 0.900000215, 1.00000024, + 0.000999987125, 0.0039999485, 0.00899988413, 0.015999794, 0.0249996781, + 0.0359995365, 0.0489993691, 0.063999176, 0.0809989572, 0.0999987125, + 0.809999943, 0.998001039}; rule.UpdateValue(value, value + embed_dim, grad); diff --git a/paddle/fluid/framework/fleet/metrics.cc b/paddle/fluid/framework/fleet/metrics.cc index 5801860f66566..57fe43fb44624 100644 --- a/paddle/fluid/framework/fleet/metrics.cc +++ b/paddle/fluid/framework/fleet/metrics.cc @@ -219,7 +219,7 @@ void BasicAucCalculator::calculate_bucket_error() { } } } else { - double* table[2] = {&_table[0][0], &_table[1][0]}; + double* table[2] = {&_table[0][0], &_table[1][0]}; // NOLINT for (int i = 0; i < _table_size; i++) { double click = table[1][i]; double show = table[0][i] + table[1][i]; diff --git a/paddle/fluid/framework/heter_section_worker.cc b/paddle/fluid/framework/heter_section_worker.cc index 65902f6c2d0c7..cecfa39d3c16b 100644 --- a/paddle/fluid/framework/heter_section_worker.cc +++ b/paddle/fluid/framework/heter_section_worker.cc @@ -507,7 +507,7 @@ void HeterSectionWorker::PrintFetchVars() { if (thread_id_ == 0 && batch_num_ % batch_per_print == 0) { time_t curtime; time(&curtime); - char mbstr[80]; + char mbstr[80]; // NOLINT std::strftime( mbstr, sizeof(mbstr), "%Y-%m-%d %H:%M:%S", std::localtime(&curtime)); std::stringstream ss; diff --git a/paddle/fluid/framework/io/shell.cc b/paddle/fluid/framework/io/shell.cc index cc893fefbb34f..fa449c1b10867 100644 --- a/paddle/fluid/framework/io/shell.cc +++ b/paddle/fluid/framework/io/shell.cc @@ -58,7 +58,7 @@ static int close_open_fds_internal() { long d_ino = 0; // NOLINT off_t d_off; unsigned short d_reclen = 0; // NOLINT - char d_name[256]; + char d_name[256]; // NOLINT }; int dir_fd = -1; @@ -66,7 +66,7 @@ static int close_open_fds_internal() { PADDLE_THROW(platform::errors::Unavailable("Failed to open proc/self/fd.")); return -1; } - char buffer[sizeof(linux_dirent)]; + char buffer[sizeof(linux_dirent)]; // NOLINT for (;;) { int bytes = 0; @@ -187,8 +187,8 @@ std::shared_ptr shell_popen(const std::string& cmd, std::string real_cmd = "set -o pipefail; " + cmd; - int pipe_fds[2]; - if (pipe(pipe_fds) != 0) { + std::array pipe_fds; + if (pipe(pipe_fds.data()) != 0) { *err_no = -1; return nullptr; } @@ -300,17 +300,17 @@ std::pair, std::shared_ptr> shell_p2open( std::string real_cmd = "set -o pipefail; " + cmd; - int pipein_fds[2]; - int pipeout_fds[2]; - if (pipe(pipein_fds) != 0) { + std::array pipein_fds; + std::array pipeout_fds; + if (pipe(pipein_fds.data()) != 0) { return {nullptr, nullptr}; } - if (pipe(pipeout_fds) != 0) { + if (pipe(pipeout_fds.data()) != 0) { return {nullptr, nullptr}; } - int child_pid = - shell_p2open_fork_internal(real_cmd.c_str(), pipein_fds, pipeout_fds); + int child_pid = shell_p2open_fork_internal( + real_cmd.c_str(), pipein_fds.data(), pipeout_fds.data()); close(pipein_fds[1]); close(pipeout_fds[0]); diff --git a/paddle/fluid/operators/controlflow/pylayer_op.cc b/paddle/fluid/operators/controlflow/pylayer_op.cc index c4b06f326a703..bd83c99a0c62d 100644 --- a/paddle/fluid/operators/controlflow/pylayer_op.cc +++ b/paddle/fluid/operators/controlflow/pylayer_op.cc @@ -26,11 +26,12 @@ namespace { // NOLINT enum class PyLayerBlockIndex { kFORWARD = 0, kBACKWARD = 1, kNONE = 2 }; } // namespace -const char PyLayerOp::kInputs[] = "Input"; -const char PyLayerOp::kOutputs[] = "Out"; -const char PyLayerOp::kScope[] = "Scope"; -const char PyLayerOp::kSkipEagerDeletionVars[] = "skip_eager_deletion_vars"; -const char PyLayerOp::kBlocks[] = "blocks"; +const char PyLayerOp::kInputs[] = "Input"; // NOLINT +const char PyLayerOp::kOutputs[] = "Out"; // NOLINT +const char PyLayerOp::kScope[] = "Scope"; // NOLINT +const char PyLayerOp::kSkipEagerDeletionVars[] = + "skip_eager_deletion_vars"; // NOLINT +const char PyLayerOp::kBlocks[] = "blocks"; // NOLINT void PyLayerOp::CreateInterpreter( const platform::Place &dev_place, diff --git a/paddle/fluid/operators/nccl/nccl_op.cc b/paddle/fluid/operators/nccl/nccl_op.cc index 8b06aa653c070..c5a1097e2f157 100644 --- a/paddle/fluid/operators/nccl/nccl_op.cc +++ b/paddle/fluid/operators/nccl/nccl_op.cc @@ -18,7 +18,7 @@ limitations under the License. */ namespace paddle { namespace operators { -static constexpr char kParallelScopes[] = "parallel_scopes"; +static constexpr char kParallelScopes[] = "parallel_scopes"; // NOLINT // NCCLinitOp class NCCLInitOp : public framework::OperatorBase { diff --git a/paddle/fluid/pir/dialect/operator/ir/manual_op.cc b/paddle/fluid/pir/dialect/operator/ir/manual_op.cc index ec61f6c7dd88d..b7cebeaf27f47 100644 --- a/paddle/fluid/pir/dialect/operator/ir/manual_op.cc +++ b/paddle/fluid/pir/dialect/operator/ir/manual_op.cc @@ -535,8 +535,10 @@ std::vector AddNArrayOp::InferMeta( return argument_outputs; } -const char *FusedGemmEpilogueOp::attributes_name[3] = { - "trans_x", "trans_y", "activation"}; +const char *FusedGemmEpilogueOp::attributes_name[3] = { // NOLINT + "trans_x", + "trans_y", + "activation"}; OpInfoTuple FusedGemmEpilogueOp::GetOpInfo() { std::vector inputs = { @@ -849,8 +851,10 @@ std::vector FusedGemmEpilogueOp::InferMeta( return argument_outputs; } -const char *FusedGemmEpilogueGradOp::attributes_name[3] = { - "trans_x", "trans_y", "activation_grad"}; +const char *FusedGemmEpilogueGradOp::attributes_name[3] = { // NOLINT + "trans_x", + "trans_y", + "activation_grad"}; OpInfoTuple FusedGemmEpilogueGradOp::GetOpInfo() { std::vector inputs = { @@ -1171,7 +1175,7 @@ std::vector FusedGemmEpilogueGradOp::InferMeta( return argument_outputs; } -const char *SplitGradOp::attributes_name[1] = {"axis"}; +const char *SplitGradOp::attributes_name[1] = {"axis"}; // NOLINT OpInfoTuple SplitGradOp::GetOpInfo() { std::vector inputs = { @@ -1360,7 +1364,7 @@ std::vector SplitGradOp::InferMeta( return argument_outputs; } -const char *CreateArrayOp::attributes_name[1] = {"dtype"}; +const char *CreateArrayOp::attributes_name[1] = {"dtype"}; // NOLINT OpInfoTuple CreateArrayOp::GetOpInfo() { std::vector inputs = {}; diff --git a/paddle/fluid/platform/collective_helper.cc b/paddle/fluid/platform/collective_helper.cc index 3444f71639b46..e3be121820684 100644 --- a/paddle/fluid/platform/collective_helper.cc +++ b/paddle/fluid/platform/collective_helper.cc @@ -133,7 +133,7 @@ void NCCLCommContext::CreateAllNCCLComms(const std::vector& dev_ids, dev_ids.size())); const int kDevices = dev_ids.size(); - ncclComm_t comms[kDevices]; + ncclComm_t comms[kDevices]; // NOLINT PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::ncclCommInitAll( comms, dev_ids.size(), dev_ids.data())); @@ -169,7 +169,7 @@ void NCCLCommContext::CreateNCCLCommMultiTrainer( VLOG(1) << "Begin CreateNCCLCommMultiTrainer. device number: " << kDevices << ", ntrainers: " << ntrainers << ", train_id: " << train_id << ", rind_id: " << ring_id; - ncclComm_t comms[kDevices]; + ncclComm_t comms[kDevices]; // NOLINT { PADDLE_ENFORCE_GPU_SUCCESS(dynload::ncclGroupStart()); for (int i = 0; i < kDevices; i++) { diff --git a/paddle/fluid/platform/profiler/cpu_utilization.cc b/paddle/fluid/platform/profiler/cpu_utilization.cc index e84256f49f078..d373ac32ea6aa 100644 --- a/paddle/fluid/platform/profiler/cpu_utilization.cc +++ b/paddle/fluid/platform/profiler/cpu_utilization.cc @@ -24,6 +24,7 @@ // limitations under the License. #include "paddle/fluid/platform/profiler/cpu_utilization.h" +#include namespace paddle { namespace platform { @@ -53,16 +54,16 @@ void CpuUtilization::RecordBeginTimeInfo() { #elif defined(__linux__) start_ = times(&process_tms_start_); #define proc_path_size 1024 - static char proc_stat_path[proc_path_size] = "/proc/stat"; + static char proc_stat_path[proc_path_size] = "/proc/stat"; // NOLINTf FILE *stat_file = fopen(proc_stat_path, "r"); if (stat_file != nullptr) { - char temp_str[200]; + std::array temp_str; uint64_t temp_lu; int retval = fscanf(stat_file, "%s %" PRIu64 "%" PRIu64 "%" PRIu64 "%" PRIu64 "%" PRIu64 "%" PRIu64 "%" PRIu64 "%" PRIu64 "%" PRIu64 "%" PRIu64, - temp_str, + temp_str.data(), &system_tms_start_.tms_utime, &nice_time_start_, &system_tms_start_.tms_stime, @@ -98,16 +99,16 @@ void CpuUtilization::RecordEndTimeInfo() { #elif defined(__linux__) end_ = times(&process_tms_end_); #define proc_path_size 1024 - static char proc_stat_path[proc_path_size] = "/proc/stat"; + static char proc_stat_path[proc_path_size] = "/proc/stat"; // NOLINT FILE *stat_file = fopen(proc_stat_path, "r"); if (stat_file != nullptr) { - char temp_str[200]; + std::array temp_str; uint64_t temp_lu; int retval = fscanf(stat_file, "%s %" PRIu64 "%" PRIu64 "%" PRIu64 "%" PRIu64 "%" PRIu64 "%" PRIu64 "%" PRIu64 "%" PRIu64 "%" PRIu64 "%" PRIu64, - temp_str, + temp_str.data(), &system_tms_end_.tms_utime, &nice_time_end_, &system_tms_end_.tms_stime, diff --git a/paddle/fluid/pybind/eager_method.cc b/paddle/fluid/pybind/eager_method.cc index 6fe07282a2223..16d5fea43fe76 100644 --- a/paddle/fluid/pybind/eager_method.cc +++ b/paddle/fluid/pybind/eager_method.cc @@ -603,7 +603,7 @@ static PyObject* tensor_method__copy_to(TensorObject* self, EAGER_CATCH_AND_THROW_RETURN_NULL } -PyDoc_STRVAR(tensor_reconstruct_from___doc__, +PyDoc_STRVAR(tensor_reconstruct_from___doc__, // NOLINT R"DOC(reconstruct_from_($self, other/) -- @@ -786,7 +786,7 @@ Enables this Tensor to have their grad populated during backward(). It is a no-o >>> print(y.grad) Tensor(shape=[3], dtype=float32, place=Place(cpu), stop_gradient=False, [1., 1., 1.]) -)DOC"); +)DOC"); // NOLINT static PyObject* tensor_retain_grads(TensorObject* self, PyObject* args, @@ -1219,7 +1219,7 @@ static PyObject* tensor_method_detach_(TensorObject* self, Py_INCREF(reinterpret_cast(self)); return reinterpret_cast(self); EAGER_CATCH_AND_THROW_RETURN_NULL -} +} // NOLINT PyDoc_STRVAR(tensor_method_get_tensor__doc__, R"DOC(get_tensor($self, /) -- @@ -1243,7 +1243,7 @@ Returns the underline tensor in the origin Tensor. - layout: NCHW - dtype: float32 - data: [1] -)DOC"); +)DOC"); // NOLINT static PyObject* tensor_method_get_underline_tensor(TensorObject* self, PyObject* args, @@ -2197,7 +2197,7 @@ Returns the total number of non zero elements in input SparseCooTensor/SparseCsr >>> coo.nnz() 3 -)DOC"); +)DOC"); // NOLINT static PyObject* tensor_method_get_non_zero_nums(TensorObject* self, PyObject* args, @@ -2247,7 +2247,7 @@ Returns the indices of non zero elements in input SparseCooTensor. [[0, 1, 2], [1, 2, 0]]) -)DOC"); +)DOC"); // NOLINT static PyObject* tensor_method_get_non_zero_indices(TensorObject* self, PyObject* args, @@ -2290,7 +2290,7 @@ Returns the values of non zero elements in input SparseCooTensor. Tensor(shape=[3], dtype=float32, place=Place(gpu:0), stop_gradient=True, [1., 2., 3.]) -)DOC"); +)DOC"); // NOLINT static PyObject* tensor_method_get_non_zero_elements(TensorObject* self, PyObject* args, @@ -2344,7 +2344,7 @@ Returns the compressed row index of non zero elements in input SparseCsrTensor. Tensor(shape=[4], dtype=int64, place=Place(gpu:0), stop_gradient=True, [0, 2, 3, 5]) -)DOC"); +)DOC"); // NOLINT static PyObject* tensor_method_get_non_zero_crows(TensorObject* self, PyObject* args, @@ -2388,7 +2388,7 @@ Returns the column index of non zero elements in input SparseCsrTensor. Tensor(shape=[5], dtype=int64, place=Place(gpu:0), stop_gradient=True, [1, 3, 2, 0, 1]) -)DOC"); +)DOC"); // NOLINT static PyObject* tensor_method_get_non_zero_cols(TensorObject* self, PyObject* args, @@ -2422,7 +2422,7 @@ Whether the Tensor is a Dense Tensor. >>> x = paddle.to_tensor([1.0], stop_gradient=False) >>> print(x.is_dense()) True -)DOC"); +)DOC"); // NOLINT static PyObject* tensor_method_is_dense(TensorObject* self, PyObject* args, @@ -2452,7 +2452,7 @@ Whether the Tensor is a Distributed Tensor. >>> x = paddle.to_tensor([1.0], stop_gradient=False) >>> print(x.is_dist()) False -)DOC"); +)DOC"); // NOLINT static PyObject* tensor_method_is_dist(TensorObject* self, PyObject* args, @@ -2489,7 +2489,8 @@ When input is SparseCooTensor/SparseCsrTensor, will return True. When input is D >>> coo.is_sparse() True -)DOC"); +)DOC"); // NOLINT + static PyObject* tensor_method_is_sparse(TensorObject* self, PyObject* args, PyObject* kwargs) { @@ -2526,7 +2527,7 @@ When input is SparseCooTensor, will return True. When input is DenseTensor/Spars >>> coo.is_sparse_coo() True -)DOC"); +)DOC"); // NOLINT static PyObject* tensor_method_is_sparse_coo(TensorObject* self, PyObject* args, @@ -2564,7 +2565,7 @@ When input is SparseCsrTensor, will return True. When input is DenseTensor/Spars >>> csr.is_sparse_csr() True -)DOC"); +)DOC"); // NOLINT static PyObject* tensor_method_is_sparse_csr(TensorObject* self, PyObject* args, @@ -2607,7 +2608,7 @@ When input is SparseCooTensor, will convert `COO` to `CSR` . When input is Dense cols=[1, 2, 0], values=[1., 2., 3.]) -)DOC"); +)DOC"); // NOLINT static PyObject* tensor_method_to_sparse_csr(TensorObject* self, PyObject* args, @@ -2654,7 +2655,7 @@ Any two type Tensor among DenseTensor/SparseCooTensor/SparseCsrTensor are suppor >>> x.is_same_shape(z) False -)DOC"); +)DOC"); // NOLINT static PyObject* tensor_method_is_same_shape(TensorObject* self, PyObject* args, @@ -2957,7 +2958,7 @@ Returns the address of the first element of current Tensor. >>> # doctest: +SKIP('return the address') 93220864 >>> # doctest: -SKIP -)DOC"); +)DOC"); // NOLINT static PyObject* tensor_data_ptr(TensorObject* self, PyObject* args, @@ -3019,7 +3020,7 @@ Returns the strides of current Tensor. >>> y = x[1] >>> print(y.get_strides()) [] -)DOC"); +)DOC"); // NOLINT static PyObject* tensor_method_strides(TensorObject* self, PyObject* args, @@ -3061,7 +3062,7 @@ If self tensor is already contiguous, this function returns the current Tensor. >>> y = y.contiguous() >>> print(y) Tensor(shape=[], dtype=int64, place=Place(cpu), stop_gradient=True, 2) -)DOC"); +)DOC"); // NOLINT static PyObject* tensor_contiguous(TensorObject* self, PyObject* args, @@ -3110,7 +3111,8 @@ Whether the Tensor is contiguous. >>> x = paddle.to_tensor([1, 2, 3]) >>> y = x[1] >>> print(y.is_contiguous()) -)DOC"); +)DOC"); // NOLINT + static PyObject* tensor_is_contiguous(TensorObject* self, PyObject* args, PyObject* kwargs) { diff --git a/paddle/fluid/pybind/eager_properties.cc b/paddle/fluid/pybind/eager_properties.cc index 2a2b94b715abd..fa926618bdf8d 100644 --- a/paddle/fluid/pybind/eager_properties.cc +++ b/paddle/fluid/pybind/eager_properties.cc @@ -40,7 +40,7 @@ namespace pybind { extern PyTypeObject* p_tensor_type; -PyDoc_STRVAR(tensor_name__doc__, +PyDoc_STRVAR(tensor_name__doc__, // NOLINT R"DOC(name Tensor's name. @@ -75,7 +75,7 @@ PyObject* tensor_properties_get_name(TensorObject* self, void* closure) { EAGER_CATCH_AND_THROW_RETURN_NULL } -PyDoc_STRVAR(tensor_type__doc__, +PyDoc_STRVAR(tensor_type__doc__, // NOLINT R"DOC(type Tensor's type. @@ -165,7 +165,7 @@ int tensor_properties_set_name(TensorObject* self, EAGER_CATCH_AND_THROW_RETURN_NEG } -PyDoc_STRVAR(tensor_stop_gradient__doc__, +PyDoc_STRVAR(tensor_stop_gradient__doc__, // NOLINT R"DOC(stop_gradient Tensor's stop_gradient. @@ -195,7 +195,7 @@ PyObject* tensor_properties_get_stop_gradient(TensorObject* self, EAGER_CATCH_AND_THROW_RETURN_NULL } -PyDoc_STRVAR(tensor_data__doc__, +PyDoc_STRVAR(tensor_data__doc__, // NOLINT R"DOC(data Tensor's self. @@ -258,7 +258,7 @@ int tensor_properties_set_data(TensorObject* self, EAGER_CATCH_AND_THROW_RETURN_NEG } -PyDoc_STRVAR(tensor_grad__doc__, +PyDoc_STRVAR(tensor_grad__doc__, // NOLINT R"DOC(grad Tensor's grad Tensor. @@ -356,7 +356,7 @@ int tensor_properties_set_stop_gradient(TensorObject* self, EAGER_CATCH_AND_THROW_RETURN_NEG } -PyDoc_STRVAR(tensor_persistable__doc__, +PyDoc_STRVAR(tensor_persistable__doc__, // NOLINT R"DOC(persistable Tensor's persistable. @@ -395,7 +395,7 @@ int tensor_properties_set_persistable(TensorObject* self, EAGER_CATCH_AND_THROW_RETURN_NEG } -PyDoc_STRVAR(tensor_process_mesh__doc__, +PyDoc_STRVAR(tensor_process_mesh__doc__, // NOLINT R"DOC(process_mesh Get process_mesh property from shard tensor. @@ -441,7 +441,7 @@ PyObject* tensor_properties_get_process_mesh(TensorObject* self, EAGER_CATCH_AND_THROW_RETURN_NULL } -PyDoc_STRVAR(tensor_placements__doc__, +PyDoc_STRVAR(tensor_placements__doc__, // NOLINT R"DOC(placements Get placements property from shard tensor. @@ -487,7 +487,7 @@ PyObject* tensor_properties_get_placements(TensorObject* self, void* closure) { EAGER_CATCH_AND_THROW_RETURN_NULL } -PyDoc_STRVAR(tensor_num_shard__doc__, +PyDoc_STRVAR(tensor_num_shard__doc__, // NOLINT R"DOC(num_shard Tensor's num_shard. @@ -553,7 +553,7 @@ PyObject* tensor_properties_get_local_shape(TensorObject* self, void* closure) { EAGER_CATCH_AND_THROW_RETURN_NULL } -PyDoc_STRVAR(tensor_shape__doc__, +PyDoc_STRVAR(tensor_shape__doc__, // NOLINT R"DOC(shape Tensor's shape. @@ -640,7 +640,7 @@ PyObject* tensor_properties_get_shape(TensorObject* self, void* closure) { EAGER_CATCH_AND_THROW_RETURN_NULL } -PyDoc_STRVAR(tensor_strides__doc__, +PyDoc_STRVAR(tensor_strides__doc__, // NOLINT R"DOC(strides Tensor's strides. @@ -679,7 +679,7 @@ PyObject* tensor_properties_get_strides(TensorObject* self, void* closure) { EAGER_CATCH_AND_THROW_RETURN_NULL } -PyDoc_STRVAR(tensor_offset__doc__, +PyDoc_STRVAR(tensor_offset__doc__, // NOLINT R"DOC(offset The address of the first element relative to the offset of the video memory. @@ -726,7 +726,7 @@ PyObject* tensor_properties_get_offset(TensorObject* self, void* closure) { EAGER_CATCH_AND_THROW_RETURN_NULL } -PyDoc_STRVAR(tensor_layout__doc__, +PyDoc_STRVAR(tensor_layout__doc__, // NOLINT R"DOC(layout Tensor's memory layout. @@ -761,7 +761,7 @@ PyObject* tensor_properties_get_layout(TensorObject* self, void* closure) { EAGER_CATCH_AND_THROW_RETURN_NULL } -PyDoc_STRVAR(tensor_place__doc__, +PyDoc_STRVAR(tensor_place__doc__, // NOLINT R"DOC(place The device Tensor's memory locate. @@ -828,7 +828,7 @@ PyObject* tensor_properties_get_placements_str(TensorObject* self, EAGER_CATCH_AND_THROW_RETURN_NULL } -PyDoc_STRVAR(tensor_dtype__doc__, +PyDoc_STRVAR(tensor_dtype__doc__, // NOLINT R"DOC(dtype Tensor's data type. diff --git a/paddle/fluid/pybind/eval_frame_tools.cc b/paddle/fluid/pybind/eval_frame_tools.cc index da78ce66373e8..504dbc5b9fa01 100644 --- a/paddle/fluid/pybind/eval_frame_tools.cc +++ b/paddle/fluid/pybind/eval_frame_tools.cc @@ -34,7 +34,7 @@ class TreeNode { private: int is_prefix; - TreeNode* children[256]; + TreeNode* children[256]; // NOLINT }; void TreeNode::clear() { diff --git a/paddle/phi/kernels/fusion/cpu/self_dp_attention_kernel.cc b/paddle/phi/kernels/fusion/cpu/self_dp_attention_kernel.cc index 56107c31d6d9c..0d3189187351c 100644 --- a/paddle/phi/kernels/fusion/cpu/self_dp_attention_kernel.cc +++ b/paddle/phi/kernels/fusion/cpu/self_dp_attention_kernel.cc @@ -161,8 +161,8 @@ void sgemm(const float* A, int ldc = n; float alpha = 1; float beta = 0; - char ta[] = "N"; - char tb[] = "N"; + std::array ta = {"N"}; + std::array tb = {"N"}; if (transa) ta[0] = 'T'; if (transb) tb[0] = 'T'; diff --git a/test/cpp/fluid/framework/tensor_util_test.cc b/test/cpp/fluid/framework/tensor_util_test.cc index 6b9c25750ac07..80140dfdbe1c1 100644 --- a/test/cpp/fluid/framework/tensor_util_test.cc +++ b/test/cpp/fluid/framework/tensor_util_test.cc @@ -68,8 +68,8 @@ TEST(TensorCopy, Tensor) { int* src_ptr = src_tensor.mutable_data(common::make_ddim({3, 3}), platform::CPUPlace()); - int arr[9] = {1, 2, 3, 4, 5, 6, 7, 8, 9}; - memcpy(src_ptr, arr, 9 * sizeof(int)); + std::array arr = {1, 2, 3, 4, 5, 6, 7, 8, 9}; + memcpy(src_ptr, arr.data(), 9 * sizeof(int)); // CPU phi::DenseTensor to GPU phi::DenseTensor auto gpu_place = new platform::CUDAPlace(0); diff --git a/test/cpp/fluid/math/im2col_test.cc b/test/cpp/fluid/math/im2col_test.cc index f3925bce95869..36968d7ab68fc 100644 --- a/test/cpp/fluid/math/im2col_test.cc +++ b/test/cpp/fluid/math/im2col_test.cc @@ -207,8 +207,8 @@ void testIm2col() { (input_width - filter_size + padding[2] + padding[3]) / stride[1] + 1; float* input_ptr = input_tmp.mutable_data( {1, input_height, input_width}, paddle::platform::CPUPlace()); - float arr[6] = {0, 1, 2, 3, 4, 5}; - memcpy(input_ptr, arr, 6 * sizeof(float)); + std::array arr = {0, 1, 2, 3, 4, 5}; + memcpy(input_ptr, arr.data(), 6 * sizeof(float)); auto* place = new paddle::platform::CUDAPlace(); auto* context = new phi::GPUContext(*place); @@ -235,8 +235,8 @@ void testIm2col() { im2col(*context, input, dilation, stride, padding, &output_cfo); im2col_ocf(*context, input, dilation, stride, padding, &output_ocf); - float out_cfo_data[] = {0, 1, 1, 2, 3, 4, 4, 5}; - float out_ocf_data[] = {0, 1, 3, 4, 1, 2, 4, 5}; + std::array out_cfo_data = {0, 1, 1, 2, 3, 4, 4, 5}; + std::array out_ocf_data = {0, 1, 3, 4, 1, 2, 4, 5}; float* out_cfo_ptr; if (paddle::platform::is_cpu_place(*place)) { @@ -268,7 +268,7 @@ void testIm2col() { col2im; phi::funcs::Col2ImFunctor col2im_ocf; - float col2im_data[] = {0, 2, 2, 3, 8, 5}; + std::array col2im_data = {0, 2, 2, 3, 8, 5}; memset(input_ptr, 0, 6 * sizeof(float)); if (paddle::platform::is_cpu_place(*place)) { diff --git a/test/cpp/fluid/math/vol2col_test.cc b/test/cpp/fluid/math/vol2col_test.cc index 9a6f14c3685cb..12fd0085ee661 100644 --- a/test/cpp/fluid/math/vol2col_test.cc +++ b/test/cpp/fluid/math/vol2col_test.cc @@ -187,8 +187,8 @@ void testVol2col() { float* input_ptr = input_tmp.mutable_data({1, input_depth, input_height, input_width}, paddle::platform::CPUPlace()); - float arr[12] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; - memcpy(input_ptr, arr, 12 * sizeof(float)); + std::array arr = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + memcpy(input_ptr, arr.data(), 12 * sizeof(float)); if (paddle::platform::is_cpu_place(*place)) { input = input_tmp; @@ -207,7 +207,8 @@ void testVol2col() { phi::funcs::Vol2ColFunctor vol2col; vol2col(*context, input, dilations, strides, paddings, &output); - float vol_2_col[] = {0, 1, 1, 2, 3, 4, 4, 5, 6, 7, 7, 8, 9, 10, 10, 11}; + std::array vol_2_col = { + 0, 1, 1, 2, 3, 4, 4, 5, 6, 7, 7, 8, 9, 10, 10, 11}; float* out_cfo_ptr; if (paddle::platform::is_cpu_place(*place)) { out_cfo_ptr = output.data(); @@ -222,7 +223,7 @@ void testVol2col() { } // Col2Vol test - float col_2_vol[] = {0, 2, 2, 3, 8, 5, 6, 14, 8, 9, 20, 11}; + std::array col_2_vol = {0, 2, 2, 3, 8, 5, 6, 14, 8, 9, 20, 11}; memset(input_ptr, 0, 12 * sizeof(float)); if (paddle::platform::is_cpu_place(*place)) { input = input_tmp; diff --git a/test/cpp/inference/api/analysis_predictor_tester.cc b/test/cpp/inference/api/analysis_predictor_tester.cc index 3d87140d9c05a..138063c98adfb 100644 --- a/test/cpp/inference/api/analysis_predictor_tester.cc +++ b/test/cpp/inference/api/analysis_predictor_tester.cc @@ -56,10 +56,10 @@ TEST(AnalysisPredictor, analysis_off) { LOG(INFO) << "scope parameters " << predictor->scope_->LocalVarNames().size(); // 2. Dummy Input Data - int64_t data[4] = {1, 2, 3, 4}; + std::array input_data = {1, 2, 3, 4}; PaddleTensor tensor; tensor.shape = std::vector({4, 1}); - tensor.data.Reset(data, sizeof(data)); + tensor.data.Reset(input_data.data(), sizeof(input_data)); tensor.dtype = PaddleDType::INT64; std::vector inputs(4, tensor); @@ -109,10 +109,10 @@ TEST(AnalysisPredictor, analysis_on) { ASSERT_EQ(predictor->GetOutputTypes().size(), 1UL); ASSERT_EQ(predictor->GetOutputTensorShape().size(), 1UL); // 2. Dummy Input Data - int64_t data[4] = {1, 2, 3, 4}; + std::array input_data = {1, 2, 3, 4}; PaddleTensor tensor; tensor.shape = std::vector({4, 1}); - tensor.data.Reset(data, sizeof(data)); + tensor.data.Reset(input_data.data(), sizeof(input_data)); tensor.dtype = PaddleDType::INT64; std::vector inputs(4, tensor); @@ -242,10 +242,10 @@ TEST(AnalysisPredictor, Clone) { << framework::GenScopeTreeDebugInfo(root_scope); // 2. Dummy Input Data - int64_t data[4] = {1, 2, 3, 4}; + std::array input_data = {1, 2, 3, 4}; PaddleTensor tensor; tensor.shape = std::vector({4, 1}); - tensor.data.Reset(data, sizeof(data)); + tensor.data.Reset(input_data.data(), sizeof(input_data)); tensor.dtype = PaddleDType::INT64; std::vector inputs(4, tensor); diff --git a/test/cpp/inference/api/analyzer_capi_exp_gpu_tester.cc b/test/cpp/inference/api/analyzer_capi_exp_gpu_tester.cc index 3ff0d86f59916..61d5966d6d92d 100644 --- a/test/cpp/inference/api/analyzer_capi_exp_gpu_tester.cc +++ b/test/cpp/inference/api/analyzer_capi_exp_gpu_tester.cc @@ -64,17 +64,17 @@ TEST(PD_Config, gpu_interface) { EXPECT_TRUE(trt_enable); const char* tensor_name = "image"; - size_t shapes_num[1] = {4}; - int32_t min_shape[4] = {1, 3, 36, 36}; - int32_t max_shape[4] = {1, 3, 224, 224}; - int32_t opt_shape[4] = {1, 3, 224, 224}; - int32_t* min_shape_ptr = min_shape; - int32_t* max_shape_ptr = max_shape; - int32_t* opt_shape_ptr = opt_shape; + std::array shapes_num = {4}; + std::array min_shape = {1, 3, 36, 36}; + std::array max_shape = {1, 3, 224, 224}; + std::array opt_shape = {1, 3, 224, 224}; + int32_t* min_shape_ptr = min_shape.data(); + int32_t* max_shape_ptr = max_shape.data(); + int32_t* opt_shape_ptr = opt_shape.data(); PD_ConfigSetTrtDynamicShapeInfo(config, 1, &tensor_name, - shapes_num, + shapes_num.data(), &min_shape_ptr, &max_shape_ptr, &opt_shape_ptr, diff --git a/test/cpp/inference/api/analyzer_capi_exp_int_tester.cc b/test/cpp/inference/api/analyzer_capi_exp_int_tester.cc index 65d740b229d47..cb3a4db6702c5 100644 --- a/test/cpp/inference/api/analyzer_capi_exp_int_tester.cc +++ b/test/cpp/inference/api/analyzer_capi_exp_int_tester.cc @@ -45,16 +45,16 @@ void predictor_run() { EXPECT_EQ(in_infos->size, 2u); PD_IOInfos* out_infos = PD_PredictorGetOutputInfos(predictor); - int32_t shape_0[4] = {1, 3, 224, 224}; - float data_0[1 * 3 * 224 * 224] = {0}; + std::array shape_0 = {1, 3, 224, 224}; + std::array data_0 = {0}; PD_Tensor* input_0 = PD_PredictorGetInputHandle(predictor, "image"); - PD_TensorReshape(input_0, 4, shape_0); - PD_TensorCopyFromCpuFloat(input_0, data_0); - int32_t shape_1[2] = {1, 1}; - int64_t data_1[1] = {0}; + PD_TensorReshape(input_0, 4, shape_0.data()); + PD_TensorCopyFromCpuFloat(input_0, data_0.data()); + std::array shape_1 = {1, 1}; + std::array data_1 = {0}; PD_Tensor* input_1 = PD_PredictorGetInputHandle(predictor, "label"); - PD_TensorReshape(input_1, 2, shape_1); - PD_TensorCopyFromCpuInt64(input_1, data_1); + PD_TensorReshape(input_1, 2, shape_1.data()); + PD_TensorCopyFromCpuInt64(input_1, data_1.data()); LOG(INFO) << "Run Inference in CAPI encapsulation. "; EXPECT_TRUE(PD_PredictorRun(predictor)); diff --git a/test/cpp/inference/api/analyzer_capi_exp_ner_tester.cc b/test/cpp/inference/api/analyzer_capi_exp_ner_tester.cc index 98abb7926ccd9..e83ed41fc85bf 100644 --- a/test/cpp/inference/api/analyzer_capi_exp_ner_tester.cc +++ b/test/cpp/inference/api/analyzer_capi_exp_ner_tester.cc @@ -47,28 +47,29 @@ TEST(PD_PredictorRun, predictor_run) { PD_OneDimArrayCstr *input_names = PD_PredictorGetInputNames(predictor); EXPECT_EQ(input_names->size, 2u); LOG(INFO) << "Predictor start run!"; - PD_Tensor *inputs[2]; + PD_Tensor *inputs[2]; // NOLINT inputs[0] = PD_PredictorGetInputHandle(predictor, input_names->data[0]); inputs[1] = PD_PredictorGetInputHandle(predictor, input_names->data[1]); LOG(INFO) << "Predictor start run!"; // inputs[0]: word, use lod memory in stack - int32_t shape_0[2] = {11, 1}; - int64_t data_0[11 * 1] = {12673, 9763, 905, 284, 45, 7474, 20, 17, 1, 4, 9}; - size_t lod_layer_0[2] = {0, 11}; + std::array shape_0 = {11, 1}; + std::array data_0 = { + 12673, 9763, 905, 284, 45, 7474, 20, 17, 1, 4, 9}; + std::array lod_layer_0 = {0, 11}; PD_OneDimArraySize layer_0; layer_0.size = 2; - layer_0.data = lod_layer_0; + layer_0.data = lod_layer_0.data(); PD_OneDimArraySize *layer_0_ptr = &layer_0; PD_TwoDimArraySize lod_0; lod_0.size = 1; lod_0.data = &layer_0_ptr; - PD_TensorReshape(inputs[0], 2, shape_0); - PD_TensorCopyFromCpuInt64(inputs[0], data_0); + PD_TensorReshape(inputs[0], 2, shape_0.data()); + PD_TensorCopyFromCpuInt64(inputs[0], data_0.data()); PD_TensorSetLod(inputs[0], &lod_0); // inputs[1]: mention, use lod memory in heap - int32_t shape_1[2] = {11, 1}; - int64_t data_1[11 * 1] = {27, 0, 0, 33, 34, 33, 0, 0, 0, 1, 2}; + std::array shape_1 = {11, 1}; + std::array data_1 = {27, 0, 0, 33, 34, 33, 0, 0, 0, 1, 2}; PD_TwoDimArraySize *lod_1_ptr = new PD_TwoDimArraySize(); lod_1_ptr->size = 1; lod_1_ptr->data = new PD_OneDimArraySize *[1]; @@ -78,8 +79,8 @@ TEST(PD_PredictorRun, predictor_run) { lod_1_ptr->data[0]->data[0] = 0; lod_1_ptr->data[0]->data[1] = 11; - PD_TensorReshape(inputs[1], 2, shape_1); - PD_TensorCopyFromCpuInt64(inputs[1], data_1); + PD_TensorReshape(inputs[1], 2, shape_1.data()); + PD_TensorCopyFromCpuInt64(inputs[1], data_1.data()); PD_TensorSetLod(inputs[1], lod_1_ptr); // retrieve the lod memory delete[] lod_1_ptr->data[0]->data; diff --git a/test/cpp/inference/api/analyzer_capi_exp_pd_tensor_tester.cc b/test/cpp/inference/api/analyzer_capi_exp_pd_tensor_tester.cc index 7a32aefb16d30..40a88d7506dbc 100644 --- a/test/cpp/inference/api/analyzer_capi_exp_pd_tensor_tester.cc +++ b/test/cpp/inference/api/analyzer_capi_exp_pd_tensor_tester.cc @@ -45,11 +45,11 @@ void PD_run() { PD_Tensor* tensor = PD_PredictorGetInputHandle(predictor, input_names->data[0]); - int32_t shapes[4] = {1, 3, 300, 300}; + std::array shapes = {1, 3, 300, 300}; std::vector input(1 * 3 * 300 * 300, 0); int32_t size; PD_PlaceType place; - PD_TensorReshape(tensor, 4, shapes); + PD_TensorReshape(tensor, 4, shapes.data()); PD_TensorCopyFromCpuFloat(tensor, input.data()); PD_TensorDataFloat(tensor, &place, &size); PD_TensorMutableDataFloat(tensor, place); @@ -98,11 +98,11 @@ TEST(PD_Tensor, int32) { PD_OneDimArrayCstr* input_names = PD_PredictorGetInputNames(predictor); PD_Tensor* tensor = PD_PredictorGetInputHandle(predictor, input_names->data[0]); - int32_t shapes[4] = {1, 3, 300, 300}; + std::array shapes = {1, 3, 300, 300}; std::vector input(1 * 3 * 300 * 300, 0); int32_t size; PD_PlaceType place; - PD_TensorReshape(tensor, 4, shapes); + PD_TensorReshape(tensor, 4, shapes.data()); PD_TensorCopyFromCpuInt32(tensor, input.data()); int32_t* data_ptr = PD_TensorDataInt32(tensor, &place, &size); EXPECT_EQ(place, PD_PLACE_CPU); @@ -129,11 +129,11 @@ TEST(PD_Tensor, int64) { PD_OneDimArrayCstr* input_names = PD_PredictorGetInputNames(predictor); PD_Tensor* tensor = PD_PredictorGetInputHandle(predictor, input_names->data[0]); - int32_t shapes[4] = {1, 3, 300, 300}; + std::array shapes = {1, 3, 300, 300}; std::vector input(1 * 3 * 300 * 300, 0); int32_t size; PD_PlaceType place; - PD_TensorReshape(tensor, 4, shapes); + PD_TensorReshape(tensor, 4, shapes.data()); PD_TensorCopyFromCpuInt64(tensor, input.data()); int64_t* data_ptr = PD_TensorDataInt64(tensor, &place, &size); EXPECT_EQ(place, PD_PLACE_CPU); @@ -160,12 +160,12 @@ TEST(PD_Tensor, uint8) { PD_OneDimArrayCstr* input_names = PD_PredictorGetInputNames(predictor); PD_Tensor* tensor = PD_PredictorGetInputHandle(predictor, input_names->data[0]); - int32_t shapes[4] = {1, 3, 300, 300}; - uint8_t input[1 * 3 * 300 * 300] = {0}; + std::array shapes = {1, 3, 300, 300}; + std::array input = {0}; int32_t size; PD_PlaceType place; - PD_TensorReshape(tensor, 4, shapes); - PD_TensorCopyFromCpuUint8(tensor, input); + PD_TensorReshape(tensor, 4, shapes.data()); + PD_TensorCopyFromCpuUint8(tensor, input.data()); uint8_t* data_ptr = PD_TensorDataUint8(tensor, &place, &size); EXPECT_EQ(place, PD_PLACE_CPU); EXPECT_EQ(size, 1 * 3 * 300 * 300); @@ -174,7 +174,7 @@ TEST(PD_Tensor, uint8) { PD_DataType data_type = PD_TensorGetDataType(tensor); EXPECT_EQ(data_type, PD_DATA_UINT8); - PD_TensorCopyToCpuUint8(tensor, input); + PD_TensorCopyToCpuUint8(tensor, input.data()); PD_TensorDestroy(tensor); PD_OneDimArrayCstrDestroy(input_names); diff --git a/test/cpp/inference/api/analyzer_capi_exp_pd_threads_tester.cc b/test/cpp/inference/api/analyzer_capi_exp_pd_threads_tester.cc index 7cd5ac7e7d482..b06c637c86e47 100644 --- a/test/cpp/inference/api/analyzer_capi_exp_pd_threads_tester.cc +++ b/test/cpp/inference/api/analyzer_capi_exp_pd_threads_tester.cc @@ -84,13 +84,13 @@ void threads_run(int thread_num) { reinterpret_cast(malloc(thread_num * sizeof(pthread_t))); RunParameter* params = reinterpret_cast( malloc(thread_num * sizeof(RunParameter))); - int32_t shapes[4] = {1, 3, 300, 300}; + std::array shapes = {1, 3, 300, 300}; float* input = reinterpret_cast(malloc(1 * 3 * 300 * 300 * sizeof(float))); memset(input, 0, 1 * 3 * 300 * 300 * sizeof(float)); for (int i = 0; i < thread_num; ++i) { params[i].predictor = PD_PredictorClone(predictor); - params[i].shapes = shapes; + params[i].shapes = shapes.data(); params[i].shape_size = 4; params[i].input_data = input; params[i].out_size = 0; diff --git a/test/cpp/inference/api/analyzer_capi_exp_tester.cc b/test/cpp/inference/api/analyzer_capi_exp_tester.cc index 3d5fbd5a0451f..17610f7834039 100644 --- a/test/cpp/inference/api/analyzer_capi_exp_tester.cc +++ b/test/cpp/inference/api/analyzer_capi_exp_tester.cc @@ -53,8 +53,8 @@ void predictor_run() { const int width = 318; float *input = new float[batch_size * channels * height * width](); - int32_t shape[4] = {batch_size, channels, height, width}; - PD_TensorReshape(tensor, 4, shape); + std::array shape = {batch_size, channels, height, width}; + PD_TensorReshape(tensor, 4, shape.data()); PD_TensorCopyFromCpuFloat(tensor, input); EXPECT_TRUE(PD_PredictorRun(predictor)); diff --git a/test/cpp/inference/api/analyzer_dam_tester.cc b/test/cpp/inference/api/analyzer_dam_tester.cc index ea31fe3760b53..3770aac10e371 100644 --- a/test/cpp/inference/api/analyzer_dam_tester.cc +++ b/test/cpp/inference/api/analyzer_dam_tester.cc @@ -120,8 +120,8 @@ struct DataRecord { void PrepareInputs(std::vector *input_slots, DataRecord *data, int batch_size) { - PaddleTensor turns_tensor[FLAGS_max_turn_num]; - PaddleTensor turns_mask_tensor[FLAGS_max_turn_num]; + PaddleTensor turns_tensor[FLAGS_max_turn_num]; // NOLINT + PaddleTensor turns_mask_tensor[FLAGS_max_turn_num]; // NOLINT PaddleTensor response_tensor; PaddleTensor response_mask_tensor; std::string turn_pre = "turn_"; diff --git a/test/cpp/inference/api/analyzer_lac_tester.cc b/test/cpp/inference/api/analyzer_lac_tester.cc index 9bdb819e5fbd6..ef057227c226c 100644 --- a/test/cpp/inference/api/analyzer_lac_tester.cc +++ b/test/cpp/inference/api/analyzer_lac_tester.cc @@ -139,7 +139,7 @@ TEST(Analyzer_LAC, profile) { if (FLAGS_num_threads == 1 && !FLAGS_test_all_data) { // the first inference result - const int64_t lac_ref_data[] = { + const std::array lac_ref_data = { 24, 25, 25, 25, 38, 30, 31, 14, 15, 44, 24, 25, 25, 25, 25, 25, 44, 24, 25, 25, 25, 36, 42, 43, 44, 14, 15, 44, 14, 15, 44, 14, 15, 44, 38, 39, 14, 15, 44, 22, 23, 23, 23, 23, 23, 23, 23}; diff --git a/test/cpp/inference/api/analyzer_ner_tester.cc b/test/cpp/inference/api/analyzer_ner_tester.cc index 8027603b7eb15..a1bd037640412 100644 --- a/test/cpp/inference/api/analyzer_ner_tester.cc +++ b/test/cpp/inference/api/analyzer_ner_tester.cc @@ -120,7 +120,7 @@ void profile(bool memory_load = false) { if (FLAGS_num_threads == 1 && !FLAGS_test_all_data) { // the first inference result - const int chinese_ner_result_data[] = { + const std::array chinese_ner_result_data = { 30, 45, 41, 48, 17, 26, 48, 39, 38, 16, 25}; PADDLE_ENFORCE_GT(outputs.size(), 0, diff --git a/test/cpp/inference/api/analyzer_rnn1_tester.cc b/test/cpp/inference/api/analyzer_rnn1_tester.cc index 14a5aa40a4512..72c53ccbdd815 100644 --- a/test/cpp/inference/api/analyzer_rnn1_tester.cc +++ b/test/cpp/inference/api/analyzer_rnn1_tester.cc @@ -191,11 +191,13 @@ void PrepareZeroCopyInputs(ZeroCopyTensor *lod_attention_tensor, minute_tensor->SetLoD({one_batch.lod3}); // assign data - float arr0[] = {0, 0}; + std::array arr0 = {0, 0}; std::vector zeros(batch_size * 15, 0); + std::copy_n(arr0.data(), + 2, + lod_attention_tensor->mutable_data(PaddlePlace::kCPU)); std::copy_n( - arr0, 2, lod_attention_tensor->mutable_data(PaddlePlace::kCPU)); - std::copy_n(arr0, 2, data_tensor->mutable_data(PaddlePlace::kCPU)); + arr0.data(), 2, data_tensor->mutable_data(PaddlePlace::kCPU)); std::copy_n(zeros.begin(), zeros.size(), cell_init_tensor->mutable_data(PaddlePlace::kCPU)); diff --git a/test/cpp/inference/api/trt_dynamic_shape_ernie_test.cc b/test/cpp/inference/api/trt_dynamic_shape_ernie_test.cc index b28a8eab95d4b..d26946c76856e 100644 --- a/test/cpp/inference/api/trt_dynamic_shape_ernie_test.cc +++ b/test/cpp/inference/api/trt_dynamic_shape_ernie_test.cc @@ -33,22 +33,22 @@ void run(const AnalysisConfig& config, std::vector* out_data, int bs) { const int run_seq_len = 128; size_t len = run_batch * run_seq_len; - int32_t i0_bs1[run_seq_len] = { + std::array i0_bs1 = { 1, 3558, 4, 75, 491, 89, 340, 313, 93, 4, 255, 10, 75, 321, 4095, 1902, 4, 134, 49, 75, 311, 14, 44, 178, 543, 15, 12043, 2, 75, 201, 340, 9, 14, 44, 486, 218, 1140, 279, 12043, 2}; - int32_t i1_bs1[run_seq_len] = { + std::array i1_bs1 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - int32_t i2_bs1[run_seq_len] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, - 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, - 30, 31, 32, 33, 34, 35, 36, 37, 38, 39}; - float i3_bs1[run_seq_len] = { + std::array i2_bs1 = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39}; + std::array i3_bs1 = { 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0}; diff --git a/test/cpp/inference/api/trt_dynamic_shape_transformer_prune_test.cc b/test/cpp/inference/api/trt_dynamic_shape_transformer_prune_test.cc index 1f6fa900268d6..515330ec11085 100644 --- a/test/cpp/inference/api/trt_dynamic_shape_transformer_prune_test.cc +++ b/test/cpp/inference/api/trt_dynamic_shape_transformer_prune_test.cc @@ -33,44 +33,44 @@ void run(const AnalysisConfig& config, std::vector* out_data) { tmp_input.reserve(run_batch * run_seq_len); tmp_four_input.reserve(run_batch * run_seq_len); - int64_t i0[run_seq_len] = { + std::array i0 = { 1, 3558, 4, 75, 491, 89, 340, 313, 93, 4, 255, 10, 75, 321, 4095, 1902, 4, 134, 49, 75, 311, 14, 44, 178, 543, 15, 12043, 2, 75, 201, 340, 9, 14, 44, 486, 218, 1140, 279, 12043, 2}; - int64_t i1[run_seq_len] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, - 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, - 30, 31, 32, 33, 34, 35, 36, 37, 38, 39}; - int64_t i2[run_seq_len] = { + std::array i1 = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39}; + std::array i2 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - float i3[run_seq_len] = {1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, - 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, - 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, - 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0}; + std::array i3 = { + 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, + 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, + 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0}; // first input auto input_t = predictor->GetInputTensor(input_names[0]); input_t->Reshape({run_batch, run_seq_len, 1}); - input_t->copy_from_cpu(i0); + input_t->copy_from_cpu(i0.data()); // second input auto input_t2 = predictor->GetInputTensor(input_names[1]); input_t2->Reshape({run_batch, run_seq_len, 1}); - input_t2->copy_from_cpu(i1); + input_t2->copy_from_cpu(i1.data()); // third input. auto input_t3 = predictor->GetInputTensor(input_names[2]); input_t3->Reshape({run_batch, run_seq_len, 1}); - input_t3->copy_from_cpu(i2); + input_t3->copy_from_cpu(i2.data()); auto input_t4 = predictor->GetInputTensor(input_names[3]); input_t4->Reshape({run_batch, run_seq_len, 1}); - input_t4->copy_from_cpu(i3); + input_t4->copy_from_cpu(i3.data()); ASSERT_TRUE(predictor->ZeroCopyRun()); diff --git a/test/cpp/inference/api/trt_rebind_stream_test.cc b/test/cpp/inference/api/trt_rebind_stream_test.cc index 1f6d5bd8adc68..361335a46be16 100644 --- a/test/cpp/inference/api/trt_rebind_stream_test.cc +++ b/test/cpp/inference/api/trt_rebind_stream_test.cc @@ -41,8 +41,8 @@ TEST(ReBindStream_single, use_gpu) { auto predictor = paddle_infer::CreatePredictor(config); auto x_t = predictor->GetInputHandle("x"); x_t->Reshape({1, 3, 224, 224}); - float x_data[3 * 224 * 224] = {0}; - x_t->CopyFromCpu(x_data); + std::array x_data = {0}; + x_t->CopyFromCpu(x_data.data()); ASSERT_TRUE(predictor->Run()); cudaDeviceSynchronize(); ASSERT_TRUE(paddle_infer::experimental::InternalUtils::RunWithExternalStream( diff --git a/test/cpp/new_executor/standalone_executor_test.cc b/test/cpp/new_executor/standalone_executor_test.cc index 5a2cb41831f7d..67f7aec8c8dfe 100644 --- a/test/cpp/new_executor/standalone_executor_test.cc +++ b/test/cpp/new_executor/standalone_executor_test.cc @@ -284,8 +284,8 @@ TEST(InterpreterCore, workqueue_multiplexing) { add->SetInput("Y", {"b"}); add->SetOutput("Out", {"c"}); - float data_a[] = {0, 1, 2, 3}; - float data_b[] = {0.0, 0.1, 0.2, 0.3}; + std::array data_a = {0, 1, 2, 3}; + std::array data_b = {0.0, 0.1, 0.2, 0.3}; phi::DDim dims = common::make_ddim({2, 2}); const platform::CPUPlace place = platform::CPUPlace(); @@ -293,8 +293,8 @@ TEST(InterpreterCore, workqueue_multiplexing) { phi::DenseTensor tensor_a = phi::DenseTensor(); phi::DenseTensor tensor_b = phi::DenseTensor(); - std::copy_n(data_a, 4, tensor_a.mutable_data(dims, place)); - std::copy_n(data_b, 4, tensor_b.mutable_data(dims, place)); + std::copy_n(data_a.data(), 4, tensor_a.mutable_data(dims, place)); + std::copy_n(data_b.data(), 4, tensor_b.mutable_data(dims, place)); TestShareWorkQueue( program, {"a", "b"}, {tensor_a, tensor_b}, {"c"}, {0.0, 1.1, 2.2, 3.3}); diff --git a/test/cpp/phi/api/test_from_blob.cc b/test/cpp/phi/api/test_from_blob.cc index c51a184e7eb6f..f936a2445ebfc 100644 --- a/test/cpp/phi/api/test_from_blob.cc +++ b/test/cpp/phi/api/test_from_blob.cc @@ -84,8 +84,8 @@ using phi::memory_utils::Copy; TEST(GetPlaceFromPtr, GPU) { using paddle::GetPlaceFromPtr; - float cpu_data[6]; - auto cpu_data_place = GetPlaceFromPtr(cpu_data); + std::array cpu_data; + auto cpu_data_place = GetPlaceFromPtr(cpu_data.data()); ASSERT_EQ(cpu_data_place, phi::CPUPlace()); std::cout << "cpu_data_place: " << cpu_data_place << std::endl; @@ -109,7 +109,7 @@ TEST(GetPlaceFromPtr, GPU) { TEST(from_blob, GPU) { // 1. create data - float cpu_data[6] = {0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f}; + std::array cpu_data = {0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f}; phi::GPUPlace gpu0(0); phi::Allocator* allocator = paddle::GetAllocator(gpu0); auto gpu_allocation = allocator->Allocate(sizeof(cpu_data)); @@ -119,7 +119,7 @@ TEST(from_blob, GPU) { Copy(gpu0, gpu_data, phi::CPUPlace(), - cpu_data, + cpu_data.data(), sizeof(cpu_data), ctx->stream()); @@ -137,9 +137,9 @@ TEST(from_blob, GPU) { // 3.2 check tensor values auto* gpu_tensor_data = gpu_tensor.template data(); - float gpu_tensor_data_cpu[6]; + std::array gpu_tensor_data_cpu; Copy(phi::CPUPlace(), - gpu_tensor_data_cpu, + gpu_tensor_data_cpu.data(), gpu0, gpu_tensor_data, sizeof(cpu_data), @@ -155,9 +155,9 @@ TEST(from_blob, GPU) { // 3.4 test other API auto gpu_tensor_pow = paddle::experimental::pow(gpu_tensor, 2); auto* gpu_tensor_pow_data = gpu_tensor_pow.template data(); - float gpu_tensor_pow_data_cpu[6]; + std::array gpu_tensor_pow_data_cpu; Copy(phi::CPUPlace(), - gpu_tensor_pow_data_cpu, + gpu_tensor_pow_data_cpu.data(), gpu0, gpu_tensor_pow_data, sizeof(cpu_data), diff --git a/test/cpp/phi/core/test_custom_kernel.cc b/test/cpp/phi/core/test_custom_kernel.cc index b4a9e9da61913..d32d6eb2ff4f1 100644 --- a/test/cpp/phi/core/test_custom_kernel.cc +++ b/test/cpp/phi/core/test_custom_kernel.cc @@ -214,7 +214,7 @@ TEST(CustomKernel, custom_kernel_dot) { auto* dense_y_data = dev_ctx->template Alloc(dense_y.get()); // dot x,y and result - uint8_t sum[2] = {0, 0}; + std::array sum = {0, 0}; for (size_t i = 0; i < 2; ++i) { for (size_t j = 0; j < 3; ++j) { dense_x_data[i * 3 + j] = (i * 3 + j); diff --git a/test/cpp/phi/kernels/strided_memcpy_test.cc b/test/cpp/phi/kernels/strided_memcpy_test.cc index 9bd893bcd10ab..6fb0014956c46 100644 --- a/test/cpp/phi/kernels/strided_memcpy_test.cc +++ b/test/cpp/phi/kernels/strided_memcpy_test.cc @@ -79,7 +79,7 @@ TEST(StridedMemcpy, CPUConcat) { #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) TEST(StridedMemcpy, GPUCrop) { // clang-format off - int src[] = { + std::array src = { 0, 1, 2, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, @@ -95,11 +95,12 @@ TEST(StridedMemcpy, GPUCrop) { auto src_allocation = phi::memory_utils::Alloc(gpu0, sizeof(src)); int* gpu_src = reinterpret_cast(src_allocation->ptr()); - memory_utils::Copy(gpu0, gpu_src, cpu, src, sizeof(src), ctx->stream()); + memory_utils::Copy( + gpu0, gpu_src, cpu, src.data(), sizeof(src), ctx->stream()); phi::DDim src_stride({5, 1}); - int dst[4]; + std::array dst; auto dst_allocation = phi::memory_utils::Alloc(gpu0, sizeof(dst)); int* gpu_dst = reinterpret_cast(dst_allocation->ptr()); @@ -109,7 +110,8 @@ TEST(StridedMemcpy, GPUCrop) { phi::funcs::StridedMemcpy( *ctx, gpu_src + 1, src_stride, dst_dim, dst_stride, gpu_dst); - memory_utils::Copy(cpu, dst, gpu0, gpu_dst, sizeof(dst), ctx->stream()); + memory_utils::Copy( + cpu, dst.data(), gpu0, gpu_dst, sizeof(dst), ctx->stream()); ctx->Wait(); ASSERT_EQ(1, dst[0]); @@ -120,7 +122,7 @@ TEST(StridedMemcpy, GPUCrop) { TEST(StridedMemcpy, GPUConcat) { // clang-format off - int src[] = { + std::array src = { 1, 2, 3, 4 }; @@ -134,9 +136,10 @@ TEST(StridedMemcpy, GPUConcat) { auto gpu_src_allocation = phi::memory_utils::Alloc(gpu0, sizeof(src)); int* gpu_src = reinterpret_cast(gpu_src_allocation->ptr()); - memory_utils::Copy(gpu0, gpu_src, cpu, src, sizeof(src), ctx->stream()); + memory_utils::Copy( + gpu0, gpu_src, cpu, src.data(), sizeof(src), ctx->stream()); - int dst[8]; + std::array dst; auto gpu_dst_allocation = phi::memory_utils::Alloc(gpu0, sizeof(dst)); int* gpu_dst = reinterpret_cast(gpu_dst_allocation->ptr()); @@ -149,11 +152,12 @@ TEST(StridedMemcpy, GPUConcat) { phi::funcs::StridedMemcpy( *ctx, gpu_src, src_stride, dst_dim, dst_stride, gpu_dst + 2); - memory_utils::Copy(cpu, dst, gpu0, gpu_dst, sizeof(dst), ctx->stream()); + memory_utils::Copy( + cpu, dst.data(), gpu0, gpu_dst, sizeof(dst), ctx->stream()); ctx->Wait(); // clang-format off - int expect_dst[] = { + std::array expect_dst = { 1, 2, 1, 2, 3, 4, 3, 4 }; diff --git a/test/cpp/pir/tools/test_op.cc b/test/cpp/pir/tools/test_op.cc index de7eaa1fb9972..cbcd78a64c27e 100644 --- a/test/cpp/pir/tools/test_op.cc +++ b/test/cpp/pir/tools/test_op.cc @@ -35,7 +35,8 @@ void BranchOp::VerifySig() const { IR_ENFORCE((*this)->successor(0), "successor[0] can't be nullptr"); } -const char *Operation1::attributes_name[2] = {"op1_attr1", "op1_attr2"}; +const char *Operation1::attributes_name[2] = {"op1_attr1", + "op1_attr2"}; // NOLINT void Operation1::Build(pir::Builder &builder, // NOLINT pir::OperationArgument &argument) { // NOLINT From 4d0be7f12b2c6d6ee629c2bc5d9dd587ae5f8f6e Mon Sep 17 00:00:00 2001 From: cyberslack_lee Date: Fri, 1 Mar 2024 10:57:48 +0800 Subject: [PATCH 213/282] [clang-tidy] NO.24 enable hicpp-exception-baseclass (#61691) --- test/cpp/inference/api/analyzer_bert_tester.cc | 10 +++++++--- test/cpp/pir/core/ir_program_test.cc | 9 ++++++--- test/cpp/pir/pass/pass_manager_test.cc | 11 +++++++---- test/cpp/pir/pattern_rewrite/pattern_rewrite_test.cc | 11 ++++++----- test/cpp/pir/tools/test_op.cc | 10 +++++++--- 5 files changed, 33 insertions(+), 18 deletions(-) diff --git a/test/cpp/inference/api/analyzer_bert_tester.cc b/test/cpp/inference/api/analyzer_bert_tester.cc index 0ad6e6cc90298..9f60c72cb0bdf 100644 --- a/test/cpp/inference/api/analyzer_bert_tester.cc +++ b/test/cpp/inference/api/analyzer_bert_tester.cc @@ -13,6 +13,8 @@ // limitations under the License. #include "paddle/fluid/framework/transfer_scope_cache.h" +#include "paddle/fluid/platform/errors.h" +#include "paddle/phi/core/enforce.h" #include "test/cpp/inference/api/tester_helper.h" namespace paddle { @@ -159,7 +161,7 @@ void profile(bool use_mkldnn, bool use_bfloat16) { std::vector> LoadInputData() { if (FLAGS_infer_data.empty()) { LOG(ERROR) << "please set input data path"; - throw "missing input data path"; + PADDLE_THROW(platform::errors::NotFound("Missing input data path")); } std::ifstream fin(FLAGS_infer_data); @@ -190,7 +192,8 @@ std::vector ParseInputStreamToVector( const std::string &line) { const auto fields = Split(line, ';'); - if (fields.size() < 5) throw "invalid input line"; + if (fields.size() < 5) + PADDLE_THROW(platform::errors::Fatal("Invalid input line")); std::vector tensors; @@ -228,7 +231,8 @@ AnalysisConfig SetConfig(bool use_mkldnn, bool use_bfloat16) { template paddle::PaddleTensor ParseTensor(const std::string &field) { const auto data = Split(field, ':'); - if (data.size() < 2) throw "invalid data field"; + if (data.size() < 2) + PADDLE_THROW(platform::errors::Fatal("Invalid data field")); std::string shape_str = data[0]; const auto shape = Split(shape_str, ' '); diff --git a/test/cpp/pir/core/ir_program_test.cc b/test/cpp/pir/core/ir_program_test.cc index 0dce6f95c08c7..2957782145a28 100644 --- a/test/cpp/pir/core/ir_program_test.cc +++ b/test/cpp/pir/core/ir_program_test.cc @@ -34,8 +34,9 @@ // paddle/fluid/pir/dialect/CMakeLists.txt. #include "paddle/fluid/pir/dialect/operator/ir/pd_op.h" #include "paddle/fluid/pir/dialect/operator/transforms/param_to_variable.h" +#include "paddle/fluid/platform/errors.h" +#include "paddle/phi/core/enforce.h" #include "test/cpp/pir/tools/macros_utils.h" - class AddOp : public pir::Op { public: using Op::Op; @@ -51,10 +52,12 @@ class AddOp : public pir::Op { }; void AddOp::VerifySig() { if (num_operands() != 2) { - throw("The size of inputs must be equal to 2."); + PADDLE_THROW(paddle::platform::errors::Fatal( + "The size of inputs must be equal to 2.")); } if (num_results() != 1) { - throw("The size of outputs must be equal to 1."); + PADDLE_THROW(paddle::platform::errors::Fatal( + "The size of outputs must be equal to 1.")); } } void AddOp::Build(pir::Builder &, diff --git a/test/cpp/pir/pass/pass_manager_test.cc b/test/cpp/pir/pass/pass_manager_test.cc index f4f4a25bd40b6..2a1c9a4ae4fdd 100644 --- a/test/cpp/pir/pass/pass_manager_test.cc +++ b/test/cpp/pir/pass/pass_manager_test.cc @@ -17,12 +17,13 @@ // NOTE(zhangbo9674): File pd_op.h is generated by op_gen.py, see details in // paddle/fluid/pir/dialect/CMakeLists.txt. -#include "paddle/fluid/pir/dialect/operator/ir/pd_op.h" - #include "paddle/fluid/pir/dialect/operator/interface/op_yaml_info.h" #include "paddle/fluid/pir/dialect/operator/ir/op_dialect.h" #include "paddle/fluid/pir/dialect/operator/ir/op_type.h" +#include "paddle/fluid/pir/dialect/operator/ir/pd_op.h" #include "paddle/fluid/pir/dialect/operator/utils/utils.h" +#include "paddle/fluid/platform/errors.h" +#include "paddle/phi/core/enforce.h" #include "paddle/phi/kernels/elementwise_add_kernel.h" #include "paddle/pir/include/core/builtin_dialect.h" #include "paddle/pir/include/core/builtin_op.h" @@ -79,10 +80,12 @@ class AddOp : public pir::Op { }; void AddOp::VerifySig() { if (num_operands() != 2) { - throw("The size of inputs must be equal to 2."); + PADDLE_THROW(paddle::platform::errors::Fatal( + "The size of inputs must be equal to 2.")); } if (num_results() != 1) { - throw("The size of outputs must be equal to 1."); + PADDLE_THROW(paddle::platform::errors::Fatal( + "The size of outputs must be equal to 1.")); } } void AddOp::Build(pir::Builder &, diff --git a/test/cpp/pir/pattern_rewrite/pattern_rewrite_test.cc b/test/cpp/pir/pattern_rewrite/pattern_rewrite_test.cc index 9c18ba550e00d..70f0f5ec0760a 100644 --- a/test/cpp/pir/pattern_rewrite/pattern_rewrite_test.cc +++ b/test/cpp/pir/pattern_rewrite/pattern_rewrite_test.cc @@ -20,6 +20,7 @@ #include #include +#include "paddle/common/enforce.h" #include "paddle/fluid/framework/scope.h" #include "paddle/fluid/pir/dialect/operator/ir/op_attribute.h" #include "paddle/fluid/pir/dialect/operator/ir/op_dialect.h" @@ -31,8 +32,7 @@ #include "paddle/fluid/pir/transforms/fusion/conv2d_add_fuse_pass.h" #include "paddle/fluid/pir/transforms/fusion/conv2d_bn_fuse_pass.h" #include "paddle/fluid/pir/transforms/transform_general_functions.h" - -#include "paddle/common/enforce.h" +#include "paddle/fluid/platform/errors.h" #include "paddle/pir/include/core/builder.h" #include "paddle/pir/include/core/builtin_attribute.h" #include "paddle/pir/include/core/builtin_dialect.h" @@ -54,7 +54,6 @@ #include "paddle/common/ddim.h" #include "paddle/phi/common/place.h" #include "paddle/phi/core/kernel_registry.h" - #include "test/cpp/pir/tools/macros_utils.h" PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT); @@ -85,11 +84,13 @@ void Operation1::VerifySig() { auto &attributes = this->attributes(); if (attributes.count("op2_attr1") == 0 || (!attributes.at("op2_attr1").isa())) { - throw("Type of attribute: parameter_name is not right."); + PADDLE_THROW(paddle::platform::errors::InvalidArgument( + "Type of attribute: parameter_name is not right.")); } if (attributes.count("op2_attr2") == 0 || (!attributes.at("op2_attr2").isa())) { - throw("Type of attribute: parameter_name is not right."); + PADDLE_THROW(paddle::platform::errors::InvalidArgument( + "Type of attribute: parameter_name is not right.")); } } const char *Operation1::attributes_name[attributes_num] = { // NOLINT diff --git a/test/cpp/pir/tools/test_op.cc b/test/cpp/pir/tools/test_op.cc index cbcd78a64c27e..6bfb0767b3d43 100644 --- a/test/cpp/pir/tools/test_op.cc +++ b/test/cpp/pir/tools/test_op.cc @@ -11,10 +11,12 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + #include "test/cpp/pir/tools/test_op.h" #include "paddle/common/enforce.h" +#include "paddle/fluid/platform/errors.h" +#include "paddle/phi/core/enforce.h" #include "paddle/pir/include/core/builtin_attribute.h" - namespace test { void RegionOp::Build(pir::Builder &builder, pir::OperationArgument &argument) { @@ -50,11 +52,13 @@ void Operation1::VerifySig() const { auto &attributes = this->attributes(); if (attributes.count("op1_attr1") == 0 || !attributes.at("op1_attr1").isa()) { - throw("Type of attribute: parameter_name is not right."); + PADDLE_THROW(paddle::platform::errors::Fatal( + "Type of attribute: parameter_name is not right.")); } if (attributes.count("op1_attr2") == 0 || !attributes.at("op1_attr2").isa()) { - throw("Type of attribute: parameter_name is not right."); + PADDLE_THROW(paddle::platform::errors::Fatal( + "Type of attribute: parameter_name is not right.")); } } From 3ff45072a154547692594206036e9e50e08d0f15 Mon Sep 17 00:00:00 2001 From: cyberslack_lee Date: Fri, 1 Mar 2024 10:58:24 +0800 Subject: [PATCH 214/282] [clang-tidy] NO.7 bugprone-branch-clone (#61735) --- .../fleet_executor/compute_interceptor.cc | 4 +- .../distributed/fleet_executor/dist_model.cc | 2 +- .../custom_operator/custom_operator_utils.cc | 4 +- paddle/fluid/eager/grad_tensor_holder.cc | 2 +- paddle/fluid/framework/data_feed.cc | 8 +- paddle/fluid/framework/data_set.cc | 14 ++-- .../framework/details/nan_inf_utils_detail.cc | 2 +- paddle/fluid/framework/dist_multi_trainer.cc | 2 +- paddle/fluid/framework/executor.cc | 2 +- .../fluid/framework/heter_section_worker.cc | 2 +- paddle/fluid/framework/infershape_utils.cc | 4 +- .../framework/ir/coalesce_grad_tensor_pass.cc | 2 +- .../framework/ir/generate_pass_tester.cc | 2 +- .../framework/ir/identity_op_clean_pass.cc | 2 +- ...ute_propagate_scales_mkldnn_pass_tester.cc | 2 +- .../ir/mkldnn/cpu_quantize_pass_tester.cc | 5 +- .../mkldnn/cpu_quantize_squash_pass_tester.cc | 2 +- ...t8_scale_calculation_mkldnn_pass_tester.cc | 9 +- .../multi_devices_graph_pass.cc | 6 +- .../framework/ir/transfer_layout_elim_pass.cc | 2 +- .../garbage_collector/garbage_collector.cc | 8 +- .../no_event_garbage_collector.cc | 7 +- .../new_executor/new_executor_defs.cc | 2 +- .../framework/new_executor/pir_interpreter.cc | 4 +- .../new_executor/standalone_executor.cc | 2 +- paddle/fluid/framework/operator.cc | 15 ++-- paddle/fluid/framework/section_worker.cc | 2 +- paddle/fluid/imperative/amp_auto_cast.cc | 5 +- .../fluid/imperative/gradient_accumulator.cc | 4 +- paddle/fluid/imperative/layout_autotune.cc | 2 +- paddle/fluid/imperative/nccl_context.cc | 2 +- .../fluid/imperative/partial_grad_engine.cc | 2 +- paddle/fluid/imperative/prepared_operator.cc | 6 +- paddle/fluid/imperative/reducer.cc | 4 +- paddle/fluid/imperative/var_helper.cc | 3 +- .../analysis/ir_passes/lite_subgraph_pass.cc | 14 ++-- .../analysis/passes/ir_graph_build_pass.cc | 2 +- .../fluid/inference/api/analysis_predictor.cc | 21 ++--- paddle/fluid/inference/api/api_impl.cc | 4 +- .../fluid/inference/api/mkldnn_quantizer.cc | 6 +- .../ir_adaptor/translator/op_translator.cc | 2 +- paddle/fluid/jit/property.cc | 2 +- .../fluid/operators/reader/buffered_reader.cc | 2 +- .../fluid/pir/drr/src/ir_operation_factory.cc | 2 +- paddle/fluid/platform/place.cc | 2 - paddle/fluid/platform/profiler.cc | 28 +++---- paddle/fluid/pybind/eager.cc | 2 +- paddle/fluid/pybind/eager_functions.cc | 2 +- paddle/fluid/pybind/eager_math_op_patch.cc | 4 +- paddle/fluid/pybind/eager_utils.cc | 7 +- paddle/fluid/pybind/parallel_executor.cc | 2 +- paddle/fluid/pybind/pybind.cc | 4 +- paddle/phi/core/compat/convert_utils.cc | 6 +- paddle/phi/core/kernel_registry.cc | 84 ++++++++++++------- paddle/phi/infermeta/unary.cc | 11 +-- .../phi/kernels/cpu/batch_norm_grad_kernel.cc | 2 +- paddle/phi/kernels/cpu/batch_norm_kernel.cc | 4 +- .../kernels/cpu/elementwise_divide_kernel.cc | 2 +- paddle/phi/kernels/cpu/rnn_grad_kernel.cc | 2 +- paddle/phi/kernels/cpu/rnn_kernel.cc | 2 +- paddle/phi/kernels/funcs/sequence_pooling.cc | 2 +- .../kernels/legacy/cpu/elementwise_kernel.cc | 4 +- .../details/fused_broadcast_op_handle_test.cc | 2 +- .../imperative/test_gradient_accmulator.cc | 4 +- 64 files changed, 192 insertions(+), 185 deletions(-) diff --git a/paddle/fluid/distributed/fleet_executor/compute_interceptor.cc b/paddle/fluid/distributed/fleet_executor/compute_interceptor.cc index 8da1ef87814de..5e2be03108294 100644 --- a/paddle/fluid/distributed/fleet_executor/compute_interceptor.cc +++ b/paddle/fluid/distributed/fleet_executor/compute_interceptor.cc @@ -176,7 +176,7 @@ bool ComputeInterceptor::IsInputReady() { flag = flag && (ready_size_map.at(i) != 0); } if (flag) { - if (scope_id_to_finish_flag.empty()) { + if (scope_id_to_finish_flag.empty()) { // NOLINT cur_scope_id_ = i; return true; } else if (scope_id_to_finish_flag.find(i) != @@ -303,7 +303,7 @@ void ComputeInterceptor::RunOps() { cur_scope_id_)); } - if (!cores_.empty()) { + if (!cores_.empty()) { // NOLINT cores_[cur_scope_id_]->Run(/*feed_names=*/{}, /*need_fetch=*/false); } else { for (auto op : node_->ops()) { diff --git a/paddle/fluid/distributed/fleet_executor/dist_model.cc b/paddle/fluid/distributed/fleet_executor/dist_model.cc index a1fd38295319e..4c19069b33705 100644 --- a/paddle/fluid/distributed/fleet_executor/dist_model.cc +++ b/paddle/fluid/distributed/fleet_executor/dist_model.cc @@ -215,7 +215,7 @@ bool DistModel::Init() { } bool DistModel::PreparePlace() { - if (config_.place == "GPU") { + if (config_.place == "GPU") { // NOLINT place_ = paddle::platform::CUDAPlace(config_.device_id); } else if (config_.place == "CPU") { place_ = paddle::platform::CPUPlace(); diff --git a/paddle/fluid/eager/custom_operator/custom_operator_utils.cc b/paddle/fluid/eager/custom_operator/custom_operator_utils.cc index b843e081c29be..a9272053346a7 100644 --- a/paddle/fluid/eager/custom_operator/custom_operator_utils.cc +++ b/paddle/fluid/eager/custom_operator/custom_operator_utils.cc @@ -558,7 +558,7 @@ std::vector> RunInferShapeFn( out_dims = RunInferShapeFunc(ctx, infer_shape_func, inputs, outputs, inplace_map); } else { - if (is_forward) { + if (is_forward) { // NOLINT out_dims = RunDefaultInferShapeFunc(ctx, inputs, outputs, inplace_map); } else { out_dims = @@ -592,7 +592,7 @@ std::vector> RunInferDtypeFn( out_dtypes = RunInferDtypeFunc(ctx, infer_dtype_func, inputs, outputs, inplace_map); } else { - if (is_forward) { + if (is_forward) { // NOLINT out_dtypes = RunDefaultInferDtypeFunc(ctx, inputs, outputs, inplace_map); } else { out_dtypes = diff --git a/paddle/fluid/eager/grad_tensor_holder.cc b/paddle/fluid/eager/grad_tensor_holder.cc index dac55f8f5462f..47f41b5a4f93b 100644 --- a/paddle/fluid/eager/grad_tensor_holder.cc +++ b/paddle/fluid/eager/grad_tensor_holder.cc @@ -79,7 +79,7 @@ void GradTensorHolder::CopyValueFromTensor(size_t slot_id, // Create new tensor->impl and fill it with 1.0 if (t.defined()) { // Fill 1.0, use full to support complex, one_like don't support it. - if (t.is_dense_tensor()) { + if (t.is_dense_tensor()) { // NOLINT buffer_[slot_id][rank] = paddle::experimental::full(t.shape(), 1, t.dtype(), t.place()); } else if (t.is_sparse_csr_tensor() || t.is_sparse_coo_tensor()) { diff --git a/paddle/fluid/framework/data_feed.cc b/paddle/fluid/framework/data_feed.cc index cec1f664ce0f1..9489d22e34d21 100644 --- a/paddle/fluid/framework/data_feed.cc +++ b/paddle/fluid/framework/data_feed.cc @@ -1813,7 +1813,7 @@ int PaddleBoxDataFeed::Next() { this->batch_size_ = index; VLOG(3) << "pv_batch_size_=" << this->batch_size_ << ", thread_id=" << thread_id_; - if (this->batch_size_ != 0) { + if (this->batch_size_ != 0) { // NOLINT PutToFeedVec(pv_vec); } else { VLOG(3) << "finish reading, output_pv_channel_ size=" @@ -2113,7 +2113,7 @@ void SlotRecordInMemoryDataFeed::Init(const DataFeedDesc& data_feed_desc) { finish_init_ = true; input_type_ = data_feed_desc.input_type(); size_t pos = pipe_command_.find(".so"); - if (pos != std::string::npos) { + if (pos != std::string::npos) { // NOLINT pos = pipe_command_.rfind('|'); if (pos == std::string::npos) { so_parser_name_ = pipe_command_; @@ -2129,7 +2129,7 @@ void SlotRecordInMemoryDataFeed::Init(const DataFeedDesc& data_feed_desc) { #if defined(PADDLE_WITH_GPU_GRAPH) && defined(PADDLE_WITH_HETERPS) gpu_graph_data_generator_.SetConfig(data_feed_desc); #endif - if (gpu_graph_mode_) { + if (gpu_graph_mode_) { // NOLINT train_mode_ = true; } else { train_mode_ = data_feed_desc.graph_config().gpu_graph_training(); @@ -2780,7 +2780,7 @@ int SlotRecordInMemoryDataFeed::Next() { this->batch_size_ = batch.second; VLOG(3) << "batch_size_=" << this->batch_size_ << ", thread_id=" << thread_id_; - if (this->batch_size_ != 0) { + if (this->batch_size_ != 0) { // NOLINT PutToFeedVec(&records_[batch.first], this->batch_size_); } else { VLOG(3) << "finish reading for heterps, batch size zero, thread_id=" diff --git a/paddle/fluid/framework/data_set.cc b/paddle/fluid/framework/data_set.cc index 0c48c6e1a25ad..20934879c9a13 100644 --- a/paddle/fluid/framework/data_set.cc +++ b/paddle/fluid/framework/data_set.cc @@ -966,7 +966,7 @@ void DatasetImpl::DynamicAdjustChannelNum(int channel_num, CHECK(output_channels_data_size == 0); // NOLINT cur_channel = 1; } - if (cur_channel == 0) { + if (cur_channel == 0) { // NOLINT origin_channels = &multi_output_channel_; other_channels = &multi_consume_channel_; origin_pv_channels = &multi_pv_output_; @@ -1111,8 +1111,8 @@ void DatasetImpl::CreateReaders() { if (input_pv_channel_ != nullptr) { readers_[i]->SetInputPvChannel(input_pv_channel_.get()); } - if (cur_channel_ == 0 && - static_cast(channel_idx) < multi_output_channel_.size()) { + if (cur_channel_ == 0 && static_cast(channel_idx) < + multi_output_channel_.size()) { // NOLINT readers_[i]->SetOutputChannel(multi_output_channel_[channel_idx].get()); readers_[i]->SetConsumeChannel(multi_consume_channel_[channel_idx].get()); readers_[i]->SetOutputPvChannel(multi_pv_output_[channel_idx].get()); @@ -1722,7 +1722,7 @@ void MultiSlotDataset::PreprocessChannel( const std::set& slots_to_replace, std::unordered_set& index_slots) { // NOLINT int out_channel_size = 0; - if (cur_channel_ == 0) { + if (cur_channel_ == 0) { // NOLINT for (auto& item : multi_output_channel_) { out_channel_size += static_cast(item->Size()); } @@ -1757,7 +1757,7 @@ void MultiSlotDataset::PreprocessChannel( input_channel_->ReadAll(slots_shuffle_original_data_); } else { CHECK(out_channel_size > 0); // NOLINT - if (cur_channel_ == 0) { + if (cur_channel_ == 0) { // NOLINT for (auto& item : multi_output_channel_) { std::vector vec_data; item->Close(); @@ -1792,7 +1792,7 @@ void MultiSlotDataset::PreprocessChannel( } else { // if already have original data for slots shuffle, clear channel input_channel_->Clear(); - if (cur_channel_ == 0) { + if (cur_channel_ == 0) { // NOLINT for (auto& item : multi_output_channel_) { if (!item) { continue; @@ -1809,7 +1809,7 @@ void MultiSlotDataset::PreprocessChannel( } } int end_size = 0; - if (cur_channel_ == 0) { + if (cur_channel_ == 0) { // NOLINT for (auto& item : multi_output_channel_) { if (!item) { continue; diff --git a/paddle/fluid/framework/details/nan_inf_utils_detail.cc b/paddle/fluid/framework/details/nan_inf_utils_detail.cc index 551a10f1ccacd..d18cee16b19a6 100644 --- a/paddle/fluid/framework/details/nan_inf_utils_detail.cc +++ b/paddle/fluid/framework/details/nan_inf_utils_detail.cc @@ -264,7 +264,7 @@ void CheckOpHasNanOrInf(const framework::OperatorBase& op, if (IsSkipOp(op)) return; - if (op_var_nan_inf_white_list().count(op.Type()) == 0) { + if (op_var_nan_inf_white_list().count(op.Type()) == 0) { // NOLINT // NOTE. vname may destruct in the end of this func. for (auto& vname : op.OutputVars(true)) { auto* var = exec_scope.FindVar(vname); diff --git a/paddle/fluid/framework/dist_multi_trainer.cc b/paddle/fluid/framework/dist_multi_trainer.cc index 6fd95267ef6ab..119b6e569cef3 100644 --- a/paddle/fluid/framework/dist_multi_trainer.cc +++ b/paddle/fluid/framework/dist_multi_trainer.cc @@ -157,7 +157,7 @@ void DistMultiTrainer::Run() { std::vector> wait_futures; CHECK_EQ(static_cast(pool.size()), thread_num_); for (int i = 0; i < thread_num_; ++i) { - if (!debug_) { + if (!debug_) { // NOLINT wait_futures.emplace_back( pool[i]->Run([this, i]() { workers_[i]->TrainFiles(); })); } else { diff --git a/paddle/fluid/framework/executor.cc b/paddle/fluid/framework/executor.cc index d935e9ea066bd..fbc2565e755fa 100644 --- a/paddle/fluid/framework/executor.cc +++ b/paddle/fluid/framework/executor.cc @@ -99,7 +99,7 @@ void Executor::CreateVariables(const ProgramDesc& pdesc, while (ancestor_scope->parent()) { ancestor_scope = ancestor_scope->parent(); } - if (ancestor_scope != scope) { + if (ancestor_scope != scope) { // NOLINT for (auto& var : global_block.AllVars()) { if (var->Name() == framework::kEmptyVarName) { continue; diff --git a/paddle/fluid/framework/heter_section_worker.cc b/paddle/fluid/framework/heter_section_worker.cc index cecfa39d3c16b..942f776b2323f 100644 --- a/paddle/fluid/framework/heter_section_worker.cc +++ b/paddle/fluid/framework/heter_section_worker.cc @@ -126,7 +126,7 @@ void HeterSectionWorker::Initialize(const TrainerDesc& desc) { bool is_first_stage = (pipeline_stage_ == 0); bool is_last_stage = (pipeline_stage_ + 1 == num_pipeline_stages_); - if (is_first_stage) { + if (is_first_stage) { // NOLINT for (auto& op_desc : program_->Block(0).AllOps()) { auto op = std::move(OpRegistry::CreateOp(*op_desc)); auto op_type = op->Type(); diff --git a/paddle/fluid/framework/infershape_utils.cc b/paddle/fluid/framework/infershape_utils.cc index bcf72be80decb..932e467e23dc0 100644 --- a/paddle/fluid/framework/infershape_utils.cc +++ b/paddle/fluid/framework/infershape_utils.cc @@ -658,7 +658,7 @@ CompatInferMetaContext BuildInferMetaContext(InferShapeContext* ctx, if (attr_ptr && !is_attr_var) { auto& attr = *attr_ptr; switch (AttrTypeID(attr)) { - case framework::proto::AttrType::INTS: + case framework::proto::AttrType::INTS: // NOLINT infer_meta_context.EmplaceBackAttr(std::move( phi::IntArray(PADDLE_GET_CONST(std::vector, attr)))); break; @@ -836,7 +836,7 @@ CompatInferMetaContext BuildInferMetaContext(InferShapeContext* ctx, attr_names[i])); } break; - case phi::AttributeType::FLOAT32S: + case phi::AttributeType::FLOAT32S: // NOLINT infer_meta_context.EmplaceBackAttr( PADDLE_GET_CONST(std::vector, attr)); break; diff --git a/paddle/fluid/framework/ir/coalesce_grad_tensor_pass.cc b/paddle/fluid/framework/ir/coalesce_grad_tensor_pass.cc index 44cb004fec172..966f4ea14967d 100644 --- a/paddle/fluid/framework/ir/coalesce_grad_tensor_pass.cc +++ b/paddle/fluid/framework/ir/coalesce_grad_tensor_pass.cc @@ -134,7 +134,7 @@ class CoalesceGradTensorPass : public ir::Pass { auto &pinned_var_set = graph->GetOrInit(details::kPinnedVars); - if (IsUnifiedDtype(p_g_dense_grad, vars_info)) { + if (IsUnifiedDtype(p_g_dense_grad, vars_info)) { // NOLINT RecordGradients(p_g_dense_grad, vars_info, &pinned_var_set); CoalesceTensors(vars_info, p_g_dense_grad, &result); } else { diff --git a/paddle/fluid/framework/ir/generate_pass_tester.cc b/paddle/fluid/framework/ir/generate_pass_tester.cc index 760e1e8ce4ef8..58a3741a924aa 100644 --- a/paddle/fluid/framework/ir/generate_pass_tester.cc +++ b/paddle/fluid/framework/ir/generate_pass_tester.cc @@ -25,7 +25,7 @@ REGISTER_GENERATE_PASS(generate_fc_fuse) { VLOG(3) << "exec lambda func."; auto mul = OP_(mul)({{"X", x}, {"Y", y}}).Out("Out"); auto ewadd = OP_(elementwise_add)({{"X", mul}, {"Y", z}}).Out("Out"); - if (with_relu) { + if (with_relu) { // NOLINT return OP_(relu)({"X", ewadd}).Out("Out"); } else { return ewadd; diff --git a/paddle/fluid/framework/ir/identity_op_clean_pass.cc b/paddle/fluid/framework/ir/identity_op_clean_pass.cc index ab9df0ae4abee..55316c1b82310 100644 --- a/paddle/fluid/framework/ir/identity_op_clean_pass.cc +++ b/paddle/fluid/framework/ir/identity_op_clean_pass.cc @@ -70,7 +70,7 @@ FindUselessOpPattern::FindUselessOpPattern(PDPattern* pattern, auto in_dtype = x->Op()->GetAttrIfExists("in_dtype"); auto out_dtype = x->Op()->GetAttrIfExists("out_dtype"); return in_dtype == out_dtype; - } else if (op_type == "c_identity") { + } else if (op_type == "c_identity") { // NOLINT return true; } else if (op_type == "assign") { const auto& in_name = x->Op()->Input("X")[0]; diff --git a/paddle/fluid/framework/ir/mkldnn/compute_propagate_scales_mkldnn_pass_tester.cc b/paddle/fluid/framework/ir/mkldnn/compute_propagate_scales_mkldnn_pass_tester.cc index 0f0d385569083..c09a2d1ffbb8d 100644 --- a/paddle/fluid/framework/ir/mkldnn/compute_propagate_scales_mkldnn_pass_tester.cc +++ b/paddle/fluid/framework/ir/mkldnn/compute_propagate_scales_mkldnn_pass_tester.cc @@ -161,7 +161,7 @@ class ComputePropagateScalesMkldnnPassTest : public testing::Test { begin(wh[i]), end(wh[i]), wh_tensor->mutable_data(phi::CPUPlace()) + i * wh[0].size()); - if (type == "gru") { + if (type == "gru") { // NOLINT ComputeGruWeightScales( graph, &scope, wx_name, wh_name, &var_quant_scales); } else { diff --git a/paddle/fluid/framework/ir/mkldnn/cpu_quantize_pass_tester.cc b/paddle/fluid/framework/ir/mkldnn/cpu_quantize_pass_tester.cc index bad886ae40cdf..c7e15e24216aa 100644 --- a/paddle/fluid/framework/ir/mkldnn/cpu_quantize_pass_tester.cc +++ b/paddle/fluid/framework/ir/mkldnn/cpu_quantize_pass_tester.cc @@ -61,7 +61,7 @@ void SetOp(ProgramDesc* prog, op->SetOutput("Output", {outputs[0]}); } else if (type == "pool2d" || type == "fused_transpose" || type == "reshape2" || type == "nearest_interp" || - type == "nearest_interp_v2") { + type == "nearest_interp_v2" || type == "dropout") { op->SetInput("X", {inputs[0]}); op->SetOutput("Out", {outputs[0]}); } else if (type == "slice") { @@ -70,9 +70,6 @@ void SetOp(ProgramDesc* prog, } else if (type == "split") { op->SetInput("X", {inputs[0]}); op->SetOutput("Out", {outputs}); - } else if (type == "dropout") { - op->SetInput("X", {inputs[0]}); - op->SetOutput("Out", {outputs[0]}); } else if (type == "fc") { op->SetInput("Input", {inputs[0]}); if (inputs.size() > 1) op->SetInput("W", {inputs[1]}); diff --git a/paddle/fluid/framework/ir/mkldnn/cpu_quantize_squash_pass_tester.cc b/paddle/fluid/framework/ir/mkldnn/cpu_quantize_squash_pass_tester.cc index d2c6d981c3a2e..89e57108b17ef 100644 --- a/paddle/fluid/framework/ir/mkldnn/cpu_quantize_squash_pass_tester.cc +++ b/paddle/fluid/framework/ir/mkldnn/cpu_quantize_squash_pass_tester.cc @@ -41,7 +41,7 @@ void SetOp(ProgramDesc* prog, if (type != "dropout" && type != "quantize" && type != "dequantize") { op->SetAttr("mkldnn_data_type", mkldnn_data_type); } - if (type == "pool2d") { + if (type == "pool2d") { // NOLINT op->SetInput("X", {inputs[0]}); op->SetOutput("Out", {outputs[0]}); if (!scale.empty()) op->SetAttr("Scale_in", scale[0]); diff --git a/paddle/fluid/framework/ir/mkldnn/int8_scale_calculation_mkldnn_pass_tester.cc b/paddle/fluid/framework/ir/mkldnn/int8_scale_calculation_mkldnn_pass_tester.cc index 44856c086dc93..fde7fb07b9108 100644 --- a/paddle/fluid/framework/ir/mkldnn/int8_scale_calculation_mkldnn_pass_tester.cc +++ b/paddle/fluid/framework/ir/mkldnn/int8_scale_calculation_mkldnn_pass_tester.cc @@ -70,14 +70,7 @@ ProgramDesc BuildProgramDesc(bool convWithExistingBias, } } - if (convWithExistingBias) { - SetOp(&prog, - "conv2d", - "conv", - std::vector({"c", "weights", "conv_bias"}), - std::vector({"f"}), - scale_weights); - } else if (scale_weights.size() > 1) { + if (convWithExistingBias || scale_weights.size() > 1) { SetOp(&prog, "conv2d", "conv", diff --git a/paddle/fluid/framework/ir/multi_devices_graph_pass/multi_devices_graph_pass.cc b/paddle/fluid/framework/ir/multi_devices_graph_pass/multi_devices_graph_pass.cc index 295ef57cfdfea..cc20f52180871 100644 --- a/paddle/fluid/framework/ir/multi_devices_graph_pass/multi_devices_graph_pass.cc +++ b/paddle/fluid/framework/ir/multi_devices_graph_pass/multi_devices_graph_pass.cc @@ -933,7 +933,7 @@ bool ReduceSSAGraphBuilder::DealWithSpecialOp(ir::Graph *result, void ReduceSSAGraphBuilder::InsertPostprocessOps(ir::Graph *result) const { if (UseGPU()) { - if (strategy_.fuse_broadcast_ops_ == true) { + if (strategy_.fuse_broadcast_ops_ == true) { // NOLINT CreateFusedBroadcastOp(result, bcast_var_name_set_); } else { for (size_t dev_id = 0; dev_id < bcast_var_name_set_.size(); ++dev_id) { @@ -1193,7 +1193,7 @@ int DistSSAGraphBuilder::CreateRPCOp(ir::Graph *result, ir::Node *node) const { node->Op()->Type())); // Create fetch_barrier op handle to enable output on all devices. // **NOTE** fetch_barrier should output variables list same as recv op does. - if (node->Op()->Type() == "fetch_barrier") { + if (node->Op()->Type() == "fetch_barrier") { // NOLINT result->Get(kGraphOps).emplace_back( new details::FetchBarrierOpHandle( result->CreateOpNode(node->Op()), local_scopes_, places_)); @@ -1354,7 +1354,7 @@ void DistSSAGraphBuilder::InsertPostprocessOps(ir::Graph *result) const { strategy_.reduce_ == details::BuildStrategy::ReduceStrategy::kReduce) { return; } - if (strategy_.fuse_broadcast_ops_ == true) { + if (strategy_.fuse_broadcast_ops_ == true) { // NOLINT CreateFusedBroadcastOp(result, bcast_var_name_set_); } else { for (size_t dev_id = 0; dev_id < bcast_var_name_set_.size(); ++dev_id) { diff --git a/paddle/fluid/framework/ir/transfer_layout_elim_pass.cc b/paddle/fluid/framework/ir/transfer_layout_elim_pass.cc index 3a9a2c81889ee..ac3441eb7e737 100644 --- a/paddle/fluid/framework/ir/transfer_layout_elim_pass.cc +++ b/paddle/fluid/framework/ir/transfer_layout_elim_pass.cc @@ -239,7 +239,7 @@ void TransferLayoutElimPass::ApplyImpl(ir::Graph *graph) const { FusePassBase::Init(pattern_name, graph); auto transfer_format = [&](std::string data_format) -> std::string { - if (data_format == "NCHW") { + if (data_format == "NCHW") { // NOLINT return "NHWC"; } else if (data_format == "NHWC") { return "NCHW"; diff --git a/paddle/fluid/framework/new_executor/garbage_collector/garbage_collector.cc b/paddle/fluid/framework/new_executor/garbage_collector/garbage_collector.cc index 166853e2b18da..0d73e2d3fede9 100644 --- a/paddle/fluid/framework/new_executor/garbage_collector/garbage_collector.cc +++ b/paddle/fluid/framework/new_executor/garbage_collector/garbage_collector.cc @@ -32,14 +32,14 @@ CreateInterpreterCoreGarbageCollector( const platform::Place& place, const std::vector>& vec_instruction) { if (platform::is_gpu_place(place)) { - if (IsInterpretercoreFastGCEnabled()) { + if (IsInterpretercoreFastGCEnabled()) { // NOLINT return std::unique_ptr( new InterpreterCoreFastGarbageCollector()); } else { return std::unique_ptr( new InterpreterCoreEventGarbageCollector(vec_instruction)); } - } else if (platform::is_xpu_place(place)) { + } else if (platform::is_xpu_place(place)) { // NOLINT // Because there is no multi-stream on XPU device, fast GC can // be used. // Previously, XPU used no_event GC. But `Wait` in no_event GC @@ -62,14 +62,14 @@ CreateInterpreterCoreGarbageCollector( const platform::Place& place, const std::vector& vec_instruction) { if (platform::is_gpu_place(place)) { - if (IsInterpretercoreFastGCEnabled()) { + if (IsInterpretercoreFastGCEnabled()) { // NOLINT return std::unique_ptr( new InterpreterCoreFastGarbageCollector()); } else { return std::unique_ptr( new InterpreterCoreEventGarbageCollector(vec_instruction)); } - } else if (platform::is_xpu_place(place)) { + } else if (platform::is_xpu_place(place)) { // NOLINT // Because there is no multi-stream on XPU device, fast GC can // be used. // Previously, XPU used no_event GC. But `Wait` in no_event GC diff --git a/paddle/fluid/framework/new_executor/garbage_collector/no_event_garbage_collector.cc b/paddle/fluid/framework/new_executor/garbage_collector/no_event_garbage_collector.cc index 3b7ebc18f36da..d236e740679dd 100644 --- a/paddle/fluid/framework/new_executor/garbage_collector/no_event_garbage_collector.cc +++ b/paddle/fluid/framework/new_executor/garbage_collector/no_event_garbage_collector.cc @@ -49,9 +49,10 @@ void InterpreterCoreNoEventGarbageCollector::Add( if (var->IsType()) { Add(var->GetMutable()->MoveMemoryHolder(), ctx); - } else if (var->IsType< - operators::reader:: - OrderedMultiDeviceLoDTensorBlockingQueueHolder>()) { + } else if ( + var->IsType< + operators::reader:: + OrderedMultiDeviceLoDTensorBlockingQueueHolder>()) { // NOLINT // TODO(xiongkun03) in old executor, this type of variable is not support // eager deletion. so we just leave it here ? } else if (var->IsType()) { diff --git a/paddle/fluid/framework/new_executor/new_executor_defs.cc b/paddle/fluid/framework/new_executor/new_executor_defs.cc index b3ec52029bb5b..6c9e5b4a877d5 100644 --- a/paddle/fluid/framework/new_executor/new_executor_defs.cc +++ b/paddle/fluid/framework/new_executor/new_executor_defs.cc @@ -94,7 +94,7 @@ void VariableScope::AddVar(const std::string& name, auto id = VarSize(); name2id_[name] = static_cast(id); vec_meta_info_.emplace_back(0, var_desc); - if (local_scope_ != nullptr) { + if (local_scope_ != nullptr) { // NOLINT var_list_.push_back(local_scope_->FindVar(name)); } else { var_list_.push_back(scope_->FindVar(name)); diff --git a/paddle/fluid/framework/new_executor/pir_interpreter.cc b/paddle/fluid/framework/new_executor/pir_interpreter.cc index 236f18dfb223c..3690c67ac58f4 100644 --- a/paddle/fluid/framework/new_executor/pir_interpreter.cc +++ b/paddle/fluid/framework/new_executor/pir_interpreter.cc @@ -702,7 +702,7 @@ void PirInterpreter::BuildInstruction() { continue; } } else if (op.dialect()->name() == "pd_op") { - if (op.isa()) { + if (op.isa()) { // NOLINT vec_instruction_base_.emplace_back(std::make_unique( op_idx++, place_, &op, value_exe_info_.get(), execution_config_)); sub_blocks_.insert( @@ -751,7 +751,7 @@ void PirInterpreter::BuildInstruction() { } VLOG(6) << "process " << op_name; - if (op.isa()) { + if (op.isa()) { // NOLINT CREATE_INSTR(LegacyKernelInstruction); } else { CREATE_INSTR(PhiKernelInstruction); diff --git a/paddle/fluid/framework/new_executor/standalone_executor.cc b/paddle/fluid/framework/new_executor/standalone_executor.cc index 2bb0a7197774e..74e09a15d6246 100644 --- a/paddle/fluid/framework/new_executor/standalone_executor.cc +++ b/paddle/fluid/framework/new_executor/standalone_executor.cc @@ -57,7 +57,7 @@ StandaloneExecutor::StandaloneExecutor(const platform::Place& place, const std::string& job_type = job->Type(); std::shared_ptr program = nullptr; std::shared_ptr<::pir::Program> ir_program = nullptr; - if (FLAGS_enable_pir_api || FLAGS_enable_pir_in_executor) { + if (FLAGS_enable_pir_api || FLAGS_enable_pir_in_executor) { // NOLINT ir_program = plan_.IrProgram(job_type); } else { // NOTE (liuchenghao): std::make_shared will duplicate ProgramDesc object, diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc index 99ccbbe50d241..55fc19ad2be1c 100644 --- a/paddle/fluid/framework/operator.cc +++ b/paddle/fluid/framework/operator.cc @@ -1754,7 +1754,8 @@ void OperatorWithKernel::RunImpl(const Scope& scope, std::string phi_kernel_name; if (phi::KernelFactory::Instance().HasCompatiblePhiKernel(type_)) { if (kernel_signature_ == nullptr || phi_kernel_ == nullptr) { - if (phi::KernelFactory::Instance().HasStructuredKernel(type_)) { + if (phi::KernelFactory::Instance().HasStructuredKernel( + type_)) { // NOLINT kernel_signature_ = std::make_unique(type_.c_str()); } else { @@ -1989,7 +1990,7 @@ void OperatorWithKernel::RunImpl(const Scope& scope, 1, platform::EventRole::kInnerOp); if (need_prepare_data_) { - if (fallback_to_cpu) { + if (fallback_to_cpu) { // NOLINT transfer_scope = PrepareData(scope, phi_cpu_kernel_key, &transfered_inplace_vars, @@ -2278,7 +2279,7 @@ OpKernelType OperatorWithKernel::InnerGetExpectedKernelType( phi::KernelKey OperatorWithKernel::ChoosePhiKernel( const ExecutionContext& ctx) const { std::string phi_kernel_name; - if (phi::KernelFactory::Instance().HasStructuredKernel(type_)) { + if (phi::KernelFactory::Instance().HasStructuredKernel(type_)) { // NOLINT kernel_signature_ = std::make_unique(type_.c_str()); } else { kernel_signature_ = std::make_unique( @@ -3104,7 +3105,7 @@ static void SetDnnAttrIntoDeviceContext( case proto::AttrType::STRING: one_dnn_ctx->SetDnnAttr(attr_name, PADDLE_GET_CONST(std::string, attr)); break; - case proto::AttrType::INTS: + case proto::AttrType::INTS: // NOLINT one_dnn_ctx->SetDnnAttr(attr_name, PADDLE_GET_CONST(std::vector, attr)); break; @@ -3358,7 +3359,7 @@ void OperatorWithKernel::BuildPhiKernelContext( case phi::AttributeType::INT_ARRAY: if (attr_iter != Attrs().end()) { switch (AttrTypeID(attr_iter->second)) { - case proto::AttrType::INTS: + case proto::AttrType::INTS: // NOLINT phi_kernel_context->EmplaceBackAttr(std::move(phi::IntArray( PADDLE_GET_CONST(std::vector, attr_iter->second)))); break; @@ -3497,7 +3498,7 @@ void OperatorWithKernel::BuildPhiKernelContext( phi_kernel_context->EmplaceBackAttr( PADDLE_GET_CONST(int64_t, attr_iter->second)); break; - case phi::AttributeType::INT32S: + case phi::AttributeType::INT32S: // NOLINT phi_kernel_context->EmplaceBackAttr( PADDLE_GET_CONST(std::vector, attr_iter->second)); break; @@ -3536,7 +3537,7 @@ void OperatorWithKernel::BuildPhiKernelContext( attr_names[i])); } break; - case phi::AttributeType::FLOAT32S: + case phi::AttributeType::FLOAT32S: // NOLINT phi_kernel_context->EmplaceBackAttr( PADDLE_GET_CONST(std::vector, attr_iter->second)); break; diff --git a/paddle/fluid/framework/section_worker.cc b/paddle/fluid/framework/section_worker.cc index 91d24cc70552c..19e09ab5edf8d 100644 --- a/paddle/fluid/framework/section_worker.cc +++ b/paddle/fluid/framework/section_worker.cc @@ -238,7 +238,7 @@ void SectionWorker::TrainFiles() { #endif } // max_memory_size >= 0 - if (schedule_mode_ == 0) { + if (schedule_mode_ == 0) { // NOLINT RunFThenB(gc); } else { Run1F1B(gc); diff --git a/paddle/fluid/imperative/amp_auto_cast.cc b/paddle/fluid/imperative/amp_auto_cast.cc index 50df994014004..c2aab61851fb5 100644 --- a/paddle/fluid/imperative/amp_auto_cast.cc +++ b/paddle/fluid/imperative/amp_auto_cast.cc @@ -185,7 +185,7 @@ AmpOperators::GetMutableUnsupportedOps(const phi::DataType& data_type) { true, phi::errors::InvalidArgument( "The data_type mismatch. It should be FLOAT16 or BFLOAT16.")); - if (data_type == phi::DataType::FLOAT16) { + if (data_type == phi::DataType::FLOAT16) { // NOLINT return unsupported_fp16_ops_; } else { return unsupported_bf16_ops_; @@ -375,7 +375,8 @@ template NameVarMap AutoCastInputs(const std::string& op_type, const NameVarMap& ins) { NameVarMap new_ins(ins); - if (AmpOperators::Instance().GetMutableAllowOps()->count(op_type)) { + if (AmpOperators::Instance().GetMutableAllowOps()->count( + op_type)) { // NOLINT for (auto& pair : new_ins) { // NOTE(zhiqiu): batch_norm and layer_norm support only input x is fp16. if ((op_type == "batch_norm" || op_type == "layer_norm" || diff --git a/paddle/fluid/imperative/gradient_accumulator.cc b/paddle/fluid/imperative/gradient_accumulator.cc index 8f4dfbbcdc977..d9c91a4c6b0a0 100644 --- a/paddle/fluid/imperative/gradient_accumulator.cc +++ b/paddle/fluid/imperative/gradient_accumulator.cc @@ -518,7 +518,7 @@ void VariableWrapperAdd(std::shared_ptr var, static platform::Place GetPlaceOfVar( const std::shared_ptr& var) { platform::Place place; - if (var->Var().IsType()) { + if (var->Var().IsType()) { // NOLINT place = var->Var().Get().place(); } else if (var->Var().IsType()) { place = var->Var().Get().place(); @@ -735,7 +735,7 @@ void SortedGradientAccumulator::SumGrad(std::shared_ptr var, } #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) - if (paddle::platform::is_gpu_place(place)) { + if (paddle::platform::is_gpu_place(place)) { // NOLINT // sum selected rows firstly for (auto& var_info : tmp_grad_vars_) { if (!var_info.var->Var().IsType()) { diff --git a/paddle/fluid/imperative/layout_autotune.cc b/paddle/fluid/imperative/layout_autotune.cc index 006021488aa57..7836572b0c426 100644 --- a/paddle/fluid/imperative/layout_autotune.cc +++ b/paddle/fluid/imperative/layout_autotune.cc @@ -145,7 +145,7 @@ LayoutAutotuneGuard::LayoutAutotuneGuard(std::shared_ptr tracer, } LayoutAutotuneGuard::~LayoutAutotuneGuard() { - if (pre_layout_autotune_) { + if (pre_layout_autotune_) { // NOLINT tracer_->EnableLayoutAutoTune(); } else { tracer_->DisableLayoutAutoTune(); diff --git a/paddle/fluid/imperative/nccl_context.cc b/paddle/fluid/imperative/nccl_context.cc index d70d40808f915..3ed9b97bfc362 100644 --- a/paddle/fluid/imperative/nccl_context.cc +++ b/paddle/fluid/imperative/nccl_context.cc @@ -67,7 +67,7 @@ void NCCLParallelContext::Init() { std::vector nccl_ids; nccl_ids.resize(strategy_.nrings_); - if (strategy_.local_rank_ == 0) { + if (strategy_.local_rank_ == 0) { // NOLINT // generate the unique ncclid on the root worker for (auto &nccl_id : nccl_ids) { platform::dynload::ncclGetUniqueId(&nccl_id); diff --git a/paddle/fluid/imperative/partial_grad_engine.cc b/paddle/fluid/imperative/partial_grad_engine.cc index 0a5d44a1e1e57..47a3605ecc7be 100644 --- a/paddle/fluid/imperative/partial_grad_engine.cc +++ b/paddle/fluid/imperative/partial_grad_engine.cc @@ -366,7 +366,7 @@ class GradientAccumulationInfo { if (!grad_var_) { grad_var_ = std::make_shared(true, mapped_grad_var_->Name()); grad_var_->SetOverriddenStopGradient(false); - if (sort_gradient_) { + if (sort_gradient_) { // NOLINT accumulator_ = std::make_unique( grad_var_->SharedVar().get()); } else { diff --git a/paddle/fluid/imperative/prepared_operator.cc b/paddle/fluid/imperative/prepared_operator.cc index 8129ea244f489..a60c81a4c22d9 100644 --- a/paddle/fluid/imperative/prepared_operator.cc +++ b/paddle/fluid/imperative/prepared_operator.cc @@ -660,7 +660,7 @@ void PreparedOp::Run(const NameVarMap& ins, const NameVarMap& outs, const framework::AttributeMap& attrs, const framework::AttributeMap& default_attrs) { - if (run_phi_kernel_) { + if (run_phi_kernel_) { // NOLINT PreparedOpRunPtImpl(op_, kernel_key_, arg_map_fn_, @@ -692,7 +692,7 @@ void PreparedOp::Run(const NameVarMap& ins, const NameVarMap& outs, const framework::AttributeMap& attrs, const framework::AttributeMap& default_attrs) { - if (run_phi_kernel_) { + if (run_phi_kernel_) { // NOLINT PreparedOpRunPtImpl(op_, kernel_key_, arg_map_fn_, @@ -724,7 +724,7 @@ void PreparedOp::Run(const NameVarMap& ins, const NameVarMap& outs, const framework::AttributeMap& attrs, const framework::AttributeMap& default_attrs) { - if (run_phi_kernel_) { + if (run_phi_kernel_) { // NOLINT PreparedOpRunPtImpl(op_, kernel_key_, arg_map_fn_, diff --git a/paddle/fluid/imperative/reducer.cc b/paddle/fluid/imperative/reducer.cc index 461c2d3ff4bb8..5b8dc28d03111 100644 --- a/paddle/fluid/imperative/reducer.cc +++ b/paddle/fluid/imperative/reducer.cc @@ -227,7 +227,7 @@ void SplitTensorsWithType( void Group::ConcatTensors(const platform::DeviceContext &context) { auto place = context.GetPlace(); - if (platform::is_gpu_place(place)) { + if (platform::is_gpu_place(place)) { // NOLINT #if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL) ConcatTensorsWithType(static_cast(context), dense_tensors_, @@ -263,7 +263,7 @@ void Group::ConcatTensors(const platform::DeviceContext &context) { void Group::SplitTensors(const platform::DeviceContext &context) { auto place = context.GetPlace(); - if (platform::is_gpu_place(place)) { + if (platform::is_gpu_place(place)) { // NOLINT #if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL) SplitTensorsWithType(static_cast(context), &dense_contents_, diff --git a/paddle/fluid/imperative/var_helper.cc b/paddle/fluid/imperative/var_helper.cc index bafea5a720d3a..9561962935ffe 100644 --- a/paddle/fluid/imperative/var_helper.cc +++ b/paddle/fluid/imperative/var_helper.cc @@ -50,7 +50,8 @@ void InitializeVariable(paddle::framework::Variable *var, var->GetMutable(); } else if (var_type == paddle::framework::proto::VarType::FEED_MINIBATCH) { var->GetMutable(); - } else if (var_type == paddle::framework::proto::VarType::FETCH_LIST) { + } else if (var_type == + paddle::framework::proto::VarType::FETCH_LIST) { // NOLINT var->GetMutable(); } else if (var_type == paddle::framework::proto::VarType::STEP_SCOPES) { var->GetMutable>(); diff --git a/paddle/fluid/inference/analysis/ir_passes/lite_subgraph_pass.cc b/paddle/fluid/inference/analysis/ir_passes/lite_subgraph_pass.cc index f8a4d4d15af72..dcdf8405cc2f8 100644 --- a/paddle/fluid/inference/analysis/ir_passes/lite_subgraph_pass.cc +++ b/paddle/fluid/inference/analysis/ir_passes/lite_subgraph_pass.cc @@ -271,7 +271,7 @@ void LiteSubgraphPass::SetUpEngine( Get>("nnadapter_model_cache_token"); lite_api::TargetType target_type = TARGET(kX86); - if (use_gpu) { + if (use_gpu) { // NOLINT target_type = TARGET(kCUDA); } else if (use_xpu) { target_type = TARGET(kXPU); @@ -417,13 +417,11 @@ void LiteSubgraphPass::ApplyImpl(framework::ir::Graph* graph) const { auto& lite_ops_filter = Get>("lite_ops_filter"); auto teller = [&lite_ops_filter](const Node* node) { - if (!node->IsOp() || !node->Op()) - return false; - else if (node->Op()->Type() == "feed" || node->Op()->Type() == "fetch") - return false; - else if (std::find(lite_ops_filter.begin(), - lite_ops_filter.end(), - node->Op()->Type()) != lite_ops_filter.end()) + if (!node->IsOp() || !node->Op() || node->Op()->Type() == "feed" || + node->Op()->Type() == "fetch" || + std::find(lite_ops_filter.begin(), + lite_ops_filter.end(), + node->Op()->Type()) != lite_ops_filter.end()) return false; return inference::lite::OpTeller::Global().Tell(node->Op()->Type(), *node->Op()); diff --git a/paddle/fluid/inference/analysis/passes/ir_graph_build_pass.cc b/paddle/fluid/inference/analysis/passes/ir_graph_build_pass.cc index 8106dfbb9e6aa..ea97be8f90a60 100644 --- a/paddle/fluid/inference/analysis/passes/ir_graph_build_pass.cc +++ b/paddle/fluid/inference/analysis/passes/ir_graph_build_pass.cc @@ -121,7 +121,7 @@ std::unique_ptr IrGraphBuildPass::LoadModel( bool model_from_memory, bool skip_load_params) { framework::Executor exe(place); - if (!model_from_memory) { + if (!model_from_memory) { // NOLINT return Load(&exe, scope, program_path, params_path, !skip_load_params); } else { return LoadFromMemory(&exe, scope, program_path, params_path); diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index 35ff7eb608b6a..9b05b9f78572e 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -1301,7 +1301,7 @@ bool AnalysisPredictor::LoadConverterConfig( int64_t key = std::stoll(one_line[0]); for (size_t i = 1; i < one_line.size(); ++i) { int64_t val = std::stoll(one_line[i]); - if (ring_to_rank) { + if (ring_to_rank) { // NOLINT if (ring_id_to_ranks->find(key) == ring_id_to_ranks->end()) { ring_id_to_ranks->insert({key, std::vector()}); } @@ -1441,7 +1441,7 @@ bool AnalysisPredictor::Run(const std::vector &inputs, HookCollectShapeRangeInfo(); } - if (config_.new_executor_enabled()) { + if (config_.new_executor_enabled()) { // NOLINT executor_->RunInterpreterCore(); } else { // Run the inference program @@ -1514,7 +1514,7 @@ bool AnalysisPredictor::Run(const std::vector &inputs, HookCollectShapeRangeInfo(); } - if (config_.new_executor_enabled()) { + if (config_.new_executor_enabled()) { // NOLINT executor_->RunInterpreterCore(); } else { // Run the inference program @@ -1937,7 +1937,7 @@ void AnalysisPredictor::PrepareArgument() { if (deleted_passes.count(pass)) continue; pass_builder->AppendPass(pass); } - } else if (config_.use_xpu()) { + } else if (config_.use_xpu()) { // NOLINT // All passes support fp16. Not reset pass_builder. } else if (config_.use_custom_device()) { // All passes support fp16. Not reset pass_builder. @@ -2060,7 +2060,8 @@ void AnalysisPredictor::OptimizeInferenceProgram() { #else if (config_.mkldnn_enabled() || (config_.tensorrt_engine_enabled() && - config_.tensorrt_precision_mode_ == AnalysisConfig::Precision::kInt8)) { + config_.tensorrt_precision_mode_ == + AnalysisConfig::Precision::kInt8)) { // NOLINT argument_->PartiallyRelease(); } else { argument_.reset(nullptr); @@ -2354,7 +2355,7 @@ std::unique_ptr AnalysisPredictor::GetInputTensor( const std::string &name) { framework::Scope *scope = nullptr; #if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE) - if (config_.dist_config().use_dist_model()) { + if (config_.dist_config().use_dist_model()) { // NOLINT scope = scope_.get(); } else { scope = executor_->GetScope(); @@ -2405,7 +2406,7 @@ std::unique_ptr AnalysisPredictor::GetOutputTensor( const std::string &name) { framework::Scope *scope; // NOLINT #if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE) - if (config_.dist_config().use_dist_model()) { + if (config_.dist_config().use_dist_model()) { // NOLINT scope = scope_.get(); } else { scope = executor_->GetScope(); @@ -2455,7 +2456,7 @@ std::unique_ptr AnalysisPredictor::GetOutputTensor( bool AnalysisPredictor::ZeroCopyRun(bool switch_stream) { inference::DisplayMemoryInfo(place_, "before run"); #if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE) - if (config_.dist_config().use_dist_model()) { + if (config_.dist_config().use_dist_model()) { // NOLINT VLOG(3) << "ZeroCopyRun will use the fleet executor."; fleet_exe_->Run(config_.dist_config().carrier_id()); return true; @@ -2514,7 +2515,7 @@ bool AnalysisPredictor::ZeroCopyRun(bool switch_stream) { } #endif - if (config_.new_executor_enabled()) { + if (config_.new_executor_enabled()) { // NOLINT executor_->RunInterpreterCore({}, false, switch_stream); } else { executor_->Run(); @@ -2780,7 +2781,7 @@ void AnalysisPredictor::StatisticShapeRangeInfo() { bool AnalysisPredictor::LoadProgramDesc() { // Initialize the inference program std::string filename; - if (!config_.model_dir().empty()) { + if (!config_.model_dir().empty()) { // NOLINT filename = config_.model_dir() + "/__model__"; } else if (!config_.prog_file().empty()) { // All parameters are saved in a single file. diff --git a/paddle/fluid/inference/api/api_impl.cc b/paddle/fluid/inference/api/api_impl.cc index c8eaa1c3ebd1e..1ae582feb4acf 100644 --- a/paddle/fluid/inference/api/api_impl.cc +++ b/paddle/fluid/inference/api/api_impl.cc @@ -101,7 +101,7 @@ bool NativePaddlePredictor::Init( executor_ = std::make_unique(place_); // Initialize the inference program - if (!config_.model_dir.empty()) { + if (!config_.model_dir.empty()) { // NOLINT // Parameters are saved in separate files sited in // the specified `dirname`. inference_program_ = paddle::inference::Load( @@ -286,7 +286,7 @@ bool NativePaddlePredictor::SetFeed(const std::vector &inputs, } input.set_lod(lod); int idx = -1; - if (config_.specify_input_name) { + if (config_.specify_input_name) { // NOLINT idx = static_cast(feed_names_[inputs[i].name]); } else { idx = PADDLE_GET_CONST(int, feeds_[i]->GetAttr("col")); diff --git a/paddle/fluid/inference/api/mkldnn_quantizer.cc b/paddle/fluid/inference/api/mkldnn_quantizer.cc index 46ae4624ea9e8..76222b84d4624 100644 --- a/paddle/fluid/inference/api/mkldnn_quantizer.cc +++ b/paddle/fluid/inference/api/mkldnn_quantizer.cc @@ -78,7 +78,7 @@ void AnalysisPredictor::MkldnnQuantizer::CalculateScalesForRNNWeights( check_var(wh_var, wh_name); phi::DenseTensor* wx_tensor = wx_var->GetMutable(); phi::DenseTensor* wh_tensor = wh_var->GetMutable(); - if (gru) { + if (gru) { // NOLINT scales_[wx_name] = GetMaxChGRUScalingFactor(*wx_tensor, *wh_tensor); } else { scales_[wx_name] = GetMaxChLSTMScalingFactor(*wx_tensor, *wh_tensor); @@ -215,6 +215,7 @@ void AnalysisPredictor::MkldnnQuantizer::CalculateSingleScale( switch (rule) { case ScaleAlgo::MAX: + case ScaleAlgo::KL: scales_[var_name] = GetMaxScalingFactor(var_tensor, is_unsigned); break; case ScaleAlgo::MAX_CH: @@ -227,9 +228,6 @@ void AnalysisPredictor::MkldnnQuantizer::CalculateSingleScale( is_unsigned, /*is_transposed*/ true); break; - case ScaleAlgo::KL: - scales_[var_name] = GetKLScalingFactor(var_tensor, is_unsigned); - break; default: throw std::runtime_error( "MkldnnQuantizer: Unexpected ScaleAlgo specified."); diff --git a/paddle/fluid/ir_adaptor/translator/op_translator.cc b/paddle/fluid/ir_adaptor/translator/op_translator.cc index b7081609f2f90..bf5acda9c1bbd 100644 --- a/paddle/fluid/ir_adaptor/translator/op_translator.cc +++ b/paddle/fluid/ir_adaptor/translator/op_translator.cc @@ -1903,7 +1903,7 @@ struct FillConstant2FullTranscriber : public OpTranscriber { } } switch (place_type) { - case -1: + case -1: // NOLINT attribute_map["place"] = paddle::dialect::PlaceAttribute::get( ctx, phi::Place(phi::AllocationType::UNDEFINED)); break; diff --git a/paddle/fluid/jit/property.cc b/paddle/fluid/jit/property.cc index 687468df83a3d..37c426bb5401b 100644 --- a/paddle/fluid/jit/property.cc +++ b/paddle/fluid/jit/property.cc @@ -99,7 +99,7 @@ std::unordered_map> Property::Values() { case ValueProto::STRING: *var->GetMutable() = GetString(n); break; - case ValueProto::FLOATS: + case ValueProto::FLOATS: // NOLINT *var->GetMutable>() = GetFloats(n); break; case ValueProto::INTS: diff --git a/paddle/fluid/operators/reader/buffered_reader.cc b/paddle/fluid/operators/reader/buffered_reader.cc index b73ffe4319be7..cc5034c86f90f 100644 --- a/paddle/fluid/operators/reader/buffered_reader.cc +++ b/paddle/fluid/operators/reader/buffered_reader.cc @@ -380,7 +380,7 @@ void BufferedReader::ReadNextImpl(paddle::framework::LoDTensorArray *out) { return; } - if (platform::is_gpu_place(place_)) { + if (platform::is_gpu_place(place_)) { // NOLINT *out = std::move(cuda_buffer_[i]); } else if (platform::is_xpu_place(place_)) { *out = std::move(xpu_buffer_[i]); diff --git a/paddle/fluid/pir/drr/src/ir_operation_factory.cc b/paddle/fluid/pir/drr/src/ir_operation_factory.cc index f792ccbdaff92..61c12c281e139 100644 --- a/paddle/fluid/pir/drr/src/ir_operation_factory.cc +++ b/paddle/fluid/pir/drr/src/ir_operation_factory.cc @@ -81,7 +81,7 @@ pir::Attribute CreateIrAttribute(const std::any& obj) { std::any_cast(obj)); } else if (obj.type() == typeid(phi::Place)) { return IrAttrbuteCreator()(std::any_cast(obj)); - } else if (obj.type() == typeid(std::vector)) { + } else if (obj.type() == typeid(std::vector)) { // NOLINT return IrAttrbuteCreator>()( std::any_cast>(obj)); } else if (obj.type() == typeid(std::vector)) { diff --git a/paddle/fluid/platform/place.cc b/paddle/fluid/platform/place.cc index 118ba7d6b782c..df66cc63e3986 100644 --- a/paddle/fluid/platform/place.cc +++ b/paddle/fluid/platform/place.cc @@ -62,8 +62,6 @@ bool is_same_place(const Place &p1, const Place &p2) { if (places_are_same_class(p1, p2)) { if (is_cpu_place(p1) || is_cuda_pinned_place(p1)) { return true; - } else if (is_xpu_place(p1) || is_ipu_place(p1) || is_custom_place(p1)) { - return p1 == p2; } else { return p1 == p2; } diff --git a/paddle/fluid/platform/profiler.cc b/paddle/fluid/platform/profiler.cc index 816ae57ff4c06..2630b36d0e8ad 100644 --- a/paddle/fluid/platform/profiler.cc +++ b/paddle/fluid/platform/profiler.cc @@ -200,8 +200,8 @@ RecordMemEvent::RecordMemEvent(const void *ptr, RecordMemEvent::size_cache["gpu"][place.GetDeviceId()][3]; RecordMemEvent::has_initialized["gpu"][place.GetDeviceId()] = true; } else { - current_allocated = - DEVICE_MEMORY_STAT_CURRENT_VALUE(Allocated, place.GetDeviceId()); + current_allocated = DEVICE_MEMORY_STAT_CURRENT_VALUE( + Allocated, place.GetDeviceId()); // NOLINT peak_allocated = DEVICE_MEMORY_STAT_PEAK_VALUE(Allocated, place.GetDeviceId()); RecordMemEvent::size_cache["gpu"][place.GetDeviceId()][0] = @@ -283,10 +283,10 @@ RecordMemEvent::RecordMemEvent(const void *ptr, RecordMemEvent::size_cache["gpu"][place.GetDeviceId()][3]; RecordMemEvent::has_initialized["gpu"][place.GetDeviceId()] = true; } else { - current_reserved = - DEVICE_MEMORY_STAT_CURRENT_VALUE(Reserved, place.GetDeviceId()); - peak_reserved = - DEVICE_MEMORY_STAT_PEAK_VALUE(Reserved, place.GetDeviceId()); + current_reserved = DEVICE_MEMORY_STAT_CURRENT_VALUE( + Reserved, place.GetDeviceId()); // NOLINT + peak_reserved = DEVICE_MEMORY_STAT_PEAK_VALUE( + Reserved, place.GetDeviceId()); // NOLINT RecordMemEvent::size_cache["gpu"][place.GetDeviceId()][1] = current_reserved; RecordMemEvent::size_cache["gpu"][place.GetDeviceId()][3] = @@ -366,10 +366,10 @@ RecordMemEvent::RecordMemEvent(const void *ptr, RecordMemEvent::size_cache["gpu"][place.GetDeviceId()][3]; RecordMemEvent::has_initialized["gpu"][place.GetDeviceId()] = true; } else { - current_allocated = - DEVICE_MEMORY_STAT_CURRENT_VALUE(Allocated, place.GetDeviceId()); - peak_allocated = - DEVICE_MEMORY_STAT_PEAK_VALUE(Allocated, place.GetDeviceId()); + current_allocated = DEVICE_MEMORY_STAT_CURRENT_VALUE( + Allocated, place.GetDeviceId()); // NOLINT + peak_allocated = DEVICE_MEMORY_STAT_PEAK_VALUE( + Allocated, place.GetDeviceId()); // NOLINT RecordMemEvent::size_cache["gpu"][place.GetDeviceId()][0] = current_allocated; RecordMemEvent::size_cache["gpu"][place.GetDeviceId()][2] = @@ -449,10 +449,10 @@ RecordMemEvent::RecordMemEvent(const void *ptr, RecordMemEvent::size_cache["gpu"][place.GetDeviceId()][3]; RecordMemEvent::has_initialized["gpu"][place.GetDeviceId()] = true; } else { - current_reserved = - DEVICE_MEMORY_STAT_CURRENT_VALUE(Reserved, place.GetDeviceId()); - peak_reserved = - DEVICE_MEMORY_STAT_PEAK_VALUE(Reserved, place.GetDeviceId()); + current_reserved = DEVICE_MEMORY_STAT_CURRENT_VALUE( + Reserved, place.GetDeviceId()); // NOLINT + peak_reserved = DEVICE_MEMORY_STAT_PEAK_VALUE( + Reserved, place.GetDeviceId()); // NOLINT RecordMemEvent::size_cache["gpu"][place.GetDeviceId()][1] = current_reserved; RecordMemEvent::size_cache["gpu"][place.GetDeviceId()][3] = diff --git a/paddle/fluid/pybind/eager.cc b/paddle/fluid/pybind/eager.cc index 3cb3ccf964ec8..00b6ba994233f 100644 --- a/paddle/fluid/pybind/eager.cc +++ b/paddle/fluid/pybind/eager.cc @@ -442,7 +442,7 @@ Placements ParsePlacementsArgs( Placements placements; const std::string& placements_key = "placements"; - if (kw_order_map[placements_key] <= args_num) { + if (kw_order_map[placements_key] <= args_num) { // NOLINT placements = CastPyArg2VectorOfPlacement( PyTuple_GET_ITEM(args, kw_order_map[placements_key] - 1), kw_order_map[placements_key] - 1); diff --git a/paddle/fluid/pybind/eager_functions.cc b/paddle/fluid/pybind/eager_functions.cc index 0a72208f36ccc..812be85b653af 100644 --- a/paddle/fluid/pybind/eager_functions.cc +++ b/paddle/fluid/pybind/eager_functions.cc @@ -644,7 +644,7 @@ PyObject* eager_api_run_custom_op(PyObject* self, } else if (attr_type_str == "std::string") { ctx.EmplaceBackAttr( CastPyArg2AttrString(obj, attr_start_idx + i)); // NOLINT - } else if (attr_type_str == "std::vector") { + } else if (attr_type_str == "std::vector") { // NOLINT ctx.EmplaceBackAttr(CastPyArg2VectorOfInt(obj, attr_start_idx + i)); } else if (attr_type_str == "std::vector") { ctx.EmplaceBackAttr(CastPyArg2VectorOfFloat(obj, attr_start_idx + i)); diff --git a/paddle/fluid/pybind/eager_math_op_patch.cc b/paddle/fluid/pybind/eager_math_op_patch.cc index 21fd549cb0b2d..17b36e9237e78 100644 --- a/paddle/fluid/pybind/eager_math_op_patch.cc +++ b/paddle/fluid/pybind/eager_math_op_patch.cc @@ -818,10 +818,10 @@ static PyObject* tensor__rdiv__method(TensorObject* self, bool has_other_double = false; if (PyFloat_Check(other_obj) || PyCheckInteger(other_obj) || IsNumpyType(other_obj)) { - if (PyFloat_Check(other_obj)) { + if (PyFloat_Check(other_obj)) { // NOLINT other_double = CastPyArg2Double(other_obj, "__rdiv__", 0); has_other_double = true; - } else if (PyCheckInteger(other_obj) || IsNumpyType(other_obj)) { + } else if (PyCheckInteger(other_obj) || IsNumpyType(other_obj)) { // NOLINT other_double = CastPyArg2Double(other_obj, "__rdiv__", 0); has_other_double = true; } diff --git a/paddle/fluid/pybind/eager_utils.cc b/paddle/fluid/pybind/eager_utils.cc index c6a2db061594b..851e498bac8b3 100644 --- a/paddle/fluid/pybind/eager_utils.cc +++ b/paddle/fluid/pybind/eager_utils.cc @@ -647,7 +647,7 @@ std::vector> CastPyArg2VectorOfVectorOfSize_t( platform::Place CastPyArg2Place(PyObject* obj, ssize_t arg_pos) { platform::Place place; - if (PyObject_TypeCheck(obj, g_place_pytype)) { + if (PyObject_TypeCheck(obj, g_place_pytype)) { // NOLINT place = ::pybind11::handle(obj).cast(); } else if (PyObject_TypeCheck(obj, g_cudaplace_pytype)) { place = ::pybind11::handle(obj).cast(); @@ -761,7 +761,8 @@ std::vector CastPyArg2VectorOfTensorBase(PyObject* obj, i)); } } - } else if (PyObject_TypeCheck(obj, g_framework_lodtensorarray_pytype)) { + } else if (PyObject_TypeCheck(obj, + g_framework_lodtensorarray_pytype)) { // NOLINT for (auto& tensor : (::pybind11::handle(obj).cast())) { result.emplace_back(tensor); @@ -788,7 +789,7 @@ using phi::distributed::Shard; Placements CastPyArg2VectorOfPlacement(PyObject* obj, ssize_t arg_pos) { Placements result; auto check_and_emplace = [&](PyObject* item, ssize_t i) { - if (PyObject_TypeCheck(item, g_placement_shard_pytype)) { + if (PyObject_TypeCheck(item, g_placement_shard_pytype)) { // NOLINT result.emplace_back( std::make_shared(::pybind11::handle(item).cast())); } else if (PyObject_TypeCheck(item, g_placement_replicated_pytype)) { diff --git a/paddle/fluid/pybind/parallel_executor.cc b/paddle/fluid/pybind/parallel_executor.cc index 9060e158c9ed9..1b567fb51ba1e 100644 --- a/paddle/fluid/pybind/parallel_executor.cc +++ b/paddle/fluid/pybind/parallel_executor.cc @@ -931,7 +931,7 @@ void BindParallelExecutor(pybind11::module &m) { // NOLINT .def_property( "memory_optimize", [](const BuildStrategy &self) -> py::object { - if (self.memory_optimize_) { + if (self.memory_optimize_) { // NOLINT return py::cast(self.memory_optimize_.get()); } else { return py::cast(nullptr); diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index ffaef54bb9da9..1d71676ba4314 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -1243,7 +1243,7 @@ All parameter, weight, gradient are variables in Paddle. py::return_value_policy::reference) .def("get_bytes", [](Variable &self) { - if (self.IsType()) { + if (self.IsType()) { // NOLINT return py::bytes(*(self.GetMutable())); } else { return py::bytes( @@ -2232,7 +2232,7 @@ All parameter, weight, gradient are variables in Paddle. const std::string &var_name, size_t index) -> py::object { auto &var = framework::GetFetchVariable(scope, var_name, index); - if (data_is_lod_tensor(var)) { + if (data_is_lod_tensor(var)) { // NOLINT return py::cast(PADDLE_GET(phi::DenseTensor, var)); } else { return py::cast(PADDLE_GET(LoDTensorArray, var)); diff --git a/paddle/phi/core/compat/convert_utils.cc b/paddle/phi/core/compat/convert_utils.cc index d4c5de0dbe6dc..37053cc0c09ec 100644 --- a/paddle/phi/core/compat/convert_utils.cc +++ b/paddle/phi/core/compat/convert_utils.cc @@ -63,6 +63,7 @@ phi::Place TransToPhiPlace(const Backend& backend, bool set_device_id) { return phi::Place(); #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) case phi::Backend::GPU: + case phi::Backend::GPUDNN: return phi::GPUPlace( set_device_id ? phi::backends::gpu::GetCurrentDeviceId() : 0); #endif @@ -70,11 +71,6 @@ phi::Place TransToPhiPlace(const Backend& backend, bool set_device_id) { case phi::Backend::ONEDNN: // NOLINT return phi::CPUPlace(); #endif -#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) - case phi::Backend::GPUDNN: - return phi::GPUPlace( - set_device_id ? phi::backends::gpu::GetCurrentDeviceId() : 0); -#endif #if defined(PADDLE_WITH_XPU) case phi::Backend::XPU: return phi::XPUPlace( diff --git a/paddle/phi/core/kernel_registry.cc b/paddle/phi/core/kernel_registry.cc index fa9d531b6534d..6ce1af187e9a3 100644 --- a/paddle/phi/core/kernel_registry.cc +++ b/paddle/phi/core/kernel_registry.cc @@ -47,139 +47,159 @@ void SetKernelArgsDef(const std::vector& args_type, ) { #endif // do nothing, skip context arg now - } else if (arg_type == std::type_index(typeid(const DenseTensor&))) { + } else if (arg_type == + std::type_index(typeid(const DenseTensor&))) { // NOLINT args_def->AppendInput(default_key.backend(), default_tensor_layout, default_key.dtype(), arg_type); } else if (arg_type == - std::type_index(typeid(const paddle::optional&))) { + std::type_index( + typeid(const paddle::optional&))) { // NOLINT args_def->AppendInput(default_key.backend(), default_tensor_layout, default_key.dtype(), arg_type); } else if (arg_type == - std::type_index(typeid( - const paddle::optional>&))) { + std::type_index( + typeid(const paddle::optional< + std::vector>&))) { // NOLINT args_def->AppendInput(default_key.backend(), default_tensor_layout, default_key.dtype(), arg_type); } else if (arg_type == - std::type_index(typeid(const paddle::optional&))) { + std::type_index( + typeid(const paddle::optional&))) { // NOLINT args_def->AppendInput(default_key.backend(), default_tensor_layout, default_key.dtype(), arg_type); - } else if (arg_type == std::type_index(typeid( - const std::vector&))) { + } else if (arg_type == + std::type_index( + typeid(const std::vector&))) { // NOLINT args_def->AppendInput(default_key.backend(), default_tensor_layout, default_key.dtype(), arg_type); } else if (arg_type == - std::type_index(typeid(const phi::ExtendedTensor&))) { + std::type_index(typeid(const phi::ExtendedTensor&))) { // NOLINT args_def->AppendInput(default_key.backend(), default_tensor_layout, default_key.dtype(), arg_type); - } else if (arg_type == std::type_index(typeid( - const std::vector&))) { + } else if (arg_type == + std::type_index(typeid( + const std::vector&))) { // NOLINT args_def->AppendInput(default_key.backend(), default_tensor_layout, default_key.dtype(), arg_type); - } else if (arg_type == std::type_index(typeid( - const std::vector&))) { + } else if (arg_type == + std::type_index(typeid( + const std::vector&))) { // NOLINT args_def->AppendInput(default_key.backend(), default_tensor_layout, default_key.dtype(), arg_type); } else if (arg_type == - std::type_index(typeid(const std::vector&))) { + std::type_index( + typeid(const std::vector&))) { // NOLINT args_def->AppendInput(default_key.backend(), default_tensor_layout, default_key.dtype(), arg_type); - } else if (arg_type == std::type_index(typeid( - const std::vector&))) { + } else if (arg_type == + std::type_index( + typeid(const std::vector&))) { // NOLINT args_def->AppendInput(default_key.backend(), default_tensor_layout, default_key.dtype(), arg_type); - } else if (arg_type == std::type_index(typeid(const SelectedRows&))) { + } else if (arg_type == + std::type_index(typeid(const SelectedRows&))) { // NOLINT args_def->AppendInput(default_key.backend(), default_tensor_layout, default_key.dtype(), arg_type); - } else if (arg_type == std::type_index(typeid(const StringTensor&))) { + } else if (arg_type == + std::type_index(typeid(const StringTensor&))) { // NOLINT args_def->AppendInput(default_key.backend(), default_tensor_layout, default_key.dtype(), arg_type); - } else if (arg_type == std::type_index(typeid(const SparseCooTensor&))) { + } else if (arg_type == + std::type_index(typeid(const SparseCooTensor&))) { // NOLINT args_def->AppendInput(default_key.backend(), default_tensor_layout, default_key.dtype(), arg_type); - } else if (arg_type == std::type_index(typeid( - paddle::optional))) { + } else if (arg_type == + std::type_index(typeid( + paddle::optional))) { // NOLINT args_def->AppendInput(default_key.backend(), default_tensor_layout, default_key.dtype(), arg_type); - } else if (arg_type == std::type_index(typeid(const SparseCsrTensor&))) { + } else if (arg_type == + std::type_index(typeid(const SparseCsrTensor&))) { // NOLINT args_def->AppendInput(default_key.backend(), default_tensor_layout, default_key.dtype(), arg_type); - } else if (arg_type == std::type_index(typeid( - paddle::optional))) { + } else if (arg_type == + std::type_index(typeid( + paddle::optional))) { // NOLINT args_def->AppendInput(default_key.backend(), default_tensor_layout, default_key.dtype(), arg_type); - } else if (arg_type == std::type_index(typeid(const TensorArray&))) { + } else if (arg_type == + std::type_index(typeid(const TensorArray&))) { // NOLINT args_def->AppendInput(default_key.backend(), default_tensor_layout, default_key.dtype(), arg_type); - } else if (arg_type == std::type_index(typeid(DenseTensor*))) { + } else if (arg_type == std::type_index(typeid(DenseTensor*))) { // NOLINT args_def->AppendOutput(default_key.backend(), default_tensor_layout, default_key.dtype(), arg_type); - } else if (arg_type == std::type_index(typeid(std::vector))) { + } else if (arg_type == + std::type_index(typeid(std::vector))) { // NOLINT args_def->AppendOutput(default_key.backend(), default_tensor_layout, default_key.dtype(), arg_type); - } else if (arg_type == std::type_index(typeid(SelectedRows*))) { + } else if (arg_type == std::type_index(typeid(SelectedRows*))) { // NOLINT args_def->AppendOutput(default_key.backend(), default_tensor_layout, default_key.dtype(), arg_type); - } else if (arg_type == std::type_index(typeid(TensorArray*))) { + } else if (arg_type == std::type_index(typeid(TensorArray*))) { // NOLINT args_def->AppendOutput(default_key.backend(), default_tensor_layout, default_key.dtype(), arg_type); - } else if (arg_type == std::type_index(typeid(SparseCooTensor*))) { + } else if (arg_type == + std::type_index(typeid(SparseCooTensor*))) { // NOLINT args_def->AppendOutput(default_key.backend(), default_tensor_layout, default_key.dtype(), arg_type); - } else if (arg_type == std::type_index(typeid(SparseCsrTensor*))) { + } else if (arg_type == + std::type_index(typeid(SparseCsrTensor*))) { // NOLINT args_def->AppendOutput(default_key.backend(), default_tensor_layout, default_key.dtype(), arg_type); - } else if (arg_type == std::type_index(typeid(StringTensor*))) { + } else if (arg_type == std::type_index(typeid(StringTensor*))) { // NOLINT args_def->AppendOutput(default_key.backend(), default_tensor_layout, default_key.dtype(), arg_type); - } else if (arg_type == std::type_index(typeid(ExtendedTensor*))) { + } else if (arg_type == + std::type_index(typeid(ExtendedTensor*))) { // NOLINT args_def->AppendOutput(default_key.backend(), default_tensor_layout, default_key.dtype(), diff --git a/paddle/phi/infermeta/unary.cc b/paddle/phi/infermeta/unary.cc index 5648ff0d469a3..b064a9f73bad6 100644 --- a/paddle/phi/infermeta/unary.cc +++ b/paddle/phi/infermeta/unary.cc @@ -236,7 +236,7 @@ void ArgMinMaxInferMeta(const MetaTensor& x, if (!config.is_runtime && axis.FromTensor()) { std::vector vec; if (flatten) { - if (keepdims) { + if (keepdims) { // NOLINT vec = std::vector(x.dims().size(), -1); } else { vec = {}; @@ -307,7 +307,7 @@ void ArgMinMaxInferMeta(const MetaTensor& x, std::vector vec; if (flatten) { - if (keepdims) { + if (keepdims) { // NOLINT vec = std::vector(x.dims().size(), 1); } else { vec = {}; @@ -4034,7 +4034,8 @@ void SplitInferMeta(const MetaTensor& x, if ((sections.FromTensor() && !config.is_runtime) || axis_value == -1 || (axis_value >= 0 && x.dims().at(axis_value) <= 0)) { std::vector out_dims; - if ((sections.FromTensor() && !config.is_runtime) || axis_value == -1) { + if ((sections.FromTensor() && !config.is_runtime) || + axis_value == -1) { // NOLINT out_dims = std::vector( sections_data.size(), common::make_ddim(std::vector(x.dims().size(), -1))); @@ -4126,7 +4127,7 @@ void SplitWithNumInferMeta(const MetaTensor& x, // fill out dims with -1 if (axis_value == -1 || (axis_value >= 0 && x.dims().at(axis_value) <= 0)) { std::vector out_dims; - if (axis_value == -1) { + if (axis_value == -1) { // NOLINT out_dims = std::vector( num, common::make_ddim(std::vector(x.dims().size(), -1))); } else { @@ -5415,7 +5416,7 @@ void WeightQuantizeInferMeta(const MetaTensor& x, } std::vector dim_out; - if (algo == "weight_only_int8" || algo == "llm.int8") { + if (algo == "weight_only_int8" || algo == "llm.int8") { // NOLINT dim_out = std::vector({x_dims[1], x_dims[0]}); } else if (algo == "weight_only_int4") { dim_out = std::vector({x_dims[1] / 2, x_dims[0]}); diff --git a/paddle/phi/kernels/cpu/batch_norm_grad_kernel.cc b/paddle/phi/kernels/cpu/batch_norm_grad_kernel.cc index 1bdf25dd4eb82..e9c5ae6a39e4a 100644 --- a/paddle/phi/kernels/cpu/batch_norm_grad_kernel.cc +++ b/paddle/phi/kernels/cpu/batch_norm_grad_kernel.cc @@ -611,7 +611,7 @@ void BatchNormDoubleGradKernel( EigenArrayMap ddy_arr( ctx.template Alloc(&transformed_ddy), C, sample_size); ddy_arr.setZero(); - if (use_global_stats) { + if (use_global_stats) { // NOLINT // math: ddy = r * ddx * inv_var + ddbias + // ddscale * (x - mean) * inv_var if (ddX) { diff --git a/paddle/phi/kernels/cpu/batch_norm_kernel.cc b/paddle/phi/kernels/cpu/batch_norm_kernel.cc index 39d53fec10a9f..f6d5e97dc7245 100644 --- a/paddle/phi/kernels/cpu/batch_norm_kernel.cc +++ b/paddle/phi/kernels/cpu/batch_norm_kernel.cc @@ -159,7 +159,7 @@ void BatchNormKernel(const Context& ctx, // use SavedMean and SavedVariance to do normalize Eigen::Array inv_std(C); - if (global_stats) { + if (global_stats) { // NOLINT ConstEigenVectorArrayMap var_arr(variance.data(), C); inv_std = (var_arr + epsilon).sqrt().inverse(); } else { @@ -178,7 +178,7 @@ void BatchNormKernel(const Context& ctx, auto* Bias = bias.get_ptr(); Eigen::Array new_scale(C); Eigen::Array new_bias(C); - if (Scale && Bias) { + if (Scale && Bias) { // NOLINT ConstEigenVectorArrayMap scale_arr(Scale->data(), C); ConstEigenVectorArrayMap bias_arr(Bias->data(), C); new_scale = inv_std * scale_arr; diff --git a/paddle/phi/kernels/cpu/elementwise_divide_kernel.cc b/paddle/phi/kernels/cpu/elementwise_divide_kernel.cc index b7fdefe023e73..ed80148344e1f 100644 --- a/paddle/phi/kernels/cpu/elementwise_divide_kernel.cc +++ b/paddle/phi/kernels/cpu/elementwise_divide_kernel.cc @@ -35,7 +35,7 @@ void DivideKernel(const Context& dev_ctx, } else { auto x_dims = x.dims(); auto y_dims = y.dims(); - if (x_dims.size() >= y_dims.size()) { + if (x_dims.size() >= y_dims.size()) { // NOLINT funcs::ElementwiseCompute, T>( dev_ctx, x, y, funcs::DivideFunctor(), out, -1); } else { diff --git a/paddle/phi/kernels/cpu/rnn_grad_kernel.cc b/paddle/phi/kernels/cpu/rnn_grad_kernel.cc index a48d05b8d783e..8b26bf31de9bb 100644 --- a/paddle/phi/kernels/cpu/rnn_grad_kernel.cc +++ b/paddle/phi/kernels/cpu/rnn_grad_kernel.cc @@ -1311,7 +1311,7 @@ void RnnGradKernel(const Context& dev_ctx, pre_state_grad, weight_grad_list); // run gru - } else if (is_rnn_relu(mode)) { + } else if (is_rnn_relu(mode)) { // NOLINT gate_num = 1; RnnGradFunc, SingleGradLayer, diff --git a/paddle/phi/kernels/cpu/rnn_kernel.cc b/paddle/phi/kernels/cpu/rnn_kernel.cc index a0035c6db4a75..5b594089793c8 100644 --- a/paddle/phi/kernels/cpu/rnn_kernel.cc +++ b/paddle/phi/kernels/cpu/rnn_kernel.cc @@ -868,7 +868,7 @@ void RnnKernel(const Context& dev_ctx, is_test, seed, reserve); - } else if (is_rnn_relu(mode)) { + } else if (is_rnn_relu(mode)) { // NOLINT gate_num = 1; RnnFunc { int64_t h = static_cast(lod[i + 1] - lod[i]); auto in_e = EigenMatrix::From(in_t, common::make_ddim({h, w})); auto out_e = EigenVector::Flatten(out_t); - if (pooltype == "AVERAGE") { + if (pooltype == "AVERAGE") { // NOLINT out_e.device(place) = in_e.mean(Eigen::array({{0}})); } else if (pooltype == "SQRT") { out_e.device(place) = in_e.sum(Eigen::array({{0}})) / diff --git a/paddle/phi/kernels/legacy/cpu/elementwise_kernel.cc b/paddle/phi/kernels/legacy/cpu/elementwise_kernel.cc index dafbf2889277d..84ebbf04fee11 100644 --- a/paddle/phi/kernels/legacy/cpu/elementwise_kernel.cc +++ b/paddle/phi/kernels/legacy/cpu/elementwise_kernel.cc @@ -55,7 +55,7 @@ void RemainderRawKernel(const Context& dev_ctx, dev_ctx.template Alloc(out); auto x_dims = x.dims(); auto y_dims = y.dims(); - if (x_dims.size() >= y_dims.size()) { + if (x_dims.size() >= y_dims.size()) { // NOLINT funcs::ElementwiseCompute, T>( dev_ctx, x, y, funcs::RemainderFunctor(), out, axis); } else { @@ -74,7 +74,7 @@ void FloorDivideRawKernel(const Context& dev_ctx, dev_ctx.template Alloc(out); auto x_dims = x.dims(); auto y_dims = y.dims(); - if (x_dims.size() >= y_dims.size()) { + if (x_dims.size() >= y_dims.size()) { // NOLINT funcs::ElementwiseCompute, T>( dev_ctx, x, y, funcs::FloorDivideFunctor(), out, axis); } else { diff --git a/test/cpp/fluid/framework/details/fused_broadcast_op_handle_test.cc b/test/cpp/fluid/framework/details/fused_broadcast_op_handle_test.cc index 786b857a80dcc..aee187d77f484 100644 --- a/test/cpp/fluid/framework/details/fused_broadcast_op_handle_test.cc +++ b/test/cpp/fluid/framework/details/fused_broadcast_op_handle_test.cc @@ -56,7 +56,7 @@ struct TestFusedBroadcastOpHandle : TestBroadcastOpHandle { // create op handle node nodes_.emplace_back( ir::CreateNodeForTest("fused_broadcast", ir::Node::Type::kOperation)); - if (use_device_ == p::kCUDA) { + if (use_device_ == p::kCUDA) { // NOLINT #if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL) op_handle_ = new FusedBroadcastOpHandle( nodes_.back().get(), local_scopes_, place_list_, nccl_ctxs_.get()); diff --git a/test/cpp/imperative/test_gradient_accmulator.cc b/test/cpp/imperative/test_gradient_accmulator.cc index b7b571fa196ad..12e2325873c47 100644 --- a/test/cpp/imperative/test_gradient_accmulator.cc +++ b/test/cpp/imperative/test_gradient_accmulator.cc @@ -376,7 +376,7 @@ static framework::Variable RandomSelectedRows(framework::DDim dims, static std::unique_ptr CreateAccumulator( const std::shared_ptr& var, bool sort_gradient) { - if (sort_gradient) { + if (sort_gradient) { // NOLINT return std::unique_ptr( new SortedGradientAccumulator(var.get())); } else { @@ -400,7 +400,7 @@ static void TestGradientAccumulatorTestUnchangeInput( std::mt19937 engine(seed); auto create_var = [&](bool use_tensor) { - if (use_tensor) { + if (use_tensor) { // NOLINT return RandomTensor(dim, place); } else { return RandomSelectedRows(dim, place, dist(engine)); From 8d1d18f09906f82aebfae2eb1bf404d36633ecd5 Mon Sep 17 00:00:00 2001 From: zyfncg Date: Fri, 1 Mar 2024 11:02:46 +0800 Subject: [PATCH 215/282] [CINN] Add test for llama inference (#62153) * fix cmake patch command to avoid patching twice error * add test for infer llama * fix bug of test * fix bug * revert other commit * add llama forward test * pulish log * remove shape pass flag --------- Co-authored-by: Silver Ling --- test/ir/pir/cinn/CMakeLists.txt | 1 + test/ir/pir/cinn/inference/CMakeLists.txt | 23 + .../pir/cinn/inference/test_llama_forward.py | 687 ++++++++++++++++++ .../cinn/inference/test_llama_postprocess.py | 123 ++++ 4 files changed, 834 insertions(+) create mode 100644 test/ir/pir/cinn/inference/CMakeLists.txt create mode 100644 test/ir/pir/cinn/inference/test_llama_forward.py create mode 100644 test/ir/pir/cinn/inference/test_llama_postprocess.py diff --git a/test/ir/pir/cinn/CMakeLists.txt b/test/ir/pir/cinn/CMakeLists.txt index 3daedfb5b4f6e..7a7d98dc37ba3 100644 --- a/test/ir/pir/cinn/CMakeLists.txt +++ b/test/ir/pir/cinn/CMakeLists.txt @@ -1,5 +1,6 @@ add_subdirectory(adt) add_subdirectory(symbolic) +add_subdirectory(inference) add_subdirectory(sub_graphs) if(WITH_GPU) diff --git a/test/ir/pir/cinn/inference/CMakeLists.txt b/test/ir/pir/cinn/inference/CMakeLists.txt new file mode 100644 index 0000000000000..c5ff7c9573d5e --- /dev/null +++ b/test/ir/pir/cinn/inference/CMakeLists.txt @@ -0,0 +1,23 @@ +if(WITH_GPU) + file( + GLOB CINN_PIR_INFER_TEST + RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" + "test_*.py") + + foreach(cinn_pir_test_name ${CINN_PIR_INFER_TEST}) + string(REGEX REPLACE ".py" "" cinn_pir_test_name ${cinn_pir_test_name}) + add_test( + NAME ${cinn_pir_test_name} + COMMAND + ${CMAKE_COMMAND} -E env + PYTHONPATH=${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/python/:$ENV{PYTHONPATH} + FLAGS_prim_enable_dynamic=True FLAGS_prim_all=True + FLAGS_enable_pir_api=1 FLAGS_cinn_bucket_compile=True + ${PYTHON_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/${cinn_pir_test_name}.py + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + set_tests_properties(${cinn_pir_test_name} PROPERTIES LABELS + "RUN_TYPE=CINN") + endforeach() + +endif() diff --git a/test/ir/pir/cinn/inference/test_llama_forward.py b/test/ir/pir/cinn/inference/test_llama_forward.py new file mode 100644 index 0000000000000..7c456ce3921d4 --- /dev/null +++ b/test/ir/pir/cinn/inference/test_llama_forward.py @@ -0,0 +1,687 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import sys +import unittest +from os.path import dirname +from typing import Optional, Tuple + +import numpy as np + +import paddle +import paddle.nn.functional as F +from paddle import nn +from paddle.incubate.nn.functional import swiglu +from paddle.static import InputSpec + +sys.path.append(dirname(dirname(__file__))) + +import utils + + +class LlamaConfig: + def __init__( + self, + vocab_size=32000, + hidden_size=4096, + intermediate_size=11008, + max_position_embeddings=2048, + seq_length=2048, + num_hidden_layers=1, + num_attention_heads=32, + num_key_value_heads=32, + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + ): + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.max_position_embeddings = max_position_embeddings + self.seq_length = seq_length + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.num_key_value_heads = num_key_value_heads + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + + +class LlamaRotaryEmbedding(nn.Layer): + def __init__(self, dim, max_position_embeddings=2048, base=10000): + super().__init__() + self.dim = dim + self.max_position_embeddings = max_position_embeddings + self.base = base + # [dim / 2] + self.inv_freq = 1.0 / ( + self.base + ** ( + paddle.cast(paddle.arange(0, self.dim, 2), dtype="float32") + / self.dim + ) + ) + self._set_cos_sin_cache(seq_len=max_position_embeddings) + + def _set_cos_sin_cache(self, seq_len): + self.max_seq_len_cached = seq_len + # [seq_len] + t = paddle.arange(seq_len, dtype="float32") + # [seq_len, dim/2] + freqs = paddle.einsum("i,j->ij", t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + # [seq_len, dim] + emb = paddle.concat([freqs, freqs], axis=-1) + # [1, seqlen, 1, dim] + self.cos_cached = emb.cos()[None, :, None, :] + self.sin_cached = emb.sin()[None, :, None, :] + + def forward(self, x, seq_len=None): + # x: [bs, num_attention_heads, seq_len, head_size] + cos = self.cos_cached[:, :seq_len, :, :] + sin = self.sin_cached[:, :seq_len, :, :] + return ( + cos.cast(x.dtype) if cos.dtype != x.dtype else cos, + sin.cast(x.dtype) if sin.dtype != x.dtype else sin, + ) + + +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return paddle.concat([-x2, x1], axis=-1) # shape is the same as x + + +def apply_rotary_pos_emb(q, k, cos, sin, position_ids): + if position_ids is None: + # Note: Only for LlamaForCausalLMPipe model pretraining + cos = cos[:, : q.shape[1], :, :] # [bs, seq_len, 1, dim] + sin = sin[:, : q.shape[1], :, :] # [bs, seq_len, 1, dim] + else: + cos = cos.squeeze(axis=[0, 2]) # [seq_len, dim] + sin = sin.squeeze(axis=[0, 2]) # [seq_len, dim] + cos = cos[position_ids].unsqueeze(2) # [bs, seq_len, 1, dim] + sin = sin[position_ids].unsqueeze(2) # [bs, seq_len, 1, dim] + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +def _make_causal_mask(input_ids_shape, past_key_values_length): + """ + Make causal mask used for self-attention + """ + batch_size, target_length = input_ids_shape # target_length: seq_len + + mask = paddle.tril( + paddle.ones((target_length, target_length), dtype="bool") + ) + + if past_key_values_length > 0: + # [tgt_len, tgt_len + past_len] + mask = paddle.concat( + [ + paddle.ones( + [target_length, past_key_values_length], dtype="bool" + ), + mask, + ], + axis=-1, + ) + + # [bs, 1, tgt_len, tgt_len + past_len] + return mask[None, None, :, :].expand( + [batch_size, 1, target_length, target_length + past_key_values_length] + ) + + +def _expand_2d_mask(mask, dtype, tgt_length): + """ + Expands attention_mask from `[batch_size, src_length]` to `[batch_size, 1, tgt_length, src_length]`. + """ + batch_size, src_length = mask.shape[0], mask.shape[-1] + tgt_length = tgt_length if tgt_length is not None else src_length + + mask = mask[:, None, None, :].astype("bool") + mask.stop_gradient = True + expanded_mask = mask.expand([batch_size, 1, tgt_length, src_length]) + + return expanded_mask + + +def get_triangle_upper_mask(x, mask=None): + if mask is not None: + return mask + # [bsz, n_head, q_len, kv_seq_len] + shape = x.shape + # [bsz, 1, q_len, kv_seq_len] + shape[1] = 1 + mask = paddle.full(shape, paddle.finfo(x.dtype).min, dtype=x.dtype) + mask = paddle.triu(mask, diagonal=1) + mask.stop_gradient = True + return mask + + +def scaled_dot_product_attention( + query_states, + config, + key_states, + value_states, + attention_mask, + output_attentions, +): + bsz, q_len, num_heads, head_dim = query_states.shape + _, kv_seq_len, _, _ = value_states.shape + + # [ bz, seqlen, nhead, head_dim] -> [bs, nhead, seq_len, head_dim] + query_states = paddle.transpose(query_states, [0, 2, 1, 3]) + # merge with the next tranpose + key_states = paddle.transpose(key_states, [0, 2, 1, 3]) + value_states = paddle.transpose(value_states, [0, 2, 1, 3]) + + # matmul and devide by sqrt(head_dim) + attn_weights = paddle.matmul( + query_states / math.sqrt(head_dim), key_states.transpose([0, 1, 3, 2]) + ) + + # NOTE: we only call get_triangle_upper_mask under PP setup + # FIXME ZHUI when we use pipeline parallel, the attention_mask can be None + # we just make it triangle_upper_mask + if attention_mask is None: + attention_mask = get_triangle_upper_mask(attn_weights) + attention_mask = attention_mask.reshape([bsz, 1, q_len, kv_seq_len]) + + attn_weights = attn_weights + attention_mask + attn_weights = F.softmax(attn_weights, axis=-1, dtype="float32").astype( + query_states.dtype + ) + + attn_output = paddle.matmul(attn_weights, value_states) + attn_output = attn_output.transpose([0, 2, 1, 3]) + + attn_output = attn_output.reshape([bsz, q_len, head_dim * num_heads]) + return (attn_output, attn_weights) if output_attentions else attn_output + + +class LlamaMLP(nn.Layer): + def __init__(self, config): + super().__init__() + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + + self.gate_proj = nn.Linear( + self.hidden_size, self.intermediate_size, bias_attr=False + ) + self.up_proj = nn.Linear( + self.hidden_size, self.intermediate_size, bias_attr=False + ) + self.down_proj = nn.Linear( + self.intermediate_size, self.hidden_size, bias_attr=False + ) + + def forward(self, x): + x = swiglu(self.gate_proj(x), self.up_proj(x)) + out = self.down_proj(x) + return out + + +class LlamaRMSNorm(nn.Layer): + def __init__(self, config): + super().__init__() + self.hidden_size = config.hidden_size + self.weight = paddle.create_parameter( + shape=[self.hidden_size], + dtype=paddle.get_default_dtype(), + default_initializer=nn.initializer.Constant(1.0), + ) + self.variance_epsilon = config.rms_norm_eps + self.config = config + + def forward(self, hidden_states): + hidden_states = hidden_states.astype("float32") + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = ( + paddle.rsqrt(variance + self.variance_epsilon) * hidden_states + ) + + if self.weight.dtype in [paddle.float16, paddle.bfloat16]: + hidden_states = paddle.cast(hidden_states, self.weight.dtype) + return hidden_states * self.weight + + +class LlamaAttention(nn.Layer): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config: LlamaConfig): + super().__init__() + + self.config = config + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + + self.head_dim = self.hidden_size // config.num_attention_heads + + self.num_key_value_heads = config.num_key_value_heads + assert config.num_attention_heads // config.num_key_value_heads + self.num_key_value_groups = ( + config.num_attention_heads // config.num_key_value_heads + ) + self.gqa_or_mqa = ( + config.num_attention_heads != config.num_key_value_heads + ) + + self.max_position_embeddings = config.max_position_embeddings + self.seq_length = config.seq_length + + self.q_proj = nn.Linear( + self.hidden_size, + self.hidden_size, + bias_attr=False, + ) + self.k_proj = nn.Linear( + self.hidden_size, + self.config.num_key_value_heads * self.head_dim, + bias_attr=False, + ) + self.v_proj = nn.Linear( + self.hidden_size, + self.config.num_key_value_heads * self.head_dim, + bias_attr=False, + ) + + self.o_proj = nn.Linear( + self.hidden_size, + self.hidden_size, + bias_attr=False, + ) + + self._init_rope() + + def _init_rope(self): + self.rotary_emb = LlamaRotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + ) + + def forward( + self, + hidden_states, + position_ids: Optional[Tuple[paddle.Tensor]] = None, + past_key_value: Optional[Tuple[paddle.Tensor]] = None, + attention_mask: Optional[paddle.Tensor] = None, + output_attentions: bool = False, + use_cache: bool = False, + ) -> Tuple[ + paddle.Tensor, Optional[paddle.Tensor], Optional[Tuple[paddle.Tensor]] + ]: + """Input shape: Batch x Time x Channel""" + # [bs, seq_len, num_head * head_dim] -> [seq_len / n, bs, num_head * head_dim] (n is model parallelism) + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + target_query_shape = [0, 0, self.num_heads, self.head_dim] + target_key_value_shape = [0, 0, self.num_key_value_heads, self.head_dim] + query_states = query_states.reshape(shape=target_query_shape) + key_states = key_states.reshape(shape=target_key_value_shape) + value_states = value_states.reshape(shape=target_key_value_shape) + + kv_seq_len = key_states.shape[-3] + + if past_key_value is not None: + kv_seq_len += past_key_value[0].shape[-3] + + cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) + query_states, key_states = apply_rotary_pos_emb( + query_states, key_states, cos, sin, position_ids + ) + + # [bs, seq_len, num_head, head_dim] + if past_key_value is not None: + # reuse k, v, self_attention + key_states = paddle.concat([past_key_value[0], key_states], axis=1) + value_states = paddle.concat( + [past_key_value[1], value_states], axis=1 + ) + + past_key_value = (key_states, value_states) if use_cache else None + + outputs = scaled_dot_product_attention( + query_states, + self.config, + key_states, + value_states, + attention_mask, + output_attentions, + ) + if output_attentions: + attn_output, attn_weights = outputs + else: + attn_output = outputs + + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + outputs = (attn_output,) + + if output_attentions: + outputs += (attn_weights,) + + if use_cache: + outputs += (past_key_value,) + + if type(outputs) is tuple and len(outputs) == 1: + outputs = outputs[0] + + return outputs + + +class LlamaDecoderLayer(nn.Layer): + def __init__(self, config): + super().__init__() + self.hidden_size = config.hidden_size + self.self_attn = LlamaAttention(config) + self.mlp = LlamaMLP(config) + self.input_layernorm = LlamaRMSNorm(config) + self.post_attention_layernorm = LlamaRMSNorm(config) + + def forward( + self, + hidden_states: paddle.Tensor, + position_ids: Optional[Tuple[paddle.Tensor]] = None, + attention_mask: Optional[paddle.Tensor] = None, + output_attentions: Optional[bool] = False, + past_key_value: Optional[Tuple[paddle.Tensor]] = None, + use_cache: Optional[bool] = False, + ) -> Tuple[paddle.Tensor, Optional[Tuple[paddle.Tensor, paddle.Tensor]]]: + """ + Args: + hidden_states (`paddle.Tensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`paddle.Tensor`, *optional*): attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + use_cache (`bool`, *optional*): + If set to `True`, `cache` key value states are returned and can be used to speed up decoding + (see `cache`). + cache (`Tuple(paddle.Tensor)`, *optional*): cached past key and value projection states + """ + + # [bs * seq_len, embed_dim] -> [seq_len * bs / n, embed_dim] (sequence_parallel) + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + + # Self Attention + outputs = self.self_attn( + hidden_states, + position_ids, + past_key_value, + attention_mask, + output_attentions, + use_cache, + ) + + if type(outputs) is tuple: + hidden_states = outputs[0] + else: + hidden_states = outputs + + if output_attentions: + self_attn_weights = outputs[1] + + if use_cache: + present_key_value = outputs[2 if output_attentions else 1] + + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + outputs = (hidden_states,) + + if output_attentions: + outputs += (self_attn_weights,) + + if use_cache: + outputs += (present_key_value,) + + # remove empty tuple for pipeline parallel + if type(outputs) is tuple and len(outputs) == 1: + outputs = outputs[0] + + return outputs + + +class LlamaModel(nn.Layer): + def __init__(self, config: LlamaConfig): + super().__init__() + self.config = config + self.vocab_size = config.vocab_size + self.hidden_size = config.hidden_size + + self.embed_tokens = nn.Embedding( + self.vocab_size, + self.hidden_size, + ) + + self.layers = nn.LayerList( + [LlamaDecoderLayer(config) for _ in range(config.num_hidden_layers)] + ) + self.norm = LlamaRMSNorm(config) + + @staticmethod + def _prepare_decoder_attention_mask( + attention_mask, input_shape, past_key_values_length, dtype + ): + if attention_mask is not None: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + if len(attention_mask.shape) == 2: + expanded_attn_mask = _expand_2d_mask( + attention_mask, dtype, tgt_length=input_shape[-1] + ) + # For decoding phase in generation, seq_length = 1, we don't need to add causal mask + if input_shape[-1] > 1: + combined_attention_mask = _make_causal_mask( + input_shape, + past_key_values_length=past_key_values_length, + ) + expanded_attn_mask = ( + expanded_attn_mask & combined_attention_mask + ) + # [bsz, seq_len, seq_len] -> [bsz, 1, seq_len, seq_len] + elif len(attention_mask.shape) == 3: + expanded_attn_mask = attention_mask.unsqueeze(1).astype("bool") + # if attention_mask is already 4-D, do nothing + else: + expanded_attn_mask = attention_mask + else: + expanded_attn_mask = _make_causal_mask( + input_shape, past_key_values_length=past_key_values_length + ) + # Convert bool attention_mask to float attention mask, which will be added to attention_scores later + expanded_attn_mask = paddle.where( + expanded_attn_mask, 0.0, paddle.finfo(dtype).min + ).astype(dtype) + return expanded_attn_mask + + def forward( + self, + input_ids=None, + position_ids=None, + attention_mask=None, + use_cache=None, + ): + output_attentions = False + output_hidden_states = False + use_cache = ( + use_cache if use_cache is not None else self.config.use_cache + ) + + # retrieve input_ids + if input_ids is not None: + batch_size, seq_length = input_ids.shape + else: + raise ValueError("You have to specify either decoder_input_ids") + + past_key_values = tuple([None] * len(self.layers)) + # NOTE: to make cache can be clear in-time + past_key_values = list(past_key_values) + + seq_length_with_past = seq_length + cache_length = 0 + if past_key_values[0] is not None: + cache_length = paddle.shape(past_key_values[0][0])[1] + seq_length_with_past += cache_length + inputs_embeds = self.embed_tokens(input_ids) + + # embed positions + if attention_mask is None: + # [bs, seq_len] + attention_mask = paddle.ones( + (batch_size, seq_length_with_past), dtype=paddle.bool + ) + + if position_ids is None: + position_ids = paddle.arange(seq_length, dtype="int64").expand( + (batch_size, seq_length) + ) + + attention_mask = self._prepare_decoder_attention_mask( + attention_mask, + (batch_size, seq_length), + cache_length, + inputs_embeds.dtype, + ) # [bs, 1, seq_len, seq_len] + + hidden_states = inputs_embeds + + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + next_decoder_cache = () if use_cache else None + + for idx, (decoder_layer) in enumerate(self.layers): + if output_hidden_states: + all_hidden_states += (hidden_states,) + past_key_value = ( + past_key_values[idx] if past_key_values is not None else None + ) + + has_gradient = not hidden_states.stop_gradient + + layer_outputs = decoder_layer( + hidden_states, + position_ids, + attention_mask, + output_attentions, + past_key_value, + use_cache, + ) + + # NOTE: clear outdate cache after it has been used for memory saving + past_key_value = past_key_values[idx] = None + if type(layer_outputs) is tuple: + hidden_states = layer_outputs[0] + else: + hidden_states = layer_outputs + + if output_attentions: + all_self_attns += (layer_outputs[1],) + + if use_cache: + next_decoder_cache += ( + layer_outputs[2 if output_attentions else 1], + ) + + hidden_states = self.norm(hidden_states) + + return hidden_states + + +class TestLlamaModel(unittest.TestCase): + def setUp(self): + paddle.seed(2024) + self.prepare_data() + + def prepare_data(self): + self.config = LlamaConfig() + self.input_ids = paddle.to_tensor( + [ + [ + 1, + 29871, + 31201, + 236, + 138, + 141, + 30287, + 30557, + 30015, + 233, + 187, + 172, + 31969, + 31325, + 31043, + 30374, + 30024, + ] + ], + dtype="int64", + ) + self.position_ids = paddle.to_tensor( + [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]], + dtype="int64", + ) + self.attention_mask = paddle.to_tensor( + [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], dtype="int64" + ) + + def check_jit_kernel_info(self, static_fn): + utils.check_jit_kernel_number(static_fn, 1) + utils.check_jit_kernel_structure(static_fn, {utils.JIT_KERNEL_NAME: 1}) + + def eval(self, use_cinn): + paddle.seed(2024) + net = LlamaModel(self.config) + input_spec = [ + InputSpec(shape=[None, None], dtype='int64'), # input_ids + InputSpec(shape=[None, None], dtype='int64'), # position_ids + InputSpec(shape=[None, None], dtype='int64'), # attention_mask + ] + net = utils.apply_to_static(net, use_cinn, input_spec) + net.eval() + out = net(self.input_ids, self.position_ids, self.attention_mask) + return out + + def test_eval(self): + dy_out = self.eval(use_cinn=False) + if utils.unittest_use_cinn(): + cinn_out = self.eval(use_cinn=True) + np.testing.assert_allclose( + cinn_out.numpy(), dy_out.numpy(), atol=1e-6, rtol=1e-6 + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/ir/pir/cinn/inference/test_llama_postprocess.py b/test/ir/pir/cinn/inference/test_llama_postprocess.py new file mode 100644 index 0000000000000..dad923b4e98f7 --- /dev/null +++ b/test/ir/pir/cinn/inference/test_llama_postprocess.py @@ -0,0 +1,123 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import sys +import unittest +from os.path import dirname + +import numpy as np + +import paddle +import paddle.nn.functional as F +from paddle import nn +from paddle.static import InputSpec + +sys.path.append(dirname(dirname(__file__))) + +import utils + + +class LlamaPostProcess(nn.Layer): + def __init__(self): + super().__init__() + + def update_scores_for_generation( + self, scores, next_scores, length, unfinished_flag + ): + # update scores + unfinished_scores = (scores * length + next_scores) / (length + 1) + scores = paddle.where(unfinished_flag, unfinished_scores, scores) + return scores + + def _post_process_( + self, logits, input_ids, cur_len, origin_len, scores, unfinished_flag + ): + # [batch_size, vocab_size] + logits = logits[:, -1, :] + probs = F.softmax(logits) + + temperature = paddle.full([1], 1) + top_p = paddle.full([1], 0) + + # sample + origin_probs = F.log_softmax(logits) + # compute next_tokens + logits = logits / temperature + top_ps_tensor = paddle.full( + shape=[paddle.shape(probs)[0], 1], + fill_value=top_p, + dtype=probs.dtype, + ) + _, next_tokens = paddle.tensor.top_p_sampling(probs, top_ps_tensor) + + next_scores = paddle.index_sample(origin_probs, next_tokens) + scores = self.update_scores_for_generation( + scores, next_scores, cur_len - origin_len, unfinished_flag + ) + + input_ids = paddle.concat([input_ids, next_tokens], axis=1) + + return input_ids, scores + + def forward(self, logits, input_ids): + batch_size, cur_len = paddle.shape(input_ids) + origin_len = paddle.shape(input_ids)[1] + unfinished_flag = paddle.full([batch_size, 1], True, dtype="bool") + scores = paddle.full( + [batch_size, 1], 0.0, dtype=paddle.get_default_dtype() + ) + return self._post_process_( + logits, input_ids, cur_len, origin_len, scores, unfinished_flag + ) + + +class TestLlamaPostProcess(unittest.TestCase): + def setUp(self): + paddle.seed(2024) + self.prepare_data() + + def prepare_data(self): + self.shape = [1, 2048, 768] + self.logits = paddle.randn([1, 256, 3200], dtype="float32") + self.input_ids = paddle.randint(0, 512, [1, 32], dtype="int64") + + def check_jit_kernel_info(self, static_fn): + utils.check_jit_kernel_number(static_fn, 1) + utils.check_jit_kernel_structure(static_fn, {utils.JIT_KERNEL_NAME: 1}) + + def eval(self, use_cinn): + paddle.seed(2024) + net = LlamaPostProcess() + input_spec = [ + InputSpec(shape=[None, None, None], dtype='float32'), # logits + InputSpec(shape=[None, None], dtype='int64'), # input_ids + ] + net = utils.apply_to_static(net, use_cinn, input_spec) + net.eval() + # paddle.jit.save(net, sys.path.join(dirname(__file__), "post_model")) + out = net(self.logits, self.input_ids) + if use_cinn: + self.check_jit_kernel_info(net.forward) + return out + + def test_eval(self): + dy_out = self.eval(use_cinn=False) + if utils.unittest_use_cinn(): + cinn_out = self.eval(use_cinn=True) + np.testing.assert_allclose( + cinn_out.numpy(), dy_out.numpy(), atol=1e-6, rtol=1e-6 + ) + + +if __name__ == '__main__': + unittest.main() From f9f6d408482897915dedaa7764bfb30feb73367c Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 1 Mar 2024 11:15:45 +0800 Subject: [PATCH 216/282] Fix calibraion calibration, etc (#62259) --- .../analysis/ir_passes/tensorrt_subgraph_pass.cc | 2 +- paddle/fluid/inference/api/paddle_analysis_config.h | 8 ++++---- paddle/fluid/inference/api/resource_manager.cc | 10 +++++----- paddle/fluid/inference/api/resource_manager.h | 2 +- paddle/fluid/inference/capi/pd_config.cc | 4 ++-- paddle/fluid/inference/capi/pd_predictor.cc | 2 +- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc b/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc index 5b2bed7745fcf..1b29ba37f5e66 100644 --- a/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc +++ b/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc @@ -754,7 +754,7 @@ std::string TensorRtSubgraphPass::CreateTensorRTOp( bool calibration_mode = (enable_int8 && calibration_data.empty() && use_calib_mode); if (calibration_mode) { - // calibraion mode means generate int8 calibration table data process. + // calibration mode means generate int8 calibration table data process. return calibration_engine_key; } diff --git a/paddle/fluid/inference/api/paddle_analysis_config.h b/paddle/fluid/inference/api/paddle_analysis_config.h index cae544ff2c234..134c0799ec663 100644 --- a/paddle/fluid/inference/api/paddle_analysis_config.h +++ b/paddle/fluid/inference/api/paddle_analysis_config.h @@ -253,7 +253,7 @@ struct PD_INFER_DECL AnalysisConfig { void SetModel(const std::string& model_dir) { model_dir_ = model_dir; } /// - /// \brief Set the combined model with two specific pathes for program and + /// \brief Set the combined model with two specific paths for program and /// parameters. /// /// \param prog_file_path model file path of the combined model. @@ -596,12 +596,12 @@ struct PD_INFER_DECL AnalysisConfig { /// \brief Control whether to perform IR graph optimization. /// If turned off, the AnalysisConfig will act just like a NativeConfig. /// - /// \param x Whether the ir graph optimization is actived. + /// \param x Whether the ir graph optimization is activated. /// void SwitchIrOptim(int x = true) { enable_ir_optim_ = x; } /// /// \brief A boolean state telling whether the ir graph optimization is - /// actived. + /// activated. /// /// \return bool Whether to use ir graph optimization. /// @@ -1213,7 +1213,7 @@ struct PD_INFER_DECL AnalysisConfig { std::string SerializeInfoCache(); protected: - // Model pathes. + // Model paths. std::string model_dir_; mutable std::string prog_file_; mutable std::string params_file_; diff --git a/paddle/fluid/inference/api/resource_manager.cc b/paddle/fluid/inference/api/resource_manager.cc index b18ca6e1c2a55..9f8a6651ebdf8 100644 --- a/paddle/fluid/inference/api/resource_manager.cc +++ b/paddle/fluid/inference/api/resource_manager.cc @@ -191,7 +191,7 @@ void GPUContextResource::InitGpuEigenDevice() { gpu_eigen_device_ = std::make_unique(eigen_stream_.get()); } -void GPUContextResource::InitDnnHanlde() { +void GPUContextResource::InitDnnHandle() { phi::InitDnnHandle(&dnn_handle_, stream_, place_); } @@ -237,7 +237,7 @@ dnnHandle_t GPUContextResource::GetDnnHandle() const { return dnn_handle_; } std::function GPUContextResource::GetDnnHandleCreator() { return [&]() -> phi::dnnHandle_t { - InitDnnHanlde(); + InitDnnHandle(); return dnn_handle_; }; } @@ -367,7 +367,7 @@ ResourceManager& ResourceManager::Instance() { } void ResourceManager::InitCPUResource() { - std::lock_guard lock_gurad(cpu_mutex_); + std::lock_guard lock_guard(cpu_mutex_); if (cpu_resource_ == nullptr) { cpu_resource_ = std::make_unique(); } @@ -382,7 +382,7 @@ CPUContextResource* ResourceManager::GetCPUResource() const { #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) void* ResourceManager::InitGPUResource(const phi::Place& place, void* stream) { - std::lock_guard lock_gurad(gpu_mutex_); + std::lock_guard lock_guard(gpu_mutex_); if (gpu_resources_.count(stream)) { Increase(stream); return stream; @@ -427,7 +427,7 @@ GPUContextResource* ResourceManager::GetGPUResource(void* stream) const { void ResourceManager::GpuResourceSwitchStream(void* old_stream, void* new_stream) { // NOTE: add lock to support stream rebind in multi-thread - std::lock_guard lock_gurad(gpu_mutex_); + std::lock_guard lock_guard(gpu_mutex_); if (old_stream == new_stream) return; PADDLE_ENFORCE_EQ( gpu_resources_.count(old_stream), diff --git a/paddle/fluid/inference/api/resource_manager.h b/paddle/fluid/inference/api/resource_manager.h index 1f4d4ea420e1b..25b4050e7c4dd 100644 --- a/paddle/fluid/inference/api/resource_manager.h +++ b/paddle/fluid/inference/api/resource_manager.h @@ -88,7 +88,7 @@ class GPUContextResource { void DestroyGPUResource(); void InitGpuProperties(); void InitGpuEigenDevice(); - void InitDnnHanlde(); + void InitDnnHandle(); void DestroyDnnHandle(); void DestroyBlasHandle(); void InitBlasLtHandle(); diff --git a/paddle/fluid/inference/capi/pd_config.cc b/paddle/fluid/inference/capi/pd_config.cc index 5197b8dede192..c2c8036ece7a8 100644 --- a/paddle/fluid/inference/capi/pd_config.cc +++ b/paddle/fluid/inference/capi/pd_config.cc @@ -275,7 +275,7 @@ void PD_EnableDlnne( int max_batch_size, bool use_static_batch, std::string weight_share_mode, - std::unordered_set disable_nodes_by_ouputs, + std::unordered_set disable_nodes_by_outputs, std::map> dlnne_input_shape_dict, bool use_calib_mode, PD_ACPrecision precision_mode) { @@ -287,7 +287,7 @@ void PD_EnableDlnne( max_batch_size, use_static_batch, weight_share_mode, - disable_nodes_by_ouputs, + disable_nodes_by_outputs, dlnne_input_shape_dict, use_calib_mode, precision_mode); diff --git a/paddle/fluid/inference/capi/pd_predictor.cc b/paddle/fluid/inference/capi/pd_predictor.cc index 39575a196e4f9..72f1b6c277153 100644 --- a/paddle/fluid/inference/capi/pd_predictor.cc +++ b/paddle/fluid/inference/capi/pd_predictor.cc @@ -92,7 +92,7 @@ bool PD_PredictorRun(const PD_AnalysisConfig* config, config, paddle::platform::errors::InvalidArgument( "The pointer of analysis configuration shouldn't be nullptr")); - VLOG(3) << "Predoctor: PD_PredictorRun. "; + VLOG(3) << "Predictor: PD_PredictorRun. "; static std::map> predictors; if (!predictors.count(config->config.model_dir())) { From 512d594060232ea1131ff3379ed0dd769f0ef4ed Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 1 Mar 2024 11:16:12 +0800 Subject: [PATCH 217/282] Fix is_sparese is_sparse, etc (#62258) --- .../fluid/distributed/collective/reducer.cc | 2 +- .../distributed/ps/service/brpc_ps_client.cc | 6 ++--- .../distributed/ps/service/brpc_ps_server.cc | 22 +++++++++---------- .../ps/service/coordinator_client.h | 4 ++-- .../ps/service/graph_brpc_server.cc | 2 +- paddle/fluid/imperative/prepared_operator.h | 2 +- paddle/fluid/imperative/reducer.cc | 6 +++-- 7 files changed, 23 insertions(+), 21 deletions(-) diff --git a/paddle/fluid/distributed/collective/reducer.cc b/paddle/fluid/distributed/collective/reducer.cc index 68ccd8f52fa10..df41993bb9bd2 100644 --- a/paddle/fluid/distributed/collective/reducer.cc +++ b/paddle/fluid/distributed/collective/reducer.cc @@ -894,7 +894,7 @@ void EagerReducer::MarkVarReady(const size_t var_index, "The sparse parameter[%d][%s] should have gradient. " "Currently, DataParallel does not support sparse " "parameters without generating gradients during training. " - "For example, if is_sparese=True is used in Embedding, " + "For example, if is_sparse=True is used in Embedding, " "the current step of this parameter cannot generate gradient " "because of stop_gradient/detach, where error will occur.", var_index, diff --git a/paddle/fluid/distributed/ps/service/brpc_ps_client.cc b/paddle/fluid/distributed/ps/service/brpc_ps_client.cc index 89150deff544a..fa9f16db05b6e 100644 --- a/paddle/fluid/distributed/ps/service/brpc_ps_client.cc +++ b/paddle/fluid/distributed/ps/service/brpc_ps_client.cc @@ -402,7 +402,7 @@ int DownpourBrpcClosure::check_response(size_t request_idx, int cmd_id) { int DownpourBrpcClosure::check_save_response(size_t request_idx, int cmd_id) { int32_t feasign_size = 0; if (_cntls[request_idx]->Failed()) { - LOG(ERROR) << "resquest cmd_id:" << cmd_id + LOG(ERROR) << "request cmd_id:" << cmd_id << " failed, " "err:" << _cntls[request_idx]->ErrorText(); @@ -426,7 +426,7 @@ std::string DownpourBrpcClosure::get_response(size_t request_idx, int cmd_id) { int FlClientBrpcClosure::check_response(size_t request_idx, int cmd_id) { if (_cntls[request_idx]->Failed()) { - LOG(ERROR) << "resquest cmd_id:" << cmd_id + LOG(ERROR) << "request cmd_id:" << cmd_id << " failed, " "err:" << _cntls[request_idx]->ErrorText(); @@ -1712,7 +1712,7 @@ void BrpcPsClient::PushSparseTaskConsume() { merge_status[shard_idx].wait(); } - // meger到task_list[0] + // merge到task_list[0] auto async_task = new SparseAsyncTask(*(task_list[0].get())); task_queue->Put(std::move(async_task)); diff --git a/paddle/fluid/distributed/ps/service/brpc_ps_server.cc b/paddle/fluid/distributed/ps/service/brpc_ps_server.cc index 8d73a563d79f1..b1c58ba7acda4 100644 --- a/paddle/fluid/distributed/ps/service/brpc_ps_server.cc +++ b/paddle/fluid/distributed/ps/service/brpc_ps_server.cc @@ -262,7 +262,7 @@ void BrpcPsService::service(google::protobuf::RpcController *cntl_base, brpc::ClosureGuard done_guard(done); std::string log_label("ReceiveCmd-"); if (!request->has_table_id()) { - set_response_code(*response, -1, "PsRequestMessage.tabel_id is required"); + set_response_code(*response, -1, "PsRequestMessage.table_id is required"); return; } @@ -307,7 +307,7 @@ int32_t BrpcPsService::PullDense(Table *table, set_response_code( response, -1, - "PsRequestMessage.datas is requeired at least 1 for num of dense"); + "PsRequestMessage.datas is required at least 1 for num of dense"); return 0; } CostTimer timer("pserver_server_pull_dense"); @@ -409,7 +409,7 @@ int32_t BrpcPsService::Barrier(Table *table, if (request.params_size() < 1) { set_response_code(response, -1, - "PsRequestMessage.params is requeired at " + "PsRequestMessage.params is required at " "least 1 for num of sparse_key"); return 0; } @@ -436,7 +436,7 @@ int32_t BrpcPsService::PushSparseParam(Table *table, if (request.params_size() < 1) { set_response_code(response, -1, - "PsRequestMessage.params is requeired at " + "PsRequestMessage.params is required at " "least 1 for num of sparse_key"); return 0; } @@ -515,7 +515,7 @@ int32_t BrpcPsService::PullSparse(Table *table, if (request.params_size() < 1) { set_response_code(response, -1, - "PsRequestMessage.params is requeired at " + "PsRequestMessage.params is required at " "least 1 for num of sparse_key"); return 0; } @@ -565,7 +565,7 @@ int32_t BrpcPsService::PushSparse(Table *table, if (request.params_size() < 1) { set_response_code(response, -1, - "PsRequestMessage.params is requeired at " + "PsRequestMessage.params is required at " "least 1 for num of sparse_key"); return 0; } @@ -616,7 +616,7 @@ int32_t BrpcPsService::LoadOneTable(Table *table, set_response_code( response, -1, - "PsRequestMessage.datas is requeired at least 2 for path & load_param"); + "PsRequestMessage.datas is required at least 2 for path & load_param"); return -1; } if (table->Load(request.params(0), request.params(1)) != 0) { @@ -649,7 +649,7 @@ int32_t BrpcPsService::SaveOneTable(Table *table, set_response_code( response, -1, - "PsRequestMessage.datas is requeired at least 2, path&mode"); + "PsRequestMessage.datas is required at least 2, path&mode"); return -1; } table->Flush(); @@ -691,7 +691,7 @@ int32_t BrpcPsService::SaveCacheTable(Table *table, set_response_code( response, -1, - "PsRequestMessage.datas is requeired at least 3, path&mode"); + "PsRequestMessage.datas is required at least 3, path&mode"); return -1; } table->Flush(); @@ -717,7 +717,7 @@ int32_t BrpcPsService::CacheShuffle(Table *table, if (request.params_size() < 3) { set_response_code(response, -1, - "PsRequestMessage.datas is requeired at least 3, " + "PsRequestMessage.datas is required at least 3, " "path&mode&cache_threshold"); return -1; } @@ -805,7 +805,7 @@ int32_t BrpcPsService::ShrinkTable(Table *table, set_response_code( response, -1, - "PsRequestMessage.datas is requeired at least 1, threshold"); + "PsRequestMessage.datas is required at least 1, threshold"); return -1; } table->Flush(); diff --git a/paddle/fluid/distributed/ps/service/coordinator_client.h b/paddle/fluid/distributed/ps/service/coordinator_client.h index 8db08c3fc7999..f0d1116fca268 100644 --- a/paddle/fluid/distributed/ps/service/coordinator_client.h +++ b/paddle/fluid/distributed/ps/service/coordinator_client.h @@ -81,7 +81,7 @@ class CoordinatorServiceHandle { lck.unlock(); VLOG(0) << "last_round_total_fl_clients_num: " << last_round_total_fl_clients_num - << ", has recved fl client num: " << _fl_clients_count.load(); + << ", has received fl client num: " << _fl_clients_count.load(); return; } @@ -102,7 +102,7 @@ class CoordinatorServiceHandle { timeline.Pause(); query_wait_time += timeline.ElapsedSec(); } - // LOG(WARNNING) << "fl-ps > query_wait_time exceed!"; + // LOG(WARNING) << "fl-ps > query_wait_time exceed!"; return true; }; diff --git a/paddle/fluid/distributed/ps/service/graph_brpc_server.cc b/paddle/fluid/distributed/ps/service/graph_brpc_server.cc index 0a8867bb66e11..df0c1a8fd3a6c 100644 --- a/paddle/fluid/distributed/ps/service/graph_brpc_server.cc +++ b/paddle/fluid/distributed/ps/service/graph_brpc_server.cc @@ -247,7 +247,7 @@ void GraphBrpcService::service(google::protobuf::RpcController *cntl_base, brpc::ClosureGuard done_guard(done); std::string log_label("ReceiveCmd-"); if (!request->has_table_id()) { - set_response_code(*response, -1, "PsRequestMessage.tabel_id is required"); + set_response_code(*response, -1, "PsRequestMessage.table_id is required"); return; } diff --git a/paddle/fluid/imperative/prepared_operator.h b/paddle/fluid/imperative/prepared_operator.h index 70c36b27d31c0..42a50cec23558 100644 --- a/paddle/fluid/imperative/prepared_operator.h +++ b/paddle/fluid/imperative/prepared_operator.h @@ -559,7 +559,7 @@ void BuildDygraphPhiKernelContext(const phi::KernelSignature& kernel_signature, PADDLE_ENFORCE_NOT_NULL( attr_ptr, platform::errors::NotFound("(%s) is not found in AttributeMap when " - "buildind dygraph KernelContext.", + "building dygraph KernelContext.", attr_names[i])); auto& attr = *attr_ptr; switch (attr_defs[i].type_index) { diff --git a/paddle/fluid/imperative/reducer.cc b/paddle/fluid/imperative/reducer.cc index 5b8dc28d03111..93e6b10e6488e 100644 --- a/paddle/fluid/imperative/reducer.cc +++ b/paddle/fluid/imperative/reducer.cc @@ -493,8 +493,10 @@ void Reducer::PrepareDeps(const std::unordered_set &init_nodes) { "using PyLayer in a DataParallel model, you can skip gradient " "synchronization among multiple cards by 'no_sync', and " "manually implement 'all_reduce' before model optimization. " - "There is an example showing specific implemetation processing " - "in offical docs: https://www.paddlepaddle.org.cn/documentation" + "There is an example showing specific implementation " + "processing " + "in official docs: " + "https://www.paddlepaddle.org.cn/documentation" "/docs/api/paddle/DataParallel_cn.html")); } ++node_deps_[grad_pending_node.get()]; From 6b3f074c0e960a3e5f9235362005fe2340d96cd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=98=A5=E4=B9=94?= <83450930+Liyulingyue@users.noreply.github.com> Date: Fri, 1 Mar 2024 11:20:47 +0800 Subject: [PATCH 218/282] =?UTF-8?q?=E3=80=90paddle=5Ftest=20No.27=E3=80=91?= =?UTF-8?q?replace=20parts=20of=20cc=5Ftest=20with=20paddle=5Ftest=20=20(#?= =?UTF-8?q?61675)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update CMakeLists.txt * add TEST_API and rm use_it_self_op * fix code-style * Update CMakeLists.txt * Apply suggestions from code review * Update CMakeLists.txt * Update test_common_infer_shape_functions.cc * replace cc with paddle_test * Update selected_rows.h * delete use_op_itself * Update CMakeLists.txt * add TEST_API * Update copy_cross_scope_test.cc * try to add TEST_API * try to add TEST_API * Update CMakeLists.txt --- paddle/fluid/framework/shape_inference.h | 7 ++- paddle/fluid/imperative/var_helper.h | 2 +- .../memory/allocation/allocator_facade.h | 13 ++-- paddle/fluid/memory/memcpy.cc | 34 +++++------ paddle/fluid/memory/memcpy.h | 4 +- .../operators/common_infer_shape_functions.h | 7 ++- paddle/phi/core/selected_rows.h | 3 +- test/cpp/fluid/CMakeLists.txt | 60 ++++--------------- test/cpp/fluid/copy_cross_scope_test.cc | 2 - test/cpp/fluid/save_load_combine_op_test.cc | 5 -- test/cpp/fluid/save_load_op_test.cc | 4 -- test/cpp/fluid/share_buffer_op_test.cc | 8 --- 12 files changed, 50 insertions(+), 99 deletions(-) diff --git a/paddle/fluid/framework/shape_inference.h b/paddle/fluid/framework/shape_inference.h index 49603b34255db..427d4be4558e9 100644 --- a/paddle/fluid/framework/shape_inference.h +++ b/paddle/fluid/framework/shape_inference.h @@ -78,13 +78,14 @@ class InferShapeContext { virtual DDim GetInputDim(const std::string &name) const = 0; virtual std::vector GetInputsDim(const std::string &name) const = 0; - virtual std::vector GetReaderDims(const std::string &name) const; + TEST_API virtual std::vector GetReaderDims( + const std::string &name) const; virtual void SetOutputDim(const std::string &name, const DDim &dim) = 0; virtual void SetOutputsDim(const std::string &name, const std::vector &dims) = 0; - virtual void SetReaderDims(const std::string &name, - const std::vector &dims); + TEST_API virtual void SetReaderDims(const std::string &name, + const std::vector &dims); virtual std::string GetInputNameByIdx(size_t idx) const = 0; virtual std::string GetOutputNameByIdx(size_t idx) const = 0; virtual AttrReader Attrs() const = 0; diff --git a/paddle/fluid/imperative/var_helper.h b/paddle/fluid/imperative/var_helper.h index ebf3e49c51870..1a74d987e7e2b 100644 --- a/paddle/fluid/imperative/var_helper.h +++ b/paddle/fluid/imperative/var_helper.h @@ -40,7 +40,7 @@ void InitializeVariable(paddle::framework::Variable* var, template const paddle::platform::Place& GetPlace(const std::shared_ptr& var); template -const std::string& GetNameFromVar(std::shared_ptr var); +TEST_API const std::string& GetNameFromVar(std::shared_ptr var); template bool CheckCachedKey(std::shared_ptr tensor, const phi::KernelKey& key); diff --git a/paddle/fluid/memory/allocation/allocator_facade.h b/paddle/fluid/memory/allocation/allocator_facade.h index f80fcac1b2a38..f0f321b887b59 100644 --- a/paddle/fluid/memory/allocation/allocator_facade.h +++ b/paddle/fluid/memory/allocation/allocator_facade.h @@ -49,11 +49,12 @@ class AllocatorFacade { const AllocatorFacade& operator=(const AllocatorFacade& o) = delete; ~AllocatorFacade(); - static AllocatorFacade& Instance(); + TEST_API static AllocatorFacade& Instance(); AllocatorFacadePrivate* GetPrivate() const; - const std::shared_ptr& GetAllocator(const platform::Place& place); + TEST_API const std::shared_ptr& GetAllocator( + const platform::Place& place); void* GetBasePtr(const std::shared_ptr& allocation); @@ -88,8 +89,8 @@ class AllocatorFacade { void RecordStream(std::shared_ptr allocation, gpuStream_t stream); void EraseStream(std::shared_ptr allocation, gpuStream_t stream); - const std::shared_ptr& GetAllocator(const platform::Place& place, - gpuStream_t stream); + TEST_API const std::shared_ptr& GetAllocator( + const platform::Place& place, gpuStream_t stream); gpuStream_t GetStream(const std::shared_ptr& allocation) const; void SetDefaultStream(const platform::CUDAPlace& place, gpuStream_t stream); #endif @@ -104,8 +105,8 @@ class AllocatorFacade { phi::stream::stream_t stream); void RecordStream(std::shared_ptr allocation, phi::stream::stream_t stream); - const std::shared_ptr& GetAllocator(const platform::Place& place, - phi::stream::stream_t stream); + TEST_API const std::shared_ptr& GetAllocator( + const platform::Place& place, phi::stream::stream_t stream); phi::stream::stream_t GetStream( const std::shared_ptr& allocation) const; void SetDefaultStream(const platform::CustomPlace& place, diff --git a/paddle/fluid/memory/memcpy.cc b/paddle/fluid/memory/memcpy.cc index 7cdf93514c52c..6ba7b4ac1d613 100644 --- a/paddle/fluid/memory/memcpy.cc +++ b/paddle/fluid/memory/memcpy.cc @@ -638,12 +638,12 @@ void Copy(phi::Place dst_place, // NOTE: only for (CPUPlace, CUDAPlace and CUDAPinnedPlace) -> (CPUPlace). template <> -void Copy(phi::CPUPlace dst_place, - void* dst, - phi::Place src_place, - const void* src, - size_t num, - void* stream) { +TEST_API void Copy(phi::CPUPlace dst_place, + void* dst, + phi::Place src_place, + const void* src, + size_t num, + void* stream) { Copy(phi::Place(dst_place.GetType()), dst, src_place, src, num, stream); } @@ -835,11 +835,11 @@ TEST_API void Copy(phi::Place dst_place, // NOTE: Only for (CPUPlace and PinnedPlace) -> (CPUPlace). template <> -void Copy(phi::CPUPlace dst_place, - void* dst, - phi::Place src_place, - const void* src, - size_t num) { +TEST_API void Copy(phi::CPUPlace dst_place, + void* dst, + phi::Place src_place, + const void* src, + size_t num) { Copy(phi::Place(dst_place.GetType()), dst, src_place, src, num); } @@ -872,12 +872,12 @@ void Copy(phi::Place dst_place, } template <> -void Copy(phi::CPUPlace dst_place, - void* dst, - phi::Place src_place, - const void* src, - size_t num, - void* stream) { +TEST_API void Copy(phi::CPUPlace dst_place, + void* dst, + phi::Place src_place, + const void* src, + size_t num, + void* stream) { Copy(phi::Place(dst_place.GetType()), dst, src_place, src, num, stream); } diff --git a/paddle/fluid/memory/memcpy.h b/paddle/fluid/memory/memcpy.h index c8d9208c48219..b0a9234817f0a 100644 --- a/paddle/fluid/memory/memcpy.h +++ b/paddle/fluid/memory/memcpy.h @@ -31,7 +31,7 @@ namespace memory { * */ template -void Copy(DstPlace, void* dst, SrcPlace, const void* src, size_t num); +TEST_API void Copy(DstPlace, void* dst, SrcPlace, const void* src, size_t num); /** * \brief Copy memory from one place to another place. @@ -51,7 +51,7 @@ void Copy(DstPlace, void* dst, SrcPlace, const void* src, size_t num); * */ template -void Copy( +TEST_API void Copy( DstPlace, void* dst, SrcPlace, const void* src, size_t num, void* stream); } // namespace memory } // namespace paddle diff --git a/paddle/fluid/operators/common_infer_shape_functions.h b/paddle/fluid/operators/common_infer_shape_functions.h index 5ce21b1de529b..a61686f3f7544 100644 --- a/paddle/fluid/operators/common_infer_shape_functions.h +++ b/paddle/fluid/operators/common_infer_shape_functions.h @@ -34,12 +34,13 @@ framework::DDim BroadcastTwoDims(const framework::DDim& x_dims, int axis = -1); } // shape input(0) -> output(0) without change. -void UnaryOpUnchangedInferShape(framework::InferShapeContext* ctx); +TEST_API void UnaryOpUnchangedInferShape(framework::InferShapeContext* ctx); // shape input(0) -> output(0) without change, check if axis in range [-Rank(x), // Rank(x)-1] -void UnaryOpUnchangedInferShapeCheckAxis(framework::InferShapeContext* ctx); +TEST_API void UnaryOpUnchangedInferShapeCheckAxis( + framework::InferShapeContext* ctx); // broadcast input(0) and input(1) -> output(0) -void BinaryOpBroadcastInferShape(framework::InferShapeContext* ctx); +TEST_API void BinaryOpBroadcastInferShape(framework::InferShapeContext* ctx); } // namespace operators } // namespace paddle diff --git a/paddle/phi/core/selected_rows.h b/paddle/phi/core/selected_rows.h index 7674a8e8722bc..145f7e7d3b2e4 100644 --- a/paddle/phi/core/selected_rows.h +++ b/paddle/phi/core/selected_rows.h @@ -42,7 +42,8 @@ class SelectedRows : public TensorBase, * */ public: - SelectedRows(const std::vector& rows, const int64_t& height); + TEST_API SelectedRows(const std::vector& rows, + const int64_t& height); TEST_API SelectedRows(); diff --git a/test/cpp/fluid/CMakeLists.txt b/test/cpp/fluid/CMakeLists.txt index f49eefb4354d0..3a8f9326764cb 100644 --- a/test/cpp/fluid/CMakeLists.txt +++ b/test/cpp/fluid/CMakeLists.txt @@ -33,14 +33,12 @@ endif() set(COMMON_OP_DEPS ${COMMON_OP_DEPS} executor) if(WITH_XPU) - cc_test( - beam_search_decode_op_xpu_test - SRCS beam_search_decode_op_xpu_test.cc - DEPS lod_tensor) + paddle_test(beam_search_decode_op_xpu_test SRCS + beam_search_decode_op_xpu_test.cc) set(COMMON_OP_DEPS ${COMMON_OP_DEPS} xpulib) endif() -cc_test( +nv_test( test_common_infer_shape_functions SRCS test_common_infer_shape_functions.cc DEPS common_infer_shape_functions @@ -51,30 +49,12 @@ cc_test( phi common generated_static_op) -cc_test( - gather_test - SRCS gather_test.cc - DEPS tensor) -cc_test( - assign_op_test - SRCS assign_op_test.cc - DEPS generated_static_op) -cc_test( - scatter_test - SRCS scatter_test.cc - DEPS tensor phi common) -cc_test( - beam_search_decode_op_test - SRCS beam_search_decode_op_test.cc - DEPS lod_tensor) -cc_test( - save_load_op_test - SRCS save_load_op_test.cc - DEPS save_op load_op) -cc_test( - save_load_combine_op_test - SRCS save_load_combine_op_test.cc - DEPS save_combine_op load_combine_op) +paddle_test(gather_test SRCS gather_test.cc) +paddle_test(assign_op_test SRCS assign_op_test.cc) +paddle_test(scatter_test SRCS scatter_test.cc DEPS common) +paddle_test(beam_search_decode_op_test SRCS beam_search_decode_op_test.cc) +paddle_test(save_load_op_test SRCS save_load_op_test.cc) +paddle_test(save_load_combine_op_test SRCS save_load_combine_op_test.cc) if(WITH_CINN) set(CINN_DEPS python) endif() @@ -109,15 +89,10 @@ elseif(WITH_ROCM) test_leaky_relu_grad_grad_functor.cu DEPS tensor device_context eigen3) else() - cc_test( - test_leaky_relu_grad_grad_functor - SRCS test_leaky_relu_grad_grad_functor.cc - DEPS tensor device_context eigen3) + paddle_test(test_leaky_relu_grad_grad_functor SRCS + test_leaky_relu_grad_grad_functor.cc) endif() -cc_test( - share_buffer_op_cpp_test - SRCS share_buffer_op_test.cc - DEPS lod_tensor device_context generated_static_op) +paddle_test(share_buffer_op_cpp_test SRCS share_buffer_op_test.cc) if(WITH_CINN) paddle_test(op_debug_string_test SRCS op_debug_string_test.cc) @@ -126,16 +101,7 @@ else() endif() if(WITH_GPU) - cc_test( - copy_cross_scope_test - SRCS copy_cross_scope_test.cc - DEPS op_registry - copy_cross_scope_op - scope - device_context - enforce - executor - common) + paddle_test(copy_cross_scope_test SRCS copy_cross_scope_test.cc) endif() if(WITH_ONNXRUNTIME AND WIN32) diff --git a/test/cpp/fluid/copy_cross_scope_test.cc b/test/cpp/fluid/copy_cross_scope_test.cc index f6f7eb31cb8e6..3d2033d77fe80 100644 --- a/test/cpp/fluid/copy_cross_scope_test.cc +++ b/test/cpp/fluid/copy_cross_scope_test.cc @@ -33,8 +33,6 @@ limitations under the License. */ namespace f = paddle::framework; namespace p = paddle::platform; -USE_NO_KERNEL_OP(copy_cross_scope); - template void Compare1(f::Scope* scope, const p::DeviceContext& ctx, diff --git a/test/cpp/fluid/save_load_combine_op_test.cc b/test/cpp/fluid/save_load_combine_op_test.cc index 8f85676b1ba55..f97409d6535ab 100644 --- a/test/cpp/fluid/save_load_combine_op_test.cc +++ b/test/cpp/fluid/save_load_combine_op_test.cc @@ -22,11 +22,6 @@ limitations under the License. */ #include "paddle/fluid/platform/float16.h" #include "paddle/phi/core/kernel_registry.h" -USE_OP_ITSELF(save_combine); -USE_OP_ITSELF(load_combine); -PD_DECLARE_KERNEL(save_combine_tensor, CPU, ALL_LAYOUT); -PD_DECLARE_KERNEL(load_combine, CPU, ALL_LAYOUT); - template T* CreateForSaveCombineOp(int x, int y, diff --git a/test/cpp/fluid/save_load_op_test.cc b/test/cpp/fluid/save_load_op_test.cc index 5ec376b71de17..5ddb0afb03616 100644 --- a/test/cpp/fluid/save_load_op_test.cc +++ b/test/cpp/fluid/save_load_op_test.cc @@ -17,12 +17,8 @@ limitations under the License. */ #include "paddle/fluid/platform/float16.h" #include "paddle/phi/core/kernel_registry.h" -USE_OP_ITSELF(save); -PD_DECLARE_KERNEL(save, CPU, ALL_LAYOUT); PD_DECLARE_KERNEL(save_sr, CPU, ALL_LAYOUT); PD_DECLARE_KERNEL(cast, CPU, ALL_LAYOUT); -USE_OP_ITSELF(load); -PD_DECLARE_KERNEL(load, CPU, ALL_LAYOUT); PD_DECLARE_KERNEL(load_sr, CPU, ALL_LAYOUT); TEST(SaveLoadOp, CPU) { diff --git a/test/cpp/fluid/share_buffer_op_test.cc b/test/cpp/fluid/share_buffer_op_test.cc index d576ba6ecfcea..eb042acf06ff2 100644 --- a/test/cpp/fluid/share_buffer_op_test.cc +++ b/test/cpp/fluid/share_buffer_op_test.cc @@ -20,14 +20,6 @@ #include "paddle/fluid/platform/place.h" #include "paddle/phi/core/kernel_registry.h" -USE_OP_ITSELF(share_buffer); - -PD_DECLARE_KERNEL(share_buffer, CPU, ALL_LAYOUT); - -#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) -PD_DECLARE_KERNEL(share_buffer, GPU, ALL_LAYOUT); -#endif - namespace paddle { namespace framework { From 7620c500fa7b85790661a50265c23b1bf32d3b63 Mon Sep 17 00:00:00 2001 From: Tian <121000916+SylarTiaNII@users.noreply.github.com> Date: Fri, 1 Mar 2024 11:21:06 +0800 Subject: [PATCH 219/282] [Distributed] fix sharding overlap comm on npu (#62236) --- .../fleet/meta_parallel/sharding/group_sharded_utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py index 046143c79842f..552d36afb1dda 100644 --- a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py +++ b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py @@ -341,7 +341,10 @@ def cvt_to_device(x, dev_id, blocking=True): elif paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(dev_id) else: - raise OSError( - "Only supported compiled paddle with gpu/rocm and xpu, but current version is compiled with cpu." - ) + supported_custom_devices = ["npu"] + place = paddle.framework._current_expected_place() + if place.get_device_type() not in supported_custom_devices: + raise OSError( + "Only supported compiled paddle with gpu/rocm and xpu, but current version is compiled with cpu." + ) return x._copy_to(place, blocking) From 85ba93655e6ed9e0eb4f04ef62bbfb312796f3f4 Mon Sep 17 00:00:00 2001 From: ming1753 <61511741+ming1753@users.noreply.github.com> Date: Fri, 1 Mar 2024 14:33:27 +0800 Subject: [PATCH 220/282] fix delete scale and zero_point var bug (#62225) * fix delete scale and zero_point var bug --- .../ir/delete_quant_dequant_linear_op_pass.cc | 17 +++++++---------- paddle/fluid/framework/ir/fuse_pass_base.h | 5 +++++ .../trt_delete_weight_dequant_linear_op_pass.cc | 17 +++++++---------- .../passes/save_optimized_model_pass.cc | 12 ++++++++++-- 4 files changed, 29 insertions(+), 22 deletions(-) diff --git a/paddle/fluid/framework/ir/delete_quant_dequant_linear_op_pass.cc b/paddle/fluid/framework/ir/delete_quant_dequant_linear_op_pass.cc index 9d4006e6f3943..b8a5dfdaa9465 100644 --- a/paddle/fluid/framework/ir/delete_quant_dequant_linear_op_pass.cc +++ b/paddle/fluid/framework/ir/delete_quant_dequant_linear_op_pass.cc @@ -124,14 +124,18 @@ void DeleteQuantDequantLinearOpPass::ApplyImpl(ir::Graph* graph) const { return; } */ - std::unordered_set nodes2rm = {}; - - // delete Scale and ZeroPoint tensor in scope + // Scale and ZeroPoint tensor should be removed in save_optimized_model_pass std::vector vars2rm = {}; vars2rm.emplace_back(quantize_linear_op->Op()->Input("Scale")[0]); vars2rm.emplace_back(quantize_linear_op->Op()->Input("ZeroPoint")[0]); vars2rm.emplace_back(dequantize_linear_op->Op()->Input("Scale")[0]); vars2rm.emplace_back(dequantize_linear_op->Op()->Input("ZeroPoint")[0]); + auto& scale_and_zero_point_param = g->GetOrInit>( + framework::ir::kScaleAndZeroPointParamAttr); + scale_and_zero_point_param.insert( + scale_and_zero_point_param.end(), vars2rm.begin(), vars2rm.end()); + + std::unordered_set nodes2rm = {}; // Get input scale from tensor const phi::DenseTensor& input_scale_tensor = @@ -182,13 +186,6 @@ void DeleteQuantDequantLinearOpPass::ApplyImpl(ir::Graph* graph) const { nodes2rm.insert(dequantize_linear_op); nodes2rm.insert(dequantize_linear_op_out); GraphSafeRemoveNodes(graph, nodes2rm); - - for (auto& var_name : vars2rm) { - if (scope->FindVar(var_name)) { - scope->EraseVars({var_name}); - } - } - found_count++; }; gpd(graph, handler); diff --git a/paddle/fluid/framework/ir/fuse_pass_base.h b/paddle/fluid/framework/ir/fuse_pass_base.h index bc5fc2a16d393..d8522f1aeaabe 100644 --- a/paddle/fluid/framework/ir/fuse_pass_base.h +++ b/paddle/fluid/framework/ir/fuse_pass_base.h @@ -40,6 +40,11 @@ static const char kFuseStatisAttr[] = "__fuse_statis__"; // allocation. static const char kRepetitiveParamAttr[] = "__repetitive_param__"; +// scale and zero point of the quantized/dequantized op should be removed in +// save_optimized_model_pass. +static const char kScaleAndZeroPointParamAttr[] = + "__scale_and_zero_point_param__"; + enum FuseOptions { DO_NOT_FUSE, // fusing will not be done FUSE_NATIVE, // fusing will be done without MKL-DNN diff --git a/paddle/fluid/framework/ir/trt_delete_weight_dequant_linear_op_pass.cc b/paddle/fluid/framework/ir/trt_delete_weight_dequant_linear_op_pass.cc index b780c07fda0a6..6bc9cb324d80d 100644 --- a/paddle/fluid/framework/ir/trt_delete_weight_dequant_linear_op_pass.cc +++ b/paddle/fluid/framework/ir/trt_delete_weight_dequant_linear_op_pass.cc @@ -231,13 +231,17 @@ void TrtDeleteWeightQuantDequantLinearOpPass::ApplyImpl( return; } */ - std::unordered_set nodes2rm = {}; - - // delete Scale and ZeroPoint tensor in scope + // Scale and ZeroPoint tensor should be removed in save_optimized_model_pass std::vector vars2rm = {}; vars2rm.emplace_back(weight_dequantize_linear_op->Op()->Input("Scale")[0]); vars2rm.emplace_back( weight_dequantize_linear_op->Op()->Input("ZeroPoint")[0]); + auto& scale_and_zero_point_param = g->GetOrInit>( + framework::ir::kScaleAndZeroPointParamAttr); + scale_and_zero_point_param.insert( + scale_and_zero_point_param.end(), vars2rm.begin(), vars2rm.end()); + + std::unordered_set nodes2rm = {}; int bit_length = PADDLE_GET_CONST( int, weight_dequantize_linear_op->Op()->GetAttr("bit_length")); @@ -363,13 +367,6 @@ void TrtDeleteWeightQuantDequantLinearOpPass::ApplyImpl( } GraphSafeRemoveNodes(graph, nodes2rm); - - for (auto& var_name : vars2rm) { - if (scope->FindVar(var_name)) { - scope->EraseVars({var_name}); - } - } - found_count++; }; gpd(graph, handler); diff --git a/paddle/fluid/inference/analysis/passes/save_optimized_model_pass.cc b/paddle/fluid/inference/analysis/passes/save_optimized_model_pass.cc index 8d988de162100..89b49df107390 100644 --- a/paddle/fluid/inference/analysis/passes/save_optimized_model_pass.cc +++ b/paddle/fluid/inference/analysis/passes/save_optimized_model_pass.cc @@ -16,6 +16,7 @@ limitations under the License. */ #include #include "paddle/fluid/framework/executor.h" +#include "paddle/fluid/framework/ir/fuse_pass_base.h" #include "paddle/fluid/framework/ir/graph_helper.h" #include "paddle/fluid/framework/scope.h" @@ -37,10 +38,17 @@ void SaveOptimizedModelPass::SaveOptimizedModel(Argument* argument) { framework::ir::GraphToProgram(*graph, &optimized_program_desc); - // Some vars may be deleted by pass, so we need to remove them in block + // Remove the scale and zero point parameters from optimized program. + auto scale_and_zero_point_param = graph->GetOrInit>( + framework::ir::kScaleAndZeroPointParamAttr); framework::BlockDesc* block = optimized_program_desc.MutableBlock(0); for (auto& var_desc : block->AllVars()) { - if (var_desc->Persistable() && !scope.FindVar(var_desc->Name())) { + auto var_name = var_desc->Name(); + if (var_desc->Persistable() && scope.FindVar(var_name) && + std::count(scale_and_zero_point_param.begin(), + scale_and_zero_point_param.end(), + var_name) > 0) { + scope.EraseVars({var_name}); block->RemoveVar(var_desc->Name()); } } From 9c1ff4b922eb7096fed049d777374a8202c5cde7 Mon Sep 17 00:00:00 2001 From: cyber-pioneer <116002591+cyber-pioneer@users.noreply.github.com> Date: Fri, 1 Mar 2024 14:33:46 +0800 Subject: [PATCH 221/282] [Prim][PIR] Add simple llama config for llama eval test (#62208) * add llama config program txt * polish test case * polish code * fix code * fix file path * fix test case * fix test case --- test/ir/pir/cinn/symbolic/CMakeLists.txt | 13 + test/ir/pir/cinn/symbolic/simple_llama.config | 252 ++++++++++++++++++ .../pir/cinn/symbolic/test_simple_llama_dy.py | 217 +++++++++++++++ 3 files changed, 482 insertions(+) create mode 100644 test/ir/pir/cinn/symbolic/simple_llama.config create mode 100644 test/ir/pir/cinn/symbolic/test_simple_llama_dy.py diff --git a/test/ir/pir/cinn/symbolic/CMakeLists.txt b/test/ir/pir/cinn/symbolic/CMakeLists.txt index 9f26f4dd17269..9d2fc16e2c638 100644 --- a/test/ir/pir/cinn/symbolic/CMakeLists.txt +++ b/test/ir/pir/cinn/symbolic/CMakeLists.txt @@ -7,6 +7,7 @@ if(WITH_GPU) list( REMOVE_ITEM CINN_PIR_SYMBOLIC_TEST + test_simple_llama_dy.py test_cinn_reduce_symbolic_demo.py test_if_st.py test_if_dy.py @@ -71,6 +72,18 @@ if(WITH_GPU) WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_tests_properties(test_llama_if_dy PROPERTIES LABELS "RUN_TYPE=CINN") + add_test( + NAME test_simple_llama_dy + COMMAND + ${CMAKE_COMMAND} -E env + PYTHONPATH=${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/python/:$ENV{PYTHONPATH} + FLAGS_prim_enable_dynamic=true FLAGS_prim_check_ops=true + FLAGS_enable_pir_api=true FLAGS_cinn_bucket_compile=false + FLAGS_pir_apply_shape_optimization_pass=false ${PYTHON_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/test_simple_llama_dy.py + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + set_tests_properties(test_simple_llama_dy PROPERTIES LABELS "RUN_TYPE=CINN") + add_test( NAME test_decomp_inference_predictor_run COMMAND diff --git a/test/ir/pir/cinn/symbolic/simple_llama.config b/test/ir/pir/cinn/symbolic/simple_llama.config new file mode 100644 index 0000000000000..ef3193a8cc735 --- /dev/null +++ b/test/ir/pir/cinn/symbolic/simple_llama.config @@ -0,0 +1,252 @@ +{ + (%0) = "pd_op.data" () {dtype:(pd_op.DataType)float16,name:"embedding_0.w_0",place:(pd_op.Place)Place(undefined:0),shape:(pd_op.IntArray)[32000,4096],stop_gradient:[true]} : () -> pd_op.tensor<32000x4096xf16> + (%1) = "pd_op.data" () {dtype:(pd_op.DataType)float16,name:"create_parameter_0.w_0",place:(pd_op.Place)Place(undefined:0),shape:(pd_op.IntArray)[4096],stop_gradient:[true]} : () -> pd_op.tensor<4096xf16> + (%2) = "pd_op.data" () {dtype:(pd_op.DataType)float16,name:"linear_0.w_0",place:(pd_op.Place)Place(undefined:0),shape:(pd_op.IntArray)[4096,4096],stop_gradient:[true]} : () -> pd_op.tensor<4096x4096xf16> + (%3) = "pd_op.data" () {dtype:(pd_op.DataType)float16,name:"linear_1.w_0",place:(pd_op.Place)Place(undefined:0),shape:(pd_op.IntArray)[4096,4096],stop_gradient:[true]} : () -> pd_op.tensor<4096x4096xf16> + (%4) = "pd_op.data" () {dtype:(pd_op.DataType)float16,name:"linear_2.w_0",place:(pd_op.Place)Place(undefined:0),shape:(pd_op.IntArray)[4096,4096],stop_gradient:[true]} : () -> pd_op.tensor<4096x4096xf16> + (%5) = "pd_op.data" () {dtype:(pd_op.DataType)float32,name:"eager_tmp_1",place:(pd_op.Place)Place(undefined:0),shape:(pd_op.IntArray)[1,2048,1,128],stop_gradient:[true]} : () -> pd_op.tensor<1x2048x1x128xf32> + (%6) = "pd_op.data" () {dtype:(pd_op.DataType)float32,name:"eager_tmp_2",place:(pd_op.Place)Place(undefined:0),shape:(pd_op.IntArray)[1,2048,1,128],stop_gradient:[true]} : () -> pd_op.tensor<1x2048x1x128xf32> + (%7) = "pd_op.data" () {dtype:(pd_op.DataType)float16,name:"linear_3.w_0",place:(pd_op.Place)Place(undefined:0),shape:(pd_op.IntArray)[4096,4096],stop_gradient:[true]} : () -> pd_op.tensor<4096x4096xf16> + (%8) = "pd_op.data" () {dtype:(pd_op.DataType)float16,name:"create_parameter_1.w_0",place:(pd_op.Place)Place(undefined:0),shape:(pd_op.IntArray)[4096],stop_gradient:[true]} : () -> pd_op.tensor<4096xf16> + (%9) = "pd_op.data" () {dtype:(pd_op.DataType)float16,name:"linear_4.w_0",place:(pd_op.Place)Place(undefined:0),shape:(pd_op.IntArray)[4096,11008],stop_gradient:[true]} : () -> pd_op.tensor<4096x11008xf16> + (%10) = "pd_op.data" () {dtype:(pd_op.DataType)float16,name:"linear_5.w_0",place:(pd_op.Place)Place(undefined:0),shape:(pd_op.IntArray)[4096,11008],stop_gradient:[true]} : () -> pd_op.tensor<4096x11008xf16> + (%11) = "pd_op.data" () {dtype:(pd_op.DataType)float16,name:"linear_6.w_0",place:(pd_op.Place)Place(undefined:0),shape:(pd_op.IntArray)[11008,4096],stop_gradient:[true]} : () -> pd_op.tensor<11008x4096xf16> + (%12) = "pd_op.data" () {dtype:(pd_op.DataType)float16,name:"create_parameter_2.w_0",place:(pd_op.Place)Place(undefined:0),shape:(pd_op.IntArray)[4096],stop_gradient:[true]} : () -> pd_op.tensor<4096xf16> + (%13) = "pd_op.data" () {dtype:(pd_op.DataType)float16,name:"llama_lm_head_0.w_0",place:(pd_op.Place)Place(undefined:0),shape:(pd_op.IntArray)[4096,32000],stop_gradient:[true]} : () -> pd_op.tensor<4096x32000xf16> + (%14) = "pd_op.data" () {dtype:(pd_op.DataType)float32,name:"top_p",place:(pd_op.Place)Place(undefined:0),shape:(pd_op.IntArray)[1],stop_gradient:[true]} : () -> pd_op.tensor<1xf32> + (%15) = "pd_op.data" () {dtype:(pd_op.DataType)int64,name:"position_ids",place:(pd_op.Place)Place(undefined:0),shape:(pd_op.IntArray)[-1,-1],stop_gradient:[true]} : () -> pd_op.tensor<-1x-1xi64> + (%16) = "pd_op.data" () {dtype:(pd_op.DataType)int64,name:"attention_mask",place:(pd_op.Place)Place(undefined:0),shape:(pd_op.IntArray)[-1,-1],stop_gradient:[true]} : () -> pd_op.tensor<-1x-1xi64> + (%17) = "pd_op.data" () {dtype:(pd_op.DataType)int64,name:"input_ids",place:(pd_op.Place)Place(undefined:0),shape:(pd_op.IntArray)[-1,-1],stop_gradient:[true]} : () -> pd_op.tensor<-1x-1xi64> + (%18) = "pd_op.shape" (%17) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1xi64>) -> pd_op.tensor<2xi32> + (%19) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)0]} : () -> pd_op.tensor<1xi64> + (%20) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)1]} : () -> pd_op.tensor<1xi64> + (%21) = "pd_op.slice" (%18, %19, %20) {axes:[(Int64)0],decrease_axis:[(Int64)0],infer_flags:[(Int64)1],is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<2xi32>, pd_op.tensor<1xi64>, pd_op.tensor<1xi64>) -> pd_op.tensor + (%22) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)1]} : () -> pd_op.tensor<1xi64> + (%23) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)2]} : () -> pd_op.tensor<1xi64> + (%24) = "pd_op.slice" (%18, %22, %23) {axes:[(Int64)0],decrease_axis:[(Int64)0],infer_flags:[(Int64)1],is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<2xi32>, pd_op.tensor<1xi64>, pd_op.tensor<1xi64>) -> pd_op.tensor + (%25) = "pd_op.cast" (%24) {dtype:(pd_op.DataType)int64,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor) -> pd_op.tensor + (%26) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)1]} : () -> pd_op.tensor<1xi64> + (%27) = "pd_op.full_with_tensor" (%26, %25) {dtype:(pd_op.DataType)int64,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<1xi64>, pd_op.tensor) -> pd_op.tensor<1xi64> + (%28) = "pd_op.shape" (%17) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1xi64>) -> pd_op.tensor<2xi32> + (%29) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)1]} : () -> pd_op.tensor<1xi64> + (%30) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)2]} : () -> pd_op.tensor<1xi64> + (%31) = "pd_op.slice" (%28, %29, %30) {axes:[(Int64)0],decrease_axis:[(Int64)0],infer_flags:[(Int64)1],is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<2xi32>, pd_op.tensor<1xi64>, pd_op.tensor<1xi64>) -> pd_op.tensor + (%32) = "pd_op.cast" (%31) {dtype:(pd_op.DataType)int64,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor) -> pd_op.tensor + (%33) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)1]} : () -> pd_op.tensor<1xi64> + (%34) = "pd_op.full_with_tensor" (%33, %32) {dtype:(pd_op.DataType)int64,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<1xi64>, pd_op.tensor) -> pd_op.tensor<1xi64> + (%35) = "pd_op.full" () {dtype:(pd_op.DataType)int32,is_persistable:[false],place:(pd_op.Place)Place(cpu),shape:(pd_op.IntArray)[],stop_gradient:[false],value:(Float)1} : () -> pd_op.tensor + (%36) = "builtin.combine" (%21, %35) {} : (pd_op.tensor, pd_op.tensor) -> vec[pd_op.tensor,pd_op.tensor] + (%37) = "pd_op.stack" (%36) {axis:(Int32)0,stop_gradient:[true]} : (vec[pd_op.tensor,pd_op.tensor]) -> pd_op.tensor<2xi32> + (%38) = "pd_op.full" () {dtype:(pd_op.DataType)float32,place:(pd_op.Place)Place(cpu),shape:(pd_op.IntArray)[1],stop_gradient:[true],value:(Float)1} : () -> pd_op.tensor<1xf32> + (%39) = "pd_op.full_with_tensor" (%37, %38) {dtype:(pd_op.DataType)bool,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<2xi32>, pd_op.tensor<1xf32>) -> pd_op.tensor<-1x1xb> + (%40) = "pd_op.full" () {dtype:(pd_op.DataType)int32,is_persistable:[false],place:(pd_op.Place)Place(cpu),shape:(pd_op.IntArray)[],stop_gradient:[false],value:(Float)1} : () -> pd_op.tensor + (%41) = "builtin.combine" (%21, %40) {} : (pd_op.tensor, pd_op.tensor) -> vec[pd_op.tensor,pd_op.tensor] + (%42) = "pd_op.stack" (%41) {axis:(Int32)0,stop_gradient:[true]} : (vec[pd_op.tensor,pd_op.tensor]) -> pd_op.tensor<2xi32> + (%43) = "pd_op.full" () {dtype:(pd_op.DataType)float32,place:(pd_op.Place)Place(cpu),shape:(pd_op.IntArray)[1],stop_gradient:[true],value:(Float)0} : () -> pd_op.tensor<1xf32> + (%44) = "pd_op.full_with_tensor" (%42, %43) {dtype:(pd_op.DataType)float16,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<2xi32>, pd_op.tensor<1xf32>) -> pd_op.tensor<-1x1xf16> + (%45) = "pd_op.shape" (%17) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1xi64>) -> pd_op.tensor<2xi32> + (%46) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)1]} : () -> pd_op.tensor<1xi64> + (%47) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)2]} : () -> pd_op.tensor<1xi64> + (%48) = "pd_op.slice" (%45, %46, %47) {axes:[(Int64)0],decrease_axis:[(Int64)0],infer_flags:[(Int64)1],is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<2xi32>, pd_op.tensor<1xi64>, pd_op.tensor<1xi64>) -> pd_op.tensor + (%49) = "pd_op.embedding" (%17, %0) {is_persistable:[false],padding_idx:(Int64)-1,sparse:false,stop_gradient:[false]} : (pd_op.tensor<-1x-1xi64>, pd_op.tensor<32000x4096xf16>) -> pd_op.tensor<-1x-1x4096xf16> + (%50) = "pd_op.shape" (%16) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1xi64>) -> pd_op.tensor<2xi32> + (%51) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)0]} : () -> pd_op.tensor<1xi64> + (%52) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)1]} : () -> pd_op.tensor<1xi64> + (%53) = "pd_op.slice" (%50, %51, %52) {axes:[(Int64)0],decrease_axis:[(Int64)0],infer_flags:[(Int64)1],is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<2xi32>, pd_op.tensor<1xi64>, pd_op.tensor<1xi64>) -> pd_op.tensor + (%54) = "pd_op.shape" (%16) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1xi64>) -> pd_op.tensor<2xi32> + (%55) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)1]} : () -> pd_op.tensor<1xi64> + (%56) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)2]} : () -> pd_op.tensor<1xi64> + (%57) = "pd_op.slice" (%54, %55, %56) {axes:[(Int64)0],decrease_axis:[(Int64)0],infer_flags:[(Int64)1],is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<2xi32>, pd_op.tensor<1xi64>, pd_op.tensor<1xi64>) -> pd_op.tensor + (%58) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)1,(Int64)2]} : () -> pd_op.tensor<2xi64> + (%59, %60) = "pd_op.unsqueeze" (%16, %58) {is_persistable:[false,false],stop_gradient:[false,false]} : (pd_op.tensor<-1x-1xi64>, pd_op.tensor<2xi64>) -> pd_op.tensor<-1x1x1x-1xi64>, <> + (%61) = "pd_op.cast" (%59) {dtype:(pd_op.DataType)bool,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x1x1x-1xi64>) -> pd_op.tensor<-1x1x1x-1xb> + (%62) = "pd_op.full" () {dtype:(pd_op.DataType)int32,is_persistable:[false],place:(pd_op.Place)Place(cpu),shape:(pd_op.IntArray)[1],stop_gradient:[false],value:(Float)1} : () -> pd_op.tensor<1xi32> + (%63) = "builtin.combine" (%53, %62, %48, %57) {} : (pd_op.tensor, pd_op.tensor<1xi32>, pd_op.tensor, pd_op.tensor) -> vec[pd_op.tensor,pd_op.tensor<1xi32>,pd_op.tensor,pd_op.tensor] + (%64) = "pd_op.expand" (%61, %63) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x1x1x-1xb>, vec[pd_op.tensor,pd_op.tensor<1xi32>,pd_op.tensor,pd_op.tensor]) -> pd_op.tensor<-1x1x-1x-1xb> + (%65) = "pd_op.full" () {dtype:(pd_op.DataType)float64,is_persistable:[false],place:(pd_op.Place)Place(undefined:0),shape:(pd_op.IntArray)[1],stop_gradient:[false],value:(Float)0} : () -> pd_op.tensor<1xf64> + (%66) = "pd_op.full" () {dtype:(pd_op.DataType)float64,is_persistable:[false],place:(pd_op.Place)Place(undefined:0),shape:(pd_op.IntArray)[1],stop_gradient:[false],value:(Float)-65504} : () -> pd_op.tensor<1xf64> + (%67) = "pd_op.full" () {dtype:(pd_op.DataType)float32,place:(pd_op.Place)Place(cpu),shape:(pd_op.IntArray)[1],stop_gradient:[true],value:(Float)0} : () -> pd_op.tensor<1xf32> + (%68) = "pd_op.full_like" (%65, %67) {dtype:(pd_op.DataType)float64,is_persistable:[false],place:(pd_op.Place)Place(undefined:0),stop_gradient:[false]} : (pd_op.tensor<1xf64>, pd_op.tensor<1xf32>) -> pd_op.tensor<1xf64> + (%69) = "pd_op.full" () {dtype:(pd_op.DataType)float32,place:(pd_op.Place)Place(cpu),shape:(pd_op.IntArray)[1],stop_gradient:[true],value:(Float)0} : () -> pd_op.tensor<1xf32> + (%70) = "pd_op.full_like" (%66, %69) {dtype:(pd_op.DataType)float64,is_persistable:[false],place:(pd_op.Place)Place(undefined:0),stop_gradient:[false]} : (pd_op.tensor<1xf64>, pd_op.tensor<1xf32>) -> pd_op.tensor<1xf64> + (%71) = "pd_op.full" () {dtype:(pd_op.DataType)float32,place:(pd_op.Place)Place(cpu),shape:(pd_op.IntArray)[1],stop_gradient:[true],value:(Float)0} : () -> pd_op.tensor<1xf32> + (%72) = "pd_op.full_like" (%64, %71) {dtype:(pd_op.DataType)bool,is_persistable:[false],place:(pd_op.Place)Place(undefined:0),stop_gradient:[false]} : (pd_op.tensor<-1x1x-1x-1xb>, pd_op.tensor<1xf32>) -> pd_op.tensor<-1x1x-1x-1xb> + (%73) = "pd_op.cast" (%72) {dtype:(pd_op.DataType)float64,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x1x-1x-1xb>) -> pd_op.tensor<-1x1x-1x-1xf64> + (%74) = "pd_op.cast" (%64) {dtype:(pd_op.DataType)float64,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x1x-1x-1xb>) -> pd_op.tensor<-1x1x-1x-1xf64> + (%75) = "pd_op.add" (%68, %70) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<1xf64>, pd_op.tensor<1xf64>) -> pd_op.tensor<1xf64> + (%76) = "pd_op.add" (%75, %73) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<1xf64>, pd_op.tensor<-1x1x-1x-1xf64>) -> pd_op.tensor<-1x1x-1x-1xf64> + (%77) = "pd_op.add" (%65, %76) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<1xf64>, pd_op.tensor<-1x1x-1x-1xf64>) -> pd_op.tensor<-1x1x-1x-1xf64> + (%78) = "pd_op.add" (%66, %76) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<1xf64>, pd_op.tensor<-1x1x-1x-1xf64>) -> pd_op.tensor<-1x1x-1x-1xf64> + (%79) = "pd_op.add" (%74, %76) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x1x-1x-1xf64>, pd_op.tensor<-1x1x-1x-1xf64>) -> pd_op.tensor<-1x1x-1x-1xf64> + (%80) = "pd_op.cast" (%79) {dtype:(pd_op.DataType)bool,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x1x-1x-1xf64>) -> pd_op.tensor<-1x1x-1x-1xb> + (%81) = "pd_op.where" (%80, %77, %78) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x1x-1x-1xb>, pd_op.tensor<-1x1x-1x-1xf64>, pd_op.tensor<-1x1x-1x-1xf64>) -> pd_op.tensor<-1x1x-1x-1xf64> + (%82) = "pd_op.cast" (%81) {dtype:(pd_op.DataType)float16,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x1x-1x-1xf64>) -> pd_op.tensor<-1x1x-1x-1xf16> + (%83) = "pd_op.cast" (%49) {dtype:(pd_op.DataType)float32,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x4096xf16>) -> pd_op.tensor<-1x-1x4096xf32> + (%84) = "pd_op.pow" (%83) {is_persistable:[false],stop_gradient:[false],y:(Float)2} : (pd_op.tensor<-1x-1x4096xf32>) -> pd_op.tensor<-1x-1x4096xf32> + (%85) = "pd_op.mean" (%84) {axis:(pd_op.IntArray)[-1],is_persistable:[false],keepdim:true,stop_gradient:[false]} : (pd_op.tensor<-1x-1x4096xf32>) -> pd_op.tensor<-1x-1x1xf32> + (%86) = "pd_op.full" () {dtype:(pd_op.DataType)float32,place:(pd_op.Place)Place(cpu),shape:(pd_op.IntArray)[1],stop_gradient:[true],value:(Float)1} : () -> pd_op.tensor<1xf32> + (%87) = "pd_op.scale" (%85, %86) {bias:(Float)1e-06,bias_after_scale:true,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x1xf32>, pd_op.tensor<1xf32>) -> pd_op.tensor<-1x-1x1xf32> + (%88) = "pd_op.rsqrt" (%87) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x1xf32>) -> pd_op.tensor<-1x-1x1xf32> + (%89) = "pd_op.multiply" (%88, %83) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x1xf32>, pd_op.tensor<-1x-1x4096xf32>) -> pd_op.tensor<-1x-1x4096xf32> + (%90) = "pd_op.cast" (%89) {dtype:(pd_op.DataType)float16,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x4096xf32>) -> pd_op.tensor<-1x-1x4096xf16> + (%91) = "pd_op.multiply" (%90, %1) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x4096xf16>, pd_op.tensor<4096xf16>) -> pd_op.tensor<-1x-1x4096xf16> + (%92) = "pd_op.matmul" (%91, %2) {is_persistable:[false],stop_gradient:[false],transpose_x:false,transpose_y:false} : (pd_op.tensor<-1x-1x4096xf16>, pd_op.tensor<4096x4096xf16>) -> pd_op.tensor<-1x-1x4096xf16> + (%93) = "pd_op.matmul" (%91, %3) {is_persistable:[false],stop_gradient:[false],transpose_x:false,transpose_y:false} : (pd_op.tensor<-1x-1x4096xf16>, pd_op.tensor<4096x4096xf16>) -> pd_op.tensor<-1x-1x4096xf16> + (%94) = "pd_op.matmul" (%91, %4) {is_persistable:[false],stop_gradient:[false],transpose_x:false,transpose_y:false} : (pd_op.tensor<-1x-1x4096xf16>, pd_op.tensor<4096x4096xf16>) -> pd_op.tensor<-1x-1x4096xf16> + (%95) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)0,(Int64)0,(Int64)32,(Int64)128]} : () -> pd_op.tensor<4xi64> + (%96, %97) = "pd_op.reshape" (%92, %95) {is_persistable:[false,false],stop_gradient:[false,false]} : (pd_op.tensor<-1x-1x4096xf16>, pd_op.tensor<4xi64>) -> pd_op.tensor<-1x-1x32x128xf16>, pd_op.tensor<0x-1x-1x4096xf16> + (%98) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)0,(Int64)0,(Int64)32,(Int64)128]} : () -> pd_op.tensor<4xi64> + (%99, %100) = "pd_op.reshape" (%93, %98) {is_persistable:[false,false],stop_gradient:[false,false]} : (pd_op.tensor<-1x-1x4096xf16>, pd_op.tensor<4xi64>) -> pd_op.tensor<-1x-1x32x128xf16>, pd_op.tensor<0x-1x-1x4096xf16> + (%101) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)0,(Int64)0,(Int64)32,(Int64)128]} : () -> pd_op.tensor<4xi64> + (%102, %103) = "pd_op.reshape" (%94, %101) {is_persistable:[false,false],stop_gradient:[false,false]} : (pd_op.tensor<-1x-1x4096xf16>, pd_op.tensor<4xi64>) -> pd_op.tensor<-1x-1x32x128xf16>, pd_op.tensor<0x-1x-1x4096xf16> + (%104) = "pd_op.shape" (%99) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x32x128xf16>) -> pd_op.tensor<4xi32> + (%105) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)1]} : () -> pd_op.tensor<1xi64> + (%106) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)2]} : () -> pd_op.tensor<1xi64> + (%107) = "pd_op.slice" (%104, %105, %106) {axes:[(Int64)0],decrease_axis:[(Int64)0],infer_flags:[(Int64)1],is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<4xi32>, pd_op.tensor<1xi64>, pd_op.tensor<1xi64>) -> pd_op.tensor + (%108) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)0]} : () -> pd_op.tensor<1xi64> + (%109) = "builtin.combine" (%107) {} : (pd_op.tensor) -> vec[pd_op.tensor] + (%110) = "pd_op.slice" (%5, %108, %109) {axes:[(Int64)1],decrease_axis:[],infer_flags:[(Int64)-1],is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<1x2048x1x128xf32>, pd_op.tensor<1xi64>, vec[pd_op.tensor]) -> pd_op.tensor<1x-1x1x128xf32> + (%111) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)0]} : () -> pd_op.tensor<1xi64> + (%112) = "builtin.combine" (%107) {} : (pd_op.tensor) -> vec[pd_op.tensor] + (%113) = "pd_op.slice" (%6, %111, %112) {axes:[(Int64)1],decrease_axis:[],infer_flags:[(Int64)-1],is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<1x2048x1x128xf32>, pd_op.tensor<1xi64>, vec[pd_op.tensor]) -> pd_op.tensor<1x-1x1x128xf32> + (%114) = "pd_op.cast" (%110) {dtype:(pd_op.DataType)float16,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<1x-1x1x128xf32>) -> pd_op.tensor<1x-1x1x128xf16> + (%115) = "pd_op.cast" (%113) {dtype:(pd_op.DataType)float16,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<1x-1x1x128xf32>) -> pd_op.tensor<1x-1x1x128xf16> + (%116) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)0,(Int64)2]} : () -> pd_op.tensor<2xi64> + (%117, %118) = "pd_op.squeeze" (%114, %116) {is_persistable:[false,false],stop_gradient:[false,false]} : (pd_op.tensor<1x-1x1x128xf16>, pd_op.tensor<2xi64>) -> pd_op.tensor<-1x128xf16>, <> + (%119) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)0,(Int64)2]} : () -> pd_op.tensor<2xi64> + (%120, %121) = "pd_op.squeeze" (%115, %119) {is_persistable:[false,false],stop_gradient:[false,false]} : (pd_op.tensor<1x-1x1x128xf16>, pd_op.tensor<2xi64>) -> pd_op.tensor<-1x128xf16>, <> + (%122) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)-1]} : () -> pd_op.tensor<1xi64> + (%123, %124) = "pd_op.unsqueeze" (%15, %122) {is_persistable:[false,false],stop_gradient:[false,false]} : (pd_op.tensor<-1x-1xi64>, pd_op.tensor<1xi64>) -> pd_op.tensor<-1x-1x1xi64>, <> + (%125) = "pd_op.gather_nd" (%117, %123) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x128xf16>, pd_op.tensor<-1x-1x1xi64>) -> pd_op.tensor<-1x-1x128xf16> + (%126) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)2]} : () -> pd_op.tensor<1xi64> + (%127, %128) = "pd_op.unsqueeze" (%125, %126) {is_persistable:[false,false],stop_gradient:[false,false]} : (pd_op.tensor<-1x-1x128xf16>, pd_op.tensor<1xi64>) -> pd_op.tensor<-1x-1x1x128xf16>, <> + (%129) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)-1]} : () -> pd_op.tensor<1xi64> + (%130, %131) = "pd_op.unsqueeze" (%15, %129) {is_persistable:[false,false],stop_gradient:[false,false]} : (pd_op.tensor<-1x-1xi64>, pd_op.tensor<1xi64>) -> pd_op.tensor<-1x-1x1xi64>, <> + (%132) = "pd_op.gather_nd" (%120, %130) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x128xf16>, pd_op.tensor<-1x-1x1xi64>) -> pd_op.tensor<-1x-1x128xf16> + (%133) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)2]} : () -> pd_op.tensor<1xi64> + (%134, %135) = "pd_op.unsqueeze" (%132, %133) {is_persistable:[false,false],stop_gradient:[false,false]} : (pd_op.tensor<-1x-1x128xf16>, pd_op.tensor<1xi64>) -> pd_op.tensor<-1x-1x1x128xf16>, <> + (%136) = "pd_op.multiply" (%96, %127) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x32x128xf16>, pd_op.tensor<-1x-1x1x128xf16>) -> pd_op.tensor<-1x-1x32x128xf16> + (%137) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)0]} : () -> pd_op.tensor<1xi64> + (%138) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)64]} : () -> pd_op.tensor<1xi64> + (%139) = "pd_op.slice" (%96, %137, %138) {axes:[(Int64)3],decrease_axis:[],infer_flags:[(Int64)1],is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x32x128xf16>, pd_op.tensor<1xi64>, pd_op.tensor<1xi64>) -> pd_op.tensor<-1x-1x32x64xf16> + (%140) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)64]} : () -> pd_op.tensor<1xi64> + (%141) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)2147483647]} : () -> pd_op.tensor<1xi64> + (%142) = "pd_op.slice" (%96, %140, %141) {axes:[(Int64)3],decrease_axis:[],infer_flags:[(Int64)1],is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x32x128xf16>, pd_op.tensor<1xi64>, pd_op.tensor<1xi64>) -> pd_op.tensor<-1x-1x32x64xf16> + (%143) = "pd_op.full" () {dtype:(pd_op.DataType)float32,place:(pd_op.Place)Place(cpu),shape:(pd_op.IntArray)[1],stop_gradient:[true],value:(Float)-1} : () -> pd_op.tensor<1xf32> + (%144) = "pd_op.scale" (%142, %143) {bias:(Float)0,bias_after_scale:true,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x32x64xf16>, pd_op.tensor<1xf32>) -> pd_op.tensor<-1x-1x32x64xf16> + (%145) = "builtin.combine" (%144, %139) {} : (pd_op.tensor<-1x-1x32x64xf16>, pd_op.tensor<-1x-1x32x64xf16>) -> vec[pd_op.tensor<-1x-1x32x64xf16>,pd_op.tensor<-1x-1x32x64xf16>] + (%146) = "pd_op.full" () {dtype:(pd_op.DataType)int32,place:(pd_op.Place)Place(cpu),shape:(pd_op.IntArray)[1],stop_gradient:[true],value:(Float)-1} : () -> pd_op.tensor<1xi32> + (%147) = "pd_op.concat" (%145, %146) {is_persistable:[false],stop_gradient:[false]} : (vec[pd_op.tensor<-1x-1x32x64xf16>,pd_op.tensor<-1x-1x32x64xf16>], pd_op.tensor<1xi32>) -> pd_op.tensor<-1x-1x32x128xf16> + (%148) = "pd_op.multiply" (%147, %134) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x32x128xf16>, pd_op.tensor<-1x-1x1x128xf16>) -> pd_op.tensor<-1x-1x32x128xf16> + (%149) = "pd_op.add" (%136, %148) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x32x128xf16>, pd_op.tensor<-1x-1x32x128xf16>) -> pd_op.tensor<-1x-1x32x128xf16> + (%150) = "pd_op.multiply" (%99, %127) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x32x128xf16>, pd_op.tensor<-1x-1x1x128xf16>) -> pd_op.tensor<-1x-1x32x128xf16> + (%151) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)0]} : () -> pd_op.tensor<1xi64> + (%152) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)64]} : () -> pd_op.tensor<1xi64> + (%153) = "pd_op.slice" (%99, %151, %152) {axes:[(Int64)3],decrease_axis:[],infer_flags:[(Int64)1],is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x32x128xf16>, pd_op.tensor<1xi64>, pd_op.tensor<1xi64>) -> pd_op.tensor<-1x-1x32x64xf16> + (%154) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)64]} : () -> pd_op.tensor<1xi64> + (%155) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)2147483647]} : () -> pd_op.tensor<1xi64> + (%156) = "pd_op.slice" (%99, %154, %155) {axes:[(Int64)3],decrease_axis:[],infer_flags:[(Int64)1],is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x32x128xf16>, pd_op.tensor<1xi64>, pd_op.tensor<1xi64>) -> pd_op.tensor<-1x-1x32x64xf16> + (%157) = "pd_op.full" () {dtype:(pd_op.DataType)float32,place:(pd_op.Place)Place(cpu),shape:(pd_op.IntArray)[1],stop_gradient:[true],value:(Float)-1} : () -> pd_op.tensor<1xf32> + (%158) = "pd_op.scale" (%156, %157) {bias:(Float)0,bias_after_scale:true,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x32x64xf16>, pd_op.tensor<1xf32>) -> pd_op.tensor<-1x-1x32x64xf16> + (%159) = "builtin.combine" (%158, %153) {} : (pd_op.tensor<-1x-1x32x64xf16>, pd_op.tensor<-1x-1x32x64xf16>) -> vec[pd_op.tensor<-1x-1x32x64xf16>,pd_op.tensor<-1x-1x32x64xf16>] + (%160) = "pd_op.full" () {dtype:(pd_op.DataType)int32,place:(pd_op.Place)Place(cpu),shape:(pd_op.IntArray)[1],stop_gradient:[true],value:(Float)-1} : () -> pd_op.tensor<1xi32> + (%161) = "pd_op.concat" (%159, %160) {is_persistable:[false],stop_gradient:[false]} : (vec[pd_op.tensor<-1x-1x32x64xf16>,pd_op.tensor<-1x-1x32x64xf16>], pd_op.tensor<1xi32>) -> pd_op.tensor<-1x-1x32x128xf16> + (%162) = "pd_op.multiply" (%161, %134) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x32x128xf16>, pd_op.tensor<-1x-1x1x128xf16>) -> pd_op.tensor<-1x-1x32x128xf16> + (%163) = "pd_op.add" (%150, %162) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x32x128xf16>, pd_op.tensor<-1x-1x32x128xf16>) -> pd_op.tensor<-1x-1x32x128xf16> + (%164) = "pd_op.shape" (%149) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x32x128xf16>) -> pd_op.tensor<4xi32> + (%165) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)0]} : () -> pd_op.tensor<1xi64> + (%166) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)1]} : () -> pd_op.tensor<1xi64> + (%167) = "pd_op.slice" (%164, %165, %166) {axes:[(Int64)0],decrease_axis:[(Int64)0],infer_flags:[(Int64)1],is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<4xi32>, pd_op.tensor<1xi64>, pd_op.tensor<1xi64>) -> pd_op.tensor + (%168) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)1]} : () -> pd_op.tensor<1xi64> + (%169) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)2]} : () -> pd_op.tensor<1xi64> + (%170) = "pd_op.slice" (%164, %168, %169) {axes:[(Int64)0],decrease_axis:[(Int64)0],infer_flags:[(Int64)1],is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<4xi32>, pd_op.tensor<1xi64>, pd_op.tensor<1xi64>) -> pd_op.tensor + (%171) = "pd_op.shape" (%102) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x32x128xf16>) -> pd_op.tensor<4xi32> + (%172) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)1]} : () -> pd_op.tensor<1xi64> + (%173) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)2]} : () -> pd_op.tensor<1xi64> + (%174) = "pd_op.slice" (%171, %172, %173) {axes:[(Int64)0],decrease_axis:[(Int64)0],infer_flags:[(Int64)1],is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<4xi32>, pd_op.tensor<1xi64>, pd_op.tensor<1xi64>) -> pd_op.tensor + (%175) = "pd_op.transpose" (%149) {is_persistable:[false],perm:[(Int32)0,(Int32)2,(Int32)1,(Int32)3],stop_gradient:[false]} : (pd_op.tensor<-1x-1x32x128xf16>) -> pd_op.tensor<-1x32x-1x128xf16> + (%176) = "pd_op.transpose" (%163) {is_persistable:[false],perm:[(Int32)0,(Int32)2,(Int32)1,(Int32)3],stop_gradient:[false]} : (pd_op.tensor<-1x-1x32x128xf16>) -> pd_op.tensor<-1x32x-1x128xf16> + (%177) = "pd_op.transpose" (%102) {is_persistable:[false],perm:[(Int32)0,(Int32)2,(Int32)1,(Int32)3],stop_gradient:[false]} : (pd_op.tensor<-1x-1x32x128xf16>) -> pd_op.tensor<-1x32x-1x128xf16> + (%178) = "pd_op.full" () {dtype:(pd_op.DataType)float32,place:(pd_op.Place)Place(cpu),shape:(pd_op.IntArray)[1],stop_gradient:[true],value:(Float)0.0883883} : () -> pd_op.tensor<1xf32> + (%179) = "pd_op.scale" (%175, %178) {bias:(Float)0,bias_after_scale:true,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x32x-1x128xf16>, pd_op.tensor<1xf32>) -> pd_op.tensor<-1x32x-1x128xf16> + (%180) = "pd_op.transpose" (%176) {is_persistable:[false],perm:[(Int32)0,(Int32)1,(Int32)3,(Int32)2],stop_gradient:[false]} : (pd_op.tensor<-1x32x-1x128xf16>) -> pd_op.tensor<-1x32x128x-1xf16> + (%181) = "pd_op.matmul" (%179, %180) {is_persistable:[false],stop_gradient:[false],transpose_x:false,transpose_y:false} : (pd_op.tensor<-1x32x-1x128xf16>, pd_op.tensor<-1x32x128x-1xf16>) -> pd_op.tensor<-1x32x-1x-1xf16> + (%182) = "pd_op.full" () {dtype:(pd_op.DataType)int32,is_persistable:[false],place:(pd_op.Place)Place(cpu),shape:(pd_op.IntArray)[1],stop_gradient:[false],value:(Float)1} : () -> pd_op.tensor<1xi32> + (%183) = "builtin.combine" (%167, %182, %170, %174) {} : (pd_op.tensor, pd_op.tensor<1xi32>, pd_op.tensor, pd_op.tensor) -> vec[pd_op.tensor,pd_op.tensor<1xi32>,pd_op.tensor,pd_op.tensor] + (%184, %185) = "pd_op.reshape" (%82, %183) {is_persistable:[false,false],stop_gradient:[false,false]} : (pd_op.tensor<-1x1x-1x-1xf16>, vec[pd_op.tensor,pd_op.tensor<1xi32>,pd_op.tensor,pd_op.tensor]) -> pd_op.tensor<-1x1x-1x-1xf16>, pd_op.tensor<0x-1x1x-1x-1xf16> + (%186) = "pd_op.add" (%181, %184) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x32x-1x-1xf16>, pd_op.tensor<-1x1x-1x-1xf16>) -> pd_op.tensor<-1x32x-1x-1xf16> + (%187) = "pd_op.cast" (%186) {dtype:(pd_op.DataType)float32,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x32x-1x-1xf16>) -> pd_op.tensor<-1x32x-1x-1xf32> + (%188) = "pd_op.softmax" (%187) {axis:(Int32)-1,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x32x-1x-1xf32>) -> pd_op.tensor<-1x32x-1x-1xf32> + (%189) = "pd_op.cast" (%188) {dtype:(pd_op.DataType)float16,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x32x-1x-1xf32>) -> pd_op.tensor<-1x32x-1x-1xf16> + (%190) = "pd_op.matmul" (%189, %177) {is_persistable:[false],stop_gradient:[false],transpose_x:false,transpose_y:false} : (pd_op.tensor<-1x32x-1x-1xf16>, pd_op.tensor<-1x32x-1x128xf16>) -> pd_op.tensor<-1x32x-1x128xf16> + (%191) = "pd_op.transpose" (%190) {is_persistable:[false],perm:[(Int32)0,(Int32)2,(Int32)1,(Int32)3],stop_gradient:[false]} : (pd_op.tensor<-1x32x-1x128xf16>) -> pd_op.tensor<-1x-1x32x128xf16> + (%192) = "pd_op.full" () {dtype:(pd_op.DataType)int32,is_persistable:[false],place:(pd_op.Place)Place(cpu),shape:(pd_op.IntArray)[1],stop_gradient:[false],value:(Float)4096} : () -> pd_op.tensor<1xi32> + (%193) = "builtin.combine" (%167, %170, %192) {} : (pd_op.tensor, pd_op.tensor, pd_op.tensor<1xi32>) -> vec[pd_op.tensor,pd_op.tensor,pd_op.tensor<1xi32>] + (%194, %195) = "pd_op.reshape" (%191, %193) {is_persistable:[false,false],stop_gradient:[false,false]} : (pd_op.tensor<-1x-1x32x128xf16>, vec[pd_op.tensor,pd_op.tensor,pd_op.tensor<1xi32>]) -> pd_op.tensor<-1x-1x4096xf16>, pd_op.tensor<0x-1x-1x32x128xf16> + (%196) = "pd_op.matmul" (%194, %7) {is_persistable:[false],stop_gradient:[false],transpose_x:false,transpose_y:false} : (pd_op.tensor<-1x-1x4096xf16>, pd_op.tensor<4096x4096xf16>) -> pd_op.tensor<-1x-1x4096xf16> + (%197) = "pd_op.add" (%49, %196) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x4096xf16>, pd_op.tensor<-1x-1x4096xf16>) -> pd_op.tensor<-1x-1x4096xf16> + (%198) = "pd_op.cast" (%197) {dtype:(pd_op.DataType)float32,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x4096xf16>) -> pd_op.tensor<-1x-1x4096xf32> + (%199) = "pd_op.pow" (%198) {is_persistable:[false],stop_gradient:[false],y:(Float)2} : (pd_op.tensor<-1x-1x4096xf32>) -> pd_op.tensor<-1x-1x4096xf32> + (%200) = "pd_op.mean" (%199) {axis:(pd_op.IntArray)[-1],is_persistable:[false],keepdim:true,stop_gradient:[false]} : (pd_op.tensor<-1x-1x4096xf32>) -> pd_op.tensor<-1x-1x1xf32> + (%201) = "pd_op.full" () {dtype:(pd_op.DataType)float32,place:(pd_op.Place)Place(cpu),shape:(pd_op.IntArray)[1],stop_gradient:[true],value:(Float)1} : () -> pd_op.tensor<1xf32> + (%202) = "pd_op.scale" (%200, %201) {bias:(Float)1e-06,bias_after_scale:true,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x1xf32>, pd_op.tensor<1xf32>) -> pd_op.tensor<-1x-1x1xf32> + (%203) = "pd_op.rsqrt" (%202) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x1xf32>) -> pd_op.tensor<-1x-1x1xf32> + (%204) = "pd_op.multiply" (%203, %198) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x1xf32>, pd_op.tensor<-1x-1x4096xf32>) -> pd_op.tensor<-1x-1x4096xf32> + (%205) = "pd_op.cast" (%204) {dtype:(pd_op.DataType)float16,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x4096xf32>) -> pd_op.tensor<-1x-1x4096xf16> + (%206) = "pd_op.multiply" (%205, %8) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x4096xf16>, pd_op.tensor<4096xf16>) -> pd_op.tensor<-1x-1x4096xf16> + (%207) = "pd_op.matmul" (%206, %9) {is_persistable:[false],stop_gradient:[false],transpose_x:false,transpose_y:false} : (pd_op.tensor<-1x-1x4096xf16>, pd_op.tensor<4096x11008xf16>) -> pd_op.tensor<-1x-1x11008xf16> + (%208) = "pd_op.matmul" (%206, %10) {is_persistable:[false],stop_gradient:[false],transpose_x:false,transpose_y:false} : (pd_op.tensor<-1x-1x4096xf16>, pd_op.tensor<4096x11008xf16>) -> pd_op.tensor<-1x-1x11008xf16> + (%209) = "pd_op.swiglu" (%207, %208) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x11008xf16>, pd_op.tensor<-1x-1x11008xf16>) -> pd_op.tensor<-1x-1x11008xf16> + (%210) = "pd_op.matmul" (%209, %11) {is_persistable:[false],stop_gradient:[false],transpose_x:false,transpose_y:false} : (pd_op.tensor<-1x-1x11008xf16>, pd_op.tensor<11008x4096xf16>) -> pd_op.tensor<-1x-1x4096xf16> + (%211) = "pd_op.add" (%197, %210) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x4096xf16>, pd_op.tensor<-1x-1x4096xf16>) -> pd_op.tensor<-1x-1x4096xf16> + (%212) = "pd_op.cast" (%211) {dtype:(pd_op.DataType)float32,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x4096xf16>) -> pd_op.tensor<-1x-1x4096xf32> + (%213) = "pd_op.pow" (%212) {is_persistable:[false],stop_gradient:[false],y:(Float)2} : (pd_op.tensor<-1x-1x4096xf32>) -> pd_op.tensor<-1x-1x4096xf32> + (%214) = "pd_op.mean" (%213) {axis:(pd_op.IntArray)[-1],is_persistable:[false],keepdim:true,stop_gradient:[false]} : (pd_op.tensor<-1x-1x4096xf32>) -> pd_op.tensor<-1x-1x1xf32> + (%215) = "pd_op.full" () {dtype:(pd_op.DataType)float32,place:(pd_op.Place)Place(cpu),shape:(pd_op.IntArray)[1],stop_gradient:[true],value:(Float)1} : () -> pd_op.tensor<1xf32> + (%216) = "pd_op.scale" (%214, %215) {bias:(Float)1e-06,bias_after_scale:true,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x1xf32>, pd_op.tensor<1xf32>) -> pd_op.tensor<-1x-1x1xf32> + (%217) = "pd_op.rsqrt" (%216) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x1xf32>) -> pd_op.tensor<-1x-1x1xf32> + (%218) = "pd_op.multiply" (%217, %212) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x1xf32>, pd_op.tensor<-1x-1x4096xf32>) -> pd_op.tensor<-1x-1x4096xf32> + (%219) = "pd_op.cast" (%218) {dtype:(pd_op.DataType)float16,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x4096xf32>) -> pd_op.tensor<-1x-1x4096xf16> + (%220) = "pd_op.multiply" (%219, %12) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x4096xf16>, pd_op.tensor<4096xf16>) -> pd_op.tensor<-1x-1x4096xf16> + (%221) = "pd_op.matmul" (%220, %13) {is_persistable:[false],stop_gradient:[false],transpose_x:false,transpose_y:false} : (pd_op.tensor<-1x-1x4096xf16>, pd_op.tensor<4096x32000xf16>) -> pd_op.tensor<-1x-1x32000xf16> + (%222) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)-1]} : () -> pd_op.tensor<1xi64> + (%223) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)2147483647]} : () -> pd_op.tensor<1xi64> + (%224) = "pd_op.slice" (%221, %222, %223) {axes:[(Int64)1],decrease_axis:[(Int64)1],infer_flags:[(Int64)1],is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1x32000xf16>, pd_op.tensor<1xi64>, pd_op.tensor<1xi64>) -> pd_op.tensor<-1x32000xf16> + (%225) = "pd_op.softmax" (%224) {axis:(Int32)-1,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x32000xf16>) -> pd_op.tensor<-1x32000xf16> + (%226) = "pd_op.log_softmax" (%224) {axis:(Int32)-1,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x32000xf16>) -> pd_op.tensor<-1x32000xf16> + (%227) = "pd_op.shape" (%225) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x32000xf16>) -> pd_op.tensor<2xi32> + (%228) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)0]} : () -> pd_op.tensor<1xi64> + (%229) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)1]} : () -> pd_op.tensor<1xi64> + (%230) = "pd_op.slice" (%227, %228, %229) {axes:[(Int64)0],decrease_axis:[(Int64)0],infer_flags:[(Int64)1],is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<2xi32>, pd_op.tensor<1xi64>, pd_op.tensor<1xi64>) -> pd_op.tensor + (%231) = "pd_op.cast" (%14) {dtype:(pd_op.DataType)float16,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<1xf32>) -> pd_op.tensor<1xf16> + (%232) = "pd_op.full" () {dtype:(pd_op.DataType)int32,is_persistable:[false],place:(pd_op.Place)Place(cpu),shape:(pd_op.IntArray)[],stop_gradient:[false],value:(Float)1} : () -> pd_op.tensor + (%233) = "builtin.combine" (%230, %232) {} : (pd_op.tensor, pd_op.tensor) -> vec[pd_op.tensor,pd_op.tensor] + (%234) = "pd_op.stack" (%233) {axis:(Int32)0,stop_gradient:[true]} : (vec[pd_op.tensor,pd_op.tensor]) -> pd_op.tensor<2xi32> + (%235) = "pd_op.full_with_tensor" (%234, %231) {dtype:(pd_op.DataType)float16,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<2xi32>, pd_op.tensor<1xf16>) -> pd_op.tensor<-1x1xf16> + (%236, %237) = "pd_op.top_p_sampling" (%225, %235, <>) {is_persistable:[false,false],seed:(Int32)-1,stop_gradient:[false,false]} : (pd_op.tensor<-1x32000xf16>, pd_op.tensor<-1x1xf16>, <>) -> pd_op.tensor<-1x1xf16>, pd_op.tensor<-1x1xi64> + (%238) = "pd_op.index_sample" (%226, %237) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x32000xf16>, pd_op.tensor<-1x1xi64>) -> pd_op.tensor<-1x1xf16> + (%239) = "pd_op.subtract" (%27, %34) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<1xi64>, pd_op.tensor<1xi64>) -> pd_op.tensor<1xi64> + (%240) = "pd_op.cast" (%239) {dtype:(pd_op.DataType)float16,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<1xi64>) -> pd_op.tensor<1xf16> + (%241) = "pd_op.multiply" (%44, %240) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x1xf16>, pd_op.tensor<1xf16>) -> pd_op.tensor<-1x1xf16> + (%242) = "pd_op.add" (%241, %238) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x1xf16>, pd_op.tensor<-1x1xf16>) -> pd_op.tensor<-1x1xf16> + (%243) = "pd_op.full" () {dtype:(pd_op.DataType)float32,place:(pd_op.Place)Place(cpu),shape:(pd_op.IntArray)[1],stop_gradient:[true],value:(Float)1} : () -> pd_op.tensor<1xf32> + (%244) = "pd_op.scale" (%239, %243) {bias:(Float)1,bias_after_scale:true,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<1xi64>, pd_op.tensor<1xf32>) -> pd_op.tensor<1xi64> + (%245) = "pd_op.cast" (%244) {dtype:(pd_op.DataType)float16,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<1xi64>) -> pd_op.tensor<1xf16> + (%246) = "pd_op.divide" (%242, %245) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x1xf16>, pd_op.tensor<1xf16>) -> pd_op.tensor<-1x1xf16> + (%247) = "pd_op.where" (%39, %246, %44) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x1xb>, pd_op.tensor<-1x1xf16>, pd_op.tensor<-1x1xf16>) -> pd_op.tensor<-1x1xf16> + (%248) = "pd_op.full" () {dtype:(pd_op.DataType)float32,place:(pd_op.Place)Place(cpu),shape:(pd_op.IntArray)[1],stop_gradient:[true],value:(Float)0} : () -> pd_op.tensor<1xf32> + (%249) = "pd_op.full_like" (%237, %248) {dtype:(pd_op.DataType)int64,is_persistable:[false],place:(pd_op.Place)Place(undefined:0),stop_gradient:[false]} : (pd_op.tensor<-1x1xi64>, pd_op.tensor<1xf32>) -> pd_op.tensor<-1x1xi64> + (%250) = "pd_op.where" (%39, %237, %249) {is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x1xb>, pd_op.tensor<-1x1xi64>, pd_op.tensor<-1x1xi64>) -> pd_op.tensor<-1x1xi64> + (%251) = "builtin.combine" (%17, %250) {} : (pd_op.tensor<-1x-1xi64>, pd_op.tensor<-1x1xi64>) -> vec[pd_op.tensor<-1x-1xi64>,pd_op.tensor<-1x1xi64>] + (%252) = "pd_op.full" () {dtype:(pd_op.DataType)int32,place:(pd_op.Place)Place(cpu),shape:(pd_op.IntArray)[1],stop_gradient:[true],value:(Float)1} : () -> pd_op.tensor<1xi32> + (%253) = "pd_op.concat" (%251, %252) {is_persistable:[false],stop_gradient:[false]} : (vec[pd_op.tensor<-1x-1xi64>,pd_op.tensor<-1x1xi64>], pd_op.tensor<1xi32>) -> pd_op.tensor<-1x-1xi64> + (%254) = "builtin.combine" (%31) {} : (pd_op.tensor) -> vec[pd_op.tensor] + (%255) = "pd_op.full_int_array" () {dtype:(pd_op.DataType)int64,place:(pd_op.Place)Place(cpu),stop_gradient:[true],value:[(Int64)2147483647]} : () -> pd_op.tensor<1xi64> + (%256) = "pd_op.slice" (%253, %254, %255) {axes:[(Int64)1],decrease_axis:[],infer_flags:[(Int64)-1],is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1xi64>, vec[pd_op.tensor], pd_op.tensor<1xi64>) -> pd_op.tensor<-1x-1xi64> + (%257) = "pd_op.full" () {dtype:(pd_op.DataType)float32,place:(pd_op.Place)Place(cpu),shape:(pd_op.IntArray)[1],stop_gradient:[true],value:(Float)1} : () -> pd_op.tensor<1xf32> + (%258) = "pd_op.scale" (%256, %257) {bias:(Float)0,bias_after_scale:true,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x-1xi64>, pd_op.tensor<1xf32>) -> pd_op.tensor<-1x-1xi64> + (%259) = "pd_op.full" () {dtype:(pd_op.DataType)float32,place:(pd_op.Place)Place(cpu),shape:(pd_op.IntArray)[1],stop_gradient:[true],value:(Float)1} : () -> pd_op.tensor<1xf32> + (%260) = "pd_op.scale" (%247, %259) {bias:(Float)0,bias_after_scale:true,is_persistable:[false],stop_gradient:[false]} : (pd_op.tensor<-1x1xf16>, pd_op.tensor<1xf32>) -> pd_op.tensor<-1x1xf16> + (%261) = "pd_op.fetch" (%258) {col:(Int32)0,name:"save_infer_model/scale_0.tmp_0"} : (pd_op.tensor<-1x-1xi64>) -> pd_op.tensor<-1x-1xi64> + (%262) = "pd_op.fetch" (%260) {col:(Int32)1,name:"save_infer_model/scale_1.tmp_0"} : (pd_op.tensor<-1x1xf16>) -> pd_op.tensor<-1x1xf16> +} diff --git a/test/ir/pir/cinn/symbolic/test_simple_llama_dy.py b/test/ir/pir/cinn/symbolic/test_simple_llama_dy.py new file mode 100644 index 0000000000000..b23818368f30b --- /dev/null +++ b/test/ir/pir/cinn/symbolic/test_simple_llama_dy.py @@ -0,0 +1,217 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import sys +import unittest + +import numpy as np + +import paddle +from paddle.base import core +from paddle.base.data_feeder import convert_dtype + +np.random.seed(2024) + + +class ProgramInfo: + def __init__(self, program, feeds, fetchs): + self.program = program + # {name: [shape, dtype]} + self.feeds = feeds + # {name: shape} + self.fetchs = fetchs + + def random_feeds(self): + feed_dict = {} + for name, info in self.feeds.items(): + data = np.random.uniform(low=-0.5, high=0.5, size=info[0]).astype( + convert_dtype(info[1]) + ) + feed_dict[name] = data + + return feed_dict + + def fetch_list(self): + return list(self.fetchs.keys()) + + +class Parser: + def __init__(self): + self.feed_op_name = 'pd_op.data' + self.fetch_op_name = 'pd_op.fetch' + self.have_dy_shape = False + + def run(self, file): + program = self.load_from(file) + for op in program.global_block().ops: + if op.name() == "pd_op.reshape": + if ( + op.result(1).initialized() + and not op.result(1).use_empty() + and op.result(1).first_use().owner().name() == "pd_op.fetch" + ): + program.global_block().remove_op( + op.result(1).first_use().owner() + ) + + if op.name() == "pd_op.squeeze": + if ( + op.result(1).initialized() + and not op.result(1).use_empty() + and op.result(1).first_use().owner().name() == "pd_op.fetch" + ): + program.global_block().remove_op( + op.result(1).first_use().owner() + ) + + if op.name() == "pd_op.unsqueeze": + if ( + op.result(1).initialized() + and not op.result(1).use_empty() + and op.result(1).first_use().owner().name() == "pd_op.fetch" + ): + program.global_block().remove_op( + op.result(1).first_use().owner() + ) + + if ( + op.name() == "pd_op.batch_norm_" + or op.name() == "pd_op.batch_norm" + ): + if ( + op.result(5).initialized() + and not op.result(5).use_empty() + and op.result(5).first_use().owner().name() == "pd_op.fetch" + ): + program.global_block().remove_op( + op.result(5).first_use().owner() + ) + + feeds = self.parse_feeds(program) + fetchs = self.parse_fetchs(program) + + return ProgramInfo(program, feeds, fetchs) + + def load_from(self, file): + with open(file, 'r') as f: + content = f.read() + + return paddle.pir.parse_program(content) + + def parse_feeds(self, program): + feeds = {} + for op in program.global_block().ops: + if op.name() == self.feed_op_name: + in_val = op.result(0) + # shape, dtype + shape = [] + for s in in_val.shape: + if s == -1: + s = 1 + self.have_dy_shape = True + shape.append(s) + info = [shape, in_val.dtype] + feeds[op.attrs()['name']] = info + + return feeds + + def parse_fetchs(self, program): + fetchs = {} + for op in program.global_block().ops: + if op.name() == self.fetch_op_name: + in_val = op.operand_source(0) + fetchs[op.attrs()['name']] = in_val.shape + + return fetchs + + +class TestTask(unittest.TestCase): + def setUp(self): + paddle.enable_static() + file_dir = os.path.dirname(os.path.abspath(__file__)) + self.file_path = os.path.join(file_dir, args.file_path) + + def test_phi(self): + self.check_infer(enable_cinn=False) + + def test_llama_eval(self): + parser = Parser() + program_info = parser.run(self.file_path) + + feed = program_info.random_feeds() + fetch_list = program_info.fetch_list() + + base_out = self.run_program(program_info.program, feed, fetch_list) + + cinn_out = self.run_program( + program_info.program, + feed, + fetch_list, + enable_cinn=False, + prim_mode=True, + ) + + for cinn_res, base_res in zip(cinn_out, base_out): + np.testing.assert_allclose(cinn_res, base_res, atol=5e-3, rtol=5e-3) + + def check_infer(self, enable_cinn): + parser = Parser() + program_info = parser.run(self.file_path) + if not parser.have_dy_shape: + feed = program_info.random_feeds() + fetch_list = program_info.fetch_list() + + return self.run_program( + program_info.program, feed, fetch_list, enable_cinn + ) + + def run_program( + self, program, feed, fetch_list, enable_cinn=False, prim_mode=False + ): + if prim_mode: + core._set_prim_forward_enabled(True) + paddle.decomposition.decomp.decompose(program, []) + core._set_prim_forward_enabled(False) + if enable_cinn: + fwd_pm = paddle.base.libpaddle.pir.PassManager() + paddle.base.libpaddle.pir.add_cinn_pass(fwd_pm, program) + fwd_pm.run(program) + + exe = paddle.static.Executor(paddle.CUDAPlace(0)) + outs = exe._run_pir_impl( + program, + feed=feed, + fetch_list=fetch_list, + feed_var_name="feed", + fetch_var_name='fetch', + scope=None, + return_numpy=True, + ) + return outs + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + '--file_path', + default="simple_llama.config", + help='input file', + dest='file_path', + ) + parser.add_argument('unittest_args', nargs='*') + args = parser.parse_args() + sys.argv[1:] = args.unittest_args + unittest.main() From 5859683678591106b3df649950993a59bbcf575b Mon Sep 17 00:00:00 2001 From: wanghuancoder Date: Fri, 1 Mar 2024 14:34:12 +0800 Subject: [PATCH 222/282] pir onednn elemetwise datalayout trans (#62265) --- .../instruction/onednn/onednn_instruction.cc | 68 +++++++++++-------- .../instruction/onednn/onednn_instruction.h | 2 + 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/paddle/fluid/framework/new_executor/instruction/onednn/onednn_instruction.cc b/paddle/fluid/framework/new_executor/instruction/onednn/onednn_instruction.cc index aa3df67535747..923d745b49d68 100644 --- a/paddle/fluid/framework/new_executor/instruction/onednn/onednn_instruction.cc +++ b/paddle/fluid/framework/new_executor/instruction/onednn/onednn_instruction.cc @@ -245,16 +245,16 @@ OneDNNPhiKernelInstruction::OneDNNPhiKernelInstruction( } VLOG(6) << "finish process infer meta context"; - auto kernel_name = + auto kernel_name_ = op_attributes.at("kernel_name").dyn_cast().AsString(); - auto kernel_key = op_attributes.at("kernel_key") - .dyn_cast() - .data(); + auto kernel_key_ = op_attributes.at("kernel_key") + .dyn_cast() + .data(); phi_kernel_ = new phi::Kernel( - phi::KernelFactory::Instance().SelectKernel(kernel_name, kernel_key)); + phi::KernelFactory::Instance().SelectKernel(kernel_name_, kernel_key_)); PADDLE_ENFORCE_EQ( - phi_kernel_->IsValid(), true, "not found kernel for [%s]", kernel_name); + phi_kernel_->IsValid(), true, "not found kernel for [%s]", kernel_name_); VLOG(6) << "finish process select kernel"; BuildPhiContext {}_optional; - if( {}.impl() ) {}_optional = paddle::make_optional({}); + if ({}.impl()) {}_optional = paddle::make_optional({}); """ CREATE_RECOVER_OPTIONAL_VECTOR_TENSOR_TEMPLATE = """ paddle::optional> {}_optional; - if( !{}.empty() ) {}_optional = paddle::make_optional>({}); + if (!{}.empty()) {}_optional = paddle::make_optional>({}); """ SET_GRAD_OUT_DIST_ATTR_TEMPLATE = """ @@ -593,20 +593,20 @@ class {} : public egr::GradNodeBase {{ CHECK_NAN_AND_INF_TEMPLATE_FORWARD = """ if (FLAGS_check_nan_inf) {{ - egr::CheckTensorHasNanOrInf("{}", {}); + egr::CheckTensorHasNanOrInf("{}", {}); }} """ CHECK_NAN_AND_INF_TEMPLATE_BACKWARD = """ if (FLAGS_check_nan_inf) {{ - try{{ - egr::CheckTensorHasNanOrInf("{}", {}); - }} catch(...) {{ - LOG(WARNING) << "There are nan/inf in ({})"; - auto forward_trace = GetForwardTrace(); - std::cout<SetTensorWrapper_{name}(*{name}_clone);}""".format_map( {"indent": indent, "name": name} @@ -1102,13 +1098,13 @@ def GenerateNodeCreationCodes(self, for_backward=False, is_inplaced=False): or (name in self.optional_inputs) ): if for_backward is False: - set_tensor_wrappers = f"{indent}if({name}) grad_node->SetTensorWrapper_{name}(*{name});" + set_tensor_wrappers = f"{indent}if ({name}) grad_node->SetTensorWrapper_{name}(*{name});" else: - set_tensor_wrappers = f"{indent}if({name}_optional) grad_node->SetTensorWrapper_{name}(*{name}_optional);" + set_tensor_wrappers = f"{indent}if ({name}_optional) grad_node->SetTensorWrapper_{name}(*{name}_optional);" else: need_pre_contiguous_set.add(name) - set_tensor_wrappers = f"{indent}if({name}) grad_node->SetTensorWrapper_{name}(*{name}_tmp);" + set_tensor_wrappers = f"{indent}if ({name}) grad_node->SetTensorWrapper_{name}(*{name}_tmp);" else: if is_inplace_input: set_tensor_wrappers = f"{indent}auto {name}_clone = paddle::experimental::assign({name});\n{indent}grad_node->SetTensorWrapper_{name}({name}_clone);" @@ -1127,9 +1123,9 @@ def GenerateNodeCreationCodes(self, for_backward=False, is_inplaced=False): else: # Forwad's output as backward's input if num_fwd_outputs > 1: # Aligned with forward output position - assert ( - name in forward_outputs_position_map.keys() - ), AssertMessage(name, forward_outputs_position_map.keys()) + assert name in forward_outputs_position_map, AssertMessage( + name, forward_outputs_position_map.keys() + ) set_tensor_wrappers = ( f"{indent}grad_node->SetTensorWrapper_{name}({name});" @@ -1185,9 +1181,9 @@ def GenerateNodeCreationCodes(self, for_backward=False, is_inplaced=False): if is_optional: if for_backward is False: - set_grad_out_meta = f"{indent}if({name}.get_ptr() != nullptr) grad_node->SetGradOutMeta(*({name}.get_ptr()), {pos});" + set_grad_out_meta = f"{indent}if ({name}.get_ptr() != nullptr) grad_node->SetGradOutMeta(*({name}.get_ptr()), {pos});" else: - set_grad_out_meta = f"{indent}if({name}_optional.get_ptr() != nullptr) grad_node->SetGradOutMeta(*({name}_optional.get_ptr()), {pos});" + set_grad_out_meta = f"{indent}if ({name}_optional.get_ptr() != nullptr) grad_node->SetGradOutMeta(*({name}_optional.get_ptr()), {pos});" else: if ( is_special_forward_api @@ -1209,7 +1205,7 @@ def GenerateNodeCreationCodes(self, for_backward=False, is_inplaced=False): set_out_rank_list = [] set_history_list = [] set_grad_in_meta_list = [] - num_outputs = len(forward_outputs_position_map.keys()) + num_outputs = len(forward_outputs_position_map) for name, (_, pos) in forward_outputs_position_map.items(): output_autograd_meta_name = GetAutoGradMetaName(name) set_out_rank = f"""{indent}if ({output_autograd_meta_name}) {{ @@ -1358,7 +1354,7 @@ def GenerateForwardLayoutAutotune( intermediate_outputs = self.intermediate_outputs forward_attrs_list = self.forward_attrs_list forward_outputs_position_map = self.forward_outputs_position_map - num_outputs = len(forward_outputs_position_map.keys()) - len( + num_outputs = len(forward_outputs_position_map) - len( intermediate_outputs ) # for layout autotune attr @@ -1481,9 +1477,7 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): indent = GetIndent(1) # Get Function Args - num_inputs = len(forward_attrs_list) + len( - forward_inputs_position_map.keys() - ) + num_inputs = len(forward_attrs_list) + len(forward_inputs_position_map) inputs_args_definition_list = ["" for i in range(num_inputs)] inputs_args_declaration_list = ["" for i in range(num_inputs)] inputs_call_list = ["" for i in range(num_inputs)] @@ -1512,7 +1506,7 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): self.is_forward_only and is_inplaced and forward_inplace_map - and name in forward_inplace_map.keys() + and name in forward_inplace_map ): arg_str = f"paddle::optional& {name}" else: @@ -1535,7 +1529,7 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): if ( is_inplaced and forward_inplace_map - and name in forward_inplace_map.keys() + and name in forward_inplace_map ): arg_str = f"paddle::Tensor& {name}" amp_tensors_vector_list.append(f"{{{name}}}") @@ -1558,7 +1552,7 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): self.is_forward_only and is_inplaced and forward_inplace_map - and name in forward_inplace_map.keys() + and name in forward_inplace_map ): arg_str = f"paddle::optional>& {name}" else: @@ -1576,7 +1570,7 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): if ( is_inplaced and forward_inplace_map - and name in forward_inplace_map.keys() + and name in forward_inplace_map ): arg_str = f"std::vector& {name}" else: @@ -1623,7 +1617,7 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): if is_inplaced and len(forward_outputs_position_map) == 1: api_out_type = "auto&" forward_call_str = f"{indent}{api_out_type} api_result = paddle::experimental::{namespace}{function_name}({inputs_call_args_str});" - num_outputs = len(forward_outputs_position_map.keys()) - len( + num_outputs = len(forward_outputs_position_map) - len( intermediate_outputs ) @@ -1710,7 +1704,7 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): self.forward_api_name[-1] != '_' or self.forward_api_name == 'assign_out_' ): - for inplace_name in forward_inplace_map.keys(): + for inplace_name in forward_inplace_map: if ( not self.is_forward_only and forward_api_name not in inplace_check_blacklist @@ -1765,7 +1759,7 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): # 2. Get Output AutoGradMeta outputs_autograd_meta_list = [] - num_fwd_outputs = len(forward_outputs_position_map.keys()) + num_fwd_outputs = len(forward_outputs_position_map) for name, (rtype, pos) in forward_outputs_position_map.items(): output_autograd_meta_name = GetAutoGradMetaName(name) @@ -1882,13 +1876,13 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): for name, (ttype, pos) in forward_inputs_position_map.items(): var_str += f"\n{indent} const char* TENSOR_{name.upper()}_TEMPLATE = \" \\n( {name} , [%s]), \";" var_str += f"\n{indent} std::string input_{name}_str = paddle::string::Sprintf(TENSOR_{name.upper()}_TEMPLATE, egr::EagerUtils::TensorStr({name}));" - var_str += f"\n{indent} input_str += input_{name}_str; " + var_str += f"\n{indent} input_str += input_{name}_str;" before_log_str = BEFORE_LOG_PRINT_TEMPLATE.format(var_str) for name, (ttype, pos) in forward_outputs_position_map.items(): var_str += f"\n{indent} const char* TENSOR_{name.upper()}_TEMPLATE = \" \\n( {name} , [%s]), \";" var_str += f"\n{indent} std::string output_{name}_str = paddle::string::Sprintf(TENSOR_{name.upper()}_TEMPLATE, egr::EagerUtils::TensorStr({name}));" - var_str += f"\n{indent} output_str += output_{name}_str; " + var_str += f"\n{indent} output_str += output_{name}_str;" log_str = AFTER_LOG_PRINT_TEMPLATE.format(var_str) @@ -1958,10 +1952,7 @@ def GenerateInplacedForwardDygraphFunctions(self): forward_api_name = self.forward_api_name forward_api_contents = self.forward_api_contents - if ( - forward_api_name != "sum" - and "inplace" in forward_api_contents.keys() - ): + if forward_api_name != "sum" and "inplace" in forward_api_contents: # Function Definition and Declaration Generation self.GenerateForwardDefinitionAndDeclaration(is_inplaced=True) self.UpdateCoreOpsInformation(is_inplaced=True) @@ -1976,10 +1967,8 @@ def UpdateCoreOpsInformation(self, is_inplaced): forward_outputs_position_map = self.forward_outputs_position_map forward_attrs_list = self.forward_attrs_list - num_args = len(forward_inputs_position_map.keys()) + len( - forward_attrs_list - ) - num_returns = len(forward_outputs_position_map.keys()) + num_args = len(forward_inputs_position_map) + len(forward_attrs_list) + num_returns = len(forward_outputs_position_map) fwd_api_name = "" + forward_api_name core_ops_returns_info[fwd_api_name] = ["" for i in range(num_returns)] @@ -2042,7 +2031,7 @@ def __init__( def TransformToNextGradName(self, string): name_mapping = self.to_next_grad_name_mapping - if string in name_mapping.keys(): + if string in name_mapping: return name_mapping[string] return string @@ -2072,6 +2061,7 @@ def RecordGrad2NextGradNameMapping(self, next_node_generator): self.to_next_grad_name_mapping[grad_ret_name] = next_ret_name def GenerateHigherOrderNodeCreationCode(self): + indent = GetIndent(1) has_higher_order_node = False namespace = self.namespace grad_api_contents = self.grad_api_contents @@ -2081,6 +2071,7 @@ def GenerateHigherOrderNodeCreationCode(self): next_grad_node_creation_str = "" next_grad_node_out_list = [] next_node_generator = None + if next_grad_api_contents: # Fake forward_api_contents and backward_api_contents forward_api_contents = grad_api_contents @@ -2107,30 +2098,43 @@ def GenerateHigherOrderNodeCreationCode(self): is_composite_grad_api = ( False if self.composite_func_info == {} else True ) - if is_composite_grad_api: if next_grad_node_creation_str != '': + next_grad_node_creation_str = [ + line if len(line) else line + for line in next_grad_node_creation_str.split("\n") + ] + next_grad_node_creation_str = [ + (indent + line if i >= 1 and len(line) else line) + for line in next_grad_node_creation_str + ] + next_grad_node_creation_str = [ + (indent + line if len(line) else line) + for line in next_grad_node_creation_str + ] + next_grad_node_creation_str = "\n".join( + next_grad_node_creation_str + ) next_grad_node_creation_str = f""" - if (!paddle::prim::PrimCommonUtils::IsEagerPrimEnabled()) {{ - {next_grad_node_creation_str} - }} - """ + if (!paddle::prim::PrimCommonUtils::IsEagerPrimEnabled() || need_skip) {{ +{next_grad_node_creation_str} + }} +""" else: if not ( self.grad_api_contents["backward_op"] in prim_white_list or is_invoke_forward_api ): next_grad_node_creation_str = f""" - if (!paddle::prim::PrimCommonUtils::IsEagerPrimEnabled()) {{ - if(trace_backward) {{ - PADDLE_THROW(phi::errors::Unavailable( - \"The Op {self.backward_api_name} doesn't have any grad\" - \"op. If you don't intend calculating higher order\" - \"derivatives, please set `create_graph`to False.\")); + if (!paddle::prim::PrimCommonUtils::IsEagerPrimEnabled() || need_skip) {{ + if (trace_backward) {{ + PADDLE_THROW(phi::errors::Unavailable( + \"The Op {self.backward_api_name} doesn't have any grad\" + \"op. If you don't intend calculating higher order\" + \"derivatives, please set `create_graph`to False.\")); + }} }} - }} - """ - +""" if next_node_generator is not None: has_higher_order_node = True return ( @@ -2143,7 +2147,7 @@ def GenerateHigherOrderNodeCreationCode(self): ) # TODO(Ruting):Integrate invoke and composite as composite so the rest branch canbe covered elif not is_invoke_forward_api and not is_composite_grad_api: - next_grad_node_creation_str = f""" if(trace_backward) {{ + next_grad_node_creation_str = f""" if (trace_backward) {{ PADDLE_THROW(phi::errors::Unavailable( \"The Op {self.backward_api_name} doesn't have any grad\" \"op. If you don't intend calculating higher order\" @@ -2273,8 +2277,8 @@ def GenerateNodeDefinition( # Construct grad_api function args # Order: TensorWrappers, GradTensors, Attributes grad_api_args_len = ( - len(backward_forward_inputs_map.keys()) - + len(backward_grad_inputs_map.keys()) + len(backward_forward_inputs_map) + + len(backward_grad_inputs_map) + len(backward_attrs_list) ) grad_api_args = ["" for i in range(grad_api_args_len)] @@ -2325,7 +2329,7 @@ def GenerateNodeDefinition( is_optional = name in self.optional_inputs tensor_wrapper_recover_str = f"{indent}auto {transformed_tensor_name} = egr::EagerUtils::RecoverTensorWrapper(&this->{tensor_wrapper_name});" - if backward_inplace_map and name in backward_inplace_map.keys(): + if backward_inplace_map and name in backward_inplace_map: if has_higher_order_node: if ( transformed_tensor_name @@ -2401,7 +2405,7 @@ def GenerateNodeDefinition( get_tensor_str = f"{indent}auto& {transformed_tensor_name} = hooked_grads[{fwd_position}][0];" # Inplace in backward op - if backward_inplace_map and name in backward_inplace_map.keys(): + if backward_inplace_map and name in backward_inplace_map: if has_higher_order_node: if ( transformed_tensor_name @@ -2464,7 +2468,7 @@ def GenerateNodeDefinition( get_grad_in_args_str = "\n".join(get_grad_in_args_list) # Grad Function Call String - slot_num_bwd_outputs = len(self.forward_inputs_position_map.keys()) + slot_num_bwd_outputs = len(self.forward_inputs_position_map) grad_api_namespace = f"paddle::experimental::{namespace}" composite_grad_api_namespace = f"paddle::prim::{namespace}" grad_function_prepare_str = f""" @@ -2508,7 +2512,7 @@ def GenerateNodeDefinition( backward_inplace_map and name in backward_inplace_map.values() ): - inplace_str = f""" if (api_output_{out_index} != nullptr && can_be_inplaced) {{ + inplace_str = f"""if (api_output_{out_index} != nullptr && can_be_inplaced) {{ egr::EagerUtils::HandleViewBetweenInputAndOutput({inplace_grad_input_str}, api_output_{out_index}); }}""" if has_higher_order_node: @@ -2520,7 +2524,7 @@ def GenerateNodeDefinition( }}""" need_gen_trace_backward_for_inplace = True else: - inplace_for_grad_outs_str += inplace_str + inplace_for_grad_outs_str += " " + inplace_str grad_function_prepare_str += f""" auto* api_output_{out_index} = (out_metas[{fwd_position}].empty() || out_metas[{fwd_position}][0].IsStopGradient()) ? nullptr : &returns[{fwd_position}][0];""" @@ -2570,43 +2574,112 @@ def GenerateNodeDefinition( grad_function_call_str = f""" if (trace_backward) {{ {indent}{autograd_api_out} api_output = {autograd_api}; - {out_assign_str}}} else {{ + {out_assign_str}{indent}}} else {{ {indent}{autograd_api_out} api_output = paddle::experimental::{self.namespace}{self.grad_api_contents['invoke']}; {out_assign_str}{indent}}} - """ - # TODO(Ruting):using composite only when we don't have backward kernel in the future. +""" elif is_composite_grad_api: - if composite_grad_api_name in prim_white_list: - grad_function_call_str = f""" + has_kernel_impl = "kernel" in self.grad_api_contents + + def _gen_api_call_code_block( + in_prim_white_list: bool, + has_kernel_impl: bool, + has_higher_order_node: bool, + indention: int, + ): + """This function will generate code block for calling composite or + kernel grad api as shown below. + + // Call grad_api function + + XXX <-- Generated code by this function + XXX <-- Generated code by this function + ... <-- Generated code by this function + ... <-- Generated code by this function + + // Check NaN and Inf id needed + + Args: + in_prim_white_list (bool): Whether current op in `prim_white_list`. + has_kernel_impl (bool): Whether current op has kernel implementation. + has_higher_order_node (bool): Whether current op has next grad op. + indention (int): Number of single space for whole code block indention. + """ + if in_prim_white_list: + code = f""" +bool original_global_grad = egr::Controller::Instance().HasGrad(); +if (!create_graph) {{ +{indent}egr::Controller::Instance().SetHasGrad(create_graph); +}} +{composite_grad_api_namespace}{composite_grad_api_name}{composite_template_name}({composite_grad_api_args_str}); +VLOG(4) << "Composite api {composite_grad_api_name} is called"; +if (!create_graph) {{ +{indent}egr::Controller::Instance().SetHasGrad(original_global_grad); +}} +""" + if has_higher_order_node: + code = f"auto need_skip = false;{code}" + else: + code = f""" +std::string grad_op_name = "{composite_grad_api_name}"; +auto need_skip = paddle::prim::StaticCompositeContext::Instance().CheckSkipCompOps(grad_op_name); +if (paddle::prim::PrimCommonUtils::IsEagerPrimEnabled() && !need_skip) {{ {indent}bool original_global_grad = egr::Controller::Instance().HasGrad(); -{indent}if(!create_graph){{ +{indent}if (!create_graph) {{ {indent}{indent}egr::Controller::Instance().SetHasGrad(create_graph); - }} - {indent}{composite_grad_api_namespace}{composite_grad_api_name}{composite_template_name}({composite_grad_api_args_str}); - VLOG(4) << "Composite api {composite_grad_api_name} is called "; -{indent}if(!create_graph){{ +{indent}}} +{indent}{composite_grad_api_namespace}{composite_grad_api_name}{composite_template_name}({composite_grad_api_args_str}); +{indent}VLOG(4) << "Composite api {composite_grad_api_name} is called"; +{indent}if (!create_graph) {{ {indent}{indent}egr::Controller::Instance().SetHasGrad(original_global_grad); - }} - """ +{indent}}}""" + if has_kernel_impl: + code = ( + code + + f""" +}} else {{ +{indent}{grad_api_namespace}{backward_api_name}({grad_api_args_str}); +{indent}VLOG(4) << "Fused api {backward_api_name} is called"; +}} +""" + ) + else: + code = ( + code + + f""" +}} else {{ + PADDLE_THROW(phi::errors::Unavailable( + \"The grad op of {self.backward_api_name} doesn't implemented yet.\")); +}} +""" + ) + # make indention for all line(s) in code + code = "\n".join( + [ + (f"{' ' * indention}{line}" if len(line) else line) + for line in code.split("\n") + ] + ) + + return code + + if ( + self.backward_api_name not in prim_white_list + and not has_kernel_impl + ): + grad_function_call_str = _gen_api_call_code_block( + self.backward_api_name in prim_white_list, + has_kernel_impl, + has_higher_order_node, + 0, + ) else: - grad_function_call_str = f""" - std::string grad_op_name = "{composite_grad_api_name}"; - auto need_skip = paddle::prim::StaticCompositeContext::Instance().CheckSkipCompOps(grad_op_name); - if (paddle::prim::PrimCommonUtils::IsEagerPrimEnabled() && !need_skip) {{ -{indent}bool original_global_grad = egr::Controller::Instance().HasGrad(); -{indent}if(!create_graph){{ -{indent}{indent}egr::Controller::Instance().SetHasGrad(create_graph); - }} - {indent}{composite_grad_api_namespace}{composite_grad_api_name}{composite_template_name}({composite_grad_api_args_str}); - {indent}VLOG(4) << "Composite api {composite_grad_api_name} is called "; -{indent}if(!create_graph){{ -{indent}{indent}egr::Controller::Instance().SetHasGrad(original_global_grad); - }} - }}else{{ - {indent}{grad_api_namespace}{backward_api_name}({grad_api_args_str}); - {indent}VLOG(4) << "Fused api {backward_api_name} is called "; - }} - """ + grad_function_call_str = _gen_api_call_code_block( + self.backward_api_name in prim_white_list, + has_kernel_impl, + has_higher_order_node, + 2, + ) else: grad_function_call_str = f""" {indent}{grad_api_namespace}{backward_api_name}({grad_api_args_str});""" @@ -2630,7 +2703,7 @@ def GenerateNodeDefinition( outputs_autograd_meta_list = [] # TODO(jiabin): Optimize this with SetStopGradient instead of Pass Stop gradient - num_fwd_outputs = len(backward_grad_outputs_map.keys()) + num_fwd_outputs = len(backward_grad_outputs_map) for name, ( rtype, pos, @@ -2649,7 +2722,7 @@ def GenerateNodeDefinition( auto& {transformed_tensor_name} = returns[{pos}][0]; egr::AutogradMeta* {output_autograd_meta_name} = returns[{pos}][0].initialized() ? egr::EagerUtils::autograd_meta(&{transformed_tensor_name}) : nullptr; if ({output_autograd_meta_name}) {output_autograd_meta_name}->SetStopGradient(false); - """ +""" else: assert IsVectorTensorType(rtype) @@ -2658,7 +2731,7 @@ def GenerateNodeDefinition( auto& {transformed_tensor_name} = returns[{pos}]; std::vector {output_autograd_meta_vec_name} = egr::EagerUtils::autograd_meta(&{transformed_tensor_name}); std::vector* {output_autograd_meta_name} = &{output_autograd_meta_vec_name}; - for(auto* meta : {output_autograd_meta_vec_name}){{ + for(auto* meta : {output_autograd_meta_vec_name}) {{ meta->SetStopGradient(false); }} """ @@ -2666,7 +2739,7 @@ def GenerateNodeDefinition( output_autograd_meta = f""" auto& {transformed_tensor_name} = returns[{pos}]; std::vector {output_autograd_meta_vec_name} = egr::EagerUtils::autograd_meta(&{transformed_tensor_name}); - for(auto* meta : {output_autograd_meta_vec_name}){{ + for(auto* meta : {output_autograd_meta_vec_name}) {{ meta->SetStopGradient(false); }} """ @@ -2674,7 +2747,7 @@ def GenerateNodeDefinition( outputs_autograd_meta_str = "\n".join(outputs_autograd_meta_list) - returns_str = f"{indent}if(NeedComplexToRealConversion()) HandleComplexGradToRealGrad(&returns);\n" + returns_str = f"{indent}if (NeedComplexToRealConversion()) HandleComplexGradToRealGrad(&returns);\n" returns_str += f"{indent}return returns;\n" grad_node_name = GetGradNodeName(self.backward_api_name) @@ -2689,7 +2762,7 @@ def GenerateNodeDefinition( new_name = self.TransformToNextGradName(name) var_str += f"\n{indent} const char* TENSOR_{new_name.upper()}_TEMPLATE = \" \\n( {new_name} , [%s]), \";" var_str += f"\n{indent} std::string input_{new_name}_str = paddle::string::Sprintf(TENSOR_{new_name.upper()}_TEMPLATE, egr::EagerUtils::TensorStr({new_name}));" - var_str += f"\n{indent} input_str += input_{new_name}_str; " + var_str += f"\n{indent} input_str += input_{new_name}_str;" for ( name, @@ -2698,7 +2771,7 @@ def GenerateNodeDefinition( new_name = self.TransformToNextGradName(name) var_str += f"\n{indent} const char* TENSOR_{new_name.upper()}_TEMPLATE = \" \\n( {new_name} , [%s]), \";" var_str += f"\n{indent} std::string input_{new_name}_str = paddle::string::Sprintf(TENSOR_{new_name.upper()}_TEMPLATE, egr::EagerUtils::TensorStr({new_name}));" - var_str += f"\n{indent} input_str += input_{new_name}_str; " + var_str += f"\n{indent} input_str += input_{new_name}_str;" before_log_str = BEFORE_LOG_PRINT_TEMPLATE.format(var_str) @@ -2710,7 +2783,7 @@ def GenerateNodeDefinition( new_name = self.TransformToNextGradName(name) var_str += f"\n{indent} const char* TENSOR_{new_name.upper()}_TEMPLATE = \" \\n ( {new_name} , [%s]), \";" var_str += f"\n{indent} std::string output_{new_name}_str = paddle::string::Sprintf(TENSOR_{new_name.upper()}_TEMPLATE, egr::EagerUtils::TensorStr({new_name}));" - var_str += f"\n{indent} output_str += output_{new_name}_str; " + var_str += f"\n{indent} output_str += output_{new_name}_str;" log_str = AFTER_LOG_PRINT_TEMPLATE.format(var_str) @@ -2787,7 +2860,7 @@ def __init__( def CollectIsForwardOnly(self, forward_api_contents): self.is_forward_only = ( - False if 'backward' in forward_api_contents.keys() else True + False if 'backward' in forward_api_contents else True ) def ParseYamlContents(self): @@ -2802,11 +2875,11 @@ def ParseYamlContents(self): def GetBackwardAPIContents(self, forward_api_contents): grad_api_dict = self.grad_api_dict - if 'backward' not in forward_api_contents.keys(): + if 'backward' not in forward_api_contents: return None backward_api_name = forward_api_contents['backward'] - assert backward_api_name in grad_api_dict.keys(), AssertMessage( + assert backward_api_name in grad_api_dict, AssertMessage( backward_api_name, grad_api_dict.keys() ) backward_api_contents = grad_api_dict[backward_api_name] From e5404f0cc58dd12f547ea8176177829dc203c43e Mon Sep 17 00:00:00 2001 From: zhengzhonghui Date: Fri, 1 Mar 2024 16:00:25 +0800 Subject: [PATCH 224/282] [AutoParallel] shard_dataloader support list inputs (#62229) * [AutoParallel] shard_dataloader support list inputs * add an example * fix doc example error * add doc * fix * fix * fix doc --- .../paddle/distributed/auto_parallel/api.py | 195 +++++++++++++--- .../hybrid_strategy/CMakeLists.txt | 8 + .../semi_auto_parallel_multi_inputs.py | 212 ++++++++++++++++++ .../test_semi_auto_parallel_multi_inputs.py | 57 +++++ .../hybrid_strategy/testslist.csv | 1 + 5 files changed, 448 insertions(+), 25 deletions(-) create mode 100644 test/auto_parallel/hybrid_strategy/semi_auto_parallel_multi_inputs.py create mode 100644 test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_multi_inputs.py diff --git a/python/paddle/distributed/auto_parallel/api.py b/python/paddle/distributed/auto_parallel/api.py index 28f15011190f2..c63f8ce3a58c9 100644 --- a/python/paddle/distributed/auto_parallel/api.py +++ b/python/paddle/distributed/auto_parallel/api.py @@ -2018,22 +2018,22 @@ def __init__( process_id, self._meshes ) ) + if input_keys is not None: + assert len(input_keys) == 2, "input_keys lengths must be 2" self._all_inputs_in_one_mesh = len(self._meshes) == 1 self._input_keys = input_keys self._shard_dims = self._process_shard_dims(shard_dims) - mesh_index = self._get_mesh_idx(process_id) - if mesh_index == -1: + mesh, shard_dim = self._get_mesh_and_shard_dim(process_id) + if mesh is None: + mesh = to_list(self._meshes[0])[0] + shard_dim = to_list(self._shard_dims[0])[0] dp_rank = 0 - dp_world_size = self._meshes[0].get_dim_size(self._shard_dims[0]) + dp_world_size = mesh.get_dim_size(shard_dim) else: - dp_rank = self._meshes[mesh_index].get_rank_by_dim_and_process_id( - self._shard_dims[mesh_index], process_id - ) - dp_world_size = self._meshes[mesh_index].get_dim_size( - self._shard_dims[mesh_index] - ) + dp_rank = mesh.get_rank_by_dim_and_process_id(shard_dim, process_id) + dp_world_size = mesh.get_dim_size(shard_dim) if is_dataset_splitted is True or shard_dims is None: self._dataloader = dataloader @@ -2074,7 +2074,13 @@ def __init__( def _process_shard_dims(self, shard_dims): if isinstance(shard_dims, (int, str)) or shard_dims is None: - return [shard_dims] * len(self._meshes) + res = [] + for i in range(len(self._meshes)): + if isinstance(self._meshes[i], (list, tuple)): + res.append([shard_dims] * len(self._meshes[i])) + else: + res.append(shard_dims) + return res else: if len(shard_dims) != len(self._meshes): raise ValueError( @@ -2084,16 +2090,30 @@ def _process_shard_dims(self, shard_dims): ) return shard_dims - def _get_mesh_idx(self, process_id): + def _get_mesh_and_shard_dim(self, process_id): for i in range(len(self._meshes)): - if process_id in self._meshes[i]._process_ids: - return i - return -1 + if isinstance(self._meshes[i], (list, tuple)): + for j in range(len(self._meshes[i])): + if process_id in self._meshes[i][j]._process_ids: + return self._meshes[i][j], self._shard_dims[i][j] + else: + if process_id in self._meshes[i]._process_ids: + return self._meshes[i], self._shard_dims[i] + return None, None def _process_id_in_multi_meshes(self, process_id): count = 0 - for i in range(len(self._meshes)): - if process_id in self._meshes[i]._process_ids: + flatten_meshes = [] + for mesh in self._meshes: + if isinstance(mesh, (list, tuple)): + flatten_meshes.extend(mesh) + else: + flatten_meshes.append(mesh) + + # NOTE(zhengzhonghui): User may set the same mesh for different inputs, so we need to unique the meshes + unique_meshes = list(set(flatten_meshes)) + for mesh in unique_meshes: + if process_id in mesh._process_ids: count += 1 return count > 1 @@ -2123,16 +2143,69 @@ def _get_mesh_and_placement(self, index): placements.append(dist.Replicate()) return mesh, placements + def _get_meshes_and_placements_for_list_input(self, index, length): + if self._all_inputs_in_one_mesh: + meshes = [self._meshes[0]] * length + shard_dims = [self._shard_dims[0]] * length + else: + meshes = self._meshes[index] + if isinstance(meshes, (list, tuple)): + assert len(meshes) == length + else: + meshes = [meshes] * length + shard_dims = self._shard_dims[index] + if isinstance(shard_dims, (list, tuple)): + assert len(shard_dims) == length + else: + shard_dims = [shard_dims] * length + + placements = [] + for i in range(length): + if shard_dims[i] is not None: + placement = [dist.Shard(0)] + else: + placement = [dist.Replicate()] + for _ in range(1, len(meshes[i]._shape)): + placement.append(dist.Replicate()) + placements.append(placement) + return meshes, placements + + def _dtensors_from_list_input(self, list_tensors, meshes, placements): + dist_data = [] + for j in range(len(list_tensors)): + dist_data.append( + dtensor_from_local(list_tensors[j], meshes[j], placements[j]) + ) + return dist_data + def _get_batch(self, batch_data): if isinstance(batch_data, (list, tuple)): if self._all_inputs_in_one_mesh is False: assert len(batch_data) == len(self._meshes) dist_batch_data = [] for i in range(len(batch_data)): - mesh, placements = self._get_mesh_and_placement(i) - dist_batch_data.append( - dtensor_from_local(batch_data[i], mesh, placements) - ) + input_data = batch_data[i] + if isinstance(input_data, (list, tuple)): + ( + meshes, + placements, + ) = self._get_meshes_and_placements_for_list_input( + i, len(input_data) + ) + dist_batch_data.append( + self._dtensors_from_list_input( + input_data, meshes, placements + ) + ) + elif isinstance(input_data, paddle.Tensor): + mesh, placements = self._get_mesh_and_placement(i) + dist_batch_data.append( + dtensor_from_local(input_data, mesh, placements) + ) + else: + raise ValueError( + f"Unsupported input_data type {type(input_data)}" + ) return dist_batch_data elif isinstance(batch_data, dict): if self._all_inputs_in_one_mesh is False: @@ -2140,10 +2213,26 @@ def _get_batch(self, batch_data): dist_batch_data = {} for i in range(len(self._input_keys)): key = self._input_keys[i] - mesh, placements = self._get_mesh_and_placement(i) - dist_batch_data[key] = dtensor_from_local( - batch_data[key], mesh, placements - ) + input_data = batch_data[key] + if isinstance(input_data, (list, tuple)): + ( + meshes, + placements, + ) = self._get_meshes_and_placements_for_list_input( + i, len(input_data) + ) + dist_batch_data[key] = self._dtensors_from_list_input( + input_data, meshes, placements + ) + elif isinstance(input_data, paddle.Tensor): + mesh, placements = self._get_mesh_and_placement(i) + dist_batch_data[key] = dtensor_from_local( + batch_data[key], mesh, placements + ) + else: + raise ValueError( + f"Unsupported input_data type {type(input_data)}" + ) return dist_batch_data else: raise ValueError(f"Unsupported batch_data type {type(batch_data)}") @@ -2173,7 +2262,9 @@ def shard_dataloader( only if is_dataset_splitted is False and shard_dims is not None, it will do split. Args: - dataloader (paddle.io.DataLoader): The dataloader to be sharded. + dataloader (paddle.io.DataLoader): The dataloader to be sharded. the output of dataloader + must be a list or dict of paddle.Tensor with 2 elements, i.e. [input_data, label] or + {"input_data": input_data, "label": label}, input_data and label can be a list to support multiple inputs. meshes (ProcessMesh|list[ProcessMesh]|tuple[ProcessMesh]): The mesh list of the dataloader. Identify which mesh the input is on. if len(meshes) == 1 or type(meshes) == ProcessMesh, all the inputs are on the same mesh. @@ -2191,6 +2282,7 @@ def shard_dataloader( Examples: .. code-block:: python + :name: example-1 >>> import paddle >>> import paddle.distributed as dist @@ -2286,6 +2378,59 @@ def shard_dataloader( >>> # RUN_STATIC=1 python -u -m paddle.distributed.launch --gpus "0,1,2,3,4,5,6,7" {test_case}.py >>> # RUN_STATIC=0 python -u -m paddle.distributed.launch --gpus "0,1,2,3,4,5,6,7" {test_case}.py + .. code-block:: python + :name: example-2 + + >>> import paddle + >>> import paddle.distributed as dist + >>> from paddle.io import BatchSampler, DataLoader, Dataset + >>> import numpy as np + >>> mesh0 = dist.ProcessMesh([[0, 1], [2, 3]], dim_names=['dp', 'mp']) + >>> mesh1 = dist.ProcessMesh([[4, 5], [6, 7]], dim_names=['dp', 'mp']) + >>> class RandomDataset(Dataset): + ... def __init__(self, seq_len, hidden, num_samples=8): + ... super().__init__() + ... self.seq_len = seq_len + ... self.hidden = hidden + ... self.num_samples = num_samples + ... self.inputs1 = [ + ... np.random.uniform(size=[self.seq_len, self.hidden]).astype( + ... "float32" + ... ) + ... for _ in range(num_samples) + ... ] + ... self.inputs2 = [ + ... np.random.uniform(size=[self.seq_len, self.hidden]).astype( + ... "float32" + ... ) + ... for _ in range(num_samples) + ... ] + ... self.labels = [ + ... np.array(index, dtype="float32") for index in range(num_samples) + ... ] + ... def __getitem__(self, index): + ... return { + ... "inputs": [self.inputs1[index], self.inputs2[index]], + ... "label": self.labels[index], + ... } + ... def __len__(self): + ... return self.num_samples + + >>> dataset = RandomDataset(4, 8) + >>> sampler = BatchSampler( + ... dataset, + ... batch_size=2, + ... ) + >>> dataloader = DataLoader( + ... dataset, + ... batch_sampler=sampler, + ... ) + >>> dist_dataloader = dist.shard_dataloader( + ... dataloader=dataloader, + ... meshes=[mesh0, mesh1], # or [[mesh0, mesh0], mesh1] + ... shard_dims="dp", + ... input_keys=["inputs", "label"], + ... ) """ return ShardDataloader( diff --git a/test/auto_parallel/hybrid_strategy/CMakeLists.txt b/test/auto_parallel/hybrid_strategy/CMakeLists.txt index 08a9f42c02a1f..063b1b5873e74 100644 --- a/test/auto_parallel/hybrid_strategy/CMakeLists.txt +++ b/test/auto_parallel/hybrid_strategy/CMakeLists.txt @@ -73,3 +73,11 @@ if((WITH_GPU) AND (LINUX)) set_tests_properties(test_semi_auto_parallel_global_input PROPERTIES TIMEOUT "120" LABELS "RUN_TYPE=HYBRID") endif() +if((WITH_GPU) AND (LINUX)) + py_test_modules( + test_semi_auto_parallel_multi_inputs MODULES + test_semi_auto_parallel_multi_inputs ENVS + "http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python") + set_tests_properties(test_semi_auto_parallel_multi_inputs + PROPERTIES TIMEOUT "120" LABELS "RUN_TYPE=HYBRID") +endif() diff --git a/test/auto_parallel/hybrid_strategy/semi_auto_parallel_multi_inputs.py b/test/auto_parallel/hybrid_strategy/semi_auto_parallel_multi_inputs.py new file mode 100644 index 0000000000000..a7166ca901d09 --- /dev/null +++ b/test/auto_parallel/hybrid_strategy/semi_auto_parallel_multi_inputs.py @@ -0,0 +1,212 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import numpy as np + +import paddle +import paddle.distributed as dist +from paddle.io import BatchSampler, DataLoader, Dataset + +SEQ_LEN = 4 +HIDDLE_SIZE = 8 +global_mesh = dist.ProcessMesh( + [[[0, 1], [2, 3]], [[4, 5], [6, 7]]], dim_names=['pp', 'dp', 'mp'] +) +mesh0 = dist.ProcessMesh([[0, 1], [2, 3]], dim_names=['dp', 'mp']) +mesh1 = dist.ProcessMesh([[4, 5], [6, 7]], dim_names=['dp', 'mp']) + + +class MlpModel(paddle.nn.Layer): + def __init__(self, variable_initial_values, run_single_process=False): + super().__init__() + self.w0 = self.create_parameter( + shape=[HIDDLE_SIZE, HIDDLE_SIZE], + default_initializer=paddle.nn.initializer.Assign( + variable_initial_values[0] + ), + ) + self.w1 = self.create_parameter( + shape=[HIDDLE_SIZE, HIDDLE_SIZE], + default_initializer=paddle.nn.initializer.Assign( + variable_initial_values[1] + ), + ) + if run_single_process is False: + self.w0 = dist.shard_tensor( + self.w0, + mesh0, + [dist.Replicate(), dist.Shard(1)], + ) + self.w1 = dist.shard_tensor( + self.w1, + mesh1, + [dist.Replicate(), dist.Shard(0)], + ) + self.run_single_process = run_single_process + + def forward(self, input1, input2): + x = input1 + input2 + # x: [bs, seq_len, hidden] + # forward on mesh0 + y = paddle.matmul(x, self.w0) + # forward on mesh1 + if self.run_single_process is False: + y = dist.reshard(y, mesh1, [dist.Shard(0), dist.Shard(2)]) + z = paddle.matmul(y, self.w1) + return z + + +class RandomDataset(Dataset): + def __init__(self, seq_len, hidden, num_samples=8): + super().__init__() + self.seq_len = seq_len + self.hidden = hidden + self.num_samples = num_samples + self.inputs1 = [ + np.random.uniform(size=[self.seq_len, self.hidden]).astype( + "float32" + ) + for _ in range(num_samples) + ] + self.inputs2 = [ + np.random.uniform(size=[self.seq_len, self.hidden]).astype( + "float32" + ) + for _ in range(num_samples) + ] + self.labels = [ + np.array(index, dtype="float32") for index in range(num_samples) + ] + + def __getitem__(self, index): + return { + "inputs": [self.inputs1[index], self.inputs2[index]], + "label": self.labels[index], + } + + def __len__(self): + return self.num_samples + + +def create_dataloader(): + dataset = RandomDataset(SEQ_LEN, HIDDLE_SIZE) + sampler = BatchSampler( + dataset, + batch_size=2, + ) + dataloader = DataLoader( + dataset, + batch_sampler=sampler, + ) + return dataloader + + +def get_variable_initial_value(var_num=2): + res = [] + for i in range(var_num): + res.append( + paddle.uniform( + shape=[HIDDLE_SIZE, HIDDLE_SIZE], + dtype=paddle.float32, + min=-0.0001, + max=0.0001, + ) + ) + return res + + +def loss_fn(logits, label): + # logits: [bs, seq_len, hidden], label: [bs] + loss = paddle.nn.MSELoss(reduction="sum") + logits = paddle.sum(logits, axis=[1, 2]) + return loss(logits, label) + + +class TestSemiAutoParallelMultiInputs: + def __init__(self): + self._backend = os.getenv("backend") + self._seed = eval(os.getenv("seed")) + self._run_static = eval(os.getenv("run_static")) + paddle.seed(self._seed) + np.random.seed(self._seed) + paddle.set_device(self._backend) + self.dataloader = create_dataloader() + self.variable_initial_values = get_variable_initial_value() + self.single_process_loss = self.get_single_process_loss() + + def get_single_process_loss(self): + model = MlpModel( + variable_initial_values=self.variable_initial_values, + run_single_process=True, + ) + opt = paddle.optimizer.AdamW( + learning_rate=0.001, parameters=model.parameters() + ) + for step, data in enumerate(self.dataloader()): + input1, input2 = data["inputs"] + logits = model(input1, input2) + label = data["label"] + loss = loss_fn(logits, label) + loss.backward() + opt.step() + opt.clear_grad() + return loss.numpy() + + def test_basic(self): + model = MlpModel(variable_initial_values=self.variable_initial_values) + opt = paddle.optimizer.AdamW( + learning_rate=0.001, parameters=model.parameters() + ) + dist_dataloader = dist.shard_dataloader( + dataloader=self.dataloader, + meshes=[mesh0, mesh1], # or [[mesh0, mesh0], mesh1] + shard_dims="dp", + input_keys=["inputs", "label"], + ) + cur_rank = paddle.distributed.get_rank() + if self._run_static: + dist_model = dist.to_static(model, dist_dataloader, loss_fn, opt) + + for step, data in enumerate(dist_dataloader()): + input1, input2 = data["inputs"] + label = data["label"] + loss = dist_model(input1, input2, label) + + if cur_rank in [5, 7]: + loss = paddle.to_tensor(loss) + group = paddle.distributed.new_group([5, 7]) + dist.all_reduce(loss, group=group) + else: + dist_opt = dist.shard_optimizer(opt) + for step, data in enumerate(dist_dataloader()): + input1, input2 = data["inputs"] + logits = model(input1, input2) + label = data["label"] + loss = loss_fn(logits, label) + loss.backward() + dist_opt.step() + dist_opt.clear_grad() + if cur_rank in [5, 7]: + np.testing.assert_allclose( + loss.numpy(), self.single_process_loss, rtol=1e-06, verbose=True + ) + + def run_test_case(self): + self.test_basic() + + +if __name__ == '__main__': + TestSemiAutoParallelMultiInputs().run_test_case() diff --git a/test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_multi_inputs.py b/test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_multi_inputs.py new file mode 100644 index 0000000000000..e172ba1da70f5 --- /dev/null +++ b/test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_multi_inputs.py @@ -0,0 +1,57 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import collective.test_communication_api_base as test_base + + +class TestSemiAutoParallelMultiInputs(test_base.CommunicationTestDistBase): + def setUp(self): + super().setUp( + num_of_devices=8, + timeout=120, + nnode=1, + ) + self._default_envs = { + "dtype": "float32", + "seed": "1024", + } + self._changeable_envs = {"backend": ["gpu"]} + + def test_dynamic(self): + self._default_envs.update({"run_static": "0"}) + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) + for envs in envs_list: + self.run_test_case( + "semi_auto_parallel_multi_inputs.py", + user_defined_envs=envs, + ) + + def test_static(self): + self._default_envs.update({"run_static": "1"}) + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) + for envs in envs_list: + self.run_test_case( + "semi_auto_parallel_multi_inputs.py", + user_defined_envs=envs, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/auto_parallel/hybrid_strategy/testslist.csv b/test/auto_parallel/hybrid_strategy/testslist.csv index 5791b71d0d5ff..2fac60515b51a 100644 --- a/test/auto_parallel/hybrid_strategy/testslist.csv +++ b/test/auto_parallel/hybrid_strategy/testslist.csv @@ -8,3 +8,4 @@ test_semi_auto_parallel_llama_model_amp,LINUX,GPU,180,HYBRID,test_runner.py,,,ht test_semi_auto_parallel_hybrid_sharding_strategy,LINUX,GPU,120,HYBRID,test_runner.py,,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_global_mesh_reshard,LINUX,GPU,120,HYBRID,test_runner.py,,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_semi_auto_parallel_global_input,LINUX,GPU,120,HYBRID,test_runner.py,,,http_proxy=;https_proxy=;PYTHONPATH=../.., +test_semi_auto_parallel_multi_inputs,LINUX,GPU,120,HYBRID,test_runner.py,,,http_proxy=;https_proxy=;PYTHONPATH=../.., From d65b004a1bab5636d4395f33a19ca11629336255 Mon Sep 17 00:00:00 2001 From: zhangbo9674 <82555433+zhangbo9674@users.noreply.github.com> Date: Fri, 1 Mar 2024 18:48:04 +0800 Subject: [PATCH 225/282] [PIR] Set NCHW as default Layout for IrTensor (#62254) * fix * fix bug * fix --- paddle/fluid/pir/dialect/operator/ir/ir_tensor.h | 2 +- paddle/phi/core/kernel_factory.cc | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/pir/dialect/operator/ir/ir_tensor.h b/paddle/fluid/pir/dialect/operator/ir/ir_tensor.h index e2c3229b04df0..21d8a9fdd7ae5 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ir_tensor.h +++ b/paddle/fluid/pir/dialect/operator/ir/ir_tensor.h @@ -81,7 +81,7 @@ class IrTensor : public phi::TensorBase, private: phi::DDim dims_; phi::DataType dtype_{phi::DataType::FLOAT32}; - phi::DataLayout layout_{phi::DataLayout::ANY}; + phi::DataLayout layout_{phi::DataLayout::NCHW}; LoD lod_; size_t offset_{0}; }; diff --git a/paddle/phi/core/kernel_factory.cc b/paddle/phi/core/kernel_factory.cc index 35ac9e1e0db95..7f1ee799824e8 100644 --- a/paddle/phi/core/kernel_factory.cc +++ b/paddle/phi/core/kernel_factory.cc @@ -177,6 +177,22 @@ bool KernelFactory::HasKernel(const std::string& kernel_name, phi::errors::NotFound("The kernel `%s` is not registered.", kernel_name)); auto kernel_iter = iter->second.find(kernel_key); + if (kernel_iter == iter->second.end() && + kernel_key.layout() != phi::DataLayout::ALL_LAYOUT) { + phi::KernelKey any_layout_kernel_key( + kernel_key.backend(), phi::DataLayout::ALL_LAYOUT, kernel_key.dtype()); + kernel_iter = iter->second.find(any_layout_kernel_key); + } + +#if defined(PADDLE_WITH_CUSTOM_DEVICE) + if (kernel_iter == iter->second.end() && + kernel_key.backend() > phi::Backend::NUM_BACKENDS) { + kernel_iter = iter->second.find({phi::Backend::CUSTOM, + phi::DataLayout::ALL_LAYOUT, + kernel_key.dtype()}); + } +#endif + if (kernel_iter == iter->second.end()) { return false; } From 0cb9bf687a3372cf851089fd5508f4d7fafc1295 Mon Sep 17 00:00:00 2001 From: Sunny-bot1 <68891411+Sunny-bot1@users.noreply.github.com> Date: Fri, 1 Mar 2024 19:29:08 +0800 Subject: [PATCH 226/282] [Inference] Add a config api to use PIR (#61968) * add a config api for pir * fix comment * fix the enable failure * fix bug * fix bug --- paddle/fluid/inference/analysis/argument.h | 1 + .../passes/inference_op_replace_pass.cc | 4 +--- .../ir_params_sync_among_devices_pass.cc | 5 ++--- paddle/fluid/inference/api/analysis_config.cc | 1 + .../fluid/inference/api/analysis_predictor.cc | 15 ++++++------- .../inference/api/demo_ci/custom_op_demo.cc | 1 + paddle/fluid/inference/api/demo_ci/run.sh | 2 +- paddle/fluid/inference/api/helper.cc | 6 ++---- paddle/fluid/inference/api/helper.h | 2 +- .../inference/api/paddle_analysis_config.h | 14 +++++++++++++ paddle/fluid/pybind/inference_api.cc | 2 ++ .../cpp/inference/analysis/analyzer_tester.cc | 2 ++ test/custom_op/test_inference_inplace.py | 13 +++++------- test/ir/inference/auto_scan_test.py | 4 ++-- test/ir/inference/program_config.py | 1 - .../inference/test_inference_predictor_run.py | 13 +++++------- .../test_decomp_inference_predictor_run.py | 21 ++++++++----------- 17 files changed, 57 insertions(+), 50 deletions(-) diff --git a/paddle/fluid/inference/analysis/argument.h b/paddle/fluid/inference/analysis/argument.h index a87c919bbe2c1..1407a8f875a29 100644 --- a/paddle/fluid/inference/analysis/argument.h +++ b/paddle/fluid/inference/analysis/argument.h @@ -227,6 +227,7 @@ struct Argument { DECL_ARGUMENT_FIELD(use_cutlass, UseCutlass, bool); DECL_ARGUMENT_FIELD(use_fc_padding, UseFcPadding, bool); DECL_ARGUMENT_FIELD(gpu_device_id, GPUDeviceId, int); + DECL_ARGUMENT_FIELD(use_pir, UsePIR, bool); // Usually use for trt dynamic shape. // TRT will select the best kernel according to opt shape diff --git a/paddle/fluid/inference/analysis/passes/inference_op_replace_pass.cc b/paddle/fluid/inference/analysis/passes/inference_op_replace_pass.cc index b422dea840af5..993ab2e8618f4 100644 --- a/paddle/fluid/inference/analysis/passes/inference_op_replace_pass.cc +++ b/paddle/fluid/inference/analysis/passes/inference_op_replace_pass.cc @@ -16,14 +16,12 @@ #include "paddle/fluid/inference/analysis/argument.h" -COMMON_DECLARE_bool(enable_pir_in_executor); - namespace paddle { namespace inference { namespace analysis { void InferenceOpReplacePass::RunImpl(Argument* argument) { - if (FLAGS_enable_pir_in_executor) { + if (argument->use_pir()) { return; } diff --git a/paddle/fluid/inference/analysis/passes/ir_params_sync_among_devices_pass.cc b/paddle/fluid/inference/analysis/passes/ir_params_sync_among_devices_pass.cc index 2961d5c66f9f4..2e722f9a7e6e9 100644 --- a/paddle/fluid/inference/analysis/passes/ir_params_sync_among_devices_pass.cc +++ b/paddle/fluid/inference/analysis/passes/ir_params_sync_among_devices_pass.cc @@ -32,8 +32,6 @@ PD_DEFINE_bool( // NOLINT false, "Keep old mode for developers, the model is saved on cpu not device."); -COMMON_DECLARE_bool(enable_pir_in_executor); - namespace paddle { namespace inference { namespace analysis { @@ -208,9 +206,10 @@ void IrParamsSyncAmongDevicesPass::CopyParamsToXpu(Argument *argument) { #endif void IrParamsSyncAmongDevicesPass::RunImpl(Argument *argument) { - if (FLAGS_enable_pir_in_executor) { + if (argument->use_pir()) { return; } + PADDLE_ENFORCE_EQ( argument->scope_valid(), true, diff --git a/paddle/fluid/inference/api/analysis_config.cc b/paddle/fluid/inference/api/analysis_config.cc index 5987483220b8a..888e2cbe080c9 100644 --- a/paddle/fluid/inference/api/analysis_config.cc +++ b/paddle/fluid/inference/api/analysis_config.cc @@ -581,6 +581,7 @@ AnalysisConfig::AnalysisConfig(const AnalysisConfig &other) { CP_MEMBER(skip_load_params_); CP_MEMBER(use_new_executor_); + CP_MEMBER(use_pir_); if (use_gpu_) { PADDLE_ENFORCE_EQ(use_xpu_, diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index 9b05b9f78572e..1cc723cd7913e 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -134,7 +134,6 @@ #include "paddle/fluid/pir/transforms/shape_optimization_pass.h" #include "paddle/pir/include/pass/pass_manager.h" -COMMON_DECLARE_bool(enable_pir_in_executor); COMMON_DECLARE_bool(pir_apply_inplace_pass); namespace paddle { @@ -376,7 +375,7 @@ AnalysisPredictor::AnalysisPredictor(const AnalysisConfig &config) } if (config_.new_executor_enabled()) { config_.EnableMemoryOptim(false); - if (FLAGS_enable_pir_in_executor) { + if (config_.new_ir_enabled()) { config_.SwitchIrOptim(false); } } @@ -893,7 +892,7 @@ bool AnalysisPredictor::PrepareExecutor() { auto output_names = GetOutputNames(); execution_config.skip_gc_vars.insert(output_names.begin(), output_names.end()); - if (FLAGS_enable_pir_in_executor) { + if (config_.new_ir_enabled()) { pir_program_ = std::move( paddle::TranslateLegacyProgramToProgram(*inference_program_)); @@ -1715,6 +1714,7 @@ void AnalysisPredictor::PrepareArgument() { argument_->SetEnableIrOptim(config_.enable_ir_optim_); argument_->SetEnableMemoryOptim(config_.enable_memory_optim()); argument_->SetModelFromMemory(config_.model_from_memory_); + argument_->SetUsePIR(config_.new_ir_enabled()); // Analyze inference_program argument_->SetPredictorID(predictor_id_); argument_->SetRootPredictorID(root_predictor_id_); @@ -1953,14 +1953,14 @@ void AnalysisPredictor::PrepareArgument() { model_precision_ == phi::DataType::FLOAT32) { argument_->SetEnableIrOptim(true); pass_builder->ClearPasses(); - if (!FLAGS_enable_pir_in_executor) { + if (!config_.new_ir_enabled()) { pass_builder->AppendPass("map_op_to_another_pass"); pass_builder->AppendPass("simplify_with_basic_ops_pass"); pass_builder->AppendPass("is_test_pass"); pass_builder->AppendPass("constant_folding_pass"); } pass_builder->AppendPass("auto_mixed_precision_pass"); - if (!FLAGS_enable_pir_in_executor) { + if (!config_.new_ir_enabled()) { pass_builder->AppendPass("inplace_op_var_pass"); } LOG(INFO) << "This model run in GPU mixed precision mode with no ir " @@ -2083,8 +2083,9 @@ CreatePaddlePredictor( // Register custom operators compiled by the user. // This function can only be executed once per process. static std::once_flag custom_operators_registered; - std::call_once(custom_operators_registered, - []() { inference::RegisterAllCustomOperator(); }); + std::call_once(custom_operators_registered, [config]() { + inference::RegisterAllCustomOperator(config.new_ir_enabled()); + }); auto SetGflags = [](const AnalysisConfig &config) { auto SetGflag = [](const char *name, const char *value) { diff --git a/paddle/fluid/inference/api/demo_ci/custom_op_demo.cc b/paddle/fluid/inference/api/demo_ci/custom_op_demo.cc index b4c8cccb8e790..ec44238f008dc 100644 --- a/paddle/fluid/inference/api/demo_ci/custom_op_demo.cc +++ b/paddle/fluid/inference/api/demo_ci/custom_op_demo.cc @@ -52,6 +52,7 @@ int main(int argc, char **argv) { config.SetModel(FLAGS_modeldir + "/custom_relu.pdmodel", FLAGS_modeldir + "/custom_relu.pdiparams"); config.EnableNewExecutor(true); + config.EnableNewIR(true); auto predictor{paddle_infer::CreatePredictor(config)}; std::vector input_shape = {1, 1, 28, 28}; std::vector input_data(1 * 1 * 28 * 28, 1); diff --git a/paddle/fluid/inference/api/demo_ci/run.sh b/paddle/fluid/inference/api/demo_ci/run.sh index 795b414258b56..3de4fd3d0335a 100755 --- a/paddle/fluid/inference/api/demo_ci/run.sh +++ b/paddle/fluid/inference/api/demo_ci/run.sh @@ -301,7 +301,7 @@ for WITH_STATIC_LIB in ON OFF; do -DCUSTOM_OPERATOR_FILES=$CUSTOM_OPERATOR_FILES \ -DWITH_ONNXRUNTIME=$WITH_ONNXRUNTIME make -j$(nproc) - FLAGS_enable_pir_in_executor=1 ./custom_op_demo \ + ./custom_op_demo \ --modeldir=$DATA_DIR/custom_op/custom_relu_infer_model if [ $? -ne 0 ]; then echo "custom_op_demo runs failed " >> ${current_dir}/test_summary.txt diff --git a/paddle/fluid/inference/api/helper.cc b/paddle/fluid/inference/api/helper.cc index e9eb090a771d2..80429055465eb 100644 --- a/paddle/fluid/inference/api/helper.cc +++ b/paddle/fluid/inference/api/helper.cc @@ -22,8 +22,6 @@ #include "paddle/phi/api/ext/op_meta_info.h" #include "paddle/pir/include/core/ir_context.h" -COMMON_DECLARE_bool(enable_pir_in_executor); - namespace paddle { namespace inference { @@ -50,11 +48,11 @@ std::string to_string>>( return ss.str(); } -void RegisterAllCustomOperator() { +void RegisterAllCustomOperator(bool use_pir) { auto &op_meta_info_map = OpMetaInfoMap::Instance(); const auto &meta_info_map = op_meta_info_map.GetMap(); for (auto &pair : meta_info_map) { - if (FLAGS_enable_pir_in_executor) { + if (use_pir) { ::pir::IrContext *ctx = ::pir::IrContext::Instance(); auto *custom_dialect = ctx->GetOrRegisterDialect(); diff --git a/paddle/fluid/inference/api/helper.h b/paddle/fluid/inference/api/helper.h index 22a5319bb0dbc..17ec8852b61df 100644 --- a/paddle/fluid/inference/api/helper.h +++ b/paddle/fluid/inference/api/helper.h @@ -431,7 +431,7 @@ static bool IsFileExists(const std::string &path) { return exists; } -void RegisterAllCustomOperator(); +void RegisterAllCustomOperator(bool use_pir); void InitGflagsFromEnv(); diff --git a/paddle/fluid/inference/api/paddle_analysis_config.h b/paddle/fluid/inference/api/paddle_analysis_config.h index 134c0799ec663..64b2de0eba3d4 100644 --- a/paddle/fluid/inference/api/paddle_analysis_config.h +++ b/paddle/fluid/inference/api/paddle_analysis_config.h @@ -879,10 +879,22 @@ struct PD_INFER_DECL AnalysisConfig { /// int tensorrt_optimization_level() { return trt_optimization_level_; } + /// \brief A boolean state telling whether to use new executor. + /// + /// \return bool whether to use new executor. + /// void EnableNewExecutor(bool x = true) { use_new_executor_ = x; } bool new_executor_enabled() const { return use_new_executor_; } + /// \brief A boolean state telling whether to use new IR. + /// + /// \return bool whether to use new IR. + /// + void EnableNewIR(bool x = true) { use_pir_ = x; } + + bool new_ir_enabled() const { return use_pir_; } + /// /// \brief Control whether to use optimized model to inference. /// @@ -1425,6 +1437,8 @@ struct PD_INFER_DECL AnalysisConfig { // PrepareProgram(). So we add this flag to control the process. bool apply_optim_{false}; bool skip_load_params_{false}; + + bool use_pir_{false}; }; } // namespace paddle diff --git a/paddle/fluid/pybind/inference_api.cc b/paddle/fluid/pybind/inference_api.cc index 268806509031e..708866b0bac34 100644 --- a/paddle/fluid/pybind/inference_api.cc +++ b/paddle/fluid/pybind/inference_api.cc @@ -869,6 +869,8 @@ void BindAnalysisConfig(py::module *m) { .def("enable_new_executor", &AnalysisConfig::EnableNewExecutor, py::arg("x") = true) + .def("enable_new_ir", &AnalysisConfig::EnableNewIR, py::arg("x") = true) + .def("new_ir_enabled", &AnalysisConfig::new_ir_enabled) .def("enable_profile", &AnalysisConfig::EnableProfile) .def("disable_glog_info", &AnalysisConfig::DisableGlogInfo) .def("glog_info_disabled", &AnalysisConfig::glog_info_disabled) diff --git a/test/cpp/inference/analysis/analyzer_tester.cc b/test/cpp/inference/analysis/analyzer_tester.cc index 611fd757c2bcf..f4a8a0f7669b0 100644 --- a/test/cpp/inference/analysis/analyzer_tester.cc +++ b/test/cpp/inference/analysis/analyzer_tester.cc @@ -33,6 +33,7 @@ TEST(Analyzer, analysis_without_tensorrt) { argument.SetModelDir(FLAGS_inference_model_dir); argument.SetEnableIrOptim(false); argument.SetUseGPU(false); + argument.SetUsePIR(false); argument.SetAnalysisPasses({"ir_graph_build_pass", "ir_analysis_pass", "ir_params_sync_among_devices_pass"}); @@ -49,6 +50,7 @@ TEST(Analyzer, analysis_with_tensorrt) { argument.SetTensorRtWorkspaceSize(1 << 20); argument.SetModelDir(FLAGS_inference_model_dir); argument.SetUseGPU(false); + argument.SetUsePIR(false); argument.SetAnalysisPasses({"ir_graph_build_pass", "ir_analysis_pass", "ir_params_sync_among_devices_pass"}); diff --git a/test/custom_op/test_inference_inplace.py b/test/custom_op/test_inference_inplace.py index 303b2b21d15dc..64219d8e148d0 100644 --- a/test/custom_op/test_inference_inplace.py +++ b/test/custom_op/test_inference_inplace.py @@ -83,10 +83,7 @@ def setUp(self): def tearDown(self): self.temp_dir.cleanup() - def enable_pir(self, flag: bool): - paddle.set_flags({'FLAGS_enable_pir_in_executor': flag}) - - def init_predictor(self): + def init_predictor(self, use_pir: bool): config = Config( os.path.join( self.temp_dir.name, @@ -100,6 +97,8 @@ def init_predictor(self): config.enable_use_gpu(256, 0) config.switch_ir_optim(False) config.enable_new_executor() + if use_pir: + config.enable_new_ir() predictor = create_predictor(config) return predictor @@ -123,11 +122,9 @@ def get_outputs(self, predictor): return outputs[0] def test_output(self): - self.enable_pir(True) - pir_predictor = self.init_predictor() + pir_predictor = self.init_predictor(True) pir_output = self.get_outputs(pir_predictor) - self.enable_pir(False) - predictor = self.init_predictor() + predictor = self.init_predictor(False) output = self.get_outputs(predictor) np.testing.assert_allclose( output.numpy().flatten(), pir_output.numpy().flatten() diff --git a/test/ir/inference/auto_scan_test.py b/test/ir/inference/auto_scan_test.py index b26725314fb1f..02bd28d7139f9 100755 --- a/test/ir/inference/auto_scan_test.py +++ b/test/ir/inference/auto_scan_test.py @@ -352,13 +352,13 @@ def run_test_config( """ Test a single case. """ - paddle.set_flags({'FLAGS_enable_pir_in_executor': True}) + pred_config.enable_new_ir(True) pred_config.switch_ir_optim(False) pred_config.enable_new_executor() result = super().run_test_config( model, params, prog_config, pred_config, feed_data ) - paddle.set_flags({'FLAGS_enable_pir_in_executor': False}) + pred_config.enable_new_ir(False) return result diff --git a/test/ir/inference/program_config.py b/test/ir/inference/program_config.py index f3d44361260f9..f64335fc4379e 100644 --- a/test/ir/inference/program_config.py +++ b/test/ir/inference/program_config.py @@ -346,7 +346,6 @@ def _cast(self) -> None: def create_fake_model(program_config): '''Create a Paddle model(in memory) according to the given config.''' - paddle.set_flags({'FLAGS_enable_pir_in_executor': False}) program_config = copy.deepcopy(program_config) program_config._cast() paddle.enable_static() diff --git a/test/ir/inference/test_inference_predictor_run.py b/test/ir/inference/test_inference_predictor_run.py index 1d8abc174f1cf..21b095d797442 100644 --- a/test/ir/inference/test_inference_predictor_run.py +++ b/test/ir/inference/test_inference_predictor_run.py @@ -62,10 +62,7 @@ def setUp(self): def tearDown(self): self.temp_dir.cleanup() - def enable_pir(self, flag: bool): - paddle.set_flags({'FLAGS_enable_pir_in_executor': flag}) - - def init_predictor(self): + def init_predictor(self, use_pir: bool): config = Config( os.path.join( self.temp_dir.name, @@ -80,6 +77,8 @@ def init_predictor(self): config.switch_ir_optim(False) # config.enable_memory_optim() config.enable_new_executor() + if use_pir: + config.enable_new_ir() predictor = create_predictor(config) return predictor @@ -117,11 +116,9 @@ def get_inorder_output(self, predictor): return outputs[0] def test_output(self): - self.enable_pir(False) - predictor = self.init_predictor() + predictor = self.init_predictor(False) output = self.get_inorder_output(predictor) - self.enable_pir(True) - pir_predictor = self.init_predictor() + pir_predictor = self.init_predictor(True) pir_output = self.get_disorder_output(pir_predictor) np.testing.assert_allclose( diff --git a/test/ir/pir/cinn/symbolic/test_decomp_inference_predictor_run.py b/test/ir/pir/cinn/symbolic/test_decomp_inference_predictor_run.py index 0a9c091f05ee7..517cd7083288a 100644 --- a/test/ir/pir/cinn/symbolic/test_decomp_inference_predictor_run.py +++ b/test/ir/pir/cinn/symbolic/test_decomp_inference_predictor_run.py @@ -68,10 +68,7 @@ def setUp(self): def tearDown(self): self.temp_dir.cleanup() - def enable_pir(self, flag: bool): - paddle.set_flags({'FLAGS_enable_pir_in_executor': flag}) - - def init_predictor(self): + def init_predictor(self, use_pir: bool): config = Config( os.path.join( self.temp_dir.name, @@ -86,6 +83,8 @@ def init_predictor(self): config.enable_use_gpu(256, 0) config.switch_ir_optim(False) config.enable_new_executor() + if use_pir: + config.enable_new_ir() predictor = create_predictor(config) return predictor @@ -118,12 +117,11 @@ def get_inorder_output(self, predictor): return outputs[0] def test_output_prim_inorder(self): - self.enable_pir(False) - predictor = self.init_predictor() + predictor = self.init_predictor(False) output = self.get_inorder_output(predictor) - self.enable_pir(True) + paddle.set_flags({'FLAGS_enable_pir_in_executor': True}) paddle.core._set_prim_all_enabled(True) - pir_predictor = self.init_predictor() + pir_predictor = self.init_predictor(True) pir_output = self.get_inorder_output(pir_predictor) paddle.core._set_prim_all_enabled(False) @@ -135,12 +133,11 @@ def test_output_prim_inorder(self): ) def test_output_prim_disorder(self): - self.enable_pir(False) - predictor = self.init_predictor() + predictor = self.init_predictor(False) output = self.get_disorder_output(predictor) - self.enable_pir(True) + paddle.set_flags({'FLAGS_enable_pir_in_executor': True}) paddle.core._set_prim_all_enabled(True) - pir_predictor = self.init_predictor() + pir_predictor = self.init_predictor(True) pir_output = self.get_disorder_output(pir_predictor) paddle.core._set_prim_all_enabled(False) From a77172c4dae94550a27d4e620f77b7222556ac31 Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 1 Mar 2024 23:12:35 +0800 Subject: [PATCH 227/282] Fix tensor_comsumer tensor_consumer,etc (#62213) --- paddle/fluid/pir/drr/src/attr_type_uilts.h | 6 ++--- .../fluid/pir/drr/src/ir_operation_factory.cc | 24 +++++++++---------- paddle/fluid/pir/drr/src/pattern_graph.cc | 20 ++++++++-------- paddle/fluid/pir/drr/src/rewrite_pattern.cc | 2 +- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/paddle/fluid/pir/drr/src/attr_type_uilts.h b/paddle/fluid/pir/drr/src/attr_type_uilts.h index 02f5a4defc155..a48ed382a7d19 100644 --- a/paddle/fluid/pir/drr/src/attr_type_uilts.h +++ b/paddle/fluid/pir/drr/src/attr_type_uilts.h @@ -48,7 +48,7 @@ PD_SPECIALIZE_CppTypeToIrAttribute(phi::IntArray, paddle::dialect::IntArrayAttribute); template -struct IrAttrbuteCreator { +struct IrAttributeCreator { typename CppTypeToIrAttribute::type operator()(T obj) const { return CppTypeToIrAttribute::type::template get( pir::IrContext::Instance(), obj); @@ -56,7 +56,7 @@ struct IrAttrbuteCreator { }; template <> -struct IrAttrbuteCreator> { +struct IrAttributeCreator> { pir::ArrayAttribute operator()(std::vector obj) const { std::vector attr_vec; attr_vec.reserve(obj.size()); @@ -69,7 +69,7 @@ struct IrAttrbuteCreator> { }; template <> -struct IrAttrbuteCreator> { +struct IrAttributeCreator> { pir::ArrayAttribute operator()(std::vector obj) const { std::vector attr_vec; attr_vec.reserve(obj.size()); diff --git a/paddle/fluid/pir/drr/src/ir_operation_factory.cc b/paddle/fluid/pir/drr/src/ir_operation_factory.cc index 61c12c281e139..bfe97d45592f7 100644 --- a/paddle/fluid/pir/drr/src/ir_operation_factory.cc +++ b/paddle/fluid/pir/drr/src/ir_operation_factory.cc @@ -65,33 +65,33 @@ void OperationFactory::RegisterManualOpCreator() { pir::Attribute CreateIrAttribute(const std::any& obj) { if (obj.type() == typeid(bool)) { - return IrAttrbuteCreator()(std::any_cast(obj)); + return IrAttributeCreator()(std::any_cast(obj)); } else if (obj.type() == typeid(int32_t)) { - return IrAttrbuteCreator()(std::any_cast(obj)); + return IrAttributeCreator()(std::any_cast(obj)); } else if (obj.type() == typeid(int64_t)) { - return IrAttrbuteCreator()(std::any_cast(obj)); + return IrAttributeCreator()(std::any_cast(obj)); } else if (obj.type() == typeid(float)) { - return IrAttrbuteCreator()(std::any_cast(obj)); + return IrAttributeCreator()(std::any_cast(obj)); } else if (obj.type() == typeid(std::string)) { - return IrAttrbuteCreator()(std::any_cast(obj)); + return IrAttributeCreator()(std::any_cast(obj)); } else if (obj.type() == typeid(const char*)) { - return IrAttrbuteCreator()(std::any_cast(obj)); + return IrAttributeCreator()(std::any_cast(obj)); } else if (obj.type() == typeid(phi::DataType)) { - return IrAttrbuteCreator()( + return IrAttributeCreator()( std::any_cast(obj)); } else if (obj.type() == typeid(phi::Place)) { - return IrAttrbuteCreator()(std::any_cast(obj)); + return IrAttributeCreator()(std::any_cast(obj)); } else if (obj.type() == typeid(std::vector)) { // NOLINT - return IrAttrbuteCreator>()( + return IrAttributeCreator>()( std::any_cast>(obj)); } else if (obj.type() == typeid(std::vector)) { - return IrAttrbuteCreator>()( + return IrAttributeCreator>()( std::any_cast>(obj)); } else if (obj.type() == typeid(std::vector)) { - return IrAttrbuteCreator>()( + return IrAttributeCreator>()( std::any_cast>(obj)); } else if (obj.type() == typeid(phi::IntArray)) { - return IrAttrbuteCreator()( + return IrAttributeCreator()( std::any_cast(obj)); } else { PADDLE_THROW( diff --git a/paddle/fluid/pir/drr/src/pattern_graph.cc b/paddle/fluid/pir/drr/src/pattern_graph.cc index eccbb30dea890..be57150ed8ffd 100644 --- a/paddle/fluid/pir/drr/src/pattern_graph.cc +++ b/paddle/fluid/pir/drr/src/pattern_graph.cc @@ -148,7 +148,7 @@ void GraphTopo::WalkGraphNodesTopoOrder( graph_->input_tensors(); const std::unordered_map> &id2owned_tensor = graph_->id2owned_tensor(); - const std::vector> &owend_opcall = + const std::vector> &owned_opcall = graph_->owned_op_call(); std::queue opcall_queue; @@ -156,7 +156,7 @@ void GraphTopo::WalkGraphNodesTopoOrder( opcall_dependent; // init opcall_dependent - for (const std::shared_ptr &opcall_sptr : owend_opcall) { + for (const std::shared_ptr &opcall_sptr : owned_opcall) { if (opcall_sptr.get()->inputs().empty()) { // opcall inputs is empty opcall_queue.push(opcall_sptr.get()); } else { @@ -174,11 +174,11 @@ void GraphTopo::WalkGraphNodesTopoOrder( "The input tensor [%s] must exists " "in pattern graph to be obtained.", tensor_name)); - for (const auto &tensor_comsumer : + for (const auto &tensor_consumer : id2owned_tensor.at(tensor_name).get()->consumers()) { - opcall_dependent[tensor_comsumer].erase(tensor_name); - if (opcall_dependent[tensor_comsumer].empty()) { - opcall_queue.push(tensor_comsumer); + opcall_dependent[tensor_consumer].erase(tensor_name); + if (opcall_dependent[tensor_consumer].empty()) { + opcall_queue.push(tensor_consumer); } } } @@ -190,10 +190,10 @@ void GraphTopo::WalkGraphNodesTopoOrder( // update opcall_dependent for (const auto &output_tensor : opcall->outputs()) { - for (const auto &tensor_comsumer : output_tensor->consumers()) { - opcall_dependent[tensor_comsumer].erase(output_tensor->name()); - if (opcall_dependent[tensor_comsumer].empty()) { - opcall_queue.push(tensor_comsumer); + for (const auto &tensor_consumer : output_tensor->consumers()) { + opcall_dependent[tensor_consumer].erase(output_tensor->name()); + if (opcall_dependent[tensor_consumer].empty()) { + opcall_queue.push(tensor_consumer); } } } diff --git a/paddle/fluid/pir/drr/src/rewrite_pattern.cc b/paddle/fluid/pir/drr/src/rewrite_pattern.cc index 04390126ddddf..46b034aca8558 100644 --- a/paddle/fluid/pir/drr/src/rewrite_pattern.cc +++ b/paddle/fluid/pir/drr/src/rewrite_pattern.cc @@ -59,7 +59,7 @@ bool DrrRewritePattern::MatchAndRewrite( if (PatternGraphMatch(op, src_match_ctx.get())) { VLOG(4) << "DRR pattern (" << pattern_name_ << ") is matched in program."; PatternGraphRewrite(*src_match_ctx, rewriter); - VLOG(4) << "DRR pattern (" << pattern_name_ << ") is rewrited in program."; + VLOG(4) << "DRR pattern (" << pattern_name_ << ") is rewritten in program."; return true; } return false; From 78254af04977586d0be32f8129236feefb9663c9 Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 1 Mar 2024 23:13:54 +0800 Subject: [PATCH 228/282] Fix Unexpceted Unexpected, etc (#62260) --- .../fast_threaded_ssa_graph_executor.cc | 4 ++-- .../framework/details/fetch_op_handle.cc | 2 +- paddle/fluid/framework/operator.cc | 10 +++++----- paddle/fluid/framework/parallel_executor.cc | 10 +++++----- paddle/fluid/framework/tensor_util.cc | 8 +++++--- paddle/fluid/framework/trainer_factory.cc | 4 ++-- paddle/fluid/operators/cvm_op.cc | 2 +- paddle/fluid/platform/float16_test.cu | 2 +- .../fluid/prim/api/manual_prim/utils/utils.h | 6 +++--- paddle/phi/kernels/prior_box_kernel.h | 20 +++++++++---------- 10 files changed, 35 insertions(+), 33 deletions(-) diff --git a/paddle/fluid/framework/details/fast_threaded_ssa_graph_executor.cc b/paddle/fluid/framework/details/fast_threaded_ssa_graph_executor.cc index 19cf30d24db40..66c62085faed2 100644 --- a/paddle/fluid/framework/details/fast_threaded_ssa_graph_executor.cc +++ b/paddle/fluid/framework/details/fast_threaded_ssa_graph_executor.cc @@ -49,8 +49,8 @@ FastThreadedSSAGraphExecutor::FastThreadedSSAGraphExecutor( /*disable_setting_default_stream_for_allocator=*/true, /*stream_priority=*/0); if (ir::IsTopologySortOperationsUnique(*graph_)) { - VLOG(10) - << "Change thread number to 1 because the toposort order is unique"; + VLOG(10) << "Change thread number to 1 because the topology sort order is " + "unique"; strategy_.num_threads_ = 1; traced_ops_.clear(); for (auto *op_node : TopologySortOperations(*graph_)) { diff --git a/paddle/fluid/framework/details/fetch_op_handle.cc b/paddle/fluid/framework/details/fetch_op_handle.cc index 27be4b7717635..25108148af349 100644 --- a/paddle/fluid/framework/details/fetch_op_handle.cc +++ b/paddle/fluid/framework/details/fetch_op_handle.cc @@ -39,7 +39,7 @@ FetchOpHandle::~FetchOpHandle() = default; void FetchOpHandle::RecordWaitEventOnCtx(platform::DeviceContext *waited_ctx) { PADDLE_THROW(platform::errors::PermissionDenied( - "No nodes need to wait FetchOp. Unexpceted Error.")); + "No nodes need to wait FetchOp. Unexpected Error.")); } static void CheckDims(const framework::DDim &tensor_dims, diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc index 55fc19ad2be1c..afe442c0a7c6f 100644 --- a/paddle/fluid/framework/operator.cc +++ b/paddle/fluid/framework/operator.cc @@ -2038,7 +2038,7 @@ void OperatorWithKernel::RunImpl(const Scope& scope, phi::KernelContext phi_kernel_context; if (enable_cache_runtime_context_ && !need_prepare_phi_data_ && !need_prepare_data_) { - // TODO(inference): Now we only suppor dense_tensor cache, we may be + // TODO(inference): Now we only support dense_tensor cache, we may be // support ScalarTensor, SparseTensor in future. bool all_dense_tensor_input_{true}; for (auto& iter : Inputs()) { @@ -2573,7 +2573,7 @@ Scope* OperatorWithKernel::PrepareData( // for some situation like InferShape(). // In this situation We cannot skip Var analysis, as // oneDNN shape of Var may differ from kNHWC Var - // In such situation corressponding resized Var + // In such situation corresponding resized Var // has to be created and registered if ((tensor_in->layout() == DataLayout::ONEDNN) && (var->IsType() == true) && @@ -3193,7 +3193,7 @@ void OperatorWithKernel::BuildPhiKernelContext( for (size_t i = 0; i < input_names.size(); ++i) { auto it = ctx.inputs.find(input_names[i]); - // calcute the start and end index of the input tensors + // calculate the start and end index of the input tensors size_t start_idx = (i == 0 ? 0 : phi_kernel_context->InputRangeAt(i - 1).second); // deal with optional here @@ -3399,7 +3399,7 @@ void OperatorWithKernel::BuildPhiKernelContext( attr_iter, Attrs().end(), platform::errors::NotFound("(%s) is not found in AttributeMap when " - "buildind static KernelContext.", + "building static KernelContext.", attr_names[i])); switch (AttrTypeID(attr_iter->second)) { case proto::AttrType::INTS: { @@ -3473,7 +3473,7 @@ void OperatorWithKernel::BuildPhiKernelContext( RuntimeAttrs().end(), platform::errors::NotFound( "(%s) is not found in AttributeMap when " - "buildind static KernelContext.", + "building static KernelContext.", attr_names[i])); } diff --git a/paddle/fluid/framework/parallel_executor.cc b/paddle/fluid/framework/parallel_executor.cc index 897e520813809..c2b6c37e7dd6e 100644 --- a/paddle/fluid/framework/parallel_executor.cc +++ b/paddle/fluid/framework/parallel_executor.cc @@ -639,15 +639,15 @@ void InitP2P(const std::vector &places) { for (int i = 0; i < count; ++i) { for (int j = 0; j < count; ++j) { if (devices[i] == devices[j]) continue; - int can_acess = -1; + int can_access = -1; #ifdef PADDLE_WITH_HIP hipError_t ret = - hipDeviceCanAccessPeer(&can_acess, devices[i], devices[j]); - if (ret != hipSuccess || can_acess != 1) { + hipDeviceCanAccessPeer(&can_access, devices[i], devices[j]); + if (ret != hipSuccess || can_access != 1) { #else cudaError_t ret = - cudaDeviceCanAccessPeer(&can_acess, devices[i], devices[j]); - if (ret != cudaSuccess || can_acess != 1) { + cudaDeviceCanAccessPeer(&can_access, devices[i], devices[j]); + if (ret != cudaSuccess || can_access != 1) { #endif LOG(WARNING) << "Cannot enable P2P access from " << devices[i] << " to " << devices[j]; diff --git a/paddle/fluid/framework/tensor_util.cc b/paddle/fluid/framework/tensor_util.cc index fafde716b7bba..bd869a0588067 100644 --- a/paddle/fluid/framework/tensor_util.cc +++ b/paddle/fluid/framework/tensor_util.cc @@ -710,8 +710,9 @@ void TensorFromStream(std::istream& is, PADDLE_THROW(platform::errors::Unimplemented( "XPUPlace is not supported when not compiled with XPU")); } else { - PADDLE_THROW(platform::errors::Unimplemented( - "CutomPlace is not supported when not compiled with CustomDevice")); + PADDLE_THROW( + platform::errors::Unimplemented("CustomPlace is not supported when " + "not compiled with CustomDevice")); } #endif } else { @@ -887,7 +888,8 @@ std::ostream& print_tensor(std::ostream& os, const phi::DenseTensor& tensor) { auto element_num = tensor.numel(); os << " - data: ["; - // Note: int8_t && uint8_t is typedf of char, ostream unable to print properly + // Note: int8_t && uint8_t is typedef of char, ostream unable to print + // properly if (typeid(int8_t) == typeid(T) || typeid(uint8_t) == typeid(T)) { if (element_num > 0) { os << signed(inspect[0]); diff --git a/paddle/fluid/framework/trainer_factory.cc b/paddle/fluid/framework/trainer_factory.cc index ba5dac4830aa1..81b2df6efc723 100644 --- a/paddle/fluid/framework/trainer_factory.cc +++ b/paddle/fluid/framework/trainer_factory.cc @@ -26,8 +26,8 @@ namespace framework { class TrainerBase; -typedef std::shared_ptr (*CreatetrainerFunction)(); -typedef std::unordered_map trainerMap; +typedef std::shared_ptr (*CreateTrainerFunction)(); +typedef std::unordered_map trainerMap; trainerMap g_trainer_map; #define REGISTER_TRAINER_CLASS(trainer_class) \ diff --git a/paddle/fluid/operators/cvm_op.cc b/paddle/fluid/operators/cvm_op.cc index 578a59130495a..1e414ff217c2f 100644 --- a/paddle/fluid/operators/cvm_op.cc +++ b/paddle/fluid/operators/cvm_op.cc @@ -127,7 +127,7 @@ class CVMOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("X", "(LodTensor, default LodTensor), a 2-D tensor with shape " "[N x D]," - " where N is the batch size and D is the emebdding dim. "); + " where N is the batch size and D is the embedding dim. "); AddInput("CVM", "(Tensor), a 2-D Tensor with shape [N x 2], where N is the batch " "size, 2 is show and click."); diff --git a/paddle/fluid/platform/float16_test.cu b/paddle/fluid/platform/float16_test.cu index 4575b54d48c9b..555f83d61675e 100644 --- a/paddle/fluid/platform/float16_test.cu +++ b/paddle/fluid/platform/float16_test.cu @@ -282,7 +282,7 @@ TEST(float16, compound_on_gpu) { TestDivAssign(6, 2, 3); } -TEST(float16, comparision_on_gpu) { +TEST(float16, comparison_on_gpu) { TestEqual(1, 1, true); TestEqual(1, 2, false); TestNotEqual(2, 3, true); diff --git a/paddle/fluid/prim/api/manual_prim/utils/utils.h b/paddle/fluid/prim/api/manual_prim/utils/utils.h index 90a25f8bf1e1f..f3b21169e57f1 100644 --- a/paddle/fluid/prim/api/manual_prim/utils/utils.h +++ b/paddle/fluid/prim/api/manual_prim/utils/utils.h @@ -29,7 +29,7 @@ namespace prim { // We put some api like utils here template Tensor empty(const paddle::experimental::IntArray& shape, - phi::DataType dype, + phi::DataType dtype, const paddle::Place& place); template @@ -37,7 +37,7 @@ Tensor empty_like(const Tensor& x, phi::DataType dtype, const paddle::Place& place); -// copy tensor for output ptr, in static need use assigh op +// copy tensor for output ptr, in static need use assign op template void by_pass(const Tensor& x, Tensor* out); @@ -114,7 +114,7 @@ static std::vector unsafe_vector_cast(const std::vector& src) { return dst; } -// This fucction compute unsqueeze dims for reshape to replace unsqueeze. +// This function compute unsqueeze dims for reshape to replace unsqueeze. static std::vector get_unsqueeze_dims( const Tensor& origin, const std::vector& axis) { auto origin_dims = origin.shape(); diff --git a/paddle/phi/kernels/prior_box_kernel.h b/paddle/phi/kernels/prior_box_kernel.h index 45a741c7a3a72..132efb7b6cc72 100644 --- a/paddle/phi/kernels/prior_box_kernel.h +++ b/paddle/phi/kernels/prior_box_kernel.h @@ -35,25 +35,25 @@ void PriorBoxKernel(const Context& ctx, DenseTensor* out, DenseTensor* var); -inline void ExpandAspectRatios(const std::vector& input_aspect_ratior, +inline void ExpandAspectRatios(const std::vector& input_aspect_ratio, bool flip, - std::vector* output_aspect_ratior) { + std::vector* output_aspect_ratio) { constexpr float epsilon = 1e-6; - output_aspect_ratior->clear(); - output_aspect_ratior->push_back(1.0f); - for (size_t i = 0; i < input_aspect_ratior.size(); ++i) { - float ar = input_aspect_ratior[i]; + output_aspect_ratio->clear(); + output_aspect_ratio->push_back(1.0f); + for (size_t i = 0; i < input_aspect_ratio.size(); ++i) { + float ar = input_aspect_ratio[i]; bool already_exist = false; - for (size_t j = 0; j < output_aspect_ratior->size(); ++j) { - if (fabs(ar - output_aspect_ratior->at(j)) < epsilon) { + for (size_t j = 0; j < output_aspect_ratio->size(); ++j) { + if (fabs(ar - output_aspect_ratio->at(j)) < epsilon) { already_exist = true; break; } } if (!already_exist) { - output_aspect_ratior->push_back(ar); + output_aspect_ratio->push_back(ar); if (flip) { - output_aspect_ratior->push_back(1.0f / ar); + output_aspect_ratio->push_back(1.0f / ar); } } } From 317fad13a6d7cfcebd69405ad8a9c5561b117daf Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 1 Mar 2024 23:15:22 +0800 Subject: [PATCH 229/282] Fix maxinum maximum, etc (#62290) --- paddle/phi/kernels/bmm_kernel.h | 2 +- .../kernels/xpu/instance_norm_grad_kernel.cc | 2 +- paddle/phi/kernels/xpu/inverse_kernel.cc | 2 +- .../phi/kernels/xpu/multiclass_nms3_kernel.cc | 2 +- paddle/phi/kernels/xpu/prelu_grad_kernel.cc | 4 +-- .../phi/kernels/xpu/reduce_max_grad_kernel.cc | 30 +++++++++---------- .../phi/kernels/xpu/reduce_min_grad_kernel.cc | 30 +++++++++---------- paddle/phi/kernels/xpu/rnn_util.h | 2 +- .../phi/kernels/xpu/set_value_grad_kernel.cc | 2 +- paddle/phi/kernels/xpu/set_value_kernel.cc | 2 +- 10 files changed, 39 insertions(+), 39 deletions(-) diff --git a/paddle/phi/kernels/bmm_kernel.h b/paddle/phi/kernels/bmm_kernel.h index 09e7f9647b68e..6d3733bf750d3 100644 --- a/paddle/phi/kernels/bmm_kernel.h +++ b/paddle/phi/kernels/bmm_kernel.h @@ -22,7 +22,7 @@ namespace phi { * @brief Bmm Kernel. * Applies batched matrix multiplication to two tensors. * - * Both of the two input tensors must be three-dementional + * Both of the two input tensors must be three-dimensional * and share the same batch size. * if x is a (b, m, k) tensor, y is a (b, k, n) tensor, * the output will be a (b, m, n) tensor. diff --git a/paddle/phi/kernels/xpu/instance_norm_grad_kernel.cc b/paddle/phi/kernels/xpu/instance_norm_grad_kernel.cc index dba0e2ccfd765..f1a217ed81ad3 100644 --- a/paddle/phi/kernels/xpu/instance_norm_grad_kernel.cc +++ b/paddle/phi/kernels/xpu/instance_norm_grad_kernel.cc @@ -39,7 +39,7 @@ void InstanceNormGradKernel(const Context& dev_ctx, true, phi::errors::InvalidArgument( "The size of input's dimensions should be less equal than 5", - "and the dimension of D should be eaual to 1", + "and the dimension of D should be equal to 1", "But received: the size of input's dimensions is [%d]", x_dims.size())); diff --git a/paddle/phi/kernels/xpu/inverse_kernel.cc b/paddle/phi/kernels/xpu/inverse_kernel.cc index a48baa508ade0..966fcc97e0ab0 100644 --- a/paddle/phi/kernels/xpu/inverse_kernel.cc +++ b/paddle/phi/kernels/xpu/inverse_kernel.cc @@ -41,7 +41,7 @@ void InverseKernel(const Context& dev_ctx, 8192, phi::errors::InvalidArgument( "The size of a single matrix (%d bytes) exceeds the " - "maxinum numbers of bytes xpu supports (8192).", + "maximum numbers of bytes xpu supports (8192).", n * n * sizeof(T))); auto RAII_GUARD = xpu::ctx_guard(dev_ctx.x_context()); auto* info_xpu = RAII_GUARD.alloc_l3_or_gm(batch); diff --git a/paddle/phi/kernels/xpu/multiclass_nms3_kernel.cc b/paddle/phi/kernels/xpu/multiclass_nms3_kernel.cc index 17746e4eeff0a..2f343ccc6b494 100644 --- a/paddle/phi/kernels/xpu/multiclass_nms3_kernel.cc +++ b/paddle/phi/kernels/xpu/multiclass_nms3_kernel.cc @@ -90,7 +90,7 @@ void MultiClassNMSKernel(const Context& ctx, PADDLE_ENFORCE_EQ( boxes_count == score_dims[0], true, - phi::errors::InvalidArgument("boxes_count shuold equal score_dims[0].", + phi::errors::InvalidArgument("boxes_count should equal score_dims[0].", "But received: (%d) and (%d)", boxes_count, score_dims[0])); diff --git a/paddle/phi/kernels/xpu/prelu_grad_kernel.cc b/paddle/phi/kernels/xpu/prelu_grad_kernel.cc index fa43c90883766..b7c2157d55f43 100644 --- a/paddle/phi/kernels/xpu/prelu_grad_kernel.cc +++ b/paddle/phi/kernels/xpu/prelu_grad_kernel.cc @@ -60,9 +60,9 @@ void PReluGradKernel(const Context& dev_ctx, } } - // mode = 0: channel_nchw, slope_shape = {c}, default. meanwhile, xhsape = {n, + // mode = 0: channel_nchw, slope_shape = {c}, default. meanwhile, xshape = {n, // c, h, w} - // mode = 1, channel_nhwc, slope_shape = {c}, meanwhile, xhsape = {n, h, w, c} + // mode = 1, channel_nhwc, slope_shape = {c}, meanwhile, xshape = {n, h, w, c} // mode = 2, elementwise, slope_shape = {c*h*w} // mode = 3, single slope, slope_shape = {1} diff --git a/paddle/phi/kernels/xpu/reduce_max_grad_kernel.cc b/paddle/phi/kernels/xpu/reduce_max_grad_kernel.cc index 846250c067740..aa8736d84b71f 100644 --- a/paddle/phi/kernels/xpu/reduce_max_grad_kernel.cc +++ b/paddle/phi/kernels/xpu/reduce_max_grad_kernel.cc @@ -60,23 +60,23 @@ void ReduceMaxGradKernel(const Context& dev_ctx, } } - T* brocast1 = nullptr; - T* brocast2 = nullptr; + T* broadcast1 = nullptr; + T* broadcast2 = nullptr; bool* equal = nullptr; xpu::ctx_guard RAII_GUARD(dev_ctx.x_context()); - brocast1 = RAII_GUARD.alloc_l3_or_gm(x.numel()); + broadcast1 = RAII_GUARD.alloc_l3_or_gm(x.numel()); PADDLE_ENFORCE_NOT_NULL( - brocast1, errors::ResourceExhausted("XPU has no enough memory")); + broadcast1, errors::ResourceExhausted("XPU has no enough memory")); equal = RAII_GUARD.alloc_l3_or_gm(x.numel()); PADDLE_ENFORCE_NOT_NULL( equal, errors::ResourceExhausted("XPU has no enough memory")); - brocast2 = RAII_GUARD.alloc_l3_or_gm(x.numel()); + broadcast2 = RAII_GUARD.alloc_l3_or_gm(x.numel()); PADDLE_ENFORCE_NOT_NULL( - brocast2, errors::ResourceExhausted("XPU has no enough memory")); + broadcast2, errors::ResourceExhausted("XPU has no enough memory")); // use [1] to replace [], because xpu not support [] if (xdims.size() == 0) { @@ -86,25 +86,25 @@ void ReduceMaxGradKernel(const Context& dev_ctx, ydims = std::vector({1}); } - // step 1. brocast out and out_grad - int r = - xpu::broadcast(dev_ctx.x_context(), out_data, brocast1, ydims, xdims); + // step 1. broadcast out and out_grad + int r = xpu::broadcast( + dev_ctx.x_context(), out_data, broadcast1, ydims, xdims); PADDLE_ENFORCE_XDNN_SUCCESS(r, "broadcast"); r = xpu::broadcast( - dev_ctx.x_context(), out_grad_data, brocast2, ydims, xdims); + dev_ctx.x_context(), out_grad_data, broadcast2, ydims, xdims); PADDLE_ENFORCE_XDNN_SUCCESS(r, "broadcast"); - // step 2. comparse out_brocast and x - r = xpu::equal(dev_ctx.x_context(), x_data, brocast1, equal, x.numel()); + // step 2. compare out_broadcast and x + r = xpu::equal(dev_ctx.x_context(), x_data, broadcast1, equal, x.numel()); PADDLE_ENFORCE_XDNN_SUCCESS(r, "equal"); // step 3. get x_grad - r = xpu::constant(dev_ctx.x_context(), brocast1, x.numel(), 0); + r = xpu::constant(dev_ctx.x_context(), broadcast1, x.numel(), 0); PADDLE_ENFORCE_XDNN_SUCCESS(r, "constant"); r = xpu::select(dev_ctx.x_context(), equal, - brocast2, - brocast1, + broadcast2, + broadcast1, x_grad_data, xdims, xdims); diff --git a/paddle/phi/kernels/xpu/reduce_min_grad_kernel.cc b/paddle/phi/kernels/xpu/reduce_min_grad_kernel.cc index 9019cb0834d72..aefcc74b45091 100644 --- a/paddle/phi/kernels/xpu/reduce_min_grad_kernel.cc +++ b/paddle/phi/kernels/xpu/reduce_min_grad_kernel.cc @@ -60,23 +60,23 @@ void ReduceMinGradKernel(const Context& dev_ctx, } } - T* brocast1 = nullptr; - T* brocast2 = nullptr; + T* broadcast1 = nullptr; + T* broadcast2 = nullptr; bool* equal = nullptr; xpu::ctx_guard RAII_GUARD(dev_ctx.x_context()); - brocast1 = RAII_GUARD.alloc_l3_or_gm(x.numel()); + broadcast1 = RAII_GUARD.alloc_l3_or_gm(x.numel()); PADDLE_ENFORCE_NOT_NULL( - brocast1, errors::ResourceExhausted("XPU has no enough memory")); + broadcast1, errors::ResourceExhausted("XPU has no enough memory")); equal = RAII_GUARD.alloc_l3_or_gm(x.numel()); PADDLE_ENFORCE_NOT_NULL( equal, errors::ResourceExhausted("XPU has no enough memory")); - brocast2 = RAII_GUARD.alloc_l3_or_gm(x.numel()); + broadcast2 = RAII_GUARD.alloc_l3_or_gm(x.numel()); PADDLE_ENFORCE_NOT_NULL( - brocast2, errors::ResourceExhausted("XPU has no enough memory")); + broadcast2, errors::ResourceExhausted("XPU has no enough memory")); // use [1] to replace [], because xpu not support [] if (xdims.size() == 0) { @@ -86,25 +86,25 @@ void ReduceMinGradKernel(const Context& dev_ctx, ydims = std::vector({1}); } - // step 1. brocast out and out_grad - int r = - xpu::broadcast(dev_ctx.x_context(), out_data, brocast1, ydims, xdims); + // step 1. broadcast out and out_grad + int r = xpu::broadcast( + dev_ctx.x_context(), out_data, broadcast1, ydims, xdims); PADDLE_ENFORCE_XDNN_SUCCESS(r, "broadcast"); r = xpu::broadcast( - dev_ctx.x_context(), out_grad_data, brocast2, ydims, xdims); + dev_ctx.x_context(), out_grad_data, broadcast2, ydims, xdims); PADDLE_ENFORCE_XDNN_SUCCESS(r, "broadcast"); - // step 2. comparse out_brocast and x - r = xpu::equal(dev_ctx.x_context(), x_data, brocast1, equal, x.numel()); + // step 2. compare out_broadcast and x + r = xpu::equal(dev_ctx.x_context(), x_data, broadcast1, equal, x.numel()); PADDLE_ENFORCE_XDNN_SUCCESS(r, "equal"); // step 3. get x_grad - r = xpu::constant(dev_ctx.x_context(), brocast1, x.numel(), 0); + r = xpu::constant(dev_ctx.x_context(), broadcast1, x.numel(), 0); PADDLE_ENFORCE_XDNN_SUCCESS(r, "constant"); r = xpu::select(dev_ctx.x_context(), equal, - brocast2, - brocast1, + broadcast2, + broadcast1, x_grad_data, xdims, xdims); diff --git a/paddle/phi/kernels/xpu/rnn_util.h b/paddle/phi/kernels/xpu/rnn_util.h index 5310b35e64dc3..7948bb2defa0c 100644 --- a/paddle/phi/kernels/xpu/rnn_util.h +++ b/paddle/phi/kernels/xpu/rnn_util.h @@ -23,7 +23,7 @@ void ResetParameterVector(const std::vector& raw_params_vec, const int& num_layers, const bool& is_bidirec, std::vector>* params_vec) { - // the parameter raw seuquence is [FWhi, FWhh, BWhi, BWhh] * num_layers + // the parameter raw sequence is [FWhi, FWhh, BWhi, BWhh] * num_layers // + [FBhi, FBhh, BBhi, BBhh] * num_layers, we will reset the parameter to // ([FWhi, FWhh, FBhi, FBhh] + [BWhi, BWhh, BBhi, BBhh]) * num_layers const int& direction_num = is_bidirec ? 2 : 1; diff --git a/paddle/phi/kernels/xpu/set_value_grad_kernel.cc b/paddle/phi/kernels/xpu/set_value_grad_kernel.cc index c5d33ae4ac8d0..227d6b39c9f28 100644 --- a/paddle/phi/kernels/xpu/set_value_grad_kernel.cc +++ b/paddle/phi/kernels/xpu/set_value_grad_kernel.cc @@ -203,7 +203,7 @@ void SetValueGradImpl(const Context& dev_ctx, auto value_grad_dims = value_grad->dims(); auto fake_value_grad_dims = out_dims; - // Create an extented shape according to the rules of broadcast. + // Create an extended shape according to the rules of broadcast. auto value_grad_dims_size = value_grad_dims.size(); int num_decrease = 0; diff --git a/paddle/phi/kernels/xpu/set_value_kernel.cc b/paddle/phi/kernels/xpu/set_value_kernel.cc index c457a6d21fd8a..60b0fff7d9d7c 100644 --- a/paddle/phi/kernels/xpu/set_value_kernel.cc +++ b/paddle/phi/kernels/xpu/set_value_kernel.cc @@ -263,7 +263,7 @@ void SetValueKernelImpl(const Context& dev_ctx, const std::vector& decrease_axes, const std::vector& none_axes, DenseTensor* out) { - // rank是xtensor的维度信息 + // rank是x tensor的维度信息 const int rank = x.dims().size(); switch (rank) { From 13d74009555434d6327a00a01aee68fc111c14bb Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 1 Mar 2024 23:17:04 +0800 Subject: [PATCH 230/282] Update kernel_backward.h (#62288) --- .../fusion/cutlass/memory_efficient_attention/kernel_backward.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/paddle/phi/kernels/fusion/cutlass/memory_efficient_attention/kernel_backward.h b/paddle/phi/kernels/fusion/cutlass/memory_efficient_attention/kernel_backward.h index 31ce0bd3574ee..2bd3ac2db5f5b 100644 --- a/paddle/phi/kernels/fusion/cutlass/memory_efficient_attention/kernel_backward.h +++ b/paddle/phi/kernels/fusion/cutlass/memory_efficient_attention/kernel_backward.h @@ -492,8 +492,6 @@ struct AttentionBackwardKernel { scalar_t, // ElementC accum_t // ElementAccumulator >; - static constexpr auto kOptimalAlignement = - std::max(DefaultConfig::kAlignmentA, DefaultConfig::kAlignmentB); static constexpr auto kMinimumAlignment = GemmType::kMinimumAlignment; struct MatmulQK { From 06d3a5de0321e2d23787a1a6ea1e4572e294585b Mon Sep 17 00:00:00 2001 From: Yuanle Liu Date: Sat, 2 Mar 2024 04:32:36 +0800 Subject: [PATCH 231/282] Fix copy *.h on paddle/pir dir introduced from PR#61863 (#62293) --- python/setup.py.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/setup.py.in b/python/setup.py.in index 9fd352ddd26be..3ba1dc05e4976 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -876,7 +876,7 @@ headers = ( # init headers list(find_files('init_phi.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/platform')) + # phi init headers # init headers - list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/pir/include')) + # pir init headers + list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/pir/include', recursive=True)) + # pir init headers # init headers list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/pir/drr/include')) + # drr init headers # init headers From cbe8810bbea29c28cc99ccd764134dd30fb61e84 Mon Sep 17 00:00:00 2001 From: lanxianghit <47554610+lanxianghit@users.noreply.github.com> Date: Sat, 2 Mar 2024 08:19:07 +0800 Subject: [PATCH 232/282] [PIR][DynamicShape] Fix bug in slice op's InferSymbolicShape (#62247) * Fix bug in slice op's InferSymbolicShape * add more tests * fix ci --- .../infer_symbolic_shape/infer_sym_utils.cc | 11 + .../infer_symbolic_shape/infer_sym_utils.h | 8 + .../paddle_op_infer_sym.cc | 241 +++++++++++------- .../shape_dialect/shape_optimization_test.cc | 8 +- .../cinn/symbolic/test_op_infer_sym_shape.py | 58 +++++ 5 files changed, 231 insertions(+), 95 deletions(-) diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.cc index 4e5f5df08732a..5675429b5c65f 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.cc @@ -16,6 +16,17 @@ namespace paddle::dialect::details { +std::optional> VecExpr2Int64(const ExprVec &expr_vec) { + std::vector int64vec; + for (auto item : expr_vec) { + if (!item.isa()) { + return std::nullopt; + } + int64vec.push_back(item.Get()); + } + return int64vec; +} + bool ReduceInferDim(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis, const std::vector &axis, diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.h index 8a14e40e6337a..d2d508ff5890d 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.h @@ -17,6 +17,12 @@ #include "paddle/fluid/pir/dialect/operator/ir/pd_op.h" #include "paddle/pir/include/dialect/shape/utils/shape_analysis.h" +// To make codes shorter +using ExprVec = std::vector; +using ShapeOrData = symbol::ShapeOrDataDimExprs; +using TensorExprs = symbol::TensorShapeOrDataDimExprs; +using TensorListExprs = symbol::TensorListShapeOrDataDimExprs; + namespace paddle::dialect::details { template struct AttributeTrait; @@ -60,6 +66,8 @@ std::vector GetVectorAttr(const ::pir::Operation *op, return vec_res; } +std::optional> VecExpr2Int64(const ExprVec &expr_vec); + bool ReduceInferDim(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis, const std::vector &axis, diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc index d95f109563518..1be26c82f4c21 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc @@ -19,11 +19,6 @@ namespace paddle::dialect { -// To make codes shorter -using ShapeOrData = symbol::ShapeOrDataDimExprs; -using TensorExprs = symbol::TensorShapeOrDataDimExprs; -using TensorListExprs = symbol::TensorListShapeOrDataDimExprs; - bool DataOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { const auto &attributes = op->attributes(); @@ -270,9 +265,104 @@ bool FullIntArrayOpInferSymbolicShape( return true; } +inline void CheckAndUpdateSliceAttrs( + const ExprVec &in_dims, + const std::vector &axes, + ExprVec *starts_p, + ExprVec *ends_p, + std::vector *infer_flags = nullptr) { + auto vec_int64 = details::VecExpr2Int64(*starts_p); + IR_ENFORCE(vec_int64.has_value(), + "for slice op, all the elements in `starts` must be int64_t"); + std::vector starts_int = vec_int64.value(); + + vec_int64 = details::VecExpr2Int64(*ends_p); + IR_ENFORCE(vec_int64.has_value(), + "for slice op, all the elements in `ends` must be int64_t"); + std::vector ends_int = vec_int64.value(); + + ExprVec &starts = *starts_p; + ExprVec &ends = *ends_p; + auto IsMaxInt = [](const symbol::DimExpr &expr) { + return expr.isa() && + expr.Get() == + static_cast(std::numeric_limits::max()); + }; + + for (size_t i = 0; i < axes.size(); ++i) { + int64_t axis = axes[i]; + + if (infer_flags != nullptr && (*infer_flags)[i] == -1) { + PADDLE_THROW( + phi::errors::Unimplemented("SliceOpInferSymbolicShape CAN NOT " + "deal with -1 in infer_flags now")); + } + + // For both start and end can be negtive or positive, we need to handle the + // following different arrangements. + ends[i] = IsMaxInt(ends[i]) ? in_dims[axis] : ends[i]; + + bool both_negative_or_positive = (starts_int[i] >= 0 && ends_int[i] >= 0) || + (starts_int[i] <= 0 && ends_int[i] <= 0); + bool start_negative_end_positive = starts_int[i] <= 0 && ends_int[i] >= 0; + bool start_positive_end_negative = starts_int[i] >= 0 && ends_int[i] <= 0; + + if (both_negative_or_positive) { + continue; + } else if (start_negative_end_positive) { + starts[i] = starts[i] + in_dims[axis]; + } else if (start_positive_end_negative) { + starts[i] = starts[i] - in_dims[axis]; + } else { + LOG(FATAL) << "Dead code"; + } + } +} + +inline ExprVec GetSliceDims(const ExprVec &in_dims, + const std::vector &axes, + const ExprVec &starts, + const ExprVec &ends, + std::vector *infer_flags = nullptr) { + ExprVec slice_dims(in_dims); + + for (size_t i = 0; i < axes.size(); ++i) { + int64_t axis = axes[i]; + + if (infer_flags != nullptr && (*infer_flags)[i] == -1) { + PADDLE_THROW( + phi::errors::Unimplemented("SliceOpInferSymbolicShape CAN NOT " + "deal with -1 in infer_flags now")); + } + + slice_dims[axis] = ends[i] - starts[i]; + } + + return slice_dims; +} + +inline ExprVec GetDecreasedDims(const ExprVec &slice_dims, + const std::vector &decrease_axes) { + ExprVec decreased_dims(slice_dims); + std::vector decrease_flag(slice_dims.size(), 0); + if (decrease_axes.size() > 0) { + for (size_t i = 0; i < decrease_axes.size(); ++i) { + int64_t axis = decrease_axes[i]; + decrease_flag[axis] = 1; + } + ExprVec new_shape; + for (size_t i = 0; i < slice_dims.size(); ++i) { + if (decrease_flag[i] == 0) { + new_shape.emplace_back(slice_dims[i]); + } + } + decreased_dims = new_shape; + } + return decreased_dims; +} + bool SliceOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - // TODO(zhangbopd): Not implemented yet. pir::Value operand_source = op->operand_source(0); pir::Value operand_starts = op->operand_source(1); pir::Value operand_ends = op->operand_source(2); @@ -285,107 +375,76 @@ bool SliceOpInferSymbolicShape(pir::Operation *op, const symbol::ShapeOrDataDimExprs &ends_shape_data = shape_analysis->GetShapeOrDataForValue(operand_ends); - // Currently, we DO NOT support the case that any element in `axes` `starts` - // or `ends` is a Symbol. const std::vector axes = [&] { - const auto &attributes = op->attributes(); - pir::Attribute attr_axes = attributes.at("axes"); - - const auto &axes_vec = attr_axes.dyn_cast().AsVector(); - std::vector axes; + std::vector axes_vec = details::GetVectorAttr(op, "axes"); int64_t rank = int64_t(operand_shape_or_data.shape().size()); - for (auto item : axes_vec) { - int64_t axis = item.dyn_cast().data(); - axes.emplace_back(axis >= 0 ? axis : std::max(int64_t(0), axis + rank)); + for (size_t i = 0; i < axes_vec.size(); i++) { + int64_t axis = axes_vec[i]; + axes_vec[i] = axis >= 0 ? axis : std::max(int64_t(0), axis + rank); } - return axes; + return axes_vec; }(); - const std::vector starts = [&] { - std::vector starts; - for (auto item : starts_shape_data.data().value()) { - IR_ENFORCE(item.isa(), - "Currently, we DO NOT support the case that any element in " - "`starts` is a Symbol."); - starts.push_back(item.Get()); - } - return starts; - }(); + // Currently, we DO NOT support any element in `starts` is a Symbol. + ExprVec starts = starts_shape_data.data().value(); + ExprVec ends = ends_shape_data.data().value(); - const std::vector ends = [&] { - std::vector ends; - for (auto item : ends_shape_data.data().value()) { - IR_ENFORCE(item.isa(), - "Currently, we DO NOT support the case that any element in " - "`ends` is a Symbol."); - ends.push_back(item.Get()); + std::vector infer_flags = [op, &axes] { + std::vector infer_flags_t = + details::GetVectorAttr(op, "infer_flags"); + if (infer_flags_t.empty()) { + infer_flags_t = std::vector(axes.size(), 1); } - return ends; + return infer_flags_t; }(); - // When `pd.slice` is operating on a tensor which is produced by a `pd.shape` - // op, the reseult should be written into data. - const auto &GetDataDimExprs = [&]() -> symbol::ShapeOrDataDimExprs { - const std::vector out_data = [&] { - std::vector out_data; - const int64_t start = - starts[0] < 0 - ? starts[0] + operand_shape_or_data.data().value().size() - : starts[0]; - const int64_t end = - static_cast(std::numeric_limits::max()) == ends[0] - ? operand_shape_or_data.data().value().size() - : ends[0]; - - for (int64_t i = start; i < end; i++) { - out_data.push_back(operand_shape_or_data.data().value()[i]); - } - return out_data; - }(); - const std::vector shape{std::int64_t(out_data.size())}; - return symbol::ShapeOrDataDimExprs{ - symbol::TensorShapeOrDataDimExprs(shape, out_data)}; - }; + const std::vector decrease_axis = + details::GetVectorAttr(op, "decrease_axis"); - // Othewise, the reseult should be written into the shape. const auto &GetShapeDimExprs = [&]() -> symbol::ShapeOrDataDimExprs { - std::vector out_shape = operand_shape_or_data.shape(); + const ExprVec &in_dims = operand_shape_or_data.shape(); + CheckAndUpdateSliceAttrs(in_dims, axes, &starts, &ends, &infer_flags); + ExprVec slice_dims = + GetSliceDims(in_dims, axes, starts, ends, &infer_flags); + ExprVec out_dims = GetDecreasedDims(slice_dims, decrease_axis); - const std::vector &dim_expr_starts = - starts_shape_data.data().value(); - const std::vector &dim_expr_ends = - ends_shape_data.data().value(); + return symbol::ShapeOrDataDimExprs{ + symbol::TensorShapeOrDataDimExprs(out_dims)}; + }; - // For both start and end can be negtive or positive, we need to handle the - // following different arrangements. - auto IsMaxInt = [](const symbol::DimExpr &expr) { - return expr.isa() && - expr.Get() == - static_cast(std::numeric_limits::max()); - }; - for (size_t i = 0; i < axes.size(); ++i) { - const int64_t axis = axes[i]; - auto end = - IsMaxInt(dim_expr_ends[i]) ? out_shape[axis] : dim_expr_ends[i]; - - bool both_negative_or_positive = - (starts[i] >= 0 && ends[i] >= 0) || (starts[i] <= 0 && ends[i] <= 0); - bool start_negative_end_positive = starts[i] <= 0 && ends[i] >= 0; - bool start_positive_end_negative = starts[i] >= 0 && ends[i] <= 0; - - if (both_negative_or_positive) { - out_shape[axis] = end - dim_expr_starts[i]; - } else if (start_negative_end_positive) { - out_shape[axis] = end - dim_expr_starts[i] - out_shape[axis]; - } else if (start_positive_end_negative) { - out_shape[axis] = out_shape[axis] - dim_expr_starts[i] + end; - } else { - LOG(FATAL) << "Dead code"; - } + // When `pd.slice` is operating on a tensor which is produced by a `pd.shape` + // op, the reseult should be written into data. + const auto &GetDataDimExprs = [&]() -> symbol::ShapeOrDataDimExprs { + std::vector out_data; + + // Currently, we DO NOT support the case that any element in `axes` `starts` + // or `ends` is a Symbol. + auto vec_int64 = details::VecExpr2Int64(starts); + IR_ENFORCE(vec_int64.has_value(), + "for slice op, all the elements in `starts` must be int64_t"); + std::vector starts_int = vec_int64.value(); + + vec_int64 = details::VecExpr2Int64(ends); + IR_ENFORCE(vec_int64.has_value(), + "for slice op, all the elements in `ends` must be int64_t"); + std::vector ends_int = vec_int64.value(); + + const int64_t start = + starts_int[0] < 0 + ? starts_int[0] + operand_shape_or_data.data().value().size() + : starts_int[0]; + const int64_t end = + static_cast(std::numeric_limits::max()) == ends_int[0] + ? operand_shape_or_data.data().value().size() + : ends_int[0]; + + for (int64_t i = start; i < end; i++) { + out_data.push_back(operand_shape_or_data.data().value()[i]); } + const std::vector shape{std::int64_t(out_data.size())}; return symbol::ShapeOrDataDimExprs{ - symbol::TensorShapeOrDataDimExprs(out_shape)}; + symbol::TensorShapeOrDataDimExprs(shape, out_data)}; }; symbol::ShapeOrDataDimExprs shape_data = diff --git a/test/cpp/pir/shape_dialect/shape_optimization_test.cc b/test/cpp/pir/shape_dialect/shape_optimization_test.cc index b48f84db4d1b8..faefec6e7ec41 100644 --- a/test/cpp/pir/shape_dialect/shape_optimization_test.cc +++ b/test/cpp/pir/shape_dialect/shape_optimization_test.cc @@ -122,10 +122,10 @@ TEST(shape_optimization, shape_optimization_pass) { "Mul(Mul(Mul(Mul(1, S1), 128), 32), 1 / (128))"); EXPECT_EQ(cast_res.shape()[3], 2); - EXPECT_EQ(symbol::ToString(relu_res.shape()[0]), "Add(Add(S2, -2), -2)"); - EXPECT_EQ(symbol::ToString(relu_res.shape()[1]), "Add(Add(S3, -2), -2)"); - EXPECT_EQ(symbol::ToString(relu_res.shape()[2]), "Add(Add(S4, -2), -2)"); - EXPECT_EQ(symbol::ToString(relu_res.shape()[3]), "Add(Add(S5, -2), -2)"); + EXPECT_EQ(symbol::ToString(relu_res.shape()[0]), "Add(-2, -Add(2, -S2))"); + EXPECT_EQ(symbol::ToString(relu_res.shape()[1]), "Add(-2, -Add(2, -S3))"); + EXPECT_EQ(symbol::ToString(relu_res.shape()[2]), "Add(-2, -Add(2, -S4))"); + EXPECT_EQ(symbol::ToString(relu_res.shape()[3]), "Add(-2, -Add(2, -S5))"); EXPECT_EQ(subtract_res.shape()[0], 1); EXPECT_EQ(subtract_res.shape()[1], 64); diff --git a/test/ir/pir/cinn/symbolic/test_op_infer_sym_shape.py b/test/ir/pir/cinn/symbolic/test_op_infer_sym_shape.py index 61ca48f19d797..4ab27bf657eac 100644 --- a/test/ir/pir/cinn/symbolic/test_op_infer_sym_shape.py +++ b/test/ir/pir/cinn/symbolic/test_op_infer_sym_shape.py @@ -459,5 +459,63 @@ def test_eval_symbolic(self): return True +class SliceNet(paddle.nn.Layer): + def __init__(self): + super().__init__() + + def forward(self, x): + out = x[:, -1, :] + out = x[1:3, 0:2, 2:4] + + axes = [0, 1, 2] + starts = [-3, 0, 2] + ends = [3, 2, 4] + out = paddle.slice(x, axes=axes, starts=starts, ends=ends) + + return out + + +class TestSliceOpInferSymbolicShape(TestBase): + def prepare_data(self): + self.cases = [np.random.rand(4, 5, 6)] + + self.expected = [ + [ + 'shape[S0, S2], data[NULL]', + 'shape[2, 2, 2], data[NULL]', + 'shape[Add(3, -Add(-3, S0)), 2, 2]', + ] + ] + + def test_eval_symbolic(self): + net = SliceNet() + + for i in range(len(self.cases)): + x = self.cases[i] + x_spec = InputSpec( + shape=[None for index in range(len(x.shape))], dtype='float32' + ) + + input_spec = [x_spec] + net = apply_to_static(net, False, input_spec) + net.eval() + + # check the infer result + sym_shape_str_list = get_sym_shape_str_for_op( + net, input_spec, 'pd_op.slice' + ) + np.testing.assert_equal( + len(sym_shape_str_list), len(self.expected[i]) + ) + for j in range(len(sym_shape_str_list)): + np.testing.assert_equal( + sym_shape_str_list[j].find(self.expected[i][j]), + 0, + f'in case i,j = {i},{j}: output shape ({sym_shape_str_list[0]}) is not expected {(self.expected[i][j])}', + ) + + return True + + if __name__ == '__main__': unittest.main() From f445bd8d31a8dc283d63dc282dc09082bf77a059 Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Sat, 2 Mar 2024 08:48:30 +0800 Subject: [PATCH 233/282] [DRR]Fix SegmentFault for BlockArgument while applying pass in Llama2 infer (#62283) * [DRR]Fix SegmentFault for BlockArgument while applying pass in Llama2 infer * fix typo --- paddle/fluid/pir/drr/src/rewrite_pattern.cc | 137 ++++++++++++-------- 1 file changed, 85 insertions(+), 52 deletions(-) diff --git a/paddle/fluid/pir/drr/src/rewrite_pattern.cc b/paddle/fluid/pir/drr/src/rewrite_pattern.cc index 46b034aca8558..e19d5ae224c7d 100644 --- a/paddle/fluid/pir/drr/src/rewrite_pattern.cc +++ b/paddle/fluid/pir/drr/src/rewrite_pattern.cc @@ -258,95 +258,128 @@ bool DrrRewritePattern::MatchFromOutputToInput( std::unordered_set ir_visited; std::queue drr_q; std::queue ir_q; - bool matched = true; - size_t step = 0; - for (auto it = output_op_map.begin(); it != output_op_map.end(); ++it) { - VLOG(6) << "match (" << it->first->name() << " @" << it->first << " : @" - << it->second << ") in source_pattern_graph "; - drr_q.push(it->first); - drr_visited.insert(it->first); - ir_q.push(it->second); - ir_visited.insert(it->second); - } - while (!drr_q.empty()) { - if (!matched) break; - auto* drr_node = drr_q.front(); - auto* ir_node = ir_q.front(); - drr_q.pop(); - ir_q.pop(); + // Initialize DRR matched queue. + const auto& InitDrrQueue = [&]() -> void { + for (auto it = output_op_map.begin(); it != output_op_map.end(); ++it) { + VLOG(6) << "match (" << it->first->name() << " @" << it->first << " : @" + << it->second << ") in source_pattern_graph "; + drr_q.push(it->first); + drr_visited.insert(it->first); + ir_q.push(it->second); + ir_visited.insert(it->second); + } + }; + // Check whether DrrNode and Operation have the same Operands and Results + // information. + const auto& IsSameOperandsAndResults = + [](const OpCall* drr_node, const pir::Operation* ir_node) -> bool { if (drr_node->name() != ir_node->name()) { - matched = false; VLOG(8) << "Match failed: drr_node(" << drr_node->name() << ") != pir_node(" << ir_node->name() << ")."; - break; + return false; } const auto& drr_input_tensors = drr_node->inputs(); auto ir_input_value_size = ir_node->num_operands(); if (drr_input_tensors.size() != ir_input_value_size) { - matched = false; VLOG(8) << drr_node->name() << " Match failed: drr input tensors(" << drr_input_tensors.size() << ") != pir input tensors(" << ir_input_value_size << ")."; - break; + return false; } if (drr_node->outputs().size() != ir_node->num_results()) { - matched = false; VLOG(8) << drr_node->name() << " Match failed: drr output tensors(" << drr_node->outputs().size() << ") != pir output tensors(" << ir_node->num_results() << ")."; + return false; + } + return true; + }; + // Check whether source_pattern_match_ctx has visited Operation's Operands. + const auto& HasVisitedOperands = [&](const Tensor* drr_input_tensor, + pir::Value ir_value) -> bool { + const auto& tensor_name = drr_input_tensor->name(); + if (ir_value.isa()) { + VLOG(8) << "Match Attention! Found BlockArgument as input of " + << tensor_name; + } + return source_pattern_match_ctx->tensor_map().count(tensor_name) != 0 && + ir_value != source_pattern_match_ctx->tensor_map().at(tensor_name); + }; + // Update drr_q et.al information. Return false if faild. + const auto& TryUpdateDrrQueue = [&](const OpCall* drr_producer_op, + pir::Operation* ir_producer_op) -> bool { + // still return true if both visited. + if (drr_visited.count(drr_producer_op) && + ir_visited.count(ir_producer_op)) { + return true; + } + // insert map if both not visited. + if (!drr_visited.count(drr_producer_op) && + !ir_visited.count(ir_producer_op)) { + drr_q.push(drr_producer_op); + ir_q.push(ir_producer_op); + drr_visited.insert(drr_producer_op); + ir_visited.insert(ir_producer_op); + return true; + } + return false; + }; + + // Step 1: Initialize DRR matched queue. + bool matched = true; + size_t step = 0; + InitDrrQueue(); + + while (!drr_q.empty()) { + if (!matched) break; + auto* drr_node = drr_q.front(); + auto* ir_node = ir_q.front(); + drr_q.pop(); + ir_q.pop(); + if (!IsSameOperandsAndResults(drr_node, ir_node)) { + matched = false; break; } + // Step 1: Bind Operation of current op to match_ctx. source_pattern_match_ctx->BindIrOperation(drr_node, ir_node); - // binding input_tensor of current_op + + // Step 2: Bind input_tensor of current op to match_ctx. + const auto& drr_input_tensors = drr_node->inputs(); + auto ir_input_values = ir_node->operands_source(); for (size_t i = 0; i < drr_input_tensors.size(); ++i) { - if (source_pattern_match_ctx->tensor_map().count( - drr_input_tensors[i]->name()) != 0 && - ir_node->operand(i).source() != - source_pattern_match_ctx->tensor_map().at( - drr_input_tensors[i]->name())) { + if (HasVisitedOperands(drr_input_tensors[i], ir_input_values[i])) { matched = false; VLOG(8) << " tensor_map key[" << drr_input_tensors[i]->name() << "] already exists,but value is different!"; break; - } else { - source_pattern_match_ctx->BindIrValue(drr_input_tensors[i]->name(), - ir_node->operand(i).source()); - } - - if (ir_node->operand_source(i).isa()) { - VLOG(8) << "Match Attention! Found BlockArgument as input of " - << drr_node->name(); } - + source_pattern_match_ctx->BindIrValue(drr_input_tensors[i]->name(), + ir_input_values[i]); + // Skip it while drr_producer_op is nullptr for trigger pattern boundary. auto* drr_producer_op = drr_input_tensors[i]->producer(); if (drr_producer_op == nullptr) { continue; } - + // Check whether tensor and value have the same use_count. if (drr_input_tensors[i]->consumers().size() != - ir_node->operand(i).source().use_count()) { + ir_input_values[i].use_count()) { matched = false; VLOG(8) << drr_node->name() << " Match failed: consumers of drr intput[" << i << "] { " << drr_node->outputs().size() << " } != consumers of pir intput[" << i << "] { " - << ir_node->operand(i).source().use_count() << " }."; + << ir_input_values[i].use_count() << " }."; break; } - auto* ir_producer_op = ir_node->operand_source(i).defining_op(); - // bfs producer_op of current_op - if (drr_visited.count(drr_producer_op) && - ir_visited.count(ir_producer_op)) { - continue; + auto* ir_producer_op = ir_input_values[i].defining_op(); + // Tigger early stop while operand is BlockArgument with + // producer_op==nullptr. + if (drr_producer_op && ir_producer_op == nullptr) { + matched = false; + break; } - - if (!drr_visited.count(drr_producer_op) && - !ir_visited.count(ir_producer_op)) { - drr_q.push(drr_producer_op); - ir_q.push(ir_producer_op); - drr_visited.insert(drr_producer_op); - ir_visited.insert(ir_producer_op); - } else { + // bfs producer_op of current_op + if (!TryUpdateDrrQueue(drr_producer_op, ir_producer_op)) { matched = false; VLOG(8) << "Match failed: status of visiting for" << drr_node->name() << " is different."; From 98f48ba2947739636c18e986f5fadfa8f5041cf5 Mon Sep 17 00:00:00 2001 From: feifei-111 <2364819892@qq.com> Date: Sat, 2 Mar 2024 10:16:32 +0800 Subject: [PATCH 234/282] [SOT] fix bug in llm stable diffusion (#62257) --- .../executor/opcode_executor.py | 19 ++++- .../executor/variables/__init__.py | 2 +- .../executor/variables/callable.py | 6 +- .../instruction_utils/opcode_analysis.py | 74 ++++++++++++------- .../paddle/jit/sot/utils/paddle_api_config.py | 1 - test/sot/test_break_graph.py | 15 ++++ 6 files changed, 82 insertions(+), 35 deletions(-) diff --git a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py index 3dfa9fb1b733b..7f28346922d91 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py +++ b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py @@ -88,6 +88,7 @@ TensorVariable, TupleVariable, UserDefinedFunctionVariable, + UserDefinedGeneratorFunctionVariable, VariableBase, VariableFactory, ) @@ -1318,11 +1319,21 @@ def g(z=x): default_args, closure, ) - self.stack.push( - UserDefinedFunctionVariable( - new_fn, self._graph, DummyTracker(related_list) + # new_fn is created for which is binded with Variables + # so new_fn.__module__ is a ConstantVariable + # can not use VariableFactory.from_value + if inspect.isgeneratorfunction(new_fn): + self.stack.push( + UserDefinedGeneratorFunctionVariable( + new_fn, self._graph, DummyTracker(related_list) + ) + ) + else: + self.stack.push( + UserDefinedFunctionVariable( + new_fn, self._graph, DummyTracker(related_list) + ) ) - ) def GET_ITER(self, instr: Instruction): source_obj = self.stack.pop() diff --git a/python/paddle/jit/sot/opcode_translator/executor/variables/__init__.py b/python/paddle/jit/sot/opcode_translator/executor/variables/__init__.py index 989c23e110abd..3d53d1fce93dc 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/variables/__init__.py +++ b/python/paddle/jit/sot/opcode_translator/executor/variables/__init__.py @@ -44,7 +44,7 @@ PaddleApiVariable, PaddleLayerVariable, UserDefinedFunctionVariable, - UserDefinedGeneratorVariable, + UserDefinedGeneratorFunctionVariable, UserDefinedLayerVariable, ) from .container import ( # noqa: F401 diff --git a/python/paddle/jit/sot/opcode_translator/executor/variables/callable.py b/python/paddle/jit/sot/opcode_translator/executor/variables/callable.py index 0e6ba7ec1e33f..1648ebcf79b4d 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/variables/callable.py +++ b/python/paddle/jit/sot/opcode_translator/executor/variables/callable.py @@ -681,9 +681,9 @@ def main_info(self) -> dict[str, Any]: } -class UserDefinedGeneratorVariable(FunctionVariable): +class UserDefinedGeneratorFunctionVariable(FunctionVariable): """ - UserDefinedGeneratorVariable is a subclass of FunctionVariable used to wrap a user-defined generator. + UserDefinedGeneratorFunctionVariable is a subclass of FunctionVariable used to wrap a user-defined generator. Args: fn (Callable[..., Any]): The user-defined generator to be wrapped. graph(FunctionGraph): The FunctionGraph object that this variable is associated with. @@ -711,7 +711,7 @@ def main_info(self) -> dict[str, Any]: ) def from_value(value: Any, graph: FunctionGraph, tracker: Tracker): if inspect.isgeneratorfunction(value): - return UserDefinedGeneratorVariable(value, graph, tracker) + return UserDefinedGeneratorFunctionVariable(value, graph, tracker) return None diff --git a/python/paddle/jit/sot/opcode_translator/instruction_utils/opcode_analysis.py b/python/paddle/jit/sot/opcode_translator/instruction_utils/opcode_analysis.py index 93722f42c9602..3d7c1cb7d1f46 100644 --- a/python/paddle/jit/sot/opcode_translator/instruction_utils/opcode_analysis.py +++ b/python/paddle/jit/sot/opcode_translator/instruction_utils/opcode_analysis.py @@ -23,21 +23,19 @@ ALL_JUMP, HAS_FREE, HAS_LOCAL, - RETURN, UNCONDITIONAL_JUMP, ) @dataclasses.dataclass -class State: +class NameRecorder: reads: OrderedSet[str] writes: OrderedSet[str] - visited: OrderedSet[int] def __or__(self, other): reads = self.reads | other.reads writes = self.writes | other.writes - return State(reads, writes, OrderedSet()) + return NameRecorder(reads, writes) def is_read_opcode(opname): @@ -90,46 +88,70 @@ def analysis_used_names( Returns: State: The analysis result. """ - root_state = State(OrderedSet(), OrderedSet(), OrderedSet()) - - def fork(state: State, start: int, jump: bool, jump_target: int) -> State: + name_recorder = NameRecorder(OrderedSet(), OrderedSet()) + + # start idx and writes names can decide the analysis result below + # so, just check the pair of (idx, writes), to skip repeat simulation + # (writes can decide if a name should be add to reads) + # one idx can has multi writes for whom is not subset with each other + # if A is subset of B, we just record A, simulate A might add more reads + visited_states = {} + + def check_and_update_visited_states(idx, writes): + writes = set(writes) + + if idx in visited_states: + history = visited_states[idx] + for record in history: + if record.issubset(writes): + return True + elif writes.issubset(record): + history.remove(record) + history.append(writes) + return False + else: + visited_states[idx] = [writes] + + return False + + def fork( + name_recorder: NameRecorder, start: int, jump: bool, jump_target: int + ) -> NameRecorder: new_start = start + 1 if not jump else jump_target - new_state = State( - OrderedSet(state.reads), - OrderedSet(state.writes), - OrderedSet(state.visited), + new_state = NameRecorder( + OrderedSet(name_recorder.reads), + OrderedSet(name_recorder.writes), ) return walk(new_state, new_start) - def walk(state: State, start: int) -> State: + def walk(name_recorder: NameRecorder, start: int) -> NameRecorder: end = len(instructions) if stop_instr_idx is None else stop_instr_idx for i in range(start, end): - if i in state.visited: - return state - state.visited.add(i) + if check_and_update_visited_states(i, name_recorder.writes): + return name_recorder instr = instructions[i] if instr.opname in HAS_LOCAL | HAS_FREE: if is_read_opcode(instr.opname) and instr.argval not in ( - state.writes + name_recorder.writes ): - state.reads.add(instr.argval) + name_recorder.reads.add(instr.argval) elif is_write_opcode(instr.opname): - state.writes.add(instr.argval) + name_recorder.writes.add(instr.argval) elif instr.opname in ALL_JUMP: assert instr.jump_to is not None target_idx = instructions.index(instr.jump_to) # Fork to two branches, jump or not - jump_branch = fork(state, i, True, target_idx) + jump_branch = fork(name_recorder, i, True, target_idx) not_jump_branch = ( - fork(state, i, False, target_idx) + fork(name_recorder, i, False, target_idx) if instr.opname not in UNCONDITIONAL_JUMP - else State(OrderedSet(), OrderedSet(), OrderedSet()) + else NameRecorder(OrderedSet(), OrderedSet()) ) return jump_branch | not_jump_branch - elif instr.opname in RETURN: - return state - return state + elif instr.opname == "RETURN_VALUE": + return name_recorder + return name_recorder - state = walk(root_state, current_instr_idx) - return state.reads, state.writes + name_recorder = walk(name_recorder, current_instr_idx) + return name_recorder.reads, name_recorder.writes diff --git a/python/paddle/jit/sot/utils/paddle_api_config.py b/python/paddle/jit/sot/utils/paddle_api_config.py index 8a5cde9e65716..24b58bda9b83b 100644 --- a/python/paddle/jit/sot/utils/paddle_api_config.py +++ b/python/paddle/jit/sot/utils/paddle_api_config.py @@ -82,7 +82,6 @@ def get_paddle_api(): # considered as paddle module？ paddle_api_module_prefix = { "paddle.nn.functional", - "paddle.nn.layer.activation", } break_graph_set = set() diff --git a/test/sot/test_break_graph.py b/test/sot/test_break_graph.py index b6908f4d229b5..58cab6d48b0a3 100644 --- a/test/sot/test_break_graph.py +++ b/test/sot/test_break_graph.py @@ -185,5 +185,20 @@ def test_break_graph_in_layer(self): self.assert_results(net.forward, x) +def dummy(*args): + return None + + +def break_graph_call_generator_function(x): + return dummy(y for y in x) + + +class TestBreakGraphCallGeneratorFunction(TestCaseBase): + def test_break_graph_when_call_generator_function(self): + x = paddle.rand([1], dtype=paddle.float32) + y = paddle.rand([1], dtype=paddle.float32) + self.assert_results(break_graph_call_generator_function, [x, y]) + + if __name__ == "__main__": unittest.main() From eabf863247fef18d5d7912817c9a1a95d3ddf23f Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Sat, 2 Mar 2024 11:02:44 +0800 Subject: [PATCH 235/282] [Dy2St][PIR] Add view op to inplace info (#62300) --- paddle/fluid/pybind/pir.cc | 5 ++ test/dygraph_to_static/test_deal_inplace.py | 53 +++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 test/dygraph_to_static/test_deal_inplace.py diff --git a/paddle/fluid/pybind/pir.cc b/paddle/fluid/pybind/pir.cc index 45fe7263e692c..d28b274348201 100644 --- a/paddle/fluid/pybind/pir.cc +++ b/paddle/fluid/pybind/pir.cc @@ -1360,7 +1360,12 @@ std::map GetOpInplaceInfo(const pir::Operation *op) { const std::string &inplace_name = yaml_parser.InplaceName(value_name); inplace_info[i] = yaml_parser.InputName2Id().at(inplace_name); } + if (yaml_parser.HasView(value_name)) { + const std::string &view_name = yaml_parser.ViewName(value_name); + inplace_info[i] = yaml_parser.InputName2Id().at(view_name); + } } + return inplace_info; } diff --git a/test/dygraph_to_static/test_deal_inplace.py b/test/dygraph_to_static/test_deal_inplace.py new file mode 100644 index 0000000000000..3984dd729db0a --- /dev/null +++ b/test/dygraph_to_static/test_deal_inplace.py @@ -0,0 +1,53 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np +from dygraph_to_static_utils import ( + Dy2StTestBase, + test_pir_only, +) + +import paddle + + +def fn_with_inplace_op(inplace_op, x): + y = inplace_op(x) + z = inplace_op(x) + return y + z + + +class TestDealInplace(Dy2StTestBase): + def run_test(self, dygraph_fn, *inputs): + dygraph_out = dygraph_fn(*inputs) + static_fn = paddle.jit.to_static(dygraph_fn) + static_out = static_fn(*inputs) + np.testing.assert_allclose(dygraph_out.numpy(), static_out.numpy()) + + @test_pir_only + def test_deal_view(self): + bn_layer = paddle.nn.BatchNorm2D(10) + x = paddle.to_tensor(np.random.random((2, 10, 3, 3)).astype('float32')) + self.run_test(fn_with_inplace_op, bn_layer, x) + + @test_pir_only + def test_deal_inplace(self): + sigmoid_layer = paddle.nn.Sigmoid() + x = paddle.to_tensor(np.random.random((2, 10, 3, 3)).astype('float32')) + self.run_test(fn_with_inplace_op, sigmoid_layer, x) + + +if __name__ == '__main__': + unittest.main() From 6f608ca9d2c84db75e7bff4ce7a9be9a321a1fba Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Sat, 2 Mar 2024 12:31:30 +0800 Subject: [PATCH 236/282] [PT] Set NCHW as default Layout for type translator (#62263) * [PT] Set NCHW as default Layout for type translator * fix randint * fix typo * fix delt --- .../ir_adaptor/translator/op_translator.cc | 2 +- .../ir_adaptor/translator/type_translator.cc | 89 +++++++++---------- 2 files changed, 44 insertions(+), 47 deletions(-) diff --git a/paddle/fluid/ir_adaptor/translator/op_translator.cc b/paddle/fluid/ir_adaptor/translator/op_translator.cc index bf5acda9c1bbd..3466c074ed994 100644 --- a/paddle/fluid/ir_adaptor/translator/op_translator.cc +++ b/paddle/fluid/ir_adaptor/translator/op_translator.cc @@ -2746,7 +2746,7 @@ struct RandIntOpTranscriber : public OpTranscriber { paddle::dialect::DenseTensorTypeStorage::Dim dim = common::make_ddim(var->GetShape()); paddle::dialect::DenseTensorTypeStorage::DataLayout layout = - paddle::dialect::DenseTensorTypeStorage::DataLayout::UNDEFINED; + paddle::dialect::DenseTensorTypeStorage::DataLayout::NCHW; paddle::dialect::DenseTensorTypeStorage::LoD lod = {}; size_t offset = 0; pir::Type translated_var_type = paddle::dialect::DenseTensorType::get( diff --git a/paddle/fluid/ir_adaptor/translator/type_translator.cc b/paddle/fluid/ir_adaptor/translator/type_translator.cc index 7cd297cf46b62..4378ef5285ceb 100644 --- a/paddle/fluid/ir_adaptor/translator/type_translator.cc +++ b/paddle/fluid/ir_adaptor/translator/type_translator.cc @@ -30,8 +30,48 @@ using DenseTensorType = paddle::dialect::DenseTensorType; using DenseTensorTypeStorage = paddle::dialect::DenseTensorTypeStorage; using SelectedRowsType = paddle::dialect::SelectedRowsType; using SelectedRowsTypeStorage = paddle::dialect::SelectedRowsTypeStorage; +using DataLayout = DenseTensorTypeStorage::DataLayout; +using LoD = DenseTensorTypeStorage::LoD; TypeTranslator::TypeTranslator() { + const auto& HandleTensor = [&](pir::IrContext* ctx, + const VarDesc& var_desc) -> pir::Type { + VLOG(10) << "[vartype translating]" + << "[" << var_desc.Name() << "] from LOD_TENSOR"; + const pir::Type dtype = + this->operator[](var_desc.GetDataType())(ctx, var_desc); + const auto dim = common::make_ddim(var_desc.GetShape()); + const auto layout = DataLayout::NCHW; + const LoD lod = {}; + const size_t offset = 0; + return DenseTensorType::get(ctx, dtype, dim, layout, lod, offset); + }; + const auto& HandleTensorArray = [&](pir::IrContext* ctx, + const VarDesc& var_desc) -> pir::Type { + VLOG(10) << "[vartype translating]" + << "[" << var_desc.Name() << "] from LOD_TENSOR_ARRAY"; + const pir::Type dtype = + this->operator[](var_desc.GetDataType())(ctx, var_desc); + const auto dims = common::make_ddim(var_desc.GetShape()); + const auto layout = DataLayout::NCHW; + return paddle::dialect::DenseTensorArrayType::get(ctx, dtype, dims, layout); + }; + + const auto& HandleSelectedRows = [&](pir::IrContext* ctx, + const VarDesc& var_desc) -> pir::Type { + VLOG(10) << "[vartype translating]" + << "[" << var_desc.Name() << "] from SELECTED_ROWS"; + const pir::Type dtype = + this->operator[](var_desc.GetDataType())(ctx, var_desc); + const auto dim = common::make_ddim(var_desc.GetShape()); + const auto layout = DataLayout::NCHW; + const LoD lod = {}; + const size_t offset = 0; + pir::Type SelectedRows = + SelectedRowsType::get(ctx, dtype, dim, layout, lod, offset); + return SelectedRows; + }; + handlers = { {VarType::BOOL, [&](pir::IrContext* ctx, const VarDesc& var_desc) -> pir::Type { @@ -81,52 +121,9 @@ TypeTranslator::TypeTranslator() { [&](pir::IrContext* ctx, const VarDesc& var_desc) -> pir::Type { return pir::Complex128Type::get(ctx); }}, - {VarType::LOD_TENSOR, - [&](pir::IrContext* ctx, const VarDesc& var_desc) -> pir::Type { - VLOG(10) << "[vartype translating]" - << "[" << var_desc.Name() << "] from LOD_TENSOR"; - - pir::Type dtype = - this->operator[](var_desc.GetDataType())(ctx, var_desc); - DenseTensorTypeStorage::Dim dim = - common::make_ddim(var_desc.GetShape()); - DenseTensorTypeStorage::DataLayout layout = - DenseTensorTypeStorage::DataLayout::UNDEFINED; - DenseTensorTypeStorage::LoD lod = {}; - size_t offset = 0; - return DenseTensorType::get(ctx, dtype, dim, layout, lod, offset); - }}, - {VarType::LOD_TENSOR_ARRAY, - [&](pir::IrContext* ctx, const VarDesc& var_desc) -> pir::Type { - VLOG(10) << "[vartype translating]" - << "[" << var_desc.Name() << "] from LOD_TENSOR_ARRAY"; - pir::Type dtype = - this->operator[](var_desc.GetDataType())(ctx, var_desc); - phi::DDim dims = common::make_ddim(var_desc.GetShape()); - DenseTensorTypeStorage::DataLayout layout = - DenseTensorTypeStorage::DataLayout::UNDEFINED; - - return paddle::dialect::DenseTensorArrayType::get( - ctx, dtype, dims, layout); - }}, - {VarType::SELECTED_ROWS, - [&](pir::IrContext* ctx, const VarDesc& var_desc) -> pir::Type { - VLOG(10) << "[vartype translating]" - << "[" << var_desc.Name() << "] from SELECTED_ROWS"; - - pir::Type dtype = - this->operator[](var_desc.GetDataType())(ctx, var_desc); - - SelectedRowsTypeStorage::Dim dim = - common::make_ddim(var_desc.GetShape()); - SelectedRowsTypeStorage::DataLayout layout = - SelectedRowsTypeStorage::DataLayout::UNDEFINED; - SelectedRowsTypeStorage::LoD lod = {}; - size_t offset = 0; - pir::Type SelectedRows = - SelectedRowsType::get(ctx, dtype, dim, layout, lod, offset); - return SelectedRows; - }}, + {VarType::LOD_TENSOR, HandleTensor}, + {VarType::LOD_TENSOR_ARRAY, HandleTensorArray}, + {VarType::SELECTED_ROWS, HandleSelectedRows}, }; } From 94018aecdeddb4169232655631f5b1cc762f8c8f Mon Sep 17 00:00:00 2001 From: hong <43953930+phlrain@users.noreply.github.com> Date: Sat, 2 Mar 2024 12:38:16 +0800 Subject: [PATCH 237/282] [CINN]Fix group op attribuge hash bug (#62309) * fix group op attribute hash bug * fix bug --- paddle/cinn/hlir/dialect/operator/ir/attribute_storage.h | 5 +++++ .../dialect/operator/transforms/cinn_group_cluster_pass.cc | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/paddle/cinn/hlir/dialect/operator/ir/attribute_storage.h b/paddle/cinn/hlir/dialect/operator/ir/attribute_storage.h index 61a2ae3268e05..d338dcd84b04d 100644 --- a/paddle/cinn/hlir/dialect/operator/ir/attribute_storage.h +++ b/paddle/cinn/hlir/dialect/operator/ir/attribute_storage.h @@ -71,6 +71,11 @@ struct GroupInfoAttributeStorage : public pir::AttributeStorage { static std::size_t HashValue(const ParamKey& key) { size_t hash_value = std::hash{}(key.group_id); + for (auto op : key.ops) { + hash_value = + pir::detail::hash_combine(hash_value, std::hash()(op)); + } + for (auto d : key.loop_ranges) { hash_value = pir::detail::hash_combine(hash_value, std::hash()(d)); diff --git a/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_cluster_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_cluster_pass.cc index 9f9856004646f..f0069a55a4cde 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_cluster_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_cluster_pass.cc @@ -252,7 +252,7 @@ cinn::dialect::GroupInfo BuildGroupInfo( const GroupClusterNode& node, const std::unordered_map<::pir::Operation*, std::vector>& new_align_info) { - cinn::dialect::GroupInfo group_info({}); + cinn::dialect::GroupInfo group_info(vec_new_op_list); group_info.group_id = BuildGroupId(vec_new_op_list); group_info.loop_ranges = node.loop_ranges; group_info.reduce_axis = node.reduce_axis; From 8b4219b0b84b42df40ebb439440ce5445d769884 Mon Sep 17 00:00:00 2001 From: Bo Zhang <105368690+zhangbopd@users.noreply.github.com> Date: Sat, 2 Mar 2024 15:10:35 +0800 Subject: [PATCH 238/282] add argmax & argmin (#62312) --- .../infer_symbolic_shape/infer_sym_utils.h | 3 + .../infer_symbolic_shape.h | 1 + .../paddle_op_infer_sym.cc | 13 -- .../paddle_op_infer_sym.h | 5 - .../infer_symbolic_shape/unary_infer_sym.cc | 77 ++++++++++++ .../infer_symbolic_shape/unary_infer_sym.h | 26 ++++ .../pir/transforms/shape_optimization_pass.cc | 4 +- .../symbolic/test_unary_op_infer_sym_shape.py | 112 ++++++++++++++++++ 8 files changed, 220 insertions(+), 21 deletions(-) create mode 100644 paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/unary_infer_sym.cc create mode 100644 paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/unary_infer_sym.h create mode 100644 test/ir/pir/cinn/symbolic/test_unary_op_infer_sym_shape.py diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.h index d2d508ff5890d..f5193b3f7ff5b 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.h @@ -17,6 +17,9 @@ #include "paddle/fluid/pir/dialect/operator/ir/pd_op.h" #include "paddle/pir/include/dialect/shape/utils/shape_analysis.h" +#define GET_BOOL_ATTR(op, str) \ + op->attributes().at(str).dyn_cast().data(); + // To make codes shorter using ExprVec = std::vector; using ShapeOrData = symbol::ShapeOrDataDimExprs; diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_symbolic_shape.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_symbolic_shape.h index 4e1946acd75f1..515eaaca1b348 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_symbolic_shape.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_symbolic_shape.h @@ -18,6 +18,7 @@ #include "paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_element_wise_binary.h" #include "paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h" #include "paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h" +#include "paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/unary_infer_sym.h" #include "paddle/pir/include/dialect/shape/utils/shape_analysis.h" // Type inference is currently modelled executionally for operation creation diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc index 1be26c82f4c21..d7ee4fb6781b0 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc @@ -1174,19 +1174,6 @@ bool SplitOpInferSymbolicShape(pir::Operation *op, // Not Impelmented Ops. -bool ArgmaxOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool ArgminOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} - bool AsComplexOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { PADDLE_THROW(phi::errors::Unimplemented( diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h index cf5e650023fa9..f23e84c27f55d 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h @@ -114,11 +114,6 @@ bool SplitOpInferSymbolicShape(pir::Operation *op, // Not Impelmented Ops. -bool ArgmaxOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool ArgminOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); - bool AsComplexOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool AsRealOpInferSymbolicShape(pir::Operation *op, diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/unary_infer_sym.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/unary_infer_sym.cc new file mode 100644 index 0000000000000..d82fc12521998 --- /dev/null +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/unary_infer_sym.cc @@ -0,0 +1,77 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/unary_infer_sym.h" +#include "paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.h" +// #include "paddle/fluid/pir/dialect/operator/ir/op_attribute.h" + +namespace paddle::dialect { + +bool ArgmaxOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + bool flatten = GET_BOOL_ATTR(op, "flatten"); + bool keepdims = GET_BOOL_ATTR(op, "keepdims"); + + const auto &input_shape_or_data = + shape_analysis->GetShapeOrDataForValue(op->operand_source(0)); + + const auto &axis_shape_or_data = + shape_analysis->GetShapeOrDataForValue(op->operand_source(1)); + int axis = + static_cast(axis_shape_or_data.data().value()[0].Get()); + + const std::vector &input_sym_shape = + input_shape_or_data.data().has_value() + ? input_shape_or_data.data().value() + : input_shape_or_data.shape(); + + int rank = input_sym_shape.size(); + if (axis < 0) axis += rank; + + const auto &out_sym_shape = [&] { + std::vector out_sym_shape; + if (flatten) { + if (keepdims) { + out_sym_shape.emplace_back(std::int64_t(rank)); + } else { + out_sym_shape.emplace_back(std::int64_t(0)); + } + } else { + for (int i = 0; i < axis; i++) { + out_sym_shape.emplace_back(input_sym_shape[i]); + } + if (keepdims) { + out_sym_shape.emplace_back(std::int64_t(1)); + } + + for (int i = axis + 1; i < rank; i++) { + out_sym_shape.emplace_back(input_sym_shape[i]); + } + } + return out_sym_shape; + }(); + + symbol::ShapeOrDataDimExprs shape_data{ + symbol::TensorShapeOrDataDimExprs(out_sym_shape)}; + + shape_analysis->SetShapeOrDataForValue(op->result(0), shape_data); + return true; +} + +bool ArgminOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return ArgmaxOpInferSymbolicShape(op, shape_analysis); +} + +} // namespace paddle::dialect diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/unary_infer_sym.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/unary_infer_sym.h new file mode 100644 index 0000000000000..832a6a7a074c3 --- /dev/null +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/unary_infer_sym.h @@ -0,0 +1,26 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/pir/include/dialect/shape/utils/shape_analysis.h" + +namespace paddle::dialect { + +bool ArgmaxOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool ArgminOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); + +} // namespace paddle::dialect diff --git a/paddle/fluid/pir/transforms/shape_optimization_pass.cc b/paddle/fluid/pir/transforms/shape_optimization_pass.cc index d9cf96f78efe9..85f4a5a5eef49 100644 --- a/paddle/fluid/pir/transforms/shape_optimization_pass.cc +++ b/paddle/fluid/pir/transforms/shape_optimization_pass.cc @@ -23,7 +23,7 @@ COMMON_DECLARE_bool(pir_apply_shape_optimization_pass); -const int vlog_level = 3; +constexpr int vlog_level = 3; namespace pir { namespace { @@ -144,8 +144,6 @@ void InferSymExprForBlock(const Block& block, &op, shape_analysis->GetShapeOrDataForValue(op.result(0))); } } else { - VLOG(vlog_level) << op.name() + - " DOES NOT have InferSymbolicShapeInterface!"; PADDLE_THROW(phi::errors::Unimplemented( op.name() + " DOES NOT have InferSymbolicShapeInterface!")); } diff --git a/test/ir/pir/cinn/symbolic/test_unary_op_infer_sym_shape.py b/test/ir/pir/cinn/symbolic/test_unary_op_infer_sym_shape.py new file mode 100644 index 0000000000000..5260475b45f1e --- /dev/null +++ b/test/ir/pir/cinn/symbolic/test_unary_op_infer_sym_shape.py @@ -0,0 +1,112 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np + +import paddle +from paddle.static import InputSpec + + +def get_sym_shape_str_for_op(net, input_spec, op_name='builtin.shadow_output'): + forward_program = net.forward.get_concrete_program(*input_spec)[ + 1 + ].infer_program.forward_program + all_sym_shape_str = [] + for op in forward_program.global_block().ops: + if op.name() == op_name: + all_sym_shape_str.append(op.attrs()['sym_shape_str']) + + return all_sym_shape_str + + +def apply_to_static(net, use_cinn, input_spec=None): + build_strategy = paddle.static.BuildStrategy() + build_strategy.build_cinn_pass = use_cinn + return paddle.jit.to_static( + net, + input_spec=input_spec, + build_strategy=build_strategy, + full_graph=True, + ) + + +class TestBase(unittest.TestCase): + def setUp(self): + paddle.seed(2022) + self.prepare_data() + + def prepare_data(self): + pass + + def test_eval_symbolic(self): + pass + + +class ArgMaxMinNet(paddle.nn.Layer): + def __init__(self): + super().__init__() + + def forward(self, x): + argmax_out = paddle.argmax(x) + argmin_out = paddle.argmin(x, axis=-1) + return argmax_out, argmin_out + + +class TestArgMaxMinOpInferSymbolicShape(TestBase): + def prepare_data(self): + self.cases = [np.random.rand(4, 5, 6)] + self.expected = [ + [ + 'shape[0], data[NULL]', + 'shape[S0, S1], data[NULL]', + ] + ] + + def test_eval_symbolic(self): + net = ArgMaxMinNet() + + for i in range(len(self.cases)): + x = self.cases[i] + x_spec = InputSpec( + shape=[None for index in range(len(x.shape))], dtype='float32' + ) + + input_spec = [x_spec] + net = apply_to_static(net, False, input_spec) + net.eval() + + # check the infer result + sym_shape_str_list = get_sym_shape_str_for_op( + net, input_spec, 'pd_op.argmax' + ) + sym_shape_str_list += get_sym_shape_str_for_op( + net, input_spec, 'pd_op.argmin' + ) + np.testing.assert_equal( + len(sym_shape_str_list), len(self.expected[i]) + ) + for j in range(len(sym_shape_str_list)): + np.testing.assert_equal( + sym_shape_str_list[j].find(self.expected[i][j]), + 0, + f'in case i,j = {i},{j}: output shape ({sym_shape_str_list[0]}) is not expected {(self.expected[i][j])}', + ) + + return True + + +if __name__ == '__main__': + unittest.main() From 6fccb8f20c283abcbf28d0ed7e82be9c83e7ce45 Mon Sep 17 00:00:00 2001 From: xiongkun Date: Sat, 2 Mar 2024 17:09:09 +0800 Subject: [PATCH 239/282] [CINN] uniform all the 0 and reduce deleted axis (#61608) * uniform all the 0 and reduce deleted axis * remove one shape for keepdim cases. * fix by code review * fix some error in 0d format --- paddle/cinn/ast_gen_ius/ast_gen.cc | 86 +++++++++++++++++++++++++----- paddle/cinn/hlir/pe/reduction.cc | 8 +++ paddle/cinn/ir/ir.cc | 5 +- paddle/cinn/ir/ir.h | 15 ++++-- paddle/cinn/lang/compute.cc | 7 +++ paddle/cinn/pybind/ir/ir_api.cc | 1 + paddle/cinn/runtime/flags.cc | 4 ++ 7 files changed, 107 insertions(+), 19 deletions(-) diff --git a/paddle/cinn/ast_gen_ius/ast_gen.cc b/paddle/cinn/ast_gen_ius/ast_gen.cc index 009158d3f9cce..57b10fb7ca884 100644 --- a/paddle/cinn/ast_gen_ius/ast_gen.cc +++ b/paddle/cinn/ast_gen_ius/ast_gen.cc @@ -22,6 +22,7 @@ #include "paddle/cinn/optim/replace_var_with_expr.h" PD_DECLARE_bool(cinn_new_group_scheduler); +PD_DECLARE_bool(group_schedule_tiling_first); PD_DECLARE_bool(cinn_bucket_compile); namespace cinn { @@ -93,9 +94,21 @@ ir::Expr AstGen::Build(const ir::Tensor& tensor, TensorGroup* tensor_group) { std::vector iter_values; // reduce body and reduce init schedule block should have different objects // for same axis so we re-create objects + VLOG(4) << "FLAGS_group_schedule_tiling_first = " + << FLAGS_group_schedule_tiling_first; std::vector axis_vars = cinn::common::GenDefaultAxis(axis_len); + const std::vector& reduce_axis = tensor->reduce_axis; + VLOG(4) << "ast gen: tensor init_body is " << init_body; for (int i = 0; i < shape.size(); ++i) { - if (FLAGS_cinn_new_group_scheduler && shape[i] == Expr(1)) { + bool is_keep_dim = axis[i]->is_keepdim; + if (FLAGS_group_schedule_tiling_first && is_keep_dim) { + // if tiling first, we need to replace the reduce axis with 0, but don't + // deal with the non-reduce axis + optim::ReplaceVarWithExpr(&init_body, axis[i], Expr(0)); + continue; + } + if (!FLAGS_group_schedule_tiling_first && + FLAGS_cinn_new_group_scheduler && shape[i] == Expr(1)) { optim::ReplaceVarWithExpr(&init_body, axis[i], Expr(0)); continue; } @@ -105,21 +118,25 @@ ir::Expr AstGen::Build(const ir::Tensor& tensor, TensorGroup* tensor_group) { /*is_reduce = */ false)); optim::ReplaceVarWithExpr(&init_body, axis[i], block_vars.back()); axis_vars[i]->is_reduce_axis = false; - if (shape[i] == Expr(1)) { + if (!FLAGS_group_schedule_tiling_first && shape[i] == Expr(1)) { iter_values.push_back(Expr(0)); } else { iter_values.push_back(axis_vars[i]); } } + VLOG(4) << "iter_value.size() and block_vars.size() is " + << iter_values.size() << " " << block_vars.size(); init_body = ir::ScheduleBlockRealize::Make( iter_values, ir::ScheduleBlock::Make( block_vars, {}, {}, reduce_init_name, init_body)); // For the remaining reduce axis, make reduce body - const std::vector& reduce_axis = tensor->reduce_axis; ir::Expr reduce_body = ConvertReduceBody(tensor->body(), tensor, axis_exprs); + + VLOG(4) << "ast gen: reduce body is " << reduce_body; + // create schedule block itervars, i0,i1... std::vector reduce_block_vars; std::vector reduce_iter_values; @@ -127,7 +144,15 @@ ir::Expr AstGen::Build(const ir::Tensor& tensor, TensorGroup* tensor_group) { // for same axis so we re-create objects std::vector reduce_axis_vars = cinn::common::GenDefaultAxis(axis_len); for (int i = 0; i < shape.size(); ++i) { - if (FLAGS_cinn_new_group_scheduler && shape[i] == Expr(1)) { + bool is_keep_dim = axis[i]->is_keepdim; + if (FLAGS_group_schedule_tiling_first && is_keep_dim) { + // if tiling first, we need to replace the reduce axis with 0, but don't + // deal with the non-reduce axis + optim::ReplaceVarWithExpr(&reduce_body, axis[i], Expr(0)); + continue; + } + if (!FLAGS_group_schedule_tiling_first && + FLAGS_cinn_new_group_scheduler && shape[i] == Expr(1)) { optim::ReplaceVarWithExpr(&reduce_body, axis[i], Expr(0)); continue; } @@ -136,12 +161,13 @@ ir::Expr AstGen::Build(const ir::Tensor& tensor, TensorGroup* tensor_group) { cinn::UniqName("i" + std::to_string(i)), /*is_reduce = */ false)); reduce_axis_vars[i]->is_reduce_axis = false; - if (shape[i] == Expr(1)) { + if (!FLAGS_group_schedule_tiling_first && shape[i] == Expr(1)) { reduce_iter_values.push_back(Expr(0)); } else { reduce_iter_values.push_back(axis_vars[i]); } } + VLOG(4) << "ast gen: reduce body is after replace 0" << reduce_body; for (int i = 0; i < reduce_axis.size(); ++i) { int count = shape.size() + i; reduce_block_vars.push_back( @@ -155,14 +181,43 @@ ir::Expr AstGen::Build(const ir::Tensor& tensor, TensorGroup* tensor_group) { } int non_zero_axis_size = 0; - for (int i = 0; i < axis.size(); ++i) { - if (FLAGS_cinn_new_group_scheduler && shape[i] == Expr(1)) { - continue; + if (FLAGS_group_schedule_tiling_first) { + std::vector non_reduce_axis_vars = [&]() { + std::vector res; + for (int i = 0; i < shape.size(); ++i) { + bool is_keep_dim = axis[i]->is_keepdim; + if (!is_keep_dim) { + res.push_back(axis[i]); + } + } + return res; + }(); + for (int i = 0; i < non_reduce_axis_vars.size(); ++i) { + optim::ReplaceVarWithExpr( + &reduce_body, non_reduce_axis_vars[i], reduce_block_vars[i]); + ++non_zero_axis_size; } - optim::ReplaceVarWithExpr( - &reduce_body, axis[i], reduce_block_vars[non_zero_axis_size]); - ++non_zero_axis_size; + } else { + for (int i = 0; i < axis.size(); ++i) { + if (!FLAGS_group_schedule_tiling_first && + FLAGS_cinn_new_group_scheduler && shape[i] == Expr(1)) { + continue; + } + optim::ReplaceVarWithExpr( + &reduce_body, axis[i], reduce_block_vars[non_zero_axis_size]); + ++non_zero_axis_size; + } + } + + VLOG(4) << "to replace : " << non_zero_axis_size << " " + << reduce_block_vars.size(); + for (auto i = 0; i < reduce_block_vars.size(); i++) { + VLOG(4) << "reduce_block_vars[" << i << "] = " << reduce_block_vars[i]; + } + for (auto i = 0; i < reduce_axis.size(); i++) { + VLOG(4) << "reduce_axis[" << i << "] = " << reduce_axis[i]; } + VLOG(4) << "before replace body: " << reduce_body; for (int i = non_zero_axis_size; i < reduce_block_vars.size(); ++i) { optim::ReplaceVarWithExpr(&reduce_body, reduce_axis[i - non_zero_axis_size], @@ -185,7 +240,12 @@ ir::Expr AstGen::Build(const ir::Tensor& tensor, TensorGroup* tensor_group) { // Put the two parts together ir::Expr body = ir::Block::Make({init_body, reduce_body}); for (int i = static_cast(axis_len) - 1; i >= 0; --i) { - if (!FLAGS_cinn_bucket_compile && shape[i] == Expr(1)) { + bool is_keep_dim = axis[i]->is_keepdim; + if (FLAGS_group_schedule_tiling_first && is_keep_dim) { + continue; + } + if (!FLAGS_group_schedule_tiling_first && !FLAGS_cinn_bucket_compile && + shape[i] == Expr(1)) { continue; } ir::Var loop_var = axis[i]; @@ -210,7 +270,7 @@ ir::Expr AstGen::Build(const ir::Tensor& tensor, TensorGroup* tensor_group) { Expr(0), shape[i], cinn::UniqName("i" + std::to_string(i)), false)); optim::ReplaceVarWithExpr(&body, axis[i], block_vars[i]); axis_vars[i]->is_reduce_axis = false; - if (shape[i] == Expr(1)) { + if (!FLAGS_group_schedule_tiling_first && shape[i] == Expr(1)) { iter_values.push_back(Expr(0)); } else { iter_values.push_back(axis_vars[i]); diff --git a/paddle/cinn/hlir/pe/reduction.cc b/paddle/cinn/hlir/pe/reduction.cc index 7e33a1475e48b..605a1b3d6443f 100644 --- a/paddle/cinn/hlir/pe/reduction.cc +++ b/paddle/cinn/hlir/pe/reduction.cc @@ -166,6 +166,14 @@ Tensor DoReduce(const Tensor& tensor, int indice_cnt = 0; int reduce_cnt = 0; + // Set keepdim flags of indices. + if (tensor->shape.size() == indices.size()) { + for (const auto& i : real_axes) { + VLOG(4) << "Set is_keepdim = true for var(" << i << ")"; + indices[i].as_var_ref()->is_keepdim = true; + } + } + for (size_t i = 0; i < tensor->shape.size(); ++i) { bool squeeze_i = std::find(squeeze_axes.begin(), squeeze_axes.end(), i) != squeeze_axes.end(); diff --git a/paddle/cinn/ir/ir.cc b/paddle/cinn/ir/ir.cc index 2e194200d1993..f3c64790551ca 100644 --- a/paddle/cinn/ir/ir.cc +++ b/paddle/cinn/ir/ir.cc @@ -218,11 +218,13 @@ Expr _Var_::Make(Expr lower_bound, Expr upper_bound, const std::string &name, bool is_reduce_axis, - bool is_symbolic_constant) { + bool is_symbolic_constant, + bool is_keepdim) { auto *n = make_shared<_Var_>(); n->lower_bound = lower_bound; n->upper_bound = upper_bound; n->is_reduce_axis = is_reduce_axis; + n->is_keepdim = is_keepdim; n->is_symbolic_constant = is_symbolic_constant; n->name = name; n->set_type(lower_bound.type()); @@ -233,6 +235,7 @@ Expr _Var_::Copy() const { auto *n = make_shared<_Var_>(); n->name = name; n->is_reduce_axis = is_reduce_axis; + n->is_keepdim = is_keepdim; n->lower_bound = lower_bound; n->upper_bound = upper_bound; n->set_type(type()); diff --git a/paddle/cinn/ir/ir.h b/paddle/cinn/ir/ir.h index c02517f9836fc..5a1f9f6a1f739 100644 --- a/paddle/cinn/ir/ir.h +++ b/paddle/cinn/ir/ir.h @@ -381,6 +381,7 @@ struct _Var_ : public ExprNode<_Var_> { std::string name; bool is_reduce_axis{false}; + bool is_keepdim{false}; bool is_symbolic_constant{false}; //! Lower bound and upper bound of a axis. // @{ @@ -401,7 +402,8 @@ struct _Var_ : public ExprNode<_Var_> { Expr upper_bound, const std::string& name, bool is_reduce, - bool is_symbolic_constant = false); + bool is_symbolic_constant = false, + bool is_keepdim = false); void Verify() const override; @@ -419,12 +421,14 @@ struct Var : public IrNodeRef { Var(Expr lower_bound, Expr upper_bound, const std::string& name, - bool is_reduce = false) - : Var(_Var_::Make(lower_bound, upper_bound, name, is_reduce)) {} + bool is_reduce = false, + bool is_keepdim = false) + : Var(_Var_::Make( + lower_bound, upper_bound, name, is_reduce, false, is_keepdim)) {} Var(int upper_bound, const std::string& name) - : Var(_Var_::Make(Expr(0), Expr(upper_bound), name, false)) {} + : Var(_Var_::Make(Expr(0), Expr(upper_bound), name, false, false)) {} Var(Expr upper_bound, const std::string& name) - : Var(_Var_::Make(Expr(0), upper_bound, name, false)) {} + : Var(_Var_::Make(Expr(0), upper_bound, name, false, false)) {} operator Expr() { return Expr(get()); } operator Expr() const { @@ -977,6 +981,7 @@ struct ScheduleBlock : public ExprNode { std::map attrs; std::string name; Expr body; + int32_t reduce_type{-1}; // 0 for warp reduce, 1 for block reduce static Expr Make(const std::vector& iter_vars, const std::vector& read_buffers, diff --git a/paddle/cinn/lang/compute.cc b/paddle/cinn/lang/compute.cc index 4828eaac64e13..bd195fd26a639 100644 --- a/paddle/cinn/lang/compute.cc +++ b/paddle/cinn/lang/compute.cc @@ -187,6 +187,13 @@ ir::Tensor Compute(const std::vector &domain, domain_without_reduce_axis, op, reduce_axis); + const auto set_keep_dim_for_tensor = [&]() { + for (int i = 0; i < _axis.size(); ++i) { + const auto &axis_var = _axis.at(i); + tensor->axis_[i]->is_keepdim = axis_var.as_var_ref()->is_keepdim; + } + }; + set_keep_dim_for_tensor(); return tensor; } diff --git a/paddle/cinn/pybind/ir/ir_api.cc b/paddle/cinn/pybind/ir/ir_api.cc index 56dff498dd710..efebf1206a867 100644 --- a/paddle/cinn/pybind/ir/ir_api.cc +++ b/paddle/cinn/pybind/ir/ir_api.cc @@ -383,6 +383,7 @@ void BindIrIr(py::module *m) { ir::Expr, const std::string &, bool, + bool, bool>(&ir::_Var_::Make)) .def("copy", &ir::_Var_::Copy); diff --git a/paddle/cinn/runtime/flags.cc b/paddle/cinn/runtime/flags.cc index 89512913e8fa9..c9f0760d43e80 100644 --- a/paddle/cinn/runtime/flags.cc +++ b/paddle/cinn/runtime/flags.cc @@ -69,6 +69,10 @@ PD_DEFINE_bool(cinn_bucket_compile, BoolFromEnv("FLAGS_cinn_bucket_compile", false), "Whether to enable bucket compile for dynamic shape."); +PD_DEFINE_bool(group_schedule_tiling_first, + BoolFromEnv("FLAGS_group_schedule_tiling_first", false), + "Whether to enable new group scheduler tiling first strategy."); + PD_DEFINE_bool(cinn_use_common_subexpression_elimination, BoolFromEnv("FLAGS_cinn_use_common_subexpression_elimination", false), From 87bbe044546820c9cceba15dd0cb13a8b8b40bbe Mon Sep 17 00:00:00 2001 From: Tian <121000916+SylarTiaNII@users.noreply.github.com> Date: Sat, 2 Mar 2024 18:06:26 +0800 Subject: [PATCH 240/282] [Distributed] modify comm data type in eager comm connection (#62306) --- python/paddle/distributed/collective.py | 4 +++- python/paddle/distributed/parallel.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/python/paddle/distributed/collective.py b/python/paddle/distributed/collective.py index ead61419af4d6..f988ccc4a052b 100644 --- a/python/paddle/distributed/collective.py +++ b/python/paddle/distributed/collective.py @@ -245,7 +245,9 @@ def new_group(ranks=None, backend=None, timeout=_default_timeout): if int(os.getenv("FLAGS_eager_communication_connection", 0)) == 1: paddle.distributed.all_reduce( - paddle.zeros([1], dtype=paddle.uint8), group=group, sync_op=True + paddle.zeros([1], dtype=paddle.float32), + group=group, + sync_op=True, ) return group diff --git a/python/paddle/distributed/parallel.py b/python/paddle/distributed/parallel.py index 483407695e42d..816af6f91530d 100644 --- a/python/paddle/distributed/parallel.py +++ b/python/paddle/distributed/parallel.py @@ -1122,7 +1122,9 @@ def init_parallel_env(): if int(os.getenv("FLAGS_eager_communication_connection", 0)) == 1: paddle.distributed.all_reduce( - paddle.zeros([1], dtype=paddle.uint8), group=group, sync_op=True + paddle.zeros([1], dtype=paddle.float32), + group=group, + sync_op=True, ) return group From 121c0f64925d908cfff01eb60dd0b624a2b96752 Mon Sep 17 00:00:00 2001 From: Tian <121000916+SylarTiaNII@users.noreply.github.com> Date: Sat, 2 Mar 2024 18:10:07 +0800 Subject: [PATCH 241/282] [Distributed] fix sharding tensor fusion on npu (#62305) --- .../distributed/fleet/utils/tensor_fusion_helper.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python/paddle/distributed/fleet/utils/tensor_fusion_helper.py b/python/paddle/distributed/fleet/utils/tensor_fusion_helper.py index 0ea2d12b292a9..4be5a5d2d27ee 100644 --- a/python/paddle/distributed/fleet/utils/tensor_fusion_helper.py +++ b/python/paddle/distributed/fleet/utils/tensor_fusion_helper.py @@ -54,11 +54,12 @@ def get_current_device_type(): device_type = "gpu" elif paddle.is_compiled_with_xpu(): device_type = "xpu" - elif paddle.is_compiled_with_custom_device(): - current_device = _current_expected_place_() - device_type = current_device.get_device_type() else: - device_type = "unknown" + current_device = _current_expected_place_() + try: + device_type = current_device.get_device_type() + except: + device_type = "unknown" assert ( device_type in alignment.keys() ), f"tensor fusion helper now only support {alignment.keys()}, but got device {device_type} instead." From 16031cb95844479fa0c49ff87f51c8c1fa3d7ec7 Mon Sep 17 00:00:00 2001 From: hong <43953930+phlrain@users.noreply.github.com> Date: Sat, 2 Mar 2024 22:57:36 +0800 Subject: [PATCH 242/282] optimize dynamic reshape pass (#62318) --- .../transforms/dynamic_reshape_pass.cc | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/paddle/cinn/hlir/dialect/operator/transforms/dynamic_reshape_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/dynamic_reshape_pass.cc index 60c9edca4fb3c..d873ceb3c5ac7 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/dynamic_reshape_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/dynamic_reshape_pass.cc @@ -28,14 +28,26 @@ bool ReplaceOpWithReshapeOp(pir::Operation* op, pir::ShapeConstraintIRAnalysis* shape_analysis, pir::PatternRewriter& rewriter) { // NOLINT pir::Value output = op->result(0); - // The value of shape attribute is fake, we only use the output shape info - // in shape analysis. - std::vector shape( - output.type().dyn_cast().dims().size(), 1); - shape[0] = -1; - - auto cinn_reshape = - rewriter.Build(op->operand_source(0), shape); + // Try to Get more detail output info + const auto& GetOupputShape = [&]() -> std::vector { + std::vector shape = phi::vectorize( + output.type().dyn_cast().dims()); + + if (shape_analysis->HasShapeOrDataForValue(op->result(0))) { + auto shape_info = + shape_analysis->GetShapeOrDataForValue(op->result(0)).shape(); + + for (size_t i = 0; i < shape_info.size(); ++i) { + if (shape_info[i].isa()) { + shape[i] = shape_info[i].Get(); + } + } + } + return shape; + }; + + auto cinn_reshape = rewriter.Build( + op->operand_source(0), GetOupputShape()); shape_analysis->SetShapeOrDataForValue( cinn_reshape.result(0), shape_analysis->GetShapeOrDataForValue(output)); From 62ce0947424d90f4705ce6a2b30562ef79b8aba9 Mon Sep 17 00:00:00 2001 From: hong <43953930+phlrain@users.noreply.github.com> Date: Sun, 3 Mar 2024 10:35:01 +0800 Subject: [PATCH 243/282] [CINN]Add remove unchanged pd reshape pass (#62316) * add remove unchanged pd reshape pass * support dyshape * fix bug --- .../remove_unchanged_reshape_pass.cc | 72 ++++++++++++------- 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/paddle/cinn/hlir/dialect/operator/transforms/remove_unchanged_reshape_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/remove_unchanged_reshape_pass.cc index 1f885ef0185e0..a65ed952383b7 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/remove_unchanged_reshape_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/remove_unchanged_reshape_pass.cc @@ -33,29 +33,50 @@ namespace cinn { namespace dialect { namespace ir { -class RemoveUnchangedReshapePattern - : public pir::OpRewritePattern { - public: - using pir::OpRewritePattern::OpRewritePattern; - - bool MatchAndRewrite(cinn::dialect::ReshapeOp op, - pir::PatternRewriter &rewriter) const override { - auto in_dim = op->operand_source(0) - .type() - .dyn_cast() - .dims(); - auto out_dim = op->result(0) - .type() - .dyn_cast() - .dims(); - - if (in_dim == out_dim) { - rewriter.ReplaceAllUsesWith(op->result(0), op->operand_source(0)); - rewriter.EraseOp(op); - return true; +bool RemoveOp(pir::Operation* op, pir::PatternRewriter* rewriter) { + const auto& IsSameShape = [&]() -> bool { + if (op->operand_source(0) + .type() + .dyn_cast() + .IsDynamicShape() || + op->result(0) + .type() + .dyn_cast() + .IsDynamicShape()) { + pir::ShapeConstraintIRAnalysis& shape_analysis = + pir::ShapeAnalysisManager::Instance().Get(op->GetParentProgram()); + + return shape_analysis.GetShapeOrDataForValue(op->operand_source(0)) + .shape() == + shape_analysis.GetShapeOrDataForValue(op->result(0)).shape(); } - return false; + return (op->operand_source(0) + .type() + .dyn_cast() + .dims()) == (op->result(0) + .type() + .dyn_cast() + .dims()); + }; + + if (IsSameShape()) { + rewriter->ReplaceAllUsesWith(op->result(0), op->operand_source(0)); + rewriter->EraseOp(op); + return true; + } + + return false; +} + +template +class RemoveUnchangedReshapePattern : public pir::OpRewritePattern { + public: + using pir::OpRewritePattern::OpRewritePattern; + + bool MatchAndRewrite(OPTYPE op, + pir::PatternRewriter& rewriter) const override { + return RemoveOp(op, &rewriter); } }; @@ -65,7 +86,7 @@ class MergeReshapePattern using pir::OpRewritePattern::OpRewritePattern; bool MatchAndRewrite(cinn::dialect::ReshapeOp op, - pir::PatternRewriter &rewriter) const override { + pir::PatternRewriter& rewriter) const override { if (auto pre_shape = op->operand_source(0) .defining_op() ->dyn_cast()) { @@ -83,17 +104,18 @@ class RemoveUnchangedReshapePass : public pir::PatternRewritePass { RemoveUnchangedReshapePass() : pir::PatternRewritePass("remove_unchanged_reshape_pass", 1) {} - pir::RewritePatternSet InitializePatterns(pir::IrContext *context) override { + pir::RewritePatternSet InitializePatterns(pir::IrContext* context) override { pir::RewritePatternSet ps(context); // remove out_shape equal in_shape reshape op - ps.Add(context); + ps.Add>(context); + ps.Add>(context); ps.Add(context); return ps; } - bool CanApplyOn(pir::Operation *op) const override { + bool CanApplyOn(pir::Operation* op) const override { return op->num_regions() > 0; } }; From 4ffb7da786cef844deb3cf8ad7f95d56000bd010 Mon Sep 17 00:00:00 2001 From: gouzil <66515297+gouzil@users.noreply.github.com> Date: Sun, 3 Mar 2024 22:12:59 +0800 Subject: [PATCH 244/282] [Cleanup] clean F403 for `python/paddle/distributed/passes/__init__.py` (#62332) --- python/paddle/distributed/passes/__init__.py | 131 ++++++++++++++++--- 1 file changed, 112 insertions(+), 19 deletions(-) diff --git a/python/paddle/distributed/passes/__init__.py b/python/paddle/distributed/passes/__init__.py index e78cc5bbd0081..ad540fbdda043 100644 --- a/python/paddle/distributed/passes/__init__.py +++ b/python/paddle/distributed/passes/__init__.py @@ -14,25 +14,118 @@ from .pass_base import new_pass, PassManager, PassContext -from .auto_parallel_gradient_merge import * # noqa: F403 -from .auto_parallel_sharding import * # noqa: F403 -from .auto_parallel_amp import * # noqa: F403 -from .auto_parallel_master_grad import * # noqa: F403 -from .auto_parallel_fp16 import * # noqa: F403 -from .auto_parallel_recompute import * # noqa: F403 -from .auto_parallel_quantization import * # noqa: F403 -from .auto_parallel_data_parallel_optimization import * # noqa: F403 -from .auto_parallel_grad_clip import * # noqa: F403 -from .auto_parallel_fused_linear_promotion import * # noqa: F403 -from .auto_parallel_supplement_explicit_dependencies import * # noqa: F403 -from .auto_parallel_pipeline import * # noqa: F403 -from .auto_parallel_sequence_parallel_optimization import * # noqa: F403 -from .allreduce_matmul_grad_overlapping import * # noqa: F403 -from .cpp_pass import * # noqa: F403 -from .fuse_all_reduce import * # noqa: F403 -from .pipeline_scheduler_pass import * # noqa: F403 -from .ps_trainer_pass import * # noqa: F403 -from .ps_server_pass import * # noqa: F403 +from .auto_parallel_gradient_merge import ( # noqa: F401 + parse_program, + GradientMergePass, +) +from .auto_parallel_sharding import ( # noqa: F401 + ShardingPass, + is_sharding_param_broadcast_op, + partition_by_use_order, + partition_by_greedy_even, + partition_parameters, + re_order_program, + group_param, + ShardingInfo, + VarGroup, +) +from .auto_parallel_amp import ( # noqa: F401 + AMPLists, + AMPState, + AMPPass, +) +from .auto_parallel_master_grad import ( # noqa: F401 + get_output_in_varlist, + MasterGradPass, +) +from .auto_parallel_fp16 import ( # noqa: F401 + set_op_dtype_to_fp16, + set_auto_cast_attr, + FP16State, + cast_startup_program, + FP16Pass, +) +from .auto_parallel_recompute import ( # noqa: F401 + RecomputeState, + RecomputePass, +) +from .auto_parallel_quantization import QuantizationPass # noqa: F401 +from .auto_parallel_data_parallel_optimization import ( # noqa: F401 + DataParallelOptimizationPass, + GradientsGroup, +) +from .auto_parallel_grad_clip import ( # noqa: F401 + ClipHelper, + ClipGradByGlobalNormPass, +) +from .auto_parallel_fused_linear_promotion import ( # noqa: F401 + FusedLinearPromotionPass, +) +from .auto_parallel_supplement_explicit_dependencies import ( # noqa: F401 + AutoParalSupplementDepPass, +) +from .auto_parallel_pipeline import is_reshard_op, PipelinePass # noqa: F401 +from .auto_parallel_sequence_parallel_optimization import ( # noqa: F401 + SequenceParallelOptimizationPass, +) +from .allreduce_matmul_grad_overlapping import ( # noqa: F401 + AllreduceMatmulGradOverlappingPass, +) +from .cpp_pass import ( # noqa: F401 + FuseElementwiseAddActPass, + FuseBatchNormActPass, + FuseBatchNormAddActPass, + FuseReluDepthwiseConvPass, + FusedAttentionPass, + FusedFeedforwardPass, + FuseGemmEpiloguePass, + FuseAdamWPass, + FuseDotProductAttentionPass, + FuseOptimizerPass, + InplaceAddtoOpPass, + FuseResUnitPass, + BuildCINNPass, +) +from .fuse_all_reduce import ( # noqa: F401 + find_adjacent_match_sequences, + insert_fuse_all_reduce_ops, + has_same_attrs, + filter_all_collective_op_indices, + find_all_fuse_all_reduce_groups, + split_fuse_all_reduce_groups_by_deps, + insert_coalesce_tensor_ops, + insert_fuse_all_reduce_by_memory_size, + FuseAllReducePass, +) +from .pipeline_scheduler_pass import ( # noqa: F401 + PipelineFThenBPass, + Pipeline1F1BPass, + PipelineEager1F1BPass, + PipelineVirtualPipelinePass, + apply_pass, +) +from .ps_trainer_pass import ( # noqa: F401 + AppendSendOpsPass, + DistributedOpsPass, + DeleteOptimizesPass, + DeleteExtraOptimizerPass, + FakeInitOpsPass, + PsGpuPass, + PsTranspilePass, + SplitHeterWorkerOpsPass, + SplitTrainerOpsPass, + SetHeterPipelineOptPass, + SplitFlOpsPass, +) +from .ps_server_pass import ( # noqa: F401 + AddLrDecayTablePass, + AddListenAndServPass, + AddRpcGlobalFlagsPass, + AddOptimizerPass, + AddGeoOptimizerPass, + BuildPserverStartupProgramPass, + DeleteUnusedInStartupPass, +) __all__ = [ From 775cbdc4ae72235ced37c2f0a60e23b651bf6f5e Mon Sep 17 00:00:00 2001 From: zyfncg Date: Mon, 4 Mar 2024 00:54:30 +0800 Subject: [PATCH 245/282] Fix unittest of if and while with dynamic shape (#61972) * fix third_party patch bug * fix * Add InferSymbolicShape interface for cinn.broadcast op * clean code * fix cmake patch command to avoid patching twice error * Add more ops' InferSymbolicShape * bug fix * bug fix * add cinn_BC * fix concat * Add InferSymbolicShape for if op * update while test * ci fix * bug fix * add while infer * yield * update * fix confilct * process 0D Tensor * fix conflict * fix conflict * fix some bug of if * refector lower cinn pass * delete unused code * update * polish code * fix bug * fix broadcase * fix bug * fix bug of expand * fix bug * fix static shape bug * fix bug * polish code * fix bug * fix test_subgraph_checker --------- Co-authored-by: risemeup1 <515586620@qq.com> Co-authored-by: lanxianghit Co-authored-by: zhangbopd <1299246947@qq.com> Co-authored-by: Silver Ling --- .../hlir/dialect/operator/ir/manual_op.cc | 15 ++++ .../cinn/hlir/dialect/operator/ir/manual_op.h | 5 +- .../add_broadcast_to_elementwise_pass.cc | 36 +++++++- .../add_broadcast_to_elementwise_pass.h | 2 + .../operator/transforms/add_cinn_pass.cc | 42 ++++++++-- .../transforms/dynamic_reshape_pass.cc | 31 ++----- ...e_shape_ops_into_generate_shape_op_pass.cc | 2 +- ...ove_generate_shape_ops_to_prologue_pass.cc | 30 ++++--- .../group_merge/op_with_group_merge_util.h | 5 ++ .../transforms/insert_broadcast_pass.cc | 11 +-- .../transforms/lower_cinn_fusion_op_pass.cc | 3 +- .../operator/transforms/pd_to_cinn_pass.cc | 2 +- .../transforms/replace_dynamic_expand_pass.cc | 31 ++----- .../hlir/framework/pir/op_lowering_impl.cc | 18 +++- paddle/cinn/hlir/framework/pir/utils.cc | 84 +++++++++++++++++++ paddle/cinn/ir/schedule/ir_schedule_util.cc | 14 ++-- .../infer_symbolic_shape/cinn_op_infer_sym.h | 3 - .../fluid/pir/transforms/build_cinn_pass.cc | 25 ++++-- .../pir/transforms/sub_graph_detector.cc | 9 +- test/ir/pir/cinn/symbolic/CMakeLists.txt | 2 - .../ir/pir/cinn/symbolic/test_dyshape_rope.py | 4 +- test/ir/pir/cinn/symbolic/test_if_dy.py | 20 +++-- test/ir/pir/cinn/test_subgraph_checker.py | 2 +- 23 files changed, 282 insertions(+), 114 deletions(-) diff --git a/paddle/cinn/hlir/dialect/operator/ir/manual_op.cc b/paddle/cinn/hlir/dialect/operator/ir/manual_op.cc index 54299cc2ff7ff..aa4a02005437d 100644 --- a/paddle/cinn/hlir/dialect/operator/ir/manual_op.cc +++ b/paddle/cinn/hlir/dialect/operator/ir/manual_op.cc @@ -24,6 +24,7 @@ #include "paddle/fluid/pir/dialect/operator/ir/ir_tensor.h" #include "paddle/fluid/pir/dialect/operator/ir/op_type.h" #include "paddle/fluid/pir/dialect/operator/utils/utils.h" +#include "paddle/fluid/pir/transforms/shape_optimization_pass.h" #include "paddle/pir/include/core/builtin_type.h" #include "paddle/pir/include/core/op_base.h" #include "paddle/pir/include/dialect/control_flow/ir/cf_op.h" @@ -104,6 +105,20 @@ void GroupOp::Print(pir::IrPrinter& printer) { os << " \n }"; } +bool GroupOp::InferSymbolicShape( + ::pir::ShapeConstraintIRAnalysis* shape_analysis) { + ::pir::InferSymExprForBlock(*block(), shape_analysis); + + for (uint32_t rst_idx = 0; rst_idx < num_results(); rst_idx++) { + auto inner_yield_value = block()->back().operand_source(rst_idx); + const auto& shape = + shape_analysis->GetShapeOrDataForValue(inner_yield_value); + shape_analysis->SetShapeOrDataForValue(result(rst_idx), shape); + } + + return true; +} + void FusionOp::Build(pir::Builder& builder, pir::OperationArgument& argument, const std::vector& output_types) { diff --git a/paddle/cinn/hlir/dialect/operator/ir/manual_op.h b/paddle/cinn/hlir/dialect/operator/ir/manual_op.h index bb9917cfbfa63..1a0fa3dba75c3 100644 --- a/paddle/cinn/hlir/dialect/operator/ir/manual_op.h +++ b/paddle/cinn/hlir/dialect/operator/ir/manual_op.h @@ -29,7 +29,8 @@ namespace cinn { namespace dialect { -class IR_API GroupOp : public pir::Op { +class IR_API GroupOp + : public pir::Op { public: using Op::Op; static const char *name() { return "cinn_op.group"; } @@ -51,6 +52,8 @@ class IR_API GroupOp : public pir::Op { pir::Block *block(); std::vector GetOperators(); + bool InferSymbolicShape(pir::ShapeConstraintIRAnalysis *shape_analysis); + void VerifySig(); void Print(pir::IrPrinter &printer); // NOLINT }; diff --git a/paddle/cinn/hlir/dialect/operator/transforms/add_broadcast_to_elementwise_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/add_broadcast_to_elementwise_pass.cc index ff0fa6381c08f..abdae97fc7d0b 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/add_broadcast_to_elementwise_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/add_broadcast_to_elementwise_pass.cc @@ -15,6 +15,7 @@ #include "paddle/cinn/hlir/dialect/operator/transforms/add_broadcast_to_elementwise_pass.h" #include "paddle/cinn/hlir/dialect/operator/ir/cinn_op.h" +#include "paddle/cinn/hlir/dialect/operator/ir/manual_op.h" #include "paddle/cinn/hlir/framework/pir/utils.h" #include "paddle/common/ddim.h" #include "paddle/fluid/pir/dialect/operator/ir/op_attribute.h" @@ -173,6 +174,23 @@ class AddBroadcastToElementwisePattern : public pir::OpRewritePattern { } }; +class DeleteUselessBroadcastPattern + : public pir::OpRewritePattern { + public: + using pir::OpRewritePattern::OpRewritePattern; + + bool MatchAndRewrite(cinn::dialect::BroadcastOp broadcast, + pir::PatternRewriter& rewriter) const override { + if (!broadcast->GetParentOp()->isa()) { + rewriter.ReplaceAllUsesWith(broadcast.result(0), + broadcast->operand_source(0)); + rewriter.EraseOp(broadcast); + return true; + } + return false; + } +}; + class AddBroadcastToElementwisePass : public pir::PatternRewritePass { public: AddBroadcastToElementwisePass() @@ -224,7 +242,19 @@ class AddBroadcastToElementwisePass : public pir::PatternRewritePass { } bool CanApplyOn(pir::Operation* op) const override { - return op->isa() && op->num_regions() > 0; + return op->num_regions() > 0; + } +}; + +class DeleteUselessBroadcastPass : public pir::PatternRewritePass { + public: + DeleteUselessBroadcastPass() + : pir::PatternRewritePass("delete_useless_broadcast_pass", 1) {} + + pir::RewritePatternSet InitializePatterns(pir::IrContext* context) override { + pir::RewritePatternSet ps(context); + ps.Add(context); + return ps; } }; @@ -232,6 +262,10 @@ std::unique_ptr CreateAddBroadcastToElementwisePass() { return std::make_unique(); } +std::unique_ptr CreateDeleteUselessBroadcastPass() { + return std::make_unique(); +} + } // namespace ir } // namespace dialect } // namespace cinn diff --git a/paddle/cinn/hlir/dialect/operator/transforms/add_broadcast_to_elementwise_pass.h b/paddle/cinn/hlir/dialect/operator/transforms/add_broadcast_to_elementwise_pass.h index d4778a17a1fbd..6b2226d385733 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/add_broadcast_to_elementwise_pass.h +++ b/paddle/cinn/hlir/dialect/operator/transforms/add_broadcast_to_elementwise_pass.h @@ -23,6 +23,8 @@ namespace ir { std::unique_ptr CreateAddBroadcastToElementwisePass(); +std::unique_ptr CreateDeleteUselessBroadcastPass(); + } // namespace ir } // namespace dialect } // namespace cinn diff --git a/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.cc index 24c05b6b006c3..1c8e9b9bf725e 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.cc @@ -94,27 +94,56 @@ void ApplyCinnPreprocessPass( pass_manager->AddPass(cinn::dialect::ir::CreateSimplifyDimExprPass()); pass_manager->AddPass( cinn::dialect::ir::CreateSubstituteDimExprBasedOnConstraintsPass()); - pass_manager->AddPass(cinn::dialect::ir::CreateInsertBroadcastPass()); pass_manager->AddPass(cinn::dialect::ir::CreateConvert0DTo1DPass()); - pass_manager->AddPass(pir::CreateShapeOptimizationPass()); pass_manager->AddPass( cinn::dialect::ir::CreateFuseShapeOpsIntoGenerateShapeOpPass()); pass_manager->AddPass(pir::CreateDeadCodeEliminationPass()); pass_manager->AddPass(pir::CreateShapeOptimizationPass()); } + pass_manager->Run(program); +} + +void ApplyBuildGroupOpPass( + ::pir::Program* program, + const std::function()>& + CreatePassManager) { + std::shared_ptr pass_manager = CreatePassManager(); pass_manager->AddPass(pir::CreateBuildCinnPass()); + if (HasDynamicShape(*program)) { + pass_manager->AddPass(cinn::dialect::ir::CreateInsertBroadcastPass()); + } + pass_manager->Run(program); +} + +void ApplyGroupOpPass(::pir::Program* program, + const std::function()>& + CreatePassManager) { + std::shared_ptr pass_manager = CreatePassManager(); + if (HasDynamicShape(*program)) { + pass_manager->AddPass(::pir::CreateShapeOptimizationPass()); + pass_manager->AddPass( + cinn::dialect::ir::CreateFuseShapeOpsIntoGenerateShapeOpPass()); + pass_manager->AddPass( + cinn::dialect::ir::CreateMoveGenerateShapeOpsToProloguePass()); + } - pass_manager->AddPass( - cinn::dialect::ir::CreateMoveGenerateShapeOpsToProloguePass()); pass_manager->AddPass(cinn::dialect::ir::CreateDynamicReshapeOpPass()); pass_manager->AddPass(cinn::dialect::ir::CreateReplaceDynamicExpandOpPass()); - pass_manager->AddPass(cinn::dialect::ir::CreateDivideGroupOpToFusionOpPass()); pass_manager->AddPass(pir::CreateDeadCodeEliminationPass()); pass_manager->Run(program); } +void ApplyDivideGroupOpToFusionOpPass( + ::pir::Program* program, + const std::function()>& + CreatePassManager) { + std::shared_ptr pass_manager = CreatePassManager(); + pass_manager->AddPass(cinn::dialect::ir::CreateDivideGroupOpToFusionOpPass()); + pass_manager->Run(program); +} + void ApplyCinnLowerPass( ::pir::Program* program, const std::function()>& @@ -148,6 +177,9 @@ void ApplyCinnPass(::pir::Program* program, const std::function()>& CreatePassManager) { ApplyCinnPreprocessPass(program, CreatePassManager); + ApplyBuildGroupOpPass(program, CreatePassManager); + ApplyGroupOpPass(program, CreatePassManager); + ApplyDivideGroupOpToFusionOpPass(program, CreatePassManager); ApplyCinnLowerPass(program, CreatePassManager); } diff --git a/paddle/cinn/hlir/dialect/operator/transforms/dynamic_reshape_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/dynamic_reshape_pass.cc index d873ceb3c5ac7..4aef88b8dcd41 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/dynamic_reshape_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/dynamic_reshape_pass.cc @@ -109,43 +109,22 @@ class DynamicUnsqueezeOpPattern } }; -class DynamicReshapeOpPass : public pir::Pass { +class DynamicReshapeOpPass : public pir::PatternRewritePass { public: DynamicReshapeOpPass() - : pir::Pass("cinn_dynamic_reshape_op_pass", /*opt_level=*/1) {} + : pir::PatternRewritePass("cinn_dynamic_reshape_op_pass", 1) {} - bool Initialize(pir::IrContext* context) override { + pir::RewritePatternSet InitializePatterns(pir::IrContext* context) override { pir::RewritePatternSet ps(context); ps.Add(context); ps.Add(context); ps.Add(context); - patterns_ = pir::FrozenRewritePatternSet(std::move(ps)); - return true; - } - - void Run(pir::Operation* op) override { - pir::GreedyRewriteConfig cfg; - cfg.use_top_down_traversal = true; - cfg.max_iterations = 10; - for (uint32_t i = 0; i < op->num_regions(); ++i) { - for (auto& block : op->region(i)) { - for (auto& op : block) { - if (op.isa()) { - auto [_, num_rewrites] = - pir::ApplyPatternsGreedily(&op, patterns_, cfg); - AddStatistics(num_rewrites); - } - } - } - } + return ps; } bool CanApplyOn(pir::Operation* op) const override { - return op->num_regions() > 0; + return op->isa() && op->num_regions() > 0; } - - private: - pir::FrozenRewritePatternSet patterns_; }; std::unique_ptr CreateDynamicReshapeOpPass() { diff --git a/paddle/cinn/hlir/dialect/operator/transforms/fuse_shape_ops_into_generate_shape_op_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/fuse_shape_ops_into_generate_shape_op_pass.cc index f396e79925a37..064035b8b3b19 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/fuse_shape_ops_into_generate_shape_op_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/fuse_shape_ops_into_generate_shape_op_pass.cc @@ -206,7 +206,7 @@ class FuseShapeOpsIntoGenerateShapeOpPass : public pir::PatternRewritePass { } bool CanApplyOn(pir::Operation* op) const override { - return op->isa() && op->num_regions() > 0; + return op->num_regions() > 0; } }; diff --git a/paddle/cinn/hlir/dialect/operator/transforms/group_merge/move_generate_shape_ops_to_prologue_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/group_merge/move_generate_shape_ops_to_prologue_pass.cc index b2dfea14d4d67..f395a1fb3e28b 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/group_merge/move_generate_shape_ops_to_prologue_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/group_merge/move_generate_shape_ops_to_prologue_pass.cc @@ -67,22 +67,32 @@ class GroupOpGenerateShapeOpsPattern } }; -class MoveGenerateShapeOpsToProloguePass : public pir::PatternRewritePass { +class MoveGenerateShapeOpsToProloguePass : public pir::Pass { public: MoveGenerateShapeOpsToProloguePass() - : pir::PatternRewritePass("move_generate_shape_ops_to_prologue", 1) {} + : pir::Pass("move_generate_shape_ops_to_prologue", /*opt_level=*/1) {} - pir::RewritePatternSet InitializePatterns(pir::IrContext* context) override { - pir::RewritePatternSet ps(context); - ps.Add(context); - return ps; + void Run(pir::Operation* op) override { + auto group_op = op->dyn_cast(); + CHECK(group_op); + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + pir::ShapeConstraintIRAnalysis& shape_analysis = + pir::ShapeAnalysisManager::Instance().Get(group_op->GetParentProgram()); + ShapeOrDataDimExprsAccessor dim_exprs_accessor{ + .GetShapeOrDataDimExprs = + [&](pir::Value value) -> const symbol::ShapeOrDataDimExprs& { + return shape_analysis.GetShapeOrDataForValue(value); + }, + .SetShapeOrDataDimExprs = + [&](pir::Value value, + const symbol::ShapeOrDataDimExprs& dim_exprs) { + shape_analysis.SetShapeOrDataForValue(value, dim_exprs); + }}; + MoveGenerateShapeOpsToPrologue(ctx, group_op.block(), dim_exprs_accessor); } bool CanApplyOn(pir::Operation* op) const override { - if (!(op->isa() && op->num_regions() > 0)) return false; - auto* program = op->GetParentProgram(); - VLOG(4) << "Before MoveGenerateShapeOpsToProloguePass: " << *program; - return true; + return op->isa() && op->num_regions() > 0; } }; diff --git a/paddle/cinn/hlir/dialect/operator/transforms/group_merge/op_with_group_merge_util.h b/paddle/cinn/hlir/dialect/operator/transforms/group_merge/op_with_group_merge_util.h index 41dd5c9089c71..038e49b8b553a 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/group_merge/op_with_group_merge_util.h +++ b/paddle/cinn/hlir/dialect/operator/transforms/group_merge/op_with_group_merge_util.h @@ -246,6 +246,11 @@ inline bool horizontal_or_vertical_reduce_relation( // check producer has same shape with reducer op. auto reduce_shape = ::common::vectorize(GetFirstInputShape(reducer)); auto reduce_axes = GetVectorAttr(reducer, "dim"); + if (reduce_axes.empty()) { + for (size_t i = 0; i < reduce_shape.size(); ++i) { + reduce_axes.push_back(i); + } + } for (auto& axis : reduce_axes) { // if axis = -1, set as shape.size() - 1 diff --git a/paddle/cinn/hlir/dialect/operator/transforms/insert_broadcast_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/insert_broadcast_pass.cc index f7eea680a3b61..022077d24916a 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/insert_broadcast_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/insert_broadcast_pass.cc @@ -15,6 +15,7 @@ #include "paddle/cinn/hlir/dialect/operator/transforms/insert_broadcast_pass.h" #include "paddle/cinn/hlir/dialect/operator/ir/cinn_op.h" +#include "paddle/cinn/hlir/dialect/operator/ir/manual_op.h" #include "paddle/cinn/hlir/framework/pir/utils.h" #include "paddle/common/ddim.h" #include "paddle/fluid/pir/dialect/operator/ir/manual_op.h" @@ -51,12 +52,13 @@ bool ProcessOp(pir::Operation* op, pir::PatternRewriter* rewriter) { const auto& y_shape = shape_analysis.GetShapeOrDataForValue(y); const auto& out_shape = shape_analysis.GetShapeOrDataForValue(op->result(0)); - bool has_insert_broadcast = false; + if (x_shape == y_shape) { + return false; + } pir::Value output_dim_tensor = GetOutputDimTensor(rewriter, x, y); if (x_shape.shape() != out_shape.shape() || x_shape.data() != out_shape.data()) { - has_insert_broadcast = true; pir::Value broadcasted_x = rewriter->Build(x, output_dim_tensor).out(); op->operand(0).set_source(broadcasted_x); @@ -64,13 +66,12 @@ bool ProcessOp(pir::Operation* op, pir::PatternRewriter* rewriter) { } if (y_shape.shape() != out_shape.shape() || y_shape.data() != out_shape.data()) { - has_insert_broadcast = true; pir::Value broadcasted_y = rewriter->Build(y, output_dim_tensor).out(); op->operand(1).set_source(broadcasted_y); shape_analysis.SetShapeOrDataForValue(broadcasted_y, out_shape); } - return has_insert_broadcast; + return true; } } // namespace @@ -120,7 +121,7 @@ class InsertBroadcastPass : public pir::PatternRewritePass { } bool CanApplyOn(pir::Operation* op) const override { - return op->isa() && op->num_regions() > 0; + return op->isa() && op->num_regions() > 0; } }; diff --git a/paddle/cinn/hlir/dialect/operator/transforms/lower_cinn_fusion_op_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/lower_cinn_fusion_op_pass.cc index a2393a09fae21..c725d33257cc3 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/lower_cinn_fusion_op_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/lower_cinn_fusion_op_pass.cc @@ -618,7 +618,6 @@ CreateGroupShapeOrDataExprs( } return value2shape; } - class FusionOpPattern : public pir::OpRewritePattern { public: explicit FusionOpPattern(::pir::IrContext* context) @@ -772,7 +771,7 @@ class LowerCinnDyShapeFusionOpPass : public pir::PatternRewritePass { } bool CanApplyOn(pir::Operation* op) const override { - return op->isa() && op->num_regions() > 0; + return op->num_regions() > 0; } }; diff --git a/paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.cc index ad6c7b9a060da..03a510863a61b 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.cc @@ -740,7 +740,7 @@ pir::RewritePatternSet PdOpToCinnOpPass::InitializePatterns( } bool PdOpToCinnOpPass::CanApplyOn(pir::Operation *op) const { - return op->isa() && op->num_regions() > 0; + return op->num_regions() > 0; } std::unique_ptr CreatePdOpToCinnOpPass() { diff --git a/paddle/cinn/hlir/dialect/operator/transforms/replace_dynamic_expand_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/replace_dynamic_expand_pass.cc index 85bdf3985c8a5..32615b4cce69c 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/replace_dynamic_expand_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/replace_dynamic_expand_pass.cc @@ -93,41 +93,20 @@ class DynamicExpandOpPattern } }; -class ReplaceDynamicExpandOpPass : public pir::Pass { +class ReplaceDynamicExpandOpPass : public pir::PatternRewritePass { public: ReplaceDynamicExpandOpPass() - : pir::Pass("replace_dynamic_expand_op_pass", /*opt_level=*/1) {} + : pir::PatternRewritePass("replace_dynamic_expand_op_pass", 1) {} - bool Initialize(pir::IrContext* context) override { + pir::RewritePatternSet InitializePatterns(pir::IrContext* context) override { pir::RewritePatternSet ps(context); ps.Add(context); - patterns_ = pir::FrozenRewritePatternSet(std::move(ps)); - return true; - } - - void Run(pir::Operation* op) override { - pir::GreedyRewriteConfig cfg; - cfg.use_top_down_traversal = true; - cfg.max_iterations = 10; - for (uint32_t i = 0; i < op->num_regions(); ++i) { - for (auto& block : op->region(i)) { - for (auto& op : block) { - if (op.isa()) { - const auto& [_, num_rewrites] = - pir::ApplyPatternsGreedily(&op, patterns_, cfg); - AddStatistics(num_rewrites); - } - } - } - } + return ps; } bool CanApplyOn(pir::Operation* op) const override { - return op->num_regions() > 0; + return op->isa() && op->num_regions() > 0; } - - private: - pir::FrozenRewritePatternSet patterns_; }; std::unique_ptr CreateReplaceDynamicExpandOpPass() { diff --git a/paddle/cinn/hlir/framework/pir/op_lowering_impl.cc b/paddle/cinn/hlir/framework/pir/op_lowering_impl.cc index 828437f0f4abe..032431feda354 100644 --- a/paddle/cinn/hlir/framework/pir/op_lowering_impl.cc +++ b/paddle/cinn/hlir/framework/pir/op_lowering_impl.cc @@ -726,12 +726,18 @@ ir::Tensor OpLowererImpl::GetTensor(const GroupPtr& group, std::vector sym_shape; ForEachDimExpr( [&](const auto& sym) { sym_shape.emplace_back(input_id, sym); }); + if (sym_shape.empty()) { + sym_shape.emplace_back(input_id, symbol::DimExpr{1}); + } return lang::CreatePlaceHolder( sym_shape, CompatibleInfo::ConvertIRType(dtype), input_id); } else { - return lang::CreatePlaceHolder(::common::vectorize(type_info.dims()), - CompatibleInfo::ConvertIRType(dtype), - input_id); + auto shape = ::common::vectorize(type_info.dims()); + if (shape.empty()) { + shape.push_back(1); + } + return lang::CreatePlaceHolder( + shape, CompatibleInfo::ConvertIRType(dtype), input_id); } } @@ -783,6 +789,9 @@ void OpLowererImpl::CollectOutputInfo(::pir::Operation* op, out_types->push_back(CompatibleInfo::ConvertIRType(type_info.dtype())); auto out_shape = ::common::vectorize(type_info.dims()); + if (out_shape.empty()) { + out_shape.push_back(1); + } out_shapes->push_back(std::move(out_shape)); } } @@ -819,6 +828,9 @@ void OpLowererImpl::CollectOutputInfo( std::vector sym_shape; ForEachDimExpr( [&](const auto& sym) { sym_shape.emplace_back(output_id, sym); }); + if (sym_shape.empty()) { + sym_shape.emplace_back(output_id, symbol::DimExpr{1}); + } out_shapes->emplace_back(std::move(sym_shape)); } } diff --git a/paddle/cinn/hlir/framework/pir/utils.cc b/paddle/cinn/hlir/framework/pir/utils.cc index 83fe4ed5ef16c..7d0acaa3cc92b 100644 --- a/paddle/cinn/hlir/framework/pir/utils.cc +++ b/paddle/cinn/hlir/framework/pir/utils.cc @@ -32,6 +32,7 @@ #include "paddle/pir/include/dialect/control_flow/ir/cf_dialect.h" #include "paddle/pir/include/dialect/control_flow/ir/cf_op.h" #include "paddle/pir/include/dialect/shape/ir/shape_attribute.h" +#include "paddle/pir/include/dialect/shape/utils/shape_analysis.h" PD_DECLARE_string(allow_cinn_ops); PD_DECLARE_string(deny_cinn_ops); @@ -177,6 +178,86 @@ bool AllInputDenseTensor(const ::pir::Operation& op) { return true; } +bool IsSmallNumelOp(const ::pir::Operation& op) { + auto GetNumElementsFromDim = [](const ::pir::DDim& dim) -> int64_t { + if (::common::contain_unknown_dim(dim)) { + return std::numeric_limits::max(); + } else { + return ::common::product(dim); + } + }; + + auto GetNumElementsFromValue = [&](const ::pir::Value& value) { + int64_t numel = -1; + if (value && value.type()) { + auto type = value.type().dyn_cast<::pir::DenseTensorType>(); + if (type) { + numel = GetNumElementsFromDim(type.dims()); + } + } + return numel; + }; + const int64_t max_value_numel = [&] { + int64_t max_value_numel = -1; + if (op.num_operands() == 0) { // no input + return max_value_numel; + } + + for (uint32_t i = 0; i < op.num_operands(); ++i) { + max_value_numel = std::max(GetNumElementsFromValue(op.operand_source(i)), + max_value_numel); + } + for (uint32_t i = 0; i < op.num_results(); ++i) { + max_value_numel = + std::max(GetNumElementsFromValue(op.result(i)), max_value_numel); + } + return max_value_numel; + }(); + + // max value check + if (0 <= max_value_numel && max_value_numel < 32) { + return true; + } + + return false; +} + +bool IsShapeComputeOp(const ::pir::Operation& op) { + const auto& shape_analysis = ::pir::ShapeAnalysisManager::Instance().Get( + op.GetParent()->parent_program()); + if (op.num_operands() == 0) { + return false; + } + bool all_input_has_shape_data = true; + for (uint32_t i = 0; i < op.num_operands(); ++i) { + if (shape_analysis.HasShapeOrDataForValue(op.operand_source(i))) { + const auto& shape_expr = + shape_analysis.GetShapeOrDataForValue(op.operand_source(i)); + if (shape_expr.isa() && + shape_expr.data()) { // has shape data + continue; + } + } + all_input_has_shape_data = false; + break; + } + return all_input_has_shape_data; +} + +// TODO(zyfncg): This function is a temporary solution, we need to remove it in +// the future. +bool IsTempDenySpecialOp(const ::pir::Operation& op) { + if (op.name() == "cinn_op.generate_shape") { + return false; + } + + if (IsShapeComputeOp(op) || IsSmallNumelOp(op)) { + return true; + } + + return false; +} + bool IsRegisteredInCINN(const ::pir::Operation& op) { if (CompatibleInfo::OP_NAMES.find(op.name()) != CompatibleInfo::OP_NAMES.end()) { @@ -192,6 +273,9 @@ bool IsSupportForCinn(const ::pir::Operation& op) { << "So mark IsSupportForCinn: " << false; return false; } + if (IsTempDenySpecialOp(op)) { + return false; + } auto allow_ops = StringSplit(FLAGS_allow_cinn_ops, kDelim); auto deny_ops = StringSplit(FLAGS_deny_cinn_ops, kDelim); LOG_FIRST_N(INFO, 1) << "The allowed Cinn Ops: " << GetDebugInfo(allow_ops); diff --git a/paddle/cinn/ir/schedule/ir_schedule_util.cc b/paddle/cinn/ir/schedule/ir_schedule_util.cc index 739f17d06e80a..62f036d3583d9 100644 --- a/paddle/cinn/ir/schedule/ir_schedule_util.cc +++ b/paddle/cinn/ir/schedule/ir_schedule_util.cc @@ -264,18 +264,20 @@ std::vector ValidateFactors(const std::vector& factors, if (!has_minus_one) { if (product < total_extent) { std::ostringstream os; - os << "In Split, the factors' product should be not larger than or equal " - "to original loop's extent!" - << std::endl; + os << "In Split, the factors' product[" << product + << "] should be not larger than or equal " + "to original loop's extent[" + << total_extent << "]!" << std::endl; throw IRScheduleErrorHandler(primitive, os.str(), module_expr); } return validated_factors; } else { if (product > total_extent) { std::ostringstream os; - os << "In Split, the factors' product should be not larger than or equal " - "to original loop's extent!" - << std::endl; + os << "In Split, the factors' product[" << product + << "] should be not larger than or equal " + "to original loop's extent[" + << total_extent << "]!" << std::endl; throw IRScheduleErrorHandler(primitive, os.str(), module_expr); } int minus_one_candidate = static_cast( diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.h index b98f8e02d66e9..34dcbd89d711f 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.h @@ -20,9 +20,6 @@ namespace cinn::dialect { bool BroadcastOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool SliceOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); - bool ConcatOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); diff --git a/paddle/fluid/pir/transforms/build_cinn_pass.cc b/paddle/fluid/pir/transforms/build_cinn_pass.cc index 48c872c23b527..34d9fde7831c8 100644 --- a/paddle/fluid/pir/transforms/build_cinn_pass.cc +++ b/paddle/fluid/pir/transforms/build_cinn_pass.cc @@ -14,6 +14,7 @@ #include "paddle/fluid/pir/transforms/build_cinn_pass.h" +#include "paddle/cinn/hlir/dialect/operator/ir/manual_op.h" #include "paddle/cinn/hlir/framework/pir/utils.h" #include "paddle/fluid/pir/transforms/sub_graph_detector.h" #include "paddle/pir/include/core/builtin_op.h" @@ -29,22 +30,28 @@ class BuildCinnPass : public pir::Pass { BuildCinnPass() : pir::Pass("build_cinn_pass", /*opt_level=*/1) {} void Run(pir::Operation* op) override { - auto module_op = op->dyn_cast(); - IR_ENFORCE(module_op, "build_cinn_pass should run on module op."); - auto& block = module_op.block(); + for (uint32_t i = 0; i < op->num_regions(); ++i) { + for (auto& block : op->region(i)) { + ProcessBlock(&block); + } + } + } + + bool CanApplyOn(pir::Operation* op) const override { + return op->num_regions() > 0 && !op->isa() && + !op->isa(); + } + private: + void ProcessBlock(pir::Block* block) { std::vector groups = - ::pir::SubgraphDetector(&block, CompatibleInfo::IsSupportCinn)(); + ::pir::SubgraphDetector(block, CompatibleInfo::IsSupportCinn)(); AddStatistics(groups.size()); for (auto& group_ops : groups) { VLOG(4) << "current group_ops.size(): " << group_ops.size(); - ::pir::ReplaceWithGroupOp(&block, group_ops); + ::pir::ReplaceWithGroupOp(block, group_ops); } } - - bool CanApplyOn(pir::Operation* op) const override { - return op->isa() && op->num_regions() > 0; - } }; } // namespace diff --git a/paddle/fluid/pir/transforms/sub_graph_detector.cc b/paddle/fluid/pir/transforms/sub_graph_detector.cc index 0e9547f7642c7..24d2c61f98d4c 100644 --- a/paddle/fluid/pir/transforms/sub_graph_detector.cc +++ b/paddle/fluid/pir/transforms/sub_graph_detector.cc @@ -83,7 +83,8 @@ std::vector InverselyTopologicalSort(pir::Block* block) { } auto* defined_op = operand.source().defining_op(); --pending_count[defined_op]; - if (defined_op && pending_count[defined_op] == 0) { + if (defined_op && pending_count[defined_op] == 0 && + defined_op->GetParent() == block) { queue.push(defined_op); } } @@ -109,7 +110,8 @@ std::vector GetProducerOpsReverseSort( continue; } auto* source_op = operand.source().defining_op(); - if (source_op && !producers.count(source_op)) { + if (source_op && !producers.count(source_op) && + source_op->GetParent() == op->GetParent()) { producers.insert(source_op); PADDLE_ENFORCE( op2id.count(source_op), @@ -134,7 +136,8 @@ std::unordered_set GetProducerOps(pir::Operation* op) { if (!operand || !(operand.source())) { continue; } - if (auto* source_op = operand.source().defining_op()) { + auto* source_op = operand.source().defining_op(); + if (source_op && source_op->GetParent() == op->GetParent()) { producers.insert(source_op); } } diff --git a/test/ir/pir/cinn/symbolic/CMakeLists.txt b/test/ir/pir/cinn/symbolic/CMakeLists.txt index 9d2fc16e2c638..3a330e6527530 100644 --- a/test/ir/pir/cinn/symbolic/CMakeLists.txt +++ b/test/ir/pir/cinn/symbolic/CMakeLists.txt @@ -54,7 +54,6 @@ if(WITH_GPU) COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/python/:$ENV{PYTHONPATH} - FLAGS_cinn_convert_dynamic_dim_to_static_dim=S0:2048 FLAGS_cinn_bucket_compile=True FLAGS_enable_pir_api=1 ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_if_dy.py WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) @@ -207,7 +206,6 @@ if(WITH_GPU) COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/python/:$ENV{PYTHONPATH} - FLAGS_cinn_convert_dynamic_dim_to_static_dim=S0:2048 FLAGS_cinn_bucket_compile=True FLAGS_enable_pir_api=1 ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_while_dy.py WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/test/ir/pir/cinn/symbolic/test_dyshape_rope.py b/test/ir/pir/cinn/symbolic/test_dyshape_rope.py index 23897178f50b3..ee11bc73876b1 100644 --- a/test/ir/pir/cinn/symbolic/test_dyshape_rope.py +++ b/test/ir/pir/cinn/symbolic/test_dyshape_rope.py @@ -92,14 +92,14 @@ def check_jit_kernel_info(self, static_fn): }, }, 'else_0': { - 'if_0_0': { + 'if_0_0': {utils.JIT_KERNEL_NAME: 1}, + 'else_0_0': { 'if_0_0_0': {utils.JIT_KERNEL_NAME: 1}, 'else_0_0_0': { 'if_0_0_0_0': {utils.JIT_KERNEL_NAME: 1}, 'else_0_0_0_0': {utils.JIT_KERNEL_NAME: 1}, }, }, - 'else_0_0': {utils.JIT_KERNEL_NAME: 1}, }, }, ) diff --git a/test/ir/pir/cinn/symbolic/test_if_dy.py b/test/ir/pir/cinn/symbolic/test_if_dy.py index 0a9bd93354a5a..fc77fdbba5d7e 100644 --- a/test/ir/pir/cinn/symbolic/test_if_dy.py +++ b/test/ir/pir/cinn/symbolic/test_if_dy.py @@ -53,8 +53,15 @@ def prepare_data(self): self.x.stop_gradient = False def check_jit_kernel_info(self, static_fn): - utils.check_jit_kernel_number(static_fn, 1) - utils.check_jit_kernel_structure(static_fn, {utils.JIT_KERNEL_NAME: 1}) + utils.check_jit_kernel_number(static_fn, 2) + utils.check_jit_kernel_structure( + static_fn, + { + 'if_0': {utils.JIT_KERNEL_NAME: 1}, + 'else_0': {}, + utils.JIT_KERNEL_NAME: 1, + }, + ) def eval(self, use_cinn): net = IfSubgraph() @@ -70,11 +77,10 @@ def eval(self, use_cinn): def test_eval(self): dy_out = self.eval(use_cinn=False) - if utils.unittest_use_cinn(): - cinn_out = self.eval(use_cinn=True) - np.testing.assert_allclose( - cinn_out.numpy(), dy_out.numpy(), atol=1e-6, rtol=1e-6 - ) + cinn_out = self.eval(use_cinn=True) + np.testing.assert_allclose( + cinn_out.numpy(), dy_out.numpy(), atol=1e-6, rtol=1e-6 + ) if __name__ == '__main__': diff --git a/test/ir/pir/cinn/test_subgraph_checker.py b/test/ir/pir/cinn/test_subgraph_checker.py index 8f3b791358a80..9a5672c462b18 100644 --- a/test/ir/pir/cinn/test_subgraph_checker.py +++ b/test/ir/pir/cinn/test_subgraph_checker.py @@ -32,7 +32,7 @@ def create_program(self, enable_prim=False): main_program = paddle.static.Program() with paddle.static.program_guard(main_program): - x = paddle.static.data(shape=[4, 4], name='pt_input_0') + x = paddle.static.data(shape=[16, 4], name='pt_input_0') out = paddle.nn.functional.softmax(x) fetch_out = paddle._pir_ops.fetch(out, out_name, 0) fetch_out.persistable = True From cb8ae07d1a051699dcec7382e59fed8ec0a91982 Mon Sep 17 00:00:00 2001 From: ShenLiang <1422485404@qq.com> Date: Mon, 4 Mar 2024 09:46:24 +0800 Subject: [PATCH 246/282] Revert "set default in p2p_overlap (#62051)" (#62296) This reverts commit 488f2d536f0f794fdbb787785af3e14f95d767c5. --- paddle/fluid/framework/distributed_strategy.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/framework/distributed_strategy.proto b/paddle/fluid/framework/distributed_strategy.proto index 27c7a7a7af276..58460fcf9064b 100755 --- a/paddle/fluid/framework/distributed_strategy.proto +++ b/paddle/fluid/framework/distributed_strategy.proto @@ -82,7 +82,7 @@ message PpConfig { optional bool sharding_comm_overlap = 4 [ default = false ]; optional bool profiling = 5 [ default = false ]; optional bool release_gradients = 6 [ default = false ]; - optional bool overlap_p2p_comm = 7 [default = true]; + optional bool overlap_p2p_comm = 7 [default = false]; } message DygraphShardingConfig { From adb8bc231f32d2e074b998783ac88aeadb692bae Mon Sep 17 00:00:00 2001 From: wanghuancoder Date: Mon, 4 Mar 2024 10:20:26 +0800 Subject: [PATCH 247/282] [PIR] add some check if for onednn kernel (#62269) * add some check if for onednn kernel --- paddle/phi/core/kernel_context.h | 4 ++++ paddle/phi/kernels/onednn/add_n_kernel.cc | 17 ++++++++++++- paddle/phi/kernels/onednn/sgd_kernel.cc | 24 +++++++++++++++++-- .../phi/kernels/onednn/slice_grad_kernel.cc | 11 ++++++++- paddle/phi/kernels/onednn/slice_kernel.cc | 16 ++++++++++++- paddle/phi/kernels/onednn/split_kernel.cc | 15 ++++++++++-- 6 files changed, 80 insertions(+), 7 deletions(-) diff --git a/paddle/phi/core/kernel_context.h b/paddle/phi/core/kernel_context.h index b40978edf1225..947af3af1d089 100644 --- a/paddle/phi/core/kernel_context.h +++ b/paddle/phi/core/kernel_context.h @@ -114,6 +114,10 @@ class KernelContext { return paddle::none; } + const TensorBase* MutableIutputAt(size_t idx) const { + return inputs_.at(idx); + } + template TensorType* MutableOutputAt(size_t idx) { return static_cast(outputs_.at(idx)); diff --git a/paddle/phi/kernels/onednn/add_n_kernel.cc b/paddle/phi/kernels/onednn/add_n_kernel.cc index f852254043e87..454d6851cfeac 100644 --- a/paddle/phi/kernels/onednn/add_n_kernel.cc +++ b/paddle/phi/kernels/onednn/add_n_kernel.cc @@ -17,6 +17,19 @@ #include "paddle/phi/core/kernel_registry.h" namespace phi { +bool AddNCheckIfOneDNNSupport(const KernelContext* ctx) { + for (size_t i = 0; i < ctx->InputsSize(); i++) { + if (!DenseTensor::classof(ctx->MutableIutputAt(i))) { + return false; + } + } + KernelContext* ctx_tmp = const_cast(ctx); + if (!DenseTensor::classof(ctx_tmp->MutableOutputAt(0))) { + return false; + } + return true; +} + namespace funcs { template class SumOneDNNHandler : public OneDNNHandlerNoCachingT { @@ -122,4 +135,6 @@ void AddNKernel(const Context& dev_ctx, } // namespace phi PD_REGISTER_KERNEL( - add_n, OneDNN, ONEDNN, phi::AddNKernel, float, phi::dtype::bfloat16) {} + add_n, OneDNN, ONEDNN, phi::AddNKernel, float, phi::dtype::bfloat16) { + kernel->check_if_onednn_kernel_support_ = phi::AddNCheckIfOneDNNSupport; +} diff --git a/paddle/phi/kernels/onednn/sgd_kernel.cc b/paddle/phi/kernels/onednn/sgd_kernel.cc index 6ceba6b2cf7b7..007af969e2787 100644 --- a/paddle/phi/kernels/onednn/sgd_kernel.cc +++ b/paddle/phi/kernels/onednn/sgd_kernel.cc @@ -20,6 +20,22 @@ namespace phi { +bool SgdCheckIfOneDNNSupport(const KernelContext* ctx) { + if (DenseTensor::classof(ctx->MutableIutputAt(0)) && + DenseTensor::classof(ctx->MutableIutputAt(2))) { + return true; + } + return false; +} + +bool SgdSparseCheckIfOneDNNSupport(const KernelContext* ctx) { + if (DenseTensor::classof(ctx->MutableIutputAt(0)) && + SelectedRows::classof(ctx->MutableIutputAt(2))) { + return true; + } + return false; +} + template void SGDDenseKernel(const Context& dev_ctx, const DenseTensor& param, @@ -82,11 +98,15 @@ void SGDDenseParamSparseGradKernel( } // namespace phi PD_REGISTER_KERNEL( - sgd, OneDNN, ONEDNN, phi::SGDDenseKernel, float, phi::dtype::bfloat16) {} + sgd, OneDNN, ONEDNN, phi::SGDDenseKernel, float, phi::dtype::bfloat16) { + kernel->check_if_onednn_kernel_support_ = phi::SgdCheckIfOneDNNSupport; +} PD_REGISTER_KERNEL(sgd_dense_param_sparse_grad, OneDNN, ONEDNN, phi::SGDDenseParamSparseGradKernel, float, - phi::dtype::bfloat16) {} + phi::dtype::bfloat16) { + kernel->check_if_onednn_kernel_support_ = phi::SgdSparseCheckIfOneDNNSupport; +} diff --git a/paddle/phi/kernels/onednn/slice_grad_kernel.cc b/paddle/phi/kernels/onednn/slice_grad_kernel.cc index a929751433ab9..e2d4aa59c9d46 100644 --- a/paddle/phi/kernels/onednn/slice_grad_kernel.cc +++ b/paddle/phi/kernels/onednn/slice_grad_kernel.cc @@ -19,6 +19,13 @@ namespace phi { +bool SliceGradCheckIfOneDNNSupport(const KernelContext* ctx) { + if (ctx->InputAt(1).mem_desc().get_inner_nblks() == 0) { + return true; + } + return false; +} + template void SliceGradKernel(const Context& dev_ctx, const DenseTensor& input UNUSED, @@ -83,4 +90,6 @@ PD_REGISTER_KERNEL(slice_grad, ONEDNN, phi::SliceGradKernel, float, - phi::dtype::bfloat16) {} + phi::dtype::bfloat16) { + kernel->check_if_onednn_kernel_support_ = phi::SliceGradCheckIfOneDNNSupport; +} diff --git a/paddle/phi/kernels/onednn/slice_kernel.cc b/paddle/phi/kernels/onednn/slice_kernel.cc index aeff6168f047c..41116033d7237 100644 --- a/paddle/phi/kernels/onednn/slice_kernel.cc +++ b/paddle/phi/kernels/onednn/slice_kernel.cc @@ -19,6 +19,18 @@ namespace phi { +bool SliceCheckIfOneDNNSupport(const KernelContext* ctx) { + auto x = ctx->InputAt(0); + auto vec_dims = common::vectorize(x.dims()); + bool all_zero_dims = std::all_of( + vec_dims.cbegin(), vec_dims.cend(), [](int64_t i) { return i == 0; }); + + if (!all_zero_dims && x.mem_desc().get_inner_nblks() == 0) { + return true; + } + return false; +} + template void SliceKernel(const Context& dev_ctx, const DenseTensor& x, @@ -106,4 +118,6 @@ PD_REGISTER_KERNEL(slice, float, int8_t, uint8_t, - phi::dtype::bfloat16) {} + phi::dtype::bfloat16) { + kernel->check_if_onednn_kernel_support_ = phi::SliceCheckIfOneDNNSupport; +} diff --git a/paddle/phi/kernels/onednn/split_kernel.cc b/paddle/phi/kernels/onednn/split_kernel.cc index cf0cd1d62a020..713324774ab20 100644 --- a/paddle/phi/kernels/onednn/split_kernel.cc +++ b/paddle/phi/kernels/onednn/split_kernel.cc @@ -19,6 +19,13 @@ namespace phi { +bool SplitCheckIfOneDNNSupport(const KernelContext* ctx) { + if (ctx->InputAt(0).mem_desc().get_inner_nblks() == 0) { + return true; + } + return false; +} + const std::vector get_slice_strides( const std::vector& out_vec_dims, const dnnl::memory::desc& full_md, @@ -104,7 +111,9 @@ PD_REGISTER_KERNEL(split, float, phi::dtype::bfloat16, int8_t, - uint8_t) {} + uint8_t) { + kernel->check_if_onednn_kernel_support_ = phi::SplitCheckIfOneDNNSupport; +} PD_REGISTER_KERNEL(split_with_num, OneDNN, @@ -113,4 +122,6 @@ PD_REGISTER_KERNEL(split_with_num, float, phi::dtype::bfloat16, int8_t, - uint8_t) {} + uint8_t) { + kernel->check_if_onednn_kernel_support_ = phi::SplitCheckIfOneDNNSupport; +} From de1777b145df0a3318dab2da2093e1a1e325227f Mon Sep 17 00:00:00 2001 From: gouzil <66515297+gouzil@users.noreply.github.com> Date: Mon, 4 Mar 2024 10:37:38 +0800 Subject: [PATCH 248/282] [SOT][3.12] replace `POP_JUMP_{BACKWARD,FORWARD}_IF_{TRUE,FALSE}` to `POP_JUMP_IF_{TRUE,FALSE}` (#62155) --- .../executor/opcode_executor.py | 25 ++++++-- .../executor/opcode_inline_executor.py | 4 ++ .../executor/pycode_generator.py | 2 +- .../instruction_utils/instruction_pass.py | 59 ++++++++++++++----- .../instruction_utils/instruction_utils.py | 15 +++-- .../instruction_utils/opcode_info.py | 4 +- test/sot/skip_files_py312 | 5 -- 7 files changed, 84 insertions(+), 30 deletions(-) diff --git a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py index 7f28346922d91..8c6f4818f4689 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py +++ b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py @@ -1697,8 +1697,9 @@ def FOR_ITER(self, instr): self._inline_call_for_loop(iterator, instr) self._lasti = self.indexof(instr.jump_to) - next_instr = self._instructions[self._lasti] - self._lasti += int(next_instr.opname == 'END_FOR') + if sys.version_info >= (3, 12): + assert self._instructions[self._lasti].opname == "END_FOR" + self._lasti += 1 except BreakGraphError as e: log(3, f"[BreakGraph] FOR_ITER sim for loop failed for: {e}\n") if backup_iter_idx: @@ -2071,10 +2072,17 @@ def create_after_loop_fn(): return None pycode_gen = PyCodeGen(self._frame) origin_instrs = get_instructions(pycode_gen._origin_code) + resume_fn_end_idx = loop_body_end_idx + + # skip resume END_FOR in python3.12 + if sys.version_info >= (3, 12): + assert origin_instrs[loop_body_end_idx].opname == "END_FOR" + resume_fn_end_idx += 1 + pycode_gen.set_function_inputs( after_loop_fn_inputs, stack_size=len(self.stack) - 1 ) - pycode_gen.extend_instrs(origin_instrs[loop_body_end_idx:]) + pycode_gen.extend_instrs(origin_instrs[resume_fn_end_idx:]) # the resume_fn contains return code, so we don't need set output here # global vars are updated correctly, and need local vars will return after_loop_fn = pycode_gen.create_function() @@ -2138,8 +2146,13 @@ def create_after_loop_fn(): self._graph.pycode_gen.gen_jump( for_iter, direction=JumpDirection.BACKWARD ) + + if sys.version_info >= (3, 12): + end_for = self._graph.pycode_gen.add_instr("END_FOR") + nop = self._graph.pycode_gen.add_instr("NOP") - for_iter.jump_to = nop + + for_iter.jump_to = end_for if sys.version_info >= (3, 12) else nop jump_if_break.jump_to = nop # 9. prepare inputs and call after_loop_fn @@ -2209,6 +2222,8 @@ def create_inline_call_fn(): for_iter_instr, direction=JumpDirection.BACKWARD ) + if sys.version_info >= (3, 12): + end_for = pycode_gen.add_instr("END_FOR") nop_for_break = pycode_gen.add_instr("NOP") # 2.4. relocate jumps @@ -2223,6 +2238,8 @@ def create_inline_call_fn(): instr.jump_to = nop_for_break jump.jump_to = for_iter_instr + if sys.version_info >= (3, 12): + for_iter_instr.jump_to = end_for pycode_gen.set_function_outputs(output_var_names) inline_call_fn = pycode_gen.create_function() diff --git a/python/paddle/jit/sot/opcode_translator/executor/opcode_inline_executor.py b/python/paddle/jit/sot/opcode_translator/executor/opcode_inline_executor.py index 306166aa7d872..98cb2da36d02a 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/opcode_inline_executor.py +++ b/python/paddle/jit/sot/opcode_translator/executor/opcode_inline_executor.py @@ -17,6 +17,7 @@ import contextlib import inspect import re +import sys from typing import TYPE_CHECKING from ...profiler import event_register @@ -316,6 +317,9 @@ def FOR_ITER(self, instr: Instruction): self.stack.pop() assert isinstance(instr.jump_to, Instruction) self._lasti = self.indexof(instr.jump_to) + if sys.version_info >= (3, 12): + assert self._instructions[self._lasti].opname == "END_FOR" + self._lasti += 1 else: self._graph.remove_global_guarded_variable(iterator) diff --git a/python/paddle/jit/sot/opcode_translator/executor/pycode_generator.py b/python/paddle/jit/sot/opcode_translator/executor/pycode_generator.py index ce25cabd6f2d4..472013d8919bb 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/pycode_generator.py +++ b/python/paddle/jit/sot/opcode_translator/executor/pycode_generator.py @@ -956,7 +956,7 @@ def gen_pop_jump( direction: JumpDirection = JumpDirection.FORWARD, suffix: PopJumpCond = PopJumpCond.NONE, ) -> Instruction: - if sys.version_info >= (3, 11): + if sys.version_info >= (3, 11) and sys.version_info < (3, 12): return self.add_instr( f"POP_JUMP_{direction.value}_IF_{suffix.value}", jump_to=jump_to ) diff --git a/python/paddle/jit/sot/opcode_translator/instruction_utils/instruction_pass.py b/python/paddle/jit/sot/opcode_translator/instruction_utils/instruction_pass.py index 5b0cc17fc808f..e790f720ee3f8 100644 --- a/python/paddle/jit/sot/opcode_translator/instruction_utils/instruction_pass.py +++ b/python/paddle/jit/sot/opcode_translator/instruction_utils/instruction_pass.py @@ -12,21 +12,32 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING + from paddle.jit.sot.utils import log, log_do from ...utils import InnerError from .instruction_utils import instrs_info from .stack_analyse import StackAnalyser +if TYPE_CHECKING: + from .instruction_utils import Instruction + -def apply_instr_pass(instrs, code_options): +def apply_instr_pass(instrs: list[Instruction], code_options): log(4, f"[Opcode Pass]: Original New Code {code_options['co_name']}:\n") log_do(4, lambda: print(instrs_info(instrs))) - supported_passes = ( + supported_passes = [ remove_load_store_pass, remove_duplicate_resume, check_precall_followed_by_call, - ) + ] + + if sys.version_info >= (3, 12): + supported_passes.append(check_for_iter_jump_to) for instr_pass in supported_passes: instr_pass(instrs, code_options) @@ -38,7 +49,7 @@ def apply_instr_pass(instrs, code_options): log_do(4, lambda: print(instrs_info(instrs))) -def find_stored_once_local_vars(instrs, code_options): +def find_stored_once_local_vars(instrs: list[Instruction], code_options): """ find out the local var names which is only stored once """ @@ -61,13 +72,13 @@ def find_stored_once_local_vars(instrs, code_options): return stored_once -def find_loaded_once_local_vars(instrs, code_options): +def find_loaded_once_local_vars(instrs: list[Instruction], code_options): """ find out the local var names which is only stored once """ loaded_vars = {} for instr in instrs: - if instr.opname == "LOAD_FAST": + if instr.opname in ["LOAD_FAST", "LOAD_FAST_CHECK"]: if instr.argval in loaded_vars: loaded_vars[instr.argval] += 1 else: @@ -77,14 +88,14 @@ def find_loaded_once_local_vars(instrs, code_options): return loaded_once -def find_related_local_opcodes(instrs, code_options): +def find_related_local_opcodes(instrs: list[Instruction], code_options): """ - find out the opcode pairs consist with LOAD_FAST and STORE_FAST + find out the opcode pairs consist with LOAD_FAST and STORE_FAST and LOAD_FAST_CHECK """ stack = [] opcode_pairs = [] for instr in instrs: - if instr.opname == "LOAD_FAST": + if instr.opname in ["LOAD_FAST", "LOAD_FAST_CHECK"]: stack.append(instr) elif instr.opname == "STORE_FAST": if len(stack) > 0 and stack[-1] is not None: @@ -105,7 +116,7 @@ def find_related_local_opcodes(instrs, code_options): return opcode_pairs -def remove_load_store_pass(instrs, code_options): +def remove_load_store_pass(instrs: list[Instruction], code_options): """ This question is extremely complex, so we just simplify it as 'remove renames which is between var names who only stored once' @@ -158,7 +169,8 @@ def code_exist(opname, argval, instrs): if a_name != b_name: for instr in instrs: if ( - instr.opname in ("LOAD_FAST", "STORE_FAST") + instr.opname + in ("LOAD_FAST_CHECK", "LOAD_FAST", "STORE_FAST") and instr.argval == b_name ): instr.argval = a_name @@ -211,7 +223,13 @@ def code_exist(opname, argval, instrs): code_range = instrs[last_store_idx : instrs.index(store_b)] if ( not code_exist("STORE_FAST", b_name, code_range) + and not code_exist("LOAD_FAST_CHECK", b_name, code_range) and not code_exist("LOAD_FAST", b_name, code_range) + and not code_exist( + "LOAD_FAST_CHECK", + a_name, + instrs[instrs.index(store_b) :], + ) and not code_exist( "LOAD_FAST", a_name, instrs[instrs.index(store_b) :] ) @@ -222,7 +240,8 @@ def code_exist(opname, argval, instrs): instrs.remove(store_b) for instr in instrs[last_store_idx:]: if ( - instr.opname in ("LOAD_FAST", "STORE_FAST") + instr.opname + in ("LOAD_FAST_CHECK", "LOAD_FAST", "STORE_FAST") and instr.argval == a_name ): instr.argval = b_name @@ -245,6 +264,7 @@ def code_exist(opname, argval, instrs): and opcode2 not in jump_target and opcode1.opname == "STORE_FAST" and opcode2.opname == "LOAD_FAST" + and opcode2.opname == "LOAD_FAST_CHECK" and opcode1.argval == opcode2.argval and opcode1.argval in loaded_once ): @@ -255,7 +275,7 @@ def code_exist(opname, argval, instrs): idx += 1 -def remove_duplicate_resume(instrs, code_options): +def remove_duplicate_resume(instrs: list[Instruction], code_options): resumes = list(filter(lambda instr: instr.opname == "RESUME", instrs)) if not resumes: return @@ -263,7 +283,7 @@ def remove_duplicate_resume(instrs, code_options): instrs.remove(resume) -def check_precall_followed_by_call(instrs, code_options): +def check_precall_followed_by_call(instrs: list[Instruction], code_options): """ PRECALL should be followed by CALL, otherwise it will cause a segmentation fault """ @@ -272,3 +292,14 @@ def check_precall_followed_by_call(instrs, code_options): raise InnerError( f"PRECALL is not followed by CALL in {code_options['co_name']}" ) + + +def check_for_iter_jump_to(instrs: list[Instruction], code_options): + """ + Check if the `jump_to` of FOR_ITER is END_FOR, in Python3.12+ + """ + for instr in instrs: + if instr.opname == "FOR_ITER": + assert instr.jump_to is not None + if instr.jump_to.opname != "END_FOR": + raise InnerError("FOR_ITER jump_to is not END_FOR") diff --git a/python/paddle/jit/sot/opcode_translator/instruction_utils/instruction_utils.py b/python/paddle/jit/sot/opcode_translator/instruction_utils/instruction_utils.py index 2965c8e6bc056..c30e21f8fb096 100644 --- a/python/paddle/jit/sot/opcode_translator/instruction_utils/instruction_utils.py +++ b/python/paddle/jit/sot/opcode_translator/instruction_utils/instruction_utils.py @@ -21,7 +21,13 @@ from typing import TYPE_CHECKING, Any from ...utils import InnerError -from .opcode_info import ABS_JUMP, ALL_JUMP, REL_BWD_JUMP, REL_JUMP +from .opcode_info import ( + ABS_JUMP, + ALL_JUMP, + PYOPCODE_CACHE_SIZE, + REL_BWD_JUMP, + REL_JUMP, +) if TYPE_CHECKING: import types @@ -239,7 +245,8 @@ def relocate_jump_target(instructions: list[Instruction]) -> None: if instr.opname in ABS_JUMP: new_arg = jump_target else: # instr.opname in REL_JUMP - new_arg = jump_target - instr.offset - 2 + cache_size = PYOPCODE_CACHE_SIZE.get(instr.opname, 0) + new_arg = jump_target - (2 * cache_size) - instr.offset - 2 if instr.opname in REL_BWD_JUMP: new_arg = -new_arg @@ -315,12 +322,12 @@ def bind_ex_arg_with_instr(ex_arg, instr): return modify_completed -def modify_vars(instructions, code_options): +def modify_vars(instructions: list[Instruction], code_options): co_names = code_options['co_names'] co_varnames = code_options['co_varnames'] co_freevars = code_options['co_freevars'] for instrs in instructions: - if instrs.opname == 'LOAD_FAST' or instrs.opname == 'STORE_FAST': + if instrs.opname in ['LOAD_FAST', 'LOAD_FAST_CHECK', 'STORE_FAST']: assert ( instrs.argval in co_varnames ), f"`{instrs.argval}` not in {co_varnames}" diff --git a/python/paddle/jit/sot/opcode_translator/instruction_utils/opcode_info.py b/python/paddle/jit/sot/opcode_translator/instruction_utils/opcode_info.py index 2dc69b7565672..d310f84993013 100644 --- a/python/paddle/jit/sot/opcode_translator/instruction_utils/opcode_info.py +++ b/python/paddle/jit/sot/opcode_translator/instruction_utils/opcode_info.py @@ -45,7 +45,7 @@ class PopJumpCond(Enum): NOT_NONE = "NOT_NONE" -def get_pyopcode_cache_size() -> dict[str, int]: +def _get_pyopcode_cache_size() -> dict[str, int]: if sys.version_info >= (3, 11) and sys.version_info < (3, 12): # Cache for some opcodes, it's for Python 3.11+ # https://github.com/python/cpython/blob/3.11/Include/internal/pycore_opcode.h#L41-L53 @@ -87,4 +87,4 @@ def get_pyopcode_cache_size() -> dict[str, int]: return {} -PYOPCODE_CACHE_SIZE = get_pyopcode_cache_size() +PYOPCODE_CACHE_SIZE = _get_pyopcode_cache_size() diff --git a/test/sot/skip_files_py312 b/test/sot/skip_files_py312 index 4d3ee9050ad6c..82cabe1866d19 100644 --- a/test/sot/skip_files_py312 +++ b/test/sot/skip_files_py312 @@ -1,9 +1,4 @@ ./test_11_jumps.py -./test_12_for_loop.py -./test_builtin_zip.py -./test_inplace_api.py -./test_min_graph_size.py ./test_side_effects.py -./test_sot_cost_model.py ./test_sot_resnet.py ./test_sot_resnet50_backward.py From 6ae38f7444a042312687cbf934cd82c03370a50b Mon Sep 17 00:00:00 2001 From: liuzhenhai93 Date: Mon, 4 Mar 2024 10:41:03 +0800 Subject: [PATCH 249/282] dynamic_to_static_global_norm_grad_clip_pass (#62285) --- python/paddle/distributed/passes/auto_parallel_grad_clip.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/paddle/distributed/passes/auto_parallel_grad_clip.py b/python/paddle/distributed/passes/auto_parallel_grad_clip.py index cc376ec009db2..02ab29c1ef3fa 100644 --- a/python/paddle/distributed/passes/auto_parallel_grad_clip.py +++ b/python/paddle/distributed/passes/auto_parallel_grad_clip.py @@ -38,6 +38,7 @@ insert_dependencies_for_vars, is_gradient_clip_op, is_optimize_op, + is_reshard_op, ) from .auto_parallel_sharding import ShardingPass from .pass_base import PassBase, register_pass @@ -431,7 +432,7 @@ def _remove_no_need_ops_vars(self, block): op.desc.set_input("X", reserved_vars) for idx, op in reversed(list(enumerate(block.ops))): - if not is_optimize_op(op): + if not (is_optimize_op(op) or is_reshard_op(op)): break if not is_gradient_clip_op(op): continue @@ -439,7 +440,7 @@ def _remove_no_need_ops_vars(self, block): block._remove_op(idx, sync=False) for idx, op in reversed(list(enumerate(block.ops))): - if not is_optimize_op(op): + if not (is_optimize_op(op) or is_reshard_op(op)): break if not is_gradient_clip_op(op): continue From 9fd6f7b3cdec6741719664fd590da4f98560a0d0 Mon Sep 17 00:00:00 2001 From: lzydev Date: Mon, 4 Mar 2024 10:41:28 +0800 Subject: [PATCH 250/282] change the decorate (#62276) --- python/paddle/amp/auto_cast.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/python/paddle/amp/auto_cast.py b/python/paddle/amp/auto_cast.py index 5a271171e09ce..3063b14b7e3be 100644 --- a/python/paddle/amp/auto_cast.py +++ b/python/paddle/amp/auto_cast.py @@ -737,13 +737,11 @@ def amp_decorate( for opt in optimizers: _set_multi_precision(opt, use_multi_precision) - # support master_grad - if master_grad: - amp_global_state().use_master_grad = True - for idx in range(len(models)): - amp_global_state().model_parameters.extend( - models[idx].parameters() - ) + # support master_grad + if master_grad: + amp_global_state().use_master_grad = True + for idx in range(len(models)): + amp_global_state().model_parameters.extend(models[idx].parameters()) if save_dtype is not None: if save_dtype not in ['float16', 'bfloat16', 'float32', 'float64']: From 492615f515e0939521119ce91ac295a7cb98634d Mon Sep 17 00:00:00 2001 From: Roc <30228238+sljlp@users.noreply.github.com> Date: Mon, 4 Mar 2024 10:51:27 +0800 Subject: [PATCH 251/282] add kernel for fused_layernorm (#62228) --- paddle/phi/backends/xpu/xpu2_op_list.cc | 2 + .../fusion/xpu/fused_layernorm_kernel.cc | 177 ++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 paddle/phi/kernels/fusion/xpu/fused_layernorm_kernel.cc diff --git a/paddle/phi/backends/xpu/xpu2_op_list.cc b/paddle/phi/backends/xpu/xpu2_op_list.cc index 14d761a1f1479..ae67044b5ca28 100644 --- a/paddle/phi/backends/xpu/xpu2_op_list.cc +++ b/paddle/phi/backends/xpu/xpu2_op_list.cc @@ -1174,6 +1174,8 @@ XPUOpMap& get_kl2_ops() { XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, {"fused_gemm_epilogue_grad", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, + {"fused_bias_residual_layernorm", + XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, {"fused_attention", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, {"fused_attention_grad", diff --git a/paddle/phi/kernels/fusion/xpu/fused_layernorm_kernel.cc b/paddle/phi/kernels/fusion/xpu/fused_layernorm_kernel.cc new file mode 100644 index 0000000000000..833caa6688787 --- /dev/null +++ b/paddle/phi/kernels/fusion/xpu/fused_layernorm_kernel.cc @@ -0,0 +1,177 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/backends/xpu/enforce_xpu.h" +#include "paddle/phi/common/amp_type_traits.h" +#include "paddle/phi/core/kernel_registry.h" + +namespace phi { + +namespace fusion { + +template +void FusedLayerNormKernel(const Context& dev_ctx, + const DenseTensor& x, + const paddle::optional& bias, + const paddle::optional& residual, + const paddle::optional& norm_weight, + const paddle::optional& norm_bias, + const float epsilon, + const float residual_alpha, + const int begin_norm_axis, + const float quant_scale, + const int quant_round_type, + const float quant_max_bound, + const float quant_min_bound, + DenseTensor* out, + DenseTensor* residual_out, + DenseTensor* mean, + DenseTensor* variance) { + int r = xpu::SUCCESS; + auto xpu_ctx = static_cast(&dev_ctx); + using XPUType = typename XPUTypeTrait::Type; + auto x_shape = x.dims(); + int m = 1; + int n = 1; + for (int i = 0; i < begin_norm_axis; i++) { + m *= x_shape[i]; + } + for (int i = begin_norm_axis; i < x_shape.size(); i++) { + n *= x_shape[i]; + } + + dev_ctx.template Alloc(out); + dev_ctx.template Alloc(mean); + dev_ctx.template Alloc(variance); + + DenseTensor residual_alpha_tmp; + residual_alpha_tmp.Resize({1}); + + DenseTensor residual_alpha_ptr; + residual_alpha_ptr.Resize({1}); + + dev_ctx.template Alloc(&residual_alpha_tmp); + dev_ctx.template Alloc(&residual_alpha_ptr); + + r = baidu::xpu::api::constant(xpu_ctx->x_context(), + residual_alpha_tmp.data(), + 1, + residual_alpha); + PADDLE_ENFORCE_XDNN_SUCCESS(r, "constant"); + + r = baidu::xpu::api::cast_v2( + xpu_ctx->x_context(), + residual_alpha_tmp.data(), + reinterpret_cast(residual_alpha_ptr.data()), + 1); + PADDLE_ENFORCE_XDNN_SUCCESS(r, "cast_v2"); + + if (residual) { + dev_ctx.template Alloc(residual_out); + r = baidu::xpu::api::broadcast_mul( + xpu_ctx->x_context(), + reinterpret_cast(residual.get().data()), + reinterpret_cast(residual_alpha_ptr.data()), + reinterpret_cast(const_cast(residual.get().data())), + {m, n}, + {1}); + PADDLE_ENFORCE_XDNN_SUCCESS(r, "broadcast_mul"); + } + + if (!norm_weight && !norm_bias) { + if (bias) { + r = baidu::xpu::api::broadcast_add( + xpu_ctx->x_context(), + reinterpret_cast(out->data()), + reinterpret_cast(bias.get().data()), + reinterpret_cast(out->data()), + {m, n}, + {n}); + PADDLE_ENFORCE_XDNN_SUCCESS(r, "broadcast_add"); + } + if (residual) { + r = baidu::xpu::api::add( + xpu_ctx->x_context(), + reinterpret_cast(out->data()), + reinterpret_cast(residual.get().data()), + reinterpret_cast(out->data()), + m * n); + PADDLE_ENFORCE_XDNN_SUCCESS(r, "add"); + } + + r = baidu::xpu::api::add(xpu_ctx->x_context(), + reinterpret_cast(out->data()), + reinterpret_cast(x.data()), + reinterpret_cast(out->data()), + m * n); + PADDLE_ENFORCE_XDNN_SUCCESS(r, "add"); + return; + } else { + if (bias) { + r = baidu::xpu::api::broadcast_add( + xpu_ctx->x_context(), + reinterpret_cast(x.data()), + reinterpret_cast(bias.get().data()), + reinterpret_cast(const_cast((x.data()))), + {m, n}, + {n}); + PADDLE_ENFORCE_XDNN_SUCCESS(r, "broadcast_add"); + } + if (residual) { + r = baidu::xpu::api::add_layer_norm_fusion( + xpu_ctx->x_context(), + reinterpret_cast(x.data()), + reinterpret_cast(residual.get().data()), + reinterpret_cast(out->data()), + m, + n, + epsilon, + norm_weight.get().data(), + norm_bias.get().data(), + mean->data(), + variance->data(), + reinterpret_cast(residual_out->data())); + PADDLE_ENFORCE_XDNN_SUCCESS(r, "add_layer_norm_fusion"); + } else { + r = baidu::xpu::api::layer_norm( + xpu_ctx->x_context(), + reinterpret_cast(x.data()), + reinterpret_cast(out->data()), + m, + n, + epsilon, + norm_weight.get().data(), + norm_bias.get().data(), + mean->data(), + variance->data()); + PADDLE_ENFORCE_XDNN_SUCCESS(r, "layer_norm"); + } + if (quant_scale > 0.0f) { + PD_THROW("NOT supported quant int8. "); + } else { + return; + } + } +} + +} // namespace fusion + +} // namespace phi + +PD_REGISTER_KERNEL(fused_bias_residual_layernorm, + XPU, + ALL_LAYOUT, + phi::fusion::FusedLayerNormKernel, + float, + phi::dtype::float16) {} From 3716973068b4a5c3044c31105220125e29480557 Mon Sep 17 00:00:00 2001 From: Roc <30228238+sljlp@users.noreply.github.com> Date: Mon, 4 Mar 2024 10:52:55 +0800 Subject: [PATCH 252/282] [XPU] add xpu kernel for fused_bias_act (#62232) --- paddle/phi/backends/xpu/xpu2_op_list.cc | 2 + .../fusion/xpu/fused_bias_act_kernel.cc | 138 ++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 paddle/phi/kernels/fusion/xpu/fused_bias_act_kernel.cc diff --git a/paddle/phi/backends/xpu/xpu2_op_list.cc b/paddle/phi/backends/xpu/xpu2_op_list.cc index ae67044b5ca28..171894b9b9f6f 100644 --- a/paddle/phi/backends/xpu/xpu2_op_list.cc +++ b/paddle/phi/backends/xpu/xpu2_op_list.cc @@ -1180,6 +1180,8 @@ XPUOpMap& get_kl2_ops() { XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, {"fused_attention_grad", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, + {"fused_bias_act", + XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, {"fused_feedforward", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, {"fused_feedforward_grad", diff --git a/paddle/phi/kernels/fusion/xpu/fused_bias_act_kernel.cc b/paddle/phi/kernels/fusion/xpu/fused_bias_act_kernel.cc new file mode 100644 index 0000000000000..d36d7416a023a --- /dev/null +++ b/paddle/phi/kernels/fusion/xpu/fused_bias_act_kernel.cc @@ -0,0 +1,138 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/backends/xpu/enforce_xpu.h" +#include "paddle/phi/backends/xpu/xpu_context.h" +#include "paddle/phi/core/dense_tensor.h" +#include "paddle/phi/core/kernel_registry.h" + +namespace phi { +namespace fusion { + +template +static void DispatchComputeImpl(const phi::XPUContext *xpu_ctx, + const DenseTensor &x, + const DenseTensor *bias, + const DenseTensor &dequant_scales, + const DenseTensor &shift, + const DenseTensor &smooth, + const std::string &act_method, + const float quant_scale, + const int quant_round_type, + const float quant_max_bound, + const float quant_min_bound, + DenseTensor *out) { + PADDLE_THROW( + phi::errors::Unimplemented("fused_bias_act with smooth " + "quant on xpu is not implemented yet.")); +} + +template +static void ComputeImpl(const phi::XPUContext *xpu_ctx, + const DenseTensor &x, + const paddle::optional &bias, + const std::string &act_method, + DenseTensor *out) { + using XPUType = typename XPUTypeTrait::Type; + int rows = x.dims()[0]; + int cols = x.dims()[1]; + int r = 0; + if (bias) { + r = baidu::xpu::api::broadcast_add( + xpu_ctx->x_context(), + reinterpret_cast(x.data()), + reinterpret_cast(bias.get().data()), + reinterpret_cast(const_cast(x.data())), + {rows, cols}, + {1, cols}); + PD_CHECK(r == 0, "baidu::xpu::api::broadcast_add failed."); + } + if (act_method == "geglu") { + PD_THROW( + "NOT supported GeGLU. " + "Currently Only Support SwiGLU, GeLU, ReLU"); + } else if (act_method == "swiglu") { + r = baidu::xpu::api::swiglu( + xpu_ctx->x_context(), + reinterpret_cast(x.data()), + reinterpret_cast(out->data()), + {rows, cols}, + 1, + true); + PD_CHECK(r == 0, "baidu::xpu::api::swiglu failed."); + } else if (act_method == "gelu") { + r = baidu::xpu::api::gelu( + xpu_ctx->x_context(), + reinterpret_cast(x.data()), + reinterpret_cast(out->data()), + rows * cols); + PD_CHECK(r == 0, "baidu::xpu::api::gelu failed."); + } else if (act_method == "relu") { + r = baidu::xpu::api::relu( + xpu_ctx->x_context(), + reinterpret_cast(x.data()), + reinterpret_cast(out->data()), + rows * cols); + PD_CHECK(r == 0, "baidu::xpu::api::relu failed."); + } else { + PD_THROW( + "NOT supported. " + "Currently Only Support SwiGLU, GeLU, ReLU"); + } +} + +template +void FusedBiasActKernel(const Context &dev_ctx, + const DenseTensor &x, + const paddle::optional &bias, + const paddle::optional &dequant_scales, + const paddle::optional &shift, + const paddle::optional &smooth, + const std::string &act_method, + const std::string &compute_dtype, + float quant_scale, + int quant_round_type, + float quant_max_bound, + float quant_min_bound, + DenseTensor *out) { + auto xpu_ctx = static_cast(&dev_ctx); + dev_ctx.template Alloc(out); + + if (dequant_scales && dequant_scales.get().numel() > 0) { + return DispatchComputeImpl(xpu_ctx, + x, + bias ? &(bias.get()) : nullptr, + dequant_scales.get(), + shift.get(), + smooth.get(), + act_method, + quant_scale, + quant_round_type, + quant_max_bound, + quant_min_bound, + out); + } else { + return ComputeImpl(xpu_ctx, x, bias, act_method, out); + } +} + +} // namespace fusion +} // namespace phi + +PD_REGISTER_KERNEL(fused_bias_act, + XPU, + ALL_LAYOUT, + phi::fusion::FusedBiasActKernel, + float, + phi::dtype::float16) {} From ab7acef4043604afff1bb1f26f55b7a2a6fd6308 Mon Sep 17 00:00:00 2001 From: NeroLoh <745827440@qq.com> Date: Mon, 4 Mar 2024 10:53:57 +0800 Subject: [PATCH 253/282] [xpu]strided slice op support reverse stride (#62268) --- paddle/phi/kernels/xpu/stride_slice_kernel.cc | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/paddle/phi/kernels/xpu/stride_slice_kernel.cc b/paddle/phi/kernels/xpu/stride_slice_kernel.cc index 5aee59729b52e..22562cbf6b29c 100644 --- a/paddle/phi/kernels/xpu/stride_slice_kernel.cc +++ b/paddle/phi/kernels/xpu/stride_slice_kernel.cc @@ -66,15 +66,10 @@ void StridedSliceRawKernel(const Context& dev_ctx, int num = axes.size(); for (int i = 0; i < num; ++i) { - PADDLE_ENFORCE_EQ( - strides_[i] > 0, - true, - errors::InvalidArgument("Currently, XPU strided slice kernel does not ", - "support reverse strided slice.")); int cur_axe = axes[i]; int st = starts_[i]; if (st > xshape[cur_axe]) { - st = xshape[cur_axe]; + st = xshape[cur_axe] - 1; } if (st < 0) { st += xshape[cur_axe]; @@ -86,17 +81,12 @@ void StridedSliceRawKernel(const Context& dev_ctx, end = xshape[cur_axe]; } if (end < 0) { - end += xshape[cur_axe]; + if (strides_[i] > 0) { + end += xshape[cur_axe]; + } } ends_in[cur_axe] = end; - PADDLE_ENFORCE_EQ( - st < end, - true, - errors::InvalidArgument("End index should be larger than", - "start Index, this OP does not support", - "reverse operator.")); - strides_in[cur_axe] = strides_[i]; } From 476403b570fdcf97df8b60b4b5eb1b778a6b3342 Mon Sep 17 00:00:00 2001 From: cyberslack_lee Date: Mon, 4 Mar 2024 11:17:09 +0800 Subject: [PATCH 254/282] =?UTF-8?q?=E3=80=90PIR=20Dist=20Op=20Reg=20No.7?= =?UTF-8?q?=E3=80=8123=E3=80=91=20reg=20c=5Freduce=5Fprod=20c=5Freduce=5Fm?= =?UTF-8?q?ax=20(#62270)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix * fix * add reduce_max --- .../pir/dialect/op_generator/ops_api_gen.py | 4 ++ paddle/fluid/pir/dialect/operator/ir/ops.yaml | 20 +++++++++ .../fluid/pir/dialect/operator/utils/utils.cc | 2 + paddle/phi/api/yaml/op_compat.yaml | 12 ++++++ test/ir/pir/translator/CMakeLists.txt | 2 + .../test_c_reduce_max_translator.py | 42 +++++++++++++++++++ .../test_c_reduce_prod_translator.py | 42 +++++++++++++++++++ 7 files changed, 124 insertions(+) create mode 100644 test/ir/pir/translator/test_c_reduce_max_translator.py create mode 100644 test/ir/pir/translator/test_c_reduce_prod_translator.py diff --git a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py index 534ea49a61f45..2cbcb29f705b3 100644 --- a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py @@ -158,8 +158,12 @@ 'soft_relu', 'uniform_random_batch_size_like', 'match_matrix_tensor', + 'c_reduce_max', + 'c_reduce_max_', 'c_reduce_min', 'c_reduce_min_', + 'c_reduce_prod', + 'c_reduce_prod_', 'push_sparse_v2', 'push_sparse_v2_', 'partial_send', diff --git a/paddle/fluid/pir/dialect/operator/ir/ops.yaml b/paddle/fluid/pir/dialect/operator/ir/ops.yaml index 7e05e5b79de8d..d856c58a75550 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops.yaml @@ -218,6 +218,16 @@ func : c_identity inplace : (x -> out) +- op : c_reduce_max + args : (Tensor x, int ring_id, int root_id, bool use_calc_stream) + output : Tensor(out) + infer_meta : + func : DistReduceInferMeta + param : [x] + kernel : + func : c_reduce_max + inplace : (x -> out) + - op : c_reduce_min args : (Tensor x, int ring_id, int root_id, bool use_calc_stream) output : Tensor(out) @@ -228,6 +238,16 @@ func : c_reduce_min inplace : (x -> out) +- op : c_reduce_prod + args : (Tensor x, int ring_id, int root_id, bool use_calc_stream) + output : Tensor(out) + infer_meta : + func : DistReduceInferMeta + param : [x] + kernel : + func : c_reduce_prod + inplace : (x -> out) + - op : c_reduce_sum args : (Tensor x, int ring_id, int root_id, bool use_calc_stream) output : Tensor(out) diff --git a/paddle/fluid/pir/dialect/operator/utils/utils.cc b/paddle/fluid/pir/dialect/operator/utils/utils.cc index 931c7d4b33624..c17a7fb6839cc 100644 --- a/paddle/fluid/pir/dialect/operator/utils/utils.cc +++ b/paddle/fluid/pir/dialect/operator/utils/utils.cc @@ -86,7 +86,9 @@ const std::unordered_set LegacyOpList = { paddle::onednn::dialect::MultiGruOp::name(), paddle::onednn::dialect::FusionLstmOp::name(), #endif + CReduceMaxOp::name(), CReduceMinOp::name(), + CReduceProdOp::name(), PushSparseV2Op::name(), PartialSendOp::name()}; diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml index 840ce5ef29de3..44a66c60e8078 100755 --- a/paddle/phi/api/yaml/op_compat.yaml +++ b/paddle/phi/api/yaml/op_compat.yaml @@ -3549,12 +3549,24 @@ outputs : out: Out +- op: c_reduce_max + inputs : + x : X + outputs : + out: Out + - op: c_reduce_min inputs : x : X outputs : out: Out +- op: c_reduce_prod + inputs : + x : X + outputs : + out: Out + - op: c_reduce_sum inputs : x : X diff --git a/test/ir/pir/translator/CMakeLists.txt b/test/ir/pir/translator/CMakeLists.txt index 2dd89d3406c92..76820d1a9a153 100644 --- a/test/ir/pir/translator/CMakeLists.txt +++ b/test/ir/pir/translator/CMakeLists.txt @@ -10,6 +10,8 @@ list(APPEND DISTRIBUTED_OP_TRANSLATOR_TEST test_c_allreduce_prod_translator) list(APPEND DISTRIBUTED_OP_TRANSLATOR_TEST test_distributed_lookup_table_translate) list(APPEND DISTRIBUTED_OP_TRANSLATOR_TEST test_partial_send_translator) +list(APPEND DISTRIBUTED_OP_TRANSLATOR_TEST test_c_reduce_max_translator) +list(APPEND DISTRIBUTED_OP_TRANSLATOR_TEST test_c_reduce_prod_translator) if(NOT WITH_DISTRIBUTE) list(REMOVE_ITEM TEST_INTERP_CASES ${DISTRIBUTED_OP_TRANSLATOR_TEST}) diff --git a/test/ir/pir/translator/test_c_reduce_max_translator.py b/test/ir/pir/translator/test_c_reduce_max_translator.py new file mode 100644 index 0000000000000..c40624ad74fbb --- /dev/null +++ b/test/ir/pir/translator/test_c_reduce_max_translator.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import test_op_translator + +import paddle +from paddle.base.layer_helper import LayerHelper + + +class TestCReduceMaxOpTranslator(test_op_translator.TestOpTranslator): + def append_op(self): + self.op_type = "c_reduce_max" + x = paddle.ones(shape=(100, 2, 3), dtype='float32') + y = paddle.ones(shape=(100, 2, 3), dtype='float32') + attrs = {'ring_id': 0, 'root_id': 0, 'use_calc_stream': False} + helper = LayerHelper(self.op_type) + helper.append_op( + type=self.op_type, + inputs={"X": x}, + outputs={"Out": y}, + attrs=attrs, + ) + + def test_translator(self): + self.check() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/ir/pir/translator/test_c_reduce_prod_translator.py b/test/ir/pir/translator/test_c_reduce_prod_translator.py new file mode 100644 index 0000000000000..34caa22d77b9f --- /dev/null +++ b/test/ir/pir/translator/test_c_reduce_prod_translator.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import test_op_translator + +import paddle +from paddle.base.layer_helper import LayerHelper + + +class TestCReduceProdOpTranslator(test_op_translator.TestOpTranslator): + def append_op(self): + self.op_type = "c_reduce_prod" + x = paddle.ones(shape=(100, 2, 3), dtype='float32') + y = paddle.ones(shape=(100, 2, 3), dtype='float32') + attrs = {'ring_id': 0, 'root_id': 0, 'use_calc_stream': False} + helper = LayerHelper(self.op_type) + helper.append_op( + type=self.op_type, + inputs={"X": x}, + outputs={"Out": y}, + attrs=attrs, + ) + + def test_translator(self): + self.check() + + +if __name__ == "__main__": + unittest.main() From 98fcb19ab828ea486b0242e1665e8dc68645eace Mon Sep 17 00:00:00 2001 From: Bo Zhang <105368690+zhangbopd@users.noreply.github.com> Date: Mon, 4 Mar 2024 11:21:07 +0800 Subject: [PATCH 255/282] [PIR][DynamicShape] Fix Expand Op's and Full_With_Tensor OP 's InferSymShap (#62326) * rm expand from yaml * fix expand && full_with_tensor --- .../paddle_op_infer_sym.cc | 21 +++++++++++------ .../paddle_op_infer_sym.h | 6 ++--- .../same_operands_and_result.cc | 5 +--- .../same_operands_and_result.h | 2 -- .../dialect/operator/ir/manual_onednn_op.cc | 6 ++--- .../pir/dialect/operator/ir/manual_op.cc | 23 ++++++++++++++----- paddle/phi/api/yaml/ops.yaml | 1 - 7 files changed, 37 insertions(+), 27 deletions(-) diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc index d7ee4fb6781b0..4b31c94280ed2 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc @@ -983,13 +983,6 @@ bool SparseWeightEmbeddingOpInferSymbolicShape( return true; } -bool ExpandOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} - bool MatmulOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { // x_dims can't be const or ref here, in case to be broadcasted @@ -1494,4 +1487,18 @@ bool UniqueOpInferSymbolicShape( return true; } +bool FullWithTensorOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + pir::Value operand_source = op->operand_source(0); + const symbol::ShapeOrDataDimExprs &operand_shape_or_data = + shape_analysis->GetShapeOrDataForValue(operand_source); + + const auto &out_shape = operand_shape_or_data.data().has_value() + ? operand_shape_or_data.data().value() + : operand_shape_or_data.shape(); + + shape_analysis->SetShapeOrDataForValue( + op->result(0), symbol::TensorShapeOrDataDimExprs(out_shape)); + return true; +} } // namespace paddle::dialect diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h index f23e84c27f55d..f46128a34d0d3 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h @@ -82,9 +82,6 @@ bool EmbeddingOpInferSymbolicShape( bool SparseWeightEmbeddingOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool ExpandOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); - bool MatmulOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); @@ -205,5 +202,6 @@ bool UniformOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool UniqueOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - +bool FullWithTensorOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); } // namespace paddle::dialect diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.cc index 68ca785e0fbb0..bb540647d0219 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.cc @@ -210,10 +210,7 @@ bool Floor_OpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); } -bool FullWithTensorOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - return SameOperandsAndResultShape(op, shape_analysis); -} + bool ImagOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { return SameOperandsAndResultShape(op, shape_analysis); diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h index c671d9da22818..e82223c812585 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/same_operands_and_result.h @@ -109,8 +109,6 @@ bool FloorOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool Floor_OpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool FullWithTensorOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool ImagOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool IncrementOpInferSymbolicShape( diff --git a/paddle/fluid/pir/dialect/operator/ir/manual_onednn_op.cc b/paddle/fluid/pir/dialect/operator/ir/manual_onednn_op.cc index 352677f0047c8..a66d4d8eb8b51 100644 --- a/paddle/fluid/pir/dialect/operator/ir/manual_onednn_op.cc +++ b/paddle/fluid/pir/dialect/operator/ir/manual_onednn_op.cc @@ -18,7 +18,6 @@ paddle::onednn::dialect::ExpandOp #include "paddle/fluid/pir/dialect/operator/ir/manual_onednn_op.h" #include "paddle/fluid/pir/dialect/kernel/ir/kernel_type.h" -#include "paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h" #include "paddle/fluid/pir/dialect/operator/ir/ir_meta_tensor.h" #include "paddle/fluid/pir/dialect/operator/ir/ir_selected_rows.h" #include "paddle/fluid/pir/dialect/operator/ir/ir_tensor.h" @@ -334,8 +333,9 @@ phi::DataType ExpandOp::GetKernelTypeForVar( bool ExpandOp::InferSymbolicShape( pir::ShapeConstraintIRAnalysis* shape_analysis) { VLOG(4) << "Infer symbolic shape for op: ExpandOp"; - return paddle::dialect::ExpandOpInferSymbolicShape(this->operation(), - shape_analysis); + PADDLE_THROW(phi::errors::Unimplemented( + " ExpandOp's InferSymbolicShape interface is NOT implemented now.")); + return true; } } // namespace dialect diff --git a/paddle/fluid/pir/dialect/operator/ir/manual_op.cc b/paddle/fluid/pir/dialect/operator/ir/manual_op.cc index b7cebeaf27f47..5a930b04fdf64 100644 --- a/paddle/fluid/pir/dialect/operator/ir/manual_op.cc +++ b/paddle/fluid/pir/dialect/operator/ir/manual_op.cc @@ -3276,8 +3276,8 @@ void ExpandOp::Build(pir::Builder &builder, bool ExpandOp::InferSymbolicShape( pir::ShapeConstraintIRAnalysis *shape_analysis) { - const auto x_shape_or_data = shape_analysis->GetShapeOrDataForValue(x()); - const auto expand_shape_shape_or_data = + const auto &x_shape_or_data = shape_analysis->GetShapeOrDataForValue(x()); + const auto &expand_shape_shape_or_data = shape_analysis->GetShapeOrDataForValue(shape()); const std::vector &x_dims = [&] { @@ -3292,12 +3292,23 @@ bool ExpandOp::InferSymbolicShape( const std::vector &expand_shape = [&] { std::vector dims; - if (expand_shape_shape_or_data.data().has_value()) { - dims = expand_shape_shape_or_data.data().value(); + + if (expand_shape_shape_or_data + .isa()) { + const auto &dims_list = + expand_shape_shape_or_data + .dyn_cast(); + for (const auto &shape_data : dims_list) { + const auto &dim_expr = shape_data.data().has_value() + ? shape_data.data().value()[0] + : shape_data.shape()[0]; + dims.emplace_back(dim_expr); + } } else { - dims = expand_shape_shape_or_data.shape(); + dims = expand_shape_shape_or_data.data().has_value() + ? expand_shape_shape_or_data.data().value() + : expand_shape_shape_or_data.shape(); } - if (dims.empty()) { dims = std::vector(x_dims.size(), -1); } diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml index 5b8d2132c519d..5156073182e67 100755 --- a/paddle/phi/api/yaml/ops.yaml +++ b/paddle/phi/api/yaml/ops.yaml @@ -941,7 +941,6 @@ func : expand data_type : x backward : expand_grad - interfaces : paddle::dialect::InferSymbolicShapeInterface - op : expand_as args : (Tensor x, Tensor y, int[] target_shape = {}) From 3ca79b620a1c1890e78ebd1ac67307d5bb608632 Mon Sep 17 00:00:00 2001 From: LiYuRio <63526175+LiYuRio@users.noreply.github.com> Date: Mon, 4 Mar 2024 11:21:52 +0800 Subject: [PATCH 256/282] make sharding dynamic to static (#62230) --- .../paddle/distributed/auto_parallel/api.py | 127 +++++++++++++++--- .../semi_auto_parallel_sharding_stage_1.py | 32 ++++- .../semi_auto_parallel_sharding_stage_3.py | 30 +++++ .../semi_auto_parallel_dist_to_static_api.py | 17 +-- .../semi_auto_parallel_sharding_stage_1.py | 27 +++- .../semi_auto_parallel_sharding_stage_3.py | 25 ++++ 6 files changed, 230 insertions(+), 28 deletions(-) diff --git a/python/paddle/distributed/auto_parallel/api.py b/python/paddle/distributed/auto_parallel/api.py index c63f8ce3a58c9..45eb7c8c2491c 100644 --- a/python/paddle/distributed/auto_parallel/api.py +++ b/python/paddle/distributed/auto_parallel/api.py @@ -551,15 +551,15 @@ def replicate_layer_params_and_buffers( ) -def get_placement_with_sharding(param): +def get_placement_with_sharding(param, sharding_mesh_axis): shard_axis = -1 for placement in param.placements: if isinstance(placement, dist.Shard): - # the parameter can't be shard twice on different mesh now - # assert here in case + # the parameter can't be shard twice with sharding on different mesh now + # for example, [Shard(0), Shard(1)], assert here in case assert ( shard_axis == -1 - ), "The parameter can't be shard twich even in different mesh now." + ), "The parameter can't be shard twice even in different mesh now." shard_axis = placement.get_dim() placement_with_sharding = None @@ -568,14 +568,8 @@ def get_placement_with_sharding(param): placement_with_sharding = dist.Shard(dim) new_placements = param.placements - for mesh_axis, placement in enumerate(param.placements): - # we need to keep the placement replicate if the it is out of tensor's dim - if ( - isinstance(placement, dist.Replicate) - and placement_with_sharding is not None - ): - new_placements[mesh_axis] = placement_with_sharding - break + if placement_with_sharding is not None: + new_placements[sharding_mesh_axis] = placement_with_sharding return new_placements @@ -604,14 +598,61 @@ def __init__(self, optimizer, shard_fn=None): self._shard_clip = True self._inner_opt = optimizer self._shard_fn = shard_fn + self._sharding_mesh_axis = None + self._sharding_degree = None - # Invoke shard_fn if it is not None to shard parameters - if self._shard_fn is not None and isinstance( - self._shard_fn, ShardingStage3 - ): + if isinstance(self._shard_fn, (ShardingStage1, ShardingStage3)): + self._set_and_check_sharding_prop_from_param() + self._shard_fn._set_sharding_mesh_axis(self._sharding_mesh_axis) + + # Invoke shard_parameter in sharding stage 3 strategy + if isinstance(self._shard_fn, ShardingStage3): for param in self._inner_opt._parameter_list: self._shard_fn._shard_parameter(param) + def _set_and_check_sharding_prop_from_param(self): + if len(self._shard_fn._mesh._shape) == 1: + self._sharding_degree = self._shard_fn._mesh.get_dim_size(0) + self._sharding_mesh_axis = 0 + else: + param_list = self._inner_opt._parameter_list + for param in param_list: + if not param.is_dist(): + continue + mesh = param.process_mesh + placements = param.placements + + if self._sharding_degree is None: + # set the sharding degree if it has not been set + if any( + isinstance(placement, dist.Shard) + for placement in placements + ): + for idx, placement in enumerate(placements): + if isinstance(placement, dist.Replicate): + self._sharding_degree = mesh.dim_size(idx) + self._sharding_mesh_axis = idx + break + else: + # check the placement on sharding axis is Replicate + assert isinstance( + placements[self._sharding_mesh_axis], dist.Replicate + ), "The placement on sharding_mesh_axis should be Replicate" + # check the sharding degree since it has already been set + if any( + isinstance(placement, dist.Shard) + for placement in placements + ): + for idx, placement in enumerate(placements): + if isinstance(placement, dist.Replicate): + assert ( + mesh.dim_size(idx) == self._sharding_degree + ), "The sharding degree of all parameters must be equal currently." + + assert ( + self._sharding_degree is not None + ), "The sharding degree is None in ShardOptimizer" + def _shard_accumulator(self, param): # create the accumulators self._inner_opt._create_accumulators(self.target_block, [param]) @@ -804,11 +845,17 @@ class ShardingStage1: >>> # python -m paddle.distributed.launch --gpus=0,1 {test_case}.py """ + def __init__(self, mesh): + self._mesh = mesh + self._sharding_mesh_axis = None + def __call__(self, key, param, accumulator): if param.is_dist(): # Only deal with momentum in optimizer, beta should be replicated cross param's mesh if 'beta' not in key: - placements = get_placement_with_sharding(param) + placements = get_placement_with_sharding( + param, self._sharding_mesh_axis + ) else: placements = [ dist.Replicate() @@ -821,6 +868,9 @@ def __call__(self, key, param, accumulator): ) return accumulator + def _set_sharding_mesh_axis(self, sharding_mesh_axis): + self._sharding_mesh_axis = sharding_mesh_axis + class ShardingStage3: """ @@ -862,6 +912,10 @@ class ShardingStage3: def __init__(self, mesh): self._mesh = mesh + self._sharding_mesh_axis = None + + def _set_sharding_mesh_axis(self, sharding_mesh_axis): + self._sharding_mesh_axis = sharding_mesh_axis def _shard_parameter(self, param): if param.is_dense(): @@ -870,11 +924,21 @@ def _shard_parameter(self, param): placements.append(dist.Replicate()) param._to_dist_(placements, self._mesh) - new_placements = get_placement_with_sharding(param) + new_placements = get_placement_with_sharding( + param, self._sharding_mesh_axis + ) shard_param = dist.reshard(param, param.process_mesh, new_placements) # change the holder of param to new shard_param param.get_tensor()._share_data_with(shard_param.get_tensor()) + def _unshard_parameter(self, param): + new_placements = param.placements + if isinstance(new_placements[self._sharding_mesh_axis], dist.Shard): + new_placements[self._sharding_mesh_axis] = dist.Replicate() + + new_param = dist.reshard(param, param.process_mesh, new_placements) + param.get_tensor()._share_data_with(new_param.get_tensor()) + def __call__(self, key, param, accumulator): if param.is_dist(): # Only deal with momentum in optimizer, beta should be replicated cross param's mesh @@ -1893,8 +1957,35 @@ def to_static( >>> # python -m paddle.distributed.launch {test_case}.py """ if isinstance(optimizer, _ShardOptimizer): + shard_fn = optimizer._shard_fn + sharding_degree = optimizer._sharding_degree optimizer = optimizer._inner_opt + if shard_fn is not None: + strategy = dist.Strategy() if strategy is None else strategy + + # Deduce sharding degree for static + # Note: Because limitation of architecture, we need to ensure that + # all parameters are sharded by the same mesh axis + assert ( + sharding_degree is not None + ), "Sharding degree can not be None." + + if isinstance(shard_fn, ShardingStage1): + strategy.sharding.enable = True + strategy.sharding.stage = 1 + strategy.sharding.degree = sharding_degree + elif isinstance(shard_fn, ShardingStage3): + strategy.sharding.enable = True + strategy.sharding.stage = 3 + strategy.sharding.degree = sharding_degree + for param in optimizer._parameter_list: + shard_fn._unshard_parameter(param) + else: + raise NotImplementedError( + "Only sharding stage 1 and 3 can to_static for now. User-defined shard_fn and sharding stage 2 will be supported later." + ) + dist_model = DistModel(layer, loader, loss, optimizer, strategy) return dist_model diff --git a/test/auto_parallel/hybrid_strategy/semi_auto_parallel_sharding_stage_1.py b/test/auto_parallel/hybrid_strategy/semi_auto_parallel_sharding_stage_1.py index 10b53fa0f443c..6a8c8513f5450 100644 --- a/test/auto_parallel/hybrid_strategy/semi_auto_parallel_sharding_stage_1.py +++ b/test/auto_parallel/hybrid_strategy/semi_auto_parallel_sharding_stage_1.py @@ -15,9 +15,14 @@ import os import numpy as np +from auto_parallel.semi_auto_parallel_dist_to_static_api import ( + DemoNet, + create_data_loader, +) import paddle import paddle.distributed as dist +from paddle import nn class TestSemiAutoParallelShardingStage1: @@ -59,7 +64,7 @@ def test_sharding_stage_1_with_mp(self): batch = dist.shard_tensor(batch, self._mesh, [dist.Shard(0)]) # shard optimizer with stage 1 fn opt = paddle.optimizer.AdamW(parameters=linear.parameters()) - opt = dist.shard_optimizer(opt, dist.ShardingStage1()) + opt = dist.shard_optimizer(opt, dist.ShardingStage1(self._mesh)) for _ in range(5): loss = linear(batch) loss.backward() @@ -68,6 +73,30 @@ def test_sharding_stage_1_with_mp(self): self.check_tensor_eq(self.weight, linear.weight.numpy()) self.check_tensor_eq(self.bias, linear.bias.numpy()) + def test_sharding_stage_1_with_mp_to_static(self): + data_loader = create_data_loader() + layer = DemoNet( + self._mesh, "sharding_with_mp_demonet", shard_weight=True + ) + opt = paddle.optimizer.SGD( + learning_rate=0.1, parameters=layer.parameters() + ) + opt = dist.shard_optimizer(opt, dist.ShardingStage1(self._mesh)) + loss_fn = nn.MSELoss() + + dist_loader = dist.shard_dataloader( + dataloader=data_loader, + meshes=[self._mesh], + shard_dims=0, + ) + + dist_model = dist.to_static(layer, dist_loader, loss_fn, opt) + + dist_model.train() + for epoch in range(2): + for batch_id, (image, label) in enumerate(dist_loader()): + loss = dist_model(image, label) + def run_test_case(self): if self._backend == "cpu": paddle.set_device("cpu") @@ -78,6 +107,7 @@ def run_test_case(self): self.get_single_card_rst() self.test_sharding_stage_1_with_mp() + self.test_sharding_stage_1_with_mp_to_static() if __name__ == '__main__': diff --git a/test/auto_parallel/hybrid_strategy/semi_auto_parallel_sharding_stage_3.py b/test/auto_parallel/hybrid_strategy/semi_auto_parallel_sharding_stage_3.py index 143e1963c5041..1cb3ff15dc1f9 100644 --- a/test/auto_parallel/hybrid_strategy/semi_auto_parallel_sharding_stage_3.py +++ b/test/auto_parallel/hybrid_strategy/semi_auto_parallel_sharding_stage_3.py @@ -15,9 +15,14 @@ import os import numpy as np +from auto_parallel.semi_auto_parallel_dist_to_static_api import ( + DemoNet, + create_data_loader, +) import paddle import paddle.distributed as dist +from paddle import nn class TestSemiAutoParallelShardingStage3: @@ -68,6 +73,30 @@ def test_sharding_stage_3_with_mp(self): self.check_tensor_eq(self.weight, linear.weight.numpy()) self.check_tensor_eq(self.bias, linear.bias.numpy()) + def test_sharding_stage_3_with_mp_to_static(self): + data_loader = create_data_loader() + layer = DemoNet( + self._mesh, "sharding_with_mp_demonet", shard_weight=True + ) + opt = paddle.optimizer.SGD( + learning_rate=0.1, parameters=layer.parameters() + ) + opt = dist.shard_optimizer(opt, dist.ShardingStage3(self._mesh)) + loss_fn = nn.MSELoss() + + dist_loader = dist.shard_dataloader( + dataloader=data_loader, + meshes=[self._mesh], + shard_dims=0, + ) + + dist_model = dist.to_static(layer, dist_loader, loss_fn, opt) + + dist_model.train() + for epoch in range(2): + for batch_id, (image, label) in enumerate(dist_loader()): + loss = dist_model(image, label) + def run_test_case(self): if self._backend == "cpu": paddle.set_device("cpu") @@ -78,6 +107,7 @@ def run_test_case(self): self.get_single_card_rst() self.test_sharding_stage_3_with_mp() + self.test_sharding_stage_3_with_mp_to_static() if __name__ == '__main__': diff --git a/test/auto_parallel/semi_auto_parallel_dist_to_static_api.py b/test/auto_parallel/semi_auto_parallel_dist_to_static_api.py index fd6ec758086d9..0e166f0457d33 100644 --- a/test/auto_parallel/semi_auto_parallel_dist_to_static_api.py +++ b/test/auto_parallel/semi_auto_parallel_dist_to_static_api.py @@ -37,6 +37,14 @@ def create_numpy_like_random(name): ) +def create_data_loader(): + images = np.random.rand(BATCH_SIZE, IMAGE_SIZE).astype('float32') + labels = np.random.rand(BATCH_SIZE, CLASS_NUM).astype('float32') + dataset = RandomDataset(images, labels, BATCH_SIZE) + loader = DataLoader(dataset, batch_size=BATCH_SIZE) + return loader + + class RandomDataset(paddle.io.Dataset): def __init__(self, images, labels, num_samples): self.images = images @@ -96,20 +104,13 @@ class TestSimpleNetForSemiAutoParallel(unittest.TestCase): def __init__(self): self._seed = eval(os.getenv("seed")) self.set_random_seed(self._seed) - self.data_loader = self.create_data_loader() + self.data_loader = create_data_loader() def set_random_seed(self, seed): random.seed(seed) np.random.seed(seed) paddle.seed(seed) - def create_data_loader(self): - images = np.random.rand(BATCH_SIZE, IMAGE_SIZE).astype('float32') - labels = np.random.rand(BATCH_SIZE, CLASS_NUM).astype('float32') - dataset = RandomDataset(images, labels, BATCH_SIZE) - loader = DataLoader(dataset, batch_size=BATCH_SIZE) - return loader - def get_program_test(self, dist_model): with self.assertRaises(ValueError): main_program = dist_model.dist_main_program() diff --git a/test/auto_parallel/semi_auto_parallel_sharding_stage_1.py b/test/auto_parallel/semi_auto_parallel_sharding_stage_1.py index ffe1d5725f1d1..4d762b07b0591 100644 --- a/test/auto_parallel/semi_auto_parallel_sharding_stage_1.py +++ b/test/auto_parallel/semi_auto_parallel_sharding_stage_1.py @@ -15,9 +15,11 @@ import os import numpy as np +from semi_auto_parallel_dist_to_static_api import DemoNet, create_data_loader import paddle import paddle.distributed as dist +from paddle import nn class TestSemiAutoParallelShardingStage1: @@ -50,7 +52,7 @@ def test_pure_sharding_stage_1(self): batch = dist.shard_tensor(batch, self._mesh, [dist.Shard(0)]) # shard optimizer with stage 1 fn opt = paddle.optimizer.AdamW(parameters=linear.parameters()) - opt = dist.shard_optimizer(opt, dist.ShardingStage1()) + opt = dist.shard_optimizer(opt, dist.ShardingStage1(self._mesh)) for _ in range(5): loss = linear(batch) loss.backward() @@ -59,6 +61,28 @@ def test_pure_sharding_stage_1(self): self.check_tensor_eq(self.weight, linear.weight.numpy()) self.check_tensor_eq(self.bias, linear.bias.numpy()) + def test_sharding_stage_1_to_static(self): + data_loader = create_data_loader() + layer = DemoNet(self._mesh, "sharding_demonet") + opt = paddle.optimizer.SGD( + learning_rate=0.1, parameters=layer.parameters() + ) + opt = dist.shard_optimizer(opt, dist.ShardingStage1(self._mesh)) + loss_fn = nn.MSELoss() + + dist_loader = dist.shard_dataloader( + dataloader=data_loader, + meshes=[self._mesh], + shard_dims=0, + ) + + dist_model = dist.to_static(layer, dist_loader, loss_fn, opt) + + dist_model.train() + for epoch in range(2): + for batch_id, (image, label) in enumerate(dist_loader()): + loss = dist_model(image, label) + def run_test_case(self): if self._backend == "cpu": paddle.set_device("cpu") @@ -69,6 +93,7 @@ def run_test_case(self): self.get_single_card_rst() self.test_pure_sharding_stage_1() + self.test_sharding_stage_1_to_static() if __name__ == '__main__': diff --git a/test/auto_parallel/semi_auto_parallel_sharding_stage_3.py b/test/auto_parallel/semi_auto_parallel_sharding_stage_3.py index f391ca9ef54f2..88999e415d91f 100644 --- a/test/auto_parallel/semi_auto_parallel_sharding_stage_3.py +++ b/test/auto_parallel/semi_auto_parallel_sharding_stage_3.py @@ -15,9 +15,11 @@ import os import numpy as np +from semi_auto_parallel_dist_to_static_api import DemoNet, create_data_loader import paddle import paddle.distributed as dist +from paddle import nn class TestSemiAutoParallelShardingStage3: @@ -59,6 +61,28 @@ def test_pure_sharding_stage_3(self): self.check_tensor_eq(self.weight, linear.weight.numpy()) self.check_tensor_eq(self.bias, linear.bias.numpy()) + def test_sharding_stage_3_to_static(self): + data_loader = create_data_loader() + layer = DemoNet(self._mesh, "sharding_demonet") + opt = paddle.optimizer.SGD( + learning_rate=0.1, parameters=layer.parameters() + ) + opt = dist.shard_optimizer(opt, dist.ShardingStage3(self._mesh)) + loss_fn = nn.MSELoss() + + dist_loader = dist.shard_dataloader( + dataloader=data_loader, + meshes=[self._mesh], + shard_dims=0, + ) + + dist_model = dist.to_static(layer, dist_loader, loss_fn, opt) + + dist_model.train() + for epoch in range(2): + for batch_id, (image, label) in enumerate(dist_loader()): + loss = dist_model(image, label) + def run_test_case(self): if self._backend == "cpu": paddle.set_device("cpu") @@ -69,6 +93,7 @@ def run_test_case(self): self.get_single_card_rst() self.test_pure_sharding_stage_3() + self.test_sharding_stage_3_to_static() if __name__ == '__main__': From b8b08b75f0d98becdcabe4bcc4bfa08f820aae5f Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 4 Mar 2024 11:51:19 +0800 Subject: [PATCH 257/282] Fix usless useless, etc (#62323) --- paddle/fluid/inference/CMakeLists.txt | 2 +- .../tensorrt/convert/set_value_op.cc | 2 +- .../tensorrt/dynamic_shape_infermeta.cc | 4 +- paddle/fluid/inference/tensorrt/engine.cc | 2 +- paddle/fluid/inference/tensorrt/op_teller.cc | 51 ++++++++++--------- paddle/fluid/inference/tensorrt/op_teller.h | 2 +- .../tensorrt/plugin_arg_mapping_context.cc | 2 +- .../tensorrt/test_arg_mapping_context.cc | 6 +-- .../inference/tensorrt/trt_int8_calibrator.h | 2 +- .../inference/utils/shape_range_info.proto | 2 +- paddle/fluid/inference/utils/table_printer.cc | 10 ++-- .../ir_adaptor/translator/op_compat_gen.py | 16 +++--- 12 files changed, 52 insertions(+), 49 deletions(-) diff --git a/paddle/fluid/inference/CMakeLists.txt b/paddle/fluid/inference/CMakeLists.txt index 88003c6db6ba6..bed777851641a 100644 --- a/paddle/fluid/inference/CMakeLists.txt +++ b/paddle/fluid/inference/CMakeLists.txt @@ -93,7 +93,7 @@ set(SHARED_INFERENCE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/api/details/zero_copy_tensor.cc ${CMAKE_CURRENT_SOURCE_DIR}/utils/io_utils.cc) -# NOTE(Aurelius84): For inference library, some DEPS is usless +# NOTE(Aurelius84): For inference library, some DEPS is useless # such as non-infer operator related targets et.al. list(REMOVE_ITEM fluid_modules cinn_op_dialect) # NOTE(Aurelisu84): Remove pir dialect related target DEPS for inference diff --git a/paddle/fluid/inference/tensorrt/convert/set_value_op.cc b/paddle/fluid/inference/tensorrt/convert/set_value_op.cc index 1c734d791cdde..50797b62e614d 100644 --- a/paddle/fluid/inference/tensorrt/convert/set_value_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/set_value_op.cc @@ -25,7 +25,7 @@ limitations under the License. */ PADDLE_ENFORCE_EQ(vec_##attr_name__.size(), \ 1UL, \ platform::errors::InvalidArgument( \ - "attr axes/starst/ends/steps 's size in " \ + "attr axes/starts/ends/steps 's size in " \ "set_value must be one, but got %d", \ vec_##attr_name__.size())); \ } \ diff --git a/paddle/fluid/inference/tensorrt/dynamic_shape_infermeta.cc b/paddle/fluid/inference/tensorrt/dynamic_shape_infermeta.cc index ed5f57165d710..1ac412384e2db 100644 --- a/paddle/fluid/inference/tensorrt/dynamic_shape_infermeta.cc +++ b/paddle/fluid/inference/tensorrt/dynamic_shape_infermeta.cc @@ -259,7 +259,7 @@ inline const nvinfer1::IDimensionExpr* CalcOutputSize( return output_size; } -nvinfer1::DimsExprs UnflodInferMeta( +nvinfer1::DimsExprs UnfoldInferMeta( int output_index, const nvinfer1::DimsExprs* inputs, int nb_inputs, @@ -879,7 +879,7 @@ nvinfer1::DimsExprs SolveInferMeta( PD_REGISTER_DYNAMIC_INFER_META_FN(gather_nd, GatherNdInferMeta); PD_REGISTER_DYNAMIC_INFER_META_FN(yolo_box, YoloBoxInferMeta); PD_REGISTER_DYNAMIC_INFER_META_FN(instance_norm, InstanceNormInferMeta); -PD_REGISTER_DYNAMIC_INFER_META_FN(unfold, UnflodInferMeta); +PD_REGISTER_DYNAMIC_INFER_META_FN(unfold, UnfoldInferMeta); PD_REGISTER_DYNAMIC_INFER_META_FN(scatter_nd_add, ScatterNdAddInferMeta); PD_REGISTER_DYNAMIC_INFER_META_FN(inverse, UnchangedInferMeta); PD_REGISTER_DYNAMIC_INFER_META_FN(moe, MoeInferMeta); diff --git a/paddle/fluid/inference/tensorrt/engine.cc b/paddle/fluid/inference/tensorrt/engine.cc index 6bc369de6c89c..2a14702b59d81 100644 --- a/paddle/fluid/inference/tensorrt/engine.cc +++ b/paddle/fluid/inference/tensorrt/engine.cc @@ -52,7 +52,7 @@ void TensorRTEngine::Weight::SetDataType(phi::DataType type) { #endif default: paddle::platform::errors::InvalidArgument( - "Paddle-TRT loads weighths failed, found not supported data type %s.", + "Paddle-TRT loads weights failed, found not supported data type %s.", type); break; } diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc index bb56dfe4d6f9b..da46cc80ca5a9 100644 --- a/paddle/fluid/inference/tensorrt/op_teller.cc +++ b/paddle/fluid/inference/tensorrt/op_teller.cc @@ -1460,7 +1460,7 @@ struct SimpleOpTypeSetTeller : public Teller { } if (desc.Output("Out").size() != 1) { VLOG(3) << "The input op's Output(\"Out\").size() " - "should equal to 1, but reveceid Output(\"Out\").size() = " + "should equal to 1, but received Output(\"Out\").size() = " << desc.Output("Out").size() << "."; return false; } @@ -2080,20 +2080,21 @@ struct SimpleOpTypeSetTeller : public Teller { auto inputs = desc.Inputs(); bool has_bias_qk = (inputs.find("BiasQK") == inputs.end()) ? false : true; if (has_bias_qk) { - auto* biasqk_desc = + auto* bias_qk_desc = block->FindVarRecursive(desc.Input("BiasQK").front()); - const auto biasqk_shape = biasqk_desc->GetShape(); + const auto bias_qk_shape = bias_qk_desc->GetShape(); // The BiasQK's shape requires to be // [batch, 1, 1, length] or [batch, head, length, length]. - bool has_same_shape = head_number == biasqk_shape[1] && - input_shape[1] == biasqk_shape[2] && - input_shape[1] == biasqk_shape[3]; - bool is_broadcastable = biasqk_shape[1] == 1 && biasqk_shape[2] == 1 && - input_shape[1] == biasqk_shape[3]; - is_broadcastable = - is_broadcastable || (biasqk_shape[0] == 1 && biasqk_shape[1] == 1 && - input_shape[1] == biasqk_shape[2] && - input_shape[1] == biasqk_shape[3]); + bool has_same_shape = head_number == bias_qk_shape[1] && + input_shape[1] == bias_qk_shape[2] && + input_shape[1] == bias_qk_shape[3]; + bool is_broadcastable = bias_qk_shape[1] == 1 && + bias_qk_shape[2] == 1 && + input_shape[1] == bias_qk_shape[3]; + is_broadcastable = is_broadcastable || + (bias_qk_shape[0] == 1 && bias_qk_shape[1] == 1 && + input_shape[1] == bias_qk_shape[2] && + input_shape[1] == bias_qk_shape[3]); if (!(has_same_shape || is_broadcastable)) { VLOG(3) << "The BiasQK's shape is invalid, expect [" << input_shape[0] << ", 1, 1, " << input_shape[1] << "] " @@ -2101,8 +2102,9 @@ struct SimpleOpTypeSetTeller : public Teller { << input_shape[1] << ", " << input_shape[1] << "] " << "or [" << input_shape[0] << "/1, " << 1 << ", " << input_shape[1] << ", " << input_shape[1] << "] " - << "but got [" << biasqk_shape[0] << ", " << biasqk_shape[1] - << ", " << biasqk_shape[2] << ", " << biasqk_shape[3] << "]."; + << "but got [" << bias_qk_shape[0] << ", " << bias_qk_shape[1] + << ", " << bias_qk_shape[2] << ", " << bias_qk_shape[3] + << "]."; return false; } } else { @@ -2140,23 +2142,24 @@ struct SimpleOpTypeSetTeller : public Teller { auto inputs = desc.Inputs(); bool has_bias_qk = (inputs.find("BiasQK") == inputs.end()) ? false : true; if (has_bias_qk) { - auto* biasqk_desc = + auto* bias_qk_desc = block->FindVarRecursive(desc.Input("BiasQK").front()); - const auto biasqk_shape = biasqk_desc->GetShape(); + const auto bias_qk_shape = bias_qk_desc->GetShape(); // The BiasQK's shape requires to be // [batch, 1, 1, length] or [batch, head, length, length]. - bool has_same_shape = head_number == biasqk_shape[1] && - input_shape[1] == biasqk_shape[2] && - input_shape[1] == biasqk_shape[3]; - bool is_broadcastable = biasqk_shape[1] == 1 && biasqk_shape[2] == 1 && - input_shape[1] == biasqk_shape[3]; + bool has_same_shape = head_number == bias_qk_shape[1] && + input_shape[1] == bias_qk_shape[2] && + input_shape[1] == bias_qk_shape[3]; + bool is_broadcastable = bias_qk_shape[1] == 1 && + bias_qk_shape[2] == 1 && + input_shape[1] == bias_qk_shape[3]; if (!(has_same_shape || is_broadcastable)) { VLOG(3) << "The BiasQK's shape is invalid, expect [" << input_shape[0] << ", 1, 1, " << input_shape[1] << "] or [" << input_shape[0] << ", " << head_number << ", " << input_shape[1] << ", " - << input_shape[1] << "] but [" << biasqk_shape[0] << ", " - << biasqk_shape[1] << ", " << biasqk_shape[2] << ", " - << biasqk_shape[3] << "]."; + << input_shape[1] << "] but [" << bias_qk_shape[0] << ", " + << bias_qk_shape[1] << ", " << bias_qk_shape[2] << ", " + << bias_qk_shape[3] << "]."; return false; } } else { diff --git a/paddle/fluid/inference/tensorrt/op_teller.h b/paddle/fluid/inference/tensorrt/op_teller.h index 69a9061ebdb97..9c909c2d71c06 100644 --- a/paddle/fluid/inference/tensorrt/op_teller.h +++ b/paddle/fluid/inference/tensorrt/op_teller.h @@ -34,7 +34,7 @@ namespace tensorrt { /* * Single Op teller definition. - * One can override this and define a more complex tell logic, considerring more + * One can override this and define a more complex tell logic, considering more * issues such as op_desc. */ struct Teller { diff --git a/paddle/fluid/inference/tensorrt/plugin_arg_mapping_context.cc b/paddle/fluid/inference/tensorrt/plugin_arg_mapping_context.cc index 26cb5166362b2..d4631f7057582 100644 --- a/paddle/fluid/inference/tensorrt/plugin_arg_mapping_context.cc +++ b/paddle/fluid/inference/tensorrt/plugin_arg_mapping_context.cc @@ -76,7 +76,7 @@ paddle::any PluginArgumentMappingContext::Attr( break; }; default: { - LOG(ERROR) << "Can't conver op's attribute [" << attr_name + LOG(ERROR) << "Can't cover op's attribute [" << attr_name << "] to paddle any."; } } diff --git a/paddle/fluid/inference/tensorrt/test_arg_mapping_context.cc b/paddle/fluid/inference/tensorrt/test_arg_mapping_context.cc index 97090518153d1..85dddfea2a7c7 100644 --- a/paddle/fluid/inference/tensorrt/test_arg_mapping_context.cc +++ b/paddle/fluid/inference/tensorrt/test_arg_mapping_context.cc @@ -21,7 +21,7 @@ namespace paddle { namespace inference { namespace tensorrt { -TEST(ArgMappingContexTest, BasicFunction) { +TEST(ArgMappingContextTest, BasicFunction) { paddle::framework::proto::OpDesc op; op.set_type("imaged_op"); auto *input_var = op.add_inputs(); @@ -86,8 +86,8 @@ TEST(ArgMappingContexTest, BasicFunction) { int int_attr = any_cast(context.Attr("int_attr")); EXPECT_EQ(int_attr, 1); - float flaot_attr = any_cast(context.Attr("float_attr")); - EXPECT_EQ(flaot_attr, 1); + float float_attr = any_cast(context.Attr("float_attr")); + EXPECT_EQ(float_attr, 1); std::string string_attr = any_cast(context.Attr("string_attr")); EXPECT_EQ(string_attr, "1"); diff --git a/paddle/fluid/inference/tensorrt/trt_int8_calibrator.h b/paddle/fluid/inference/tensorrt/trt_int8_calibrator.h index 82bb7a64168b4..43386ca324c54 100644 --- a/paddle/fluid/inference/tensorrt/trt_int8_calibrator.h +++ b/paddle/fluid/inference/tensorrt/trt_int8_calibrator.h @@ -87,7 +87,7 @@ class TRTCalibratorEngine { std::unique_ptr engine_; }; /* - * Manager to control the TensorRT Int8 calibration creation and deltetion. + * Manager to control the TensorRT Int8 calibration creation and deletion. */ class TRTCalibratorEngineManager { public: diff --git a/paddle/fluid/inference/utils/shape_range_info.proto b/paddle/fluid/inference/utils/shape_range_info.proto index 53f018cb59348..9e980de9d0fd5 100644 --- a/paddle/fluid/inference/utils/shape_range_info.proto +++ b/paddle/fluid/inference/utils/shape_range_info.proto @@ -16,7 +16,7 @@ syntax = "proto2"; package paddle.inference.proto; // To support trt dynamic shape, record the runtime shape -// information of all tmp tensors in the Compution graph. +// information of all tmp tensors in the Computation graph. message ShapeRangeInfos { message ShapeRangeInfo { required string name = 1; diff --git a/paddle/fluid/inference/utils/table_printer.cc b/paddle/fluid/inference/utils/table_printer.cc index ba7a8d342e352..19b4a94834a17 100644 --- a/paddle/fluid/inference/utils/table_printer.cc +++ b/paddle/fluid/inference/utils/table_printer.cc @@ -57,18 +57,18 @@ std::string TablePrinter::PrintTable() { } TablePrinter::TablePrinter(const std::vector& header) { - size_t terminal_witdh = 500; + size_t terminal_width = 500; #ifdef _WIN32 CONSOLE_SCREEN_BUFFER_INFO csbi; int ret = GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &csbi); if (ret && (csbi.dwSize.X != 0)) { - terminal_witdh = csbi.dwSize.X; + terminal_width = csbi.dwSize.X; } #else struct winsize terminal_size; int status = ioctl(STDOUT_FILENO, TIOCGWINSZ, &terminal_size); if (status == 0 && terminal_size.ws_col != 0) { - terminal_witdh = terminal_size.ws_col; + terminal_width = terminal_size.ws_col; } #endif @@ -77,8 +77,8 @@ TablePrinter::TablePrinter(const std::vector& header) { widths_.emplace_back(0); } - terminal_witdh = terminal_witdh - (2 * num_cols) - (num_cols + 1); - int avg_width = static_cast(terminal_witdh / num_cols); // NOLINT + terminal_width = terminal_width - (2 * num_cols) - (num_cols + 1); + int avg_width = static_cast(terminal_width / num_cols); // NOLINT for (size_t i = 0; i < num_cols; ++i) { shares_.emplace_back(avg_width); diff --git a/paddle/fluid/ir_adaptor/translator/op_compat_gen.py b/paddle/fluid/ir_adaptor/translator/op_compat_gen.py index 1cb0ab7a3b01a..c7f56fe025fef 100644 --- a/paddle/fluid/ir_adaptor/translator/op_compat_gen.py +++ b/paddle/fluid/ir_adaptor/translator/op_compat_gen.py @@ -48,7 +48,7 @@ def to_phi_and_fluid_op_name(op_item): op_compat_infos = yaml.safe_load(f) op_name_mappings: Dict[str, str] = {} op_arg_name_mappings: Dict[str, Dict[str, str]] = {} - op_mutable_attribues: Dict[str, Set[str]] = {} + op_mutable_attributes: Dict[str, Set[str]] = {} op_mutable_attribute_infos: Dict[str, Dict[str, List[str]]] = {} for op_compat_item in op_compat_infos: @@ -70,15 +70,15 @@ def insert_new_arg_mappings(op_name: str, arg_mapping: Dict[str, str]): def insert_new_mutable_attributes( op_name: str, mutable_attribute_infos: Dict[str, Dict[str, str]] ): - if op_name not in op_mutable_attribues: - op_mutable_attribues[op_name] = set() + if op_name not in op_mutable_attributes: + op_mutable_attributes[op_name] = set() if op_name not in op_mutable_attribute_infos: op_mutable_attribute_infos[op_name] = {} for ( attribute_name, mutable_attribute_info, ) in mutable_attribute_infos.items(): - op_mutable_attribues[op_name].add(attribute_name) + op_mutable_attributes[op_name].add(attribute_name) op_mutable_attribute_infos[op_name][attribute_name] = [] for k, v in mutable_attribute_info.items(): if k == 'tensor_name' or k == 'tensors_name': @@ -168,12 +168,12 @@ def insert_new_mutable_attributes( {"out_grad_in": "Out@GRAD", "out_grad_out": "Out@GRAD"} ) - op_name_normailzer_template = env.get_template("op_compat_info.cc.j2") + op_name_normalizer_template = env.get_template("op_compat_info.cc.j2") with open(output_source_file, 'wt') as f: - op_compat_definition = op_name_normailzer_template.render( + op_compat_definition = op_name_normalizer_template.render( op_name_pairs=op_name_mappings, op_arg_name_pairs=op_arg_name_mappings, - op_mutable_attributes=op_mutable_attribues, + op_mutable_attributes=op_mutable_attributes, op_mutable_attribute_infos=op_mutable_attribute_infos, ) f.write(op_compat_definition) @@ -184,7 +184,7 @@ def insert_new_mutable_attributes( # ===================================== def ParseArguments(): parser = argparse.ArgumentParser( - description='Generate OP Compatiable info Files By Yaml' + description='Generate OP Compatible info Files By Yaml' ) parser.add_argument('--op_compat_yaml_file', type=str) parser.add_argument('--output_source_file', type=str) From e989c159a0453e881c07a0fa58f557b97701f94a Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 4 Mar 2024 11:51:45 +0800 Subject: [PATCH 258/282] Fix cotain contain, etc (#62319) --- .../generic_and_custom_plugin_creater.cc | 54 +++++++++---------- .../tensorrt/convert/layer_norm_op.cc | 2 +- .../convert/layernorm_shift_partition_op.cc | 2 +- .../inference/tensorrt/convert/op_converter.h | 6 +-- .../convert/preln_emb_eltwise_layernorm.cc | 4 +- .../tensorrt/convert/quantize_linear_op.cc | 2 +- .../inference/tensorrt/convert/range_op.cc | 6 +-- .../inference/tensorrt/convert/reshape_op.cc | 2 +- .../tensorrt/convert/set_value_op.cc | 2 +- .../tensorrt/convert/skip_layernorm.cc | 24 +++++---- .../inference/tensorrt/convert/slice_op.cc | 2 +- .../inference/tensorrt/convert/softmax_op.cc | 2 +- .../tensorrt/convert/sparse_fc_op.cc | 2 +- .../tensorrt/convert/trans_layernorm_op.cc | 2 +- .../inference/tensorrt/convert/ut_helper.h | 2 +- 15 files changed, 59 insertions(+), 55 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc b/paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc index 5e4dfca1417f8..eefed86f141c3 100644 --- a/paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc +++ b/paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc @@ -60,7 +60,7 @@ class CustomPluginCreater : public OpConverter { CHECK(creator); // set attrs - std::vector plugindatas; + std::vector plugin_datas; auto &op_attrs_names = OpMetaInfoHelper::GetAttrs(op_info); auto &attrs = op_desc.GetAttrMap(); @@ -74,7 +74,7 @@ class CustomPluginCreater : public OpConverter { for (auto &attr_name_and_type : op_attrs_names) { auto attr_name = attr_name_and_type.substr(0, attr_name_and_type.find_first_of(":")); - nvinfer1::PluginField plugindata; + nvinfer1::PluginField plugin_data; // NOTE: to avoid string rewrite by iterator, deep copy here std::vector plugin_attr_name(attr_name.length() + 1, 0); @@ -82,47 +82,47 @@ class CustomPluginCreater : public OpConverter { attr_name.length() + 1, "%s", attr_name.c_str()); - plugindata.name = plugin_attr_name.data(); + plugin_data.name = plugin_attr_name.data(); if (op_desc.GetAttrType(attr_name) == framework::proto::AttrType::INT) { int_attrs.push_back(PADDLE_GET_CONST(int, attrs.at(attr_name))); - plugindata.data = &int_attrs.back(); - plugindata.type = nvinfer1::PluginFieldType::kINT32; - plugindata.length = 1; + plugin_data.data = &int_attrs.back(); + plugin_data.type = nvinfer1::PluginFieldType::kINT32; + plugin_data.length = 1; } else if (op_desc.GetAttrType(attr_name) == framework::proto::AttrType::FLOAT) { float_attrs.push_back(PADDLE_GET_CONST(float, attrs.at(attr_name))); - plugindata.data = &float_attrs.back(); - plugindata.type = nvinfer1::PluginFieldType::kFLOAT32; - plugindata.length = 1; + plugin_data.data = &float_attrs.back(); + plugin_data.type = nvinfer1::PluginFieldType::kFLOAT32; + plugin_data.length = 1; } else if (op_desc.GetAttrType(attr_name) == framework::proto::AttrType::BOOLEAN) { int_attrs.push_back(PADDLE_GET_CONST(bool, attrs.at(attr_name))); - plugindata.data = &int_attrs.back(); - plugindata.type = nvinfer1::PluginFieldType::kINT32; - plugindata.length = 1; + plugin_data.data = &int_attrs.back(); + plugin_data.type = nvinfer1::PluginFieldType::kINT32; + plugin_data.length = 1; } else if (op_desc.GetAttrType(attr_name) == framework::proto::AttrType::STRING) { string_attrs.push_back( PADDLE_GET_CONST(std::string, attrs.at(attr_name))); - plugindata.data = string_attrs.back().data(); - plugindata.type = nvinfer1::PluginFieldType::kCHAR; - plugindata.length = + plugin_data.data = string_attrs.back().data(); + plugin_data.type = nvinfer1::PluginFieldType::kCHAR; + plugin_data.length = string_attrs.back().size() + 1; // string ends with ‘\0’ } else if (op_desc.GetAttrType(attr_name) == framework::proto::AttrType::INTS) { ints_attrs.push_back( PADDLE_GET_CONST(std::vector, attrs.at(attr_name))); - plugindata.data = ints_attrs.back().data(); - plugindata.type = nvinfer1::PluginFieldType::kINT32; - plugindata.length = ints_attrs.back().size(); + plugin_data.data = ints_attrs.back().data(); + plugin_data.type = nvinfer1::PluginFieldType::kINT32; + plugin_data.length = ints_attrs.back().size(); } else if (op_desc.GetAttrType(attr_name) == framework::proto::AttrType::FLOATS) { floats_attrs.push_back( PADDLE_GET_CONST(std::vector, attrs.at(attr_name))); - plugindata.data = floats_attrs.back().data(); - plugindata.type = nvinfer1::PluginFieldType::kFLOAT32; - plugindata.length = floats_attrs.back().size(); + plugin_data.data = floats_attrs.back().data(); + plugin_data.type = nvinfer1::PluginFieldType::kFLOAT32; + plugin_data.length = floats_attrs.back().size(); } else if (op_desc.GetAttrType(attr_name) == framework::proto::AttrType::BOOLEANS) { auto bools_attr = @@ -130,17 +130,17 @@ class CustomPluginCreater : public OpConverter { std::vector convert_to_ints_attr; for (bool i : bools_attr) convert_to_ints_attr.push_back(i); ints_attrs.push_back(convert_to_ints_attr); - plugindata.data = ints_attrs.back().data(); - plugindata.type = nvinfer1::PluginFieldType::kINT32; - plugindata.length = ints_attrs.back().size(); + plugin_data.data = ints_attrs.back().data(); + plugin_data.type = nvinfer1::PluginFieldType::kINT32; + plugin_data.length = ints_attrs.back().size(); } else { CHECK(false) << "UNKNOWN PluginFieldType."; } - plugindatas.push_back(plugindata); + plugin_datas.push_back(plugin_data); } - nvinfer1::PluginFieldCollection plugin_fc{(int32_t)plugindatas.size(), - plugindatas.data()}; + nvinfer1::PluginFieldCollection plugin_fc{(int32_t)plugin_datas.size(), + plugin_datas.data()}; auto *plugin = creator->createPlugin(op_desc.Type().c_str(), &plugin_fc); CHECK(plugin); diff --git a/paddle/fluid/inference/tensorrt/convert/layer_norm_op.cc b/paddle/fluid/inference/tensorrt/convert/layer_norm_op.cc index 50fa54bcf90c2..43d56b0994ddd 100644 --- a/paddle/fluid/inference/tensorrt/convert/layer_norm_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/layer_norm_op.cc @@ -74,7 +74,7 @@ class LayerNormOpConverter : public OpConverter { #endif #if IS_TRT_VERSION_LT(8600) // For dynamic shape & trt<8.6, - // the shape of mean and variance will be determine in configuPlugin. + // the shape of mean and variance will be determine in configurePlugin. auto* X = engine_->GetITensor(op_desc.Input("X").front()); auto* Bias_v = scope.FindVar(op_desc.Input("Bias").front()); auto* Scale_v = scope.FindVar(op_desc.Input("Scale").front()); diff --git a/paddle/fluid/inference/tensorrt/convert/layernorm_shift_partition_op.cc b/paddle/fluid/inference/tensorrt/convert/layernorm_shift_partition_op.cc index 7cf5dea57d5d4..4f4b09b6173a2 100644 --- a/paddle/fluid/inference/tensorrt/convert/layernorm_shift_partition_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/layernorm_shift_partition_op.cc @@ -73,7 +73,7 @@ class LayerNormShiftPartitionOpConverter : public OpConverter { PADDLE_ENFORCE_EQ(bias_weight.get().count, scale_weight.get().count, platform::errors::InvalidArgument( - "The num between bias_weight and cale_weight should " + "The num between bias_weight and scale_weight should " "be equal. (%d vs %d)", bias_weight.get().count, scale_weight.get().count)); diff --git a/paddle/fluid/inference/tensorrt/convert/op_converter.h b/paddle/fluid/inference/tensorrt/convert/op_converter.h index 3b75a79d9b563..1e663fa362929 100644 --- a/paddle/fluid/inference/tensorrt/convert/op_converter.h +++ b/paddle/fluid/inference/tensorrt/convert/op_converter.h @@ -70,7 +70,7 @@ class OpConverter { 1UL, platform::errors::InvalidArgument( "The input op's Input(\"Y\")." - "size() should equal to 1, but reveceid " + "size() should equal to 1, but received " "Input(\"Y\").size() = %u.", op_desc.Input("Y").size())); int op_type_len = op_desc.Type().size(); @@ -179,7 +179,7 @@ class OpConverter { (*it)(op, scope, test_mode); size_t output_num = op_desc.OutputNames().size(); - // only one out settensordynamicRange + // only one out SetTensorDynamicRange if (op_desc.HasAttr("out_threshold")) { float out_scale = PADDLE_GET_CONST(float, op_desc.GetAttr("out_threshold")); @@ -202,7 +202,7 @@ class OpConverter { VLOG(1) << "Set out scale = " << out_scale << " for tensor " << output_name << "."; } - // outs settensordynamicRange + // outs SetTensorDynamicRange for (size_t i = 0; i < output_num; ++i) { if (op_desc.HasAttr("out_" + std::to_string(i) + "_threshold")) { float out_scale = PADDLE_GET_CONST( diff --git a/paddle/fluid/inference/tensorrt/convert/preln_emb_eltwise_layernorm.cc b/paddle/fluid/inference/tensorrt/convert/preln_emb_eltwise_layernorm.cc index 529175c7de81a..0ec1336f0e2d1 100644 --- a/paddle/fluid/inference/tensorrt/convert/preln_emb_eltwise_layernorm.cc +++ b/paddle/fluid/inference/tensorrt/convert/preln_emb_eltwise_layernorm.cc @@ -103,7 +103,7 @@ class PrelnEmbEltwiseLayerNormOpConverter : public OpConverter { slice_stride_dims); // unuseful slice_start_dims slice_layer->setInput(1, *start_tensor); slice_layer->setInput(2, *size_tensor); - slice_layer->setName(("Embeltwise_slice_layer (Output: slice_max_seqlen " + + slice_layer->setName(("EmbEltwise_slice_layer (Output: slice_max_seqlen " + op_desc.Output("Out")[0] + ")") .c_str()); engine_->SetTensorDynamicRange(slice_layer->getOutput(0), 1.0f); @@ -114,7 +114,7 @@ class PrelnEmbEltwiseLayerNormOpConverter : public OpConverter { shape_dim.nbDims = 1; shape_dim.d[0] = -1; reshape_layer->setReshapeDimensions(shape_dim); - reshape_layer->setName(("Embeltwise_reshape_layer (Output: max_seqlen " + + reshape_layer->setName(("EmbEltwise_reshape_layer (Output: max_seqlen " + op_desc.Output("Out")[0] + ")") .c_str()); engine_->SetTensorDynamicRange(reshape_layer->getOutput(0), 1.0f); diff --git a/paddle/fluid/inference/tensorrt/convert/quantize_linear_op.cc b/paddle/fluid/inference/tensorrt/convert/quantize_linear_op.cc index b37a8f327e154..74a8f56ea6c20 100644 --- a/paddle/fluid/inference/tensorrt/convert/quantize_linear_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/quantize_linear_op.cc @@ -33,7 +33,7 @@ class QuantizeLinearOpConverter : public OpConverter { // Create constant layer for scale PADDLE_ENFORCE_NOT_NULL( scale_var, - platform::errors::NotFound("Can not find %s presistale var in scope.", + platform::errors::NotFound("Can not find %s presistable var in scope.", op_desc.Input("Scale")[0])); auto* scale_t = scale_var->GetMutable(); int n_scale = scale_t->numel(); diff --git a/paddle/fluid/inference/tensorrt/convert/range_op.cc b/paddle/fluid/inference/tensorrt/convert/range_op.cc index b44d9d588744a..073b51b8c0734 100644 --- a/paddle/fluid/inference/tensorrt/convert/range_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/range_op.cc @@ -35,15 +35,15 @@ class RangeOpConverter : public OpConverter { auto output_name = op_desc.Output("Out")[0]; auto zero_tensor = Add1DConstantLayer(0, output_name + "_zero_tensor_"); - auto fquotient_tensor = FloorDiv(Sub(start, end), step); + auto f_quotient_tensor = FloorDiv(Sub(start, end), step); if (start->getType() == nvinfer1::DataType::kFLOAT) { auto* cast_int32_layer = - TRT_ENGINE_ADD_LAYER(engine_, Identity, *fquotient_tensor); + TRT_ENGINE_ADD_LAYER(engine_, Identity, *f_quotient_tensor); cast_int32_layer->setOutputType(0, nvinfer1::DataType::kINT32); cast_int32_layer->getOutput(0)->setType(nvinfer1::DataType::kINT32); quotient_tensor = cast_int32_layer->getOutput(0); } else { - quotient_tensor = fquotient_tensor; + quotient_tensor = f_quotient_tensor; } auto number_tensor = Max(Sub(zero_tensor, quotient_tensor), zero_tensor); auto* start1 = engine_->GetITensor(op_desc.Input("Start")[0]); diff --git a/paddle/fluid/inference/tensorrt/convert/reshape_op.cc b/paddle/fluid/inference/tensorrt/convert/reshape_op.cc index c31cf1b012a49..c1f226626742f 100644 --- a/paddle/fluid/inference/tensorrt/convert/reshape_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/reshape_op.cc @@ -67,7 +67,7 @@ class ReshapeOpConverter : public OpConverter { layer->getOutput(0)->getDimensions().nbDims, 0, platform::errors::InvalidArgument( - "Errors occures in Paddle-TRT reshape2 op, try to use C++ Api " + "Errors occurs in Paddle-TRT reshape2 op, try to use C++ Api " "config.Exp_DisableTensorRtOPs({\"reshape2\"})\n; or Python Api " "config.exp_disable_tensorrt_ops([\"reshape2\"]) to forbid " "reshape2 op into " diff --git a/paddle/fluid/inference/tensorrt/convert/set_value_op.cc b/paddle/fluid/inference/tensorrt/convert/set_value_op.cc index 50797b62e614d..29f95a3554fc4 100644 --- a/paddle/fluid/inference/tensorrt/convert/set_value_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/set_value_op.cc @@ -151,7 +151,7 @@ class SetValueConverter : public OpConverter { platform::errors::InvalidArgument( "ValueTensor‘s rank not equal to Input's rank, " "you should try use C++ API " - "config.exp_disable_tensorrt_ops({\"%s\"}) to forbind this op " + "config.exp_disable_tensorrt_ops({\"%s\"}) to forbid this op " "enter into TRT, " "please find the %s's real name from .pdmodel or shape.txt", output_name, diff --git a/paddle/fluid/inference/tensorrt/convert/skip_layernorm.cc b/paddle/fluid/inference/tensorrt/convert/skip_layernorm.cc index 15ef380253949..ab70ebb6ccd81 100644 --- a/paddle/fluid/inference/tensorrt/convert/skip_layernorm.cc +++ b/paddle/fluid/inference/tensorrt/convert/skip_layernorm.cc @@ -67,17 +67,19 @@ class SkipLayerNormOpConverter : public OpConverter { if ((x_rank == 2 && y_rank == 4) || (y_rank == 2 && x_rank == 4)) { if (x_rank == 2 && y_rank == 4) { - auto* reshape_before_skiplayn = + auto* reshape_before_skip_layer_n = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *input1); std::vector reshape_before_tensor; reshape_before_tensor.push_back(GetEleTensorOfShape(Shape(input1), 0)); reshape_before_tensor.push_back(GetEleTensorOfShape(Shape(input1), 1)); reshape_before_tensor.push_back(Add1DConstantLayer(1)); reshape_before_tensor.push_back(Add1DConstantLayer(1)); - reshape_before_skiplayn->setInput(1, *Concat(reshape_before_tensor)); - reshape_before_skiplayn->setName( - ("reshape_before_skiplayn(Output: " + output_name + ")").c_str()); - input1 = reshape_before_skiplayn->getOutput(0); + reshape_before_skip_layer_n->setInput(1, + *Concat(reshape_before_tensor)); + reshape_before_skip_layer_n->setName( + ("reshape_before_skip_layer_n(Output: " + output_name + ")") + .c_str()); + input1 = reshape_before_skip_layer_n->getOutput(0); if (enable_int8) { if (op_desc.HasAttr("X")) { @@ -85,17 +87,19 @@ class SkipLayerNormOpConverter : public OpConverter { } } } else { - auto* reshape_before_skiplayn = + auto* reshape_before_skip_layer_n = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *input2); std::vector reshape_before_tensor; reshape_before_tensor.push_back(GetEleTensorOfShape(Shape(input2), 0)); reshape_before_tensor.push_back(GetEleTensorOfShape(Shape(input2), 1)); reshape_before_tensor.push_back(Add1DConstantLayer(1)); reshape_before_tensor.push_back(Add1DConstantLayer(1)); - reshape_before_skiplayn->setInput(1, *Concat(reshape_before_tensor)); - reshape_before_skiplayn->setName( - ("reshape_before_skiplayn(Output: " + output_name + ")").c_str()); - input2 = reshape_before_skiplayn->getOutput(0); + reshape_before_skip_layer_n->setInput(1, + *Concat(reshape_before_tensor)); + reshape_before_skip_layer_n->setName( + ("reshape_before_skip_layer_n(Output: " + output_name + ")") + .c_str()); + input2 = reshape_before_skip_layer_n->getOutput(0); if (enable_int8) { if (op_desc.HasAttr("Y")) { diff --git a/paddle/fluid/inference/tensorrt/convert/slice_op.cc b/paddle/fluid/inference/tensorrt/convert/slice_op.cc index 4a2d38d5e0736..0e2382a2d3fa6 100644 --- a/paddle/fluid/inference/tensorrt/convert/slice_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/slice_op.cc @@ -20,7 +20,7 @@ class SliceOpConverter : public OpConverter { void operator()(const framework::proto::OpDesc& op, const framework::Scope& scope, bool test_mode) override { - // This OP is implemented by trt dynamic shpae plugin. + // This OP is implemented by trt dynamic shape plugin. // Dynamic shape plugin requires TRT version greater than 6.0. VLOG(4) << "convert slice op to tensorrt layer"; framework::OpDesc op_desc(op, nullptr); diff --git a/paddle/fluid/inference/tensorrt/convert/softmax_op.cc b/paddle/fluid/inference/tensorrt/convert/softmax_op.cc index 921402a9be5d2..483cd0711ffc6 100644 --- a/paddle/fluid/inference/tensorrt/convert/softmax_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/softmax_op.cc @@ -58,7 +58,7 @@ class SoftMaxOpConverter : public OpConverter { uint32_t axes = std::max(0, input_dims - 3); // TODO(cryoco): Poor workaround. Fix padded dims problem when TRT layers // support Nd. - // Tips: Dynammic shape alreay fixes. + // Tips: Dynamic shape already fixes. int padded_dims = 0; int explicit_batch = 0; if (engine_->with_dynamic_shape()) explicit_batch = 1; diff --git a/paddle/fluid/inference/tensorrt/convert/sparse_fc_op.cc b/paddle/fluid/inference/tensorrt/convert/sparse_fc_op.cc index bae9cccde6fa7..c143eb00d2797 100644 --- a/paddle/fluid/inference/tensorrt/convert/sparse_fc_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/sparse_fc_op.cc @@ -116,7 +116,7 @@ class SparseFcOpConverter : public OpConverter { PADDLE_ENFORCE_NOT_NULL( Y_v, platform::errors::NotFound( - "Can not find %s presistale var of sparse_fc in scope.", w_name)); + "Can not find %s presistable var of sparse_fc in scope.", w_name)); auto* Y_t = Y_v->GetMutable(); int x_num_col_dims = op_desc.HasAttr("x_num_col_dims") diff --git a/paddle/fluid/inference/tensorrt/convert/trans_layernorm_op.cc b/paddle/fluid/inference/tensorrt/convert/trans_layernorm_op.cc index dc257beb14683..a5db8ed88c4c0 100644 --- a/paddle/fluid/inference/tensorrt/convert/trans_layernorm_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/trans_layernorm_op.cc @@ -53,7 +53,7 @@ class TransLayerNormOpConverter : public OpConverter { nvinfer1::ILayer* layernorm_layer = nullptr; if (engine_->with_dynamic_shape()) { // For dynamic shape, - // the shape of mean and variance will be determine in configuPlugin. + // the shape of mean and variance will be determine in configurePlugin. std::vector mean_shape{1}; std::vector variance_shape{1}; bool with_fp16 = diff --git a/paddle/fluid/inference/tensorrt/convert/ut_helper.h b/paddle/fluid/inference/tensorrt/convert/ut_helper.h index 8901d0a43fd41..347f6f500c7c8 100644 --- a/paddle/fluid/inference/tensorrt/convert/ut_helper.h +++ b/paddle/fluid/inference/tensorrt/convert/ut_helper.h @@ -247,7 +247,7 @@ class TRTConvertValidation { std::unique_ptr op_desc_; const std::unordered_set& parameters_; framework::Scope& scope_; - // The ITensor of trt does not cotain the batch size, + // The ITensor of trt does not contain the batch size, // bug, in most cases, we need to set batch size for // fluid's tensor shape. This variable indicates // whether to add batch size to tensor shape of fluid. From b4b22d545bcafc43c84429452c0ab091caa69eb3 Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 4 Mar 2024 11:53:24 +0800 Subject: [PATCH 259/282] Fix Successed Succeed,etc (#62331) --- paddle/fluid/operators/top_k_op.cu | 2 +- paddle/phi/backends/custom/custom_device.cc | 2 +- paddle/phi/core/cuda_stream.h | 2 +- paddle/phi/core/custom_kernel.cc | 4 ++-- paddle/phi/kernels/gpu/top_k_kernel.cu | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/paddle/fluid/operators/top_k_op.cu b/paddle/fluid/operators/top_k_op.cu index ef6172b6965f2..003f670133e45 100644 --- a/paddle/fluid/operators/top_k_op.cu +++ b/paddle/fluid/operators/top_k_op.cu @@ -93,7 +93,7 @@ class TopkOpCUDAKernel : public framework::OpKernel { if ((input_width <= 1024 || k >= 128 || k == input_width)) { if (phi::funcs::SortTopk( dev_ctx, input, input_width, input_height, k, output, indices)) { - // Successed, return. + // Succeed, return. return; } else { LOG(INFO) << "TopKOP: Some errors happened when use cub sorting, use " diff --git a/paddle/phi/backends/custom/custom_device.cc b/paddle/phi/backends/custom/custom_device.cc index 4e2108cbbd9e4..53fe86492e2e9 100644 --- a/paddle/phi/backends/custom/custom_device.cc +++ b/paddle/phi/backends/custom/custom_device.cc @@ -1106,7 +1106,7 @@ void LoadCustomRuntimeLib(const std::string& dso_lib_path, void* dso_handle) { } LoadCustomRuntimeLib( runtime_params, std::move(device_interface), dso_lib_path, dso_handle); - LOG(INFO) << "Successed in loading custom runtime in lib: " << dso_lib_path; + LOG(INFO) << "Succeed in loading custom runtime in lib: " << dso_lib_path; } #undef INTERFACE_UNIMPLEMENT diff --git a/paddle/phi/core/cuda_stream.h b/paddle/phi/core/cuda_stream.h index b27770b081433..b6900cdabf2b3 100644 --- a/paddle/phi/core/cuda_stream.h +++ b/paddle/phi/core/cuda_stream.h @@ -155,7 +155,7 @@ class CUDAStream { private: Place place_; Stream stream_; - bool owned_{false}; // whether the stream is created and onwed by self + bool owned_{false}; // whether the stream is created and owned by self }; } // namespace phi diff --git a/paddle/phi/core/custom_kernel.cc b/paddle/phi/core/custom_kernel.cc index bc737fa398baf..3f694518d2dcc 100644 --- a/paddle/phi/core/custom_kernel.cc +++ b/paddle/phi/core/custom_kernel.cc @@ -55,12 +55,12 @@ void CustomKernelMap::RegisterCustomKernels() { kernels[pair.first][info_pair.first] = info_pair.second; - VLOG(3) << "Successed in registering kernel [" << pair.first << ":" + VLOG(3) << "Succeed in registering kernel [" << pair.first << ":" << info_pair.first << "] to Paddle. It will be used like native ones."; } } - LOG(INFO) << "Successed in loading " << kernels_.size() + LOG(INFO) << "Succeed in loading " << kernels_.size() << " custom kernel(s) from loaded lib(s), will be " << "used like native ones."; kernels_.clear(); diff --git a/paddle/phi/kernels/gpu/top_k_kernel.cu b/paddle/phi/kernels/gpu/top_k_kernel.cu index 1d93ef1a2790f..d946bc50adfca 100644 --- a/paddle/phi/kernels/gpu/top_k_kernel.cu +++ b/paddle/phi/kernels/gpu/top_k_kernel.cu @@ -117,7 +117,7 @@ void TopkKernel(const Context& dev_ctx, out, indices, largest)) { - // Successed, return. + // Succeed, return. return; } else { VLOG(4) << "TopKOP: Some errors happened when use cub sorting, use " From 79b66828eb9d0979764882c633762b51a0fd3f01 Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 4 Mar 2024 11:54:04 +0800 Subject: [PATCH 260/282] Fix currnet current, etc (#62330) --- paddle/phi/core/distributed/auto_parallel/dist_tensor.h | 2 +- .../phi/core/distributed/auto_parallel/inferspmd_utils.h | 2 +- paddle/phi/core/distributed/auto_parallel/proto_helper.cc | 8 ++++---- paddle/phi/core/distributed/auto_parallel/proto_helper.h | 4 ++-- .../auto_parallel/reshard/nd_mesh_reshard_function.cc | 2 +- .../auto_parallel/reshard/same_status_reshard_function.cc | 2 +- paddle/phi/core/sparse_coo_tensor.h | 4 ++-- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/paddle/phi/core/distributed/auto_parallel/dist_tensor.h b/paddle/phi/core/distributed/auto_parallel/dist_tensor.h index bf5b083aa6e6f..5af868ef01f17 100644 --- a/paddle/phi/core/distributed/auto_parallel/dist_tensor.h +++ b/paddle/phi/core/distributed/auto_parallel/dist_tensor.h @@ -79,7 +79,7 @@ class DistTensor final const Placements& placements); /// \brief Construct a empty dist tensor (for infer spmd) - /// \param dims The global dimension of the currnet Tensor. + /// \param dims The global dimension of the current Tensor. /// \param dist_attr The distributed attributes of the current tensor. DistTensor(const DDim& dims, const TensorDistAttr& dist_attr); diff --git a/paddle/phi/core/distributed/auto_parallel/inferspmd_utils.h b/paddle/phi/core/distributed/auto_parallel/inferspmd_utils.h index 71395507a0951..d2c22bcd08db0 100644 --- a/paddle/phi/core/distributed/auto_parallel/inferspmd_utils.h +++ b/paddle/phi/core/distributed/auto_parallel/inferspmd_utils.h @@ -107,7 +107,7 @@ struct InferSpmdFnImpl { } }; - // for vecotr slot + // for vector slot template struct InferSpmdFnCallHelper&, Tail...> { diff --git a/paddle/phi/core/distributed/auto_parallel/proto_helper.cc b/paddle/phi/core/distributed/auto_parallel/proto_helper.cc index e8e4197a63c08..fad63c15d63bd 100644 --- a/paddle/phi/core/distributed/auto_parallel/proto_helper.cc +++ b/paddle/phi/core/distributed/auto_parallel/proto_helper.cc @@ -35,8 +35,8 @@ auto_parallel::ProcessMeshProto to_proto(const ProcessMesh& process_mesh) { } auto_parallel::DeviceCapabilityProto to_proto( - const auto_parallel::DeviceCapability& device_capibilty) { - TO_PROTO_HELPER(device_capibilty, auto_parallel::DeviceCapabilityProto); + const auto_parallel::DeviceCapability& device_capability) { + TO_PROTO_HELPER(device_capability, auto_parallel::DeviceCapabilityProto); } auto_parallel::DeviceProto to_proto(const auto_parallel::Device& device) { @@ -44,8 +44,8 @@ auto_parallel::DeviceProto to_proto(const auto_parallel::Device& device) { } auto_parallel::LinkCapabilityProto to_proto( - const auto_parallel::LinkCapability& link_capibilty) { - TO_PROTO_HELPER(link_capibilty, auto_parallel::LinkCapabilityProto); + const auto_parallel::LinkCapability& link_capability) { + TO_PROTO_HELPER(link_capability, auto_parallel::LinkCapabilityProto); } auto_parallel::LinkProto to_proto(const auto_parallel::Link& link) { diff --git a/paddle/phi/core/distributed/auto_parallel/proto_helper.h b/paddle/phi/core/distributed/auto_parallel/proto_helper.h index 66bdf2af74406..840c0eb95f89e 100644 --- a/paddle/phi/core/distributed/auto_parallel/proto_helper.h +++ b/paddle/phi/core/distributed/auto_parallel/proto_helper.h @@ -30,10 +30,10 @@ auto_parallel::TensorDistAttrProto to_proto(const TensorDistAttr& dist_attr); auto_parallel::ProcessMeshProto to_proto(const ProcessMesh& dist_attr); auto_parallel::DeviceCapabilityProto to_proto( - const auto_parallel::DeviceCapability& device_capibilty); + const auto_parallel::DeviceCapability& device_capability); auto_parallel::DeviceProto to_proto(const auto_parallel::Device& device); auto_parallel::LinkCapabilityProto to_proto( - const auto_parallel::LinkCapability& link_capibilty); + const auto_parallel::LinkCapability& link_capability); auto_parallel::LinkProto to_proto(const auto_parallel::Link& link); auto_parallel::DeviceMeshProto to_proto(const auto_parallel::DeviceMesh& link); auto_parallel::DistributedMapperProto to_proto( diff --git a/paddle/phi/core/distributed/auto_parallel/reshard/nd_mesh_reshard_function.cc b/paddle/phi/core/distributed/auto_parallel/reshard/nd_mesh_reshard_function.cc index b7a6679590e63..7a044209677d3 100644 --- a/paddle/phi/core/distributed/auto_parallel/reshard/nd_mesh_reshard_function.cc +++ b/paddle/phi/core/distributed/auto_parallel/reshard/nd_mesh_reshard_function.cc @@ -228,7 +228,7 @@ void SameNdMeshReshardFunction::Eval(phi::DeviceContext* dev_ctx, bool is_partial = in_partial_status.count(out_mesh_axis) != 0; VLOG(3) << "Step4: out_mesh axis : " << out_mesh_axis - << "; paratial state :" << is_partial; + << "; partial state :" << is_partial; // 4.1 Calculate the dist_attr after this transform TensorDistAttr real_out_dist_attr(out->dist_attr()); std::vector real_dims_mapping = diff --git a/paddle/phi/core/distributed/auto_parallel/reshard/same_status_reshard_function.cc b/paddle/phi/core/distributed/auto_parallel/reshard/same_status_reshard_function.cc index 2869951addffc..0a86275203b51 100644 --- a/paddle/phi/core/distributed/auto_parallel/reshard/same_status_reshard_function.cc +++ b/paddle/phi/core/distributed/auto_parallel/reshard/same_status_reshard_function.cc @@ -91,7 +91,7 @@ void SameStatusReshardFunction::Eval(phi::DeviceContext* dev_ctx, if (src == cur_global_rank) { VLOG(3) << "Send from src " << src << " to dst " << dst; int64_t dst_local_rank = GetLocalRankInParticipate(all_process_ids, dst); - // Sice send kernel only has input, so we don't need to infermeta + // Since send kernel only has input, so we don't need to infermeta // actually. According to this reason, just use the kernel directly. RESHARD_FUNCTOR_WITH_COMM(dev_ctx, PSendKernel, diff --git a/paddle/phi/core/sparse_coo_tensor.h b/paddle/phi/core/sparse_coo_tensor.h index d0759bedcf557..61c8b0c3d2a5b 100644 --- a/paddle/phi/core/sparse_coo_tensor.h +++ b/paddle/phi/core/sparse_coo_tensor.h @@ -127,7 +127,7 @@ class SparseCooTensor : public TensorBase, /// \brief Test whether the non_zero_elements_ storage is allocated. /// In special cases, when nnz=0, non_zero_elements_ will not need to be - /// initialized, but it is neccessary to return true here, otherwise the + /// initialized, but it is necessary to return true here, otherwise the /// gradient will be None. return Whether the non_zero_elements_ storage is /// allocated. bool initialized() const override { @@ -189,7 +189,7 @@ class SparseCooTensor : public TensorBase, /// \brief get the sparse dim int32_t sparse_dim() const; - /// \brief get the dnese dim + /// \brief get the dense dim int32_t dense_dim() const; /// \brief Returns the meta information of the tensor. From 114e8c17006d49c9e92e08b9e95627a33a7ee68e Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 4 Mar 2024 11:56:02 +0800 Subject: [PATCH 261/282] Update op_utils.h (#62329) --- paddle/phi/core/compat/op_utils.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/paddle/phi/core/compat/op_utils.h b/paddle/phi/core/compat/op_utils.h index b2c334d89023d..12a419e5d6fcc 100644 --- a/paddle/phi/core/compat/op_utils.h +++ b/paddle/phi/core/compat/op_utils.h @@ -29,11 +29,6 @@ namespace phi { const static std::string deprecated_kernel_name = "deprecated"; // NOLINT -const std::unordered_set standard_kernel_suffixs({ - "sr", // SelectedRows kernel - "raw" // fallback kernel of original fluid op -}); - /** * Some fluid ops are no longer used under the corresponding official API * system of 2.0. These names need to correspond to the official API names From 8ae036f0401cdcb5cdf70e1b27b38b52d9b1559c Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 4 Mar 2024 11:56:26 +0800 Subject: [PATCH 262/282] Fix contians contains, etc (#62324) --- .../plugin/preln_groupnorm_act_op_plugin.h | 2 +- .../plugin/skip_groupnorm_act_op_plugin.h | 2 +- paddle/fluid/inference/utils/singleton.h | 2 +- .../memory/allocation/allocator_facade.cc | 2 +- .../fluid/memory/allocation/mmap_allocator.cc | 12 +++---- .../allocation/stream_safe_xpu_allocator.cc | 4 +-- ...l_memory_auto_growth_best_fit_allocator.cc | 5 ++- ...al_memory_auto_growth_best_fit_allocator.h | 2 +- paddle/fluid/memory/malloc.h | 2 +- paddle/fluid/memory/stats.cc | 4 +-- paddle/fluid/memory/stats.h | 36 +++++++++---------- .../operators/cinn/cinn_launch_context.cc | 8 ++--- 12 files changed, 40 insertions(+), 41 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/plugin/preln_groupnorm_act_op_plugin.h b/paddle/fluid/inference/tensorrt/plugin/preln_groupnorm_act_op_plugin.h index e4c76e2d652ee..2d5dde9190103 100644 --- a/paddle/fluid/inference/tensorrt/plugin/preln_groupnorm_act_op_plugin.h +++ b/paddle/fluid/inference/tensorrt/plugin/preln_groupnorm_act_op_plugin.h @@ -144,7 +144,7 @@ class PrelnGroupnormActPluginDynamic : public DynamicPluginTensorRT { const nvinfer1::DynamicPluginTensorDesc* out, int nbOutputs) TRT_NOEXCEPT override { // sizeof(float2) * maxBatchSize * maxNumberOfGroup. float2 - // contians two buffers for sum and squared sum; + // contains two buffers for sum and squared sum; ws_ = sizeof(float) * 2 * in[0].max.d[0] * groups_; } diff --git a/paddle/fluid/inference/tensorrt/plugin/skip_groupnorm_act_op_plugin.h b/paddle/fluid/inference/tensorrt/plugin/skip_groupnorm_act_op_plugin.h index 0a93559f5ee2c..1260bbb8e2917 100644 --- a/paddle/fluid/inference/tensorrt/plugin/skip_groupnorm_act_op_plugin.h +++ b/paddle/fluid/inference/tensorrt/plugin/skip_groupnorm_act_op_plugin.h @@ -139,7 +139,7 @@ class SkipGroupnormActPluginDynamic : public DynamicPluginTensorRT { const nvinfer1::DynamicPluginTensorDesc* out, int nbOutputs) TRT_NOEXCEPT override { // sizeof(float2) * maxBatchSize * maxNumberOfGroup. float2 - // contians two buffers for sum and squared sum; + // contains two buffers for sum and squared sum; ws_ = sizeof(float) * 2 * in[0].max.d[0] * groups_; } diff --git a/paddle/fluid/inference/utils/singleton.h b/paddle/fluid/inference/utils/singleton.h index 5c2a1bf563f21..82a50e6042c76 100644 --- a/paddle/fluid/inference/utils/singleton.h +++ b/paddle/fluid/inference/utils/singleton.h @@ -35,7 +35,7 @@ struct Singleton { }; /* - * An registor for any type. + * An Registry for any type. * NOTE not thread-safe. */ template diff --git a/paddle/fluid/memory/allocation/allocator_facade.cc b/paddle/fluid/memory/allocation/allocator_facade.cc index eff0a1891ed7b..e340d55ee02d1 100644 --- a/paddle/fluid/memory/allocation/allocator_facade.cc +++ b/paddle/fluid/memory/allocation/allocator_facade.cc @@ -232,7 +232,7 @@ class AllocatorFacadePrivate { // Note(Ruibiao): For GPU multi-stream case without CUDA graph // capturing, the 'allocators_' map(place -> Allocator) hold the - // StreamSafeCUDAAllocator relate to defaultstream (i.e., the stream + // StreamSafeCUDAAllocator relate to default stream (i.e., the stream // directly got from DeviceContext), while the 'cuda_allocators_' map // (place -> map(stream -> Allocator)) hold the StreamSafeCUDAAllocator // relate to non-default stream (i.e., the stream users pass in). The diff --git a/paddle/fluid/memory/allocation/mmap_allocator.cc b/paddle/fluid/memory/allocation/mmap_allocator.cc index 3b371ed20e59c..a4a05df1dcaa9 100644 --- a/paddle/fluid/memory/allocation/mmap_allocator.cc +++ b/paddle/fluid/memory/allocation/mmap_allocator.cc @@ -90,7 +90,7 @@ void AllocateMemoryMap( PADDLE_ENFORCE_EQ(ftruncate(fd, size), 0, platform::errors::Unavailable( - "Fruncate a file to a specified length failed!")); + "Truncate a file to a specified length failed!")); if (flags & MAPPED_SHAREDMEM) { *map_ptr_ = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); @@ -109,7 +109,7 @@ void AllocateMemoryMap( PADDLE_ENFORCE_NE(::close(fd), -1, platform::errors::Unavailable( - "Error closing memory maped file <", filename, ">")); + "Error closing memory mapped file <", filename, ">")); *fd_ = -1; } @@ -129,10 +129,10 @@ AllocateRefcountedMemoryMapAllocation(std::string filename, base_ptr = MemoryMapAllocationPool::Instance().GetById(buffer_id).mmap_ptr_; VLOG(4) << "Get a cached shm " << filename; } - void *aliged_base_ptr = + void *aligned_base_ptr = static_cast(static_cast(base_ptr) + mmap_alignment); return std::make_shared( - aliged_base_ptr, size, filename, flags, fd, buffer_id); + aligned_base_ptr, size, filename, flags, fd, buffer_id); } RefcountedMemoryMapAllocation::RefcountedMemoryMapAllocation( @@ -267,7 +267,7 @@ std::shared_ptr AllocateMemoryMapWriterAllocation( PADDLE_ENFORCE_EQ(ftruncate(fd, size), 0, platform::errors::Unavailable( - "Fruncate a file to a specified length failed!")); + "Truncate a file to a specified length failed!")); void *ptr = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); PADDLE_ENFORCE_NE(ptr, @@ -337,7 +337,7 @@ MemoryMapAllocationPool *MemoryMapAllocationPool::pool_ = nullptr; void MemoryMapAllocationPool::Insert(const MemoryMapInfo &memory_map) { std::lock_guard guard(mtx_); memory_map_allocations_.push_back(memory_map); - VLOG(4) << this << "Intsert a new shm: " << memory_map.file_name_; + VLOG(4) << this << "Insert a new shm: " << memory_map.file_name_; } int MemoryMapAllocationPool::FindFromCache(const int &flag, diff --git a/paddle/fluid/memory/allocation/stream_safe_xpu_allocator.cc b/paddle/fluid/memory/allocation/stream_safe_xpu_allocator.cc index 7f48ef5ab5007..9809b1e5358c4 100644 --- a/paddle/fluid/memory/allocation/stream_safe_xpu_allocator.cc +++ b/paddle/fluid/memory/allocation/stream_safe_xpu_allocator.cc @@ -175,8 +175,8 @@ uint64_t StreamSafeXPUAllocator::ReleaseImpl(const platform::Place& place) { } void StreamSafeXPUAllocator::ProcessUnfreedAllocations() { - // NOTE(Ruibiao): This condition is to reduce lock competion. It does not need - // to be thread-safe since here occasional misjudgments are permissible. + // NOTE(Ruibiao): This condition is to reduce lock completion. It does not + // need to be thread-safe since here occasional misjudgments are permissible. if (unfreed_allocations_.empty()) { return; } diff --git a/paddle/fluid/memory/allocation/virtual_memory_auto_growth_best_fit_allocator.cc b/paddle/fluid/memory/allocation/virtual_memory_auto_growth_best_fit_allocator.cc index 0c5bfe7bd1a90..52399df8ce5ff 100644 --- a/paddle/fluid/memory/allocation/virtual_memory_auto_growth_best_fit_allocator.cc +++ b/paddle/fluid/memory/allocation/virtual_memory_auto_growth_best_fit_allocator.cc @@ -22,9 +22,8 @@ namespace paddle { namespace memory { namespace allocation { -bool NeedSplit(size_t block_size, size_t alignment, size_t allock_size) { - return block_size > (allock_size * 2) || - (block_size - allock_size) > alignment; +bool NeedSplit(size_t block_size, size_t alignment, size_t alloc_size) { + return block_size > (alloc_size * 2) || (block_size - alloc_size) > alignment; } VirtualMemoryAutoGrowthBestFitAllocator:: diff --git a/paddle/fluid/memory/allocation/virtual_memory_auto_growth_best_fit_allocator.h b/paddle/fluid/memory/allocation/virtual_memory_auto_growth_best_fit_allocator.h index ce5cbdeb12593..b8c7e38da00b8 100644 --- a/paddle/fluid/memory/allocation/virtual_memory_auto_growth_best_fit_allocator.h +++ b/paddle/fluid/memory/allocation/virtual_memory_auto_growth_best_fit_allocator.h @@ -46,7 +46,7 @@ struct BlockAllocation : public Allocation { * Like AutoGrowthBestFitAllocator, VirtualMemoryAutoGrowthBestFitAllocator will * gradually apply to GPU for video memory as the model uses more video memory. * However, the difference is that VirtualMemoryAutoGrowthBestFitAllocator uses - * nviaid's virtual memory management technology and obtains the virtual memory + * NVIDIA's virtual memory management technology and obtains the virtual memory * address. If the video memory applied for twice is continuous, we can combine * the two video memories later. This combination can greatly reduce * fragmentation. diff --git a/paddle/fluid/memory/malloc.h b/paddle/fluid/memory/malloc.h index a9286499ec24c..dc25b85c8b040 100644 --- a/paddle/fluid/memory/malloc.h +++ b/paddle/fluid/memory/malloc.h @@ -71,7 +71,7 @@ struct ThrustAllocator { place_ = place; stream_ = stream; } - ~ThrustAllocator() { VLOG(2) << "destory allocator"; } + ~ThrustAllocator() { VLOG(2) << "destroy allocator"; } char* allocate(std::ptrdiff_t num_bytes) { VLOG(2) << "allocate " << num_bytes << " bytes"; auto storage = memory::AllocShared( diff --git a/paddle/fluid/memory/stats.cc b/paddle/fluid/memory/stats.cc index 39b01c46f389e..2d66a5b6838b0 100644 --- a/paddle/fluid/memory/stats.cc +++ b/paddle/fluid/memory/stats.cc @@ -36,7 +36,7 @@ class StatRegistry { auto it = stat_map_.find(GetStatKey(stat_type, dev_id)); if (it == stat_map_.end()) { PADDLE_THROW(platform::errors::InvalidArgument( - "The STAT type \"%s\" for device %d has not been regeistered.", + "The STAT type \"%s\" for device %d has not been registered.", stat_type.c_str(), dev_id)); } @@ -171,7 +171,7 @@ int RegisterAllStats() { return 0; } -UNUSED static int regiester_all_stats = RegisterAllStats(); +UNUSED static int register_all_stats = RegisterAllStats(); } // namespace memory } // namespace paddle diff --git a/paddle/fluid/memory/stats.h b/paddle/fluid/memory/stats.h index b6d722b62a4b0..78d20d968c968 100644 --- a/paddle/fluid/memory/stats.h +++ b/paddle/fluid/memory/stats.h @@ -42,7 +42,7 @@ struct ThreadLocalStatBase { friend std::ostream& operator<<(std::ostream& os, const ThreadLocalStatBase& stat) { - os << "{cuerrent : " << stat.current << ", peak : " << stat.peak << "}"; + os << "{current : " << stat.current << ", peak : " << stat.peak << "}"; return os; } }; @@ -136,7 +136,7 @@ void HostMemoryStatUpdate(const std::string& stat_type, void LogDeviceMemoryStats(const platform::Place& place, const std::string& op_name); -#define DEVICE_MEMORY_STAT_FUNC_SWITHCH_CASE(item, id) \ +#define DEVICE_MEMORY_STAT_FUNC_SWITCH_CASE(item, id) \ case id: \ stat = paddle::memory::Stat< \ paddle::memory::DeviceMemoryStat##item##id>::GetInstance(); \ @@ -146,22 +146,22 @@ void LogDeviceMemoryStats(const platform::Place& place, [&] { \ paddle::memory::StatBase* stat = nullptr; \ switch (id) { \ - DEVICE_MEMORY_STAT_FUNC_SWITHCH_CASE(item, 0); \ - DEVICE_MEMORY_STAT_FUNC_SWITHCH_CASE(item, 1); \ - DEVICE_MEMORY_STAT_FUNC_SWITHCH_CASE(item, 2); \ - DEVICE_MEMORY_STAT_FUNC_SWITHCH_CASE(item, 3); \ - DEVICE_MEMORY_STAT_FUNC_SWITHCH_CASE(item, 4); \ - DEVICE_MEMORY_STAT_FUNC_SWITHCH_CASE(item, 5); \ - DEVICE_MEMORY_STAT_FUNC_SWITHCH_CASE(item, 6); \ - DEVICE_MEMORY_STAT_FUNC_SWITHCH_CASE(item, 7); \ - DEVICE_MEMORY_STAT_FUNC_SWITHCH_CASE(item, 8); \ - DEVICE_MEMORY_STAT_FUNC_SWITHCH_CASE(item, 9); \ - DEVICE_MEMORY_STAT_FUNC_SWITHCH_CASE(item, 10); \ - DEVICE_MEMORY_STAT_FUNC_SWITHCH_CASE(item, 11); \ - DEVICE_MEMORY_STAT_FUNC_SWITHCH_CASE(item, 12); \ - DEVICE_MEMORY_STAT_FUNC_SWITHCH_CASE(item, 13); \ - DEVICE_MEMORY_STAT_FUNC_SWITHCH_CASE(item, 14); \ - DEVICE_MEMORY_STAT_FUNC_SWITHCH_CASE(item, 15); \ + DEVICE_MEMORY_STAT_FUNC_SWITCH_CASE(item, 0); \ + DEVICE_MEMORY_STAT_FUNC_SWITCH_CASE(item, 1); \ + DEVICE_MEMORY_STAT_FUNC_SWITCH_CASE(item, 2); \ + DEVICE_MEMORY_STAT_FUNC_SWITCH_CASE(item, 3); \ + DEVICE_MEMORY_STAT_FUNC_SWITCH_CASE(item, 4); \ + DEVICE_MEMORY_STAT_FUNC_SWITCH_CASE(item, 5); \ + DEVICE_MEMORY_STAT_FUNC_SWITCH_CASE(item, 6); \ + DEVICE_MEMORY_STAT_FUNC_SWITCH_CASE(item, 7); \ + DEVICE_MEMORY_STAT_FUNC_SWITCH_CASE(item, 8); \ + DEVICE_MEMORY_STAT_FUNC_SWITCH_CASE(item, 9); \ + DEVICE_MEMORY_STAT_FUNC_SWITCH_CASE(item, 10); \ + DEVICE_MEMORY_STAT_FUNC_SWITCH_CASE(item, 11); \ + DEVICE_MEMORY_STAT_FUNC_SWITCH_CASE(item, 12); \ + DEVICE_MEMORY_STAT_FUNC_SWITCH_CASE(item, 13); \ + DEVICE_MEMORY_STAT_FUNC_SWITCH_CASE(item, 14); \ + DEVICE_MEMORY_STAT_FUNC_SWITCH_CASE(item, 15); \ default: \ PADDLE_THROW(paddle::platform::errors::OutOfRange( \ "Only support device id between [0, 15] for device memory stats," \ diff --git a/paddle/fluid/operators/cinn/cinn_launch_context.cc b/paddle/fluid/operators/cinn/cinn_launch_context.cc index f75e77a075177..efd23f050989d 100644 --- a/paddle/fluid/operators/cinn/cinn_launch_context.cc +++ b/paddle/fluid/operators/cinn/cinn_launch_context.cc @@ -412,10 +412,10 @@ std::unique_ptr CinnLaunchContext::BuildCompiledProgram( // build a map that links the name of a Paddle variable to its VarDesc const std::unordered_set& nodes = graph.Nodes(); - std::unordered_map original_vardescs; + std::unordered_map original_var_descs; for (auto* node : nodes) { if (node->IsVar() && node->Var()) { - original_vardescs.emplace(node->Name(), node->Var()); + original_var_descs.emplace(node->Name(), node->Var()); } } @@ -433,8 +433,8 @@ std::unique_ptr CinnLaunchContext::BuildCompiledProgram( framework::VarDesc* var_desc = block->Var(var_name); var_desc->SetType(framework::proto::VarType::LOD_TENSOR); - auto res = original_vardescs.find(var_name); - if (res != original_vardescs.end()) { + auto res = original_var_descs.find(var_name); + if (res != original_var_descs.end()) { auto* ori_desc = res->second; var_desc->SetPersistable(ori_desc->Persistable()); var_desc->SetIsParameter(ori_desc->IsParameter()); From a58820650ab6c19135cc62b03c21144d4bbc1142 Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 4 Mar 2024 11:57:12 +0800 Subject: [PATCH 263/282] Fix multihead_mamul_fc multihead_matmul_fc, etc (#62317) --- .../tensorrt/convert/activation_op.cc | 6 ++-- .../tensorrt/convert/affine_channel_op.cc | 8 ++--- .../tensorrt/convert/bitwise_not_op.cc | 2 +- .../inference/tensorrt/convert/conv3d_op.cc | 2 +- .../convert/cross_multihead_matmul_op.cc | 9 +++--- .../tensorrt/convert/dequantize_linear_op.cc | 2 +- .../convert/flash_multihead_matmul_op.cc | 29 ++++++++++--------- .../generic_and_custom_plugin_creater.cc | 6 ++-- .../tensorrt/convert/multihead_matmul_op.cc | 10 +++---- .../convert/multihead_matmul_roformer_op.cc | 2 +- .../convert/qk_multihead_matmul_op.cc | 6 ++-- .../convert/sparse_multihead_matmul_op.cc | 5 ++-- 12 files changed, 45 insertions(+), 42 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/convert/activation_op.cc b/paddle/fluid/inference/tensorrt/convert/activation_op.cc index f09e5091ae9b1..f9057ab7b0a21 100644 --- a/paddle/fluid/inference/tensorrt/convert/activation_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/activation_op.cc @@ -181,9 +181,9 @@ class STanhOpConverter : public ActivationOpConverter { STanhOpConverter() { op_type_ = "stanh"; } }; -class ThreasholdedReluOpConverter : public ActivationOpConverter { +class ThresholdedReluOpConverter : public ActivationOpConverter { public: - ThreasholdedReluOpConverter() { op_type_ = "thresholded_relu"; } + ThresholdedReluOpConverter() { op_type_ = "thresholded_relu"; } }; #endif @@ -201,5 +201,5 @@ REGISTER_TRT_OP_CONVERTER(selu, SeluOpConverter); REGISTER_TRT_OP_CONVERTER(softsign, SoftsignOpConverter); REGISTER_TRT_OP_CONVERTER(softplus, SoftplusOpConverter); REGISTER_TRT_OP_CONVERTER(stanh, STanhOpConverter); -REGISTER_TRT_OP_CONVERTER(thresholded_relu, ThreasholdedReluOpConverter); +REGISTER_TRT_OP_CONVERTER(thresholded_relu, ThresholdedReluOpConverter); #endif diff --git a/paddle/fluid/inference/tensorrt/convert/affine_channel_op.cc b/paddle/fluid/inference/tensorrt/convert/affine_channel_op.cc index d7699c7c1003c..9f19b0b41096f 100644 --- a/paddle/fluid/inference/tensorrt/convert/affine_channel_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/affine_channel_op.cc @@ -36,7 +36,7 @@ class AffineChannelOpConverter : public OpConverter { std::string output_name = op_desc.Output("Out").front(); auto input_tensor = engine_->GetITensor(input_name); - auto idim = input_tensor->getDimensions(); + auto input_dim = input_tensor->getDimensions(); auto* scale_v = scope.FindVar(scale_name); auto* scale_t = scale_v->GetMutable(); @@ -49,17 +49,17 @@ class AffineChannelOpConverter : public OpConverter { engine_->GetFp32TrtWeight(bias_name, *bias_t).get().values)); // tensorrt scalend layer only support spatial dims >= 2, - // so nhwc is not availabe (spatial dims == 0) + // so nhwc is not available (spatial dims == 0) const int channel_axis = engine_->with_dynamic_shape(); TensorRTEngine::Weight scale_weights{ nvinfer1::DataType::kFLOAT, static_cast(scale_ptr), - static_cast(idim.d[channel_axis])}; + static_cast(input_dim.d[channel_axis])}; TensorRTEngine::Weight bias_weights{ nvinfer1::DataType::kFLOAT, static_cast(bias_ptr), - static_cast(idim.d[channel_axis])}; + static_cast(input_dim.d[channel_axis])}; TensorRTEngine::Weight power_weights{ nvinfer1::DataType::kFLOAT, nullptr, 0}; diff --git a/paddle/fluid/inference/tensorrt/convert/bitwise_not_op.cc b/paddle/fluid/inference/tensorrt/convert/bitwise_not_op.cc index a944527313a02..63a02d4e393e8 100644 --- a/paddle/fluid/inference/tensorrt/convert/bitwise_not_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/bitwise_not_op.cc @@ -42,7 +42,7 @@ class BitwiseNotConverter : public OpConverter { nvinfer1::Dims input_dims = input_tensor->getDimensions(); // set up a elementwise -1 tensor, can not get the dims info for - // dynamic_shape so just let it broadcaste + // dynamic_shape so just let it broadcast nvinfer1::Dims neg_one_tensor_dims; neg_one_tensor_dims.nbDims = input_dims.nbDims; for (int i = 0; i < input_dims.nbDims; ++i) { diff --git a/paddle/fluid/inference/tensorrt/convert/conv3d_op.cc b/paddle/fluid/inference/tensorrt/convert/conv3d_op.cc index 1df92f0641040..37a53d31f47b5 100644 --- a/paddle/fluid/inference/tensorrt/convert/conv3d_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/conv3d_op.cc @@ -35,7 +35,7 @@ void ConvertConv3d(TensorRTEngine* engine, auto* Y_v = scope.FindVar(filter_var_name); PADDLE_ENFORCE_NOT_NULL( Y_v, - platform::errors::NotFound("Can not find %s presistale var in scope.", + platform::errors::NotFound("Can not find %s presistable var in scope.", filter_var_name)); auto* Y_t = Y_v->GetMutable(); bool enable_int8 = op_desc.HasAttr("enable_int8"); diff --git a/paddle/fluid/inference/tensorrt/convert/cross_multihead_matmul_op.cc b/paddle/fluid/inference/tensorrt/convert/cross_multihead_matmul_op.cc index 6a1cf1951f9a6..df5665b75b34e 100644 --- a/paddle/fluid/inference/tensorrt/convert/cross_multihead_matmul_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/cross_multihead_matmul_op.cc @@ -24,8 +24,9 @@ class CrossMultiheadMatMulOpConverter : public OpConverter { void operator()(const framework::proto::OpDesc& op, const framework::Scope& scope, bool test_mode) override { - VLOG(3) << "convert a cross_multihead_mamul op to a corresponding tensorrt " - "network structure"; + VLOG(3) + << "convert a cross_multihead_matmul op to a corresponding tensorrt " + "network structure"; bool with_fp16 = engine_->WithFp16() && !engine_->disable_trt_plugin_fp16(); if (engine_->precision() == phi::DataType::INT8) { with_fp16 = true; @@ -109,7 +110,7 @@ class CrossMultiheadMatMulOpConverter : public OpConverter { weight_q, bias_q); fc_q_layer->setName( - ("multihead_mamul_fc_q(Output: " + output_name + ")").c_str()); + ("multihead_matmul_fc_q(Output: " + output_name + ")").c_str()); // add shuffle for fc layer auto* reshape_after_fc_q_layer = @@ -211,7 +212,7 @@ class CrossMultiheadMatMulOpConverter : public OpConverter { weight_kv, bias_kv); fc_layer->setName( - ("multihead_mamul_fc(Output: " + output_name + ")").c_str()); + ("multihead_matmul_fc(Output: " + output_name + ")").c_str()); // add shuffle for fc layer auto* reshape_after_fc_layer = diff --git a/paddle/fluid/inference/tensorrt/convert/dequantize_linear_op.cc b/paddle/fluid/inference/tensorrt/convert/dequantize_linear_op.cc index 9b88e14fc9efe..662769e7f24ec 100644 --- a/paddle/fluid/inference/tensorrt/convert/dequantize_linear_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/dequantize_linear_op.cc @@ -32,7 +32,7 @@ class DequantizeLinearOpConverter : public OpConverter { // Create constant layer for scale PADDLE_ENFORCE_NOT_NULL( scale_var, - platform::errors::NotFound("Can not find %s presistale var in scope.", + platform::errors::NotFound("Can not find %s presistable var in scope.", op_desc.Input("Scale")[0])); auto* scale_t = scale_var->GetMutable(); int n_scale = scale_t->numel(); diff --git a/paddle/fluid/inference/tensorrt/convert/flash_multihead_matmul_op.cc b/paddle/fluid/inference/tensorrt/convert/flash_multihead_matmul_op.cc index 8b49127cb93db..e5904a1cf7543 100644 --- a/paddle/fluid/inference/tensorrt/convert/flash_multihead_matmul_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/flash_multihead_matmul_op.cc @@ -24,11 +24,12 @@ namespace tensorrt { class FlashMultiheadMatMulOpConverter : public OpConverter { public: - void flash_multihead_mamul_trt(const framework::proto::OpDesc& op, - const framework::Scope& scope, - bool test_mode) { - VLOG(3) << "convert a flash_multihead_mamul op to a corresponding tensorrt " - "network structure\n"; + void flash_multihead_matmul_trt(const framework::proto::OpDesc& op, + const framework::Scope& scope, + bool test_mode) { + VLOG(3) + << "convert a flash_multihead_matmul op to a corresponding tensorrt " + "network structure\n"; bool with_fp16 = engine_->WithFp16() && !engine_->disable_trt_plugin_fp16(); if (engine_->precision() == phi::DataType::INT8) { @@ -138,7 +139,7 @@ class FlashMultiheadMatMulOpConverter : public OpConverter { weight, bias); fc_layer->setName( - ("multihead_mamul_fc(Output: " + output_name + ")").c_str()); + ("multihead_matmul_fc(Output: " + output_name + ")").c_str()); // add shuffle for fc layer reshape_before_mha_layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *fc_layer->getOutput(0)); @@ -243,10 +244,10 @@ class FlashMultiheadMatMulOpConverter : public OpConverter { layer, "flash_multihead_matmul", {output_name}, test_mode); } - void flash_multihead_mamul(const framework::proto::OpDesc& op, - const framework::Scope& scope, - bool test_mode) { - VLOG(3) << "convert a flash_multihead_mamul op to a " + void flash_multihead_matmul(const framework::proto::OpDesc& op, + const framework::Scope& scope, + bool test_mode) { + VLOG(3) << "convert a flash_multihead_matmul op to a " "MemoryEfficientAttention OP " "network structure\n"; framework::OpDesc op_desc(op, nullptr); @@ -310,7 +311,7 @@ class FlashMultiheadMatMulOpConverter : public OpConverter { hidden_out, weight, bias); - qkv_fc_layers[i]->setName(("multihead_mamul_fc_" + std::to_string(i) + + qkv_fc_layers[i]->setName(("multihead_matmul_fc_" + std::to_string(i) + "_(Output: " + output_name + ")") .c_str()); } else { @@ -334,7 +335,7 @@ class FlashMultiheadMatMulOpConverter : public OpConverter { matrix_operation_x, *weight_reshape_before_mm[i]->getOutput(0), matrix_operation_y); - qkv_fc_layers[i]->setName(("multihead_mamul_matmul_" + + qkv_fc_layers[i]->setName(("multihead_matmul_matmul_" + std::to_string(i) + "_(Output: " + output_name + ")") .c_str()); @@ -499,9 +500,9 @@ class FlashMultiheadMatMulOpConverter : public OpConverter { framework::OpDesc op_desc(op, nullptr); bool use_trt_fma = PADDLE_GET_CONST(bool, op_desc.GetAttr("use_trt_fma")); if (use_trt_fma) { - flash_multihead_mamul_trt(op, scope, test_mode); + flash_multihead_matmul_trt(op, scope, test_mode); } else { - flash_multihead_mamul(op, scope, test_mode); + flash_multihead_matmul(op, scope, test_mode); } } }; diff --git a/paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc b/paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc index eefed86f141c3..6ebc1278c277f 100644 --- a/paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc +++ b/paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc @@ -31,7 +31,7 @@ class CustomPluginCreater : public OpConverter { const framework::Scope &scope, bool test_mode) override { framework::OpDesc op_desc(op, nullptr); - VLOG(3) << "convert " << op_desc.Type() << " op to custom pluign layer"; + VLOG(3) << "convert " << op_desc.Type() << " op to custom plugin layer"; std::string plugin_name; @@ -175,7 +175,7 @@ class GenericPluginCreater : public OpConverter { const framework::Scope &scope, bool test_mode) override { framework::OpDesc op_desc(op, nullptr); - VLOG(3) << "convert " << op_desc.Type() << " op to generic pluign layer"; + VLOG(3) << "convert " << op_desc.Type() << " op to generic plugin layer"; CHECK(block_); const framework::BlockDesc block_desc( @@ -259,7 +259,7 @@ class CustomGenericPluginCreater : public OpConverter { bool test_mode) override { framework::OpDesc op_desc(op, nullptr); VLOG(3) << "convert " << op_desc.Type() - << " op to custom generic pluign layer"; + << " op to custom generic plugin layer"; nvinfer1::ILayer *layer = nullptr; std::vector inputs; diff --git a/paddle/fluid/inference/tensorrt/convert/multihead_matmul_op.cc b/paddle/fluid/inference/tensorrt/convert/multihead_matmul_op.cc index 4e6cab4ff907e..73c43d39357c0 100644 --- a/paddle/fluid/inference/tensorrt/convert/multihead_matmul_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/multihead_matmul_op.cc @@ -25,7 +25,7 @@ class MultiheadMatMulOpConverter : public OpConverter { void operator()(const framework::proto::OpDesc& op, const framework::Scope& scope, bool test_mode) override { - VLOG(3) << "convert a multihead_mamul op to a corresponding tensorrt " + VLOG(3) << "convert a multihead_matmul op to a corresponding tensorrt " "network structure"; framework::OpDesc op_desc(op, nullptr); // Declare inputs @@ -377,7 +377,7 @@ class MultiheadMatMulOpConverter : public OpConverter { reshape_before_multihead_layer->setInput(1, *Concat(reshape_tensor)); reshape_before_multihead_layer->setName( - ("reshape_before_multihead_mamul(Output: " + output_name + ")") + ("reshape_before_multihead_matmul(Output: " + output_name + ")") .c_str()); if (op_desc.HasAttr("fc_out_threshold")) { @@ -625,7 +625,7 @@ class MultiheadMatMulOpConverter : public OpConverter { bias); } fc_layer->setName( - ("multihead_mamul_fc(Output: " + output_name + ")").c_str()); + ("multihead_matmul_fc(Output: " + output_name + ")").c_str()); // add shuffle for CustomQKVToContextPluginDynamic layer auto* reshape_after_fc_layer = @@ -798,7 +798,7 @@ class MultiheadMatMulOpConverter : public OpConverter { reshape_before_fc_layer->setInput( 1, *Concat(reshape_before_fc_shape_tensor)); reshape_before_fc_layer->setName( - ("shuffle_before_multihead_mamul(Output: " + output_name + ")") + ("shuffle_before_multihead_matmul(Output: " + output_name + ")") .c_str()); // add layer fc @@ -834,7 +834,7 @@ class MultiheadMatMulOpConverter : public OpConverter { engine_->SetTensorDynamicRange(fc_layer->getOutput(0), out_scale); } fc_layer->setName( - ("multihead_mamul_fc(Output: " + output_name + ")").c_str()); + ("multihead_matmul_fc(Output: " + output_name + ")").c_str()); // no need to add shuffle after fc, just change it in // QkvToContextPluginDynamic diff --git a/paddle/fluid/inference/tensorrt/convert/multihead_matmul_roformer_op.cc b/paddle/fluid/inference/tensorrt/convert/multihead_matmul_roformer_op.cc index 517f5f1e7efc0..f849fff7ab1f2 100644 --- a/paddle/fluid/inference/tensorrt/convert/multihead_matmul_roformer_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/multihead_matmul_roformer_op.cc @@ -24,7 +24,7 @@ class MultiheadMatMulRoformerOpConverter : public OpConverter { void operator()(const framework::proto::OpDesc& op, const framework::Scope& scope, bool test_mode) override { - VLOG(3) << "convert a multihead_mamul_roformer op to a corresponding " + VLOG(3) << "convert a multihead_matmul_roformer op to a corresponding " "tensorrt " "network structure"; framework::OpDesc op_desc(op, nullptr); diff --git a/paddle/fluid/inference/tensorrt/convert/qk_multihead_matmul_op.cc b/paddle/fluid/inference/tensorrt/convert/qk_multihead_matmul_op.cc index 4a24e7425068f..e8ed4af9cddf7 100644 --- a/paddle/fluid/inference/tensorrt/convert/qk_multihead_matmul_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/qk_multihead_matmul_op.cc @@ -23,7 +23,7 @@ class QkMultiheadMatMulOpConverter : public OpConverter { void operator()(const framework::proto::OpDesc& op, const framework::Scope& scope, bool test_mode) override { - VLOG(3) << "convert a qk_multihead_mamul op to a corresponding tensorrt " + VLOG(3) << "convert a qk_multihead_matmul op to a corresponding tensorrt " "network structure"; framework::OpDesc op_desc(op, nullptr); @@ -142,7 +142,7 @@ class QkMultiheadMatMulOpConverter : public OpConverter { *bias_qk_tensor, elementwise_operation); merge_qk_element_layer->setName( - ("multihead_mamul_fc_qk(Output: " + output_name + ")").c_str()); + ("multihead_matmul_fc_qk(Output: " + output_name + ")").c_str()); auto* reshape_after_fc_qk_layer = TRT_ENGINE_ADD_LAYER( engine_, Shuffle, *merge_qk_element_layer->getOutput(0)); @@ -232,7 +232,7 @@ class QkMultiheadMatMulOpConverter : public OpConverter { *bias_v_tensor, elementwise_operation); merge_v_element_layer->setName( - ("multihead_mamul_fc_v(Output: " + output_name + ")").c_str()); + ("multihead_matmul_fc_v(Output: " + output_name + ")").c_str()); // add shuffle for fc layer auto* reshape_after_fc_v_layer = TRT_ENGINE_ADD_LAYER( diff --git a/paddle/fluid/inference/tensorrt/convert/sparse_multihead_matmul_op.cc b/paddle/fluid/inference/tensorrt/convert/sparse_multihead_matmul_op.cc index 74198b3066a88..a0736522e5b14 100644 --- a/paddle/fluid/inference/tensorrt/convert/sparse_multihead_matmul_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/sparse_multihead_matmul_op.cc @@ -366,7 +366,7 @@ class SparseMultiheadMatMulOpConverter : public OpConverter { } reshape_before_fc_layer->setReshapeDimensions(reshape_before_fc_dim); reshape_before_fc_layer->setName( - ("shuffle_before_sparse_multihead_mamul(Output: " + output_name + + ("shuffle_before_sparse_multihead_matmul(Output: " + output_name + ")") .c_str()); @@ -403,7 +403,8 @@ class SparseMultiheadMatMulOpConverter : public OpConverter { engine_->SetTensorDynamicRange(fc_layer->getOutput(0), out_scale); } fc_layer->setName( - ("sparse_multihead_mamul_fc(Output: " + output_name + ")").c_str()); + ("sparse_multihead_matmul_fc(Output: " + output_name + ")") + .c_str()); // no need to add shuffle after fc, just change it in // QkvToContextPluginDynamic From 1a8df18603d88542e59740360683375bc831d47a Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 4 Mar 2024 11:59:08 +0800 Subject: [PATCH 264/282] Update paddle/pir/src/core/op_operand.cc (#62311) --- paddle/pir/src/core/op_operand.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/paddle/pir/src/core/op_operand.cc b/paddle/pir/src/core/op_operand.cc index 5c27cd4943ca6..06c0d79ed9ae0 100644 --- a/paddle/pir/src/core/op_operand.cc +++ b/paddle/pir/src/core/op_operand.cc @@ -22,8 +22,8 @@ "impl_ pointer is null when call func:" #func_name \ " , in class: " #class_name ".") -#define CHECK_OPOPEREND_NULL_IMPL(func_name) \ - CHECK_NULL_IMPL(OpOpernad, func_name) +#define CHECK_OP_OPERAND_NULL_IMPL(func_name) \ + CHECK_NULL_IMPL(OpOperand, func_name) namespace pir { OpOperand &OpOperand::operator=(const OpOperand &rhs) { // NOLINT @@ -37,34 +37,34 @@ OpOperand &OpOperand::operator=(const OpOperand &rhs) { // NOLINT OpOperand::operator bool() const { return impl_ && impl_->source(); } OpOperand OpOperand::next_use() const { - CHECK_OPOPEREND_NULL_IMPL(next_use); + CHECK_OP_OPERAND_NULL_IMPL(next_use); return impl_->next_use(); } Value OpOperand::source() const { - CHECK_OPOPEREND_NULL_IMPL(source); + CHECK_OP_OPERAND_NULL_IMPL(source); return impl_->source(); } Type OpOperand::type() const { return source().type(); } void OpOperand::set_source(Value value) { - CHECK_OPOPEREND_NULL_IMPL(set_source); + CHECK_OP_OPERAND_NULL_IMPL(set_source); impl_->set_source(value); } Operation *OpOperand::owner() const { - CHECK_OPOPEREND_NULL_IMPL(owner); + CHECK_OP_OPERAND_NULL_IMPL(owner); return impl_->owner(); } uint32_t OpOperand::index() const { - CHECK_OPOPEREND_NULL_IMPL(index); + CHECK_OP_OPERAND_NULL_IMPL(index); return impl_->index(); } void OpOperand::RemoveFromUdChain() { - CHECK_OPOPEREND_NULL_IMPL(RemoveFromUdChain); + CHECK_OP_OPERAND_NULL_IMPL(RemoveFromUdChain); return impl_->RemoveFromUdChain(); } From f0eabc4c46fbd65c7e96361eadb129dea3367ee2 Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 4 Mar 2024 12:21:35 +0800 Subject: [PATCH 265/282] Change charcases char_cases (#62310) * Fix * Fix --- .../strings/gpu/strings_lower_upper_kernel.cu | 2 +- .../strings/strings_lower_upper_kernel.h | 6 ++-- paddle/phi/kernels/strings/unicode.cc | 28 +++++++++---------- paddle/phi/kernels/strings/unicode.h | 6 ++-- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/paddle/phi/kernels/strings/gpu/strings_lower_upper_kernel.cu b/paddle/phi/kernels/strings/gpu/strings_lower_upper_kernel.cu index 832d9bbf73c0b..2a238e8a49b4d 100644 --- a/paddle/phi/kernels/strings/gpu/strings_lower_upper_kernel.cu +++ b/paddle/phi/kernels/strings/gpu/strings_lower_upper_kernel.cu @@ -56,7 +56,7 @@ struct UTF8CaseConverter { pstring* out, size_t num) const { auto unicode_flag_map = GetGPUUniflagMap(); - auto cases_map = GetGPUCharcasesMap(); + auto cases_map = GetGPUCharCasesMap(); thrust::device_vector unicode_offsets(num + 1, 0); uint32_t* unicode_offsets_ptr = thrust::raw_pointer_cast(unicode_offsets.data()); diff --git a/paddle/phi/kernels/strings/strings_lower_upper_kernel.h b/paddle/phi/kernels/strings/strings_lower_upper_kernel.h index a8d7f2dda94f7..a7c1d4a0936fc 100644 --- a/paddle/phi/kernels/strings/strings_lower_upper_kernel.h +++ b/paddle/phi/kernels/strings/strings_lower_upper_kernel.h @@ -60,13 +60,13 @@ StringTensor StringUpper(const ContextT& dev_ctx, return string_out; } -template +template struct StringCaseConvertKernel { void operator()(const ContextT& dev_ctx, const StringTensor& x, bool use_utf8_encoding, StringTensor* out) { - AsciiCoverter ascii_converter; + AsciiConverter ascii_converter; UTF8Converter utf8_converter; const pstring* in_ptr = x.data(); pstring* out_ptr = dev_ctx.template Alloc(out); @@ -101,7 +101,7 @@ struct UTF8CaseConverter { pstring* out, size_t num) const { auto unicode_flag_map = GetUniFlagMap(); - auto cases_map = GetCharcasesMap(); + auto cases_map = GetCharCasesMap(); for (size_t i = 0; i < num; ++i) { uint32_t unicode_len = GetUnicodeStrLen(in[i].data(), in[i].size()); std::vector unicode_in(unicode_len, 0); diff --git a/paddle/phi/kernels/strings/unicode.cc b/paddle/phi/kernels/strings/unicode.cc index 292160e2b2db1..71d9ef36cd16d 100644 --- a/paddle/phi/kernels/strings/unicode.cc +++ b/paddle/phi/kernels/strings/unicode.cc @@ -23,7 +23,7 @@ namespace phi { namespace strings { static const void* utils_map[4] = {nullptr}; // NOLINT -static uint16_t CHARCASES_MAP[65536] = {0}; // NOLINT +static uint16_t CHAR_CASES_MAP[65536] = {0}; // NOLINT const uint8_t* GetUniFlagMap() { if (utils_map[1] == nullptr) { @@ -32,16 +32,16 @@ const uint8_t* GetUniFlagMap() { return reinterpret_cast(utils_map[1]); } -const uint16_t* GetCharcasesMap() { +const uint16_t* GetCharCasesMap() { if (utils_map[0] == nullptr) { for (uint32_t i = 0; i < 65536; ++i) { if (utf8proc_islower(static_cast(i))) { - CHARCASES_MAP[i] = utf8proc_toupper(static_cast(i)); + CHAR_CASES_MAP[i] = utf8proc_toupper(static_cast(i)); } else if (utf8proc_isupper(static_cast(i))) { - CHARCASES_MAP[i] = utf8proc_tolower(static_cast(i)); + CHAR_CASES_MAP[i] = utf8proc_tolower(static_cast(i)); } } - utils_map[0] = CHARCASES_MAP; + utils_map[0] = CHAR_CASES_MAP; } return reinterpret_cast(utils_map[0]); } @@ -67,21 +67,21 @@ const uint8_t* GetGPUUniflagMap() { return reinterpret_cast(utils_map[3]); } -const uint16_t* GetGPUCharcasesMap() { +const uint16_t* GetGPUCharCasesMap() { if (utils_map[2] == nullptr) { - const uint16_t* cpu_charcases = GetCharcasesMap(); - auto size = sizeof(CHARCASES_MAP); - uint16_t* gpu_charcases; + const uint16_t* cpu_char_cases = GetCharCasesMap(); + auto size = sizeof(CHAR_CASES_MAP); + uint16_t* gpu_char_cases; #ifdef PADDLE_WITH_HIP - hipMalloc(reinterpret_cast(&gpu_charcases), size); + hipMalloc(reinterpret_cast(&gpu_char_cases), size); phi::backends::gpu::GpuMemcpySync( - gpu_charcases, cpu_charcases, size, hipMemcpyHostToDevice); + gpu_char_cases, cpu_char_cases, size, hipMemcpyHostToDevice); #else - cudaMalloc(reinterpret_cast(&gpu_charcases), size); + cudaMalloc(reinterpret_cast(&gpu_char_cases), size); phi::backends::gpu::GpuMemcpySync( - gpu_charcases, cpu_charcases, size, cudaMemcpyHostToDevice); + gpu_char_cases, cpu_char_cases, size, cudaMemcpyHostToDevice); #endif - utils_map[2] = gpu_charcases; + utils_map[2] = gpu_char_cases; } return reinterpret_cast(utils_map[2]); } diff --git a/paddle/phi/kernels/strings/unicode.h b/paddle/phi/kernels/strings/unicode.h index 6dfb6aeb6ede6..48c07dbf8dd4f 100644 --- a/paddle/phi/kernels/strings/unicode.h +++ b/paddle/phi/kernels/strings/unicode.h @@ -169,7 +169,7 @@ HOSTDEVICE inline uint32_t GetUTF8StrLen(const uint32_t* unicode_str, // +1 means '\0' return utf8_str_count + 1; } -// Need to gurantee utf8_str has enough memory +// Need to guarantee utf8_str has enough memory HOSTDEVICE inline void GetUTF8Str(const uint32_t* unicode_str, char* utf8_str, @@ -186,12 +186,12 @@ HOSTDEVICE inline void GetUTF8Str(const uint32_t* unicode_str, } const uint8_t* GetUniFlagMap(); -const uint16_t* GetCharcasesMap(); +const uint16_t* GetCharCasesMap(); #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) const uint8_t* GetGPUUniflagMap(); -const uint16_t* GetGPUCharcasesMap(); +const uint16_t* GetGPUCharCasesMap(); #endif } // namespace strings From 5f59752c209f4a70d4c302dcba194a6ccb33dc81 Mon Sep 17 00:00:00 2001 From: ming1753 <61511741+ming1753@users.noreply.github.com> Date: Mon, 4 Mar 2024 12:32:43 +0800 Subject: [PATCH 266/282] [Inference] modify test of UseOptimizedModel API (#62275) * add to do * modify test --- .../analysis/passes/save_optimized_model_pass.cc | 1 + test/ir/inference/test_use_optimized_model_api.py | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/paddle/fluid/inference/analysis/passes/save_optimized_model_pass.cc b/paddle/fluid/inference/analysis/passes/save_optimized_model_pass.cc index 89b49df107390..aaf9439d2b9ed 100644 --- a/paddle/fluid/inference/analysis/passes/save_optimized_model_pass.cc +++ b/paddle/fluid/inference/analysis/passes/save_optimized_model_pass.cc @@ -38,6 +38,7 @@ void SaveOptimizedModelPass::SaveOptimizedModel(Argument* argument) { framework::ir::GraphToProgram(*graph, &optimized_program_desc); + // TODO(minghaipeng): Move the following code to a separate clean pass. // Remove the scale and zero point parameters from optimized program. auto scale_and_zero_point_param = graph->GetOrInit>( framework::ir::kScaleAndZeroPointParamAttr); diff --git a/test/ir/inference/test_use_optimized_model_api.py b/test/ir/inference/test_use_optimized_model_api.py index cdfcb705e8a9c..be6391933e1d7 100644 --- a/test/ir/inference/test_use_optimized_model_api.py +++ b/test/ir/inference/test_use_optimized_model_api.py @@ -18,6 +18,7 @@ from inference_pass_test import InferencePassTest import paddle +from paddle.framework import core from paddle.inference import Config, create_predictor # -------------------------- TestNet -------------------------- @@ -68,18 +69,18 @@ def setUp(self): ) def test_check_output(self): - out_origin_model = self.inference() - out_optimized_model = self.inference() - np.testing.assert_allclose( - out_origin_model, out_optimized_model, rtol=1e-5, atol=1e-2 - ) + if core.is_compiled_with_cuda(): + out_origin_model = self.inference() + out_optimized_model = self.inference() + np.testing.assert_allclose( + out_origin_model, out_optimized_model, rtol=1e-5, atol=1e-2 + ) def inference(self): # Config config = Config( self.path_prefix + ".pdmodel", self.path_prefix + ".pdiparams" ) - # if core.is_compiled_with_cuda(): config.enable_use_gpu(100, 0) config.enable_tensorrt_engine( workspace_size=1 << 30, From 602f8cff9b96d51d5c6641ed229122abd266000a Mon Sep 17 00:00:00 2001 From: wanghuancoder Date: Mon, 4 Mar 2024 12:51:45 +0800 Subject: [PATCH 267/282] add some data_format_tensors (#62262) --- paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml index af136f8a518b5..39ae6203cfd43 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml @@ -62,9 +62,11 @@ - op : depthwise_conv2d extra_args : bool is_test=false, bool fuse_relu_before_depthwise_conv=false, bool use_quantizer=false, str mkldnn_data_type="float32", bool fuse_relu=false, str fuse_activation="", float fuse_alpha=0.0, float fuse_beta=0.0, bool use_addto=false, bool fuse_residual_connection=false, float scale_in=1.0, float scale_out=1.0, float scale_in_eltwise=1.0, float[] scale_weights={1.0f}, bool force_fp32_output=false + data_format_tensors : input - op : depthwise_conv2d_grad extra_args : bool is_test=false, bool fuse_relu_before_depthwise_conv=false, bool use_quantizer=false, str mkldnn_data_type="float32", bool fuse_relu=false, str fuse_activation="", float fuse_alpha=0.0, float fuse_beta=0.0, bool use_addto=false, bool fuse_residual_connection=false, float scale_in=1.0, float scale_out=1.0, float scale_in_eltwise=1.0, float[] scale_weights={1.0f}, bool force_fp32_output=false + data_format_tensors : input, out_grad - op : divide @@ -191,6 +193,7 @@ - op : multiply_grad - op : nearest_interp + data_format_tensors : x - op : pad From d07406f7c4e8c34df6d44f2345cb4aed1b483566 Mon Sep 17 00:00:00 2001 From: tianshuo78520a <707759223@qq.com> Date: Mon, 4 Mar 2024 13:42:16 +0800 Subject: [PATCH 268/282] Test cinn test retry (#62190) * Test cinn test retry * Fix retry * fix test * Fix * Fix * Fix ut_actual_total_startTime_s --- paddle/scripts/paddle_build.sh | 108 ++++++++++++++++++++++++++------- 1 file changed, 87 insertions(+), 21 deletions(-) diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 71ee30a115ef7..63e7d013f2e56 100644 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -2464,29 +2464,95 @@ set +x matchstr='' testcase='' done <<< "$test_cases"; + + ut_actual_total_startTime_s=`date +%s` card_test "$single_card_tests" 1 -set -x - for file in `ls $tmp_dir`; do - exit_code=0 - grep -q 'The following tests FAILED:' $tmp_dir/$file||exit_code=$? - if [ $exit_code -ne 0 ]; then - failuretest='' - else - failuretest=`grep -A 10000 'The following tests FAILED:' $tmp_dir/$file | sed 's/The following tests FAILED://g'|sed '/^$/d'` - failed_test_lists="${failed_test_lists} - ${failuretest}" - break - fi - done - ut_endTime_s=`date +%s` - echo "CINN testCase Time: $[ $ut_endTime_s - $ut_startTime_s ]s" + collect_failed_tests + + # add unit test retry for CINN + rm -f $tmp_dir/* + exec_times=0 + retry_unittests_record='' + retry_time=4 + exec_time_array=('first' 'second' 'third' 'fourth') + parallel_failed_tests_exec_retry_threshold=120 + exec_retry_threshold=30 + is_retry_execuate=0 + rerun_ut_startTime_s=`date +%s` + if [ -n "$failed_test_lists" ];then + need_retry_ut_str=$(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' ) + need_retry_ut_arr=(${need_retry_ut_str}) + need_retry_ut_count=${#need_retry_ut_arr[@]} + retry_unittests=$(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' ) + while ( [ $exec_times -lt $retry_time ] ) + do + if [[ "${exec_times}" == "0" ]] ;then + if [ $need_retry_ut_count -lt $parallel_failed_tests_exec_retry_threshold ];then + is_retry_execuate=0 + else + is_retry_execuate=1 + fi + elif [[ "${exec_times}" == "1" ]] ;then + need_retry_ut_str=$(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' ) + need_retry_ut_arr=(${need_retry_ut_str}) + need_retry_ut_count=${#need_retry_ut_arr[@]} + if [ $need_retry_ut_count -lt $exec_retry_threshold ];then + is_retry_execuate=0 + else + is_retry_execuate=1 + fi + fi + if [[ "$is_retry_execuate" == "0" ]];then + set +e + retry_unittests_record="$retry_unittests_record$failed_test_lists" + failed_test_lists_ult=`echo "${failed_test_lists}" |grep -Po '[^ ].*$'` + set -e + if [[ "${exec_times}" == "1" ]] || [[ "${exec_times}" == "3" ]];then + if [[ "${failed_test_lists}" == "" ]];then + break + else + retry_unittests=$(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' ) + fi + fi + echo "=========================================" + echo "This is the ${exec_time_array[$exec_times]} time to re-run" + echo "=========================================" + echo "The following unittest will be re-run:" + echo "${retry_unittests}" + for line in ${retry_unittests[@]} ; + do + tmp_one_tmp="$( echo $single_card_tests | grep -oEi $line )" + + if [[ "$tmp_one_tmp" != "" ]]; then + if [[ "$one_card_retry" == "" ]]; then + one_card_retry="^$line$" + else + one_card_retry="$one_card_retry|^$line$" + fi + fi + + done + + if [[ "$one_card_retry" != "" ]]; then + card_test "$one_card_retry" 1 # run cases 1 job each time with single GPU + fi + exec_times=$[$exec_times+1] + failed_test_lists='' + collect_failed_tests + rm -f $tmp_dir/* + one_card_retry='' + else + break + fi + done + fi + rerun_ut_endTime_s=`date +%s` + + echo "ipipe_log_param_Rerun_TestCases_Total_Time: $[ $rerun_ut_endTime_s - $rerun_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt + ut_actual_total_endTime_s=`date +%s` + echo "ipipe_log_param_actual_TestCases_Total_Time: $[ $ut_actual_total_endTime_s - $ut_actual_total_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt if [[ "$EXIT_CODE" != "0" ]]; then - rm -f $tmp_dir/* - echo "Summary Failed Tests... " - echo "========================================" - echo "The following tests FAILED: " - echo "${failuretest}" | sort -u - exit 8; + show_ut_retry_result fi fi } From 85f915261fa4fa963f4d438b244298e30b8cc07a Mon Sep 17 00:00:00 2001 From: ZhouMengLei1999 <33919397+ZhouMengLei1999@users.noreply.github.com> Date: Mon, 4 Mar 2024 15:27:24 +0800 Subject: [PATCH 269/282] [XPU] support variable_length_memory_efficient_attention_kernel and flash_attn_unpadded_kernel (#62217) --- paddle/phi/backends/xpu/xpu2_op_list.cc | 4 + ...ength_memory_efficient_attention_kernel.cc | 122 +++++++++++++ paddle/phi/kernels/xpu/flash_attn_kernel.cc | 165 ++++++++++++++++++ 3 files changed, 291 insertions(+) create mode 100644 paddle/phi/kernels/fusion/xpu/variable_length_memory_efficient_attention_kernel.cc diff --git a/paddle/phi/backends/xpu/xpu2_op_list.cc b/paddle/phi/backends/xpu/xpu2_op_list.cc index 171894b9b9f6f..be1d1b6f11304 100644 --- a/paddle/phi/backends/xpu/xpu2_op_list.cc +++ b/paddle/phi/backends/xpu/xpu2_op_list.cc @@ -1202,6 +1202,10 @@ XPUOpMap& get_kl2_ops() { XPUKernelSet({phi::DataType::FLOAT16, phi::DataType::FLOAT32})}, {"roformer_relative_embedding_xpu", XPUKernelSet({phi::DataType::FLOAT16, phi::DataType::FLOAT32})}, + {"variable_length_memory_efficient_attention", + XPUKernelSet({phi::DataType::FLOAT16, phi::DataType::FLOAT32})}, + {"flash_attn_unpadded", + XPUKernelSet({phi::DataType::FLOAT16, phi::DataType::FLOAT32})}, }; return s_xpu2_kernels; diff --git a/paddle/phi/kernels/fusion/xpu/variable_length_memory_efficient_attention_kernel.cc b/paddle/phi/kernels/fusion/xpu/variable_length_memory_efficient_attention_kernel.cc new file mode 100644 index 0000000000000..8f6a25ddc5c86 --- /dev/null +++ b/paddle/phi/kernels/fusion/xpu/variable_length_memory_efficient_attention_kernel.cc @@ -0,0 +1,122 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/backends/xpu/enforce_xpu.h" +#include "paddle/phi/core/kernel_registry.h" + +namespace phi { +namespace fusion { + +template +void MultiHeadAttentionVariableForwardKernel( + const Context& ctx, + const DenseTensor& query, + const DenseTensor& key, + const DenseTensor& value, + const DenseTensor& seq_lens, + const DenseTensor& kv_seq_lens, + const paddle::optional& mask, + const float scale, + const bool causal, + const int pre_cache_length, + DenseTensor* output) { + xpu::ctx_guard RAII_GUARD(ctx.x_context()); + + using XPUType = typename XPUTypeTrait::Type; + + int64_t num_batches = query.dims()[0]; + int64_t num_heads = query.dims()[1]; + int64_t kv_num_heads = key.dims()[1]; + int64_t query_seq_len = query.dims()[2]; + int64_t head_size = query.dims()[3]; + std::vector mask_shape = {}; + if (mask) { + // [B, 1, S, D] + auto mask_tensor = mask.get(); + mask_shape = common::vectorize(mask_tensor.dims()); + } + + xpu::QKVAttnParam qkv_attn_param( + num_batches, /* batch */ + query_seq_len, /* max_seqlen */ + num_heads, /* head_num */ + head_size, /* head_dim */ + mask_shape, /* mask_shape */ + xpu::Activation_t::RELU, /* act */ + -1, /* last_slice_seq */ + false, /* do_fc_qkv_fusion */ + -1, /* hidden_dim */ + false, /* is_pre_norm */ + false, /* is_perchannel */ + 2, /* qkv_shape */ + AttnMacMaxPtrType_t::ATTN_WHOLE_BATCH, /* max_ptr_type */ + -1, /* ldz */ + scale /* alpha */ + ); + qkv_attn_param.key_value_head_num = kv_num_heads; + + const XPUType* mask_ptr = + mask ? reinterpret_cast(mask.get().data()) : nullptr; + auto* out_data = reinterpret_cast(ctx.template Alloc(output)); + XPUType* qk_buf = RAII_GUARD.alloc_l3_or_gm( + num_batches * num_heads * query_seq_len * query_seq_len); + float* maxptr_buf = RAII_GUARD.alloc_l3_or_gm(32); + int r = xpu::qk_attention( + ctx.x_context(), /* ctx */ + reinterpret_cast(query.data()), /* q */ + reinterpret_cast(key.data()), /* k */ + qk_buf, /* qk */ + nullptr, /* max q */ + nullptr, /* max k */ + maxptr_buf, /* max qk */ + qkv_attn_param, /* param */ + mask_ptr /* mask */ + ); + PADDLE_ENFORCE_EQ( + r, 0, phi::errors::InvalidArgument("xpu::qk_attention run failed")); + XPUType* out_tmp_buf = RAII_GUARD.alloc_l3_or_gm( + num_batches * query_seq_len * num_heads * head_size); + r = xpu::qk_v_attention( + ctx.x_context(), /* ctx */ + qk_buf, /* qk */ + reinterpret_cast(value.data()), /* v */ + out_tmp_buf, /* output */ + maxptr_buf, /* max qk */ + nullptr, /* max v */ + nullptr, /* max qkv */ + qkv_attn_param /* mask */ + ); + PADDLE_ENFORCE_EQ( + r, 0, phi::errors::InvalidArgument("xpu::qk_v_attention run failed")); + r = xpu::transpose( + ctx.x_context(), + out_tmp_buf, + out_data, + {num_batches, query_seq_len, num_heads, head_size}, + {0, 2, 1, 3}); + PADDLE_ENFORCE_EQ( + r, 0, phi::errors::InvalidArgument("xpu::transpose run failed")); +} + +} // namespace fusion +} // namespace phi + +PD_REGISTER_KERNEL(variable_length_memory_efficient_attention, + XPU, + ALL_LAYOUT, + phi::fusion::MultiHeadAttentionVariableForwardKernel, + float, + phi::dtype::float16) { + kernel->InputAt(3).SetDataType(phi::DataType::INT32); +} diff --git a/paddle/phi/kernels/xpu/flash_attn_kernel.cc b/paddle/phi/kernels/xpu/flash_attn_kernel.cc index f040ef383c539..9ea712c410d1d 100644 --- a/paddle/phi/kernels/xpu/flash_attn_kernel.cc +++ b/paddle/phi/kernels/xpu/flash_attn_kernel.cc @@ -23,6 +23,161 @@ namespace phi { +template +void FlashAttnUnpaddedKernel( + const Context& ctx, + const DenseTensor& q, + const DenseTensor& k, + const DenseTensor& v, + const DenseTensor& cu_seqlens_q, + const DenseTensor& cu_seqlens_k, + const paddle::optional& fixed_seed_offset, + const paddle::optional& attn_mask, + int64_t max_seqlen_q, + int64_t max_seqlen_k, + float scale, + float dropout, + bool causal, + bool return_softmax, + bool is_test, + const std::string& rng_name, + DenseTensor* out, + DenseTensor* softmax, + DenseTensor* softmax_lse, + DenseTensor* seed_offset) { +#ifdef PADDLE_WITH_XPU_XHPC + xpu::ctx_guard RAII_GUARD(ctx.x_context()); + // q, k, v [batch_size * seq_len, num_heads, head_dim] + std::vector dims = common::vectorize(q.dims()); + + const int batch_size = cu_seqlens_q.numel() - 1; + const int num_heads = dims[1]; + const int head_size = dims[2]; + const int num_heads_k = k.dims()[1]; + + // lod info, only support qlod == klod + std::vector qlod_vec(batch_size + 1, 0); + int r = xpu_wait(ctx.x_context()->xpu_stream); + PADDLE_ENFORCE_EQ(r, 0, "xpu_wait failed."); + r = xpu_memcpy(qlod_vec.data(), + cu_seqlens_q.data(), + sizeof(int32_t) * (batch_size + 1), + XPUMemcpyKind::XPU_DEVICE_TO_HOST); + PADDLE_ENFORCE_EQ(r, 0, "xpu_memcpy failed."); + std::vector klod_vec(batch_size + 1, 0); + r = xpu_wait(ctx.x_context()->xpu_stream); + PADDLE_ENFORCE_EQ(r, 0, "xpu_wait failed."); + r = xpu_memcpy(klod_vec.data(), + cu_seqlens_k.data(), + sizeof(int32_t) * (batch_size + 1), + XPUMemcpyKind::XPU_DEVICE_TO_HOST); + PADDLE_ENFORCE_EQ(r, 0, "xpu_memcpy failed."); + // output: softmax_lse, 训练参数，给反向用于反向重计算的L + bool is_cross_attn = false; + for (int i = 0; i < batch_size + 1; ++i) { + if (qlod_vec[i] != klod_vec[i]) { + is_cross_attn = true; + break; + } + } + + using XPUType = typename XPUTypeTrait::Type; + auto* out_data = reinterpret_cast(ctx.template Alloc(out)); + const XPUType* q_data = reinterpret_cast(q.data()); + const XPUType* k_data = reinterpret_cast(k.data()); + const XPUType* v_data = reinterpret_cast(v.data()); + if (!is_cross_attn) { + xpu::VectorParam lods{ + qlod_vec.data(), (int32_t)(qlod_vec.size()), nullptr}; + xpu::QKVAttnParam qkv_attn_param( + lods, // only support qlods == kvlods + num_heads, // head_nums + head_size, // head_dim + xpu::Activation_t::RELU, // Activation_t + -1, // last_slice_seq(unused param) + false, // do_fc_qkv_fusion(unused param) + -1, // pad_seqlen(unused param) + -1, // hidden_dim(unused param) + false, // is_pre_norm(unused param) + false, // is_perchannel(unused param) + 0, // qkv_shape + {}, // z_shape + AttnMacMaxPtrType_t::ATTN_WHOLE_BATCH, // max_ptr_type + -1, // ldz(unused param) + {}, // sqlod(unused param) + scale); // alpha + qkv_attn_param.triangle_mask_autogen = causal; + qkv_attn_param.key_value_head_num = num_heads_k; + r = xpu::qkv_attention(ctx.x_context(), + q_data, // q + k_data, // k + v_data, // v + out_data, // out + nullptr, // max_q + nullptr, // max_k + nullptr, // max_v + nullptr, // max_ctx + qkv_attn_param, + nullptr, + nullptr, + nullptr); + PADDLE_ENFORCE_EQ(r, 0, "xpu::qkv_attention failed."); + } else { + std::vector lod; + lod.reserve(2 * batch_size + 2); + int real_max_len = 0; + for (int i = 0; i < batch_size + 1; i++) { + lod.push_back(qlod_vec[i]); + if (i) + real_max_len = std::max(qlod_vec[i] - qlod_vec[i - 1], real_max_len); + } + for (int i = 0; i < batch_size + 1; i++) { + lod.push_back(klod_vec[i]); + if (i) + real_max_len = std::max(klod_vec[i] - klod_vec[i - 1], real_max_len); + } + xpu::DifSeqAttnParam dis_api_attn_param( + {lod.data(), 2 * batch_size + 2, nullptr}, num_heads, head_size); + XPUType* qk_buf = RAII_GUARD.alloc_l3_or_gm( + batch_size * num_heads * real_max_len * real_max_len); + float* qk_max_buf = RAII_GUARD.alloc_l3_or_gm(6); + r = xpu::qk_attention( + ctx.x_context(), + q_data, + k_data, + qk_buf, + nullptr, + nullptr, + qk_max_buf, + dis_api_attn_param, + nullptr); + PADDLE_ENFORCE_EQ(r, 0, "xpu::qk_attention failed."); + r = xpu::qk_v_attention( + ctx.x_context(), + qk_buf, + v_data, + out_data, + qk_max_buf, + nullptr, + nullptr, + dis_api_attn_param, + nullptr); + PADDLE_ENFORCE_EQ(r, 0, "xpu::qk_v_attention failed."); + } +#else + PADDLE_THROW(phi::errors::PreconditionNotMet( + "re-compile using -DWITH_XPU_XHPC=ON to use FlashAttnKernel")); +#endif +} + template void FlashAttnKernel(const Context& ctx, const DenseTensor& q, @@ -127,6 +282,16 @@ void FlashAttnKernel(const Context& ctx, } // namespace phi +PD_REGISTER_KERNEL(flash_attn_unpadded, + XPU, + ALL_LAYOUT, + phi::FlashAttnUnpaddedKernel, + float, + phi::dtype::float16) { + kernel->InputAt(5).SetBackend( + phi::Backend::ALL_BACKEND); // fixed_seed_offset +} + PD_REGISTER_KERNEL(flash_attn, XPU, ALL_LAYOUT, From abf2116a4a9bb693a74487fdaa937c2542b1cb75 Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 4 Mar 2024 15:55:42 +0800 Subject: [PATCH 270/282] Fix formated_axis formatted_axis, etc (#62308) --- .../infer_symbolic_shape/infer_sym_utils.cc | 10 +++---- .../paddle_op_infer_sym.cc | 6 ++-- paddle/phi/infermeta/backward.cc | 8 +++--- paddle/phi/infermeta/unary.cc | 28 +++++++++---------- paddle/phi/kernels/cpu/transpose_kernel.cc | 20 ++++++------- .../fusion/onednn/fused_transpose_kernel.cc | 6 ++-- paddle/phi/kernels/gpu/transpose_kernel.cu | 8 +++--- .../kernels/impl/transpose_grad_kernel_impl.h | 6 ++-- paddle/phi/kernels/onednn/transpose_kernel.cc | 6 ++-- .../kernels/stride/transpose_grad_kernel.cc | 6 ++-- paddle/phi/kernels/stride/transpose_kernel.cc | 8 +++--- paddle/phi/kernels/xpu/flip_kernel.cc | 8 +++--- .../phi/kernels/xpu/transpose_grad_kernel.cc | 6 ++-- paddle/phi/kernels/xpu/transpose_kernel.cc | 8 +++--- python/paddle/jit/dy2static/error.py | 12 ++++---- python/paddle/jit/dy2static/origin_info.py | 2 +- 16 files changed, 74 insertions(+), 74 deletions(-) diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.cc index 5675429b5c65f..c417df6bc79c0 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.cc @@ -35,18 +35,18 @@ bool ReduceInferDim(pir::Operation *op, auto x = op->operand_source(0); int x_rank = x.type().dyn_cast().dims().size(); - const std::vector formated_axis = [&] { - std::vector formated_axis = axis; + const std::vector formatted_axis = [&] { + std::vector formatted_axis = axis; for (size_t i = 0; i < axis.size(); ++i) { if (axis[i] < 0) { - formated_axis[i] = axis[i] + x_rank; + formatted_axis[i] = axis[i] + x_rank; } } - return formated_axis; + return formatted_axis; }(); bool full_dim = true; - std::set dims_set(formated_axis.begin(), formated_axis.end()); + std::set dims_set(formatted_axis.begin(), formatted_axis.end()); for (int64_t i = 0; i < x_rank; ++i) { if (dims_set.find(i) == dims_set.end()) { full_dim = false; diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc index 4b31c94280ed2..20cdc880f8759 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc @@ -848,7 +848,7 @@ bool TransposeOpInferSymbolicShape( int x_rank = x_dims.size(); - const std::vector formated_axis = [op, x_rank, &perm] { + const std::vector formatted_axis = [op, x_rank, &perm] { std::vector out(perm.size(), 0); std::transform(perm.begin(), perm.end(), @@ -866,11 +866,11 @@ bool TransposeOpInferSymbolicShape( return out; }(); - int axis_size = static_cast(formated_axis.size()); + int axis_size = static_cast(formatted_axis.size()); std::vector out_dims(x_dims); for (int i = 0; i < axis_size; ++i) { - out_dims[i] = x_dims[formated_axis[i]]; + out_dims[i] = x_dims[formatted_axis[i]]; } shape_analysis->SetShapeOrDataForValue(op->result(0), diff --git a/paddle/phi/infermeta/backward.cc b/paddle/phi/infermeta/backward.cc index 845a8e6835729..9f66d0ec3a9f5 100644 --- a/paddle/phi/infermeta/backward.cc +++ b/paddle/phi/infermeta/backward.cc @@ -1180,16 +1180,16 @@ void TransposeGradInferMeta(const MetaTensor& x, const std::vector& axis, MetaTensor* out) { size_t x_rank = x.dims().size(); - std::vector formated_axis = axis; + std::vector formatted_axis = axis; for (size_t i = 0; i < axis.size(); i++) { if (axis[i] < 0) { - formated_axis[i] = static_cast(axis[i] + x_rank); + formatted_axis[i] = static_cast(axis[i] + x_rank); } } std::vector reversed_axis(axis); - for (int i = 0; i < static_cast(formated_axis.size()); i++) { - reversed_axis[formated_axis[i]] = i; + for (int i = 0; i < static_cast(formatted_axis.size()); i++) { + reversed_axis[formatted_axis[i]] = i; } TransposeInferMeta(x, reversed_axis, out); diff --git a/paddle/phi/infermeta/unary.cc b/paddle/phi/infermeta/unary.cc index b064a9f73bad6..5596b9bb798e9 100644 --- a/paddle/phi/infermeta/unary.cc +++ b/paddle/phi/infermeta/unary.cc @@ -2584,7 +2584,7 @@ void NanmedianInferMeta(const MetaTensor& x, } } } else { - std::vector formated_axis; + std::vector formatted_axis; for (auto& axis : axis_list) { if (x_rank == 0) { PADDLE_ENFORCE_EQ(axis == 0 || axis == -1, @@ -2612,17 +2612,17 @@ void NanmedianInferMeta(const MetaTensor& x, } if (axis < 0) axis += x_rank; PADDLE_ENFORCE_EQ( - std::find(formated_axis.begin(), formated_axis.end(), axis), - formated_axis.end(), + std::find(formatted_axis.begin(), formatted_axis.end(), axis), + formatted_axis.end(), errors::InvalidArgument("Attr(axes) has duplicated elements: %d.", static_cast(axis))); - formated_axis.push_back(axis); + formatted_axis.push_back(axis); } for (int64_t i = 0; i < x_rank; i++) { - if (std::find(formated_axis.begin(), formated_axis.end(), i) == - formated_axis.end()) { + if (std::find(formatted_axis.begin(), formatted_axis.end(), i) == + formatted_axis.end()) { out_dim.push_back(x_dim[i]); // NOLINT } else if (keep_dim) { out_dim.push_back(1); @@ -3382,7 +3382,7 @@ DDim ReduceInferDim(const MetaTensor& x, bool reduce_all) { int x_rank = x.dims().size(); - std::vector formated_axis = axis; + std::vector formatted_axis = axis; for (size_t i = 0; i < axis.size(); ++i) { if (x_rank == 0) { PADDLE_ENFORCE_EQ( @@ -3414,12 +3414,12 @@ DDim ReduceInferDim(const MetaTensor& x, } if (axis[i] < 0) { - formated_axis[i] = axis[i] + x_rank; + formatted_axis[i] = axis[i] + x_rank; } } bool full_dim = true; - std::set dims_set(formated_axis.begin(), formated_axis.end()); + std::set dims_set(formatted_axis.begin(), formatted_axis.end()); for (int64_t i = 0; i < x_rank; ++i) { if (dims_set.find(i) == dims_set.end()) { full_dim = false; @@ -4148,7 +4148,7 @@ void SplitWithNumInferMeta(const MetaTensor& x, } } else { auto input_axis_dim = x.dims().at(axis_value); - // step1: get formated sections + // step1: get formatted sections std::vector sections_vec; PADDLE_ENFORCE_NE( num, @@ -4757,7 +4757,7 @@ void TransposeInferMeta(const MetaTensor& x, x_rank, axis_size)); - std::vector formated_axis = axis; + std::vector formatted_axis = axis; std::vector count(axis_size, 0); for (int i = 0; i < axis_size; i++) { PADDLE_ENFORCE_LT(axis[i], @@ -4780,10 +4780,10 @@ void TransposeInferMeta(const MetaTensor& x, axis[i])); if (axis[i] < 0) { - formated_axis[i] = axis[i] + x_rank; + formatted_axis[i] = axis[i] + x_rank; } PADDLE_ENFORCE_EQ( - ++count[formated_axis[i]], + ++count[formatted_axis[i]], 1, errors::InvalidArgument("Each element of axis should be unique. but " "axis[%d] is %d appear not only once", @@ -4793,7 +4793,7 @@ void TransposeInferMeta(const MetaTensor& x, phi::DDim out_dims(x_dims); for (int i = 0; i < axis_size; ++i) { - out_dims[i] = x_dims[formated_axis[i]]; + out_dims[i] = x_dims[formatted_axis[i]]; } out->set_dims(out_dims); diff --git a/paddle/phi/kernels/cpu/transpose_kernel.cc b/paddle/phi/kernels/cpu/transpose_kernel.cc index bab9d47caa9aa..67f2b2ce9b403 100644 --- a/paddle/phi/kernels/cpu/transpose_kernel.cc +++ b/paddle/phi/kernels/cpu/transpose_kernel.cc @@ -29,10 +29,10 @@ void TransposeKernel(const Context& ctx, const std::vector& axis, DenseTensor* out) { size_t x_rank = x.dims().size(); - std::vector formated_axis = axis; + std::vector formatted_axis = axis; for (size_t i = 0; i < axis.size(); i++) { if (axis[i] < 0) { - formated_axis[i] = static_cast(axis[i] + x_rank); + formatted_axis[i] = static_cast(axis[i] + x_rank); } } @@ -40,39 +40,39 @@ void TransposeKernel(const Context& ctx, if (out->numel() == 0) { return; } - int rank = static_cast(formated_axis.size()); + int rank = static_cast(formatted_axis.size()); switch (rank) { case 0: phi::Copy(ctx, x, ctx.GetPlace(), false, out); break; case 1: funcs::Transpose trans1; - trans1(ctx, x, out, formated_axis); + trans1(ctx, x, out, formatted_axis); break; case 2: funcs::Transpose trans2; - trans2(ctx, x, out, formated_axis); + trans2(ctx, x, out, formatted_axis); break; case 3: funcs::Transpose trans3; - trans3(ctx, x, out, formated_axis); + trans3(ctx, x, out, formatted_axis); break; case 4: funcs::Transpose trans4; - trans4(ctx, x, out, formated_axis); + trans4(ctx, x, out, formatted_axis); break; case 5: funcs::Transpose trans5; - trans5(ctx, x, out, formated_axis); + trans5(ctx, x, out, formatted_axis); break; case 6: funcs::Transpose trans6; - trans6(ctx, x, out, formated_axis); + trans6(ctx, x, out, formatted_axis); break; default: // for rank >= 7 situation funcs::TransposeNormal trans_normal; - trans_normal(ctx, x, out, formated_axis); + trans_normal(ctx, x, out, formatted_axis); } } diff --git a/paddle/phi/kernels/fusion/onednn/fused_transpose_kernel.cc b/paddle/phi/kernels/fusion/onednn/fused_transpose_kernel.cc index f8a2f4fe0201e..78fd2cfd964d7 100644 --- a/paddle/phi/kernels/fusion/onednn/fused_transpose_kernel.cc +++ b/paddle/phi/kernels/fusion/onednn/fused_transpose_kernel.cc @@ -69,11 +69,11 @@ void FusedTransposeKernel(const Context& dev_ctx, (phi::OneDNNContext::tls().get_cur_paddle_data_layout() == phi::DataLayout::kNHWC)) { int axis_size = static_cast(axis.size()); - std::vector formated_axis = axis; + std::vector formatted_axis = axis; std::vector count(axis_size, 0); for (int i = 0; i < axis_size; i++) { if (axis[i] < 0) { - formated_axis[i] = axis[i] + axis_size; + formatted_axis[i] = axis[i] + axis_size; } } auto dims = common::vectorize(x_dims); @@ -85,7 +85,7 @@ void FusedTransposeKernel(const Context& dev_ctx, phi::DDim out_dims(x_dims); for (size_t i = 0; i < axis.size(); i++) { - out_dims[i] = x_dims[formated_axis[i]]; // NOLINT + out_dims[i] = x_dims[formatted_axis[i]]; // NOLINT } out->Resize(out_dims); } diff --git a/paddle/phi/kernels/gpu/transpose_kernel.cu b/paddle/phi/kernels/gpu/transpose_kernel.cu index 323c228c16039..809d28ee616e6 100644 --- a/paddle/phi/kernels/gpu/transpose_kernel.cu +++ b/paddle/phi/kernels/gpu/transpose_kernel.cu @@ -31,10 +31,10 @@ void TransposeKernel(const Context& ctx, const std::vector& axis, DenseTensor* out) { size_t x_rank = x.dims().size(); - std::vector formated_axis = axis; + std::vector formatted_axis = axis; for (size_t i = 0; i < axis.size(); i++) { if (axis[i] < 0) { - formated_axis[i] = axis[i] + x_rank; + formatted_axis[i] = axis[i] + x_rank; } } @@ -42,11 +42,11 @@ void TransposeKernel(const Context& ctx, if (out->numel() == 0) { return; } - if (formated_axis.size() == 0) { + if (formatted_axis.size() == 0) { phi::Copy(ctx, x, ctx.GetPlace(), false, out); return; } - phi::funcs::TransposeGPUKernelDriver(ctx, x, formated_axis, out); + phi::funcs::TransposeGPUKernelDriver(ctx, x, formatted_axis, out); } } // namespace phi diff --git a/paddle/phi/kernels/impl/transpose_grad_kernel_impl.h b/paddle/phi/kernels/impl/transpose_grad_kernel_impl.h index f296ad995cf7f..72ed43f09e152 100644 --- a/paddle/phi/kernels/impl/transpose_grad_kernel_impl.h +++ b/paddle/phi/kernels/impl/transpose_grad_kernel_impl.h @@ -26,17 +26,17 @@ void TransposeGradKernel(const Context& dev_ctx, const std::vector& axis, DenseTensor* x_grad) { size_t axis_size = axis.size(); - std::vector formated_axis = axis; + std::vector formatted_axis = axis; for (size_t i = 0; i < axis_size; i++) { if (axis[i] < 0) { - formated_axis[i] = axis[i] + axis_size; + formatted_axis[i] = axis[i] + axis_size; } } std::vector reversed_axis(axis); dev_ctx.template Alloc(x_grad); for (size_t i = 0; i < axis_size; i++) { - reversed_axis[formated_axis[i]] = i; + reversed_axis[formatted_axis[i]] = i; } TransposeKernel(dev_ctx, out_grad, reversed_axis, x_grad); diff --git a/paddle/phi/kernels/onednn/transpose_kernel.cc b/paddle/phi/kernels/onednn/transpose_kernel.cc index ef1f3b0d87fdb..c0faaf5e6c7ba 100644 --- a/paddle/phi/kernels/onednn/transpose_kernel.cc +++ b/paddle/phi/kernels/onednn/transpose_kernel.cc @@ -33,11 +33,11 @@ void TransposeKernel(const Context& dev_ctx, (phi::OneDNNContext::tls().get_cur_paddle_data_layout() == phi::DataLayout::kNHWC)) { int axis_size = static_cast(axis.size()); - std::vector formated_axis = axis; + std::vector formatted_axis = axis; std::vector count(axis_size, 0); for (int i = 0; i < axis_size; i++) { if (axis[i] < 0) { - formated_axis[i] = axis[i] + axis_size; + formatted_axis[i] = axis[i] + axis_size; } } auto dims = common::vectorize(x_dims); @@ -49,7 +49,7 @@ void TransposeKernel(const Context& dev_ctx, phi::DDim out_dims(x_dims); for (size_t i = 0; i < axis.size(); i++) { - out_dims[i] = x_dims[formated_axis[i]]; // NOLINT + out_dims[i] = x_dims[formatted_axis[i]]; // NOLINT } out->Resize(out_dims); } diff --git a/paddle/phi/kernels/stride/transpose_grad_kernel.cc b/paddle/phi/kernels/stride/transpose_grad_kernel.cc index 51295658393c4..0da65306027d4 100644 --- a/paddle/phi/kernels/stride/transpose_grad_kernel.cc +++ b/paddle/phi/kernels/stride/transpose_grad_kernel.cc @@ -25,16 +25,16 @@ void TransposeGradStridedKernel(const Context& dev_ctx, const std::vector& axis, DenseTensor* x_grad) { size_t axis_size = axis.size(); - std::vector formated_axis = axis; + std::vector formatted_axis = axis; for (size_t i = 0; i < axis_size; i++) { if (axis[i] < 0) { - formated_axis[i] = static_cast(axis[i] + axis_size); + formatted_axis[i] = static_cast(axis[i] + axis_size); } } std::vector reversed_axis(axis); for (int i = 0; i < static_cast(axis_size); i++) { - reversed_axis[formated_axis[i]] = i; + reversed_axis[formatted_axis[i]] = i; } TransposeStridedKernel(dev_ctx, out_grad, reversed_axis, x_grad); diff --git a/paddle/phi/kernels/stride/transpose_kernel.cc b/paddle/phi/kernels/stride/transpose_kernel.cc index acdc321ad0e8a..ca09e6a768f60 100644 --- a/paddle/phi/kernels/stride/transpose_kernel.cc +++ b/paddle/phi/kernels/stride/transpose_kernel.cc @@ -24,18 +24,18 @@ void TransposeStridedKernel(const Context& ctx, const std::vector& axis, DenseTensor* out) { size_t x_rank = x.dims().size(); - std::vector formated_axis = axis; + std::vector formatted_axis = axis; for (size_t i = 0; i < axis.size(); i++) { if (axis[i] < 0) { - formated_axis[i] = static_cast(axis[i] + x_rank); + formatted_axis[i] = static_cast(axis[i] + x_rank); } } auto meta = out->meta(); auto in_stride = x.strides(); meta.strides = in_stride; - for (int i = 0; i < static_cast(formated_axis.size()); i++) { - meta.strides[i] = in_stride[formated_axis[i]]; + for (int i = 0; i < static_cast(formatted_axis.size()); i++) { + meta.strides[i] = in_stride[formatted_axis[i]]; } meta.offset = x.offset(); diff --git a/paddle/phi/kernels/xpu/flip_kernel.cc b/paddle/phi/kernels/xpu/flip_kernel.cc index 56a31197e56c7..aa44e3083b7c2 100644 --- a/paddle/phi/kernels/xpu/flip_kernel.cc +++ b/paddle/phi/kernels/xpu/flip_kernel.cc @@ -26,17 +26,17 @@ void FlipKernel(const Context& dev_ctx, DenseTensor* out) { using XPUInTDType = typename XPUTypeTrait::Type; int x_rank = x.dims().size(); - std::vector formated_axis(std::begin(axis), std::end(axis)); + std::vector formatted_axis(std::begin(axis), std::end(axis)); for (size_t i = 0; i < axis.size(); i++) { if (axis[i] < 0) { - formated_axis[i] = static_cast(axis[i] + x_rank); + formatted_axis[i] = static_cast(axis[i] + x_rank); } } dev_ctx.template Alloc(out); if (out->numel() == 0) { return; } - if (formated_axis.size() == 0) { + if (formatted_axis.size() == 0) { phi::Copy(dev_ctx, x, dev_ctx.GetPlace(), false, out); return; } @@ -52,7 +52,7 @@ void FlipKernel(const Context& dev_ctx, /* const T* x */ x_data, /* T* y */ out_data, /* const std::vector& xshape */ x_shape, - /* const std::vector& axis */ formated_axis); + /* const std::vector& axis */ formatted_axis); PADDLE_ENFORCE_XDNN_SUCCESS(r, "flip"); } diff --git a/paddle/phi/kernels/xpu/transpose_grad_kernel.cc b/paddle/phi/kernels/xpu/transpose_grad_kernel.cc index ab6be8c3347ca..a461b0dcb1b58 100644 --- a/paddle/phi/kernels/xpu/transpose_grad_kernel.cc +++ b/paddle/phi/kernels/xpu/transpose_grad_kernel.cc @@ -36,16 +36,16 @@ void TransposeGradKernel(const Context& dev_ctx, return; } - std::vector formated_axis = axis; + std::vector formatted_axis = axis; for (size_t i = 0; i < axis_size; i++) { if (axis[i] < 0) { - formated_axis[i] = axis[i] + axis_size; + formatted_axis[i] = axis[i] + axis_size; } } std::vector reversed_axis(axis); for (size_t i = 0; i < axis_size; i++) { - reversed_axis[formated_axis[i]] = i; + reversed_axis[formatted_axis[i]] = i; } std::vector out_grad_dim_vec = common::vectorize(out_grad.dims()); diff --git a/paddle/phi/kernels/xpu/transpose_kernel.cc b/paddle/phi/kernels/xpu/transpose_kernel.cc index f88e06b18e88d..4fda5e3912645 100644 --- a/paddle/phi/kernels/xpu/transpose_kernel.cc +++ b/paddle/phi/kernels/xpu/transpose_kernel.cc @@ -25,10 +25,10 @@ void TransposeKernel(const Context& dev_ctx, const std::vector& axis, DenseTensor* out) { size_t x_rank = x.dims().size(); - std::vector formated_axis = axis; + std::vector formatted_axis = axis; for (size_t i = 0; i < axis.size(); i++) { if (axis[i] < 0) { - formated_axis[i] = axis[i] + x_rank; + formatted_axis[i] = axis[i] + x_rank; } } @@ -38,7 +38,7 @@ void TransposeKernel(const Context& dev_ctx, if (out->numel() == 0) { return; } - if (formated_axis.size() == 0) { + if (formatted_axis.size() == 0) { phi::Copy(dev_ctx, x, dev_ctx.GetPlace(), false, out); return; } @@ -48,7 +48,7 @@ void TransposeKernel(const Context& dev_ctx, reinterpret_cast(x.data()), reinterpret_cast(out->data()), x_dim_vec, - formated_axis); + formatted_axis); PADDLE_ENFORCE_XDNN_SUCCESS(r, "transpose"); } diff --git a/python/paddle/jit/dy2static/error.py b/python/paddle/jit/dy2static/error.py index 2173eddac87e6..69078a913fa4e 100644 --- a/python/paddle/jit/dy2static/error.py +++ b/python/paddle/jit/dy2static/error.py @@ -75,7 +75,7 @@ def __init__(self, location, function_name, source_code): self.source_code = source_code self.error_line = '' - def formated_message(self): + def formatted_message(self): # self.source_code may be empty in some functions. # For example, decorator generated function return ( @@ -141,7 +141,7 @@ def __init__(self, location, function_name): + self.source_code[i] ) - def formated_message(self): + def formatted_message(self): msg = ( ' ' * BLANK_COUNT_BEFORE_FILE_STR + 'File "{}", line {}, in {}\n'.format( @@ -288,7 +288,7 @@ def create_message(self): dygraph_func_info.source_code, ) - message_lines.append(traceback_frame.formated_message()) + message_lines.append(traceback_frame.formatted_message()) error_line = traceback_frame.error_line message_lines.append("") @@ -304,7 +304,7 @@ def create_message(self): traceback_frame = TraceBackFrame( Location(filepath, lineno), funcname, code ) - message_lines.append(traceback_frame.formated_message()) + message_lines.append(traceback_frame.formatted_message()) message_lines.append("") # Step3: Adds error message like "TypeError: dtype must be int32, but received float32". @@ -413,7 +413,7 @@ def _simplify_error_value(self): traceback_frame = TraceBackFrame( Location(filepath, lineno), funcname, code ) - error_frame.append(traceback_frame.formated_message()) + error_frame.append(traceback_frame.formatted_message()) error_frame.append("") # Add paddle traceback after user code traceback @@ -428,7 +428,7 @@ def _simplify_error_value(self): traceback_frame = TraceBackFrame( Location(filepath, lineno), funcname, code ) - error_frame.append(traceback_frame.formated_message()) + error_frame.append(traceback_frame.formatted_message()) error_frame.append("") error_frame.extend(bottom_error_message) diff --git a/python/paddle/jit/dy2static/origin_info.py b/python/paddle/jit/dy2static/origin_info.py index 3115262c4148d..cff76af463419 100644 --- a/python/paddle/jit/dy2static/origin_info.py +++ b/python/paddle/jit/dy2static/origin_info.py @@ -69,7 +69,7 @@ def __str__(self): self.location, self.source_code, self.function_name ) - def formated_message(self): + def formatted_message(self): flag_for_origin_info = "(* user code *)" return ' File "{}", line {}, in {} {}\n\t{}'.format( self.location.filepath, From 2e95fdbfa0b3200694e9eff51abffe17026eb3af Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 4 Mar 2024 16:20:26 +0800 Subject: [PATCH 271/282] Fix dimensionss dimensions, etc (#62289) * Fix * ci --- .../kernels/fusion/xpu/bn_act_xpu_kernel.cc | 2 +- .../xpu/fused_feedforward_grad_kernel.cc | 2 +- .../fusion/xpu/multi_encoder_xpu_kernel.cc | 2 +- .../fusion/xpu/qkv_attention_xpu_kernel.cc | 2 +- .../phi/kernels/xpu/batch_norm_grad_kernel.cc | 6 ++--- paddle/phi/kernels/xpu/batch_norm_kernel.cc | 4 ++-- paddle/phi/kernels/xpu/bitwise.cc | 2 +- .../phi/kernels/xpu/embedding_grad_kernel.cc | 2 +- .../xpu/fused_attention_grad_kernel.cc | 22 +++++++++---------- .../phi/kernels/xpu/fused_attention_kernel.cc | 14 ++++++------ 10 files changed, 29 insertions(+), 29 deletions(-) diff --git a/paddle/phi/kernels/fusion/xpu/bn_act_xpu_kernel.cc b/paddle/phi/kernels/fusion/xpu/bn_act_xpu_kernel.cc index 82840ec1b3537..17ff819d346d3 100644 --- a/paddle/phi/kernels/fusion/xpu/bn_act_xpu_kernel.cc +++ b/paddle/phi/kernels/fusion/xpu/bn_act_xpu_kernel.cc @@ -69,7 +69,7 @@ void BNActXPUKernel(const Context& dev_ctx, 5, phi::errors::InvalidArgument( "The size of input X's dimensions should be less than 6." - "But received: the size of input X's dimensionss is [%d]", + "But received: the size of input X's dimensions is [%d]", x_dims.size())); bool is_nchw = data_layout_str == "NCHW"; diff --git a/paddle/phi/kernels/fusion/xpu/fused_feedforward_grad_kernel.cc b/paddle/phi/kernels/fusion/xpu/fused_feedforward_grad_kernel.cc index 29f74e8e1fe23..aeb5cb22cbe66 100644 --- a/paddle/phi/kernels/fusion/xpu/fused_feedforward_grad_kernel.cc +++ b/paddle/phi/kernels/fusion/xpu/fused_feedforward_grad_kernel.cc @@ -231,7 +231,7 @@ void FFNGrad(const phi::XPUContext& dev_ctx, std::tie(info_d_dropout1, info_dw2, a_1, b_1, a_2, b_2) = fc_info; - // if l3_total_size >= dim_feedforward * bsz_seq * sizeof(T), first transpos + // if l3_total_size >= dim_feedforward * bsz_seq * sizeof(T), first transpose if (l3_total_size >= dim_feedforward * bsz_seq * sizeof(T) && info_dw2.trans_x) { r = xpu::transpose(xpu_ctx, diff --git a/paddle/phi/kernels/fusion/xpu/multi_encoder_xpu_kernel.cc b/paddle/phi/kernels/fusion/xpu/multi_encoder_xpu_kernel.cc index 0b311eb0e65f7..8b65964671b0b 100644 --- a/paddle/phi/kernels/fusion/xpu/multi_encoder_xpu_kernel.cc +++ b/paddle/phi/kernels/fusion/xpu/multi_encoder_xpu_kernel.cc @@ -6,7 +6,7 @@ // // http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, sofint16_tare +// Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and diff --git a/paddle/phi/kernels/fusion/xpu/qkv_attention_xpu_kernel.cc b/paddle/phi/kernels/fusion/xpu/qkv_attention_xpu_kernel.cc index b08921e750a80..5c8562d6c3969 100644 --- a/paddle/phi/kernels/fusion/xpu/qkv_attention_xpu_kernel.cc +++ b/paddle/phi/kernels/fusion/xpu/qkv_attention_xpu_kernel.cc @@ -6,7 +6,7 @@ // // http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, sofint16_tare +// Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and diff --git a/paddle/phi/kernels/xpu/batch_norm_grad_kernel.cc b/paddle/phi/kernels/xpu/batch_norm_grad_kernel.cc index 454141ff4c3ea..7579d4f922d64 100644 --- a/paddle/phi/kernels/xpu/batch_norm_grad_kernel.cc +++ b/paddle/phi/kernels/xpu/batch_norm_grad_kernel.cc @@ -96,7 +96,7 @@ void BatchNormGradKernel(const Context &dev_ctx, true, phi::errors::InvalidArgument( "The 'data_layout' attribute must be NCHW or NHWC. " - "But recevived 'data_layout' is [%s].", + "But received 'data_layout' is [%s].", data_layout)); const auto data_layout_val = common::StringToDataLayout(data_layout); @@ -120,7 +120,7 @@ void BatchNormGradKernel(const Context &dev_ctx, x_dims.size() >= 2 && x_dims.size() <= 5, true, phi::errors::InvalidArgument( - "The size of input's dimensions should be between 2 and 5" + "The size of input's dimensions should be between 2 and 5. " "But received: the size of input's dimensions is [%d]", x_dims.size())); @@ -192,7 +192,7 @@ void BatchNormGradKernel(const Context &dev_ctx, const auto *global_mean = mean.get_ptr(); const auto *global_var = variance.get_ptr(); - // TODO(guozibin): hadle the situation case of N * H * W = 1 + // TODO(guozibin): handle the situation case of N * H * W = 1 int r = 0; if (is_inplace) { float *global_inv_std_data = nullptr; diff --git a/paddle/phi/kernels/xpu/batch_norm_kernel.cc b/paddle/phi/kernels/xpu/batch_norm_kernel.cc index 8427c49b43d42..81dd253460337 100644 --- a/paddle/phi/kernels/xpu/batch_norm_kernel.cc +++ b/paddle/phi/kernels/xpu/batch_norm_kernel.cc @@ -48,7 +48,7 @@ void BatchNormKernel(const Context& dev_ctx, true, phi::errors::InvalidArgument( "The 'data_layout' attribute must be NCHW or NHWC. " - "But recevived 'data_layout' is [%s].", + "But received 'data_layout' is [%s].", data_layout_str)); const auto& x_dims = x.dims(); @@ -104,7 +104,7 @@ void BatchNormKernel(const Context& dev_ctx, 5, phi::errors::InvalidArgument( "The size of input X's dimensions should be less than 6." - "But received: the size of input X's dimensionss is [%d]", + "But received: the size of input X's dimensions is [%d]", x_dims.size())); bool is_nchw = data_layout_str == "NCHW"; diff --git a/paddle/phi/kernels/xpu/bitwise.cc b/paddle/phi/kernels/xpu/bitwise.cc index dee96be39e185..c9eb0d93a66f0 100644 --- a/paddle/phi/kernels/xpu/bitwise.cc +++ b/paddle/phi/kernels/xpu/bitwise.cc @@ -39,7 +39,7 @@ void BitwiseAndKernel(const Context& ctx, const DenseTensor& y, DenseTensor* out) { // XPU api do not support bitwise operation now. - // However, because biwise and logical operation is identical for bool type, + // However, because bitwise and logical operation is identical for bool type, // we can implement bitwise_and_bool kernel by calling their logical // counterpart. Need to be changed when adding support to other types. LogicalAndKernel(ctx, x, y, out); diff --git a/paddle/phi/kernels/xpu/embedding_grad_kernel.cc b/paddle/phi/kernels/xpu/embedding_grad_kernel.cc index 3d0d0355b635f..11fd3826f4f6f 100644 --- a/paddle/phi/kernels/xpu/embedding_grad_kernel.cc +++ b/paddle/phi/kernels/xpu/embedding_grad_kernel.cc @@ -109,7 +109,7 @@ void EmbeddingSparseGradKernel(const Context& ctx, ids = CopyIdsToVector(ids_cpu); } else { PADDLE_THROW(phi::errors::Unimplemented( - "emebdding input only support int32 and int64")); + "embedding input only support int32 and int64")); } auto ids_num = static_cast(input.numel()); diff --git a/paddle/phi/kernels/xpu/fused_attention_grad_kernel.cc b/paddle/phi/kernels/xpu/fused_attention_grad_kernel.cc index c4432f82d9b26..fe989318cbcb4 100644 --- a/paddle/phi/kernels/xpu/fused_attention_grad_kernel.cc +++ b/paddle/phi/kernels/xpu/fused_attention_grad_kernel.cc @@ -224,9 +224,9 @@ void FusedAttentionGradKernel( XPUTypeT *d_dropout_grad_ptr = NULL; // dx5 [batch_size, seq_len, hidden] XPUTypeT *d_fmha_out_ptr = - NULL; // d_fmha_out [batch_size, seq_len, num_heads, head_dims] - XPUTypeT *d_fmha_out_transpos_tmp_ptr = - NULL; // d_fmha_out_transpos [batch_size, seq_len, num_heads, + NULL; // d_fmha_out [batch_size, seq_len, num_heads, head_dims] + XPUTypeT *d_fmha_out_transpose_tmp_ptr = + NULL; // d_fmha_out_transpose [batch_size, seq_len, num_heads, // head_dims] XPUTypeT *d_qk_ptr = @@ -235,7 +235,7 @@ void FusedAttentionGradKernel( XPUTypeT *d_combination_qkv_ptr = NULL; // d_combination_qkv_ptr[3, batch_size, num_heads, seq_len, // head_dims] - XPUTypeT *d_transpos_qkv_ptr = + XPUTypeT *d_transpose_qkv_ptr = NULL; // dx2 [batch_size, seq_len, 3, num_heads, head_dims] XPUTypeT *d_last_layernorm_grad_ptr = @@ -250,9 +250,9 @@ void FusedAttentionGradKernel( num_heads * head_dims); d_combination_qkv_ptr = RAII_GUARD.alloc(batch_size * seq_len * embed_dims * 3); - d_transpos_qkv_ptr = RAII_GUARD.alloc_l3_or_gm( + d_transpose_qkv_ptr = RAII_GUARD.alloc_l3_or_gm( batch_size * seq_len * embed_dims * 3); - d_fmha_out_transpos_tmp_ptr = + d_fmha_out_transpose_tmp_ptr = RAII_GUARD.alloc_l3_or_gm(batch_size * seq_len * embed_dims); d_qk_ptr = RAII_GUARD.alloc_l3_or_gm(batch_size * seq_len * seq_len * num_heads); @@ -343,7 +343,7 @@ void FusedAttentionGradKernel( XPUTypeT *d_v_out_ptr = d_k_out_ptr + qkv_size; r = xpu::transpose(xpu_ctx, d_fmha_out_ptr, - d_fmha_out_transpos_tmp_ptr, + d_fmha_out_transpose_tmp_ptr, {batch_size, seq_len, num_heads, head_dims}, {0, 2, 1, 3}); PADDLE_ENFORCE_XDNN_SUCCESS(r, "transpose"); @@ -381,7 +381,7 @@ void FusedAttentionGradKernel( false, attn_dropout_out_ptr, v_out_ptr, - d_fmha_out_transpos_tmp_ptr); + d_fmha_out_transpose_tmp_ptr); std::tie(info_d_qk, info_d_v, a_1, b_1, a_2, b_2) = fc_info; phi::MatMulXPUFunction( @@ -452,7 +452,7 @@ void FusedAttentionGradKernel( // r = xpu::transpose(xpu_ctx, d_combination_qkv_ptr, - d_transpos_qkv_ptr, + d_transpose_qkv_ptr, {3, batch_size, num_heads, seq_len, head_dims}, {1, 3, 0, 2, 4}); PADDLE_ENFORCE_XDNN_SUCCESS(r, "transpose"); @@ -487,7 +487,7 @@ void FusedAttentionGradKernel( true, use_calc_input_x_ptr, qkv_weight_ptr, - d_transpos_qkv_ptr); + d_transpose_qkv_ptr); std::tie(info_d_x, info_d_qkv_w, a_1, b_1, a_2, b_2) = fc_info; phi::MatMulXPUFunction( @@ -497,7 +497,7 @@ void FusedAttentionGradKernel( // d_qkv_bias r = xpu::reduce_sum(xpu_ctx, - d_transpos_qkv_ptr, + d_transpose_qkv_ptr, d_qkv_bias_ptr, {batch_size * seq_len, 3 * embed_dims}, {0}); diff --git a/paddle/phi/kernels/xpu/fused_attention_kernel.cc b/paddle/phi/kernels/xpu/fused_attention_kernel.cc index d18dda47866ef..b7a1c8a638648 100644 --- a/paddle/phi/kernels/xpu/fused_attention_kernel.cc +++ b/paddle/phi/kernels/xpu/fused_attention_kernel.cc @@ -199,7 +199,7 @@ void FusedAttentionKernel(const Context &dev_ctx, int l3_total_size = xpu_ctx->_l3_mgr.get_size(); - XPUTypeT *qkv_before_transpos_ptr = + XPUTypeT *qkv_before_transpose_ptr = NULL; // x2[batch_size, seq_len, 3, num_heads,head_dims] XPUTypeT *qk_ptr = NULL; // qk [batch_size, num_heads, seq_len, seq_len] XPUTypeT *qkv_ptr = NULL; // qkv[batch_size, num_heads, seq_len, head_dims] @@ -215,7 +215,7 @@ void FusedAttentionKernel(const Context &dev_ctx, std::sort(temp_vec.begin(), temp_vec.end(), std::greater()); XPUTypeT *max_gm_ptr = RAII_GUARD.alloc(temp_vec[0]); PADDLE_ENFORCE_XDNN_NOT_NULL(max_gm_ptr); - qkv_before_transpos_ptr = max_gm_ptr; + qkv_before_transpose_ptr = max_gm_ptr; qk_ptr = max_gm_ptr; qkv_ptr = max_gm_ptr; linear_out_ptr = max_gm_ptr; @@ -223,7 +223,7 @@ void FusedAttentionKernel(const Context &dev_ctx, for (size_t i = 0; i < temp_vec.size(); ++i) { if (l3_total_size >= temp_vec[i] * sizeof_t) { XPUTypeT *l3_ptr = RAII_GUARD.alloc_l3(temp_vec[i]); - qkv_before_transpos_ptr = + qkv_before_transpose_ptr = (temp_size_1 <= temp_vec[i]) ? l3_ptr : max_gm_ptr; qk_ptr = (temp_size_2 <= temp_vec[i]) ? l3_ptr : max_gm_ptr; qkv_ptr = (temp_size_3 <= temp_vec[i]) ? l3_ptr : max_gm_ptr; @@ -264,22 +264,22 @@ void FusedAttentionKernel(const Context &dev_ctx, phi::MatMulXPUFunction(xpu_ctx, x_cacl_ptr, qkv_weight_ptr, - qkv_before_transpos_ptr, + qkv_before_transpose_ptr, qkv_fc_info, 1.0f); // bias r = xpu::broadcast_add(xpu_ctx, - qkv_before_transpos_ptr, + qkv_before_transpose_ptr, qkv_bias_ptr, - qkv_before_transpos_ptr, + qkv_before_transpose_ptr, {batch_size * seq_len, 3 * num_heads * head_dims}, {3 * num_heads * head_dims}); PADDLE_ENFORCE_XDNN_SUCCESS(r, "broadcast_add"); // transpose r = xpu::transpose(xpu_ctx, - qkv_before_transpos_ptr, + qkv_before_transpose_ptr, qkv_transpose_out_ptr, {batch_size, seq_len, 3, num_heads, head_dims}, {2, 0, 3, 1, 4}); From b625897a81c56a37d9929bae67548aab539512e3 Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 4 Mar 2024 16:21:40 +0800 Subject: [PATCH 272/282] Change XPUT -> XPUType (#62307) --- .../fused/resnet_basic_block_op_xpu.cc | 425 ++++++++-------- .../fusion/xpu/conv_transpose_xpu_kernel.cc | 8 +- .../fusion/xpu/fused_rope_grad_kernel.cc | 28 +- .../kernels/fusion/xpu/fused_rope_kernel.cc | 29 +- .../phi/kernels/fusion/xpu/fused_rope_utils.h | 48 +- paddle/phi/kernels/xpu/bmm_grad_kernel.cc | 4 +- paddle/phi/kernels/xpu/bmm_kernel.cc | 4 +- paddle/phi/kernels/xpu/conv_grad_kernel.cc | 480 +++++++++--------- paddle/phi/kernels/xpu/conv_kernel.cc | 356 ++++++------- .../phi/kernels/xpu/conv_transpose_kernel.cc | 12 +- .../phi/kernels/xpu/embedding_grad_kernel.cc | 8 +- paddle/phi/kernels/xpu/index_put_kernel.cc | 20 +- paddle/phi/kernels/xpu/inverse_kernel.cc | 14 +- .../phi/kernels/xpu/multiclass_nms3_kernel.cc | 8 +- .../kernels/xpu/scatter_nd_add_grad_kernel.cc | 33 +- 15 files changed, 749 insertions(+), 728 deletions(-) diff --git a/paddle/fluid/operators/fused/resnet_basic_block_op_xpu.cc b/paddle/fluid/operators/fused/resnet_basic_block_op_xpu.cc index bd918924cdf09..f2e8add25028c 100644 --- a/paddle/fluid/operators/fused/resnet_basic_block_op_xpu.cc +++ b/paddle/fluid/operators/fused/resnet_basic_block_op_xpu.cc @@ -295,7 +295,7 @@ static inline void xpu_conv2d_grad(xpu::Context* ctx, template class ResNetBasicBlockXPUKernel : public framework::OpKernel { public: - using XPUT = typename XPUTypeTrait::Type; + using XPUType = typename XPUTypeTrait::Type; void Compute(const framework::ExecutionContext& ctx) const override { PADDLE_ENFORCE_EQ( @@ -319,20 +319,23 @@ class ResNetBasicBlockXPUKernel : public framework::OpKernel { phi::DenseTensor* output = ctx.Output("Y"); auto place = ctx.GetPlace(); - auto x_data = reinterpret_cast(x->data()); - auto conv1_filter_data = reinterpret_cast(filter1->data()); - auto conv2_filter_data = reinterpret_cast(filter2->data()); + auto x_data = reinterpret_cast(x->data()); + auto conv1_filter_data = + reinterpret_cast(filter1->data()); + auto conv2_filter_data = + reinterpret_cast(filter2->data()); auto conv1_output_data = - reinterpret_cast(conv1_output->mutable_data(place)); + reinterpret_cast(conv1_output->mutable_data(place)); auto conv2_input_data = - reinterpret_cast(conv2_input->mutable_data(place)); + reinterpret_cast(conv2_input->mutable_data(place)); auto conv2_output_data = - reinterpret_cast(conv2_output->mutable_data(place)); + reinterpret_cast(conv2_output->mutable_data(place)); auto scale1_data = scale1->data(); auto scale2_data = scale2->data(); auto bias1_data = bias1->data(); auto bias2_data = bias2->data(); - auto output_data = reinterpret_cast(output->mutable_data(place)); + auto output_data = + reinterpret_cast(output->mutable_data(place)); float* conv1_input_max_data = nullptr; float* conv1_filter_max_data = nullptr; @@ -372,18 +375,18 @@ class ResNetBasicBlockXPUKernel : public framework::OpKernel { int r = XPU_SUCCESS; // 1. short - const XPUT* z_out_data = nullptr; + const XPUType* z_out_data = nullptr; if (attr.has_shortcut) { phi::DenseTensor* conv3_out = ctx.Output("Conv3"); const phi::DenseTensor* filter3 = ctx.Input("Filter3"); auto conv3_filter_data = - reinterpret_cast(filter3->data()); + reinterpret_cast(filter3->data()); auto conv3_output_data = - reinterpret_cast(conv3_out->mutable_data(place)); + reinterpret_cast(conv3_out->mutable_data(place)); - XPUT* conv3_input_l3_data = nullptr; - XPUT* conv3_filter_l3_data = - RAII_GUARD.alloc_l3(attr.conv3_filter_numel); + XPUType* conv3_input_l3_data = nullptr; + XPUType* conv3_filter_l3_data = + RAII_GUARD.alloc_l3(attr.conv3_filter_numel); if (attr.find_max) { r = xpu::findmax_copy_fusion(dev_ctx.x_context(), @@ -420,7 +423,7 @@ class ResNetBasicBlockXPUKernel : public framework::OpKernel { auto bias3_data = bias3->data(); auto scale3_data = scale3->data(); - auto bn3_output_data = RAII_GUARD.alloc(attr.conv3_output_numel); + auto bn3_output_data = RAII_GUARD.alloc(attr.conv3_output_numel); PADDLE_ENFORCE_XDNN_NOT_NULL(bn3_output_data); if (!attr.global_stats) { @@ -438,56 +441,56 @@ class ResNetBasicBlockXPUKernel : public framework::OpKernel { auto running_mean3_data = running_mean3->mutable_data(place); auto running_var3_data = running_var3->mutable_data(place); - r = xpu::batch_norm_fusion(dev_ctx.x_context(), - conv3_output_data, - bn3_output_data, - attr.conv3_output_shape[0], - attr.conv3_output_shape[1], - attr.conv3_output_shape[3], - attr.conv3_output_shape[3], - attr.eps, - attr.momentum, - scale3_data, - bias3_data, - saved_mean3_data, - saved_invstd3_data, - running_mean3_data, - running_var3_data, - true, - nullptr, - xpu::Activation_t::LINEAR, - nullptr, - 0); + r = xpu::batch_norm_fusion(dev_ctx.x_context(), + conv3_output_data, + bn3_output_data, + attr.conv3_output_shape[0], + attr.conv3_output_shape[1], + attr.conv3_output_shape[3], + attr.conv3_output_shape[3], + attr.eps, + attr.momentum, + scale3_data, + bias3_data, + saved_mean3_data, + saved_invstd3_data, + running_mean3_data, + running_var3_data, + true, + nullptr, + xpu::Activation_t::LINEAR, + nullptr, + 0); PADDLE_ENFORCE_XDNN_SUCCESS(r, "batch_norm_fusion"); } else { const auto* mean3 = ctx.Input("Mean3"); const auto* var3 = ctx.Input("Var3"); const auto* mean3_data = mean3->data(); const auto* variance3_data = var3->data(); - r = xpu::batch_norm_infer(dev_ctx.x_context(), - conv3_output_data, - bn3_output_data, - attr.conv3_output_shape[0], - attr.conv3_output_shape[1], - attr.conv3_output_shape[2], - attr.conv3_output_shape[3], - attr.eps, - scale3_data, - bias3_data, - mean3_data, - variance3_data, - true); + r = xpu::batch_norm_infer(dev_ctx.x_context(), + conv3_output_data, + bn3_output_data, + attr.conv3_output_shape[0], + attr.conv3_output_shape[1], + attr.conv3_output_shape[2], + attr.conv3_output_shape[3], + attr.eps, + scale3_data, + bias3_data, + mean3_data, + variance3_data, + true); PADDLE_ENFORCE_XDNN_SUCCESS(r, "batch_norm_infer"); } - z_out_data = reinterpret_cast(bn3_output_data); + z_out_data = reinterpret_cast(bn3_output_data); } else { z_out_data = x_data; } // 2. conv1 - XPUT* conv1_input_l3_data = nullptr; - XPUT* conv1_filter_l3_data = - RAII_GUARD.alloc_l3(attr.conv1_filter_numel); + XPUType* conv1_input_l3_data = nullptr; + XPUType* conv1_filter_l3_data = + RAII_GUARD.alloc_l3(attr.conv1_filter_numel); if (attr.find_max) { r = xpu::findmax_copy_fusion(dev_ctx.x_context(), x_data, @@ -531,49 +534,49 @@ class ResNetBasicBlockXPUKernel : public framework::OpKernel { auto running_mean1_data = running_mean1->mutable_data(place); auto running_var1_data = running_var1->mutable_data(place); - r = xpu::batch_norm_fusion(dev_ctx.x_context(), - conv1_output_data, - conv2_input_data, - attr.conv1_output_shape[0], - attr.conv1_output_shape[1], - attr.conv1_output_shape[2], - attr.conv1_output_shape[3], - attr.eps, - attr.momentum, - scale1_data, - bias1_data, - saved_mean1_data, - saved_invstd1_data, - running_mean1_data, - running_var1_data, - true, - nullptr, - xpu::Activation_t::RELU, - nullptr, - 0); + r = xpu::batch_norm_fusion(dev_ctx.x_context(), + conv1_output_data, + conv2_input_data, + attr.conv1_output_shape[0], + attr.conv1_output_shape[1], + attr.conv1_output_shape[2], + attr.conv1_output_shape[3], + attr.eps, + attr.momentum, + scale1_data, + bias1_data, + saved_mean1_data, + saved_invstd1_data, + running_mean1_data, + running_var1_data, + true, + nullptr, + xpu::Activation_t::RELU, + nullptr, + 0); PADDLE_ENFORCE_XDNN_SUCCESS(r, "batch_norm_fusion"); } else { // bn --> relu - auto bn1_output_data = RAII_GUARD.alloc(attr.conv1_output_numel); + auto bn1_output_data = RAII_GUARD.alloc(attr.conv1_output_numel); PADDLE_ENFORCE_XDNN_NOT_NULL(bn1_output_data); const auto* mean1 = ctx.Input("Mean1"); const auto* var1 = ctx.Input("Var1"); const auto* mean_data = mean1->data(); const auto* variance_data = var1->data(); - r = xpu::batch_norm_infer(dev_ctx.x_context(), - conv1_output_data, - bn1_output_data, - attr.conv1_output_shape[0], - attr.conv1_output_shape[1], - attr.conv1_output_shape[2], - attr.conv1_output_shape[3], - attr.eps, - scale1_data, - bias1_data, - mean_data, - variance_data, - true); + r = xpu::batch_norm_infer(dev_ctx.x_context(), + conv1_output_data, + bn1_output_data, + attr.conv1_output_shape[0], + attr.conv1_output_shape[1], + attr.conv1_output_shape[2], + attr.conv1_output_shape[3], + attr.eps, + scale1_data, + bias1_data, + mean_data, + variance_data, + true); PADDLE_ENFORCE_XDNN_SUCCESS(r, "batch_norm_infer"); r = xpu::relu(dev_ctx.x_context(), @@ -584,9 +587,9 @@ class ResNetBasicBlockXPUKernel : public framework::OpKernel { } // 4. conv2 - XPUT* conv2_input_l3_data = nullptr; - XPUT* conv2_filter_l3_data = - RAII_GUARD.alloc_l3(attr.conv2_filter_numel); + XPUType* conv2_input_l3_data = nullptr; + XPUType* conv2_filter_l3_data = + RAII_GUARD.alloc_l3(attr.conv2_filter_numel); if (attr.find_max) { phi::DenseTensor* max_input2 = ctx.Output("MaxInput2"); phi::DenseTensor* max_filter2 = @@ -637,59 +640,59 @@ class ResNetBasicBlockXPUKernel : public framework::OpKernel { auto running_mean2_data = running_mean2->mutable_data(place); auto running_var2_data = running_var2->mutable_data(place); - r = xpu::batch_norm_fusion(dev_ctx.x_context(), - conv2_output_data, - output_data, - attr.conv2_output_shape[0], - attr.conv2_output_shape[1], - attr.conv2_output_shape[2], - attr.conv2_output_shape[3], - attr.eps, - attr.momentum, - scale2_data, - bias2_data, - saved_mean2_data, - saved_var2_data, - running_mean2_data, - running_var2_data, - true, - z_out_data, - xpu::Activation_t::RELU, - nullptr, - 0); + r = xpu::batch_norm_fusion(dev_ctx.x_context(), + conv2_output_data, + output_data, + attr.conv2_output_shape[0], + attr.conv2_output_shape[1], + attr.conv2_output_shape[2], + attr.conv2_output_shape[3], + attr.eps, + attr.momentum, + scale2_data, + bias2_data, + saved_mean2_data, + saved_var2_data, + running_mean2_data, + running_var2_data, + true, + z_out_data, + xpu::Activation_t::RELU, + nullptr, + 0); PADDLE_ENFORCE_XDNN_SUCCESS(r, "batch_norm_fusion"); } else { - auto bn2_out_data = RAII_GUARD.alloc(attr.conv2_output_numel); + auto bn2_out_data = RAII_GUARD.alloc(attr.conv2_output_numel); PADDLE_ENFORCE_XDNN_NOT_NULL(bn2_out_data); const auto* mean2 = ctx.Input("Mean2"); const auto* var2 = ctx.Input("Var2"); const auto* mean_data = mean2->data(); const auto* variance_data = var2->data(); - r = xpu::batch_norm_infer(dev_ctx.x_context(), - conv2_output_data, - bn2_out_data, - attr.conv2_output_shape[0], - attr.conv2_output_shape[1], - attr.conv2_output_shape[2], - attr.conv2_output_shape[3], - attr.eps, - scale2_data, - bias2_data, - mean_data, - variance_data, - true); + r = xpu::batch_norm_infer(dev_ctx.x_context(), + conv2_output_data, + bn2_out_data, + attr.conv2_output_shape[0], + attr.conv2_output_shape[1], + attr.conv2_output_shape[2], + attr.conv2_output_shape[3], + attr.eps, + scale2_data, + bias2_data, + mean_data, + variance_data, + true); PADDLE_ENFORCE_XDNN_SUCCESS(r, "batch_norm_infer"); - r = xpu::add_activation_fusion(dev_ctx.x_context(), - bn2_out_data, - z_out_data, - output_data, - output->numel(), - nullptr, - nullptr, - nullptr, - xpu::Activation_t::RELU); + r = xpu::add_activation_fusion(dev_ctx.x_context(), + bn2_out_data, + z_out_data, + output_data, + output->numel(), + nullptr, + nullptr, + nullptr, + xpu::Activation_t::RELU); PADDLE_ENFORCE_XDNN_SUCCESS(r, "add_activation_fusion"); } } @@ -698,7 +701,7 @@ class ResNetBasicBlockXPUKernel : public framework::OpKernel { template class ResNetBasicBlockGradXPUKernel : public framework::OpKernel { public: - using XPUT = typename XPUTypeTrait::Type; + using XPUType = typename XPUTypeTrait::Type; void Compute(const framework::ExecutionContext& ctx) const override { PADDLE_ENFORCE_EQ( @@ -774,19 +777,20 @@ class ResNetBasicBlockGradXPUKernel : public framework::OpKernel { ResnetBasicBlockGradAttr attr(ctx); auto place = ctx.GetPlace(); - const auto* y_grad_data = reinterpret_cast(y_grad->data()); - const auto* y_data = reinterpret_cast(y->data()); - const auto* x_data = reinterpret_cast(x->data()); + const auto* y_grad_data = + reinterpret_cast(y_grad->data()); + const auto* y_data = reinterpret_cast(y->data()); + const auto* x_data = reinterpret_cast(x->data()); const auto* conv1_output_data = - reinterpret_cast(conv1_out->data()); + reinterpret_cast(conv1_out->data()); const auto* conv1_filter_data = - reinterpret_cast(filter1->data()); + reinterpret_cast(filter1->data()); const auto* conv2_input_data = - reinterpret_cast(conv2_input->data()); + reinterpret_cast(conv2_input->data()); const auto* conv2_output_data = - reinterpret_cast(conv2_out->data()); + reinterpret_cast(conv2_out->data()); const auto* conv2_filter_data = - reinterpret_cast(filter2->data()); + reinterpret_cast(filter2->data()); const auto* scale2_data = scale2->data(); const auto* saved_mean2_data = saved_mean2->data(); @@ -826,77 +830,77 @@ class ResNetBasicBlockGradXPUKernel : public framework::OpKernel { // 0. bn2, bn2_fusion grad auto conv2_output_grad_data = - RAII_GUARD.alloc(attr.conv2_output_numel); + RAII_GUARD.alloc(attr.conv2_output_numel); PADDLE_ENFORCE_XDNN_NOT_NULL(conv2_output_grad_data); - XPUT* z_output_grad_data = nullptr; - XPUT* z_grad_data = nullptr; + XPUType* z_output_grad_data = nullptr; + XPUType* z_grad_data = nullptr; if (!attr.has_shortcut) { - z_output_grad_data = RAII_GUARD.alloc(attr.conv1_input_numel); + z_output_grad_data = RAII_GUARD.alloc(attr.conv1_input_numel); PADDLE_ENFORCE_XDNN_NOT_NULL(z_output_grad_data); z_grad_data = z_output_grad_data; } else { - z_output_grad_data = RAII_GUARD.alloc(attr.conv3_output_numel); + z_output_grad_data = RAII_GUARD.alloc(attr.conv3_output_numel); PADDLE_ENFORCE_XDNN_NOT_NULL(z_output_grad_data); - z_grad_data = RAII_GUARD.alloc(attr.conv1_input_numel); + z_grad_data = RAII_GUARD.alloc(attr.conv1_input_numel); PADDLE_ENFORCE_XDNN_NOT_NULL(z_grad_data); } - r = xpu::batch_norm_grad_fusion(dev_ctx.x_context(), - conv2_output_data, - y_data, - y_grad_data, - conv2_output_grad_data, - attr.conv2_output_shape[0], - attr.conv2_output_shape[1], - attr.conv2_output_shape[2], - attr.conv2_output_shape[3], - scale2_data, - saved_mean2_data, - saved_invstd2_data, - scale2_grad_data, - bias2_grad_data, - true, - z_output_grad_data, - xpu::Activation_t::RELU, - nullptr, - 0); + r = xpu::batch_norm_grad_fusion(dev_ctx.x_context(), + conv2_output_data, + y_data, + y_grad_data, + conv2_output_grad_data, + attr.conv2_output_shape[0], + attr.conv2_output_shape[1], + attr.conv2_output_shape[2], + attr.conv2_output_shape[3], + scale2_data, + saved_mean2_data, + saved_invstd2_data, + scale2_grad_data, + bias2_grad_data, + true, + z_output_grad_data, + xpu::Activation_t::RELU, + nullptr, + 0); PADDLE_ENFORCE_XDNN_SUCCESS(r, "batch_norm_grad_fusion"); if (attr.has_shortcut) { // bn3 grad const auto* conv3_output_data = - reinterpret_cast(conv3_out->data()); + reinterpret_cast(conv3_out->data()); const auto* scale3_data = scale3->data(); const auto* saved_mean3_data = saved_mean3->data(); const auto* saved_invstd3_data = saved_invstd3->data(); auto* scale3_grad_data = scale3_grad->mutable_data(place); auto* bias3_grad_data = bias3_grad->mutable_data(place); auto* conv3_output_grad_data = - RAII_GUARD.alloc(attr.conv3_output_numel); - - r = xpu::batch_norm_grad(dev_ctx.x_context(), - conv3_output_data, - z_output_grad_data, - conv3_output_grad_data, - attr.conv3_output_shape[0], - attr.conv3_output_shape[1], - attr.conv3_output_shape[2], - attr.conv3_output_shape[3], - scale3_data, - saved_mean3_data, - saved_invstd3_data, - scale3_grad_data, - bias3_grad_data, - true); + RAII_GUARD.alloc(attr.conv3_output_numel); + + r = xpu::batch_norm_grad(dev_ctx.x_context(), + conv3_output_data, + z_output_grad_data, + conv3_output_grad_data, + attr.conv3_output_shape[0], + attr.conv3_output_shape[1], + attr.conv3_output_shape[2], + attr.conv3_output_shape[3], + scale3_data, + saved_mean3_data, + saved_invstd3_data, + scale3_grad_data, + bias3_grad_data, + true); PADDLE_ENFORCE_XDNN_SUCCESS(r, "batch_norm_grad"); // conv3 grad auto* conv3_filter_grad_data = - reinterpret_cast(filter3_grad->mutable_data(place)); + reinterpret_cast(filter3_grad->mutable_data(place)); auto* conv3_filter_data = - reinterpret_cast(filter3->data()); + reinterpret_cast(filter3->data()); xpu_conv2d_grad(dev_ctx.x_context(), x_data, conv3_filter_data, @@ -915,9 +919,9 @@ class ResNetBasicBlockGradXPUKernel : public framework::OpKernel { // 2. conv2_grad auto* conv2_filter_grad_data = - reinterpret_cast(filter2_grad->mutable_data(place)); + reinterpret_cast(filter2_grad->mutable_data(place)); auto* conv2_input_grad_data = - RAII_GUARD.alloc(attr.conv2_input_numel); + RAII_GUARD.alloc(attr.conv2_input_numel); xpu_conv2d_grad(dev_ctx.x_context(), conv2_input_data, conv2_filter_data, @@ -935,35 +939,36 @@ class ResNetBasicBlockGradXPUKernel : public framework::OpKernel { // 3. b1 grad auto* conv1_output_grad_data = - RAII_GUARD.alloc(attr.conv1_output_numel); + RAII_GUARD.alloc(attr.conv1_output_numel); PADDLE_ENFORCE_XDNN_NOT_NULL(conv1_output_grad_data); auto* scale1_grad_data = scale1_grad->mutable_data(ctx.GetPlace()); auto* bias1_grad_data = bias1_grad->mutable_data(ctx.GetPlace()); - r = xpu::batch_norm_grad_fusion(dev_ctx.x_context(), - conv1_output_data, - conv2_input_data, - conv2_input_grad_data, - conv1_output_grad_data, - attr.conv1_output_shape[0], - attr.conv1_output_shape[1], - attr.conv1_output_shape[2], - attr.conv1_output_shape[3], - scale1_data, - saved_mean1_data, - saved_invstd1_data, - scale1_grad_data, - bias1_grad_data, - true, - nullptr, - xpu::Activation_t::RELU, - nullptr, - 0); + r = xpu::batch_norm_grad_fusion(dev_ctx.x_context(), + conv1_output_data, + conv2_input_data, + conv2_input_grad_data, + conv1_output_grad_data, + attr.conv1_output_shape[0], + attr.conv1_output_shape[1], + attr.conv1_output_shape[2], + attr.conv1_output_shape[3], + scale1_data, + saved_mean1_data, + saved_invstd1_data, + scale1_grad_data, + bias1_grad_data, + true, + nullptr, + xpu::Activation_t::RELU, + nullptr, + 0); PADDLE_ENFORCE_XDNN_SUCCESS(r, "batch_norm_grad_fusion"); // 4. conv1_grad - auto* x_grad_data = reinterpret_cast(x_grad->mutable_data(place)); + auto* x_grad_data = + reinterpret_cast(x_grad->mutable_data(place)); auto* conv1_filter_grad_data = - reinterpret_cast(filter1_grad->mutable_data(place)); + reinterpret_cast(filter1_grad->mutable_data(place)); xpu_conv2d_grad(dev_ctx.x_context(), x_data, conv1_filter_data, @@ -980,7 +985,7 @@ class ResNetBasicBlockGradXPUKernel : public framework::OpKernel { attr.group); // add z_grad to x_grad - r = xpu::add( + r = xpu::add( dev_ctx.x_context(), x_grad_data, z_grad_data, x_grad_data, x->numel()); PADDLE_ENFORCE_XDNN_SUCCESS(r, "add"); } diff --git a/paddle/phi/kernels/fusion/xpu/conv_transpose_xpu_kernel.cc b/paddle/phi/kernels/fusion/xpu/conv_transpose_xpu_kernel.cc index 58f40f3040f74..cc66ee88b0787 100644 --- a/paddle/phi/kernels/fusion/xpu/conv_transpose_xpu_kernel.cc +++ b/paddle/phi/kernels/fusion/xpu/conv_transpose_xpu_kernel.cc @@ -39,7 +39,7 @@ void Conv2dTransposeXPUKernel(const Context& ctx, const std::string& act_type, DenseTensor* out, DenseTensor* out_max) { - using XPUT = typename XPUTypeTrait::Type; + using XPUType = typename XPUTypeTrait::Type; ctx.template Alloc(out); ctx.template Alloc(out_max); @@ -71,11 +71,11 @@ void Conv2dTransposeXPUKernel(const Context& ctx, x_max.get_ptr() == nullptr ? nullptr : x_max.get_ptr()->data(); auto filter_max_data = filter_max.data(); - int r = xpu::conv2d_transpose_fusion_v2( + int r = xpu::conv2d_transpose_fusion_v2( ctx.x_context(), - reinterpret_cast(x.data()), + reinterpret_cast(x.data()), filter.data(), - reinterpret_cast(out->data()), + reinterpret_cast(out->data()), batch_size, img_yc, img_xh, diff --git a/paddle/phi/kernels/fusion/xpu/fused_rope_grad_kernel.cc b/paddle/phi/kernels/fusion/xpu/fused_rope_grad_kernel.cc index 1e988ca9ea03e..831e6dbd778d8 100644 --- a/paddle/phi/kernels/fusion/xpu/fused_rope_grad_kernel.cc +++ b/paddle/phi/kernels/fusion/xpu/fused_rope_grad_kernel.cc @@ -32,7 +32,7 @@ void FusedRopeGradKernel(const Context& dev_ctx, DenseTensor* dq, DenseTensor* dk, DenseTensor* dv) { - using XPUT = typename XPUTypeTrait::Type; + using XPUType = typename XPUTypeTrait::Type; if (dout_q.numel() <= 0) { return; } @@ -48,8 +48,8 @@ void FusedRopeGradKernel(const Context& dev_ctx, xpu::ctx_guard RAII_GUARD(dev_ctx.x_context()); int64_t sin_cos_len = batch_size * seq_len * head_dim; - auto* sin_data = RAII_GUARD.alloc_l3_or_gm(sin_cos_len); - auto* cos_data = RAII_GUARD.alloc_l3_or_gm(sin_cos_len); + auto* sin_data = RAII_GUARD.alloc_l3_or_gm(sin_cos_len); + auto* cos_data = RAII_GUARD.alloc_l3_or_gm(sin_cos_len); if (sin.get_ptr() && cos.get_ptr()) { PADDLE_ENFORCE_EQ(sin.get_ptr()->dims(), @@ -61,9 +61,9 @@ void FusedRopeGradKernel(const Context& dev_ctx, cos.get_ptr()->dims())); } - XPUGetSinCosData( + XPUGetSinCosData( dev_ctx, sin, position_ids, sin_data, batch_size, seq_len, head_dim); - XPUGetSinCosData( + XPUGetSinCosData( dev_ctx, cos, position_ids, cos_data, batch_size, seq_len, head_dim); if (use_neox_rotary_style) { @@ -72,10 +72,10 @@ void FusedRopeGradKernel(const Context& dev_ctx, phi::errors::Unimplemented("XPU do not support rotary_embedding_grad " "with use_neox_rotary_style set.")); } else { - auto* dq_data = reinterpret_cast(dev_ctx.template Alloc(dq)); - XPUFusedRotaryHalf( + auto* dq_data = reinterpret_cast(dev_ctx.template Alloc(dq)); + XPUFusedRotaryHalf( dev_ctx, - reinterpret_cast(dout_q.data()), + reinterpret_cast(dout_q.data()), sin_data, cos_data, dq_data, @@ -86,10 +86,10 @@ void FusedRopeGradKernel(const Context& dev_ctx, true); if (dout_k.get_ptr()) { - auto* dk_data = reinterpret_cast(dev_ctx.template Alloc(dk)); - XPUFusedRotaryHalf( + auto* dk_data = reinterpret_cast(dev_ctx.template Alloc(dk)); + XPUFusedRotaryHalf( dev_ctx, - reinterpret_cast(dout_k->data()), + reinterpret_cast(dout_k->data()), sin_data, cos_data, dk_data, @@ -101,10 +101,10 @@ void FusedRopeGradKernel(const Context& dev_ctx, } if (dout_v.get_ptr()) { - auto* dv_data = reinterpret_cast(dev_ctx.template Alloc(dv)); - XPUFusedRotaryHalf( + auto* dv_data = reinterpret_cast(dev_ctx.template Alloc(dv)); + XPUFusedRotaryHalf( dev_ctx, - reinterpret_cast(dout_v->data()), + reinterpret_cast(dout_v->data()), sin_data, cos_data, dv_data, diff --git a/paddle/phi/kernels/fusion/xpu/fused_rope_kernel.cc b/paddle/phi/kernels/fusion/xpu/fused_rope_kernel.cc index c8980310fb0f9..b76b467686ea9 100644 --- a/paddle/phi/kernels/fusion/xpu/fused_rope_kernel.cc +++ b/paddle/phi/kernels/fusion/xpu/fused_rope_kernel.cc @@ -33,7 +33,7 @@ void FusedRopeKernel(const Context& dev_ctx, DenseTensor* out_q, DenseTensor* out_k, DenseTensor* out_v) { - using XPUT = typename XPUTypeTrait::Type; + using XPUType = typename XPUTypeTrait::Type; if (q.numel() <= 0) { return; } @@ -54,8 +54,8 @@ void FusedRopeKernel(const Context& dev_ctx, xpu::ctx_guard RAII_GUARD(dev_ctx.x_context()); int64_t sin_cos_len = batch_size * seq_len * head_dim; - auto* sin_data = RAII_GUARD.alloc_l3_or_gm(sin_cos_len); - auto* cos_data = RAII_GUARD.alloc_l3_or_gm(sin_cos_len); + auto* sin_data = RAII_GUARD.alloc_l3_or_gm(sin_cos_len); + auto* cos_data = RAII_GUARD.alloc_l3_or_gm(sin_cos_len); if (sin.get_ptr() && cos.get_ptr()) { PADDLE_ENFORCE_EQ(sin.get_ptr()->dims(), @@ -67,9 +67,9 @@ void FusedRopeKernel(const Context& dev_ctx, cos.get_ptr()->dims())); } - XPUGetSinCosData( + XPUGetSinCosData( dev_ctx, sin, position_ids, sin_data, batch_size, seq_len, head_dim); - XPUGetSinCosData( + XPUGetSinCosData( dev_ctx, cos, position_ids, cos_data, batch_size, seq_len, head_dim); if (use_neox_rotary_style) { @@ -77,10 +77,11 @@ void FusedRopeKernel(const Context& dev_ctx, PADDLE_THROW(phi::errors::Unimplemented( "XPU do not support rotary_embedding with use_neox_rotary_style set.")); } else { - auto* outq_data = reinterpret_cast(dev_ctx.template Alloc(out_q)); - XPUFusedRotaryHalf( + auto* outq_data = + reinterpret_cast(dev_ctx.template Alloc(out_q)); + XPUFusedRotaryHalf( dev_ctx, - reinterpret_cast(q.data()), + reinterpret_cast(q.data()), sin_data, cos_data, outq_data, @@ -91,10 +92,10 @@ void FusedRopeKernel(const Context& dev_ctx, if (k) { auto* outk_data = - reinterpret_cast(dev_ctx.template Alloc(out_k)); - XPUFusedRotaryHalf( + reinterpret_cast(dev_ctx.template Alloc(out_k)); + XPUFusedRotaryHalf( dev_ctx, - reinterpret_cast(k->data()), + reinterpret_cast(k->data()), sin_data, cos_data, outk_data, @@ -106,10 +107,10 @@ void FusedRopeKernel(const Context& dev_ctx, if (v) { auto* outv_data = - reinterpret_cast(dev_ctx.template Alloc(out_v)); - XPUFusedRotaryHalf( + reinterpret_cast(dev_ctx.template Alloc(out_v)); + XPUFusedRotaryHalf( dev_ctx, - reinterpret_cast(v->data()), + reinterpret_cast(v->data()), sin_data, cos_data, outv_data, diff --git a/paddle/phi/kernels/fusion/xpu/fused_rope_utils.h b/paddle/phi/kernels/fusion/xpu/fused_rope_utils.h index 6432815b36489..393d6955d19a6 100644 --- a/paddle/phi/kernels/fusion/xpu/fused_rope_utils.h +++ b/paddle/phi/kernels/fusion/xpu/fused_rope_utils.h @@ -17,11 +17,11 @@ namespace phi { namespace fusion { -template +template void XPUGetSinCosData(const Context& dev_ctx, const paddle::optional& sin_cos, const paddle::optional& position_ids, - XPUT* sin_cos_data, + XPUType* sin_cos_data, int64_t batch_size, int64_t seq_len, int64_t head_dim) { @@ -68,22 +68,22 @@ void XPUGetSinCosData(const Context& dev_ctx, phi::errors::InvalidArgument( "The batch_size and seq_len of position_ids must be the same as " "those of q.")); - using XPUTFp16 = typename XPUTypeTrait::Type; - using XPUTBf16 = typename XPUTypeTrait::Type; - if (std::is_same::value) { - int ret = xpu::gather( + using XPUTypeFp16 = typename XPUTypeTrait::Type; + using XPUTypeBf16 = typename XPUTypeTrait::Type; + if (std::is_same::value) { + int ret = xpu::gather( dev_ctx.x_context(), - reinterpret_cast(sin_cos->data()), + reinterpret_cast(sin_cos->data()), position_ids->data(), - reinterpret_cast(sin_cos_data), + reinterpret_cast(sin_cos_data), {seq_len, head_dim}, batch_size * seq_len, 0); PADDLE_ENFORCE_XDNN_SUCCESS(ret, "gather"); } else { - int ret = xpu::gather( + int ret = xpu::gather( dev_ctx.x_context(), - reinterpret_cast(sin_cos->data()), + reinterpret_cast(sin_cos->data()), position_ids->data(), sin_cos_data, {seq_len, head_dim}, @@ -92,37 +92,37 @@ void XPUGetSinCosData(const Context& dev_ctx, PADDLE_ENFORCE_XDNN_SUCCESS(ret, "gather"); } } else { - int ret = - xpu::broadcast(dev_ctx.x_context(), - reinterpret_cast(sin_cos->data()), - sin_cos_data, - {1, seq_len, head_dim}, - {batch_size, seq_len, head_dim}); + int ret = xpu::broadcast( + dev_ctx.x_context(), + reinterpret_cast(sin_cos->data()), + sin_cos_data, + {1, seq_len, head_dim}, + {batch_size, seq_len, head_dim}); PADDLE_ENFORCE_XDNN_SUCCESS(ret, "broadcast"); } } else { int ret = xpu::constant(dev_ctx.x_context(), sin_cos_data, batch_size * seq_len * head_dim, - static_cast(0.0f)); + static_cast(0.0f)); PADDLE_ENFORCE_XDNN_SUCCESS(ret, "constant"); } } -template +template void XPUFusedRotaryHalf(const Context& dev_ctx, - const XPUT* in_data, - const XPUT* sin_data, - const XPUT* cos_data, - XPUT* out_data, + const XPUType* in_data, + const XPUType* sin_data, + const XPUType* cos_data, + XPUType* out_data, int64_t batch_size, int64_t seq_len, int64_t num_heads, int64_t head_dim, bool is_bwd = false) { - auto func = &xpu::rotary_no_freqs_embedding_v2; + auto func = &xpu::rotary_no_freqs_embedding_v2; if (is_bwd) { - func = &xpu::rotary_no_freqs_embedding_v2_grad; + func = &xpu::rotary_no_freqs_embedding_v2_grad; } int ret = diff --git a/paddle/phi/kernels/xpu/bmm_grad_kernel.cc b/paddle/phi/kernels/xpu/bmm_grad_kernel.cc index cbc98dd7ad9ac..751608552482c 100644 --- a/paddle/phi/kernels/xpu/bmm_grad_kernel.cc +++ b/paddle/phi/kernels/xpu/bmm_grad_kernel.cc @@ -25,10 +25,10 @@ void MatMul(const Context& dev_ctx, const DenseTensor& b, bool trans_b, DenseTensor* out) { - using XPUT = typename XPUTypeTrait::Type; + using XPUType = typename XPUTypeTrait::Type; dev_ctx.template Alloc(out); xpu::Context* xpu_ctx = dev_ctx.x_context(); - int fccal_type = FCCalcType(); + int fccal_type = FCCalcType(); if (fccal_type == XPUFCCalcType::FC_INT32) { MatMulXPUFunction(a, b, out, trans_a, trans_b, xpu_ctx); } else if (fccal_type == XPUFCCalcType::FC_FLOAT) { diff --git a/paddle/phi/kernels/xpu/bmm_kernel.cc b/paddle/phi/kernels/xpu/bmm_kernel.cc index ae80f12747ac1..160fabe1ec750 100644 --- a/paddle/phi/kernels/xpu/bmm_kernel.cc +++ b/paddle/phi/kernels/xpu/bmm_kernel.cc @@ -20,7 +20,7 @@ void BmmKernel(const Context& dev_ctx, const DenseTensor& x, const DenseTensor& y, DenseTensor* out) { - using XPUT = typename XPUTypeTrait::Type; + using XPUType = typename XPUTypeTrait::Type; dev_ctx.template Alloc(out); if (x.numel() == 0 || y.numel() == 0) { return; @@ -63,7 +63,7 @@ void BmmKernel(const Context& dev_ctx, y_dims[1])); xpu::Context* xpu_ctx = dev_ctx.x_context(); - int fccal_type = FCCalcType(); + int fccal_type = FCCalcType(); if (fccal_type == XPUFCCalcType::FC_INT32) { MatMulXPUFunction(x, y, out, trans_x, trans_y, xpu_ctx); } else if (fccal_type == XPUFCCalcType::FC_FLOAT) { diff --git a/paddle/phi/kernels/xpu/conv_grad_kernel.cc b/paddle/phi/kernels/xpu/conv_grad_kernel.cc index 03276ebd53b5f..356f77a850b43 100644 --- a/paddle/phi/kernels/xpu/conv_grad_kernel.cc +++ b/paddle/phi/kernels/xpu/conv_grad_kernel.cc @@ -34,7 +34,7 @@ void ConvGradKernel(const Context& dev_ctx, const std::string& data_format, DenseTensor* input_grad, DenseTensor* filter_grad) { - using XPUT = typename XPUTypeTrait::Type; + using XPUType = typename XPUTypeTrait::Type; std::vector paddings = paddings_t; std::vector dilations = dilations_t; // The filter and filter_grad will be reshaped in the calculations, @@ -69,153 +69,157 @@ void ConvGradKernel(const Context& dev_ctx, is_nchw = false; } - const XPUT* input_data = reinterpret_cast(input.data()); - const XPUT* filter_data = reinterpret_cast(filter.data()); - const XPUT* output_grad_data = - reinterpret_cast(out_grad.data()); - XPUT* input_grad_data = nullptr; + const XPUType* input_data = reinterpret_cast(input.data()); + const XPUType* filter_data = + reinterpret_cast(filter.data()); + const XPUType* output_grad_data = + reinterpret_cast(out_grad.data()); + XPUType* input_grad_data = nullptr; if (input_grad) { dev_ctx.template Alloc(input_grad); - input_grad_data = reinterpret_cast(input_grad->data()); + input_grad_data = reinterpret_cast(input_grad->data()); } - XPUT* filter_grad_data = nullptr; + XPUType* filter_grad_data = nullptr; if (filter_grad) { dev_ctx.template Alloc(filter_grad); - filter_grad_data = reinterpret_cast(filter_grad->data()); + filter_grad_data = reinterpret_cast(filter_grad->data()); } xpu::ctx_guard RAII_GUARD(dev_ctx.x_context()); - XPUT* filter_data_tmp; - XPUT* filter_grad_data_tmp; - const XPUT* filter_data_ptr = filter_data; - XPUT* filter_grad_data_ptr = filter_grad_data; + XPUType* filter_data_tmp; + XPUType* filter_grad_data_tmp; + const XPUType* filter_data_ptr = filter_data; + XPUType* filter_grad_data_ptr = filter_grad_data; if (data_format == "NHWC") { - filter_data_tmp = RAII_GUARD.alloc(filter.numel()); + filter_data_tmp = RAII_GUARD.alloc(filter.numel()); PADDLE_ENFORCE_XDNN_NOT_NULL(filter_data_tmp); - int r = xpu::transpose(dev_ctx.x_context(), - filter_data, - filter_data_tmp, - filter_shape, - {0, 2, 3, 1}); + int r = xpu::transpose(dev_ctx.x_context(), + filter_data, + filter_data_tmp, + filter_shape, + {0, 2, 3, 1}); PADDLE_ENFORCE_XDNN_SUCCESS(r, "transpose"); - filter_data_ptr = reinterpret_cast(filter_data_tmp); + filter_data_ptr = reinterpret_cast(filter_data_tmp); if (filter_grad_data != nullptr) { - filter_grad_data_tmp = RAII_GUARD.alloc(filter.numel()); + filter_grad_data_tmp = RAII_GUARD.alloc(filter.numel()); PADDLE_ENFORCE_XDNN_NOT_NULL(filter_grad_data_tmp); filter_grad_data_ptr = filter_grad_data_tmp; } } - int fccal_type = FCCalcType(); + int fccal_type = FCCalcType(); if (fccal_type == XPUFCCalcType::FC_INT32) { - int r = xpu::conv2d_grad(dev_ctx.x_context(), - input_data, - filter_data_ptr, - output_grad_data, - input_grad_data, - filter_grad_data_ptr, - batch_size, - img_c, - img_h, - img_w, - f, - ksize, - strides, - paddings, - dilations, - groups, - nullptr, - nullptr, - nullptr, - nullptr, - nullptr, - is_nchw); + int r = + xpu::conv2d_grad(dev_ctx.x_context(), + input_data, + filter_data_ptr, + output_grad_data, + input_grad_data, + filter_grad_data_ptr, + batch_size, + img_c, + img_h, + img_w, + f, + ksize, + strides, + paddings, + dilations, + groups, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + is_nchw); PADDLE_ENFORCE_XDNN_SUCCESS(r, "conv2d_grad"); } else if (fccal_type == XPUFCCalcType::FC_FLOAT) { - int r = xpu::conv2d_grad(dev_ctx.x_context(), - input_data, - filter_data_ptr, - output_grad_data, - input_grad_data, - filter_grad_data_ptr, - batch_size, - img_c, - img_h, - img_w, - f, - ksize, - strides, - paddings, - dilations, - groups, - nullptr, - nullptr, - nullptr, - nullptr, - nullptr, - is_nchw); + int r = + xpu::conv2d_grad(dev_ctx.x_context(), + input_data, + filter_data_ptr, + output_grad_data, + input_grad_data, + filter_grad_data_ptr, + batch_size, + img_c, + img_h, + img_w, + f, + ksize, + strides, + paddings, + dilations, + groups, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + is_nchw); PADDLE_ENFORCE_XDNN_SUCCESS(r, "conv2d_grad"); } else if (fccal_type == XPUFCCalcType::FC_INT32_WITH_LL) { - int r = - xpu::conv2d_grad(dev_ctx.x_context(), - input_data, - filter_data_ptr, - output_grad_data, - input_grad_data, - filter_grad_data_ptr, - batch_size, - img_c, - img_h, - img_w, - f, - ksize, - strides, - paddings, - dilations, - groups, - nullptr, - nullptr, - nullptr, - nullptr, - nullptr, - is_nchw); + int r = xpu::conv2d_grad( + dev_ctx.x_context(), + input_data, + filter_data_ptr, + output_grad_data, + input_grad_data, + filter_grad_data_ptr, + batch_size, + img_c, + img_h, + img_w, + f, + ksize, + strides, + paddings, + dilations, + groups, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + is_nchw); PADDLE_ENFORCE_XDNN_SUCCESS(r, "conv2d_grad"); } else { - int r = xpu::conv2d_grad(dev_ctx.x_context(), - input_data, - filter_data_ptr, - output_grad_data, - input_grad_data, - filter_grad_data_ptr, - batch_size, - img_c, - img_h, - img_w, - f, - ksize, - strides, - paddings, - dilations, - groups, - nullptr, - nullptr, - nullptr, - nullptr, - nullptr, - is_nchw); + int r = xpu::conv2d_grad( + dev_ctx.x_context(), + input_data, + filter_data_ptr, + output_grad_data, + input_grad_data, + filter_grad_data_ptr, + batch_size, + img_c, + img_h, + img_w, + f, + ksize, + strides, + paddings, + dilations, + groups, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + is_nchw); PADDLE_ENFORCE_XDNN_SUCCESS(r, "conv2d_grad"); } if ((filter_grad_data_ptr != nullptr) && (data_format == "NHWC")) { std::vector filter_shape_fhwc = { filter_shape[0], filter_shape[2], filter_shape[3], filter_shape[1]}; - int r = xpu::transpose(dev_ctx.x_context(), - filter_grad_data_ptr, - filter_grad_data, - filter_shape_fhwc, - {0, 3, 1, 2}); + int r = xpu::transpose(dev_ctx.x_context(), + filter_grad_data_ptr, + filter_grad_data, + filter_shape_fhwc, + {0, 3, 1, 2}); PADDLE_ENFORCE_XDNN_SUCCESS(r, "transpose"); } } @@ -260,7 +264,7 @@ void Conv3DGradKernel(const Context& dev_ctx, const std::string& data_format, DenseTensor* input_grad, DenseTensor* filter_grad) { - using XPUT = typename XPUTypeTrait::Type; + using XPUType = typename XPUTypeTrait::Type; std::vector paddings = paddings_t; std::vector dilations = dilations_t; // The filter and filter_grad will be reshaped in the calculations, @@ -292,144 +296,148 @@ void Conv3DGradKernel(const Context& dev_ctx, is_ncdhw = false; } - const XPUT* input_data = reinterpret_cast(input.data()); - const XPUT* filter_data = reinterpret_cast(filter.data()); - const XPUT* output_grad_data = - reinterpret_cast(out_grad.data()); - XPUT* input_grad_data = nullptr; + const XPUType* input_data = reinterpret_cast(input.data()); + const XPUType* filter_data = + reinterpret_cast(filter.data()); + const XPUType* output_grad_data = + reinterpret_cast(out_grad.data()); + XPUType* input_grad_data = nullptr; if (input_grad) { dev_ctx.template Alloc(input_grad); - input_grad_data = reinterpret_cast(input_grad->data()); + input_grad_data = reinterpret_cast(input_grad->data()); } - XPUT* filter_grad_data = nullptr; + XPUType* filter_grad_data = nullptr; if (filter_grad) { dev_ctx.template Alloc(filter_grad); - filter_grad_data = reinterpret_cast(filter_grad->data()); + filter_grad_data = reinterpret_cast(filter_grad->data()); } xpu::ctx_guard RAII_GUARD(dev_ctx.x_context()); - XPUT* filter_data_tmp; - XPUT* filter_grad_data_tmp; - const XPUT* filter_data_ptr = filter_data; - XPUT* filter_grad_data_ptr = filter_grad_data; + XPUType* filter_data_tmp; + XPUType* filter_grad_data_tmp; + const XPUType* filter_data_ptr = filter_data; + XPUType* filter_grad_data_ptr = filter_grad_data; if (data_format == "NDHWC") { - filter_data_tmp = RAII_GUARD.alloc(filter.numel()); + filter_data_tmp = RAII_GUARD.alloc(filter.numel()); PADDLE_ENFORCE_XDNN_NOT_NULL(filter_data_tmp); - int r = xpu::transpose(dev_ctx.x_context(), - filter_data, - filter_data_tmp, - filter_shape, - {0, 2, 3, 4, 1}); + int r = xpu::transpose(dev_ctx.x_context(), + filter_data, + filter_data_tmp, + filter_shape, + {0, 2, 3, 4, 1}); PADDLE_ENFORCE_XDNN_SUCCESS(r, "transpose"); - filter_data_ptr = reinterpret_cast(filter_data_tmp); + filter_data_ptr = reinterpret_cast(filter_data_tmp); if (filter_grad_data != nullptr) { - filter_grad_data_tmp = RAII_GUARD.alloc(filter.numel()); + filter_grad_data_tmp = RAII_GUARD.alloc(filter.numel()); PADDLE_ENFORCE_XDNN_NOT_NULL(filter_grad_data_tmp); filter_grad_data_ptr = filter_grad_data_tmp; } } - int fccal_type = FCCalcType(); + int fccal_type = FCCalcType(); if (fccal_type == XPUFCCalcType::FC_INT32) { - int r = xpu::conv3d_grad(dev_ctx.x_context(), - input_data, - filter_data_ptr, - output_grad_data, - input_grad_data, - filter_grad_data_ptr, - batch_size, - img_c, - img_d, - img_h, - img_w, - f, - ksize, - strides, - paddings, - dilations, - groups, - nullptr, - nullptr, - nullptr, - nullptr, - nullptr, - is_ncdhw); + int r = + xpu::conv3d_grad(dev_ctx.x_context(), + input_data, + filter_data_ptr, + output_grad_data, + input_grad_data, + filter_grad_data_ptr, + batch_size, + img_c, + img_d, + img_h, + img_w, + f, + ksize, + strides, + paddings, + dilations, + groups, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + is_ncdhw); PADDLE_ENFORCE_XDNN_SUCCESS(r, "conv3d_grad"); } else if (fccal_type == XPUFCCalcType::FC_FLOAT) { - int r = xpu::conv3d_grad(dev_ctx.x_context(), - input_data, - filter_data_ptr, - output_grad_data, - input_grad_data, - filter_grad_data_ptr, - batch_size, - img_c, - img_d, - img_h, - img_w, - f, - ksize, - strides, - paddings, - dilations, - groups, - nullptr, - nullptr, - nullptr, - nullptr, - nullptr, - is_ncdhw); + int r = + xpu::conv3d_grad(dev_ctx.x_context(), + input_data, + filter_data_ptr, + output_grad_data, + input_grad_data, + filter_grad_data_ptr, + batch_size, + img_c, + img_d, + img_h, + img_w, + f, + ksize, + strides, + paddings, + dilations, + groups, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + is_ncdhw); PADDLE_ENFORCE_XDNN_SUCCESS(r, "conv3d_grad"); } else if (fccal_type == XPUFCCalcType::FC_INT32_WITH_LL) { - int r = - xpu::conv3d_grad(dev_ctx.x_context(), - input_data, - filter_data_ptr, - output_grad_data, - input_grad_data, - filter_grad_data_ptr, - batch_size, - img_c, - img_d, - img_h, - img_w, - f, - ksize, - strides, - paddings, - dilations, - groups, - nullptr, - nullptr, - nullptr, - nullptr, - nullptr, - is_ncdhw); + int r = xpu::conv3d_grad( + dev_ctx.x_context(), + input_data, + filter_data_ptr, + output_grad_data, + input_grad_data, + filter_grad_data_ptr, + batch_size, + img_c, + img_d, + img_h, + img_w, + f, + ksize, + strides, + paddings, + dilations, + groups, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + is_ncdhw); PADDLE_ENFORCE_XDNN_SUCCESS(r, "conv3d_grad"); } else { - int r = xpu::conv3d_grad(dev_ctx.x_context(), - input_data, - filter_data_ptr, - output_grad_data, - input_grad_data, - filter_grad_data_ptr, - batch_size, - img_c, - img_d, - img_h, - img_w, - f, - ksize, - strides, - paddings, - dilations, - groups, - nullptr, - nullptr, - nullptr, - nullptr, - nullptr, - is_ncdhw); + int r = xpu::conv3d_grad( + dev_ctx.x_context(), + input_data, + filter_data_ptr, + output_grad_data, + input_grad_data, + filter_grad_data_ptr, + batch_size, + img_c, + img_d, + img_h, + img_w, + f, + ksize, + strides, + paddings, + dilations, + groups, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + is_ncdhw); PADDLE_ENFORCE_XDNN_SUCCESS(r, "conv3d_grad"); } @@ -439,11 +447,11 @@ void Conv3DGradKernel(const Context& dev_ctx, filter_shape[3], filter_shape[4], filter_shape[1]}; - int r = xpu::transpose(dev_ctx.x_context(), - filter_grad_data_ptr, - filter_grad_data, - filter_shape_fhwc, - {0, 4, 1, 2, 3}); + int r = xpu::transpose(dev_ctx.x_context(), + filter_grad_data_ptr, + filter_grad_data, + filter_shape_fhwc, + {0, 4, 1, 2, 3}); PADDLE_ENFORCE_XDNN_SUCCESS(r, "transpose"); } } diff --git a/paddle/phi/kernels/xpu/conv_kernel.cc b/paddle/phi/kernels/xpu/conv_kernel.cc index 0dc93d676186b..02e4bbcae1180 100644 --- a/paddle/phi/kernels/xpu/conv_kernel.cc +++ b/paddle/phi/kernels/xpu/conv_kernel.cc @@ -32,7 +32,7 @@ void ConvKernel(const Context& dev_ctx, int groups, const std::string& data_format, DenseTensor* out) { - using XPUT = typename XPUTypeTrait::Type; + using XPUType = typename XPUTypeTrait::Type; std::vector paddings = paddings_t; std::vector dilations = dilations_t; // The filter will be reshaped in the calculations, @@ -67,107 +67,109 @@ void ConvKernel(const Context& dev_ctx, is_nchw = false; } - const XPUT* input_data = reinterpret_cast(input.data()); - const XPUT* filter_data = reinterpret_cast(filter.data()); - XPUT* output_data = reinterpret_cast(out->data()); + const XPUType* input_data = reinterpret_cast(input.data()); + const XPUType* filter_data = + reinterpret_cast(filter.data()); + XPUType* output_data = reinterpret_cast(out->data()); xpu::ctx_guard RAII_GUARD(dev_ctx.x_context()); - XPUT* filter_data_tmp; - const XPUT* filter_data_ptr = filter_data; + XPUType* filter_data_tmp; + const XPUType* filter_data_ptr = filter_data; if (data_format == "NHWC") { - filter_data_tmp = RAII_GUARD.alloc(filter.numel()); + filter_data_tmp = RAII_GUARD.alloc(filter.numel()); PADDLE_ENFORCE_XDNN_NOT_NULL(filter_data_tmp); std::vector filter_shape = common::vectorize(filter.dims()); - int r = xpu::transpose(dev_ctx.x_context(), - filter_data, - filter_data_tmp, - filter_shape, - {0, 2, 3, 1}); + int r = xpu::transpose(dev_ctx.x_context(), + filter_data, + filter_data_tmp, + filter_shape, + {0, 2, 3, 1}); PADDLE_ENFORCE_XDNN_SUCCESS(r, "transpose"); - filter_data_ptr = reinterpret_cast(filter_data_tmp); + filter_data_ptr = reinterpret_cast(filter_data_tmp); } - int fccal_type = FCCalcType(); + int fccal_type = FCCalcType(); if (fccal_type == XPUFCCalcType::FC_INT32) { - int r = xpu::conv2d(dev_ctx.x_context(), - input_data, - filter_data_ptr, - output_data, - batch_size, - img_c, - img_h, - img_w, - f, - ksize, - strides, - paddings, - dilations, - groups, - nullptr, - nullptr, - nullptr, - is_nchw); + int r = xpu::conv2d(dev_ctx.x_context(), + input_data, + filter_data_ptr, + output_data, + batch_size, + img_c, + img_h, + img_w, + f, + ksize, + strides, + paddings, + dilations, + groups, + nullptr, + nullptr, + nullptr, + is_nchw); PADDLE_ENFORCE_XDNN_SUCCESS(r, "conv2d"); } else if (fccal_type == XPUFCCalcType::FC_FLOAT) { - int r = xpu::conv2d(dev_ctx.x_context(), - input_data, - filter_data_ptr, - output_data, - batch_size, - img_c, - img_h, - img_w, - f, - ksize, - strides, - paddings, - dilations, - groups, - nullptr, - nullptr, - nullptr, - is_nchw); + int r = xpu::conv2d(dev_ctx.x_context(), + input_data, + filter_data_ptr, + output_data, + batch_size, + img_c, + img_h, + img_w, + f, + ksize, + strides, + paddings, + dilations, + groups, + nullptr, + nullptr, + nullptr, + is_nchw); PADDLE_ENFORCE_XDNN_SUCCESS(r, "conv2d"); } else if (fccal_type == XPUFCCalcType::FC_INT32_WITH_LL) { - int r = xpu::conv2d(dev_ctx.x_context(), - input_data, - filter_data_ptr, - output_data, - batch_size, - img_c, - img_h, - img_w, - f, - ksize, - strides, - paddings, - dilations, - groups, - nullptr, - nullptr, - nullptr, - is_nchw); + int r = xpu::conv2d( + dev_ctx.x_context(), + input_data, + filter_data_ptr, + output_data, + batch_size, + img_c, + img_h, + img_w, + f, + ksize, + strides, + paddings, + dilations, + groups, + nullptr, + nullptr, + nullptr, + is_nchw); PADDLE_ENFORCE_XDNN_SUCCESS(r, "conv2d"); } else { - int r = xpu::conv2d(dev_ctx.x_context(), - input_data, - filter_data_ptr, - output_data, - batch_size, - img_c, - img_h, - img_w, - f, - ksize, - strides, - paddings, - dilations, - groups, - nullptr, - nullptr, - nullptr, - is_nchw); + int r = xpu::conv2d(dev_ctx.x_context(), + input_data, + filter_data_ptr, + output_data, + batch_size, + img_c, + img_h, + img_w, + f, + ksize, + strides, + paddings, + dilations, + groups, + nullptr, + nullptr, + nullptr, + is_nchw); PADDLE_ENFORCE_XDNN_SUCCESS(r, "conv2d"); } } @@ -206,7 +208,7 @@ void Conv3DKernel(const Context& dev_ctx, const std::vector& dilations_t, const std::string& data_format, DenseTensor* out) { - using XPUT = typename XPUTypeTrait::Type; + using XPUType = typename XPUTypeTrait::Type; std::vector paddings = paddings_t; std::vector dilations = dilations_t; // The filter will be reshaped in the calculations, @@ -237,112 +239,114 @@ void Conv3DKernel(const Context& dev_ctx, is_ncdhw = false; } - XPUT* output_data = reinterpret_cast(out->data()); - const XPUT* filter_data = reinterpret_cast(filter.data()); - const XPUT* input_data = reinterpret_cast(input.data()); + XPUType* output_data = reinterpret_cast(out->data()); + const XPUType* filter_data = + reinterpret_cast(filter.data()); + const XPUType* input_data = reinterpret_cast(input.data()); xpu::ctx_guard RAII_GUARD(dev_ctx.x_context()); - XPUT* filter_data_tmp; - const XPUT* filter_data_ptr = filter_data; + XPUType* filter_data_tmp; + const XPUType* filter_data_ptr = filter_data; if (data_format == "NDHWC") { - filter_data_tmp = RAII_GUARD.alloc(filter.numel()); + filter_data_tmp = RAII_GUARD.alloc(filter.numel()); PADDLE_ENFORCE_XDNN_NOT_NULL(filter_data_tmp); std::vector filter_shape = common::vectorize(filter.dims()); - int r = xpu::transpose(dev_ctx.x_context(), - filter_data, - filter_data_tmp, - filter_shape, - {0, 2, 3, 4, 1}); + int r = xpu::transpose(dev_ctx.x_context(), + filter_data, + filter_data_tmp, + filter_shape, + {0, 2, 3, 4, 1}); PADDLE_ENFORCE_XDNN_SUCCESS(r, "transpose"); - filter_data_ptr = reinterpret_cast(filter_data_tmp); + filter_data_ptr = reinterpret_cast(filter_data_tmp); } - int fccal_type = FCCalcType(); + int fccal_type = FCCalcType(); if (fccal_type == XPUFCCalcType::FC_INT32) { - int r = xpu::conv3d(dev_ctx.x_context(), - input_data, - filter_data_ptr, - output_data, - batch_size, - img_c, - img_d, - img_h, - img_w, - f, - ksize, - strides, - paddings, - dilations, - groups, - nullptr, - nullptr, - nullptr, - is_ncdhw); + int r = xpu::conv3d(dev_ctx.x_context(), + input_data, + filter_data_ptr, + output_data, + batch_size, + img_c, + img_d, + img_h, + img_w, + f, + ksize, + strides, + paddings, + dilations, + groups, + nullptr, + nullptr, + nullptr, + is_ncdhw); PADDLE_ENFORCE_XDNN_SUCCESS(r, "conv3d"); } else if (fccal_type == XPUFCCalcType::FC_FLOAT) { - int r = xpu::conv3d(dev_ctx.x_context(), - input_data, - filter_data_ptr, - output_data, - batch_size, - img_c, - img_d, - img_h, - img_w, - f, - ksize, - strides, - paddings, - dilations, - groups, - nullptr, - nullptr, - nullptr, - is_ncdhw); + int r = xpu::conv3d(dev_ctx.x_context(), + input_data, + filter_data_ptr, + output_data, + batch_size, + img_c, + img_d, + img_h, + img_w, + f, + ksize, + strides, + paddings, + dilations, + groups, + nullptr, + nullptr, + nullptr, + is_ncdhw); PADDLE_ENFORCE_XDNN_SUCCESS(r, "conv3d"); } else if (fccal_type == XPUFCCalcType::FC_INT32_WITH_LL) { - int r = xpu::conv3d(dev_ctx.x_context(), - input_data, - filter_data_ptr, - output_data, - batch_size, - img_c, - img_d, - img_h, - img_w, - f, - ksize, - strides, - paddings, - dilations, - groups, - nullptr, - nullptr, - nullptr, - is_ncdhw); + int r = xpu::conv3d( + dev_ctx.x_context(), + input_data, + filter_data_ptr, + output_data, + batch_size, + img_c, + img_d, + img_h, + img_w, + f, + ksize, + strides, + paddings, + dilations, + groups, + nullptr, + nullptr, + nullptr, + is_ncdhw); PADDLE_ENFORCE_XDNN_SUCCESS(r, "conv3d"); } else { - int r = xpu::conv3d(dev_ctx.x_context(), - input_data, - filter_data_ptr, - output_data, - batch_size, - img_c, - img_d, - img_h, - img_w, - f, - ksize, - strides, - paddings, - dilations, - groups, - nullptr, - nullptr, - nullptr, - is_ncdhw); + int r = xpu::conv3d(dev_ctx.x_context(), + input_data, + filter_data_ptr, + output_data, + batch_size, + img_c, + img_d, + img_h, + img_w, + f, + ksize, + strides, + paddings, + dilations, + groups, + nullptr, + nullptr, + nullptr, + is_ncdhw); PADDLE_ENFORCE_XDNN_SUCCESS(r, "conv3d"); } } diff --git a/paddle/phi/kernels/xpu/conv_transpose_kernel.cc b/paddle/phi/kernels/xpu/conv_transpose_kernel.cc index 2a1195e48c1f0..8dafe67056b50 100644 --- a/paddle/phi/kernels/xpu/conv_transpose_kernel.cc +++ b/paddle/phi/kernels/xpu/conv_transpose_kernel.cc @@ -51,7 +51,7 @@ void Conv2dTransposeKernel(const Context& ctx, const std::vector& dilations, const std::string& data_format, DenseTensor* out) { - using XPUT = typename XPUTypeTrait::Type; + using XPUType = typename XPUTypeTrait::Type; ctx.template Alloc(out); @@ -76,7 +76,7 @@ void Conv2dTransposeKernel(const Context& ctx, const int img_xh = static_cast(out->dims()[2]); const int img_xw = static_cast(out->dims()[3]); - int fccal_type = FCCalcType(); + int fccal_type = FCCalcType(); if (fccal_type == XPUFCCalcType::FC_INT32) { int r = xpu::conv2d_transpose_v2( ctx.x_context(), @@ -171,11 +171,11 @@ void Conv2dTransposeKernel(const Context& ctx, PADDLE_ENFORCE_XDNN_SUCCESS(r, "conv2d_transpose"); } } else { - int r = xpu::conv2d_transpose_v2( + int r = xpu::conv2d_transpose_v2( ctx.x_context(), - reinterpret_cast(x.data()), - reinterpret_cast(filter.data()), - reinterpret_cast(out->data()), + reinterpret_cast(x.data()), + reinterpret_cast(filter.data()), + reinterpret_cast(out->data()), batch_size, img_yc, img_xh, diff --git a/paddle/phi/kernels/xpu/embedding_grad_kernel.cc b/paddle/phi/kernels/xpu/embedding_grad_kernel.cc index 11fd3826f4f6f..ae1bd8d5c507d 100644 --- a/paddle/phi/kernels/xpu/embedding_grad_kernel.cc +++ b/paddle/phi/kernels/xpu/embedding_grad_kernel.cc @@ -28,7 +28,7 @@ void EmbeddingGradKernel(const Context& ctx, const DenseTensor& out_grad, int64_t padding_idx, DenseTensor* weight_grad) { - using XPUT = typename XPUTypeTrait::Type; + using XPUType = typename XPUTypeTrait::Type; DDim table_dim; table_dim = weight.dims(); @@ -63,11 +63,11 @@ void EmbeddingGradKernel(const Context& ctx, int ym = static_cast(ids_numel); int n = d_table_t->dims()[1]; - int r = xpu::embedding_grad( + int r = xpu::embedding_grad( dev_ctx.x_context(), - reinterpret_cast(d_output_data), + reinterpret_cast(d_output_data), ids_data, - reinterpret_cast(d_table_data), + reinterpret_cast(d_table_data), xm, n, ym, diff --git a/paddle/phi/kernels/xpu/index_put_kernel.cc b/paddle/phi/kernels/xpu/index_put_kernel.cc index 60c91a8e5c83c..0a86bc6cef536 100644 --- a/paddle/phi/kernels/xpu/index_put_kernel.cc +++ b/paddle/phi/kernels/xpu/index_put_kernel.cc @@ -104,7 +104,7 @@ void IndexPutKernel(const Context& dev_ctx, return; } - using XPUT = typename XPUTypeTrait::Type; + using XPUType = typename XPUTypeTrait::Type; auto out_data = dev_ctx.template Alloc(out); auto bd_dims = funcs::BroadCastTensorsDims(int_indices_v); DenseTensor res_indices(DataType::INT64); @@ -133,15 +133,15 @@ void IndexPutKernel(const Context& dev_ctx, value_data = value_bd.data(); } - int r = - xpu::index_put(dev_ctx.x_context(), - reinterpret_cast(x.data()), - reinterpret_cast(value_data), - res_indices.data(), - reinterpret_cast(out_data), - x_shape, - index_shape, - accumulate); + int r = xpu::index_put( + dev_ctx.x_context(), + reinterpret_cast(x.data()), + reinterpret_cast(value_data), + res_indices.data(), + reinterpret_cast(out_data), + x_shape, + index_shape, + accumulate); PADDLE_ENFORCE_XDNN_SUCCESS(r, "index_put"); if (dev_ctx.x_context()->xpu_stream) { dev_ctx.Wait(); diff --git a/paddle/phi/kernels/xpu/inverse_kernel.cc b/paddle/phi/kernels/xpu/inverse_kernel.cc index 966fcc97e0ab0..82d54653eb03c 100644 --- a/paddle/phi/kernels/xpu/inverse_kernel.cc +++ b/paddle/phi/kernels/xpu/inverse_kernel.cc @@ -24,7 +24,7 @@ template void InverseKernel(const Context& dev_ctx, const DenseTensor& x, DenseTensor* out) { - using XPUT = typename XPUTypeTrait::Type; + using XPUType = typename XPUTypeTrait::Type; auto out_data = dev_ctx.template Alloc(out); int64_t x_dims_len = x.dims().size(); @@ -46,12 +46,12 @@ void InverseKernel(const Context& dev_ctx, auto RAII_GUARD = xpu::ctx_guard(dev_ctx.x_context()); auto* info_xpu = RAII_GUARD.alloc_l3_or_gm(batch); // Xpu inverse api has check for singularity itself. - int r = xpu::inverse(dev_ctx.x_context(), - reinterpret_cast(x.data()), - reinterpret_cast(out_data), - info_xpu, - batch, - n); + int r = xpu::inverse(dev_ctx.x_context(), + reinterpret_cast(x.data()), + reinterpret_cast(out_data), + info_xpu, + batch, + n); PADDLE_ENFORCE_XDNN_SUCCESS(r, "inverse"); } diff --git a/paddle/phi/kernels/xpu/multiclass_nms3_kernel.cc b/paddle/phi/kernels/xpu/multiclass_nms3_kernel.cc index 2f343ccc6b494..6e1c20a366d23 100644 --- a/paddle/phi/kernels/xpu/multiclass_nms3_kernel.cc +++ b/paddle/phi/kernels/xpu/multiclass_nms3_kernel.cc @@ -38,10 +38,12 @@ void MultiClassNMSKernel(const Context& ctx, DenseTensor* out, DenseTensor* index, DenseTensor* nms_rois_num) { - using XPUT = typename XPUTypeTrait::Type; + using XPUType = typename XPUTypeTrait::Type; - const XPUT* bboxes_data = reinterpret_cast(bboxes.data()); - const XPUT* scores_data = reinterpret_cast(scores.data()); + const XPUType* bboxes_data = + reinterpret_cast(bboxes.data()); + const XPUType* scores_data = + reinterpret_cast(scores.data()); bool return_index = index != nullptr; bool has_rois_num = rois_num.get_ptr() != nullptr; diff --git a/paddle/phi/kernels/xpu/scatter_nd_add_grad_kernel.cc b/paddle/phi/kernels/xpu/scatter_nd_add_grad_kernel.cc index 37e6e91ea779e..bc08afbb7f6da 100644 --- a/paddle/phi/kernels/xpu/scatter_nd_add_grad_kernel.cc +++ b/paddle/phi/kernels/xpu/scatter_nd_add_grad_kernel.cc @@ -25,15 +25,15 @@ void ScatterNdAddGradKernel(const Context &ctx, const DenseTensor &out_grad, DenseTensor *x_grad, DenseTensor *updates_grad) { - using XPUT = typename XPUTypeTrait::Type; + using XPUType = typename XPUTypeTrait::Type; int ret = xpu::SUCCESS; const T *out_grad_data = out_grad.data(); if (x_grad) { auto *x_grad_data = ctx.template Alloc(x_grad); - ret = xpu::copy(ctx.x_context(), - reinterpret_cast(out_grad_data), - reinterpret_cast(x_grad_data), - out_grad.numel()); + ret = xpu::copy(ctx.x_context(), + reinterpret_cast(out_grad_data), + reinterpret_cast(x_grad_data), + out_grad.numel()); PADDLE_ENFORCE_XDNN_SUCCESS(ret, "copy"); } @@ -64,11 +64,12 @@ void ScatterNdAddGradKernel(const Context &ctx, out_grad_numel, remain_numel, updates_grad_numel)); - ret = xpu::broadcast(ctx.x_context(), - reinterpret_cast(out_grad_data), - reinterpret_cast(updates_grad_data), - {1, out_grad_numel}, - {remain_numel, out_grad_numel}); + ret = xpu::broadcast( + ctx.x_context(), + reinterpret_cast(out_grad_data), + reinterpret_cast(updates_grad_data), + {1, out_grad_numel}, + {remain_numel, out_grad_numel}); PADDLE_ENFORCE_XDNN_SUCCESS(ret, "broadcast"); return; } @@ -84,19 +85,19 @@ void ScatterNdAddGradKernel(const Context &ctx, nullptr}; if (index.dtype() == DataType::INT32) { - ret = xpu::gather_nd( + ret = xpu::gather_nd( ctx.x_context(), - reinterpret_cast(out_grad_data), + reinterpret_cast(out_grad_data), index.data(), - reinterpret_cast(updates_grad_data), + reinterpret_cast(updates_grad_data), out_grad_shape_param, index_shape_vec); } else { - ret = xpu::gather_nd( + ret = xpu::gather_nd( ctx.x_context(), - reinterpret_cast(out_grad_data), + reinterpret_cast(out_grad_data), index.data(), - reinterpret_cast(updates_grad_data), + reinterpret_cast(updates_grad_data), out_grad_shape_param, index_shape_vec); } From 170ba3f72e9aefcfd981c7310ef03e25157685d8 Mon Sep 17 00:00:00 2001 From: Bo Zhang <105368690+zhangbopd@users.noreply.github.com> Date: Mon, 4 Mar 2024 17:06:05 +0800 Subject: [PATCH 273/282] [PIR][DynamicShape] Fix reshape Op and add cumOp's InferSymShape (#62321) * fix reshape Op and add cumOp's InferSymShape --- .../infer_symbolic_shape/cinn_op_infer_sym.cc | 9 +- .../infer_symbolic_shape/infer_sym_utils.h | 10 +- .../paddle_op_infer_sym.cc | 154 +-------------- .../paddle_op_infer_sym.h | 25 --- .../infer_symbolic_shape/unary_infer_sym.cc | 179 +++++++++++++++++- .../infer_symbolic_shape/unary_infer_sym.h | 20 ++ paddle/phi/api/yaml/ops.yaml | 1 - test/ir/pir/cinn/symbolic/CMakeLists.txt | 13 ++ .../symbolic/test_unary_op_infer_sym_shape.py | 157 +++++++++++++++ 9 files changed, 384 insertions(+), 184 deletions(-) diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.cc index f81624427207e..932012bf0622f 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/cinn_op_infer_sym.cc @@ -93,16 +93,11 @@ bool ConcatOpInferSymbolicShape( bool ReduceInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - const auto &attr_map = op->attributes(); - PADDLE_ENFORCE( - attr_map.count("keep_dim"), - phi::errors::PreconditionNotMet( - "attr [keep_dim] MUST in attribute map for [%s] op", op->name())); - bool keepdim = attr_map.at("keep_dim").dyn_cast().data(); + bool keep_dim = GetBoolAttr(op, "keep_dim"); auto axis = paddle::dialect::details::GetVectorAttr(op, "dim"); bool reduce_all = axis.size() == 0 ? true : false; return paddle::dialect::details::ReduceInferDim( - op, shape_analysis, axis, keepdim, reduce_all); + op, shape_analysis, axis, keep_dim, reduce_all); } bool ReduceMaxOpInferSymbolicShape( diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.h index f5193b3f7ff5b..4be08cde7a619 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.h @@ -17,8 +17,14 @@ #include "paddle/fluid/pir/dialect/operator/ir/pd_op.h" #include "paddle/pir/include/dialect/shape/utils/shape_analysis.h" -#define GET_BOOL_ATTR(op, str) \ - op->attributes().at(str).dyn_cast().data(); +inline bool GetBoolAttr(const pir::Operation *op, const std::string &str) { + const auto &attr_map = op->attributes(); + PADDLE_ENFORCE( + attr_map.count(str), + phi::errors::PreconditionNotMet( + "attr [%s] MUST in attribute map for [%s] op", str, op->name())); + return attr_map.at(str).dyn_cast().data(); +} // To make codes shorter using ExprVec = std::vector; diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc index 20cdc880f8759..4c7a3ab544fb8 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.cc @@ -115,9 +115,7 @@ bool StackOpInferSymbolicShape(pir::Operation *op, bool SumOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - const auto &attributes = op->attributes(); - bool keepdim = attributes.at("keepdim").dyn_cast().data(); - + bool keepdim = GetBoolAttr(op, "keepdim"); bool reduce_all = false; auto axis_gen_op = op->operand_source(1).defining_op(); @@ -142,12 +140,8 @@ bool SumOpInferSymbolicShape(pir::Operation *op, bool ProdOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - const auto &attributes = op->attributes(); - bool keepdim = - attributes.at("keep_dim").dyn_cast().data(); - - bool reduce_all = - attributes.at("reduce_all").dyn_cast().data(); + bool keepdim = GetBoolAttr(op, "keep_dim"); + bool reduce_all = GetBoolAttr(op, "reduce_all"); auto axis_gen_op = op->operand_source(1).defining_op(); if (axis_gen_op->isa()) { @@ -166,80 +160,6 @@ bool ProdOpInferSymbolicShape(pir::Operation *op, return true; } -bool ReshapeOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - pir::Value operand_source = op->operand_source(0); - if (shape_analysis->GetShapeOrDataForValue(operand_source) - .data() - .has_value()) { - const symbol::ShapeOrDataDimExprs &operand_shape_or_data = - shape_analysis->GetShapeOrDataForValue(operand_source); - shape_analysis->SetShapeOrDataForValue(op->result(0), - operand_shape_or_data); - return true; - } - - pir::Value operand_source_shape = op->operand_source(1); - - const symbol::ShapeOrDataDimExprs &operand_shape_or_data = - shape_analysis->GetShapeOrDataForValue(operand_source_shape); - - const auto &GetProduct = [&](const auto &dim_exprs, const auto &Filter) { - symbol::DimExpr product{1}; - for (const auto &dim_expr : dim_exprs) { - if (Filter(dim_expr)) { - product = product * dim_expr; - } - } - return product; - }; - - const auto &IsNotMinusOne = [&](const symbol::DimExpr &dim_expr) { - if (dim_expr.isa()) { - return dim_expr.dyn_cast() != static_cast(-1); - } - return true; - }; - - const std::vector out_dims = [&] { - const auto &original_shape = - shape_analysis->GetShapeOrDataForValue(op->operand_source(0)).shape(); - - const auto &numel = - GetProduct(original_shape, [](const auto &) { return true; }); - - const auto &product_exclude_minus_one = - GetProduct(operand_shape_or_data.data().value(), IsNotMinusOne); - - const auto &input_dims = operand_shape_or_data.data().value(); - - std::vector out_dims; - out_dims.reserve(input_dims.size()); - for (const auto &dim_expr : input_dims) { - const auto &out_dim_expr = IsNotMinusOne(dim_expr) - ? dim_expr - : (numel / product_exclude_minus_one); - out_dims.emplace_back(out_dim_expr); - } - - return out_dims; - }(); - - symbol::ShapeOrDataDimExprs shape_data{ - symbol::TensorShapeOrDataDimExprs(out_dims)}; - - shape_analysis->SetShapeOrDataForValue(op->result(0), shape_data); - shape_analysis->SetShapeOrDataForValue( - op->result(1), - shape_analysis->GetShapeOrDataForValue(operand_source_shape)); - return true; -} - -bool Reshape_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - return ReshapeOpInferSymbolicShape(op, shape_analysis); -} - bool FullIntArrayOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { const auto &attributes = op->attributes(); @@ -1046,14 +966,12 @@ bool MatmulOpInferSymbolicShape( } } + bool transpose_x_attr = GetBoolAttr(op, "transpose_x"); + bool transpose_y_attr = GetBoolAttr(op, "transpose_y"); symbol::DimExpr out_M = - op->attributes().at("transpose_x").dyn_cast().data() - ? x_dims[ndims_x - 1] - : x_dims[ndims_x - 2]; + transpose_x_attr ? x_dims[ndims_x - 1] : x_dims[ndims_x - 2]; symbol::DimExpr out_N = - op->attributes().at("transpose_y").dyn_cast().data() - ? y_dims[ndims_y - 2] - : y_dims[ndims_y - 1]; + transpose_y_attr ? y_dims[ndims_y - 2] : y_dims[ndims_y - 1]; if (!x_broadcasted) { out_dims.emplace_back(out_M); } @@ -1069,8 +987,7 @@ bool MatmulOpInferSymbolicShape( bool MaxOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - bool keepdim = - op->attributes().at("keepdim").dyn_cast().data(); + bool keepdim = GetBoolAttr(op, "keepdim"); const std::vector axis = [&] { pir::Operation *axis_gen_op = op->operand_source(1).defining_op(); @@ -1167,61 +1084,6 @@ bool SplitOpInferSymbolicShape(pir::Operation *op, // Not Impelmented Ops. -bool AsComplexOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool AsRealOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool AsStridedOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} - -bool CummaxOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool CumminOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool CumprodOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Cumprod_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool CumsumOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} -bool Cumsum_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - PADDLE_THROW(phi::errors::Unimplemented( - op->name() + " 's InferSymbolicShape interface is NOT implemented now.")); - return true; -} bool DiagEmbedOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { PADDLE_THROW(phi::errors::Unimplemented( diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h index f46128a34d0d3..4547e476a4992 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/paddle_op_infer_sym.h @@ -32,11 +32,6 @@ bool StackOpInferSymbolicShape(pir::Operation *op, bool SumOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool ReshapeOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Reshape_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - bool FullIntArrayOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); @@ -111,26 +106,6 @@ bool SplitOpInferSymbolicShape(pir::Operation *op, // Not Impelmented Ops. -bool AsComplexOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool AsRealOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool AsStridedOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - -bool CummaxOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool CumminOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool CumprodOpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Cumprod_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); -bool CumsumOpInferSymbolicShape(pir::Operation *op, - pir::ShapeConstraintIRAnalysis *shape_analysis); -bool Cumsum_OpInferSymbolicShape( - pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); - bool DiagEmbedOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool DiagonalOpInferSymbolicShape( diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/unary_infer_sym.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/unary_infer_sym.cc index d82fc12521998..c2e17f1f8f8c6 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/unary_infer_sym.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/unary_infer_sym.cc @@ -14,14 +14,13 @@ #include "paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/unary_infer_sym.h" #include "paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/infer_sym_utils.h" -// #include "paddle/fluid/pir/dialect/operator/ir/op_attribute.h" namespace paddle::dialect { bool ArgmaxOpInferSymbolicShape( pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { - bool flatten = GET_BOOL_ATTR(op, "flatten"); - bool keepdims = GET_BOOL_ATTR(op, "keepdims"); + bool flatten = GetBoolAttr(op, "flatten"); + bool keepdims = GetBoolAttr(op, "keepdims"); const auto &input_shape_or_data = shape_analysis->GetShapeOrDataForValue(op->operand_source(0)); @@ -74,4 +73,178 @@ bool ArgminOpInferSymbolicShape( return ArgmaxOpInferSymbolicShape(op, shape_analysis); } +bool AsComplexOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + pir::Value operand_source = op->operand_source(0); + const symbol::ShapeOrDataDimExprs &operand_shape_or_data = + shape_analysis->GetShapeOrDataForValue(operand_source); + + const std::vector out_dims = [&] { + std::vector out_dims = operand_shape_or_data.shape(); + out_dims.pop_back(); + return out_dims; + }(); + + symbol::ShapeOrDataDimExprs shape_data{ + symbol::TensorShapeOrDataDimExprs(out_dims)}; + + shape_analysis->SetShapeOrDataForValue(op->result(0), shape_data); + return true; +} +bool AsRealOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + pir::Value operand_source = op->operand_source(0); + const symbol::ShapeOrDataDimExprs &operand_shape_or_data = + shape_analysis->GetShapeOrDataForValue(operand_source); + + const std::vector out_dims = [&] { + std::vector out_dims = operand_shape_or_data.shape(); + out_dims.push_back(symbol::DimExpr(2)); + return out_dims; + }(); + + symbol::ShapeOrDataDimExprs shape_data{ + symbol::TensorShapeOrDataDimExprs(out_dims)}; + + shape_analysis->SetShapeOrDataForValue(op->result(0), shape_data); + return true; +} + +bool CummaxOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + pir::Value operand_source = op->operand_source(0); + const symbol::ShapeOrDataDimExprs &operand_shape_or_data = + shape_analysis->GetShapeOrDataForValue(operand_source); + + shape_analysis->SetShapeOrDataForValue(op->result(0), operand_shape_or_data); + shape_analysis->SetShapeOrDataForValue(op->result(1), operand_shape_or_data); + return true; +} +bool CumminOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return CummaxOpInferSymbolicShape(op, shape_analysis); +} +bool CumprodOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + pir::Value operand_source = op->operand_source(0); + const symbol::ShapeOrDataDimExprs &operand_shape_or_data = + shape_analysis->GetShapeOrDataForValue(operand_source); + shape_analysis->SetShapeOrDataForValue(op->result(0), operand_shape_or_data); + return true; +} +bool Cumprod_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return CumprodOpInferSymbolicShape(op, shape_analysis); +} +bool CumsumOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + pir::Value operand_source = op->operand_source(0); + + const symbol::ShapeOrDataDimExprs &operand_shape_or_data = + shape_analysis->GetShapeOrDataForValue(operand_source); + + bool flatten = GetBoolAttr(op, "flatten"); + if (flatten) { + symbol::DimExpr product{1}; + const auto &dim_exprs = operand_shape_or_data.shape(); + for (const auto &dim_expr : dim_exprs) { + product = product * dim_expr; + } + const std::vector out_dims = {product}; + symbol::ShapeOrDataDimExprs shape_data{ + symbol::TensorShapeOrDataDimExprs(out_dims)}; + shape_analysis->SetShapeOrDataForValue(op->result(0), shape_data); + + } else { + shape_analysis->SetShapeOrDataForValue(op->result(0), + operand_shape_or_data); + } + return true; +} +bool Cumsum_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return CumsumOpInferSymbolicShape(op, shape_analysis); +} +bool ReshapeOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + pir::Value operand_source = op->operand_source(0); + if (shape_analysis->GetShapeOrDataForValue(operand_source) + .data() + .has_value()) { + const symbol::ShapeOrDataDimExprs &operand_shape_or_data = + shape_analysis->GetShapeOrDataForValue(operand_source); + shape_analysis->SetShapeOrDataForValue(op->result(0), + operand_shape_or_data); + return true; + } + + pir::Value operand_source_shape = op->operand_source(1); + + const symbol::ShapeOrDataDimExprs &operand_shape_or_data = + shape_analysis->GetShapeOrDataForValue(operand_source_shape); + + const auto &GetProduct = [&](const auto &dim_exprs, const auto &Filter) { + symbol::DimExpr product{1}; + for (const auto &dim_expr : dim_exprs) { + if (Filter(dim_expr)) { + product = product * dim_expr; + } + } + return product; + }; + + const auto &IsNotMinusOne = [&](const symbol::DimExpr &dim_expr) { + if (dim_expr.isa()) { + return dim_expr.dyn_cast() != static_cast(-1); + } + return true; + }; + + const auto &IsZero = [&](const symbol::DimExpr &dim_expr) { + if (dim_expr.isa()) { + return dim_expr.dyn_cast() == static_cast(0); + } + return false; + }; + + const std::vector out_dims = [&] { + const auto &original_shape = + shape_analysis->GetShapeOrDataForValue(op->operand_source(0)).shape(); + + const auto &numel = + GetProduct(original_shape, [](const auto &) { return true; }); + + const auto &product_exclude_minus_one = + GetProduct(operand_shape_or_data.data().value(), IsNotMinusOne); + + const auto &input_dims = operand_shape_or_data.data().value(); + + std::vector out_dims; + out_dims.reserve(input_dims.size()); + for (size_t i = 0; i < input_dims.size(); ++i) { + auto out_dim_expr = IsNotMinusOne(input_dims[i]) + ? input_dims[i] + : (numel / product_exclude_minus_one); + out_dim_expr = IsZero(input_dims[i]) ? original_shape[i] : out_dim_expr; + out_dims.emplace_back(out_dim_expr); + } + + return out_dims; + }(); + + symbol::ShapeOrDataDimExprs shape_data{ + symbol::TensorShapeOrDataDimExprs(out_dims)}; + + shape_analysis->SetShapeOrDataForValue(op->result(0), shape_data); + shape_analysis->SetShapeOrDataForValue( + op->result(1), + shape_analysis->GetShapeOrDataForValue(operand_source_shape)); + return true; +} + +bool Reshape_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return ReshapeOpInferSymbolicShape(op, shape_analysis); +} + } // namespace paddle::dialect diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/unary_infer_sym.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/unary_infer_sym.h index 832a6a7a074c3..4cbf8696a01bc 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/unary_infer_sym.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/unary_infer_sym.h @@ -22,5 +22,25 @@ bool ArgmaxOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); bool ArgminOpInferSymbolicShape(pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool AsComplexOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool AsRealOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool CummaxOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool CumminOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool CumprodOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Cumprod_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool CumsumOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Cumsum_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool ReshapeOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); +bool Reshape_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); } // namespace paddle::dialect diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml index 5156073182e67..35ccab6221eb6 100755 --- a/paddle/phi/api/yaml/ops.yaml +++ b/paddle/phi/api/yaml/ops.yaml @@ -207,7 +207,6 @@ func : as_strided backward : as_strided_grad no_need_buffer : input - interfaces : paddle::dialect::InferSymbolicShapeInterface - op : asgd_ args : (Tensor param, Tensor grad, Tensor learning_rate, Tensor d, Tensor y, Tensor n, Tensor master_param, bool multi_precision=false) diff --git a/test/ir/pir/cinn/symbolic/CMakeLists.txt b/test/ir/pir/cinn/symbolic/CMakeLists.txt index 3a330e6527530..d227d7cc8af3a 100644 --- a/test/ir/pir/cinn/symbolic/CMakeLists.txt +++ b/test/ir/pir/cinn/symbolic/CMakeLists.txt @@ -13,6 +13,7 @@ if(WITH_GPU) test_if_dy.py test_llama_if_dy.py test_decomp_inference_predictor_run.py + test_unary_op_infer_sym_shape.py test_sub_graph_for_backend.py test_sub_graph_for_frontend.py test_check_infer_symbolic.py @@ -38,6 +39,18 @@ if(WITH_GPU) "RUN_TYPE=CINN") endforeach() + add_test( + NAME test_unary_op_infer_sym_shape + COMMAND + ${CMAKE_COMMAND} -E env + PYTHONPATH=${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/python/:$ENV{PYTHONPATH} + FLAGS_enable_pir_api=1 FLAGS_cinn_bucket_compile=True FLAGS_prim_all=True + FLAGS_pir_apply_shape_optimization_pass=1 ${PYTHON_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/test_unary_op_infer_sym_shape.py + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + set_tests_properties(test_unary_op_infer_sym_shape PROPERTIES LABELS + "RUN_TYPE=CINN") + add_test( NAME test_if_st COMMAND diff --git a/test/ir/pir/cinn/symbolic/test_unary_op_infer_sym_shape.py b/test/ir/pir/cinn/symbolic/test_unary_op_infer_sym_shape.py index 5260475b45f1e..be6741661295a 100644 --- a/test/ir/pir/cinn/symbolic/test_unary_op_infer_sym_shape.py +++ b/test/ir/pir/cinn/symbolic/test_unary_op_infer_sym_shape.py @@ -108,5 +108,162 @@ def test_eval_symbolic(self): return True +class AsComplexAsRealNet(paddle.nn.Layer): + def __init__(self): + super().__init__() + + def forward(self, x): + real_res = paddle.as_complex(x) + complex_res = paddle.as_real(real_res) + return real_res, complex_res + + +class TestAsComplexAsRealOPInferSymbolicShape(TestBase): + def prepare_data(self): + self.cases = [np.random.rand(4, 5, 6)] + self.expected = [ + [ + 'shape[S0, S1], data[NULL]', + 'shape[S0, S1, 2], data[NULL]', + ] + ] + + def test_eval_symbolic(self): + net = AsComplexAsRealNet() + + for i in range(len(self.cases)): + x = self.cases[i] + x_spec = InputSpec(shape=[None, None, 2], dtype='float32') + + input_spec = [x_spec] + net = apply_to_static(net, False, input_spec) + net.eval() + + # check the infer result + sym_shape_str_list = get_sym_shape_str_for_op( + net, input_spec, 'pd_op.as_complex' + ) + sym_shape_str_list += get_sym_shape_str_for_op( + net, input_spec, 'pd_op.as_real' + ) + + np.testing.assert_equal( + len(sym_shape_str_list), len(self.expected[i]) + ) + for j in range(len(sym_shape_str_list)): + np.testing.assert_equal( + sym_shape_str_list[j].find(self.expected[i][j]), + 0, + f'in case i,j = {i},{j}: output shape ({sym_shape_str_list[0]}) is not expected {(self.expected[i][j])}', + ) + + return True + + +class CumSumProdNet(paddle.nn.Layer): + def __init__(self): + super().__init__() + + def forward(self, x): + cumsum_out = paddle.cumsum(x) + cumprod_out = paddle.cumprod(x, dim=1) + return cumsum_out, cumprod_out + + +class TestCumSumProdOpInferSymbolicShape(TestBase): + def prepare_data(self): + self.cases = [np.random.rand(4, 5, 6)] + self.expected = [ + [ + 'shape[Mul(Mul(Mul(1, S0), S1), S2)], data[NULL]', + 'shape[S0, S1, S2], data[NULL]', + ] + ] + + def test_eval_symbolic(self): + net = CumSumProdNet() + + for i in range(len(self.cases)): + x = self.cases[i] + x_spec = InputSpec( + shape=[None for index in range(len(x.shape))], dtype='float32' + ) + + input_spec = [x_spec] + net = apply_to_static(net, False, input_spec) + net.eval() + + # check the infer result + sym_shape_str_list = get_sym_shape_str_for_op( + net, input_spec, 'pd_op.cumsum' + ) + sym_shape_str_list += get_sym_shape_str_for_op( + net, input_spec, 'pd_op.cumprod' + ) + + np.testing.assert_equal( + len(sym_shape_str_list), len(self.expected[i]) + ) + for j in range(len(sym_shape_str_list)): + np.testing.assert_equal( + sym_shape_str_list[j].find(self.expected[i][j]), + 0, + f'in case i,j = {i},{j}: output shape ({sym_shape_str_list[0]}) is not expected {(self.expected[i][j])}', + ) + + return True + + +class ReshapeNet(paddle.nn.Layer): + def __init__(self): + super().__init__() + + def forward(self, x): + out1 = paddle.reshape(x, [-1, 4, 5]) + out2 = paddle.reshape(x, [0, 0, 12]) + return out1, out2 + + +class TestReshapeOpInferSymbolicShape(TestBase): + def prepare_data(self): + self.cases = [np.random.rand(4, 5, 6)] + self.expected = [ + [ + 'shape[Mul(Mul(Mul(Mul(1, S0), S1), S2), 1 / (20)), 4, 5], data[NULL]', + 'shape[S0, S1, 12], data[NULL]', + ] + ] + + def test_eval_symbolic(self): + net = ReshapeNet() + + for i in range(len(self.cases)): + x = self.cases[i] + x_spec = InputSpec( + shape=[None for index in range(len(x.shape))], dtype='float32' + ) + + input_spec = [x_spec] + net = apply_to_static(net, False, input_spec) + net.eval() + + # check the infer result + sym_shape_str_list = get_sym_shape_str_for_op( + net, input_spec, 'pd_op.reshape' + ) + + np.testing.assert_equal( + len(sym_shape_str_list), len(self.expected[i]) + ) + for j in range(len(sym_shape_str_list)): + np.testing.assert_equal( + sym_shape_str_list[j].find(self.expected[i][j]), + 0, + f'in case i,j = {i},{j}: output shape ({sym_shape_str_list[0]}) is not expected {(self.expected[i][j])}', + ) + + return True + + if __name__ == '__main__': unittest.main() From 04d499ba57d928acebf37bba4446af3b6198a132 Mon Sep 17 00:00:00 2001 From: zhangbo9674 <82555433+zhangbo9674@users.noreply.github.com> Date: Mon, 4 Mar 2024 17:25:47 +0800 Subject: [PATCH 274/282] fix (#62351) --- cmake/external/pslib.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/external/pslib.cmake b/cmake/external/pslib.cmake index d7de1aae86015..9800eab1e0992 100644 --- a/cmake/external/pslib.cmake +++ b/cmake/external/pslib.cmake @@ -69,7 +69,7 @@ ExternalProject_Add( -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${PSLIB_INSTALL_ROOT} -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} - BUILD_BYPRODUCTS ${PSLIB_LIB}) + BUILD_BYPRODUCTS ${PSLIB_LIB} ${JVM_LIB}) add_library(pslib SHARED IMPORTED GLOBAL) set_property(TARGET pslib PROPERTY IMPORTED_LOCATION ${PSLIB_LIB}) From 437293bed1b6006732671531cfb2010411a6c0cb Mon Sep 17 00:00:00 2001 From: WangZhen <23097963+0x45f@users.noreply.github.com> Date: Mon, 4 Mar 2024 19:03:49 +0800 Subject: [PATCH 275/282] fused_multi_transformer/fused_bias_dropout_residual_layer_norm to phi (#62049) --- .../fused/fused_multi_transformer_int8_op.cu | 65 +- .../fused/fused_multi_transformer_op.cu | 2508 +++++++++-------- .../fused/fused_multi_transformer_op.cu.h | 195 +- .../fused_multi_transformer_sig.cc | 58 + .../pir/dialect/op_generator/ops_api_gen.py | 1 - paddle/fluid/pir/dialect/operator/ir/ops.yaml | 10 + paddle/phi/api/yaml/fused_backward.yaml | 3 +- paddle/phi/api/yaml/fused_ops.yaml | 1 + paddle/phi/api/yaml/legacy_ops.yaml | 10 + paddle/phi/infermeta/fusion.cc | 104 +- paddle/phi/infermeta/fusion.h | 34 +- ...dropout_residual_layer_norm_grad_kernel.cu | 2 +- .../nn/functional/fused_transformer.py | 32 +- ...bias_dropout_residual_layer_norm_op_api.py | 5 +- .../test_fused_multi_transformer_op.py | 11 +- 15 files changed, 1623 insertions(+), 1416 deletions(-) create mode 100644 paddle/fluid/operators/ops_signature/fused_multi_transformer_sig.cc diff --git a/paddle/fluid/operators/fused/fused_multi_transformer_int8_op.cu b/paddle/fluid/operators/fused/fused_multi_transformer_int8_op.cu index 157a45c71c16e..a76e93f5cdcf5 100644 --- a/paddle/fluid/operators/fused/fused_multi_transformer_int8_op.cu +++ b/paddle/fluid/operators/fused/fused_multi_transformer_int8_op.cu @@ -14,6 +14,7 @@ limitations under the License. */ #include "paddle/fluid/operators/fused/attn_gemm_int8.h" #include "paddle/fluid/operators/fused/fused_multi_transformer_op.cu.h" +#include "paddle/phi/kernels/fusion/gpu/attention_layer.norm.h" namespace paddle { namespace operators { @@ -345,18 +346,18 @@ class FusedMultiTransformerINT8OpKernel : public framework::OpKernel { if (time_step) { // generation decoder stage // [2, batch_size, num_head, max_seq_len, head_size] int max_seq_len = cache_kv->dims()[3]; - fmha(dev_ctx, - qkv_out, - *qkv_bias, - *src_mask, - cache_kv_out, - &fmha_out, - bsz, - max_seq_len, - num_head, - dim_head, - time_step->data()[0], - 1. / std::sqrt(dim_head)); + phi::fusion::fmha(dev_ctx, + qkv_out, + *qkv_bias, + *src_mask, + cache_kv_out, + &fmha_out, + bsz, + max_seq_len, + num_head, + dim_head, + time_step->data()[0], + 1. / std::sqrt(dim_head)); } else if (cache_kv_out) { // generation context stage // TODO(wangxi): can remove dropout in inference fmha_compute.ComputeForward(qkv_out, @@ -387,16 +388,16 @@ class FusedMultiTransformerINT8OpKernel : public framework::OpKernel { T *cache_k_ptr = cache_kv_data; T *cache_v_ptr = cache_kv_data + cache_k_size; - write_cache_kv(dev_ctx, - cache_k_ptr, - cache_v_ptr, - k_ptr, - v_ptr, - bsz, - num_head, - seq_len, - max_seq_len, - dim_head); + phi::fusion::write_cache_kv(dev_ctx, + cache_k_ptr, + cache_v_ptr, + k_ptr, + v_ptr, + bsz, + num_head, + seq_len, + max_seq_len, + dim_head); } else { // not generation // TODO(wangxi): can remove dropout in inference fmha_compute.ComputeForward(qkv_out, @@ -427,10 +428,10 @@ class FusedMultiTransformerINT8OpKernel : public framework::OpKernel { quant_round_type, quant_max_bound, quant_min_bound); - AllReduce(output_workspace, - ring_id, - bsz * seq_len * num_head * dim_head, - dev_ctx); + phi::fusion::AllReduce(output_workspace, + ring_id, + bsz * seq_len * num_head * dim_head, + dev_ctx); } else { out_linear_compute.ComputeForward(out_linear_weights[i], &fmha_out, @@ -444,7 +445,7 @@ class FusedMultiTransformerINT8OpKernel : public framework::OpKernel { quant_round_type, quant_max_bound, quant_min_bound); - AllReduce(*buf0, ring_id, buf0->numel(), dev_ctx); + phi::fusion::AllReduce(*buf0, ring_id, buf0->numel(), dev_ctx); } #ifdef _DEBUG_FUSED_MULTI_TRANSFORMER VLOG(0) << "step4"; @@ -583,12 +584,12 @@ class FusedMultiTransformerINT8OpKernel : public framework::OpKernel { #endif if (pre_layer_norm) { - AllReduce(output_workspace, - ring_id, - bsz * seq_len * num_head * dim_head, - dev_ctx); + phi::fusion::AllReduce(output_workspace, + ring_id, + bsz * seq_len * num_head * dim_head, + dev_ctx); } else { - AllReduce(*buf0, ring_id, buf0->numel(), dev_ctx); + phi::fusion::AllReduce(*buf0, ring_id, buf0->numel(), dev_ctx); } #ifdef _DEBUG_FUSED_MULTI_TRANSFORMER VLOG(0) << "step8.1"; diff --git a/paddle/fluid/operators/fused/fused_multi_transformer_op.cu b/paddle/fluid/operators/fused/fused_multi_transformer_op.cu index e3158d74df629..75a4c7b275a8a 100644 --- a/paddle/fluid/operators/fused/fused_multi_transformer_op.cu +++ b/paddle/fluid/operators/fused/fused_multi_transformer_op.cu @@ -14,1365 +14,1393 @@ limitations under the License. */ #include "paddle/fluid/operators/fused/fused_multi_transformer_op.cu.h" -namespace paddle { -namespace operators { +#include "paddle/fluid/framework/op_registry.h" + +#include "paddle/phi/api/include/tensor.h" +#include "paddle/phi/backends/gpu/gpu_context.h" +#include "paddle/phi/backends/gpu/gpu_device_function.h" +#include "paddle/phi/core/dense_tensor.h" +#include "paddle/phi/core/enforce.h" +#include "paddle/phi/core/kernel_registry.h" +#include "paddle/phi/kernels/funcs/layer_norm_impl.cu.h" +#include "paddle/phi/kernels/fusion/gpu/attention_layer.norm.h" +#include "paddle/phi/kernels/fusion/gpu/fmha_ref.h" +#include "paddle/phi/kernels/fusion/gpu/fused_dropout_helper.h" + +namespace phi { +namespace fusion { #if CUDA_VERSION >= 11060 // Use cublasLt to fuse FFN operation. -template -class FusedMultiTransformerOpKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext &ctx) const override { - using U = LayerNormParamType; - auto &dev_ctx = ctx.cuda_device_context(); - - auto *time_step = ctx.Input("TimeStep"); - // 0. input - auto *input_x = ctx.Input("X"); - const auto input_x_dims = input_x->dims(); - int bsz = input_x_dims[0]; - int seq_len = input_x_dims[1]; - int dim_embed = input_x_dims[2]; - int bsz_seq = bsz * seq_len; - const std::string act_method = ctx.Attr("act_method"); - bool remove_padding = false; - auto *sequence_lengths = ctx.Input("SeqLengths"); - if (sequence_lengths) { - remove_padding = true; - } - phi::DenseTensor d_token_tensor; - phi::DenseTensor padding_offset_tensor; - phi::DenseTensor x_remove_padding; - bool encoder_remove_padding = (remove_padding && !time_step); - int token_num = 0; - - // remove padding in encoder - if (encoder_remove_padding) { - // just for encoder - d_token_tensor.Resize({{1}}); - auto *d_token_num = dev_ctx.Alloc( - &d_token_tensor, d_token_tensor.numel() * sizeof(int)); - // alloc the max size of padding_offset_tensor - padding_offset_tensor.Resize({{bsz_seq}}); - dev_ctx.Alloc(&padding_offset_tensor, - padding_offset_tensor.numel() * sizeof(int)); - InvokeGetPaddingOffset(dev_ctx, - &token_num, - d_token_num, - padding_offset_tensor.data(), - sequence_lengths->data(), - bsz, - seq_len); - padding_offset_tensor.Resize({{token_num}}); - x_remove_padding.Resize({{token_num, dim_embed}}); - dev_ctx.Alloc(&x_remove_padding, x_remove_padding.numel() * sizeof(T)); - InvokeRemovePadding(dev_ctx, - x_remove_padding.data(), - input_x->data(), - padding_offset_tensor.data(), - token_num, - dim_embed); - } else { - token_num = bsz_seq; - } - auto *padding_offset_data = - encoder_remove_padding ? padding_offset_tensor.data() : nullptr; - - // 1. layer norm - const auto pre_layer_norm = ctx.Attr("pre_layer_norm"); - const float epsilon = ctx.Attr("epsilon"); - auto ln_scales = ctx.MultiInput("LnScale"); - auto ln_biases = ctx.MultiInput("LnBias"); - - auto ln_compute = AttnLayerNorm(dev_ctx, epsilon, token_num, dim_embed); - phi::DenseTensor ln_mean, ln_var; - ln_mean.Resize({{token_num}}); - auto *ln_mean_data = - dev_ctx.Alloc(&ln_mean, ln_mean.numel() * sizeof(U)); - ln_var.Resize({{token_num}}); - auto *ln_var_data = dev_ctx.Alloc(&ln_var, ln_var.numel() * sizeof(U)); - - // 2. qkv - // x: qkv's input [batch_size, seq_len, dim_embed] - // y: qkv's weight: [3, num_head, dim_head, dim_embed] - auto qkv_weights = ctx.MultiInput("QKVW"); - auto qkv_biases = ctx.MultiInput("QKVBias"); - const bool trans_qkvw = ctx.Attr("trans_qkvw"); - const auto qkv_w_dims = qkv_weights[0]->dims(); - int num_head = trans_qkvw ? qkv_w_dims[1] : qkv_w_dims[2]; - int dim_head = trans_qkvw ? qkv_w_dims[2] : qkv_w_dims[3]; - int hidden_size = num_head * dim_head; - int output_size = 3 * hidden_size; - int input_size = dim_embed; - - bool compute_bias = qkv_biases.size() > 0 && time_step == nullptr; - // (transA, transB, compute_bias) = (false, trans_qkvw, false) - // Since we fused QKVBias into QKVBiasAddTransposeSplit kernel, here we set - // compute_bias as false. - auto qkv_compute = phi::fusion::AttnMatMul(dev_ctx, - false, - trans_qkvw, - token_num, - output_size, - input_size, - /*compute_bias=*/false); - - phi::DenseTensor qkv_out; - qkv_out.Resize({{token_num, 3, num_head, dim_head}}); - auto *qkv_out_data = - dev_ctx.Alloc(&qkv_out, qkv_out.numel() * sizeof(T)); - - // 2.1 rotary - auto *rotary_tensor = ctx.Input("RotaryPosEmb"); - const int rotary_emb_dims = ctx.Attr("rotary_emb_dims"); - - // 3. fmha - AttnDropoutParam attn_param( - true, "upscale_in_train", 0.0, true, true, 0, nullptr); - auto fmha_compute = - FMHARef(dev_ctx, bsz, seq_len, num_head, dim_head, attn_param); - auto *src_mask = ctx.Input("SrcMask"); - auto cache_kvs = ctx.MultiInput("CacheKV"); - auto cache_kv_outs = ctx.MultiOutput("CacheKVOut"); - // auto *time_step = ctx.Input("TimeStep"); - auto pre_caches = ctx.MultiInput("PreCaches"); - int cache_offset = 0; - if (pre_caches.size() > 0) { - cache_offset = pre_caches[0]->dims()[3]; +template +void FusedMultiTransformerKernel( + const Context &dev_ctx, + const DenseTensor &x, + const std::vector &ln_scales, + const std::vector &ln_biases, + const std::vector &qkv_weights, + const paddle::optional> &qkv_biases, + const paddle::optional> &cache_kvs, + const paddle::optional> &pre_caches, + const paddle::optional &rotary_tensor, + const paddle::optional &time_step, + const paddle::optional &seq_lengths, + const paddle::optional &src_mask, + const std::vector &out_linear_weights, + const paddle::optional> &out_linear_biases, + const std::vector &ffn_ln_scales, + const std::vector &ffn_ln_biases, + const std::vector &ffn1_weights, + const paddle::optional> &ffn1_biases, + const std::vector &ffn2_weights, + const paddle::optional> &ffn2_biases, + bool pre_layer_norm, + float epsilon, + float dropout_rate, + int rotary_emb_dims, + bool is_test, + const std::string &dropout_implementation, + const std::string &act_method, + bool trans_qkvw, + int ring_id, + std::vector cache_kv_outs, + DenseTensor *out) { + if (cache_kvs) { + for (size_t i = 0; i < cache_kv_outs.size(); i++) { + *(cache_kv_outs[i]) = *(cache_kvs.get()[i]); } + } + using U = phi::funcs::LayerNormParamType; + + auto *rotary_tensor_t = rotary_tensor.get_ptr(); + auto *seq_lengths_t = seq_lengths.get_ptr(); + auto *src_mask_t = src_mask.get_ptr(); + auto *time_step_t = time_step.get_ptr(); + + const auto input_x_dims = x.dims(); + int bsz = input_x_dims[0]; + int seq_len = input_x_dims[1]; + int dim_embed = input_x_dims[2]; + int bsz_seq = bsz * seq_len; + bool remove_padding = false; + if (seq_lengths_t) { + remove_padding = true; + } + phi::DenseTensor d_token_tensor; + phi::DenseTensor padding_offset_tensor; + phi::DenseTensor x_remove_padding; + bool encoder_remove_padding = (remove_padding && !time_step_t); + int token_num = 0; + + // remove padding in encoder + if (encoder_remove_padding) { + // just for encoder + d_token_tensor.Resize({1}); + auto *d_token_num = dev_ctx.template Alloc( + &d_token_tensor, d_token_tensor.numel() * sizeof(int)); + // alloc the max size of padding_offset_tensor + padding_offset_tensor.Resize({bsz_seq}); + dev_ctx.template Alloc(&padding_offset_tensor, + padding_offset_tensor.numel() * sizeof(int)); + InvokeGetPaddingOffset(dev_ctx, + &token_num, + d_token_num, + padding_offset_tensor.data(), + seq_lengths_t->data(), + bsz, + seq_len); + padding_offset_tensor.Resize({token_num}); + x_remove_padding.Resize({token_num, dim_embed}); + dev_ctx.template Alloc(&x_remove_padding, + x_remove_padding.numel() * sizeof(T)); + InvokeRemovePadding(dev_ctx, + x_remove_padding.data(), + x.data(), + padding_offset_tensor.data(), + token_num, + dim_embed); + } else { + token_num = bsz_seq; + } + auto *padding_offset_data = + encoder_remove_padding ? padding_offset_tensor.data() : nullptr; + + auto ln_compute = AttnLayerNorm(dev_ctx, epsilon, token_num, dim_embed); + phi::DenseTensor ln_mean, ln_var; + ln_mean.Resize({token_num}); + auto *ln_mean_data = + dev_ctx.template Alloc(&ln_mean, ln_mean.numel() * sizeof(U)); + ln_var.Resize({token_num}); + auto *ln_var_data = + dev_ctx.template Alloc(&ln_var, ln_var.numel() * sizeof(U)); + + // 2. qkv + // x: qkv's input [batch_size, seq_len, dim_embed] + // y: qkv's weight: [3, num_head, dim_head, dim_embed] + const auto qkv_w_dims = qkv_weights[0]->dims(); + int num_head = trans_qkvw ? qkv_w_dims[1] : qkv_w_dims[2]; + int dim_head = trans_qkvw ? qkv_w_dims[2] : qkv_w_dims[3]; + int hidden_size = num_head * dim_head; + int output_size = 3 * hidden_size; + int input_size = dim_embed; + + bool compute_bias = + qkv_biases && !qkv_biases.get().empty() && time_step_t == nullptr; + // (transA, transB, compute_bias) = (false, trans_qkvw, false) + // Since we fused QKVBias into QKVBiasAddTransposeSplit kernel, here we set + // compute_bias as false. + auto qkv_compute = phi::fusion::AttnMatMul(dev_ctx, + false, + trans_qkvw, + token_num, + output_size, + input_size, + /*compute_bias=*/false); + + phi::DenseTensor qkv_out; + qkv_out.Resize({token_num, 3, num_head, dim_head}); + auto *qkv_out_data = + dev_ctx.template Alloc(&qkv_out, qkv_out.numel() * sizeof(T)); + + // 3. fmha + AttnDropoutParam attn_param( + true, "upscale_in_train", 0.0, true, true, 0, nullptr); + auto fmha_compute = + FMHARef(dev_ctx, bsz, seq_len, num_head, dim_head, attn_param); + int cache_offset = 0; + if (pre_caches && pre_caches.get().size() > 0) { + cache_offset = pre_caches.get()[0]->dims()[3]; + } - auto out_seq_len = seq_len; - if (time_step) { - PADDLE_ENFORCE_EQ(time_step->place(), - platform::CPUPlace(), - platform::errors::PreconditionNotMet( - "The place of input(TimeStep) must be CPUPlace.")); - // cache_seq_len - int time_step_value = time_step->data()[0]; - PADDLE_ENFORCE_GT(time_step_value, - 0, - platform::errors::PreconditionNotMet( - "The value of time_step must > 0, but now is %d", - time_step_value)); - PADDLE_ENFORCE_EQ( - seq_len, - 1, - platform::errors::PreconditionNotMet( - "In decode stage, the seq_len of input must be 1, but now is %d", - seq_len)); - out_seq_len += time_step_value; - } else { - out_seq_len += cache_offset; - } + auto out_seq_len = seq_len; + if (time_step_t) { + PADDLE_ENFORCE_EQ(time_step_t->place(), + phi::CPUPlace(), + phi::errors::PreconditionNotMet( + "The place of input(TimeStep) must be CPUPlace.")); + // cache_seq_len + int time_step_value = time_step_t->data()[0]; + PADDLE_ENFORCE_GT(time_step_value, + 0, + phi::errors::PreconditionNotMet( + "The value of time_step_t must > 0, but now is %d", + time_step_value)); + PADDLE_ENFORCE_EQ( + seq_len, + 1, + phi::errors::PreconditionNotMet( + "In decode stage, the seq_len of input must be 1, but now is %d", + seq_len)); + out_seq_len += time_step_value; + } else { + out_seq_len += cache_offset; + } - phi::DenseTensor q_transpose_out, kv_transpose_out, qk_out; - q_transpose_out.Resize({{bsz, num_head, seq_len, dim_head}}); - auto *q_transpose_out_data = - dev_ctx.Alloc(&q_transpose_out, q_transpose_out.numel() * sizeof(T)); + phi::DenseTensor q_transpose_out, kv_transpose_out, qk_out; + q_transpose_out.Resize({bsz, num_head, seq_len, dim_head}); + auto *q_transpose_out_data = dev_ctx.template Alloc( + &q_transpose_out, q_transpose_out.numel() * sizeof(T)); - kv_transpose_out.Resize({{2, bsz, num_head, seq_len, dim_head}}); - auto *kv_transpose_out_data = dev_ctx.Alloc( - &kv_transpose_out, kv_transpose_out.numel() * sizeof(T)); + kv_transpose_out.Resize({2, bsz, num_head, seq_len, dim_head}); + auto *kv_transpose_out_data = dev_ctx.template Alloc( + &kv_transpose_out, kv_transpose_out.numel() * sizeof(T)); - qk_out.Resize({{bsz, num_head, seq_len, out_seq_len}}); - auto *qk_out_data = dev_ctx.Alloc(&qk_out, qk_out.numel() * sizeof(T)); + qk_out.Resize({bsz, num_head, seq_len, out_seq_len}); + auto *qk_out_data = + dev_ctx.template Alloc(&qk_out, qk_out.numel() * sizeof(T)); - phi::DenseTensor src_mask_out; - if (cache_offset > 0) { - src_mask_out.Resize({{bsz, num_head, seq_len, out_seq_len}}); - auto *src_mask_out_data = - dev_ctx.Alloc(&src_mask_out, src_mask_out.numel() * sizeof(T)); - } + phi::DenseTensor src_mask_out; + if (cache_offset > 0) { + src_mask_out.Resize({bsz, num_head, seq_len, out_seq_len}); + auto *src_mask_out_data = dev_ctx.template Alloc( + &src_mask_out, src_mask_out.numel() * sizeof(T)); + } - // [2, bs, num_head, cache_seq_len + seq_len, head_dim] - phi::DenseTensor pre_cache_kv_out; - if (cache_offset > 0) { - pre_cache_kv_out.Resize( - {{2, bsz, num_head, seq_len + cache_offset, dim_head}}); - auto *pre_cache_kv_out_data = dev_ctx.Alloc( - &pre_cache_kv_out, pre_cache_kv_out.numel() * sizeof(T)); - } + // [2, bs, num_head, cache_seq_len + seq_len, head_dim] + phi::DenseTensor pre_cache_kv_out; + if (cache_offset > 0) { + pre_cache_kv_out.Resize( + {{2, bsz, num_head, seq_len + cache_offset, dim_head}}); + auto *pre_cache_kv_out_data = dev_ctx.template Alloc( + &pre_cache_kv_out, pre_cache_kv_out.numel() * sizeof(T)); + } - phi::DenseTensor softmax_out; - phi::DenseTensor attn_dropout_mask_out, attn_dropout_out; - phi::DenseTensor qktv_out, fmha_out; - softmax_out.Resize({{bsz, num_head, seq_len, out_seq_len}}); - auto *softmax_out_data = - dev_ctx.Alloc(&softmax_out, softmax_out.numel() * sizeof(T)); - - attn_dropout_mask_out.Resize({{bsz, num_head, seq_len, out_seq_len}}); - auto *attn_dropout_mask_out_data = dev_ctx.Alloc( - &attn_dropout_mask_out, attn_dropout_mask_out.numel() * sizeof(T)); - attn_dropout_out.Resize({{bsz, num_head, seq_len, out_seq_len}}); - auto *attn_dropout_data_data = dev_ctx.Alloc( - &attn_dropout_out, attn_dropout_out.numel() * sizeof(T)); - - qktv_out.Resize({{bsz, num_head, seq_len, dim_head}}); - auto *qktv_out_data = - dev_ctx.Alloc(&qktv_out, qktv_out.numel() * sizeof(T)); - fmha_out.Resize({{bsz, seq_len, num_head, dim_head}}); - auto *fmha_out_data = - dev_ctx.Alloc(&fmha_out, fmha_out.numel() * sizeof(T)); - - // 4. out_linear - auto out_linear_weights = ctx.MultiInput("OutLinearW"); - auto out_linear_biases = ctx.MultiInput("OutLinearBias"); - int ring_id = ctx.Attr("ring_id"); - // (transA, transB, compute_bias) = (false, false, false) - auto out_linear_compute = phi::fusion::AttnMatMul( - dev_ctx, false, false, token_num, dim_embed, hidden_size, false); - - // 5. ln(residual + bias) - DropoutParam dropout_param2(true, 0, true, true, 0.0, nullptr, 0); - FusedDropoutLayerNormHelper fused_dropout_layernorm_helper( - dev_ctx, token_num, dim_embed, dropout_param2, epsilon); - auto ffn_ln_scales = ctx.MultiInput("FFNLnScale"); - auto ffn_ln_biases = ctx.MultiInput("FFNLnBias"); - phi::DenseTensor bias_dropout_residual_out, dropout_mask_out; - T *bias_dropout_residual_out_data = nullptr; + phi::DenseTensor softmax_out; + phi::DenseTensor attn_dropout_mask_out, attn_dropout_out; + phi::DenseTensor qktv_out, fmha_out; + softmax_out.Resize({bsz, num_head, seq_len, out_seq_len}); + auto *softmax_out_data = + dev_ctx.template Alloc(&softmax_out, softmax_out.numel() * sizeof(T)); + + attn_dropout_mask_out.Resize({bsz, num_head, seq_len, out_seq_len}); + auto *attn_dropout_mask_out_data = dev_ctx.template Alloc( + &attn_dropout_mask_out, attn_dropout_mask_out.numel() * sizeof(T)); + attn_dropout_out.Resize({bsz, num_head, seq_len, out_seq_len}); + auto *attn_dropout_data_data = dev_ctx.template Alloc( + &attn_dropout_out, attn_dropout_out.numel() * sizeof(T)); + + qktv_out.Resize({bsz, num_head, seq_len, dim_head}); + auto *qktv_out_data = + dev_ctx.template Alloc(&qktv_out, qktv_out.numel() * sizeof(T)); + fmha_out.Resize({bsz, seq_len, num_head, dim_head}); + auto *fmha_out_data = + dev_ctx.template Alloc(&fmha_out, fmha_out.numel() * sizeof(T)); + + // (transA, transB, compute_bias) = (false, false, false) + auto out_linear_compute = phi::fusion::AttnMatMul( + dev_ctx, false, false, token_num, dim_embed, hidden_size, false); + + // 5. ln(residual + bias) + DropoutParam dropout_param2(true, 0, true, true, 0.0, nullptr, 0); + FusedDropoutLayerNormHelper fused_dropout_layernorm_helper( + dev_ctx, token_num, dim_embed, dropout_param2, epsilon); + phi::DenseTensor bias_dropout_residual_out, dropout_mask_out; + T *bias_dropout_residual_out_data = nullptr; + if (pre_layer_norm) { + bias_dropout_residual_out.Resize({token_num, dim_embed}); + bias_dropout_residual_out_data = dev_ctx.template Alloc( + &bias_dropout_residual_out, + bias_dropout_residual_out.numel() * sizeof(T)); + } + dropout_mask_out.Resize({token_num, dim_embed}); + auto *dropout_mask_out_data = dev_ctx.template Alloc( + &dropout_mask_out, dropout_mask_out.numel() * sizeof(uint8_t)); + + // 6. ffn1 matmul + act + bias + auto ffn1_weight_dim = ffn1_weights[0]->dims(); + + int dim_ffn = ffn1_weight_dim[1]; + + auto ffn1_cublas_linear = CublasFusedMLP(dev_ctx); + const phi::DDim ffn1_input_shape({token_num, dim_embed}); + ffn1_cublas_linear.Setup(ffn1_input_shape, ffn1_weight_dim, false, false); + + phi::DenseTensor ffn1_out; + ffn1_out.Resize({token_num, dim_ffn}); + auto *ffn1_out_data = + dev_ctx.template Alloc(&ffn1_out, ffn1_out.numel() * sizeof(T)); + + // 7. ffn2 matmul + bias + residual. + auto ffn2_linear_compute = phi::fusion::AttnMatMul( + dev_ctx, false, false, token_num, dim_embed, dim_ffn, false); + + // 8. ffn2 Layernorm residual bias + DropoutParam ffn2_dropout_param(true, 0, true, true, 0.0, nullptr, 0); + FusedDropoutLayerNormHelper ffn2_fused_dropout_helper( + dev_ctx, token_num, dim_embed, ffn2_dropout_param, epsilon); + + // calc + auto *from_data = dev_ctx.template Alloc(out, out->numel() * sizeof(T)); + phi::DenseTensor *from_tensor = out; + phi::DenseTensor tmp_out, tmp_out_rm_padding; + tmp_out.Resize({token_num, dim_embed}); + if (encoder_remove_padding) { + tmp_out_rm_padding.Resize({token_num, dim_embed}); + auto *tmp_out_rm_padding_data = dev_ctx.template Alloc( + &tmp_out_rm_padding, tmp_out_rm_padding.numel() * sizeof(T)); + } + auto *tmp_out_data = + dev_ctx.template Alloc(&tmp_out, tmp_out.numel() * sizeof(T)); + + const T *x_data; + if (encoder_remove_padding) { + x_data = x_remove_padding.data(); + } else { + x_data = x.data(); + } + phi::DenseTensor *buf0 = nullptr; + phi::DenseTensor *buf1 = nullptr; + + // step0: x --> buf1 + // step1: buf1 --> buf0 + // step2: buf0 --> buf1 + int layers = qkv_weights.size(); + if (encoder_remove_padding) { + // In the case of variable lengths, the padding needs to be rebuilt + // eventually. So buf0 and buf1 do not need to be changed according to the + // pre_layer_norm and the number of layers. + buf0 = &tmp_out; + buf1 = &tmp_out_rm_padding; + } else { if (pre_layer_norm) { - bias_dropout_residual_out.Resize({{token_num, dim_embed}}); - bias_dropout_residual_out_data = - dev_ctx.Alloc(&bias_dropout_residual_out, - bias_dropout_residual_out.numel() * sizeof(T)); - } - dropout_mask_out.Resize({{token_num, dim_embed}}); - auto *dropout_mask_out_data = dev_ctx.Alloc( - &dropout_mask_out, dropout_mask_out.numel() * sizeof(uint8_t)); - - // 6. ffn1 matmul + act + bias - auto ffn1_weights = ctx.MultiInput("FFN1Weight"); - auto ffn1_biases = ctx.MultiInput("FFN1Bias"); - auto ffn1_weight_dim = ffn1_weights[0]->dims(); - - int dim_ffn = ffn1_weight_dim[1]; - - auto ffn1_cublas_linear = CublasFusedMLP(dev_ctx); - const phi::DDim ffn1_input_shape({token_num, dim_embed}); - ffn1_cublas_linear.Setup(ffn1_input_shape, ffn1_weight_dim, false, false); - - phi::DenseTensor ffn1_out; - ffn1_out.Resize({{token_num, dim_ffn}}); - auto *ffn1_out_data = - dev_ctx.Alloc(&ffn1_out, ffn1_out.numel() * sizeof(T)); - - // 7. ffn2 matmul + bias + residual. - auto ffn2_weights = ctx.MultiInput("FFN2Weight"); - auto ffn2_biases = ctx.MultiInput("FFN2Bias"); - - auto ffn2_linear_compute = phi::fusion::AttnMatMul( - dev_ctx, false, false, token_num, dim_embed, dim_ffn, false); - - // 8. ffn2 Layernorm residual bias - DropoutParam ffn2_dropout_param(true, 0, true, true, 0.0, nullptr, 0); - FusedDropoutLayerNormHelper ffn2_fused_dropout_helper( - dev_ctx, token_num, dim_embed, ffn2_dropout_param, epsilon); - - // calc - auto *out = ctx.Output("Out"); - auto *from_data = dev_ctx.Alloc(out, out->numel() * sizeof(T)); - phi::DenseTensor *from_tensor = out; - phi::DenseTensor tmp_out, tmp_out_rm_padding; - tmp_out.Resize({{token_num, dim_embed}}); - if (encoder_remove_padding) { - tmp_out_rm_padding.Resize({{token_num, dim_embed}}); - auto *tmp_out_rm_padding_data = dev_ctx.Alloc( - &tmp_out_rm_padding, tmp_out_rm_padding.numel() * sizeof(T)); - } - auto *tmp_out_data = - dev_ctx.Alloc(&tmp_out, tmp_out.numel() * sizeof(T)); - - const T *x_data; - if (encoder_remove_padding) { - x_data = x_remove_padding.data(); - } else { - x_data = input_x->data(); - } - phi::DenseTensor *buf0 = nullptr; - phi::DenseTensor *buf1 = nullptr; - - // step0: x --> buf1 - // step1: buf1 --> buf0 - // step2: buf0 --> buf1 - int layers = qkv_weights.size(); - if (encoder_remove_padding) { - // In the case of variable lengths, the padding needs to be rebuilt - // eventually. So buf0 and buf1 do not need to be changed according to the - // pre_layer_norm and the number of layers. - buf0 = &tmp_out; - buf1 = &tmp_out_rm_padding; - } else { - if (pre_layer_norm) { - if (layers & 1) { - // odd, set buf1 as out - buf0 = &tmp_out; - buf1 = out; - } else { - // even, set buf0 as out - buf0 = out; - buf1 = &tmp_out; - } - } else { + if (layers & 1) { + // odd, set buf1 as out buf0 = &tmp_out; buf1 = out; + } else { + // even, set buf0 as out + buf0 = out; + buf1 = &tmp_out; } + } else { + buf0 = &tmp_out; + buf1 = out; } + } - for (int i = 0; i < layers; ++i) { - // step1. layer_norm - if (i == 0 && pre_layer_norm) { - auto *ln_scale_data = ln_scales[i]->data(); - auto *ln_bias_data = ln_biases[i]->data(); - // TODO(wangxi): can remove mean var in inference - ln_compute.ComputeForward(x_data, - ln_scale_data, - ln_bias_data, - buf1->data(), - ln_mean_data, - ln_var_data); - } + for (int i = 0; i < layers; ++i) { + // step1. layer_norm + if (i == 0 && pre_layer_norm) { + auto *ln_scale_data = ln_scales[i]->data(); + auto *ln_bias_data = ln_biases[i]->data(); + // TODO(wangxi): can remove mean var in inference + ln_compute.ComputeForward(x_data, + ln_scale_data, + ln_bias_data, + buf1->data(), + ln_mean_data, + ln_var_data); + } #ifdef _DEBUG_FUSED_MULTI_TRANSFORMER - VLOG(0) << "step1"; + VLOG(0) << "step1"; #endif - // step2. qkv - const phi::DenseTensor *qkv_bias = - qkv_biases.size() > 0 ? qkv_biases[i] : nullptr; - // NOTE: in decoder stage, bias is fused in fmha - const phi::DenseTensor *bias = time_step ? nullptr : qkv_bias; - if (!pre_layer_norm && i == 0) { - const phi::DenseTensor *tmp_input_x = - (encoder_remove_padding) ? &x_remove_padding : input_x; - qkv_compute.ComputeForward( - qkv_weights[i], tmp_input_x, bias, &qkv_out, &qkv_out); - } else { - qkv_compute.ComputeForward( - qkv_weights[i], buf1, bias, &qkv_out, &qkv_out); - } + // step2. qkv + const phi::DenseTensor *qkv_bias = + qkv_biases && !qkv_biases.get().empty() ? qkv_biases.get()[i] : nullptr; + // NOTE: in decoder stage, bias is fused in fmha + const phi::DenseTensor *bias = time_step_t ? nullptr : qkv_bias; + if (!pre_layer_norm && i == 0) { + const phi::DenseTensor *tmp_input_x = + (encoder_remove_padding) ? &x_remove_padding : &x; + qkv_compute.ComputeForward( + qkv_weights[i], tmp_input_x, bias, &qkv_out, &qkv_out); + } else { + qkv_compute.ComputeForward( + qkv_weights[i], buf1, bias, &qkv_out, &qkv_out); + } #ifdef _DEBUG_FUSED_MULTI_TRANSFORMER - VLOG(0) << "step2"; + VLOG(0) << "step2"; #endif - // step3. fmha - const phi::DenseTensor *cache_kv = - cache_kvs.size() > 0 ? cache_kvs[i] : nullptr; - phi::DenseTensor *cache_kv_out = cache_kv ? cache_kv_outs[i] : nullptr; - - if (time_step) { // generation decoder stage - // [2, batch_size, num_head, max_seq_len, head_size] - int max_seq_len = cache_kv->dims()[3]; - fmha(dev_ctx, - qkv_out, - *qkv_bias, - *src_mask, - sequence_lengths, - rotary_tensor, - cache_kv_out, - &fmha_out, - bsz, - max_seq_len, - num_head, - dim_head, - time_step->data()[0], - rotary_emb_dims, - 1. / std::sqrt(dim_head)); - } else if (cache_kv_out) { // generation context stage - const phi::DenseTensor *pre_cache_kv_tensor = - pre_caches.size() > 0 ? pre_caches[i] : nullptr; - phi::DenseTensor *pre_cache_kv_out_tmp = - cache_offset > 0 ? &pre_cache_kv_out : nullptr; - phi::DenseTensor *src_mask_tmp = - cache_offset > 0 ? &src_mask_out : nullptr; - qkv_bias_add_transpose_split(dev_ctx, - q_transpose_out_data, - kv_transpose_out_data, - qkv_out_data, - qkv_bias->data(), - padding_offset_data, - token_num, - bsz, - num_head, - seq_len, - dim_head, - compute_bias); - // q_transpose_out_data [bs, head_num, seq_len, dim_head] - // kv_transpose_out_data [2, bs, head_num, seq_len, dim_head] - if (rotary_emb_dims != 0) { - auto *rotary_emb_data = rotary_tensor->data(); - const int *sequence_lengths_data = - encoder_remove_padding ? sequence_lengths->data() : nullptr; - rotary_qk(dev_ctx, - q_transpose_out_data, - kv_transpose_out_data, - q_transpose_out_data, - kv_transpose_out_data, - rotary_emb_data, - sequence_lengths_data, - rotary_emb_dims, - bsz, - num_head, - seq_len, - dim_head); - } - - phi::DenseTensor *tmp_padding_offset_tensor = - encoder_remove_padding ? &padding_offset_tensor : nullptr; - fmha_compute.ComputeForwardWithoutTranspose(pre_cache_kv_tensor, - src_mask, - tmp_padding_offset_tensor, - &q_transpose_out, - &kv_transpose_out, - pre_cache_kv_out_tmp, - &qk_out, - src_mask_tmp, - &softmax_out, - &attn_dropout_mask_out, - &attn_dropout_out, - &qktv_out, - &fmha_out, - token_num); - const T *k_ptr = nullptr; - const T *v_ptr = nullptr; - - if (cache_offset > 0) { - // [2, bsz, num_head, cache_offset + seq_len, head_dim] - const T *kv_data = pre_cache_kv_out.data(); - k_ptr = kv_data; - int64_t k_size = bsz * num_head * (seq_len + cache_offset) * dim_head; - v_ptr = k_ptr + k_size; - } else { - // [3, bsz, num_head, seq_len, head_dim] - int64_t k_size = bsz * seq_len * num_head * dim_head; - const T *q_ptr = q_transpose_out_data; - k_ptr = kv_transpose_out_data; - v_ptr = k_ptr + k_size; - } - - // [2, bsz, num_head, max_seq_len, head_dim] - int max_seq_len = cache_kv_out->dims()[3]; - T *cache_kv_data = cache_kv_out->data(); - int64_t cache_k_size = bsz * num_head * max_seq_len * dim_head; - - T *cache_k_ptr = cache_kv_data; - T *cache_v_ptr = cache_kv_data + cache_k_size; - - const int seq_len_tmp = seq_len + cache_offset; - write_cache_kv(dev_ctx, - cache_k_ptr, - cache_v_ptr, - k_ptr, - v_ptr, - bsz, - num_head, - seq_len_tmp, - max_seq_len, - dim_head); - } else { // not generation - // TODO(wangxi): can remove dropout in inference - qkv_bias_add_transpose_split(dev_ctx, - q_transpose_out_data, - kv_transpose_out_data, - qkv_out_data, - qkv_bias->data(), - padding_offset_data, - token_num, - bsz, - num_head, - seq_len, - dim_head, - compute_bias); - - // q_transpose_out_data [bs, head_num, seq_len, dim_head] - // kv_transpose_out_data [2, bs, head_num, seq_len, dim_head] - if (rotary_emb_dims != 0) { - auto *rotary_emb_data = rotary_tensor->data(); - const int *sequence_lengths_data = - encoder_remove_padding ? sequence_lengths->data() : nullptr; - rotary_qk(dev_ctx, - q_transpose_out_data, - kv_transpose_out_data, - q_transpose_out_data, - kv_transpose_out_data, - rotary_emb_data, - sequence_lengths_data, - rotary_emb_dims, - bsz, - num_head, - seq_len, - dim_head); - } - - phi::DenseTensor *tmp_padding_offset_tensor = - encoder_remove_padding ? &padding_offset_tensor : nullptr; - fmha_compute.ComputeForwardWithoutTranspose(cache_kv, - src_mask, - tmp_padding_offset_tensor, - &q_transpose_out, - &kv_transpose_out, - cache_kv_out, - &qk_out, - nullptr, - &softmax_out, - &attn_dropout_mask_out, - &attn_dropout_out, - &qktv_out, - &fmha_out, - token_num); + // step3. fmha + const phi::DenseTensor *cache_kv = + cache_kvs && cache_kvs.get().size() > 0 ? cache_kvs.get()[i] : nullptr; + phi::DenseTensor *cache_kv_out = cache_kv ? cache_kv_outs[i] : nullptr; + + if (time_step_t) { // generation decoder stage + // [2, batch_size, num_head, max_seq_len, head_size] + int max_seq_len = cache_kv->dims()[3]; + fmha(dev_ctx, + qkv_out, + *qkv_bias, + *src_mask_t, + seq_lengths_t, + rotary_tensor_t, + cache_kv_out, + &fmha_out, + bsz, + max_seq_len, + num_head, + dim_head, + time_step_t->data()[0], + rotary_emb_dims, + 1. / std::sqrt(dim_head)); + } else if (cache_kv_out) { // generation context stage + const phi::DenseTensor *pre_cache_kv_tensor = + pre_caches && pre_caches.get().size() > 0 ? pre_caches.get()[i] + : nullptr; + phi::DenseTensor *pre_cache_kv_out_tmp = + cache_offset > 0 ? &pre_cache_kv_out : nullptr; + phi::DenseTensor *src_mask_tmp = + cache_offset > 0 ? &src_mask_out : nullptr; + qkv_bias_add_transpose_split(dev_ctx, + q_transpose_out_data, + kv_transpose_out_data, + qkv_out_data, + qkv_bias->data(), + padding_offset_data, + token_num, + bsz, + num_head, + seq_len, + dim_head, + compute_bias); + // q_transpose_out_data [bs, head_num, seq_len, dim_head] + // kv_transpose_out_data [2, bs, head_num, seq_len, dim_head] + if (rotary_emb_dims != 0) { + auto *rotary_emb_data = rotary_tensor_t->data(); + const int *sequence_lengths_data = + encoder_remove_padding ? seq_lengths_t->data() : nullptr; + rotary_qk(dev_ctx, + q_transpose_out_data, + kv_transpose_out_data, + q_transpose_out_data, + kv_transpose_out_data, + rotary_emb_data, + sequence_lengths_data, + rotary_emb_dims, + bsz, + num_head, + seq_len, + dim_head); } -#ifdef _DEBUG_FUSED_MULTI_TRANSFORMER - VLOG(0) << "step3"; -#endif - if (pre_layer_norm) { - out_linear_compute.ComputeForward( - out_linear_weights[i], &fmha_out, nullptr, buf1, nullptr); - AllReduce(*buf1, ring_id, buf1->numel(), dev_ctx); + phi::DenseTensor *tmp_padding_offset_tensor = + encoder_remove_padding ? &padding_offset_tensor : nullptr; + fmha_compute.ComputeForwardWithoutTranspose(pre_cache_kv_tensor, + src_mask_t, + tmp_padding_offset_tensor, + &q_transpose_out, + &kv_transpose_out, + pre_cache_kv_out_tmp, + &qk_out, + src_mask_tmp, + &softmax_out, + &attn_dropout_mask_out, + &attn_dropout_out, + &qktv_out, + &fmha_out, + token_num); + const T *k_ptr = nullptr; + const T *v_ptr = nullptr; + + if (cache_offset > 0) { + // [2, bsz, num_head, cache_offset + seq_len, head_dim] + const T *kv_data = pre_cache_kv_out.data(); + k_ptr = kv_data; + int64_t k_size = bsz * num_head * (seq_len + cache_offset) * dim_head; + v_ptr = k_ptr + k_size; } else { - out_linear_compute.ComputeForward( - out_linear_weights[i], &fmha_out, nullptr, buf0, nullptr); - AllReduce(*buf0, ring_id, buf0->numel(), dev_ctx); + // [3, bsz, num_head, seq_len, head_dim] + int64_t k_size = bsz * seq_len * num_head * dim_head; + const T *q_ptr = q_transpose_out_data; + k_ptr = kv_transpose_out_data; + v_ptr = k_ptr + k_size; + } + + // [2, bsz, num_head, max_seq_len, head_dim] + int max_seq_len = cache_kv_out->dims()[3]; + T *cache_kv_data = cache_kv_out->data(); + int64_t cache_k_size = bsz * num_head * max_seq_len * dim_head; + + T *cache_k_ptr = cache_kv_data; + T *cache_v_ptr = cache_kv_data + cache_k_size; + + const int seq_len_tmp = seq_len + cache_offset; + write_cache_kv(dev_ctx, + cache_k_ptr, + cache_v_ptr, + k_ptr, + v_ptr, + bsz, + num_head, + seq_len_tmp, + max_seq_len, + dim_head); + } else { // not generation + // TODO(wangxi): can remove dropout in inference + qkv_bias_add_transpose_split(dev_ctx, + q_transpose_out_data, + kv_transpose_out_data, + qkv_out_data, + qkv_bias->data(), + padding_offset_data, + token_num, + bsz, + num_head, + seq_len, + dim_head, + compute_bias); + + // q_transpose_out_data [bs, head_num, seq_len, dim_head] + // kv_transpose_out_data [2, bs, head_num, seq_len, dim_head] + if (rotary_emb_dims != 0) { + auto *rotary_emb_data = rotary_tensor_t->data(); + const int *sequence_lengths_data = + encoder_remove_padding ? seq_lengths_t->data() : nullptr; + rotary_qk(dev_ctx, + q_transpose_out_data, + kv_transpose_out_data, + q_transpose_out_data, + kv_transpose_out_data, + rotary_emb_data, + sequence_lengths_data, + rotary_emb_dims, + bsz, + num_head, + seq_len, + dim_head); } + + phi::DenseTensor *tmp_padding_offset_tensor = + encoder_remove_padding ? &padding_offset_tensor : nullptr; + fmha_compute.ComputeForwardWithoutTranspose(cache_kv, + src_mask_t, + tmp_padding_offset_tensor, + &q_transpose_out, + &kv_transpose_out, + cache_kv_out, + &qk_out, + nullptr, + &softmax_out, + &attn_dropout_mask_out, + &attn_dropout_out, + &qktv_out, + &fmha_out, + token_num); + } #ifdef _DEBUG_FUSED_MULTI_TRANSFORMER - VLOG(0) << "step4"; + VLOG(0) << "step3"; #endif - // step5. ln(residual + dropout(input + bias)) - if (pre_layer_norm) { - auto *ln_scale_data = ffn_ln_scales[i]->data(); - auto *ln_bias_data = ffn_ln_biases[i]->data(); - auto *out_linear_bias_data = out_linear_biases[i]->data(); + if (pre_layer_norm) { + out_linear_compute.ComputeForward( + out_linear_weights[i], &fmha_out, nullptr, buf1, nullptr); + AllReduce(*buf1, ring_id, buf1->numel(), dev_ctx); + } else { + out_linear_compute.ComputeForward( + out_linear_weights[i], &fmha_out, nullptr, buf0, nullptr); + AllReduce(*buf0, ring_id, buf0->numel(), dev_ctx); + } +#ifdef _DEBUG_FUSED_MULTI_TRANSFORMER + VLOG(0) << "step4"; +#endif - // inplace - fused_dropout_layernorm_helper.LayernormResidualDropoutBias( - dev_ctx, - buf1->data(), - x_data, - out_linear_bias_data, - ln_scale_data, - ln_bias_data, - bias_dropout_residual_out_data, - dropout_mask_out_data, - buf1->data(), - ln_mean_data, - ln_var_data); - } else { - auto *ln_scale_data = ln_scales[i]->data(); - auto *ln_bias_data = ln_biases[i]->data(); - auto *out_linear_bias_data = out_linear_biases[i]->data(); - auto *residual_data = (i == 0 ? x_data : buf1->data()); - fused_dropout_layernorm_helper.LayernormResidualDropoutBias( - dev_ctx, - buf0->data(), - residual_data, - out_linear_bias_data, - ln_scale_data, - ln_bias_data, - buf0->data(), - dropout_mask_out_data, - buf1->data(), - ln_mean_data, - ln_var_data); - } + // step5. ln(residual + dropout(input + bias)) + if (pre_layer_norm) { + auto *ln_scale_data = ffn_ln_scales[i]->data(); + auto *ln_bias_data = ffn_ln_biases[i]->data(); + auto *out_linear_bias_data = out_linear_biases.get()[i]->data(); + + // inplace + fused_dropout_layernorm_helper.LayernormResidualDropoutBias( + dev_ctx, + buf1->data(), + x_data, + out_linear_bias_data, + ln_scale_data, + ln_bias_data, + bias_dropout_residual_out_data, + dropout_mask_out_data, + buf1->data(), + ln_mean_data, + ln_var_data); + } else { + auto *ln_scale_data = ln_scales[i]->data(); + auto *ln_bias_data = ln_biases[i]->data(); + auto *out_linear_bias_data = out_linear_biases.get()[i]->data(); + auto *residual_data = (i == 0 ? x_data : buf1->data()); + fused_dropout_layernorm_helper.LayernormResidualDropoutBias( + dev_ctx, + buf0->data(), + residual_data, + out_linear_bias_data, + ln_scale_data, + ln_bias_data, + buf0->data(), + dropout_mask_out_data, + buf1->data(), + ln_mean_data, + ln_var_data); + } #ifdef _DEBUG_FUSED_MULTI_TRANSFORMER - VLOG(0) << "step5"; + VLOG(0) << "step5"; #endif - // step6. ffn matmul1 - ffn1_cublas_linear.ComputeForward(buf1, - ffn1_weights[i], - ffn1_biases[i], - nullptr, - &ffn1_out, - act_method); + // step6. ffn matmul1 + ffn1_cublas_linear.ComputeForward(buf1, + ffn1_weights[i], + ffn1_biases.get()[i], + nullptr, + &ffn1_out, + act_method); #ifdef _DEBUG_FUSED_MULTI_TRANSFORMER - VLOG(0) << "step6"; + VLOG(0) << "step6"; #endif - // step7. ffn2 matmul - if (pre_layer_norm) { - ffn2_linear_compute.ComputeForward( - ffn2_weights[i], &ffn1_out, nullptr, buf1, nullptr); - } else { - ffn2_linear_compute.ComputeForward( - ffn2_weights[i], &ffn1_out, nullptr, buf0, nullptr); - } + // step7. ffn2 matmul + if (pre_layer_norm) { + ffn2_linear_compute.ComputeForward( + ffn2_weights[i], &ffn1_out, nullptr, buf1, nullptr); + } else { + ffn2_linear_compute.ComputeForward( + ffn2_weights[i], &ffn1_out, nullptr, buf0, nullptr); + } #ifdef _DEBUG_FUSED_MULTI_TRANSFORMER - VLOG(0) << "step7"; + VLOG(0) << "step7"; #endif - if (pre_layer_norm) { - AllReduce(*buf1, ring_id, buf1->numel(), dev_ctx); - } else { - AllReduce(*buf0, ring_id, buf0->numel(), dev_ctx); - } + if (pre_layer_norm) { + AllReduce(*buf1, ring_id, buf1->numel(), dev_ctx); + } else { + AllReduce(*buf0, ring_id, buf0->numel(), dev_ctx); + } #ifdef _DEBUG_FUSED_MULTI_TRANSFORMER - VLOG(0) << "step7.1"; + VLOG(0) << "step7.1"; #endif - // step8. layer norm + bias_add + residual - if (pre_layer_norm) { - // TODO(wangxi): remove dropout mask in inference - if (i < layers - 1) { - auto *ln_scale_data = ln_scales[i + 1]->data(); - auto *ln_bias_data = ln_biases[i + 1]->data(); - ffn2_fused_dropout_helper.LayernormResidualDropoutBias( - dev_ctx, - buf1->data(), - bias_dropout_residual_out_data, - ffn2_biases[i]->data(), - ln_scale_data, - ln_bias_data, - buf1->data(), - dropout_mask_out_data, - buf0->data(), - ln_mean_data, - ln_var_data); - } else { - ffn2_fused_dropout_helper.ResidualDropoutBias( - dev_ctx, - buf1->data(), - bias_dropout_residual_out_data, - ffn2_biases[i]->data(), - buf1->data(), - dropout_mask_out_data); - } - } else { - auto *ln_scale_data = ffn_ln_scales[i]->data(); - auto *ln_bias_data = ffn_ln_biases[i]->data(); + // step8. layer norm + bias_add + residual + if (pre_layer_norm) { + // TODO(wangxi): remove dropout mask in inference + if (i < layers - 1) { + auto *ln_scale_data = ln_scales[i + 1]->data(); + auto *ln_bias_data = ln_biases[i + 1]->data(); ffn2_fused_dropout_helper.LayernormResidualDropoutBias( dev_ctx, - buf0->data(), buf1->data(), - ffn2_biases[i]->data(), + bias_dropout_residual_out_data, + ffn2_biases.get()[i]->data(), ln_scale_data, ln_bias_data, - buf0->data(), - dropout_mask_out_data, buf1->data(), + dropout_mask_out_data, + buf0->data(), ln_mean_data, ln_var_data); + } else { + ffn2_fused_dropout_helper.ResidualDropoutBias( + dev_ctx, + buf1->data(), + bias_dropout_residual_out_data, + ffn2_biases.get()[i]->data(), + buf1->data(), + dropout_mask_out_data); } + } else { + auto *ln_scale_data = ffn_ln_scales[i]->data(); + auto *ln_bias_data = ffn_ln_biases[i]->data(); + ffn2_fused_dropout_helper.LayernormResidualDropoutBias( + dev_ctx, + buf0->data(), + buf1->data(), + ffn2_biases.get()[i]->data(), + ln_scale_data, + ln_bias_data, + buf0->data(), + dropout_mask_out_data, + buf1->data(), + ln_mean_data, + ln_var_data); + } #ifdef _DEBUG_FUSED_MULTI_TRANSFORMER - VLOG(0) << "step8"; + VLOG(0) << "step8"; #endif - if (pre_layer_norm) { - x_data = buf1->data(); - std::swap(buf0, buf1); - } + if (pre_layer_norm) { + x_data = buf1->data(); + std::swap(buf0, buf1); } - if (encoder_remove_padding) { - if (pre_layer_norm) { - InvokeRebuildPadding(dev_ctx, - from_data, - buf0->data(), - padding_offset_data, - token_num, - dim_embed); - } else { - InvokeRebuildPadding(dev_ctx, - from_data, - buf1->data(), - padding_offset_data, - token_num, - dim_embed); - } + } + if (encoder_remove_padding) { + if (pre_layer_norm) { + InvokeRebuildPadding(dev_ctx, + from_data, + buf0->data(), + padding_offset_data, + token_num, + dim_embed); + } else { + InvokeRebuildPadding(dev_ctx, + from_data, + buf1->data(), + padding_offset_data, + token_num, + dim_embed); } } -}; +} #else -template -class FusedMultiTransformerOpKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext &ctx) const override { - using U = LayerNormParamType; - auto &dev_ctx = ctx.cuda_device_context(); - - auto *time_step = ctx.Input("TimeStep"); - // 0. input - auto *input_x = ctx.Input("X"); - const auto input_x_dims = input_x->dims(); - int bsz = input_x_dims[0]; - int seq_len = input_x_dims[1]; - int dim_embed = input_x_dims[2]; - int bsz_seq = bsz * seq_len; - const std::string act_method = ctx.Attr("act_method"); - bool remove_padding = false; - auto *sequence_lengths = ctx.Input("SeqLengths"); - if (sequence_lengths) { - remove_padding = true; - } - phi::DenseTensor d_token_tensor; - phi::DenseTensor padding_offset_tensor; - phi::DenseTensor x_remove_padding; - bool encoder_remove_padding = (remove_padding && !time_step); - int token_num = 0; - - // remove padding in encoder - if (encoder_remove_padding) { - // just for encoder - d_token_tensor.Resize({{1}}); - auto *d_token_num = dev_ctx.Alloc( - &d_token_tensor, d_token_tensor.numel() * sizeof(int)); - // alloc the max size of padding_offset_tensor - padding_offset_tensor.Resize({{bsz_seq}}); - dev_ctx.Alloc(&padding_offset_tensor, - padding_offset_tensor.numel() * sizeof(int)); - InvokeGetPaddingOffset(dev_ctx, - &token_num, - d_token_num, - padding_offset_tensor.data(), - sequence_lengths->data(), - bsz, - seq_len); - padding_offset_tensor.Resize({{token_num}}); - x_remove_padding.Resize({{token_num, dim_embed}}); - dev_ctx.Alloc(&x_remove_padding, x_remove_padding.numel() * sizeof(T)); - InvokeRemovePadding(dev_ctx, - x_remove_padding.data(), - input_x->data(), - padding_offset_tensor.data(), - token_num, - dim_embed); - } else { - token_num = bsz_seq; - } - auto *padding_offset_data = - encoder_remove_padding ? padding_offset_tensor.data() : nullptr; - - // 1. layer norm - const auto pre_layer_norm = ctx.Attr("pre_layer_norm"); - const float epsilon = ctx.Attr("epsilon"); - auto ln_scales = ctx.MultiInput("LnScale"); - auto ln_biases = ctx.MultiInput("LnBias"); - - auto ln_compute = AttnLayerNorm(dev_ctx, epsilon, token_num, dim_embed); - phi::DenseTensor ln_mean, ln_var; - ln_mean.Resize({{token_num}}); - auto *ln_mean_data = - dev_ctx.Alloc(&ln_mean, ln_mean.numel() * sizeof(U)); - ln_var.Resize({{token_num}}); - auto *ln_var_data = dev_ctx.Alloc(&ln_var, ln_var.numel() * sizeof(U)); - - // 2. qkv - // x: qkv's input [batch_size, seq_len, dim_embed] - // y: qkv's weight: [3, num_head, dim_head, dim_embed] - auto qkv_weights = ctx.MultiInput("QKVW"); - auto qkv_biases = ctx.MultiInput("QKVBias"); - const bool trans_qkvw = ctx.Attr("trans_qkvw"); - const auto qkv_w_dims = qkv_weights[0]->dims(); - int num_head = trans_qkvw ? qkv_w_dims[1] : qkv_w_dims[2]; - int dim_head = trans_qkvw ? qkv_w_dims[2] : qkv_w_dims[3]; - int hidden_size = num_head * dim_head; - int output_size = 3 * hidden_size; - int input_size = dim_embed; - - bool compute_bias = qkv_biases.size() > 0 && time_step == nullptr; - // (transA, transB, compute_bias) = (false, trans_qkvw, false) - // Since we fused QKVBias into QKVBiasAddTransposeSplit kernel, here we - // set compute_bias as false. - auto qkv_compute = phi::fusion::AttnMatMul(dev_ctx, - false, - trans_qkvw, - token_num, - output_size, - input_size, - /*compute_bias=*/false); - - phi::DenseTensor qkv_out; - qkv_out.Resize({{token_num, 3, num_head, dim_head}}); - auto *qkv_out_data = - dev_ctx.Alloc(&qkv_out, qkv_out.numel() * sizeof(T)); - - // 2.1 rotary - auto *rotary_tensor = ctx.Input("RotaryPosEmb"); - const int rotary_emb_dims = ctx.Attr("rotary_emb_dims"); - - // 3. fmha - AttnDropoutParam attn_param( - true, "upscale_in_train", 0.0, true, true, 0, nullptr); - auto fmha_compute = - FMHARef(dev_ctx, bsz, seq_len, num_head, dim_head, attn_param); - auto *src_mask = ctx.Input("SrcMask"); - auto cache_kvs = ctx.MultiInput("CacheKV"); - auto cache_kv_outs = ctx.MultiOutput("CacheKVOut"); - // auto *time_step = ctx.Input("TimeStep"); - auto pre_caches = ctx.MultiInput("PreCaches"); - int cache_offset = 0; - if (pre_caches.size() > 0) { - cache_offset = pre_caches[0]->dims()[3]; +template +void FusedMultiTransformerKernel( + const Context &dev_ctx, + const DenseTensor &x, + const std::vector &ln_scales, + const std::vector &ln_biases, + const std::vector &qkv_weights, + const paddle::optional> &qkv_biases, + const paddle::optional> &cache_kvs, + const paddle::optional> &pre_caches, + const paddle::optional &rotary_tensor, + const paddle::optional &time_step, + const paddle::optional &seq_lengths, + const paddle::optional &src_mask, + const std::vector &out_linear_weights, + const paddle::optional> &out_linear_biases, + const std::vector &ffn_ln_scales, + const std::vector &ffn_ln_biases, + const std::vector &ffn1_weights, + const paddle::optional> &ffn1_biases, + const std::vector &ffn2_weights, + const paddle::optional> &ffn2_biases, + bool pre_layer_norm, + float epsilon, + float dropout_rate, + int rotary_emb_dims, + bool is_test, + const std::string &dropout_implementation, + const std::string &act_method, + bool trans_qkvw, + int ring_id, + std::vector cache_kv_outs, + DenseTensor *out) { + if (cache_kvs) { + for (size_t i = 0; i < cache_kv_outs.size(); i++) { + *(cache_kv_outs[i]) = *(cache_kvs.get()[i]); } + } + using U = phi::funcs::LayerNormParamType; + auto *rotary_tensor_t = rotary_tensor.get_ptr(); + auto *seq_lengths_t = seq_lengths.get_ptr(); + auto *src_mask_t = src_mask.get_ptr(); + auto *time_step_t = time_step.get_ptr(); + + // 0. input + const auto input_x_dims = x.dims(); + int bsz = input_x_dims[0]; + int seq_len = input_x_dims[1]; + int dim_embed = input_x_dims[2]; + int bsz_seq = bsz * seq_len; + bool remove_padding = false; + if (seq_lengths_t) { + remove_padding = true; + } + phi::DenseTensor d_token_tensor; + phi::DenseTensor padding_offset_tensor; + phi::DenseTensor x_remove_padding; + bool encoder_remove_padding = (remove_padding && !time_step_t); + int token_num = 0; + + // remove padding in encoder + if (encoder_remove_padding) { + // just for encoder + d_token_tensor.Resize({1}); + auto *d_token_num = dev_ctx.template Alloc( + &d_token_tensor, d_token_tensor.numel() * sizeof(int)); + // alloc the max size of padding_offset_tensor + padding_offset_tensor.Resize({bsz_seq}); + dev_ctx.template Alloc(&padding_offset_tensor, + padding_offset_tensor.numel() * sizeof(int)); + InvokeGetPaddingOffset(dev_ctx, + &token_num, + d_token_num, + padding_offset_tensor.data(), + seq_lengths_t->data(), + bsz, + seq_len); + padding_offset_tensor.Resize({token_num}); + x_remove_padding.Resize({token_num, dim_embed}); + dev_ctx.template Alloc(&x_remove_padding, + x_remove_padding.numel() * sizeof(T)); + InvokeRemovePadding(dev_ctx, + x_remove_padding.data(), + x.data(), + padding_offset_tensor.data(), + token_num, + dim_embed); + } else { + token_num = bsz_seq; + } + auto *padding_offset_data = + encoder_remove_padding ? padding_offset_tensor.data() : nullptr; + + // 1. layer norm + + auto ln_compute = AttnLayerNorm(dev_ctx, epsilon, token_num, dim_embed); + phi::DenseTensor ln_mean, ln_var; + ln_mean.Resize({token_num}); + auto *ln_mean_data = + dev_ctx.template Alloc(&ln_mean, ln_mean.numel() * sizeof(U)); + ln_var.Resize({token_num}); + auto *ln_var_data = + dev_ctx.template Alloc(&ln_var, ln_var.numel() * sizeof(U)); + + // 2. qkv + // x: qkv's input [batch_size, seq_len, dim_embed] + // y: qkv's weight: [3, num_head, dim_head, dim_embed] + const auto qkv_w_dims = qkv_weights[0]->dims(); + int num_head = trans_qkvw ? qkv_w_dims[1] : qkv_w_dims[2]; + int dim_head = trans_qkvw ? qkv_w_dims[2] : qkv_w_dims[3]; + int hidden_size = num_head * dim_head; + int output_size = 3 * hidden_size; + int input_size = dim_embed; + + bool compute_bias = + qkv_biases && !qkv_biases.get().empty() && time_step_t == nullptr; + // (transA, transB, compute_bias) = (false, trans_qkvw, false) + // Since we fused QKVBias into QKVBiasAddTransposeSplit kernel, here we + // set compute_bias as false. + auto qkv_compute = phi::fusion::AttnMatMul(dev_ctx, + false, + trans_qkvw, + token_num, + output_size, + input_size, + /*compute_bias=*/false); + + phi::DenseTensor qkv_out; + qkv_out.Resize({token_num, 3, num_head, dim_head}); + auto *qkv_out_data = + dev_ctx.template Alloc(&qkv_out, qkv_out.numel() * sizeof(T)); + + // 3. fmha + AttnDropoutParam attn_param( + true, "upscale_in_train", 0.0, true, true, 0, nullptr); + auto fmha_compute = + FMHARef(dev_ctx, bsz, seq_len, num_head, dim_head, attn_param); + int cache_offset = 0; + if (pre_caches && pre_caches.get().size() > 0) { + cache_offset = pre_caches.get()[0]->dims()[3]; + } - auto out_seq_len = seq_len; - if (time_step) { - PADDLE_ENFORCE_EQ(time_step->place(), - platform::CPUPlace(), - platform::errors::PreconditionNotMet( - "The place of input(TimeStep) must be CPUPlace.")); - // cache_seq_len - int time_step_value = time_step->data()[0]; - PADDLE_ENFORCE_GT(time_step_value, - 0, - platform::errors::PreconditionNotMet( - "The value of time_step must > 0, but now is %d", - time_step_value)); - PADDLE_ENFORCE_EQ( - seq_len, - 1, - platform::errors::PreconditionNotMet( - "In decode stage, the seq_len of input must be 1, but now is %d", - seq_len)); - out_seq_len += time_step_value; - } else { - out_seq_len += cache_offset; - } + auto out_seq_len = seq_len; + if (time_step_t) { + PADDLE_ENFORCE_EQ(time_step_t->place(), + phi::CPUPlace(), + phi::errors::PreconditionNotMet( + "The place of input(TimeStep) must be CPUPlace.")); + // cache_seq_len + int time_step_value = time_step_t->data()[0]; + PADDLE_ENFORCE_GT(time_step_value, + 0, + phi::errors::PreconditionNotMet( + "The value of time_step_t must > 0, but now is %d", + time_step_value)); + PADDLE_ENFORCE_EQ( + seq_len, + 1, + phi::errors::PreconditionNotMet( + "In decode stage, the seq_len of input must be 1, but now is %d", + seq_len)); + out_seq_len += time_step_value; + } else { + out_seq_len += cache_offset; + } - phi::DenseTensor q_transpose_out, kv_transpose_out, qk_out; - q_transpose_out.Resize({{bsz, num_head, seq_len, dim_head}}); - auto *q_transpose_out_data = - dev_ctx.Alloc(&q_transpose_out, q_transpose_out.numel() * sizeof(T)); + phi::DenseTensor q_transpose_out, kv_transpose_out, qk_out; + q_transpose_out.Resize({bsz, num_head, seq_len, dim_head}); + auto *q_transpose_out_data = dev_ctx.template Alloc( + &q_transpose_out, q_transpose_out.numel() * sizeof(T)); - kv_transpose_out.Resize({{2, bsz, num_head, seq_len, dim_head}}); - auto *kv_transpose_out_data = dev_ctx.Alloc( - &kv_transpose_out, kv_transpose_out.numel() * sizeof(T)); + kv_transpose_out.Resize({2, bsz, num_head, seq_len, dim_head}); + auto *kv_transpose_out_data = dev_ctx.template Alloc( + &kv_transpose_out, kv_transpose_out.numel() * sizeof(T)); - qk_out.Resize({{bsz, num_head, seq_len, out_seq_len}}); - auto *qk_out_data = dev_ctx.Alloc(&qk_out, qk_out.numel() * sizeof(T)); + qk_out.Resize({bsz, num_head, seq_len, out_seq_len}); + auto *qk_out_data = + dev_ctx.template Alloc(&qk_out, qk_out.numel() * sizeof(T)); - phi::DenseTensor src_mask_out; - if (cache_offset > 0) { - src_mask_out.Resize({{bsz, num_head, seq_len, out_seq_len}}); - auto *src_mask_out_data = - dev_ctx.Alloc(&src_mask_out, src_mask_out.numel() * sizeof(T)); - } + phi::DenseTensor src_mask_out; + if (cache_offset > 0) { + src_mask_out.Resize({bsz, num_head, seq_len, out_seq_len}); + auto *src_mask_out_data = dev_ctx.template Alloc( + &src_mask_out, src_mask_out.numel() * sizeof(T)); + } - // [2, bs, num_head, cache_seq_len + seq_len, head_dim] - phi::DenseTensor pre_cache_kv_out; - if (cache_offset > 0) { - pre_cache_kv_out.Resize( - {{2, bsz, num_head, seq_len + cache_offset, dim_head}}); - auto *pre_cache_kv_out_data = dev_ctx.Alloc( - &pre_cache_kv_out, pre_cache_kv_out.numel() * sizeof(T)); - } + // [2, bs, num_head, cache_seq_len + seq_len, head_dim] + phi::DenseTensor pre_cache_kv_out; + if (cache_offset > 0) { + pre_cache_kv_out.Resize( + {{2, bsz, num_head, seq_len + cache_offset, dim_head}}); + auto *pre_cache_kv_out_data = dev_ctx.template Alloc( + &pre_cache_kv_out, pre_cache_kv_out.numel() * sizeof(T)); + } - phi::DenseTensor softmax_out; - phi::DenseTensor attn_dropout_mask_out, attn_dropout_out; - phi::DenseTensor qktv_out, fmha_out; - softmax_out.Resize({{bsz, num_head, seq_len, out_seq_len}}); - auto *softmax_out_data = - dev_ctx.Alloc(&softmax_out, softmax_out.numel() * sizeof(T)); - - attn_dropout_mask_out.Resize({{bsz, num_head, seq_len, out_seq_len}}); - auto *attn_dropout_mask_out_data = dev_ctx.Alloc( - &attn_dropout_mask_out, attn_dropout_mask_out.numel() * sizeof(T)); - attn_dropout_out.Resize({{bsz, num_head, seq_len, out_seq_len}}); - auto *attn_dropout_data_data = dev_ctx.Alloc( - &attn_dropout_out, attn_dropout_out.numel() * sizeof(T)); - - qktv_out.Resize({{bsz, num_head, seq_len, dim_head}}); - auto *qktv_out_data = - dev_ctx.Alloc(&qktv_out, qktv_out.numel() * sizeof(T)); - fmha_out.Resize({{bsz, seq_len, num_head, dim_head}}); - auto *fmha_out_data = - dev_ctx.Alloc(&fmha_out, fmha_out.numel() * sizeof(T)); - - // 4. out_linear - auto out_linear_weights = ctx.MultiInput("OutLinearW"); - auto out_linear_biases = ctx.MultiInput("OutLinearBias"); - int ring_id = ctx.Attr("ring_id"); - // (transA, transB, compute_bias) = (false, false, false) - auto out_linear_compute = phi::fusion::AttnMatMul( - dev_ctx, false, false, token_num, dim_embed, hidden_size, false); - - // 5. ln(residual + bias) - DropoutParam dropout_param2(true, 0, true, true, 0.0, nullptr, 0); - FusedDropoutLayerNormHelper fused_dropout_layernorm_helper( - dev_ctx, token_num, dim_embed, dropout_param2, epsilon); - auto ffn_ln_scales = ctx.MultiInput("FFNLnScale"); - auto ffn_ln_biases = ctx.MultiInput("FFNLnBias"); - phi::DenseTensor bias_dropout_residual_out, dropout_mask_out; - T *bias_dropout_residual_out_data = nullptr; + phi::DenseTensor softmax_out; + phi::DenseTensor attn_dropout_mask_out, attn_dropout_out; + phi::DenseTensor qktv_out, fmha_out; + softmax_out.Resize({bsz, num_head, seq_len, out_seq_len}); + auto *softmax_out_data = + dev_ctx.template Alloc(&softmax_out, softmax_out.numel() * sizeof(T)); + + attn_dropout_mask_out.Resize({bsz, num_head, seq_len, out_seq_len}); + auto *attn_dropout_mask_out_data = dev_ctx.template Alloc( + &attn_dropout_mask_out, attn_dropout_mask_out.numel() * sizeof(T)); + attn_dropout_out.Resize({bsz, num_head, seq_len, out_seq_len}); + auto *attn_dropout_data_data = dev_ctx.template Alloc( + &attn_dropout_out, attn_dropout_out.numel() * sizeof(T)); + + qktv_out.Resize({bsz, num_head, seq_len, dim_head}); + auto *qktv_out_data = + dev_ctx.template Alloc(&qktv_out, qktv_out.numel() * sizeof(T)); + fmha_out.Resize({bsz, seq_len, num_head, dim_head}); + auto *fmha_out_data = + dev_ctx.template Alloc(&fmha_out, fmha_out.numel() * sizeof(T)); + + // 4. out_linear + // (transA, transB, compute_bias) = (false, false, false) + auto out_linear_compute = phi::fusion::AttnMatMul( + dev_ctx, false, false, token_num, dim_embed, hidden_size, false); + + // 5. ln(residual + bias) + DropoutParam dropout_param2(true, 0, true, true, 0.0, nullptr, 0); + FusedDropoutLayerNormHelper fused_dropout_layernorm_helper( + dev_ctx, token_num, dim_embed, dropout_param2, epsilon); + phi::DenseTensor bias_dropout_residual_out, dropout_mask_out; + T *bias_dropout_residual_out_data = nullptr; + if (pre_layer_norm) { + bias_dropout_residual_out.Resize({token_num, dim_embed}); + bias_dropout_residual_out_data = dev_ctx.template Alloc( + &bias_dropout_residual_out, + bias_dropout_residual_out.numel() * sizeof(T)); + } + dropout_mask_out.Resize({token_num, dim_embed}); + auto *dropout_mask_out_data = dev_ctx.template Alloc( + &dropout_mask_out, dropout_mask_out.numel() * sizeof(uint8_t)); + + // 6. ffn matmul1 + auto ffn1_weight_dim = ffn1_weights[0]->dims(); + + int dim_ffn = ffn1_weight_dim[1]; + auto ffn1_linear_compute = phi::fusion::AttnMatMul( + dev_ctx, false, false, token_num, dim_ffn, dim_embed, false); + phi::DenseTensor ffn1_out; + ffn1_out.Resize({token_num, dim_ffn}); + auto *ffn1_out_data = + dev_ctx.template Alloc(&ffn1_out, ffn1_out.numel() * sizeof(T)); + + // 7. ffn act + bias + DropoutParam ffn1_dropout_param(true, 0, true, true, 0.0, nullptr, 0); + FusedDropoutHelper fused_act_dropout_helper( + dev_ctx, token_num, dim_ffn, ffn1_dropout_param); + phi::DenseTensor ffn1_dropout_out, ffn1_dropout_mask; + ffn1_dropout_out.Resize({token_num, dim_ffn}); + auto *ffn1_dropout_out_data = dev_ctx.template Alloc( + &ffn1_dropout_out, ffn1_dropout_out.numel() * sizeof(T)); + ffn1_dropout_mask.Resize({token_num, dim_ffn}); + auto *ffn1_dropout_mask_data = dev_ctx.template Alloc( + &ffn1_dropout_mask, ffn1_dropout_mask.numel() * sizeof(uint8_t)); + + // 8. ffn2 matmul + auto ffn2_linear_compute = phi::fusion::AttnMatMul( + dev_ctx, false, false, token_num, dim_embed, dim_ffn, false); + + // 9. ffn2 residual bias + DropoutParam ffn2_dropout_param(true, 0, true, true, 0.0, nullptr, 0); + FusedDropoutLayerNormHelper ffn2_fused_dropout_helper( + dev_ctx, token_num, dim_embed, ffn2_dropout_param, epsilon); + + // calc + auto *from_data = dev_ctx.template Alloc(out, out->numel() * sizeof(T)); + phi::DenseTensor *from_tensor = out; + phi::DenseTensor tmp_out, tmp_out_rm_padding; + tmp_out.Resize({token_num, dim_embed}); + if (encoder_remove_padding) { + tmp_out_rm_padding.Resize({token_num, dim_embed}); + auto *tmp_out_rm_padding_data = dev_ctx.template Alloc( + &tmp_out_rm_padding, tmp_out_rm_padding.numel() * sizeof(T)); + } + auto *tmp_out_data = + dev_ctx.template Alloc(&tmp_out, tmp_out.numel() * sizeof(T)); + + const T *x_data; + if (encoder_remove_padding) { + x_data = x_remove_padding.data(); + } else { + x_data = x.data(); + } + phi::DenseTensor *buf0 = nullptr; + phi::DenseTensor *buf1 = nullptr; + + // step0: x --> buf1 + // step1: buf1 --> buf0 + // step2: buf0 --> buf1 + int layers = qkv_weights.size(); + if (encoder_remove_padding) { + // In the case of variable lengths, the padding needs to be rebuilt + // eventually. So buf0 and buf1 do not need to be changed according to the + // pre_layer_norm and the number of layers. + buf0 = &tmp_out; + buf1 = &tmp_out_rm_padding; + } else { if (pre_layer_norm) { - bias_dropout_residual_out.Resize({{token_num, dim_embed}}); - bias_dropout_residual_out_data = - dev_ctx.Alloc(&bias_dropout_residual_out, - bias_dropout_residual_out.numel() * sizeof(T)); - } - dropout_mask_out.Resize({{token_num, dim_embed}}); - auto *dropout_mask_out_data = dev_ctx.Alloc( - &dropout_mask_out, dropout_mask_out.numel() * sizeof(uint8_t)); - - // 6. ffn matmul1 - auto ffn1_weights = ctx.MultiInput("FFN1Weight"); - auto ffn1_biases = ctx.MultiInput("FFN1Bias"); - auto ffn1_weight_dim = ffn1_weights[0]->dims(); - - int dim_ffn = ffn1_weight_dim[1]; - auto ffn1_linear_compute = phi::fusion::AttnMatMul( - dev_ctx, false, false, token_num, dim_ffn, dim_embed, false); - phi::DenseTensor ffn1_out; - ffn1_out.Resize({{token_num, dim_ffn}}); - auto *ffn1_out_data = - dev_ctx.Alloc(&ffn1_out, ffn1_out.numel() * sizeof(T)); - - // 7. ffn act + bias - DropoutParam ffn1_dropout_param(true, 0, true, true, 0.0, nullptr, 0); - FusedDropoutHelper fused_act_dropout_helper( - dev_ctx, token_num, dim_ffn, ffn1_dropout_param); - phi::DenseTensor ffn1_dropout_out, ffn1_dropout_mask; - ffn1_dropout_out.Resize({{token_num, dim_ffn}}); - auto *ffn1_dropout_out_data = dev_ctx.Alloc( - &ffn1_dropout_out, ffn1_dropout_out.numel() * sizeof(T)); - ffn1_dropout_mask.Resize({{token_num, dim_ffn}}); - auto *ffn1_dropout_mask_data = dev_ctx.Alloc( - &ffn1_dropout_mask, ffn1_dropout_mask.numel() * sizeof(uint8_t)); - - // 8. ffn2 matmul - auto ffn2_weights = ctx.MultiInput("FFN2Weight"); - auto ffn2_biases = ctx.MultiInput("FFN2Bias"); - auto ffn2_linear_compute = phi::fusion::AttnMatMul( - dev_ctx, false, false, token_num, dim_embed, dim_ffn, false); - - // 9. ffn2 residual bias - DropoutParam ffn2_dropout_param(true, 0, true, true, 0.0, nullptr, 0); - FusedDropoutLayerNormHelper ffn2_fused_dropout_helper( - dev_ctx, token_num, dim_embed, ffn2_dropout_param, epsilon); - - // calc - auto *out = ctx.Output("Out"); - auto *from_data = dev_ctx.Alloc(out, out->numel() * sizeof(T)); - phi::DenseTensor *from_tensor = out; - phi::DenseTensor tmp_out, tmp_out_rm_padding; - tmp_out.Resize({{token_num, dim_embed}}); - if (encoder_remove_padding) { - tmp_out_rm_padding.Resize({{token_num, dim_embed}}); - auto *tmp_out_rm_padding_data = dev_ctx.Alloc( - &tmp_out_rm_padding, tmp_out_rm_padding.numel() * sizeof(T)); - } - auto *tmp_out_data = - dev_ctx.Alloc(&tmp_out, tmp_out.numel() * sizeof(T)); - - const T *x_data; - if (encoder_remove_padding) { - x_data = x_remove_padding.data(); - } else { - x_data = input_x->data(); - } - phi::DenseTensor *buf0 = nullptr; - phi::DenseTensor *buf1 = nullptr; - - // step0: x --> buf1 - // step1: buf1 --> buf0 - // step2: buf0 --> buf1 - int layers = qkv_weights.size(); - if (encoder_remove_padding) { - // In the case of variable lengths, the padding needs to be rebuilt - // eventually. So buf0 and buf1 do not need to be changed according to the - // pre_layer_norm and the number of layers. - buf0 = &tmp_out; - buf1 = &tmp_out_rm_padding; - } else { - if (pre_layer_norm) { - if (layers & 1) { - // odd, set buf1 as out - buf0 = &tmp_out; - buf1 = out; - } else { - // even, set buf0 as out - buf0 = out; - buf1 = &tmp_out; - } - } else { + if (layers & 1) { + // odd, set buf1 as out buf0 = &tmp_out; buf1 = out; + } else { + // even, set buf0 as out + buf0 = out; + buf1 = &tmp_out; } + } else { + buf0 = &tmp_out; + buf1 = out; } + } - for (int i = 0; i < layers; ++i) { - // step1. layer_norm - if (i == 0 && pre_layer_norm) { - auto *ln_scale_data = ln_scales[i]->data(); - auto *ln_bias_data = ln_biases[i]->data(); - // TODO(wangxi): can remove mean var in inference - ln_compute.ComputeForward(x_data, - ln_scale_data, - ln_bias_data, - buf1->data(), - ln_mean_data, - ln_var_data); - } + for (int i = 0; i < layers; ++i) { + // step1. layer_norm + if (i == 0 && pre_layer_norm) { + auto *ln_scale_data = ln_scales[i]->data(); + auto *ln_bias_data = ln_biases[i]->data(); + // TODO(wangxi): can remove mean var in inference + ln_compute.ComputeForward(x_data, + ln_scale_data, + ln_bias_data, + buf1->data(), + ln_mean_data, + ln_var_data); + } #ifdef _DEBUG_FUSED_MULTI_TRANSFORMER - VLOG(0) << "step1"; + VLOG(0) << "step1"; #endif - // step2. qkv - const phi::DenseTensor *qkv_bias = - qkv_biases.size() > 0 ? qkv_biases[i] : nullptr; - // NOTE: in decoder stage, bias is fused in fmha - const phi::DenseTensor *bias = time_step ? nullptr : qkv_bias; - if (!pre_layer_norm && i == 0) { - const phi::DenseTensor *tmp_input_x = - (encoder_remove_padding) ? &x_remove_padding : input_x; - qkv_compute.ComputeForward( - qkv_weights[i], tmp_input_x, bias, &qkv_out, &qkv_out); - } else { - qkv_compute.ComputeForward( - qkv_weights[i], buf1, bias, &qkv_out, &qkv_out); - } + // step2. qkv + const phi::DenseTensor *qkv_bias = + qkv_biases && !qkv_biases.get().empty() ? qkv_biases.get()[i] : nullptr; + // NOTE: in decoder stage, bias is fused in fmha + const phi::DenseTensor *bias = time_step_t ? nullptr : qkv_bias; + if (!pre_layer_norm && i == 0) { + const phi::DenseTensor *tmp_input_x = + (encoder_remove_padding) ? &x_remove_padding : &x; + qkv_compute.ComputeForward( + qkv_weights[i], tmp_input_x, bias, &qkv_out, &qkv_out); + } else { + qkv_compute.ComputeForward( + qkv_weights[i], buf1, bias, &qkv_out, &qkv_out); + } #ifdef _DEBUG_FUSED_MULTI_TRANSFORMER - VLOG(0) << "step2"; + VLOG(0) << "step2"; #endif - // step3. fmha - const phi::DenseTensor *cache_kv = - cache_kvs.size() > 0 ? cache_kvs[i] : nullptr; - phi::DenseTensor *cache_kv_out = cache_kv ? cache_kv_outs[i] : nullptr; - - if (time_step) { // generation decoder stage - // [2, batch_size, num_head, max_seq_len, head_size] - int max_seq_len = cache_kv->dims()[3]; - fmha(dev_ctx, - qkv_out, - *qkv_bias, - *src_mask, - sequence_lengths, - rotary_tensor, - cache_kv_out, - &fmha_out, - bsz, - max_seq_len, - num_head, - dim_head, - time_step->data()[0], - rotary_emb_dims, - 1. / std::sqrt(dim_head)); - } else if (cache_kv_out) { // generation context stage - const phi::DenseTensor *pre_cache_kv_tensor = - pre_caches.size() > 0 ? pre_caches[i] : nullptr; - phi::DenseTensor *pre_cache_kv_out_tmp = - cache_offset > 0 ? &pre_cache_kv_out : nullptr; - phi::DenseTensor *src_mask_tmp = - cache_offset > 0 ? &src_mask_out : nullptr; - qkv_bias_add_transpose_split(dev_ctx, - q_transpose_out_data, - kv_transpose_out_data, - qkv_out_data, - qkv_bias->data(), - padding_offset_data, - token_num, - bsz, - num_head, - seq_len, - dim_head, - compute_bias); - - // q_transpose_out_data [bs, head_num, seq_len, dim_head] - // kv_transpose_out_data [2, bs, head_num, seq_len, dim_head] - if (rotary_emb_dims != 0) { - auto *rotary_emb_data = rotary_tensor->data(); - const int *sequence_lengths_data = - encoder_remove_padding ? sequence_lengths->data() : nullptr; - rotary_qk(dev_ctx, - q_transpose_out_data, - kv_transpose_out_data, - q_transpose_out_data, - kv_transpose_out_data, - rotary_emb_data, - sequence_lengths_data, - rotary_emb_dims, - bsz, - num_head, - seq_len, - dim_head); - } - - phi::DenseTensor *tmp_padding_offset_tensor = - encoder_remove_padding ? &padding_offset_tensor : nullptr; - fmha_compute.ComputeForwardWithoutTranspose(pre_cache_kv_tensor, - src_mask, - tmp_padding_offset_tensor, - &q_transpose_out, - &kv_transpose_out, - pre_cache_kv_out_tmp, - &qk_out, - src_mask_tmp, - &softmax_out, - &attn_dropout_mask_out, - &attn_dropout_out, - &qktv_out, - &fmha_out, - token_num); - const T *k_ptr = nullptr; - const T *v_ptr = nullptr; - - if (cache_offset > 0) { - // [2, bsz, num_head, cache_offset + seq_len, head_dim] - const T *kv_data = pre_cache_kv_out.data(); - k_ptr = kv_data; - int64_t k_size = bsz * num_head * (seq_len + cache_offset) * dim_head; - v_ptr = k_ptr + k_size; - } else { - // [3, bsz, num_head, seq_len, head_dim] - int64_t k_size = bsz * seq_len * num_head * dim_head; - const T *q_ptr = q_transpose_out_data; - k_ptr = kv_transpose_out_data; - v_ptr = k_ptr + k_size; - } - - // [2, bsz, num_head, max_seq_len, head_dim] - int max_seq_len = cache_kv_out->dims()[3]; - T *cache_kv_data = cache_kv_out->data(); - int64_t cache_k_size = bsz * num_head * max_seq_len * dim_head; - - T *cache_k_ptr = cache_kv_data; - T *cache_v_ptr = cache_kv_data + cache_k_size; - - const int seq_len_tmp = seq_len + cache_offset; - write_cache_kv(dev_ctx, - cache_k_ptr, - cache_v_ptr, - k_ptr, - v_ptr, - bsz, - num_head, - seq_len_tmp, - max_seq_len, - dim_head); - } else { // not generation - // TODO(wangxi): can remove dropout in inference - qkv_bias_add_transpose_split(dev_ctx, - q_transpose_out_data, - kv_transpose_out_data, - qkv_out_data, - qkv_bias->data(), - padding_offset_data, - token_num, - bsz, - num_head, - seq_len, - dim_head, - compute_bias); - - // q_transpose_out_data [bs, head_num, seq_len, dim_head] - // kv_transpose_out_data [2, bs, head_num, seq_len, dim_head] - if (rotary_emb_dims != 0) { - auto *rotary_emb_data = rotary_tensor->data(); - const int *sequence_lengths_data = - encoder_remove_padding ? sequence_lengths->data() : nullptr; - rotary_qk(dev_ctx, - q_transpose_out_data, - kv_transpose_out_data, - q_transpose_out_data, - kv_transpose_out_data, - rotary_emb_data, - sequence_lengths_data, - rotary_emb_dims, - bsz, - num_head, - seq_len, - dim_head); - } - - phi::DenseTensor *tmp_padding_offset_tensor = - encoder_remove_padding ? &padding_offset_tensor : nullptr; - fmha_compute.ComputeForwardWithoutTranspose(cache_kv, - src_mask, - tmp_padding_offset_tensor, - &q_transpose_out, - &kv_transpose_out, - cache_kv_out, - &qk_out, - nullptr, - &softmax_out, - &attn_dropout_mask_out, - &attn_dropout_out, - &qktv_out, - &fmha_out, - token_num); + // step3. fmha + const phi::DenseTensor *cache_kv = + cache_kvs && cache_kvs.get().size() > 0 ? cache_kvs.get()[i] : nullptr; + phi::DenseTensor *cache_kv_out = cache_kv ? cache_kv_outs[i] : nullptr; + + if (time_step_t) { // generation decoder stage + // [2, batch_size, num_head, max_seq_len, head_size] + int max_seq_len = cache_kv->dims()[3]; + fmha(dev_ctx, + qkv_out, + *qkv_bias, + *src_mask_t, + seq_lengths_t, + rotary_tensor_t, + cache_kv_out, + &fmha_out, + bsz, + max_seq_len, + num_head, + dim_head, + time_step_t->data()[0], + rotary_emb_dims, + 1. / std::sqrt(dim_head)); + } else if (cache_kv_out) { // generation context stage + const phi::DenseTensor *pre_cache_kv_tensor = + pre_caches && pre_caches.get().size() > 0 ? pre_caches.get()[i] + : nullptr; + phi::DenseTensor *pre_cache_kv_out_tmp = + cache_offset > 0 ? &pre_cache_kv_out : nullptr; + phi::DenseTensor *src_mask_tmp = + cache_offset > 0 ? &src_mask_out : nullptr; + qkv_bias_add_transpose_split(dev_ctx, + q_transpose_out_data, + kv_transpose_out_data, + qkv_out_data, + qkv_bias->data(), + padding_offset_data, + token_num, + bsz, + num_head, + seq_len, + dim_head, + compute_bias); + + // q_transpose_out_data [bs, head_num, seq_len, dim_head] + // kv_transpose_out_data [2, bs, head_num, seq_len, dim_head] + if (rotary_emb_dims != 0) { + auto *rotary_emb_data = rotary_tensor_t->data(); + const int *sequence_lengths_data = + encoder_remove_padding ? seq_lengths_t->data() : nullptr; + rotary_qk(dev_ctx, + q_transpose_out_data, + kv_transpose_out_data, + q_transpose_out_data, + kv_transpose_out_data, + rotary_emb_data, + sequence_lengths_data, + rotary_emb_dims, + bsz, + num_head, + seq_len, + dim_head); } -#ifdef _DEBUG_FUSED_MULTI_TRANSFORMER - VLOG(0) << "step3"; -#endif - if (pre_layer_norm) { - out_linear_compute.ComputeForward( - out_linear_weights[i], &fmha_out, nullptr, buf1, nullptr); - AllReduce(*buf1, ring_id, buf1->numel(), dev_ctx); + phi::DenseTensor *tmp_padding_offset_tensor = + encoder_remove_padding ? &padding_offset_tensor : nullptr; + fmha_compute.ComputeForwardWithoutTranspose(pre_cache_kv_tensor, + src_mask_t, + tmp_padding_offset_tensor, + &q_transpose_out, + &kv_transpose_out, + pre_cache_kv_out_tmp, + &qk_out, + src_mask_tmp, + &softmax_out, + &attn_dropout_mask_out, + &attn_dropout_out, + &qktv_out, + &fmha_out, + token_num); + const T *k_ptr = nullptr; + const T *v_ptr = nullptr; + if (cache_offset > 0) { + // [2, bsz, num_head, cache_offset + seq_len, head_dim] + const T *kv_data = pre_cache_kv_out.data(); + k_ptr = kv_data; + int64_t k_size = bsz * num_head * (seq_len + cache_offset) * dim_head; + v_ptr = k_ptr + k_size; } else { - out_linear_compute.ComputeForward( - out_linear_weights[i], &fmha_out, nullptr, buf0, nullptr); - AllReduce(*buf0, ring_id, buf0->numel(), dev_ctx); + // [3, bsz, num_head, seq_len, head_dim] + int64_t k_size = bsz * seq_len * num_head * dim_head; + const T *q_ptr = q_transpose_out_data; + k_ptr = kv_transpose_out_data; + v_ptr = k_ptr + k_size; } + + // [2, bsz, num_head, max_seq_len, head_dim] + int max_seq_len = cache_kv_out->dims()[3]; + T *cache_kv_data = cache_kv_out->data(); + int64_t cache_k_size = bsz * num_head * max_seq_len * dim_head; + + T *cache_k_ptr = cache_kv_data; + T *cache_v_ptr = cache_kv_data + cache_k_size; + const int seq_len_tmp = seq_len + cache_offset; + write_cache_kv(dev_ctx, + cache_k_ptr, + cache_v_ptr, + k_ptr, + v_ptr, + bsz, + num_head, + seq_len_tmp, + max_seq_len, + dim_head); + } else { // not generation + // TODO(wangxi): can remove dropout in inference + qkv_bias_add_transpose_split(dev_ctx, + q_transpose_out_data, + kv_transpose_out_data, + qkv_out_data, + qkv_bias->data(), + padding_offset_data, + token_num, + bsz, + num_head, + seq_len, + dim_head, + compute_bias); + + // q_transpose_out_data [bs, head_num, seq_len, dim_head] + // kv_transpose_out_data [2, bs, head_num, seq_len, dim_head] + if (rotary_emb_dims != 0) { + auto *rotary_emb_data = rotary_tensor_t->data(); + const int *sequence_lengths_data = + encoder_remove_padding ? seq_lengths_t->data() : nullptr; + rotary_qk(dev_ctx, + q_transpose_out_data, + kv_transpose_out_data, + q_transpose_out_data, + kv_transpose_out_data, + rotary_emb_data, + sequence_lengths_data, + rotary_emb_dims, + bsz, + num_head, + seq_len, + dim_head); + } + + phi::DenseTensor *tmp_padding_offset_tensor = + encoder_remove_padding ? &padding_offset_tensor : nullptr; + fmha_compute.ComputeForwardWithoutTranspose(cache_kv, + src_mask_t, + tmp_padding_offset_tensor, + &q_transpose_out, + &kv_transpose_out, + cache_kv_out, + &qk_out, + nullptr, + &softmax_out, + &attn_dropout_mask_out, + &attn_dropout_out, + &qktv_out, + &fmha_out, + token_num); + } #ifdef _DEBUG_FUSED_MULTI_TRANSFORMER - VLOG(0) << "step4"; + VLOG(0) << "step3"; #endif - // step5. ln(residual + dropout(input + bias)) - if (pre_layer_norm) { - auto *ln_scale_data = ffn_ln_scales[i]->data(); - auto *ln_bias_data = ffn_ln_biases[i]->data(); - auto *out_linear_bias_data = out_linear_biases[i]->data(); + if (pre_layer_norm) { + out_linear_compute.ComputeForward( + out_linear_weights[i], &fmha_out, nullptr, buf1, nullptr); + AllReduce(*buf1, ring_id, buf1->numel(), dev_ctx); + } else { + out_linear_compute.ComputeForward( + out_linear_weights[i], &fmha_out, nullptr, buf0, nullptr); + AllReduce(*buf0, ring_id, buf0->numel(), dev_ctx); + } +#ifdef _DEBUG_FUSED_MULTI_TRANSFORMER + VLOG(0) << "step4"; +#endif - // inplace - fused_dropout_layernorm_helper.LayernormResidualDropoutBias( - dev_ctx, - buf1->data(), - x_data, - out_linear_bias_data, - ln_scale_data, - ln_bias_data, - bias_dropout_residual_out_data, - dropout_mask_out_data, - buf1->data(), - ln_mean_data, - ln_var_data); - } else { - auto *ln_scale_data = ln_scales[i]->data(); - auto *ln_bias_data = ln_biases[i]->data(); - auto *out_linear_bias_data = out_linear_biases[i]->data(); - auto *residual_data = (i == 0 ? x_data : buf1->data()); - fused_dropout_layernorm_helper.LayernormResidualDropoutBias( - dev_ctx, - buf0->data(), - residual_data, - out_linear_bias_data, - ln_scale_data, - ln_bias_data, - buf0->data(), - dropout_mask_out_data, - buf1->data(), - ln_mean_data, - ln_var_data); - } + // step5. ln(residual + dropout(input + bias)) + if (pre_layer_norm) { + auto *ln_scale_data = ffn_ln_scales[i]->data(); + auto *ln_bias_data = ffn_ln_biases[i]->data(); + auto *out_linear_bias_data = out_linear_biases.get()[i]->data(); + + // inplace + fused_dropout_layernorm_helper.LayernormResidualDropoutBias( + dev_ctx, + buf1->data(), + x_data, + out_linear_bias_data, + ln_scale_data, + ln_bias_data, + bias_dropout_residual_out_data, + dropout_mask_out_data, + buf1->data(), + ln_mean_data, + ln_var_data); + } else { + auto *ln_scale_data = ln_scales[i]->data(); + auto *ln_bias_data = ln_biases[i]->data(); + auto *out_linear_bias_data = out_linear_biases.get()[i]->data(); + auto *residual_data = (i == 0 ? x_data : buf1->data()); + fused_dropout_layernorm_helper.LayernormResidualDropoutBias( + dev_ctx, + buf0->data(), + residual_data, + out_linear_bias_data, + ln_scale_data, + ln_bias_data, + buf0->data(), + dropout_mask_out_data, + buf1->data(), + ln_mean_data, + ln_var_data); + } #ifdef _DEBUG_FUSED_MULTI_TRANSFORMER - VLOG(0) << "step5"; + VLOG(0) << "step5"; #endif - // step6. ffn matmul1 - ffn1_linear_compute.ComputeForward( - ffn1_weights[i], buf1, nullptr, &ffn1_out, nullptr); + // step6. ffn matmul1 + ffn1_linear_compute.ComputeForward( + ffn1_weights[i], buf1, nullptr, &ffn1_out, nullptr); #ifdef _DEBUG_FUSED_MULTI_TRANSFORMER - VLOG(0) << "step6"; + VLOG(0) << "step6"; #endif - // step7. act bias - // TODO(wangxi): remove dropout mask in inference - fused_act_dropout_helper.DropoutActBias(dev_ctx, - ffn1_out_data, - ffn1_biases[i]->data(), - act_method, - ffn1_dropout_out_data, - ffn1_dropout_mask_data); + // step7. act bias + // TODO(wangxi): remove dropout mask in inference + fused_act_dropout_helper.DropoutActBias(dev_ctx, + ffn1_out_data, + ffn1_biases.get()[i]->data(), + act_method, + ffn1_dropout_out_data, + ffn1_dropout_mask_data); #ifdef _DEBUG_FUSED_MULTI_TRANSFORMER - VLOG(0) << "step7"; + VLOG(0) << "step7"; #endif - // step8. ffn matmul2 - if (pre_layer_norm) { - ffn2_linear_compute.ComputeForward( - ffn2_weights[i], &ffn1_dropout_out, nullptr, buf1, nullptr); - } else { - ffn2_linear_compute.ComputeForward( - ffn2_weights[i], &ffn1_dropout_out, nullptr, buf0, nullptr); - } + // step8. ffn matmul2 + if (pre_layer_norm) { + ffn2_linear_compute.ComputeForward( + ffn2_weights[i], &ffn1_dropout_out, nullptr, buf1, nullptr); + } else { + ffn2_linear_compute.ComputeForward( + ffn2_weights[i], &ffn1_dropout_out, nullptr, buf0, nullptr); + } #ifdef _DEBUG_FUSED_MULTI_TRANSFORMER - VLOG(0) << "step8.0"; + VLOG(0) << "step8.0"; #endif - if (pre_layer_norm) { - AllReduce(*buf1, ring_id, buf1->numel(), dev_ctx); - } else { - AllReduce(*buf0, ring_id, buf0->numel(), dev_ctx); - } + if (pre_layer_norm) { + AllReduce(*buf1, ring_id, buf1->numel(), dev_ctx); + } else { + AllReduce(*buf0, ring_id, buf0->numel(), dev_ctx); + } #ifdef _DEBUG_FUSED_MULTI_TRANSFORMER - VLOG(0) << "step8.1"; + VLOG(0) << "step8.1"; #endif - // step9. residual bias - if (pre_layer_norm) { - // TODO(wangxi): remove dropout mask in inference - if (i < layers - 1) { - auto *ln_scale_data = ln_scales[i + 1]->data(); - auto *ln_bias_data = ln_biases[i + 1]->data(); - ffn2_fused_dropout_helper.LayernormResidualDropoutBias( - dev_ctx, - buf1->data(), - bias_dropout_residual_out_data, - ffn2_biases[i]->data(), - ln_scale_data, - ln_bias_data, - buf1->data(), - dropout_mask_out_data, - buf0->data(), - ln_mean_data, - ln_var_data); - } else { - ffn2_fused_dropout_helper.ResidualDropoutBias( - dev_ctx, - buf1->data(), - bias_dropout_residual_out_data, - ffn2_biases[i]->data(), - buf1->data(), - dropout_mask_out_data); - } - } else { - auto *ln_scale_data = ffn_ln_scales[i]->data(); - auto *ln_bias_data = ffn_ln_biases[i]->data(); + // step9. residual bias + if (pre_layer_norm) { + // TODO(wangxi): remove dropout mask in inference + if (i < layers - 1) { + auto *ln_scale_data = ln_scales[i + 1]->data(); + auto *ln_bias_data = ln_biases[i + 1]->data(); ffn2_fused_dropout_helper.LayernormResidualDropoutBias( dev_ctx, - buf0->data(), buf1->data(), - ffn2_biases[i]->data(), + bias_dropout_residual_out_data, + ffn2_biases.get()[i]->data(), ln_scale_data, ln_bias_data, - buf0->data(), - dropout_mask_out_data, buf1->data(), + dropout_mask_out_data, + buf0->data(), ln_mean_data, ln_var_data); + } else { + ffn2_fused_dropout_helper.ResidualDropoutBias( + dev_ctx, + buf1->data(), + bias_dropout_residual_out_data, + ffn2_biases.get()[i]->data(), + buf1->data(), + dropout_mask_out_data); } + } else { + auto *ln_scale_data = ffn_ln_scales[i]->data(); + auto *ln_bias_data = ffn_ln_biases[i]->data(); + ffn2_fused_dropout_helper.LayernormResidualDropoutBias( + dev_ctx, + buf0->data(), + buf1->data(), + ffn2_biases.get()[i]->data(), + ln_scale_data, + ln_bias_data, + buf0->data(), + dropout_mask_out_data, + buf1->data(), + ln_mean_data, + ln_var_data); + } #ifdef _DEBUG_FUSED_MULTI_TRANSFORMER - VLOG(0) << "step9"; + VLOG(0) << "step9"; #endif - if (pre_layer_norm) { - x_data = buf1->data(); - std::swap(buf0, buf1); - } + if (pre_layer_norm) { + x_data = buf1->data(); + std::swap(buf0, buf1); } - if (encoder_remove_padding) { - if (pre_layer_norm) { - InvokeRebuildPadding(dev_ctx, - from_data, - buf0->data(), - padding_offset_data, - token_num, - dim_embed); - } else { - InvokeRebuildPadding(dev_ctx, - from_data, - buf1->data(), - padding_offset_data, - token_num, - dim_embed); - } + } + if (encoder_remove_padding) { + if (pre_layer_norm) { + InvokeRebuildPadding(dev_ctx, + from_data, + buf0->data(), + padding_offset_data, + token_num, + dim_embed); + } else { + InvokeRebuildPadding(dev_ctx, + from_data, + buf1->data(), + padding_offset_data, + token_num, + dim_embed); } } -}; - +} #endif // CUDA_VERSION >= 11060 -} // namespace operators -} // namespace paddle - -namespace ops = paddle::operators; -namespace plat = paddle::platform; -PD_REGISTER_STRUCT_KERNEL(fused_multi_transformer, - GPU, - ALL_LAYOUT, - ops::FusedMultiTransformerOpKernel, - float, - plat::float16) {} +} // namespace fusion +} // namespace phi + +PD_REGISTER_KERNEL(fused_multi_transformer, + GPU, + ALL_LAYOUT, + phi::fusion::FusedMultiTransformerKernel, + float, + phi::dtype::float16) { + kernel->InputAt(8).SetBackend(phi::Backend::CPU); +} diff --git a/paddle/fluid/operators/fused/fused_multi_transformer_op.cu.h b/paddle/fluid/operators/fused/fused_multi_transformer_op.cu.h index 0aff1cb5365fc..415a6ba1ffdf3 100644 --- a/paddle/fluid/operators/fused/fused_multi_transformer_op.cu.h +++ b/paddle/fluid/operators/fused/fused_multi_transformer_op.cu.h @@ -31,8 +31,8 @@ limitations under the License. */ #include "paddle/fluid/operators/fused/fmha_ref.h" #include "paddle/fluid/operators/fused/fused_dropout_helper.h" #include "paddle/fluid/platform/device/gpu/gpu_dnn.h" -#include "paddle/fluid/platform/dynload/cublasLt.h" #include "paddle/phi/api/include/tensor.h" +#include "paddle/phi/backends/dynload/cublasLt.h" #include "paddle/phi/backends/gpu/gpu_device_function.h" #include "paddle/phi/core/distributed/comm_context_manager.h" #include "paddle/phi/kernels/funcs/fused_gemm_epilogue.h" @@ -49,8 +49,8 @@ COMMON_DECLARE_bool(dynamic_static_unified_comm); COMMON_DECLARE_bool(gemm_use_half_precision_compute_type); -namespace paddle { -namespace operators { +namespace phi { +namespace fusion { // for debug // #define _DEBUG_FUSED_MULTI_TRANSFORMER @@ -75,14 +75,13 @@ static void AllReduce(phi::DenseTensor &tensor, // NOLINT auto task = pg->AllReduce(in_tensor, out_tensor, opts); task->Wait(); } else { - auto dtype = platform::ToNCCLDataType( - framework::TransToProtoVarType(tensor.dtype())); + auto dtype = phi::ToNCCLDataType(tensor.dtype()); int64_t numel = tensor.numel(); const void *sendbuff = tensor.data(); auto place = ctx.GetPlace(); void *recvbuff = tensor.mutable_data(place); gpuStream_t stream = nullptr; - platform::NCCLComm *comm = nullptr; + paddle::platform::NCCLComm *comm = nullptr; phi::distributed::NCCLCommContext *comm_ctx = nullptr; const auto &comm_context_manager = @@ -92,7 +91,7 @@ static void AllReduce(phi::DenseTensor &tensor, // NOLINT // Use New Communication Library PADDLE_ENFORCE_EQ(comm_context_manager.Has(std::to_string(ring_id)), true, - platform::errors::InvalidArgument( + phi::errors::InvalidArgument( "You choose to use new communication library by " "setting environment " "variable FLAGS_dynamic_static_unified_comm True. " @@ -103,7 +102,7 @@ static void AllReduce(phi::DenseTensor &tensor, // NOLINT comm_context_manager.Get(std::to_string(ring_id))); PADDLE_ENFORCE_NE(comm_ctx, nullptr, - platform::errors::Unavailable( + phi::errors::Unavailable( "NCCLCommContext is nullptr, collective op should " "has ring_id attr.")); @@ -111,20 +110,19 @@ static void AllReduce(phi::DenseTensor &tensor, // NOLINT VLOG(3) << "new comm_context_manager has ring_id" << ring_id; } else { - comm = platform::NCCLCommContext::Instance().Get(ring_id, place); - + comm = paddle::platform::NCCLCommContext::Instance().Get(ring_id, place); stream = ctx.stream(); VLOG(3) << "old NCCLCommContext has ring_id " << ring_id; } if (comm_ctx) { comm_ctx->AllReduce(&tensor, tensor, ncclSum, stream); } else { - PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::ncclAllReduce( + PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::ncclAllReduce( sendbuff, recvbuff, count, dtype, ncclSum, comm->comm(), stream)); } } #else - PADDLE_THROW(platform::errors::Unimplemented( + PADDLE_THROW(phi::errors::Unimplemented( "PaddlePaddle should compile with NCCL or RCCL when used tensor model " "parallel op.")); #endif @@ -1310,8 +1308,8 @@ void fmha(const phi::GPUContext &dev_ctx, fmha_launch_kernel(params, dev_ctx.stream()); break; default: - PADDLE_THROW(platform::errors::Unimplemented( - "Dim_head = %d is unsupport!", dim_head)); + PADDLE_THROW( + phi::errors::Unimplemented("Dim_head = %d is unsupport!", dim_head)); } } @@ -1431,7 +1429,7 @@ void write_cache_kv(const phi::GPUContext &dev_ctx, PADDLE_ENFORCE_EQ( dim_head % x, 0, - platform::errors::PreconditionNotMet( + phi::errors::PreconditionNotMet( "dim_head=%d must be divisible by vec_size=%d", dim_head, x)); int max_size = max_seq_len * dim_head / x; @@ -1548,7 +1546,7 @@ void qkv_bias_add_transpose_split(const phi::GPUContext &dev_ctx, constexpr int PackSize = VEC_16B / sizeof(T); PADDLE_ENFORCE_EQ(size_per_head % PackSize, 0, - platform::errors::PreconditionNotMet( + phi::errors::PreconditionNotMet( "dim_head=%d must be divisible by vec_size=%d", size_per_head, PackSize)); @@ -1711,12 +1709,12 @@ void InvokeGetPaddingOffset(const phi::GPUContext &dev_ctx, const int max_seq_len) { GetPaddingOffset<<<1, 1, 0, dev_ctx.stream()>>>( d_token_num, padding_offset, sequence_lengths, batch_size, max_seq_len); - memory::Copy(platform::CPUPlace(), - h_token_num, - dev_ctx.GetPlace(), - d_token_num, - sizeof(int), - dev_ctx.stream()); + phi::memory_utils::Copy(phi::CPUPlace(), + h_token_num, + dev_ctx.GetPlace(), + d_token_num, + sizeof(int), + dev_ctx.stream()); } template @@ -1785,7 +1783,7 @@ class CublasFusedMLP { cudaDataType_t mat_type = CUDA_R_32F; cudaDataType_t scale_type = CUDA_R_32F; cublasComputeType_t compute_type = CUBLAS_COMPUTE_32F; - if (std::is_same::value) { + if (std::is_same::value) { mat_type = CUDA_R_16F; if (FLAGS_gemm_use_half_precision_compute_type) { // This option default value is true, it tends to result NaN, but get @@ -1795,7 +1793,7 @@ class CublasFusedMLP { scale_type = CUDA_R_16F; } } - if (std::is_same::value) { + if (std::is_same::value) { mat_type = CUDA_R_16BF; } if (std::is_same::value) { @@ -1804,24 +1802,24 @@ class CublasFusedMLP { compute_type = CUBLAS_COMPUTE_64F; } - PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cublasLtMatmulDescCreate( + PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cublasLtMatmulDescCreate( &operation_desc_, compute_type, scale_type)); - PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cublasLtMatrixLayoutCreate( - &x_desc_, mat_type, 1, 1, 1)); - PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cublasLtMatrixLayoutCreate( - &w_desc_, mat_type, 1, 1, 1)); - PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cublasLtMatrixLayoutCreate( + PADDLE_ENFORCE_GPU_SUCCESS( + phi::dynload::cublasLtMatrixLayoutCreate(&x_desc_, mat_type, 1, 1, 1)); + PADDLE_ENFORCE_GPU_SUCCESS( + phi::dynload::cublasLtMatrixLayoutCreate(&w_desc_, mat_type, 1, 1, 1)); + PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cublasLtMatrixLayoutCreate( &out_desc_, mat_type, 1, 1, 1)); } ~CublasFusedMLP() { PADDLE_ENFORCE_GPU_SUCCESS( - platform::dynload::cublasLtMatmulDescDestroy(operation_desc_)); + phi::dynload::cublasLtMatmulDescDestroy(operation_desc_)); PADDLE_ENFORCE_GPU_SUCCESS( - platform::dynload::cublasLtMatrixLayoutDestroy(x_desc_)); + phi::dynload::cublasLtMatrixLayoutDestroy(x_desc_)); PADDLE_ENFORCE_GPU_SUCCESS( - platform::dynload::cublasLtMatrixLayoutDestroy(w_desc_)); + phi::dynload::cublasLtMatrixLayoutDestroy(w_desc_)); PADDLE_ENFORCE_GPU_SUCCESS( - platform::dynload::cublasLtMatrixLayoutDestroy(out_desc_)); + phi::dynload::cublasLtMatrixLayoutDestroy(out_desc_)); } void Setup(const phi::DDim &x_shape, @@ -1834,18 +1832,16 @@ class CublasFusedMLP { cublasOperation_t cublas_transA = trans_x ? CUBLAS_OP_T : CUBLAS_OP_N; cublasOperation_t cublas_transB = trans_w ? CUBLAS_OP_T : CUBLAS_OP_N; - PADDLE_ENFORCE_GPU_SUCCESS( - platform::dynload::cublasLtMatmulDescSetAttribute( - operation_desc_, - CUBLASLT_MATMUL_DESC_TRANSB, - &cublas_transA, - sizeof(cublas_transA))); - PADDLE_ENFORCE_GPU_SUCCESS( - platform::dynload::cublasLtMatmulDescSetAttribute( - operation_desc_, - CUBLASLT_MATMUL_DESC_TRANSA, - &cublas_transB, - sizeof(cublas_transB))); + PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cublasLtMatmulDescSetAttribute( + operation_desc_, + CUBLASLT_MATMUL_DESC_TRANSB, + &cublas_transA, + sizeof(cublas_transA))); + PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cublasLtMatmulDescSetAttribute( + operation_desc_, + CUBLASLT_MATMUL_DESC_TRANSA, + &cublas_transB, + sizeof(cublas_transB))); SetCublasMatrixLayout(x_desc_, trans_x, M, K); SetCublasMatrixLayout(w_desc_, trans_w, K, N); @@ -1867,27 +1863,25 @@ class CublasFusedMLP { if (add_bias) { bias_data = bias->data(); } - PADDLE_ENFORCE_GPU_SUCCESS( - platform::dynload::cublasLtMatmulDescSetAttribute( - operation_desc_, - CUBLASLT_MATMUL_DESC_BIAS_POINTER, - &bias_data, - sizeof(bias_data))); + PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cublasLtMatmulDescSetAttribute( + operation_desc_, + CUBLASLT_MATMUL_DESC_BIAS_POINTER, + &bias_data, + sizeof(bias_data))); cublasLtEpilogue_t epiloque_func = GetEpilogueType(activation, add_bias); - PADDLE_ENFORCE_GPU_SUCCESS( - platform::dynload::cublasLtMatmulDescSetAttribute( - operation_desc_, - CUBLASLT_MATMUL_DESC_EPILOGUE, - &epiloque_func, - sizeof(epiloque_func))); + PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cublasLtMatmulDescSetAttribute( + operation_desc_, + CUBLASLT_MATMUL_DESC_EPILOGUE, + &epiloque_func, + sizeof(epiloque_func))); T *residual_data = add_residual ? residual->data() : out_data; cublasLtHandle_t lt_handle = dev_ctx_.cublaslt_handle(); size_t workspace_size = static_cast(4) * 1024 * 1024; cudaStream_t stream = dev_ctx_.stream(); - memory::allocation::AllocationPtr workspace = memory::Alloc( + phi::Allocator::AllocationPtr workspace = phi::memory_utils::Alloc( dev_ctx_.GetPlace(), workspace_size, phi::Stream(reinterpret_cast(dev_ctx_.stream()))); @@ -1930,23 +1924,22 @@ class CublasFusedMLP { workspace->ptr(), workspace_size); - PADDLE_ENFORCE_GPU_SUCCESS( - platform::dynload::cublasLtMatmul(lt_handle, - operation_desc_, - alpha, - w_data, - w_desc_, - x_data, - x_desc_, - beta, - residual_data, - out_desc_, - out_data, - out_desc_, - algo, - workspace->ptr(), - workspace_size, - stream)); + PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cublasLtMatmul(lt_handle, + operation_desc_, + alpha, + w_data, + w_desc_, + x_data, + x_desc_, + beta, + residual_data, + out_desc_, + out_data, + out_desc_, + algo, + workspace->ptr(), + workspace_size, + stream)); } private: @@ -1974,7 +1967,7 @@ class CublasFusedMLP { PADDLE_ENFORCE_EQ( true, false, - platform::errors::InvalidArgument( + phi::errors::InvalidArgument( "The activation attribute of fused_gemm_epilogue op should be" " one of {\"none\", \"relu\", \"gelu\"}. But received %s." "But received activation=%s.", @@ -1987,42 +1980,32 @@ class CublasFusedMLP { const uint64_t cublas_row, const uint64_t cublas_col) { cudaDataType_t mat_type = CUDA_R_32F; - if (std::is_same::value) { + if (std::is_same::value) { mat_type = CUDA_R_16F; } - if (std::is_same::value) { + if (std::is_same::value) { mat_type = CUDA_R_16BF; } if (std::is_same::value) { mat_type = CUDA_R_64F; } - PADDLE_ENFORCE_GPU_SUCCESS( - platform::dynload::cublasLtMatrixLayoutSetAttribute( - layout_desc, - CUBLASLT_MATRIX_LAYOUT_TYPE, - &mat_type, - sizeof(mat_type))); - - PADDLE_ENFORCE_GPU_SUCCESS( - platform::dynload::cublasLtMatrixLayoutSetAttribute( - layout_desc, - CUBLASLT_MATRIX_LAYOUT_ROWS, - transpose ? &cublas_row : &cublas_col, - sizeof(cublas_row))); - PADDLE_ENFORCE_GPU_SUCCESS( - platform::dynload::cublasLtMatrixLayoutSetAttribute( - layout_desc, - CUBLASLT_MATRIX_LAYOUT_COLS, - transpose ? &cublas_col : &cublas_row, - sizeof(cublas_col))); + PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cublasLtMatrixLayoutSetAttribute( + layout_desc, CUBLASLT_MATRIX_LAYOUT_TYPE, &mat_type, sizeof(mat_type))); + + PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cublasLtMatrixLayoutSetAttribute( + layout_desc, + CUBLASLT_MATRIX_LAYOUT_ROWS, + transpose ? &cublas_row : &cublas_col, + sizeof(cublas_row))); + PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cublasLtMatrixLayoutSetAttribute( + layout_desc, + CUBLASLT_MATRIX_LAYOUT_COLS, + transpose ? &cublas_col : &cublas_row, + sizeof(cublas_col))); int64_t cublas_ld = transpose ? cublas_row : cublas_col; - PADDLE_ENFORCE_GPU_SUCCESS( - platform::dynload::cublasLtMatrixLayoutSetAttribute( - layout_desc, - CUBLASLT_MATRIX_LAYOUT_LD, - &cublas_ld, - sizeof(cublas_ld))); + PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cublasLtMatrixLayoutSetAttribute( + layout_desc, CUBLASLT_MATRIX_LAYOUT_LD, &cublas_ld, sizeof(cublas_ld))); } const phi::GPUContext &dev_ctx_; @@ -2036,5 +2019,5 @@ class CublasFusedMLP { } // namespace -} // namespace operators -} // namespace paddle +} // namespace fusion +} // namespace phi diff --git a/paddle/fluid/operators/ops_signature/fused_multi_transformer_sig.cc b/paddle/fluid/operators/ops_signature/fused_multi_transformer_sig.cc new file mode 100644 index 0000000000000..184df326b79e8 --- /dev/null +++ b/paddle/fluid/operators/ops_signature/fused_multi_transformer_sig.cc @@ -0,0 +1,58 @@ +/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/phi/core/compat/op_utils.h" + +namespace phi { + +KernelSignature FusedMultiTransformerOpArgumentMapping( + const ArgumentMappingContext& ctx UNUSED) { + return KernelSignature("fused_multi_transformer", + { + "X", + "LnScale", + "LnBias", + "QKVW", + "QKVBias", + "CacheKV", + "PreCaches", + "RotaryPosEmb", + "TimeStep", + "SeqLengths", + "SrcMask", + "OutLinearW", + "OutLinearBias", + "FFNLnScale", + "FFNLnBias", + "FFN1Weight", + "FFN1Bias", + "FFN2Weight", + "FFN2Bias", + }, + {"pre_layer_norm", + "epsilon", + "dropout_rate", + "rotary_emb_dims", + "is_test", + "dropout_implementation", + "act_method", + "trans_qkvw", + "ring_id"}, + {"CacheKVOut", "Out"}); +} + +} // namespace phi + +PD_REGISTER_ARG_MAPPING_FN(fused_multi_transformer, + phi::FusedMultiTransformerOpArgumentMapping); diff --git a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py index 2cbcb29f705b3..019a384f51173 100644 --- a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py @@ -70,7 +70,6 @@ NEED_GEN_STATIC_ONLY_APIS = [ 'fetch', - 'fused_bias_dropout_residual_layer_norm', 'fused_embedding_eltwise_layernorm', 'fused_fc_elementwise_layernorm', 'fused_multi_transformer_xpu', diff --git a/paddle/fluid/pir/dialect/operator/ir/ops.yaml b/paddle/fluid/pir/dialect/operator/ir/ops.yaml index d856c58a75550..98f240f485c0d 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops.yaml @@ -683,6 +683,16 @@ view : (mean -> mean_out), (variance -> variance_out) backward : fused_bn_add_activation_grad +- op : fused_multi_transformer + args : (Tensor x, Tensor[] ln_scales, Tensor[] ln_biases, Tensor[] qkv_weights, Tensor[] qkv_biases, Tensor[] cache_kvs, Tensor[] pre_caches, Tensor rotary_tensor, Tensor time_step, Tensor seq_lengths, Tensor src_mask, Tensor[] out_linear_weights, Tensor[] out_linear_biases, Tensor[] ffn_ln_scales, Tensor[] ffn_ln_biases, Tensor[] ffn1_weights, Tensor[] ffn1_biases, Tensor[] ffn2_weights, Tensor[] ffn2_biases, bool pre_layer_norm = true, float epsilon = 1e-5, float dropout_rate = .5f, int rotary_emb_dims = 0, bool is_test = false, str dropout_implementation = "downgrade_in_infer", str act_method = "gelu", bool trans_qkvw =true, int ring_id = -1) + optional : qkv_biases, cache_kvs, pre_caches, rotary_tensor, time_step, seq_lengths, src_mask, out_linear_biases, ffn1_biases, ffn2_biases, cache_kv_outs + output : Tensor[](cache_kv_outs){out_linear_weights.size()}, Tensor(out) + infer_meta : + func : FusedMultiTransformerInferMeta + kernel : + func : fused_multi_transformer + data_type : x + - op : fused_softmax_mask args : (Tensor x, Tensor mask) output : Tensor(out) diff --git a/paddle/phi/api/yaml/fused_backward.yaml b/paddle/phi/api/yaml/fused_backward.yaml index 5c92b1a2a692f..36c3c0dde5191 100644 --- a/paddle/phi/api/yaml/fused_backward.yaml +++ b/paddle/phi/api/yaml/fused_backward.yaml @@ -6,7 +6,7 @@ - backward_op : fused_bias_dropout_residual_layer_norm_grad forward: fused_bias_dropout_residual_layer_norm (Tensor x, Tensor residual, Tensor bias, Tensor ln_scale, Tensor ln_bias, float dropout_rate, bool is_test, bool dropout_fix_seed, int dropout_seed, str dropout_implementation, float ln_epsilon) -> Tensor(y), Tensor(bias_dropout_residual_out), Tensor(dropout_mask_out), Tensor(ln_mean), Tensor(ln_variance) - args : (Tensor y_grad, Tensor x, Tensor residual, Tensor bias, Tensor ln_scale, Tensor ln_bias, Tensor ln_mean, Tensor ln_variance, Tensor bias_dropout_residual_out, Tensor dropout_mask_out, float dropout_rate = 0.5f, bool is_test = false, bool dropout_fix_seed = true, int dropout_seed = true, str dropout_implementation = "downgrade_in_infer", float ln_epsilon = 1e-5) + args : (Tensor x, Tensor residual, Tensor bias, Tensor ln_scale, Tensor ln_bias, Tensor ln_mean, Tensor ln_variance, Tensor bias_dropout_residual_out, Tensor dropout_mask_out, Tensor y_grad, float dropout_rate = 0.5f, bool is_test = false, bool dropout_fix_seed = true, int dropout_seed = true, str dropout_implementation = "downgrade_in_infer", float ln_epsilon = 1e-5) output : Tensor(x_grad), Tensor(residual_grad), Tensor(bias_grad), Tensor(ln_scale_grad), Tensor(ln_bias_grad) optional : bias, ln_scale, ln_bias, bias_grad, ln_scale_grad, ln_bias_grad infer_meta : @@ -14,6 +14,7 @@ kernel : func : fused_bias_dropout_residual_layer_norm_grad data_type : y_grad + support_dygraph_mode : true - backward_op : fused_dot_product_attention_grad forward : fused_dot_product_attention (Tensor q, Tensor k, Tensor v, Tensor mask, float scaling_factor, float dropout_probability, bool is_training, bool is_causal_masking) -> Tensor(out), Tensor(softmax_out), Tensor(rng_state) diff --git a/paddle/phi/api/yaml/fused_ops.yaml b/paddle/phi/api/yaml/fused_ops.yaml index c7b0b14606b98..ff6969194f6d6 100644 --- a/paddle/phi/api/yaml/fused_ops.yaml +++ b/paddle/phi/api/yaml/fused_ops.yaml @@ -163,6 +163,7 @@ data_type : x backward : fused_bias_dropout_residual_layer_norm_grad intermediate : bias_dropout_residual_out, dropout_mask_out, ln_mean, ln_variance + support_dygraph_mode : true - op : fused_bias_residual_layernorm args : (Tensor x, Tensor bias, Tensor residual, Tensor norm_weight, Tensor norm_bias, float epsilon, float residual_alpha, int begin_norm_axis, float quant_scale, int quant_round_type, float quant_max_bound, float quant_min_bound) diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml index 9b1d862180903..e920f8a91eb8d 100755 --- a/paddle/phi/api/yaml/legacy_ops.yaml +++ b/paddle/phi/api/yaml/legacy_ops.yaml @@ -592,6 +592,16 @@ backward: fused_gemm_epilogue_grad optional: reserve_space +- op : fused_multi_transformer + args : (Tensor x, Tensor[] ln_scales, Tensor[] ln_biases, Tensor[] qkv_weights, Tensor[] qkv_biases, Tensor[] cache_kvs, Tensor[] pre_caches, Tensor rotary_tensor, Tensor time_step, Tensor seq_lengths, Tensor src_mask, Tensor[] out_linear_weights, Tensor[] out_linear_biases, Tensor[] ffn_ln_scales, Tensor[] ffn_ln_biases, Tensor[] ffn1_weights, Tensor[] ffn1_biases, Tensor[] ffn2_weights, Tensor[] ffn2_biases, bool pre_layer_norm = true, float epsilon = 1e-5, float dropout_rate = .5f, int rotary_emb_dims = 0, bool is_test = false, str dropout_implementation = "downgrade_in_infer", str act_method = "gelu", bool trans_qkvw =true, int ring_id = -1) + optional : qkv_biases, cache_kvs, pre_caches, rotary_tensor, time_step, seq_lengths, src_mask, out_linear_biases, ffn1_biases, ffn2_biases, cache_kv_outs + output : Tensor[](cache_kv_outs){out_linear_weights.size()}, Tensor(out) + infer_meta : + func : FusedMultiTransformerInferMeta + kernel : + func : fused_multi_transformer + data_type : x + - op : fused_softmax_mask args : (Tensor x, Tensor mask) output : Tensor(out) diff --git a/paddle/phi/infermeta/fusion.cc b/paddle/phi/infermeta/fusion.cc index 4af21b36b34da..b56e7fab0bfe6 100644 --- a/paddle/phi/infermeta/fusion.cc +++ b/paddle/phi/infermeta/fusion.cc @@ -116,6 +116,108 @@ void AddLayernormXPUInferMeta(const MetaTensor& x, out->share_lod(x); } +void FusedMultiTransformerInferMeta( + const MetaTensor& x, + const std::vector& ln_scales, + const std::vector& ln_biases, + const std::vector& qkv_weights, + const paddle::optional>& qkv_biases, + const paddle::optional>& cache_kvs, + const paddle::optional>& pre_caches, + const MetaTensor& rotary_tensor, + const MetaTensor& time_step, + const MetaTensor& seq_lengths, + const MetaTensor& src_mask, + const std::vector& out_linear_weights, + const paddle::optional>& out_linear_biases, + const std::vector& ffn_ln_scales, + const std::vector& ffn_ln_biases, + const std::vector& ffn1_weights, + const paddle::optional>& ffn1_biases, + const std::vector& ffn2_weights, + const paddle::optional>& ffn2_biases, + bool pre_layer_norm, + float epsilon, + float dropout_rate, + int rotary_emb_dims, + bool is_test, + const std::string& dropout_implementation, + const std::string& act_method, + bool trans_qkvw, + int ring_id, + std::vector cache_kv_outs, + MetaTensor* out) { + // x: qkv's input [batch_size, seq_len, dim_embed] + // y: qkv's weight: [3, num_head, dim_head, dim_embed] + auto x_dim = x.dims(); + auto y_dim = qkv_weights[0]->dims(); + PADDLE_ENFORCE_EQ( + x_dim.size(), + 3, + phi::errors::InvalidArgument("The dimensions of x must be 3" + "(batch_size, seq_len, dim_embed)," + "but received dimensions of" + "Input is [%d]", + x_dim.size())); + PADDLE_ENFORCE_EQ( + y_dim.size(), + 4, + phi::errors::InvalidArgument("The dimensions of qkv_weight must be 4" + "(3, num_head, dim_head, dim_embed)," + "but received dimensions of" + "Input is [%d]", + y_dim.size())); + PADDLE_ENFORCE_EQ( + x_dim[2], + trans_qkvw ? y_dim[3] : y_dim[0], + phi::errors::InvalidArgument( + "ShapeError: the dimension of x_dim[2] and y_dim[3](trans_qkvw is " + "true) or y_dim[0](trans_qkvw is false)" + "must be equal. But received: the shape " + "of input x = [%s], and the shape of " + "input qkv_weight = [%s]", + x_dim, + y_dim)); + + if (cache_kvs && cache_kvs->size() > 0) { + // [2, batch_size, num_head, max_seq_len, head_size] + const auto& c_dim = cache_kvs.get()[0]->dims(); + + PADDLE_ENFORCE_EQ( + c_dim.size(), + 5, + phi::errors::InvalidArgument("The CacheKV must be 5 dims, but got %d", + c_dim.size())); + PADDLE_ENFORCE_EQ(c_dim[0], + 2, + phi::errors::InvalidArgument( + "The first dim of CacheKV must be 2, but got %d", + c_dim[0])); // 2 + PADDLE_ENFORCE_EQ(c_dim[1], + x_dim[0], + phi::errors::InvalidArgument( + "The second dim of CacheKV must be equal with " + "batch size %d, but got %d", + x_dim[0], + c_dim[1])); // batch_size + PADDLE_ENFORCE_EQ(c_dim[2], + trans_qkvw ? y_dim[1] : y_dim[2], + phi::errors::InvalidArgument( + "The third dim of CacheKV must be equal with num " + "head %d, but got %d", + trans_qkvw ? y_dim[1] : y_dim[2], + c_dim[2])); // num_head + PADDLE_ENFORCE_EQ(c_dim[4], + trans_qkvw ? y_dim[2] : y_dim[3], + phi::errors::InvalidArgument( + "The fifth dim of CacheKV must be equal with head " + "size %d, but got %d", + trans_qkvw ? y_dim[2] : y_dim[3], + c_dim[4])); // head_size + } + out->set_dims(x.dims()); +} + void BlockMultiheadAttentionInferMeta(const MetaTensor& qkv, const MetaTensor& key_cache, const MetaTensor& value_cache, @@ -975,7 +1077,6 @@ void FusedBiasDropoutResidualLnInferMeta( } void FusedBiasDropoutResidualLnGradInferMeta( - const MetaTensor& y_grad, const MetaTensor& x, const MetaTensor& residual, const MetaTensor& bias, @@ -985,6 +1086,7 @@ void FusedBiasDropoutResidualLnGradInferMeta( const MetaTensor& ln_variance, const MetaTensor& bias_dropout_residual_out, const MetaTensor& dropout_mask_out, + const MetaTensor& y_grad, const float dropout_rate, const bool is_test, const bool dropout_fix_seed, diff --git a/paddle/phi/infermeta/fusion.h b/paddle/phi/infermeta/fusion.h index a724000bab9f0..0a7224e39f73b 100644 --- a/paddle/phi/infermeta/fusion.h +++ b/paddle/phi/infermeta/fusion.h @@ -22,6 +22,38 @@ namespace phi { // Common InferMeta Functions for fusion operators. // NOTE: The InferMeta Functions in this file are arranged in alphabetic order. +void FusedMultiTransformerInferMeta( + const MetaTensor& x, + const std::vector& ln_scales, + const std::vector& ln_biases, + const std::vector& qkv_weights, + const paddle::optional>& qkv_biases, + const paddle::optional>& cache_kvs, + const paddle::optional>& pre_caches, + const MetaTensor& rotary_tensor, + const MetaTensor& time_step, + const MetaTensor& seq_lengths, + const MetaTensor& src_mask, + const std::vector& out_linear_weights, + const paddle::optional>& out_linear_biases, + const std::vector& ffn_ln_scales, + const std::vector& ffn_ln_biases, + const std::vector& ffn1_weights, + const paddle::optional>& ffn1_biases, + const std::vector& ffn2_weights, + const paddle::optional>& ffn2_biases, + bool pre_layer_norm, + float epsilon, + float dropout_rate, + int rotary_emb_dims, + bool is_test, + const std::string& dropout_implementation, + const std::string& act_method, + bool trans_qkvw, + int ring_id, + std::vector cache_kv_outs, + MetaTensor* out); + void AddActXPUInferMeta(const MetaTensor& x, const MetaTensor& x_max, const MetaTensor& y, @@ -755,7 +787,6 @@ void FusedBiasDropoutResidualLnInferMeta( MetaTensor* ln_variance); void FusedBiasDropoutResidualLnGradInferMeta( - const MetaTensor& y_grad, const MetaTensor& x, const MetaTensor& residual, const MetaTensor& bias, @@ -765,6 +796,7 @@ void FusedBiasDropoutResidualLnGradInferMeta( const MetaTensor& ln_variance, const MetaTensor& bias_dropout_residual_out, const MetaTensor& dropout_mask_out, + const MetaTensor& y_grad, const float dropout_rate, const bool is_test, const bool dropout_fix_seed, diff --git a/paddle/phi/kernels/fusion/gpu/fused_bias_dropout_residual_layer_norm_grad_kernel.cu b/paddle/phi/kernels/fusion/gpu/fused_bias_dropout_residual_layer_norm_grad_kernel.cu index 0f93e21553a74..60a82cfe7c198 100644 --- a/paddle/phi/kernels/fusion/gpu/fused_bias_dropout_residual_layer_norm_grad_kernel.cu +++ b/paddle/phi/kernels/fusion/gpu/fused_bias_dropout_residual_layer_norm_grad_kernel.cu @@ -30,7 +30,6 @@ namespace fusion { template void FusedBiasDropoutResidualLnGradKernel( const Context& dev_ctx, - const DenseTensor& y_grad, const DenseTensor& x, const DenseTensor& residual, const paddle::optional& bias, @@ -40,6 +39,7 @@ void FusedBiasDropoutResidualLnGradKernel( const DenseTensor& ln_variance, const DenseTensor& bias_dropout_residual_out, const DenseTensor& dropout_mask_out, + const DenseTensor& y_grad, const float dropout_rate, const bool is_test, const bool dropout_fix_seed, diff --git a/python/paddle/incubate/nn/functional/fused_transformer.py b/python/paddle/incubate/nn/functional/fused_transformer.py index 423e071bbf25b..5a25e0b91f082 100644 --- a/python/paddle/incubate/nn/functional/fused_transformer.py +++ b/python/paddle/incubate/nn/functional/fused_transformer.py @@ -413,33 +413,21 @@ def fused_bias_dropout_residual_layer_norm( x.shape[len(x.shape) - 1] == ln_bias.shape[0] ), "The dim of ln_bias must equal to the last dim of x." - if in_dynamic_mode(): + if in_dynamic_or_pir_mode(): if default_main_program().random_seed != 0: seed = default_main_program().random_seed - ( - _, - _, - _, - _, - final_out, - ) = _legacy_C_ops.fused_bias_dropout_residual_layer_norm( + final_out = _C_ops.fused_bias_dropout_residual_layer_norm( x, residual, bias, ln_scale, ln_bias, - 'dropout_rate', dropout_rate, - 'ln_epsilon', - ln_epsilon, - 'is_test', not training, - 'dropout_fix_seed', seed is not None, - 'dropout_seed', seed if seed is not None else 0, - 'dropout_implementation', mode, + ln_epsilon, ) return final_out else: @@ -1151,8 +1139,8 @@ def fused_multi_transformer( 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode ) # semantic transfer - if in_dynamic_mode(): - cache_kv_out, final_out = _legacy_C_ops.fused_multi_transformer( + if in_dynamic_or_pir_mode(): + cache_kv_out, final_out = _C_ops.fused_multi_transformer( x, ln_scales, ln_biases, @@ -1172,24 +1160,14 @@ def fused_multi_transformer( ffn1_biases, ffn2_weights, ffn2_biases, - cache_kvs, - 'pre_layer_norm', pre_layer_norm, - 'epsilon', epsilon, - 'dropout_rate', dropout_rate, - 'rotary_emb_dims', rotary_emb_dims, - 'is_test', not training, - 'dropout_implementation', mode, - 'act_method', activation, - 'trans_qkvw', trans_qkvw, - 'ring_id', ring_id, ) if cache_kvs is not None: diff --git a/test/legacy_test/test_fused_bias_dropout_residual_layer_norm_op_api.py b/test/legacy_test/test_fused_bias_dropout_residual_layer_norm_op_api.py index 9efa1cd354cb3..9827957120635 100644 --- a/test/legacy_test/test_fused_bias_dropout_residual_layer_norm_op_api.py +++ b/test/legacy_test/test_fused_bias_dropout_residual_layer_norm_op_api.py @@ -20,7 +20,7 @@ from paddle.incubate.nn.layer.fused_transformer import ( FusedBiasDropoutResidualLayerNorm, ) -from paddle.static import Program +from paddle.pir_utils import test_with_pir_api def layer_norm(x, has_scale, has_bias, weight, bias, epsilon=1e-05): @@ -164,9 +164,10 @@ def run_static(self): ) return out, linear_bias, ln_scale, ln_bias + @test_with_pir_api def test_static_api(self): paddle.enable_static() - with paddle.static.program_guard(Program()): + with paddle.static.program_guard(paddle.static.Program()): out, linear_bias, ln_scale, ln_bias = self.run_static() ref_out = compute_reference( self.x, self.residual, ln_scale, ln_bias, linear_bias diff --git a/test/legacy_test/test_fused_multi_transformer_op.py b/test/legacy_test/test_fused_multi_transformer_op.py index 63921b64e93f7..b7fec52341be6 100644 --- a/test/legacy_test/test_fused_multi_transformer_op.py +++ b/test/legacy_test/test_fused_multi_transformer_op.py @@ -27,6 +27,7 @@ from paddle.nn.layer.common import Dropout, Linear from paddle.nn.layer.norm import LayerNorm from paddle.nn.layer.transformer import _convert_attention_mask +from paddle.pir_utils import test_with_pir_api seed = 42 @@ -999,19 +1000,20 @@ def GetFusedMultiTransformerOutStatic(self): } if self.has_pre_cache: out = exe.run( - paddle.base.default_main_program(), + paddle.static.default_main_program(), feed=feed_data, - fetch_list=[final_out[0].name], + fetch_list=[final_out[0]], ) else: out = exe.run( - paddle.base.default_main_program(), + paddle.static.default_main_program(), feed=feed_data, - fetch_list=[final_out.name], + fetch_list=[final_out], ) paddle.disable_static() return out + @test_with_pir_api def test_fused_multi_transformer_op(self): if self.has_cache_kv and not self.gen_cache_kv and self.remove_padding: final_out_ref = self.GetVariableDecoderBaselineOut() @@ -1393,6 +1395,7 @@ def config(self): initializer=paddle.nn.initializer.Constant(0.0) ) + @test_with_pir_api def test_fused_multi_transformer_op(self): self.has_pre_cache = True self.remove_padding = False From fc3fb0549357ca9c56d736b0215971332ce6fb65 Mon Sep 17 00:00:00 2001 From: chen2016013 <111894720+chen2016013@users.noreply.github.com> Date: Mon, 4 Mar 2024 19:14:07 +0800 Subject: [PATCH 276/282] [Dygraph] Fix `EagerReducer::MarkVarReady()` 's lank of HasGrad() branch (#62299) * fix eagr reducer * Update reducer.cc * fix approve error --- .../fluid/distributed/collective/reducer.cc | 40 ++++++++++++------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/paddle/fluid/distributed/collective/reducer.cc b/paddle/fluid/distributed/collective/reducer.cc index df41993bb9bd2..493936e599091 100644 --- a/paddle/fluid/distributed/collective/reducer.cc +++ b/paddle/fluid/distributed/collective/reducer.cc @@ -831,23 +831,33 @@ void EagerReducer::MarkVarReady(const size_t var_index, auto &group_tensor = group.dense_tensors_[inside_group_index]; const auto length = group.length_[inside_group_index]; if (is_used_var) { - auto *autograd_meta = tensors_[var_index].get_autograd_meta(); - paddle::Tensor grad_tensor = - static_cast(autograd_meta)->Grad(); - if (grad_tensor.is_dense_tensor()) { - const auto &tensor_impl = grad_tensor.impl(); - auto dense_tensor = - std::dynamic_pointer_cast(tensor_impl); - if (!dense_tensor->meta().is_contiguous()) { - grad_tensor.set_impl(std::make_shared(std::move( - paddle::experimental::Trans2Contiguous(*dense_tensor)))); + if (HasGrad(var_index)) { + auto *autograd_meta = tensors_[var_index].get_autograd_meta(); + paddle::Tensor grad_tensor = + static_cast(autograd_meta)->Grad(); + if (grad_tensor.is_dense_tensor()) { + const auto &tensor_impl = grad_tensor.impl(); + auto dense_tensor = + std::dynamic_pointer_cast(tensor_impl); + if (!dense_tensor->meta().is_contiguous()) { + grad_tensor.set_impl(std::make_shared(std::move( + paddle::experimental::Trans2Contiguous(*dense_tensor)))); + } } - } - group_tensor - .ShareDataWith(*( - std::dynamic_pointer_cast(grad_tensor.impl()))) - .Resize({grad_tensor.numel()}); + group_tensor + .ShareDataWith(*(std::dynamic_pointer_cast( + grad_tensor.impl()))) + .Resize({grad_tensor.numel()}); + } else { + VLOG(3) << "Tensor[" << tensors_[var_index].name() + << "] doesn't have grad"; + auto *dev_ctx = + platform::DeviceContextPool::Instance().Get(inner_place_); + group_tensor.Resize({static_cast(length)}); + dev_ctx->Alloc(&group_tensor, group.dtype_); + phi::funcs::set_constant(*dev_ctx, &group_tensor, 0.0f); + } } else { // TODO(shenliang03): maybe save the memory by avoiding tensor // construction From c72c0d6b3ef652219fce1da4224b7af390206801 Mon Sep 17 00:00:00 2001 From: LiYuRio <63526175+LiYuRio@users.noreply.github.com> Date: Mon, 4 Mar 2024 19:20:36 +0800 Subject: [PATCH 277/282] support 3d mesh calculation (#62356) --- .../auto_parallel/reshard/nd_mesh_reshard_function.cc | 8 +++++--- .../semi_auto_parallel_3d_global_mesh_reshard.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/paddle/phi/core/distributed/auto_parallel/reshard/nd_mesh_reshard_function.cc b/paddle/phi/core/distributed/auto_parallel/reshard/nd_mesh_reshard_function.cc index 7a044209677d3..222e918ae540b 100644 --- a/paddle/phi/core/distributed/auto_parallel/reshard/nd_mesh_reshard_function.cc +++ b/paddle/phi/core/distributed/auto_parallel/reshard/nd_mesh_reshard_function.cc @@ -40,9 +40,11 @@ ProcessMesh GetSubProcessMesh(const ProcessMesh& mesh, int64_t axis) { std::vector process_ids; for (int64_t i = 0; i < shape_of_axis; ++i) { coord[axis] = i; - int64_t rank = coord.back(); - for (int64_t j = static_cast(coord.size() - 2); j >= 0; --j) { - rank += coord[j] * mesh.dim_size(j + 1); + int64_t rank = 0; + int64_t degree = 1; + for (int64_t j = static_cast(coord.size() - 1); j >= 0; --j) { + rank += coord[j] * degree; + degree *= mesh.dim_size(j); } process_ids.emplace_back(mesh.process_ids()[rank]); } diff --git a/test/auto_parallel/hybrid_strategy/semi_auto_parallel_3d_global_mesh_reshard.py b/test/auto_parallel/hybrid_strategy/semi_auto_parallel_3d_global_mesh_reshard.py index bdc256a8a6493..9f15b4c36c234 100644 --- a/test/auto_parallel/hybrid_strategy/semi_auto_parallel_3d_global_mesh_reshard.py +++ b/test/auto_parallel/hybrid_strategy/semi_auto_parallel_3d_global_mesh_reshard.py @@ -64,8 +64,18 @@ def test_basic(self): verbose=True, ) + def test_3d_mesh_with_any_status(self): + dense_tensor = paddle.ones(shape=[2, 6], dtype='float32') + dist_tensor = dist.shard_tensor( + dense_tensor, + self._global_mesh, + [dist.Replicate(), dist.Shard(0), dist.Replicate()], + ) + np.testing.assert_equal(dist_tensor._local_shape, [1, 6]) + def run_test_case(self): self.test_basic() + self.test_3d_mesh_with_any_status() if __name__ == '__main__': From 14b3c61d7e6a0c88fd16cca922ae7a7c406f2270 Mon Sep 17 00:00:00 2001 From: zhangbo9674 <82555433+zhangbo9674@users.noreply.github.com> Date: Mon, 4 Mar 2024 20:05:51 +0800 Subject: [PATCH 278/282] fix (#62365) --- .../new_executor/pir_adaptor/pir_adaptor_util.cc | 9 --------- 1 file changed, 9 deletions(-) diff --git a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc index 11b263f540500..952648803359f 100644 --- a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc +++ b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc @@ -480,18 +480,9 @@ void HandleForSpecialOp(pir::Operation* op, auto shape = op->attribute("shape"); auto dim = phi::make_ddim(shape.data().GetData()); auto dtype = op->attribute("dtype"); - auto place = op->attribute("place").data(); - if (place.GetType() == phi::AllocationType::UNDEFINED) { - place = phi::CPUPlace(); - } if (!common::contain_unknown_dim(dim)) { phi::DenseTensorMeta meta(dtype.data(), dim); t->set_meta(meta); - auto* dev_ctx = platform::DeviceContextPool::Instance().Get(place); - dev_ctx->Alloc(t, dtype.data()); - VLOG(10) << "[Alloc var]: " - << op->attribute("name") << " " - << t->initialized(); } } } From bdd1fe8487af0081f39e38a2d2167512462ec862 Mon Sep 17 00:00:00 2001 From: xiaoxiaohehe001 <49090790+xiaoxiaohehe001@users.noreply.github.com> Date: Mon, 4 Mar 2024 21:14:16 +0800 Subject: [PATCH 279/282] yolo_box_test_time_lower (#62368) --- test/ir/inference/test_trt_convert_yolo_box.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/ir/inference/test_trt_convert_yolo_box.py b/test/ir/inference/test_trt_convert_yolo_box.py index 343c17046d91e..079db6e203901 100644 --- a/test/ir/inference/test_trt_convert_yolo_box.py +++ b/test/ir/inference/test_trt_convert_yolo_box.py @@ -56,13 +56,13 @@ def generate_input2(attrs: list[dict[str, Any]], batch): iou_aware, iou_aware_factor, ) in product( - [1, 4], - [80, 30], + [1], + [80], [[10, 13, 16, 30, 33, 23]], - [32, 16], - [0.01, 0.02], + [32], + [0.01], [True, False], - [1.0, 0.9], + [1.0], [False, True], [0.5], ): From 5d12fb165325136edbf15e036f6ecf9585a78458 Mon Sep 17 00:00:00 2001 From: BiynXu <62832681+BiynXu@users.noreply.github.com> Date: Mon, 4 Mar 2024 22:43:26 +0800 Subject: [PATCH 280/282] Tile first schedule (#61987) * [ForTest]Trigger to Extract Subgraph for PIR+CINN in PTS Platform * fix 50 -> 100 * fix logic * [PIR+CINN]Part-1 Refine SubGraphChecker code * fix UT * upload auto-test script * fix conflict * update * update * update * update * update1 * update * update * update * support gpt running * update subgraph test * support num not divide by 128t * update * add new cinn group cluster pass * update * update * update * update * add broadcast to dy schedule * update * update * update * update * update * update * fix ir op cluster test * fix unit test * update * update * update * update * formate * update * update * formate cmakelist * add header * updat * update * fix bug of ci * fix bug * fix bug * update * update * fix broadcast bug * update * update * update * update * aadd cinn store op * add store in fusion op * uniform all the 0 and reduce deleted axis * update * add cinn store op * update * before merge op cluster * fix group cluster bug * remove one shape for keepdim cases. * support store op * remove useless output data * fix store contrain * update * update store op * update before mrege code * merge dy shape and st shape schedule * revert some code * polish code * remove some useless code * polish coden and fix group cluster bug * polish code * polish base group scheduler * polish align type * revert codegen_cuda code * revert dyshape code * Add loop_reorder_alignment_tactic * Enable loop reorder alignment * Add tile first general tactic * fix factorize_reduction * add some symbolic Compute function * Migrate partial logic to BucketLower * update dyshape workflow * fix reshape * fix dyshape new infra * remove reduce init in write-back block * fix ir copy on buffer * fix conflict * delete migrated code * open pir all path unittest * polish code * polish code * move tactic class to cc file * rename StoreOp to YieldStoreOp * polish code * polish code * polish code * fix test instruction bug * update cmakelist * polish code * cinn(test): fix factor reduce schedule ut * fix factorize reduction * fix unittest * filter unittest * fix unittest * fix unittests * fix unittests * disable unittests * fix cmake * disable unittests --------- Co-authored-by: Aurelius84 Co-authored-by: phlrain Co-authored-by: zyfncg Co-authored-by: xiongkun Co-authored-by: 6clc --- paddle/cinn/ast_gen_ius/ast_gen.cc | 2 +- .../hlir/dialect/operator/ir/manual_op.cc | 11 + .../cinn/hlir/dialect/operator/ir/manual_op.h | 17 + .../hlir/dialect/operator/ir/op_dialect.cc | 1 + .../operator/transforms/add_cinn_pass.cc | 10 +- .../transforms/add_store_in_fusion_op_pass.cc | 122 ++ .../transforms/add_store_in_fusion_op_pass.h | 28 + .../transforms/cinn_group_cluster_pass.cc | 19 +- .../transforms/lower_cinn_fusion_op_pass.cc | 13 +- .../operator/transforms/pd_to_cinn_pass.cc | 12 +- paddle/cinn/hlir/framework/op.h | 0 paddle/cinn/hlir/framework/pir/group.h | 6 + .../hlir/framework/pir/op_lowering_impl.cc | 455 ++++++- .../hlir/framework/pir/op_lowering_impl.h | 19 + paddle/cinn/hlir/framework/pir/utils.cc | 42 +- paddle/cinn/hlir/op/elementwise.cc | 134 ++ paddle/cinn/hlir/pe/broadcast.cc | 2 +- paddle/cinn/hlir/pe/elementwise.cc | 91 +- paddle/cinn/hlir/pe/elementwise.h | 3 + .../ir/group_schedule/base_group_scheduler.cc | 7 +- .../ir/group_schedule/base_group_scheduler.h | 12 +- .../dy_shape_group_scheduler.cc | 15 +- .../group_schedule/dy_shape_group_scheduler.h | 5 +- .../group_schedule/st_shape_group_scheduler.h | 5 +- .../ir/group_schedule/tactic/CMakeLists.txt | 2 + .../tactic/align_iter_space_tactic.cc | 16 + .../tactic/align_iter_space_tactic.h | 12 +- .../tactic/arrange_storage_tactic.cc | 16 + .../tactic/arrange_storage_tactic.h | 12 +- .../group_schedule/tactic/bind_cuda_tactic.cc | 16 + .../group_schedule/tactic/bind_cuda_tactic.h | 12 +- .../tactic/compute_inline_tactic.cc | 17 + .../tactic/compute_inline_tactic.h | 13 +- .../tactic/loop_reorder_alignment_tactic.cc | 188 +++ .../tactic/loop_reorder_alignment_tactic.h | 26 + .../tactic/optimize_reduction_tactic.cc | 16 + .../tactic/optimize_reduction_tactic.h | 12 +- .../group_schedule/tactic/schedule_tactic.h | 31 + .../tactic/tile_first_general_tactic.cc | 283 +++++ .../tactic/tile_first_general_tactic.h | 26 + .../ir/group_schedule/tactic/tile_tactic.cc | 16 + .../ir/group_schedule/tactic/tile_tactic.h | 12 +- paddle/cinn/ir/ir.h | 8 +- paddle/cinn/ir/schedule/factorize_reduction.h | 84 +- paddle/cinn/ir/schedule/impl/for_type.cc | 2 +- paddle/cinn/ir/schedule/impl/ir_schedule.h | 8 +- paddle/cinn/ir/schedule/impl/reduction.cc | 22 +- paddle/cinn/ir/schedule/ir_schedule.cc | 27 +- paddle/cinn/ir/schedule/ir_schedule.h | 10 +- paddle/cinn/ir/schedule/schedule_base.cc | 165 +++ paddle/cinn/ir/schedule/schedule_base.h | 24 +- paddle/cinn/ir/schedule/schedule_desc.cc | 1 + paddle/cinn/ir/utils/ir_copy.cc | 37 +- paddle/cinn/ir/utils/ir_copy.h | 12 +- paddle/cinn/ir/utils/ir_replace.cc | 4 +- paddle/cinn/optim/replace_call_with_expr.cc | 5 +- .../optim/replace_cross_thread_reduction.cc | 35 +- .../replace_cross_thread_reduction_test.cc | 2 +- paddle/cinn/optim/unroll_loops.cc | 3 +- paddle/cinn/optim/vectorize_loops.cc | 18 +- paddle/cinn/pybind/optim.cc | 5 +- .../fluid/pir/transforms/build_cinn_pass.cc | 3 + test/cpp/pir/cinn/CMakeLists.txt | 11 +- test/cpp/pir/cinn/pir_all_path_test.cc | 1128 ++++++++--------- test/cpp/pir/cinn/pir_compiler_test.cc | 213 ++-- test/ir/pir/cinn/CMakeLists.txt | 86 +- test/ir/pir/cinn/sub_graphs/CMakeLists.txt | 1 + .../pir/cinn/sub_graphs/test_sub_graph_0.py | 4 +- .../pir/cinn/sub_graphs/test_sub_graph_19.py | 4 +- .../pir/cinn/sub_graphs/test_sub_graph_32.py | 4 +- .../pir/cinn/sub_graphs/test_sub_graph_33.py | 4 +- .../pir/cinn/sub_graphs/test_sub_graph_37.py | 4 +- .../pir/cinn/sub_graphs/test_sub_graph_5.py | 4 +- .../pir/cinn/sub_graphs/test_sub_graph_50.py | 4 +- .../pir/cinn/sub_graphs/test_sub_graph_53.py | 4 +- .../pir/cinn/sub_graphs/test_sub_graph_58.py | 4 +- .../pir/cinn/sub_graphs/test_sub_graph_60.py | 4 +- .../pir/cinn/sub_graphs/test_sub_graph_68.py | 4 +- .../pir/cinn/sub_graphs/test_sub_graph_70.py | 4 +- .../pir/cinn/sub_graphs/test_sub_graph_71.py | 4 +- .../pir/cinn/sub_graphs/test_sub_graph_75.py | 4 +- .../pir/cinn/sub_graphs/test_sub_graph_76.py | 4 +- .../pir/cinn/sub_graphs/test_sub_graph_79.py | 4 +- .../pir/cinn/sub_graphs/test_sub_graph_88.py | 4 +- .../pir/cinn/sub_graphs/test_sub_graph_89.py | 4 +- .../sub_graphs/test_sub_graph_mul_method.py | 4 +- .../cinn/sub_graphs/test_sub_graph_relu6.py | 4 +- test/ir/pir/cinn/symbolic/CMakeLists.txt | 7 +- .../symbolic/test_check_infer_symbolic.py | 4 +- .../symbolic/test_cinn_sub_graph_symbolic.py | 4 +- .../cinn/symbolic/test_dyshape_rms_norm.py | 6 +- .../ir/pir/cinn/symbolic/test_dyshape_rope.py | 4 +- test/ir/pir/cinn/symbolic/test_if_dy.py | 4 +- .../ir/pir/cinn/symbolic/test_llama_mlp_dy.py | 4 +- .../symbolic/test_multiple_subgraph_dy.py | 4 +- .../symbolic/test_sub_graph_for_frontend.py | 4 +- test/ir/pir/cinn/test_cinn_sub_graph.py | 265 ++-- test/ir/pir/cinn/test_llama_sub_graph.py | 140 +- test/ir/pir/cinn/test_rms_norm.py | 5 +- test/ir/pir/cinn/test_rope.py | 4 +- test/ir/pir/cinn/test_subgraph_checker.py | 4 +- .../pir_prim/test_prim_rms_norm_st_shape.py | 114 +- 102 files changed, 3069 insertions(+), 1255 deletions(-) create mode 100644 paddle/cinn/hlir/dialect/operator/transforms/add_store_in_fusion_op_pass.cc create mode 100644 paddle/cinn/hlir/dialect/operator/transforms/add_store_in_fusion_op_pass.h mode change 100755 => 100644 paddle/cinn/hlir/framework/op.h create mode 100644 paddle/cinn/ir/group_schedule/tactic/loop_reorder_alignment_tactic.cc create mode 100644 paddle/cinn/ir/group_schedule/tactic/loop_reorder_alignment_tactic.h create mode 100644 paddle/cinn/ir/group_schedule/tactic/tile_first_general_tactic.cc create mode 100644 paddle/cinn/ir/group_schedule/tactic/tile_first_general_tactic.h diff --git a/paddle/cinn/ast_gen_ius/ast_gen.cc b/paddle/cinn/ast_gen_ius/ast_gen.cc index 57b10fb7ca884..ee1db18a69f85 100644 --- a/paddle/cinn/ast_gen_ius/ast_gen.cc +++ b/paddle/cinn/ast_gen_ius/ast_gen.cc @@ -244,7 +244,7 @@ ir::Expr AstGen::Build(const ir::Tensor& tensor, TensorGroup* tensor_group) { if (FLAGS_group_schedule_tiling_first && is_keep_dim) { continue; } - if (!FLAGS_group_schedule_tiling_first && !FLAGS_cinn_bucket_compile && + if ((!FLAGS_group_schedule_tiling_first || !FLAGS_cinn_bucket_compile) && shape[i] == Expr(1)) { continue; } diff --git a/paddle/cinn/hlir/dialect/operator/ir/manual_op.cc b/paddle/cinn/hlir/dialect/operator/ir/manual_op.cc index aa4a02005437d..d3af713a6a069 100644 --- a/paddle/cinn/hlir/dialect/operator/ir/manual_op.cc +++ b/paddle/cinn/hlir/dialect/operator/ir/manual_op.cc @@ -170,6 +170,16 @@ void FusionOp::Print(pir::IrPrinter& printer) { os << " \n }"; } +void YieldStoreOp::Build(pir::Builder& builder, + pir::OperationArgument& argument, + pir::Value x, + pir::Type output_type) { + argument.inputs = {x}; + argument.output_types = {output_type}; +} + +void YieldStoreOp::VerifySig() {} + bool ConcatOp::InferSymbolicShape( pir::ShapeConstraintIRAnalysis* shape_analysis) { VLOG(4) << "Infer symbolic shape for cinn_op.concat"; @@ -501,3 +511,4 @@ IR_DEFINE_EXPLICIT_TYPE_ID(cinn::dialect::FusionOp) IR_DEFINE_EXPLICIT_TYPE_ID(cinn::dialect::ConcatOp) IR_DEFINE_EXPLICIT_TYPE_ID(cinn::dialect::SplitOp) IR_DEFINE_EXPLICIT_TYPE_ID(cinn::dialect::GenerateShapeOp); +IR_DEFINE_EXPLICIT_TYPE_ID(cinn::dialect::YieldStoreOp); diff --git a/paddle/cinn/hlir/dialect/operator/ir/manual_op.h b/paddle/cinn/hlir/dialect/operator/ir/manual_op.h index 1a0fa3dba75c3..9273a722e25c5 100644 --- a/paddle/cinn/hlir/dialect/operator/ir/manual_op.h +++ b/paddle/cinn/hlir/dialect/operator/ir/manual_op.h @@ -82,6 +82,22 @@ class IR_API FusionOp : public pir::Op { void Print(pir::IrPrinter &printer); // NOLINT }; +// YieldStoreOp represents a store operation for +// seperate local variable and ouptut +class IR_API YieldStoreOp : public pir::Op { + public: + using Op::Op; + static const char *name() { return "cinn_op.yield_store"; } + static constexpr uint32_t attributes_num = 0; + static constexpr const char **attributes_name = nullptr; + static void Build(pir::Builder &builder, // NOLINT + pir::OperationArgument &argument, // NOLINT + pir::Value x, + pir::Type output_type); + + void VerifySig(); +}; + class IR_API ConcatOp : public pir::Op { public: @@ -170,3 +186,4 @@ IR_EXPORT_DECLARE_EXPLICIT_TYPE_ID(cinn::dialect::FusionOp) IR_EXPORT_DECLARE_EXPLICIT_TYPE_ID(cinn::dialect::ConcatOp) IR_EXPORT_DECLARE_EXPLICIT_TYPE_ID(cinn::dialect::SplitOp) IR_EXPORT_DECLARE_EXPLICIT_TYPE_ID(cinn::dialect::GenerateShapeOp); +IR_EXPORT_DECLARE_EXPLICIT_TYPE_ID(cinn::dialect::YieldStoreOp); diff --git a/paddle/cinn/hlir/dialect/operator/ir/op_dialect.cc b/paddle/cinn/hlir/dialect/operator/ir/op_dialect.cc index c07ae5a9b0cad..32a534a397018 100644 --- a/paddle/cinn/hlir/dialect/operator/ir/op_dialect.cc +++ b/paddle/cinn/hlir/dialect/operator/ir/op_dialect.cc @@ -56,6 +56,7 @@ void OperatorDialect::initialize() { RegisterOp(); RegisterOp(); RegisterOp(); + RegisterOp(); RegisterOp(); RegisterAttribute(); RegisterAttribute(); diff --git a/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.cc index 1c8e9b9bf725e..a05cbc8fe34fb 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/add_cinn_pass.cc @@ -25,6 +25,7 @@ #include "paddle/cinn/hlir/dialect/operator/ir/op_dialect.h" #include "paddle/cinn/hlir/dialect/operator/transforms/add_broadcast_to_elementwise_pass.h" +#include "paddle/cinn/hlir/dialect/operator/transforms/cinn_group_cluster_pass.h" #include "paddle/cinn/hlir/dialect/operator/transforms/dynamic_reshape_pass.h" #include "paddle/cinn/hlir/dialect/operator/transforms/fuse_shape_ops_into_generate_shape_op_pass.h" #include "paddle/cinn/hlir/dialect/operator/transforms/group_merge/check_infer_symbolic_pass.h" @@ -47,6 +48,7 @@ COMMON_DECLARE_bool(print_ir); COMMON_DECLARE_bool(check_infer_symbolic); +PD_DECLARE_bool(group_schedule_tiling_first); namespace cinn::dialect::ir { @@ -130,6 +132,7 @@ void ApplyGroupOpPass(::pir::Program* program, pass_manager->AddPass(cinn::dialect::ir::CreateDynamicReshapeOpPass()); pass_manager->AddPass(cinn::dialect::ir::CreateReplaceDynamicExpandOpPass()); + pass_manager->AddPass(cinn::dialect::ir::CreateDivideGroupOpToFusionOpPass()); pass_manager->AddPass(pir::CreateDeadCodeEliminationPass()); pass_manager->Run(program); @@ -140,7 +143,12 @@ void ApplyDivideGroupOpToFusionOpPass( const std::function()>& CreatePassManager) { std::shared_ptr pass_manager = CreatePassManager(); - pass_manager->AddPass(cinn::dialect::ir::CreateDivideGroupOpToFusionOpPass()); + if (FLAGS_group_schedule_tiling_first) { + pass_manager->AddPass(cinn::dialect::ir::CreateCinnGroupClusterPass()); + } else { + pass_manager->AddPass( + cinn::dialect::ir::CreateDivideGroupOpToFusionOpPass()); + } pass_manager->Run(program); } diff --git a/paddle/cinn/hlir/dialect/operator/transforms/add_store_in_fusion_op_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/add_store_in_fusion_op_pass.cc new file mode 100644 index 0000000000000..47fa9371fdcff --- /dev/null +++ b/paddle/cinn/hlir/dialect/operator/transforms/add_store_in_fusion_op_pass.cc @@ -0,0 +1,122 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/cinn/hlir/dialect/operator/transforms/add_store_in_fusion_op_pass.h" + +#include "paddle/cinn/hlir/dialect/operator/ir/cinn_op.h" +#include "paddle/cinn/hlir/dialect/operator/ir/manual_op.h" +#include "paddle/fluid/pir/dialect/operator/ir/pd_op.h" +#include "paddle/pir/include/core/builtin_type_interfaces.h" +#include "paddle/pir/include/dialect/control_flow/ir/cf_op.h" +#include "paddle/pir/include/pattern_rewrite/pattern_rewrite_driver.h" + +namespace cinn { +namespace dialect { +namespace ir { + +class AddYieldStoreInFusionOpPattern + : public pir::OpRewritePattern<::pir::YieldOp> { + public: + using pir::OpRewritePattern<::pir::YieldOp>::OpRewritePattern; + + bool MatchAndRewrite(::pir::YieldOp op, + pir::PatternRewriter& rewriter) const override { + for (auto i = 0; i < op->num_operands(); ++i) { + if (op->operand_source(i) + .defining_op() + ->isa()) { + auto pre_name = op->operand_source(i).defining_op()->name(); + + if (op->operand_source(i).use_count() > 1) { + continue; + } + + if ((pre_name != "cinn_op.reduce_sum") && + (pre_name != "cinn_op.reduce_max")) { + auto new_full = rewriter.Build( + op->operand_source(i).defining_op()->operand_source(0), + op->operand_source(i).type()); + + op->operand(i).set_source(new_full.result(0)); + + continue; + } + } + + if (op->operand_source(i).use_count() == 1) { + continue; + } + + auto new_full = rewriter.Build( + op->operand_source(i), op->operand_source(i).type()); + + op->operand(i).set_source(new_full.result(0)); + } + + return true; + } +}; + +class AddStoreInFusionOpPass : public pir::Pass { + public: + AddStoreInFusionOpPass() + : pir::Pass("add_store_in_fusion_op", /*opt_level=*/1) {} + + bool Initialize(pir::IrContext* context) override { + pir::RewritePatternSet ps(context); + ps.Add(context); + + patterns_ = pir::FrozenRewritePatternSet(std::move(ps)); + return true; + } + + void Run(pir::Operation* op) override { + pir::GreedyRewriteConfig cfg; + cfg.use_top_down_traversal = true; + cfg.max_iterations = 1; + for (uint32_t i = 0; i < op->num_regions(); ++i) { + for (auto& block : op->region(i)) { + for (auto& op : block) { + if (op.isa()) { + auto fusion_op = op.dyn_cast(); + if (fusion_op.GetOperators().size() == 2 && + fusion_op.GetOperators() + .front() + ->isa()) { + continue; + } + auto [_, num_rewrites] = + pir::ApplyPatternsGreedily(&op, patterns_, cfg); + AddStatistics(num_rewrites); + } + } + } + } + } + + bool CanApplyOn(pir::Operation* op) const override { + return op->num_regions() > 0; + } + + private: + pir::FrozenRewritePatternSet patterns_; +}; + +std::unique_ptr CreateAddStoreInFusionOpPass() { + return std::make_unique(); +} + +} // namespace ir +} // namespace dialect +} // namespace cinn diff --git a/paddle/cinn/hlir/dialect/operator/transforms/add_store_in_fusion_op_pass.h b/paddle/cinn/hlir/dialect/operator/transforms/add_store_in_fusion_op_pass.h new file mode 100644 index 0000000000000..403e9a13ce38b --- /dev/null +++ b/paddle/cinn/hlir/dialect/operator/transforms/add_store_in_fusion_op_pass.h @@ -0,0 +1,28 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include "paddle/pir/include/pass/pass.h" + +namespace cinn { +namespace dialect { +namespace ir { + +std::unique_ptr CreateAddStoreInFusionOpPass(); + +} // namespace ir +} // namespace dialect +} // namespace cinn diff --git a/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_cluster_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_cluster_pass.cc index f0069a55a4cde..1c4e842b79bd7 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_cluster_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_cluster_pass.cc @@ -287,10 +287,13 @@ ::pir::GroupOpsVec CloneOps( auto new_op = op->Clone(*ir_mapping, clone_options); auto& shape_analysis = pir::ShapeAnalysisManager::Instance().Get(op->GetParentProgram()); + for (size_t i = 0; i < op->num_results(); ++i) { - shape_analysis.SetShapeOrDataForValue( - new_op->result(i), - shape_analysis.GetShapeOrDataForValue(op->result(i))); + if (shape_analysis.HasShapeOrDataForValue(op->result(i))) { + shape_analysis.SetShapeOrDataForValue( + new_op->result(i), + shape_analysis.GetShapeOrDataForValue(op->result(i))); + } } vec_new_op_list.push_back(new_op); @@ -398,7 +401,13 @@ bool CanFuse(const GroupClusterNode& first, if (first.loop_ranges != second.loop_ranges) { sch_node->type = hlir::framework::pir::ScheduleAlignType::kBroadcast; - sch_node->axis_info = first.reduce_axis; + for (auto& d : first.reduce_axis) { + if (d < 0) { + sch_node->axis_info.push_back(d + first.loop_ranges.size()); + } else { + sch_node->axis_info.push_back(d); + } + } sch_node->factor_info = first.loop_ranges; } return true; @@ -531,6 +540,8 @@ void GetClusterNodeBasicInfo(::pir::Operation* op, sch_node->axis_info = cinn::dialect::ir::GetVectorAttr(op, "broadcast_axes"); sch_node->factor_info = cinn::dialect::ir::GetVectorAttr(op, "out_shape"); + } else if (op->name() == "cinn_op.generate_shape") { + // do nothing for now } else { PADDLE_THROW(phi::errors::Unimplemented( "only support elementwise, broadcast, reduce type")); diff --git a/paddle/cinn/hlir/dialect/operator/transforms/lower_cinn_fusion_op_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/lower_cinn_fusion_op_pass.cc index c725d33257cc3..b35c56690bbc2 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/lower_cinn_fusion_op_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/lower_cinn_fusion_op_pass.cc @@ -690,11 +690,23 @@ class FusionOpPattern : public pir::OpRewritePattern { std::shared_ptr RebuildGroup(cinn::dialect::FusionOp fusion_op) const { auto group = std::make_shared(); group->op_pattern_kind = cinn::hlir::framework::OpPatternKind::kElementWise; + if (fusion_op.attributes().count("group_info")) { + auto attr = fusion_op.attribute("group_info") + .dyn_cast() + .data(); + + group->op_pattern_kind = attr.op_pattern_kind; + group->loop_ranges = attr.loop_ranges; + + group->reduce_axis = attr.reduce_axis; + group->alignment_schedule_info = attr.alignment_schedule_info; + } // Rebuild ops of the group for (auto op : fusion_op.GetOperators()) { if (!op->isa<::pir::YieldOp>()) { group->ops.push_back(op); + group->ops_set.insert(op); group->op_pattern_kind = static_cast(CompatibleInfo::OpKind(*op)) > @@ -709,7 +721,6 @@ class FusionOpPattern : public pir::OpRewritePattern { for (size_t i = 0; i < yield_op->num_operands(); ++i) { auto in = yield_op->operand_source(i); group->output_values.push_back(in); - group->output_ops.insert(in.defining_op()); } diff --git a/paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.cc index 03a510863a61b..66098f0e9467a 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.cc @@ -203,13 +203,15 @@ class ReshapeOpPattern auto scale_factor_gen_op = op->operand_source(1).defining_op(); auto full_op = scale_factor_gen_op->dyn_cast(); - return flag && full_op; + auto not_combine_input = + op->result(0).use_count() == 1 && + op->result(0).first_use().owner()->name() == "builtin.combine"; + return flag && full_op && (!not_combine_input); } void Rewrite(paddle::dialect::ReshapeOp op, pir::PatternRewriter &rewriter) const override { auto scale_factor_gen_op = op->operand_source(1).defining_op(); - auto full_op = scale_factor_gen_op->dyn_cast(); // scale is generator by full op @@ -725,16 +727,10 @@ pir::RewritePatternSet PdOpToCinnOpPass::InitializePatterns( ps.Add(paddle::drr::Create(context)); ps.Add(paddle::drr::Create(context)); ps.Add(context); - ps.Add(context); - ps.Add(context); - ps.Add(context); ps.Add(context); - ps.Add(context); ps.Add(context); - ps.Add(context); ps.Add(context); ps.Add(context); - // ps.Add(paddle::drr::Create(context)); return ps; } diff --git a/paddle/cinn/hlir/framework/op.h b/paddle/cinn/hlir/framework/op.h old mode 100755 new mode 100644 diff --git a/paddle/cinn/hlir/framework/pir/group.h b/paddle/cinn/hlir/framework/pir/group.h index 29ff85d099220..acf4d86092921 100644 --- a/paddle/cinn/hlir/framework/pir/group.h +++ b/paddle/cinn/hlir/framework/pir/group.h @@ -121,6 +121,12 @@ struct Group { std::string fn_name{""}; std::map int_args_map; + std::unordered_map<::pir::Operation*, + std::vector> + alignment_schedule_info; + std::vector reduce_axis; + std::vector loop_ranges; + struct SharedGroupHasher { size_t operator()(const std::shared_ptr& group) const noexcept { return std::hash()(reinterpret_cast(group.get())); diff --git a/paddle/cinn/hlir/framework/pir/op_lowering_impl.cc b/paddle/cinn/hlir/framework/pir/op_lowering_impl.cc index 032431feda354..a277a26000589 100644 --- a/paddle/cinn/hlir/framework/pir/op_lowering_impl.cc +++ b/paddle/cinn/hlir/framework/pir/op_lowering_impl.cc @@ -19,6 +19,7 @@ #include "paddle/cinn/adt/map_expr_ctx.h" #include "paddle/cinn/ast_gen_ius/tensor_group.h" #include "paddle/cinn/backends/codegen_cuda_util.h" +#include "paddle/cinn/hlir/dialect/operator/ir/manual_op.h" #include "paddle/cinn/hlir/framework/compile_error.h" #include "paddle/cinn/hlir/framework/pir/op_lowering_util.h" #include "paddle/cinn/hlir/framework/pir/utils.h" @@ -33,6 +34,9 @@ #include "paddle/cinn/optim/transform_gpu_forloop.h" #include "paddle/common/ddim.h" #include "paddle/fluid/pir/dialect/operator/ir/op_type.h" +#include "paddle/pir/include/dialect/control_flow/ir/cf_op.h" + +#include "paddle/cinn/hlir/dialect/operator/transforms/group_merge/op_with_group_merge_util.h" PD_DECLARE_bool(cinn_use_cuda_vectorize); PD_DECLARE_bool(cinn_enable_map_expr); @@ -64,6 +68,149 @@ NodeAttr CollectAttrs(const ::pir::Operation& op) { } // namespace details +int64_t Next2Power(int64_t n) { + if (n == 1) { + return 1; + } + return int64_t(std::pow(2.0, std::ceil(std::log2(n)))); +} + +std::shared_ptr OpLowererImpl::GetGroupTileInfo( + const GroupPtr& group) { + std::shared_ptr group_tile_info = + std::make_shared(); + + const auto data_dim = group->loop_ranges; + group_tile_info->data_rank = data_dim.size(); + const auto reduce_axis = group->reduce_axis; + + std::set reduce_set; + for (auto dim : reduce_axis) { + if (dim < 0) { + dim += group_tile_info->data_rank; + } + + group_tile_info->reduce_axis_.push_back(dim); + reduce_set.insert(dim); + } + + int64_t spatial_numel = 1; + int64_t reduce_numel = 1; + + for (int64_t i = 0; i < group_tile_info->data_rank; ++i) { + if (reduce_set.count(i)) { + reduce_numel *= data_dim[i]; + } else { + spatial_numel *= data_dim[i]; + } + } + + PADDLE_ENFORCE_GT( + reduce_numel, + 0, + phi::errors::Unimplemented("negative reduce numel or flaten numel")); + + int64_t reduce_block = 1; + int64_t spatial_block = 1; + + int64_t reduce_inner_num = 1; + int64_t spatial_inner_num = 1; + int warp_num = 1; + + if (reduce_numel == 1) { + reduce_block = 1; + if (spatial_numel < 0) { + spatial_block = 1024; + + reduce_inner_num = 1; + warp_num = spatial_block / 128; + + spatial_inner_num = spatial_block / (warp_num * 32); + if (spatial_inner_num == 0) { + spatial_inner_num = 1; + } + + group_tile_info->block_num = -1; + } else { + spatial_block = Next2Power(spatial_numel); + if (spatial_block > 1024) { + spatial_block = 1024; + } + reduce_inner_num = 1; + warp_num = spatial_block / 128; + if (warp_num == 0) { + warp_num = 1; + } + spatial_inner_num = spatial_block / (warp_num * 32); + if (spatial_inner_num == 0) { + spatial_inner_num = 1; + } + + int64_t block_num = + int64_t(std::ceil(spatial_numel * 1.0 / spatial_block)); + group_tile_info->block_num = block_num; + } + } else if (reduce_numel <= 256) { + // warp reduce + reduce_block = Next2Power(reduce_numel); + spatial_block = 256 / reduce_block; + spatial_inner_num = spatial_block; + reduce_inner_num = reduce_block / 32; + if (reduce_inner_num == 0) { + reduce_inner_num = 2; + } + warp_num = 8; + } else if (reduce_numel > 256 && reduce_numel <= 2048) { + spatial_block = 1; + reduce_block = int64_t(std::ceil(reduce_numel * 1.0 / 256.0)) * 256; + warp_num = reduce_block / 256; + spatial_inner_num = 1; + reduce_inner_num = 8; + } else if (reduce_numel > 2048) { + spatial_block = 1; + reduce_block = 2048; + warp_num = 8; + reduce_inner_num = int64_t(std::ceil(reduce_numel * 1.0 / 256.0)); + spatial_inner_num = 1; + } + + group_tile_info->reduce_numel = reduce_numel; + group_tile_info->reduce_block = reduce_block; + + VLOG(6) << "block num " << group_tile_info->block_num << std::endl; + VLOG(6) << "num warp " << warp_num << std::endl; + VLOG(6) << "flatten block " << spatial_block << std::endl; + VLOG(6) << "reduce block " << reduce_block << std::endl; + VLOG(6) << "flatten inner num " << spatial_inner_num << std::endl; + VLOG(6) << "reduce inner num " << reduce_inner_num << std::endl; + + group_tile_info->warp_num = warp_num; + group_tile_info->spatial_inner_num = spatial_inner_num; + group_tile_info->reduce_inner_num = reduce_inner_num; + + if (reduce_block > 1 && reduce_block <= 256) { + group_tile_info->reduce_method = ir::WarpReduceMethod(); + } + + for (auto op : group->ops) { + if (CompatibleInfo::OpKind(*op) == OpPatternKind::kReduction) { + group_tile_info->reduce_tensor_names.insert(ValueName(op->result(0))); + } + } + + for (auto& val : group->output_values) { + group_tile_info->direct_output_var_names.insert(ValueName(val)); + } + + group_tile_info->shared_var_names = shared_var_names; + group_tile_info->thread_sync_before_names = thread_sync_before_names; + + group_tile_info->broadcast_info = broadcast_info; + group_tile_info->broadcast_to_elementwise = broadcast_to_elementwise; + + return group_tile_info; +} + OpLowererImpl::OpLowererImpl(const Target& target) : target_(target) { name_gene_ = new PrettyNamer(); } @@ -131,16 +278,52 @@ BucketLoweredFuncsWrapper OpLowererImpl::BucketLower(const GroupPtr& group, ir_sch.MergeExprs(); std::vector> cond2func_bodies; VLOG(3) << "After lower, ir is: \n" << ir_sch.GetModule().GetExprs().at(0); + + std::unordered_set<::pir::Value> inner_genevalue; + std::unordered_set<::pir::Operation*> ops_set(ops.begin(), ops.end()); + for (auto* op : ops) { + for (size_t i = 0; i < op->num_results(); ++i) { + inner_genevalue.insert(op->result(i)); + } + } + + BuildBroadcastInfo(group); + + for (auto& op : group->output_ops) { + // collect all output tensor. + if (op->name() == "cinn_op.yield_store") { + auto input_var_name = ValueName(op->operand_source(0)); + if (broadcast_info.count(input_var_name)) { + auto base_info = broadcast_info[input_var_name]; + base_info.with_constrain = true; + broadcast_info[ValueName(op->result(0))] = base_info; + } + } + + for (auto opresult : op->results()) { + if (tensor_map.count(opresult) == 0) { + continue; + } + } + } + if (apply_group_schedule) { std::unordered_set output_tensor_names; for (auto value : group->GetGroupOutputValues()) { output_tensor_names.insert(ValueName(value)); } + std::shared_ptr group_tile_info = + GetGroupTileInfo(group); std::unique_ptr group_scheduler = - ir::GroupScheduler::Make( - &ir_sch, output_tensor_names, target_, /* is_dy_shape = */ true); + ir::GroupScheduler::Make(&ir_sch, + output_tensor_names, + target_, + /* is_dy_shape = */ true, + group_tile_info); + group_scheduler->Schedule(); + cond2func_bodies = group_scheduler->GetIRs(); } else { cond2func_bodies.emplace_back(ir::Expr(true), @@ -280,8 +463,10 @@ std::vector OpLowererImpl::LowerMapExpr( for (auto value : group->GetGroupOutputValues()) { output_tensor_names.insert(ValueName(value)); } + + std::shared_ptr group_tile_info; ir::StaticShapeGroupScheduler group_scheduler( - &ir_sch, output_tensor_names, target_); + &ir_sch, output_tensor_names, target_, group_tile_info); group_scheduler.MapExprSchedule(); VLOG(3) << "After group schedule, ir is: \n" << ir_sch.GetModule().GetExprs().at(0); @@ -323,24 +508,66 @@ std::vector OpLowererImpl::LowerGroup( &group_func_arg_tensors, &tensor_map); } - std::vector func_bodies = LowerOps(group, - ops, - do_op_schedule, - schedule_determine_func, - &group_func_arg_tensors, - &tensor_map, - &tmp_tensor_info); + std::vector func_bodies = + LowerOps(group, + ops, + do_op_schedule, + &OpLowererImpl::DyShapeScheduleDetermineFunction, + &group_func_arg_tensors, + &tensor_map, + &tmp_tensor_info); + + std::unordered_set<::pir::Value> inner_genevalue; + std::unordered_set<::pir::Operation*> ops_set(ops.begin(), ops.end()); + for (auto* op : ops) { + for (size_t i = 0; i < op->num_results(); ++i) { + inner_genevalue.insert(op->result(i)); + } + } + + BuildBroadcastInfo(group); + + for (auto& op : group->output_ops) { + // collect all output tensor. + if (op->name() == "cinn_op.yield_store") { + auto input_var_name = ValueName(op->operand_source(0)); + if (broadcast_info.count(input_var_name)) { + auto base_info = broadcast_info[input_var_name]; + base_info.with_constrain = true; + broadcast_info[ValueName(op->result(0))] = base_info; + } + } + + for (auto opresult : op->results()) { + if (tensor_map.count(opresult) == 0) { + continue; + } + } + } // 2.Do group schedule. + ir::ModuleExpr mod_expr(func_bodies); - ir::IRSchedule ir_sch(mod_expr); - ir_sch.MergeExprs(); - VLOG(3) << "After lower, ir is: \n" << ir_sch.GetModule().GetExprs().at(0); - if (apply_group_schedule) { - DoGroupSchedule(ir_sch, group, tensor_map, tmp_tensor_info); - VLOG(3) << "After group schedule, ir is: \n" - << ir_sch.GetModule().GetExprs().at(0); + std::shared_ptr ir_sch = + std::make_shared(mod_expr); + + auto have_dy_shape = false; + for (auto d : group->loop_ranges) { + if (d < 0) { + have_dy_shape = true; + } } + if (have_dy_shape) { + ir_sch = std::make_shared( + mod_expr, -1, false, cinn::utils::ErrorMessageLevel::kGeneral, true); + } + ir_sch->MergeExprs(); + VLOG(3) << "After lower, ir is: \n" << ir_sch->GetModule().GetExprs().at(0); + // if (apply_group_schedule) { + DoGroupSchedule(*(ir_sch.get()), group, tensor_map, tmp_tensor_info); + VLOG(3) << "After group schedule, ir is: \n" + << ir_sch->GetModule().GetExprs().at(0); + // } // 3.Do post-processing, // including preparing function args and temporary variables, @@ -349,11 +576,140 @@ std::vector OpLowererImpl::LowerGroup( return PostProcess(group, tensor_map, do_op_schedule, - {ir_sch.GetModule().GetExprs().at(0)}, + {ir_sch->GetModule().GetExprs().at(0)}, &group_func_arg_tensors, &group_func_args); } +void OpLowererImpl::BuildBroadcastInfo(const GroupPtr& group) { + // TODO(phlrain): this is primary verion for loop aligment + // will be update by a new method + auto& align_info = group->alignment_schedule_info; + auto& ops = group->ops; + for (auto op1 : ops) { + auto it = align_info.find(op1); + if (it == align_info.end()) { + continue; + } + + PADDLE_ENFORCE_EQ( + it->second.size(), + 1, + phi::errors::Unimplemented("only suppopt one transform yet")); + + if (it->second[0].type == ScheduleAlignType::kBroadcast) { + // get broadcast op + auto broadcast_axes = it->second[0].axis_info; + auto output_shape = it->second[0].factor_info; + + phi::DDim in_dim; + + if (it->first->name() == "cinn_op.reshape") { + // TODO(phlrain): deal with reshape in a better way + if (it->first->result(0).use_count() == 1 && + it->first->result(0).first_use().owner()->isa<::pir::YieldOp>()) { + continue; + } + } + + if ((it->first->name() != "cinn_op.reshape") && + (it->first->name() != "cinn_op.broadcast") && + (it->first->num_operands() == 1)) { + in_dim = it->first->operand_source(0) + .type() + .dyn_cast() + .dims(); + } else { + in_dim = it->first->result(0) + .type() + .dyn_cast() + .dims(); + } + + cinn::ir::BroadcastInfo info; + if (in_dim.size() == 1u && in_dim[0] == 1u) { + info.full_broadcast = true; + for (size_t i = 0; i < output_shape.size(); ++i) { + info.broadcast_axes.push_back(i); + info.output_shape.push_back(output_shape[i]); + } + } else if (in_dim.size() == broadcast_axes.size()) { + if (in_dim.size() != output_shape.size()) { + info.split_first = true; + + if (broadcast_axes.size() == 1) { + std::vector temp_shape(output_shape.size(), 1); + temp_shape[broadcast_axes[0]] = output_shape[broadcast_axes[0]]; + info.split_info.emplace_back(0, temp_shape); + + for (size_t i = 0; i < output_shape.size(); ++i) { + if (i != broadcast_axes[0]) { + info.broadcast_axes.push_back(i); + info.output_shape.push_back(output_shape[i]); + } + } + } else { + throw std::runtime_error("not support multi dim broadcast yet"); + } + } else { + for (size_t i = 0; i < broadcast_axes.size(); ++i) { + if (in_dim[i] != output_shape[broadcast_axes[i]]) { + if (in_dim[i] != 1) { + throw std::runtime_error("Only support 1 - D broadcast "); + } + info.broadcast_axes.push_back(i); + info.output_shape.push_back(output_shape[broadcast_axes[i]]); + } + } + } + } else { + // only deal with broadcast axes + std::set axes_set; + for (size_t i = 0; i < broadcast_axes.size(); ++i) { + axes_set.insert(broadcast_axes[i]); + if (in_dim[broadcast_axes[i]] != 1) { + throw std::runtime_error("Only support 1 - D broadcast "); + } + + info.broadcast_axes.push_back(broadcast_axes[i]); + info.output_shape.push_back(output_shape[broadcast_axes[i]]); + } + } + PADDLE_ENFORCE_NE( + info.broadcast_axes.size(), + 0, + phi::errors::PreconditionNotMet("broadcast axes can not be zero")); + + for (size_t i = 0; i < it->first->num_operands(); ++i) { + if (!align_info.count(it->first->operand_source(i).defining_op())) { + info.first_broadcast = true; + break; + } + } + + auto op_out = it->first->result(0); + info.op_name = it->first->name(); + broadcast_info[ValueName(op_out)] = info; + + for (auto use_it = op_out.use_begin(); use_it != op_out.use_end(); + ++use_it) { + if (use_it->owner()->name() == "cf.yield") { + continue; + } + if (CompatibleInfo::OpKind(*(use_it->owner())) == + framework::kBroadcast) { + if (!info.full_broadcast) { + broadcast_to_elementwise[ValueName(use_it->owner()->result(0))] = + info; + } + } + } + } else { + throw std::runtime_error("only supportbroadcast type for now"); + } + } +} + std::vector OpLowererImpl::LowerCustomCall( const GroupPtr& group) { auto& ops = group->ops; @@ -420,6 +776,7 @@ std::vector OpLowererImpl::PostProcess( } group->output_names.clear(); + // collect all output tensor. for (auto op_result : group->GetGroupOutputValues()) { if (tensor_map.count(op_result) == 0) { @@ -489,7 +846,6 @@ std::vector OpLowererImpl::PostProcess( } } } - std::vector lowered_funcs; for (ir::Expr func_body : func_bodies) { optim::EliminateDeadScheduleBlock(&(func_body), group->output_names); @@ -524,20 +880,46 @@ std::vector OpLowererImpl::LowerOps( std::unordered_map* tmp_tensor_info) { auto& strategy = Operator::GetAttrs("CINNStrategy"); std::vector func_bodies; + std::unordered_set<::pir::Value> inner_used_value; + for (auto* op : ops) { + for (size_t i = 0; i < op->num_operands(); ++i) { + inner_used_value.insert(op->operand_source(i)); + } + } + + std::unordered_set<::pir::Operation*> not_used_op; + for (auto* op : ops) { + bool used = false; + for (size_t i = 0; i < op->num_results(); ++i) { + if (inner_used_value.count(op->result(i))) { + used = true; + break; + } + } + + if (!used) { + not_used_op.insert(op); + } + } + for (auto* op : ops) { VLOG(4) << "start lowering op:" << op->name(); + std::string cinn_op_name = CompatibleInfo::OpName(*op); + + VLOG(4) << "cinn op name " << cinn_op_name << std::endl; + // 1.Select Op impl std::vector op_func_arg_tensors = CollectInputTensor(group, op, group_func_arg_tensors, tensor_map); VLOG(4) << "input size:" << op_func_arg_tensors.size(); - std::string cinn_op_name = CompatibleInfo::OpName(*op); const hlir::framework::Operator* cinn_op = Operator::Get(cinn_op_name); std::shared_ptr op_impl = nullptr; if (FLAGS_cinn_bucket_compile) { std::vector out_types; std::vector> out_shapes; CollectOutputInfo(op, &out_types, &out_shapes, group); + CHECK_EQ(out_types.size(), out_shapes.size()); VLOG(4) << "out_types.size(): " << out_types.size(); NodeAttr node_attrs = details::CollectAttrs(*op); @@ -568,14 +950,17 @@ std::vector OpLowererImpl::LowerOps( std::vector funcs = DoOpLower( op_impl, op, tensor_map, tmp_tensor_info, &op_func_arg_tensors); - if (apply_op_schedule && (this->*schedule_determine_func)(op)) { - // 3.Perform the schedule of Op - func_bodies.push_back(DoOpSchedule(op_impl, op_func_arg_tensors, funcs)); - } else { - for (const ir::LoweredFunc& func : funcs) { - func_bodies.push_back(func->body); - } + if (ops.size() > 1 && not_used_op.count(op) && + (op->name() == "cinn_op.reshape")) { + erase_reshape.insert(op); + continue; } + + for (const ir::LoweredFunc& func : funcs) { + func_bodies.push_back(func->body); + } + + remain_ops.push_back(op); } VLOG(4) << "group_func_arg_tensors.size(): " @@ -692,13 +1077,25 @@ ir::Expr OpLowererImpl::DoGroupSchedule( const std::unordered_map<::pir::Value, ir::Tensor>& tensor_map, const std::unordered_map& tmp_tensor_info) { VLOG(3) << "using StaticShapeGroupScheduler to schedule group."; + bool have_dy_shape = false; + for (auto d : group->loop_ranges) { + if (d < 0) { + have_dy_shape = true; + } + } + + auto group_tile_info = GetGroupTileInfo(group); + std::unordered_set output_tensor_names; for (auto value : group->GetGroupOutputValues()) { output_tensor_names.insert(ValueName(value)); } std::unique_ptr group_scheduler = - ir::GroupScheduler::Make( - &ir_sch, output_tensor_names, target_, /* is_dy_shape = */ false); + ir::GroupScheduler::Make(&ir_sch, + output_tensor_names, + target_, + /* is_dy_shape = */ true, + group_tile_info); group_scheduler->Schedule(); return ir_sch.GetModule().GetExprs().at(0); } diff --git a/paddle/cinn/hlir/framework/pir/op_lowering_impl.h b/paddle/cinn/hlir/framework/pir/op_lowering_impl.h index fff73071becb9..c449e7dcc2efa 100644 --- a/paddle/cinn/hlir/framework/pir/op_lowering_impl.h +++ b/paddle/cinn/hlir/framework/pir/op_lowering_impl.h @@ -245,6 +245,9 @@ class OpLowererImpl : public OpLowererImplBase { ir::Tensor GetTensorSymbolic(const GroupPtr& group, const ::pir::Value& value); + std::shared_ptr GetGroupTileInfo( + const GroupPtr& group); + void CollectOutputInfo(::pir::Operation* op, std::vector* out_types, std::vector>* out_shapes, @@ -267,9 +270,25 @@ class OpLowererImpl : public OpLowererImplBase { common::Type GetTensorDtype(const ::pir::Value& value); + void BuildBroadcastInfo(const GroupPtr& group); + Target target_; PrettyNamer* name_gene_; + + std::vector thread_sync_before_names; + std::set shared_var_names; + std::set direct_output_var_names; + + std::vector broadcast_output_names; + + std::unordered_map broadcast_info; + std::unordered_map + broadcast_to_elementwise; + + std::unordered_set<::pir::Operation*> erase_reshape; + + std::vector<::pir::Operation*> remain_ops; }; } // namespace pir diff --git a/paddle/cinn/hlir/framework/pir/utils.cc b/paddle/cinn/hlir/framework/pir/utils.cc index 7d0acaa3cc92b..80d0597bb3ed3 100644 --- a/paddle/cinn/hlir/framework/pir/utils.cc +++ b/paddle/cinn/hlir/framework/pir/utils.cc @@ -87,7 +87,24 @@ class OpTransInfo { {"batch_norm_grad", {"ReserveSpace"}}}; std::unordered_set default_deny_ops_{ - "feed", "fetch", "conv2d", "conv2d_grad", "dropout", "matmul"}; + "feed", + "fetch", + "conv2d", + "conv2d_grad", + "dropout", + "slice", + "concat", + "gather_nd", + "pool2d", + "split", + "matmul", + "matmul_grad", + "transpose", + "embedding_grad", + "embedding", + "gather", + "arange", + }; }; std::unordered_set StringSplit(const std::string& str, @@ -132,6 +149,21 @@ bool HaveZeroDimInput(const ::pir::Operation& op) { auto tensor_type = type.dyn_cast<::pir::DenseTensorType>(); return tensor_type && tensor_type.dims().size() == 0U; }; + + auto HasNegDim = [](const ::pir::Type& type) { + auto tensor_type = type.dyn_cast<::pir::DenseTensorType>(); + + if (tensor_type) { + for (size_t i = 0; i < tensor_type.dims().size(); ++i) { + if (tensor_type.dims()[i] < 0) { + return true; + } + } + } + + return false; + }; + // Judge for vector auto HasZeroDimInVT = [&](const std::vector<::pir::Type>& types) { for (auto& type : types) { @@ -145,7 +177,7 @@ bool HaveZeroDimInput(const ::pir::Operation& op) { if (!value || !value.type()) continue; if (auto vector_type = value.type().dyn_cast<::pir::VectorType>()) { if (HasZeroDimInVT(vector_type.data())) return true; - } else if (HasZeroDim(value.type())) { + } else if (HasZeroDim(value.type()) || HasNegDim(value.type())) { return true; } } @@ -267,7 +299,7 @@ bool IsRegisteredInCINN(const ::pir::Operation& op) { } bool IsSupportForCinn(const ::pir::Operation& op) { - if (!AllInputDenseTensor(op) || HaveZeroDimInput(op) || UnimplementOps(op)) { + if (!AllInputDenseTensor(op) || UnimplementOps(op)) { VLOG(4) << "Found " << op.name() << " HaveZeroDimInput or UnimplementOps or NotAllInputDenseTensor. " << "So mark IsSupportForCinn: " << false; @@ -403,6 +435,8 @@ static utils::Attribute ConvertArrayAttribute( "ArrayAttribute"; } } + } else if (src_attr.isa<::pir::shape::SymbolAttribute>()) { + // do nothing for now } else { LOG(FATAL) << "unknown Attribute: " << src_attr; } @@ -483,7 +517,7 @@ OpPatternKind CompatibleInfo::OpKind(const ::pir::Operation& op) { auto& op_pattern_dict = Operator::GetAttrs("OpPattern"); auto op_name = CompatibleInfo::OpName(op); if (op_name == "generate_shape") { - return hlir::framework::kNonFusible; + return hlir::framework::kElementWise; } const hlir::framework::Operator* cinn_op = Operator::Get(op_name); CHECK(op_pattern_dict.Find(cinn_op)); diff --git a/paddle/cinn/hlir/op/elementwise.cc b/paddle/cinn/hlir/op/elementwise.cc index b215e0dd85952..6a9f41e84cf0b 100644 --- a/paddle/cinn/hlir/op/elementwise.cc +++ b/paddle/cinn/hlir/op/elementwise.cc @@ -73,6 +73,7 @@ std::shared_ptr StrategyForElementwise( CHECK(!args.empty()) << "The input argument of " << op_name << " compute is empty! Please check."; CINNValuePack pack_args = args[0]; + CHECK_GE(pack_args.size(), 1U) << "1 input tensor for " << op_name << " compute"; CHECK_EQ(pack_args.size(), 2U); @@ -1128,6 +1129,120 @@ std::shared_ptr StrategyForCast( return strategy; } +std::shared_ptr StrategyForCastSymbolic( + const framework::NodeAttr &attrs, + const std::vector &inputs, + const std::vector &out_type, + const std::vector> &output_shapes, + const Target &target) { + framework::CINNCompute cast_compute( + [=](lang::Args args, lang::RetValue *ret) { + CHECK(!args.empty()) + << "The input arguments of Cast compute is empty! Please check.\n"; + CINNValuePack pack_args = args[0]; + CHECK_GE(pack_args.size(), 1U) + << "at least 1 input tensors for Cast compute\n"; + Expr A = pack_args[0]; + CHECK(A.as_tensor()); + CHECK(!output_shapes.empty()); + auto tensor_A = A.as_tensor_ref(); + auto stages = CreateStages({tensor_A}); + VLOG(3) << "A shape: " << utils::Join(tensor_A->shape, ", ") + << ", output_shapes: " << utils::Join(output_shapes[0], ", "); + CHECK_EQ(pack_args.size(), 2U); + std::string tensor_name = pack_args[1].operator std::string(); + ir::Tensor out = pe::Cast(tensor_A, out_type[0], tensor_name); + std::vector res; + stages->InsertLazily(out); + res.push_back(CINNValue(out)); + CHECK(!out_type.empty()) + << "Output type of Cast is empty! Please check.\n"; + res.push_back(CINNValue(stages)); + *ret = CINNValuePack{res}; + }); + + auto strategy = std::make_shared(); + strategy->AddImpl(cast_compute, lang::PackedFunc(), "strategy.cast.x86", 1); + return strategy; +} + +std::shared_ptr StrategyForYieldStore( + const framework::NodeAttr &attrs, + const std::vector &inputs, + const std::vector &out_type, + const std::vector> &output_shapes, + const Target &target) { + framework::CINNCompute cast_compute( + [=](lang::Args args, lang::RetValue *ret) { + CHECK(!args.empty()) + << "The input arguments of Cast compute is empty! Please check.\n"; + CINNValuePack pack_args = args[0]; + CHECK_GE(pack_args.size(), 1U) + << "at least 1 input tensors for Cast compute\n"; + Expr A = pack_args[0]; + CHECK(A.as_tensor()); + CHECK(!output_shapes.empty()); + auto tensor_A = A.as_tensor_ref(); + auto stages = CreateStages({tensor_A}); + VLOG(3) << "A shape: " << utils::Join(tensor_A->shape, ", ") + << ", output_shapes: " << utils::Join(output_shapes[0], ", "); + CHECK_EQ(pack_args.size(), 2U); + std::string tensor_name = pack_args[1].operator std::string(); + ir::Tensor out = pe::Cast(tensor_A, out_type[0], tensor_name); + std::vector res; + stages->InsertLazily(out); + res.push_back(CINNValue(out)); + CHECK(!out_type.empty()) + << "Output type of Cast is empty! Please check.\n"; + res.push_back(CINNValue(stages)); + *ret = CINNValuePack{res}; + }); + + auto strategy = std::make_shared(); + strategy->AddImpl(cast_compute, + GetElementwiseScheduleFunc(output_shapes, target), + "strategy.reshape.x86", + 1); + return strategy; +} + +std::shared_ptr StrategyForYieldStoreSymbolic( + const framework::NodeAttr &attrs, + const std::vector &inputs, + const std::vector &out_type, + const std::vector> &output_shapes, + const Target &target) { + framework::CINNCompute cast_compute( + [=](lang::Args args, lang::RetValue *ret) { + CHECK(!args.empty()) + << "The input arguments of Cast compute is empty! Please check.\n"; + CINNValuePack pack_args = args[0]; + CHECK_GE(pack_args.size(), 1U) + << "at least 1 input tensors for Cast compute\n"; + Expr A = pack_args[0]; + CHECK(A.as_tensor()); + CHECK(!output_shapes.empty()); + auto tensor_A = A.as_tensor_ref(); + auto stages = CreateStages({tensor_A}); + VLOG(3) << "A shape: " << utils::Join(tensor_A->shape, ", ") + << ", output_shapes: " << utils::Join(output_shapes[0], ", "); + CHECK_EQ(pack_args.size(), 2U); + std::string tensor_name = pack_args[1].operator std::string(); + ir::Tensor out = pe::Cast(tensor_A, out_type[0], tensor_name); + std::vector res; + stages->InsertLazily(out); + res.push_back(CINNValue(out)); + CHECK(!out_type.empty()) + << "Output type of Cast is empty! Please check.\n"; + res.push_back(CINNValue(stages)); + *ret = CINNValuePack{res}; + }); + + auto strategy = std::make_shared(); + strategy->AddImpl(cast_compute, lang::PackedFunc(), "strategy.store.x86", 1); + return strategy; +} + std::vector InferDtypeForCast(const std::vector &inputs_type, const framework::AttrMapType &attrs) { CHECK(attrs.count("dtype")); @@ -1441,6 +1556,25 @@ CINN_REGISTER_HELPER(elementwise_ops) { .set_num_outputs(1) .set_attr( "CINNStrategy", cinn::hlir::op::StrategyForCast) + .set_attr( + "CINNStrategySymbolic", cinn::hlir::op::StrategyForCastSymbolic) + .set_attr("infershape", + MakeOpFunction(cinn::hlir::op::InferShapeForElementwise)) + .set_attr("inferdtype", MakeOpFunction(cinn::hlir::op::InferDtypeForCast)) + .set_attr("inferlayout", + MakeOpFunction(cinn::hlir::op::InferLayoutForElementwise)) + .set_attr( + "OpPattern", cinn::hlir::framework::OpPatternKind::kElementWise) + .set_support_level(4); + + CINN_REGISTER_OP(yield_store) + .describe("This operator is used to cast input tensor's type to target.") + .set_num_inputs(1) + .set_num_outputs(1) + .set_attr( + "CINNStrategy", cinn::hlir::op::StrategyForYieldStore) + .set_attr( + "CINNStrategySymbolic", cinn::hlir::op::StrategyForYieldStoreSymbolic) .set_attr("infershape", MakeOpFunction(cinn::hlir::op::InferShapeForElementwise)) .set_attr("inferdtype", MakeOpFunction(cinn::hlir::op::InferDtypeForCast)) diff --git a/paddle/cinn/hlir/pe/broadcast.cc b/paddle/cinn/hlir/pe/broadcast.cc index 439ff30e2691c..29189a5b1987c 100644 --- a/paddle/cinn/hlir/pe/broadcast.cc +++ b/paddle/cinn/hlir/pe/broadcast.cc @@ -357,7 +357,7 @@ Tensor BroadcastTo(const Tensor& A, [=](const std::vector& indice) { std::vector broadcast_indice; for (int idx = 0; idx < axes.size(); ++idx) { - int a_shape_i = A_shape[idx].as_int32(); + int a_shape_i = A_shape[idx].as_int64(); if (a_shape_i == 1) { broadcast_indice.push_back(ir::Expr(0)); } else if (a_shape_i == out_shape[axes[idx]]) { diff --git a/paddle/cinn/hlir/pe/elementwise.cc b/paddle/cinn/hlir/pe/elementwise.cc index 60933cd66c4b0..6bda344a413d2 100644 --- a/paddle/cinn/hlir/pe/elementwise.cc +++ b/paddle/cinn/hlir/pe/elementwise.cc @@ -197,30 +197,47 @@ ir::Tensor Reshape(const ir::Tensor& A, const std::vector& A_expr_shape = A->shape; int input_total_size = 1; int output_total_size = 1; - for (auto& i : A_expr_shape) { - CHECK(i.is_constant()) << "Input tensor's shape should be constant value."; - input_total_size *= static_cast(i.get_constant()); + std::vector A_stride_info; + int stride_base = 1; + A_stride_info.push_back(Expr(stride_base)); + + for (int i = A_expr_shape.size() - 1; i > 0; i--) { + stride_base *= static_cast(A_expr_shape[i].get_constant()); + A_stride_info.insert(A_stride_info.begin(), Expr(stride_base)); + } + + std::vector new_stride_info; + stride_base = 1; + new_stride_info.push_back(Expr(stride_base)); + + for (int i = new_shape.size() - 1; i > 0; --i) { + stride_base *= new_shape[i]; + + new_stride_info.insert(new_stride_info.begin(), Expr(stride_base)); } + for (auto& i : new_shape) { output_total_size *= i; new_expr_shape.push_back(Expr(i)); } - CHECK_EQ(input_total_size, output_total_size) - << "In op reshape, the input tensor and output tensor's total size " - "should be equal, please check!"; + auto res = Compute( new_expr_shape, [=](const std::vector& indice) { - Expr offset = Expr(0); - for (int i = 0; i < indice.size(); i++) { - offset = offset * new_expr_shape[i] + indice[i]; + Expr offset = indice[0] * new_stride_info[0]; + for (int i = 1; i < indice.size(); i++) { + offset = offset + indice[i] * new_stride_info[i]; } std::vector indice_a; for (int i = A_expr_shape.size() - 1; i >= 0; i--) { - auto temp = common::AutoSimplify(offset % A_expr_shape[i]); + auto inner_offset = offset; + if (i != (A_expr_shape.size() - 1)) { + inner_offset = inner_offset / A_stride_info[i]; + } + auto temp = inner_offset % A_expr_shape[i]; indice_a.insert(indice_a.begin(), temp); - offset = (offset - temp) / A_expr_shape[i]; } + LOG(INFO) << "indice_a = " << indice_a[0]; return A(indice_a); }, name); @@ -232,33 +249,47 @@ ir::Tensor Reshape(const ir::Tensor& A, const std::string& name) { std::vector new_expr_shape; const std::vector& A_expr_shape = A->shape; - ir::Expr input_total_size(1); - for (auto& i : A_expr_shape) { - // CHECK(i.is_constant()) << "Input tensor's shape should be constant - // value."; - input_total_size = ir::Mul::Make(input_total_size, i); + Expr input_total_size(1); + Expr output_total_size(1); + + std::vector A_stride_info; + Expr stride_base(1); + A_stride_info.push_back(stride_base); + for (int i = A_expr_shape.size() - 1; i > 0; i--) { + stride_base = stride_base * A_expr_shape[i]; + A_stride_info.insert(A_stride_info.begin(), Expr(stride_base)); + } + + std::vector new_stride_info; + stride_base = Expr(1); + new_stride_info.push_back(Expr(stride_base)); + for (int i = new_shape.size() - 1; i > 0; --i) { + stride_base = stride_base * new_shape[i]->dim_expr; + new_stride_info.insert(new_stride_info.begin(), Expr(stride_base)); } - ir::Expr output_total_size(1); + for (auto& i : new_shape) { - output_total_size = ir::Mul::Make(output_total_size, i->dim_expr); + output_total_size = output_total_size * i->dim_expr; new_expr_shape.push_back(i->dim_expr); } - // CHECK_EQ(input_total_size, output_total_size) - // << "In op reshape, the input tensor and output tensor's total size " - // "should be equal, please check!"; + auto res = Compute( new_expr_shape, [=](const std::vector& indice) { - Expr offset = Expr(0); - for (int i = 0; i < indice.size(); i++) { - offset = offset * new_expr_shape[i] + indice[i]; + Expr offset = indice[0] * new_stride_info[0]; + for (int i = 1; i < indice.size(); i++) { + offset = offset + indice[i] * new_stride_info[i]; } std::vector indice_a; for (int i = A_expr_shape.size() - 1; i >= 0; i--) { - auto temp = offset % A_expr_shape[i]; + auto inner_offset = offset; + if (i != (A_expr_shape.size() - 1)) { + inner_offset = inner_offset / A_stride_info[i]; + } + auto temp = inner_offset % A_expr_shape[i]; indice_a.insert(indice_a.begin(), temp); - offset = (offset - temp) / A_expr_shape[i]; } + LOG(INFO) << "indice_a = " << indice_a[0]; return A(indice_a); }, name); @@ -277,6 +308,14 @@ ir::Tensor Cast(const ir::Tensor& A, return res; } +ir::Tensor Store(const ir::Tensor& A, const std::string& name) { + auto res = Compute( + A->shape, + [=](const std::vector& indices) { return A(indices); }, + name); + return res; +} + ir::Tensor Arange(const float start, const float stop, const float step, diff --git a/paddle/cinn/hlir/pe/elementwise.h b/paddle/cinn/hlir/pe/elementwise.h index a9bbb71193255..64c5cccb125b7 100644 --- a/paddle/cinn/hlir/pe/elementwise.h +++ b/paddle/cinn/hlir/pe/elementwise.h @@ -139,6 +139,9 @@ ir::Tensor Cast(const ir::Tensor& A, const Type& dtype, const std::string& name = UniqName("T_Elementwise_Cast_out")); +ir::Tensor Store(const ir::Tensor& A, + const std::string& name = UniqName("T_Elementwise_Store_out")); + ir::Tensor Arange( const float start, const float stop, diff --git a/paddle/cinn/ir/group_schedule/base_group_scheduler.cc b/paddle/cinn/ir/group_schedule/base_group_scheduler.cc index a740ad268cb09..6504af8aae5f6 100644 --- a/paddle/cinn/ir/group_schedule/base_group_scheduler.cc +++ b/paddle/cinn/ir/group_schedule/base_group_scheduler.cc @@ -23,13 +23,14 @@ std::unique_ptr GroupScheduler::Make( ir::IRSchedule* ir_sch, const std::unordered_set& output_tensor_names, const cinn::common::Target& target, - bool is_dy_shape) { + bool is_dy_shape, + const std::shared_ptr& group_tile_info) { if (is_dy_shape) { return std::make_unique( - ir_sch, output_tensor_names, target); + ir_sch, output_tensor_names, target, group_tile_info); } else { return std::make_unique( - ir_sch, output_tensor_names, target); + ir_sch, output_tensor_names, target, group_tile_info); } } diff --git a/paddle/cinn/ir/group_schedule/base_group_scheduler.h b/paddle/cinn/ir/group_schedule/base_group_scheduler.h index 33cce051f1845..eb409af1cb3ce 100644 --- a/paddle/cinn/ir/group_schedule/base_group_scheduler.h +++ b/paddle/cinn/ir/group_schedule/base_group_scheduler.h @@ -14,6 +14,7 @@ #pragma once #include "paddle/cinn/common/target.h" +#include "paddle/cinn/ir/group_schedule/tactic/schedule_tactic.h" #include "paddle/cinn/ir/schedule/ir_schedule.h" #include "paddle/cinn/ir/schedule_block_graph.h" @@ -29,10 +30,12 @@ class GroupScheduler { public: GroupScheduler(ir::IRSchedule* ir_sch, const std::unordered_set& output_tensor_names, - const cinn::common::Target& target) + const cinn::common::Target& target, + const std::shared_ptr& group_tile_info) : ir_sch_(ir_sch), output_tensor_names_(output_tensor_names), - target_(target) { + target_(target), + group_tile_info_(group_tile_info) { schedule_block_graph_ = std::make_unique(*ir_sch_); } @@ -40,7 +43,8 @@ class GroupScheduler { ir::IRSchedule* ir_sch, const std::unordered_set& output_tensor_names, const cinn::common::Target& target, - bool is_dy_shape = false); + bool is_dy_shape = false, + const std::shared_ptr& group_tile_info = nullptr); virtual ~GroupScheduler() = default; @@ -57,6 +61,8 @@ class GroupScheduler { // Graph in units of ScheduleBlockNode, each node corresponds to a // ScheduleBlock in IR. std::unique_ptr schedule_block_graph_; + + std::shared_ptr group_tile_info_; }; } // namespace ir diff --git a/paddle/cinn/ir/group_schedule/dy_shape_group_scheduler.cc b/paddle/cinn/ir/group_schedule/dy_shape_group_scheduler.cc index d5a64b6d8f7f1..037c1e7ad5fec 100644 --- a/paddle/cinn/ir/group_schedule/dy_shape_group_scheduler.cc +++ b/paddle/cinn/ir/group_schedule/dy_shape_group_scheduler.cc @@ -18,11 +18,15 @@ #include "paddle/cinn/ir/group_schedule/tactic/arrange_storage_tactic.h" #include "paddle/cinn/ir/group_schedule/tactic/bind_cuda_tactic.h" #include "paddle/cinn/ir/group_schedule/tactic/compute_inline_tactic.h" +#include "paddle/cinn/ir/group_schedule/tactic/loop_reorder_alignment_tactic.h" #include "paddle/cinn/ir/group_schedule/tactic/optimize_reduction_tactic.h" +#include "paddle/cinn/ir/group_schedule/tactic/tile_first_general_tactic.h" #include "paddle/cinn/ir/group_schedule/tactic/tile_tactic.h" #include "paddle/cinn/ir/ir_analyzer/ir_analyzer.h" #include "paddle/cinn/ir/op/ir_operators.h" +PD_DECLARE_bool(cinn_bucket_compile); + namespace cinn { namespace ir { @@ -32,12 +36,8 @@ void DynamicShapeGroupScheduler::Init() { VLOG(4) << "original group func body: \n" << ir_sch_->GetModule().GetExprs()[0]; InitBuckets(); - tactics_.emplace_back(new AlignIterSpaceTactic()); - tactics_.emplace_back(new ComputeInlineTactic()); - tactics_.emplace_back(new TileTactic()); - tactics_.emplace_back(new OptimizeReductionTactic()); - tactics_.emplace_back(new BindCudaTactic()); - tactics_.emplace_back(new ArrangeStorageTactic()); + tactics_.emplace_back(CreateLoopReorderAlignmentTactic()); + tactics_.emplace_back(CreateTileFirstGeneralTactic()); } void DynamicShapeGroupScheduler::InitBuckets() { @@ -85,7 +85,8 @@ void DynamicShapeGroupScheduler::InitBuckets() { ScheduleContext schedule_context{output_names, target_, std::move(iter_space_info), - std::move(bucket_info)}; + std::move(bucket_info), + group_tile_info_}; BucketContext bucket_context{std::move(predicate), std::move(ir_sch), std::move(schedule_block_graph), diff --git a/paddle/cinn/ir/group_schedule/dy_shape_group_scheduler.h b/paddle/cinn/ir/group_schedule/dy_shape_group_scheduler.h index e226059011b63..d9bff4ef8939f 100644 --- a/paddle/cinn/ir/group_schedule/dy_shape_group_scheduler.h +++ b/paddle/cinn/ir/group_schedule/dy_shape_group_scheduler.h @@ -28,8 +28,9 @@ class DynamicShapeGroupScheduler : public GroupScheduler { DynamicShapeGroupScheduler( ir::IRSchedule* ir_sch, const std::unordered_set& output_tensor_names, - const cinn::common::Target& target) - : GroupScheduler(ir_sch, output_tensor_names, target) { + const cinn::common::Target& target, + const std::shared_ptr& group_tile_info) + : GroupScheduler(ir_sch, output_tensor_names, target, group_tile_info) { Init(); } diff --git a/paddle/cinn/ir/group_schedule/st_shape_group_scheduler.h b/paddle/cinn/ir/group_schedule/st_shape_group_scheduler.h index 337817995eb0f..d17d8618433fa 100644 --- a/paddle/cinn/ir/group_schedule/st_shape_group_scheduler.h +++ b/paddle/cinn/ir/group_schedule/st_shape_group_scheduler.h @@ -46,8 +46,9 @@ class StaticShapeGroupScheduler : public GroupScheduler { StaticShapeGroupScheduler( ir::IRSchedule* ir_sch, const std::unordered_set& output_tensor_names, - const cinn::common::Target& target) - : GroupScheduler(ir_sch, output_tensor_names, target) {} + const cinn::common::Target& target, + const std::shared_ptr& group_tile_info) + : GroupScheduler(ir_sch, output_tensor_names, target, group_tile_info) {} void Schedule() override; diff --git a/paddle/cinn/ir/group_schedule/tactic/CMakeLists.txt b/paddle/cinn/ir/group_schedule/tactic/CMakeLists.txt index e8205f7244bb1..b6a2f06760646 100644 --- a/paddle/cinn/ir/group_schedule/tactic/CMakeLists.txt +++ b/paddle/cinn/ir/group_schedule/tactic/CMakeLists.txt @@ -6,3 +6,5 @@ gather_srcs(cinnapi_src SRCS compute_inline_tactic.cc) gather_srcs(cinnapi_src SRCS optimize_reduction_tactic.cc) gather_srcs(cinnapi_src SRCS bind_cuda_tactic.cc) gather_srcs(cinnapi_src SRCS arrange_storage_tactic.cc) +gather_srcs(cinnapi_src SRCS loop_reorder_alignment_tactic.cc) +gather_srcs(cinnapi_src SRCS tile_first_general_tactic.cc) diff --git a/paddle/cinn/ir/group_schedule/tactic/align_iter_space_tactic.cc b/paddle/cinn/ir/group_schedule/tactic/align_iter_space_tactic.cc index 14fde3b148a52..dcc72e4a217d8 100644 --- a/paddle/cinn/ir/group_schedule/tactic/align_iter_space_tactic.cc +++ b/paddle/cinn/ir/group_schedule/tactic/align_iter_space_tactic.cc @@ -23,6 +23,18 @@ namespace cinn { namespace ir { +class AlignIterSpaceTactic final : public ScheduleTactic { + public: + void Init(ScheduleContext* context) override; + + void Apply(ir::IRSchedule* sch, const std::string& block_id) override; + + std::string TacticName() const override { return "AlignIterSpaceTactic"; } + + private: + ScheduleContext* context_; +}; + void AlignIterSpaceTactic::Init(ScheduleContext* context) { context_ = context; } @@ -84,5 +96,9 @@ void AlignIterSpaceTactic::Apply(ir::IRSchedule* sch, } } +std::unique_ptr CreateAlignIterSpaceTactic() { + return std::make_unique(); +} + } // namespace ir } // namespace cinn diff --git a/paddle/cinn/ir/group_schedule/tactic/align_iter_space_tactic.h b/paddle/cinn/ir/group_schedule/tactic/align_iter_space_tactic.h index ef30f80ce470b..2ac65d114c7f5 100644 --- a/paddle/cinn/ir/group_schedule/tactic/align_iter_space_tactic.h +++ b/paddle/cinn/ir/group_schedule/tactic/align_iter_space_tactic.h @@ -20,17 +20,7 @@ namespace cinn { namespace ir { -class AlignIterSpaceTactic final : public ScheduleTactic { - public: - void Init(ScheduleContext* context) override; - - void Apply(ir::IRSchedule* sch, const std::string& block_id) override; - - std::string TacticName() const override { return "AlignIterSpaceTactic"; } - - private: - ScheduleContext* context_; -}; +std::unique_ptr CreateAlignIterSpaceTactic(); } // namespace ir } // namespace cinn diff --git a/paddle/cinn/ir/group_schedule/tactic/arrange_storage_tactic.cc b/paddle/cinn/ir/group_schedule/tactic/arrange_storage_tactic.cc index 5c5398533513d..8484c0c62210e 100644 --- a/paddle/cinn/ir/group_schedule/tactic/arrange_storage_tactic.cc +++ b/paddle/cinn/ir/group_schedule/tactic/arrange_storage_tactic.cc @@ -24,6 +24,18 @@ namespace cinn { namespace ir { +class ArrangeStorageTactic final : public ScheduleTactic { + public: + void Init(ScheduleContext* context) override; + + void Apply(ir::IRSchedule* sch, const std::string& block_id) override; + + std::string TacticName() const override { return "ArrangeStorageTactic"; } + + private: + std::unordered_set output_names_; +}; + // [block_name, [var, for_node]] using VarToForMap = std::unordered_map>; @@ -420,5 +432,9 @@ void ArrangeStorageTactic::Apply(ir::IRSchedule* sch, } } +std::unique_ptr CreateArrangeStorageTactic() { + return std::make_unique(); +} + } // namespace ir } // namespace cinn diff --git a/paddle/cinn/ir/group_schedule/tactic/arrange_storage_tactic.h b/paddle/cinn/ir/group_schedule/tactic/arrange_storage_tactic.h index 994108d1662b9..25fe8047efcd0 100644 --- a/paddle/cinn/ir/group_schedule/tactic/arrange_storage_tactic.h +++ b/paddle/cinn/ir/group_schedule/tactic/arrange_storage_tactic.h @@ -21,17 +21,7 @@ namespace cinn { namespace ir { -class ArrangeStorageTactic final : public ScheduleTactic { - public: - void Init(ScheduleContext* context) override; - - void Apply(ir::IRSchedule* sch, const std::string& block_id) override; - - std::string TacticName() const override { return "ArrangeStorageTactic"; } - - private: - std::unordered_set output_names_; -}; +std::unique_ptr CreateArrangeStorageTactic(); } // namespace ir } // namespace cinn diff --git a/paddle/cinn/ir/group_schedule/tactic/bind_cuda_tactic.cc b/paddle/cinn/ir/group_schedule/tactic/bind_cuda_tactic.cc index 0fe53e779aeae..50556da0db033 100644 --- a/paddle/cinn/ir/group_schedule/tactic/bind_cuda_tactic.cc +++ b/paddle/cinn/ir/group_schedule/tactic/bind_cuda_tactic.cc @@ -19,6 +19,18 @@ namespace cinn { namespace ir { +class BindCudaTactic final : public ScheduleTactic { + public: + void Init(ScheduleContext* context) override; + + void Apply(ir::IRSchedule* sch, const std::string& block_id) override; + + std::string TacticName() const override { return "BindCudaTactic"; } + + private: + ScheduleContext* context_; +}; + void BindCudaTactic::Init(ScheduleContext* context) { context_ = context; } const std::unordered_map @@ -56,5 +68,9 @@ void BindCudaTactic::Apply(ir::IRSchedule* sch, const std::string& block_id) { } } +std::unique_ptr CreateBindCudaTactic() { + return std::make_unique(); +} + } // namespace ir } // namespace cinn diff --git a/paddle/cinn/ir/group_schedule/tactic/bind_cuda_tactic.h b/paddle/cinn/ir/group_schedule/tactic/bind_cuda_tactic.h index b66c7d1fb802c..ae2ed3985bef1 100644 --- a/paddle/cinn/ir/group_schedule/tactic/bind_cuda_tactic.h +++ b/paddle/cinn/ir/group_schedule/tactic/bind_cuda_tactic.h @@ -20,17 +20,7 @@ namespace cinn { namespace ir { -class BindCudaTactic final : public ScheduleTactic { - public: - void Init(ScheduleContext* context) override; - - void Apply(ir::IRSchedule* sch, const std::string& block_id) override; - - std::string TacticName() const override { return "BindCudaTactic"; } - - private: - ScheduleContext* context_; -}; +std::unique_ptr CreateBindCudaTactic(); } // namespace ir } // namespace cinn diff --git a/paddle/cinn/ir/group_schedule/tactic/compute_inline_tactic.cc b/paddle/cinn/ir/group_schedule/tactic/compute_inline_tactic.cc index 8da8f44d32695..5076d1ded1e69 100644 --- a/paddle/cinn/ir/group_schedule/tactic/compute_inline_tactic.cc +++ b/paddle/cinn/ir/group_schedule/tactic/compute_inline_tactic.cc @@ -25,6 +25,19 @@ namespace cinn { namespace ir { +class ComputeInlineTactic final : public ScheduleTactic { + public: + void Init(ScheduleContext* context) override; + + void Apply(ir::IRSchedule* sch, const std::string& block_id) override; + + std::string TacticName() const override { return "ComputeInlineTactic"; } + + private: + std::unordered_set output_names_; + cinn::common::Target target_; +}; + void ComputeInlineTactic::Init(ScheduleContext* context) { output_names_ = context->output_names; target_ = context->target; @@ -48,5 +61,9 @@ void ComputeInlineTactic::Apply(ir::IRSchedule* sch, << sch->GetModule().GetExprs().front(); } +std::unique_ptr CreateComputeInlineTactic() { + return std::make_unique(); +} + } // namespace ir } // namespace cinn diff --git a/paddle/cinn/ir/group_schedule/tactic/compute_inline_tactic.h b/paddle/cinn/ir/group_schedule/tactic/compute_inline_tactic.h index b03e28d579bc8..821126bfc7ecc 100644 --- a/paddle/cinn/ir/group_schedule/tactic/compute_inline_tactic.h +++ b/paddle/cinn/ir/group_schedule/tactic/compute_inline_tactic.h @@ -22,18 +22,7 @@ namespace cinn { namespace ir { -class ComputeInlineTactic final : public ScheduleTactic { - public: - void Init(ScheduleContext* context) override; - - void Apply(ir::IRSchedule* sch, const std::string& block_id) override; - - std::string TacticName() const override { return "ComputeInlineTactic"; } - - private: - std::unordered_set output_names_; - cinn::common::Target target_; -}; +std::unique_ptr CreateComputeInlineTactic(); } // namespace ir } // namespace cinn diff --git a/paddle/cinn/ir/group_schedule/tactic/loop_reorder_alignment_tactic.cc b/paddle/cinn/ir/group_schedule/tactic/loop_reorder_alignment_tactic.cc new file mode 100644 index 0000000000000..39bf104e56508 --- /dev/null +++ b/paddle/cinn/ir/group_schedule/tactic/loop_reorder_alignment_tactic.cc @@ -0,0 +1,188 @@ +// Copyright (c) 2024 CINN Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/cinn/ir/group_schedule/tactic/loop_reorder_alignment_tactic.h" +#include +#include +#include "paddle/cinn/ir/ir.h" + +namespace cinn { +namespace ir { + +class LoopReorderAlignmentTactic final : public ScheduleTactic { + public: + void Init(ScheduleContext* context) override; + + void Apply(ir::IRSchedule* sch, const std::string& block_id) override; + + std::string TacticName() const override { + return "LoopReorderAlignmentTactic"; + } + + private: + bool NeedReorderLoops(); + + std::vector GetNewOrder(); + + void UpdateBaseRank(ir::IRSchedule* sch, const std::string& block_id); + + void DoBroadcastLoop(ir::IRSchedule* sch, const std::string& block_id); + + void DoReorder(ir::IRSchedule* sch, const std::string& block_id); + + private: + ScheduleContext* context_; + size_t base_rank_; + bool need_reorder_loops_; + std::vector new_order_; +}; + +void LoopReorderAlignmentTactic::Init(ScheduleContext* context) { + context_ = context; + base_rank_ = 0; + need_reorder_loops_ = NeedReorderLoops(); + new_order_ = GetNewOrder(); +} + +void LoopReorderAlignmentTactic::Apply(ir::IRSchedule* sch, + const std::string& block_id) { + DoBroadcastLoop(sch, block_id); + + if (!ir::IsReduceInitTensorName(block_id)) { + UpdateBaseRank(sch, block_id); + } + + if (need_reorder_loops_ && !ir::IsReduceInitTensorName(block_id)) { + DoReorder(sch, block_id); + } +} + +void LoopReorderAlignmentTactic::UpdateBaseRank(ir::IRSchedule* sch, + const std::string& block_id) { + auto loops = sch->GetLoops(block_id); + if (base_rank_ == 0) { + base_rank_ = loops.size(); + } else { + if (base_rank_ != loops.size()) { + throw std::runtime_error("loops rank not same "); + } + } +} + +bool LoopReorderAlignmentTactic::NeedReorderLoops() { + const auto HasReduceAxis = [&]() { + return context_->group_tile_info->reduce_axis_.size() > 0; + }; + if (!HasReduceAxis()) { + return false; + } + + const auto HasNonLastDimReduce = [&]() { + std::vector vec_reduce_axis = + context_->group_tile_info->reduce_axis_; + std::sort(vec_reduce_axis.begin(), vec_reduce_axis.end()); + return vec_reduce_axis.front() != + context_->group_tile_info->data_rank - vec_reduce_axis.size(); + }; + + return HasNonLastDimReduce(); +} + +std::vector LoopReorderAlignmentTactic::GetNewOrder() { + std::set reduce_set(context_->group_tile_info->reduce_axis_.begin(), + context_->group_tile_info->reduce_axis_.end()); + + std::vector new_order; + for (int32_t i = 0; i < context_->group_tile_info->data_rank; ++i) { + if (!reduce_set.count(i)) { + new_order.push_back(i); + } + } + for (auto axis : context_->group_tile_info->reduce_axis_) { + new_order.push_back(axis); + } + + return new_order; +} + +void LoopReorderAlignmentTactic::DoBroadcastLoop(ir::IRSchedule* sch, + const std::string& block_id) { + const auto HasBroadcastInfo = [&](const std::string& block_id) { + return context_->group_tile_info->broadcast_info.count(block_id) > 0; + }; + const auto HasBroadcastToElementwiseInfo = [&](const std::string& block_id) { + return context_->group_tile_info->broadcast_to_elementwise.count(block_id) > + 0; + }; + const auto IsFullBroadcast = [&](const std::string& block_id) { + return context_->group_tile_info->broadcast_info[block_id].full_broadcast; + }; + const auto IsSplitFirst = [&](const std::string& block_id) { + return context_->group_tile_info->broadcast_info[block_id].split_first; + }; + + if (HasBroadcastInfo(block_id)) { + if (IsFullBroadcast(block_id)) { + std::vector vec_out_split( + context_->group_tile_info->broadcast_info[block_id] + .output_shape.size(), + 1); + + auto loops = sch->GetLoops(block_id); + sch->Split(loops[0], vec_out_split); + loops = sch->GetLoops(block_id); + } else if (IsSplitFirst(block_id)) { + for (auto& info : + context_->group_tile_info->broadcast_info[block_id].split_info) { + auto axis = info.first; + auto split_res = info.second; + + auto loops = sch->GetLoops(block_id); + sch->Split(loops[axis], split_res); + loops = sch->GetLoops(block_id); + } + } else { + // Do nothing + } + + sch->Broadcast(block_id, + context_->group_tile_info->broadcast_info[block_id]); + } + + if (HasBroadcastToElementwiseInfo(block_id)) { + sch->BroadcastToElementwise( + block_id, + context_->group_tile_info->broadcast_to_elementwise[block_id] + .broadcast_axes); + } +} + +void LoopReorderAlignmentTactic::DoReorder(ir::IRSchedule* sch, + const std::string& block_id) { + const auto IsReduceBlock = [&](const std::string& block_id) { + return context_->group_tile_info->reduce_tensor_names.count(block_id) > 0; + }; + if (!IsReduceBlock(block_id)) { + return; + } + + sch->Reorder(block_id, new_order_); +} + +std::unique_ptr CreateLoopReorderAlignmentTactic() { + return std::make_unique(); +} + +} // namespace ir +} // namespace cinn diff --git a/paddle/cinn/ir/group_schedule/tactic/loop_reorder_alignment_tactic.h b/paddle/cinn/ir/group_schedule/tactic/loop_reorder_alignment_tactic.h new file mode 100644 index 0000000000000..ee4864a5ecf92 --- /dev/null +++ b/paddle/cinn/ir/group_schedule/tactic/loop_reorder_alignment_tactic.h @@ -0,0 +1,26 @@ +// Copyright (c) 2024 CINN Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include "paddle/cinn/ir/group_schedule/tactic/schedule_tactic.h" + +namespace cinn { +namespace ir { + +std::unique_ptr CreateLoopReorderAlignmentTactic(); + +} // namespace ir +} // namespace cinn diff --git a/paddle/cinn/ir/group_schedule/tactic/optimize_reduction_tactic.cc b/paddle/cinn/ir/group_schedule/tactic/optimize_reduction_tactic.cc index c9f435704be9f..445ac32c94ab1 100644 --- a/paddle/cinn/ir/group_schedule/tactic/optimize_reduction_tactic.cc +++ b/paddle/cinn/ir/group_schedule/tactic/optimize_reduction_tactic.cc @@ -19,6 +19,18 @@ namespace cinn { namespace ir { +class OptimizeReductionTactic final : public ScheduleTactic { + public: + void Init(ScheduleContext* context) override; + + void Apply(ir::IRSchedule* sch, const std::string& block_id) override; + + std::string TacticName() const override { return "OptimizeReductionTactic"; } + + private: + ScheduleContext* context_; +}; + void OptimizeReductionTactic::Init(ScheduleContext* context) { context_ = context; } @@ -151,5 +163,9 @@ void OptimizeReductionTactic::Apply(ir::IRSchedule* sch, << sch->GetModule().GetExprs()[0]; } +std::unique_ptr CreateOptimizeReductionTactic() { + return std::make_unique(); +} + } // namespace ir } // namespace cinn diff --git a/paddle/cinn/ir/group_schedule/tactic/optimize_reduction_tactic.h b/paddle/cinn/ir/group_schedule/tactic/optimize_reduction_tactic.h index 108f674ee2253..aa2405530f917 100644 --- a/paddle/cinn/ir/group_schedule/tactic/optimize_reduction_tactic.h +++ b/paddle/cinn/ir/group_schedule/tactic/optimize_reduction_tactic.h @@ -20,17 +20,7 @@ namespace cinn { namespace ir { -class OptimizeReductionTactic final : public ScheduleTactic { - public: - void Init(ScheduleContext* context) override; - - void Apply(ir::IRSchedule* sch, const std::string& block_id) override; - - std::string TacticName() const override { return "OptimizeReductionTactic"; } - - private: - ScheduleContext* context_; -}; +std::unique_ptr CreateOptimizeReductionTactic(); } // namespace ir } // namespace cinn diff --git a/paddle/cinn/ir/group_schedule/tactic/schedule_tactic.h b/paddle/cinn/ir/group_schedule/tactic/schedule_tactic.h index 68f4ae31c7a7c..ef3d4817949b2 100644 --- a/paddle/cinn/ir/group_schedule/tactic/schedule_tactic.h +++ b/paddle/cinn/ir/group_schedule/tactic/schedule_tactic.h @@ -16,6 +16,7 @@ #include #include "paddle/cinn/common/integer_set.h" +#include "paddle/cinn/ir/ir.h" #include "paddle/cinn/ir/schedule/ir_schedule.h" #include "paddle/cinn/ir/schedule_block_graph.h" @@ -71,11 +72,41 @@ struct BucketInfo { int rb_upper_bound = UINT_MAX; }; +struct GroupTileInfo { + GroupTileInfo() {} + + std::vector reduce_axis_; + int64_t data_rank; + + int64_t block_num{-1}; + int64_t warp_num; + int64_t spatial_inner_num; + int64_t reduce_numel; + int64_t reduce_inner_num; + int64_t reduce_block; + + std::set reduce_tensor_names; + std::set temp_var_names; + + std::set shared_var_names; + std::set direct_output_var_names; + std::vector thread_sync_before_names; + + ReduceMethod reduce_method{NoneReduceMethod()}; + + std::unordered_map broadcast_info; + std::unordered_map broadcast_to_elementwise; +}; + struct ScheduleContext { + // TODO(BiynXu): Unify fields with similar meanings std::unordered_set output_names; Target target; IterativeSpaceInfo iter_space_info; BucketInfo bucket_info; + // Will tile information be modified during the schedule process? + // If so, it is necessary to store a separate copy for each context + std::shared_ptr group_tile_info; }; class ScheduleTactic { diff --git a/paddle/cinn/ir/group_schedule/tactic/tile_first_general_tactic.cc b/paddle/cinn/ir/group_schedule/tactic/tile_first_general_tactic.cc new file mode 100644 index 0000000000000..b7e584bba737f --- /dev/null +++ b/paddle/cinn/ir/group_schedule/tactic/tile_first_general_tactic.cc @@ -0,0 +1,283 @@ +// Copyright (c) 2024 CINN Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/cinn/ir/group_schedule/tactic/tile_first_general_tactic.h" +#include "paddle/cinn/common/target.h" +#include "paddle/cinn/ir/ir.h" +#include "paddle/cinn/ir/schedule/ir_schedule_util.h" + +namespace cinn { +namespace ir { + +bool IsInnerThreadSpatialLoopGT(const std::shared_ptr& tile_info, + int num) { + return tile_info->spatial_inner_num > num; +} + +bool IsInnerThreadReduceLoopGT(const std::shared_ptr& tile_info, + int num) { + return tile_info->reduce_inner_num > num; +} + +bool IsReduceBlock(const std::shared_ptr& tile_info, + const std::string& block_id) { + return tile_info->reduce_tensor_names.count(block_id) > 0; +} + +bool HasReduceAxis(const std::shared_ptr& tile_info) { + return tile_info->reduce_axis_.size() > 0; +} + +class TileFirstGeneralTactic final : public ScheduleTactic { + public: + void Init(ScheduleContext* context) override; + + void Apply(ir::IRSchedule* sch, const std::string& block_id) override; + + std::string TacticName() const override { return "TileFirstGeneralTactic"; } + + private: + void MergeFlattenAxis(ir::IRSchedule* sch, const std::string& block_id); + void MergeReduceAxis(ir::IRSchedule* sch, const std::string& block_id); + void SplitFlattenInner(ir::IRSchedule* sch, const std::string& block_id); + void SplitReduceInner(ir::IRSchedule* sch, const std::string& block_id); + void ReorderFlattenInnerWithReduceAxis(ir::IRSchedule* sch, + const std::string& block_id); + void SplitWarpNumber(ir::IRSchedule* sch, const std::string& block_id); + void Unroll(ir::IRSchedule* sch, const std::string& block_id); + void VariableTypeAssignment(ir::IRSchedule* sch, const std::string& block_id); + void SetReduceType(ir::IRSchedule* sch, const std::string& block_id); + void BindCudaInfo(ir::IRSchedule* sch, const std::string& block_id); + + private: + ScheduleContext* context_; + std::vector vec_flatten_axis_; + std::vector vec_reduce_axis_; + int reduce_current_axis_{0}; +}; + +void TileFirstGeneralTactic::Init(ScheduleContext* context) { + context_ = context; + reduce_current_axis_ = + IsInnerThreadSpatialLoopGT(context_->group_tile_info, 1) ? 2 : 1; + // reduce axis have be re-order to last + vec_flatten_axis_.clear(); + vec_reduce_axis_.clear(); + int32_t reduce_start_idx = context_->group_tile_info->data_rank - + context_->group_tile_info->reduce_axis_.size(); + for (int32_t i = 0; i < context_->group_tile_info->data_rank; ++i) { + if (i >= reduce_start_idx) { + vec_reduce_axis_.push_back(i); + } else { + vec_flatten_axis_.push_back(i); + } + } +} + +void TileFirstGeneralTactic::Apply(ir::IRSchedule* sch, + const std::string& block_id) { + if (ir::IsReduceInitTensorName(block_id)) return; + MergeFlattenAxis(sch, block_id); + MergeReduceAxis(sch, block_id); + SplitFlattenInner(sch, block_id); + SplitReduceInner(sch, block_id); + ReorderFlattenInnerWithReduceAxis(sch, block_id); + SplitWarpNumber(sch, block_id); + BindCudaInfo(sch, block_id); + VariableTypeAssignment(sch, block_id); + Unroll(sch, block_id); + SetReduceType(sch, block_id); +} + +void TileFirstGeneralTactic::MergeFlattenAxis(ir::IRSchedule* sch, + const std::string& block_id) { + if (vec_flatten_axis_.size() >= 2) { + sch->Fuse(block_id, vec_flatten_axis_); + } +} + +void TileFirstGeneralTactic::MergeReduceAxis(ir::IRSchedule* sch, + const std::string& block_id) { + if (vec_reduce_axis_.size() >= 2) { + sch->Fuse(block_id, vec_reduce_axis_); + } +} + +void TileFirstGeneralTactic::SplitFlattenInner(ir::IRSchedule* sch, + const std::string& block_id) { + if (IsInnerThreadSpatialLoopGT(context_->group_tile_info, 1)) { + auto loops = sch->GetLoops(block_id); + auto split_loops = sch->Split( + loops[0], + std::vector({-1, context_->group_tile_info->spatial_inner_num})); + } +} + +void TileFirstGeneralTactic::SplitReduceInner(ir::IRSchedule* sch, + const std::string& block_id) { + if (!IsInnerThreadReduceLoopGT(context_->group_tile_info, 1)) return; + + auto loops = sch->GetLoops(block_id); + auto reduce_loop = loops[reduce_current_axis_].As(); + + if (ir::GetLoopExtent(reduce_loop) == 1) { + return; + } + + const auto IsReduceBlockGE = [&](int64_t num) { + return context_->group_tile_info->reduce_block >= num; + }; + std::vector split_factors; + if (IsReduceBlockGE(2048)) { + split_factors.emplace_back( + std::ceil(context_->group_tile_info->reduce_numel * 1.0 / + context_->group_tile_info->reduce_inner_num)); + split_factors.emplace_back(context_->group_tile_info->reduce_inner_num); + } else { + split_factors.emplace_back( + std::ceil(context_->group_tile_info->reduce_block * 1.0 / + context_->group_tile_info->reduce_inner_num)); + split_factors.emplace_back(context_->group_tile_info->reduce_inner_num); + } + + auto split_loops = sch->Split(loops[reduce_current_axis_], split_factors); + + if (IsReduceBlock(context_->group_tile_info, block_id)) { + sch->FactorizeReduction( + split_loops[0], 0, /* with_write_back_block_init = */ false); + } +} + +void TileFirstGeneralTactic::ReorderFlattenInnerWithReduceAxis( + ir::IRSchedule* sch, const std::string& block_id) { + // re-order flatten inner num with last dim + if (IsInnerThreadSpatialLoopGT(context_->group_tile_info, 1) && + HasReduceAxis(context_->group_tile_info)) { + auto loops = sch->GetLoops(block_id); + sch->Reorder({loops[2], loops[1]}); + if (IsReduceBlock(context_->group_tile_info, block_id)) { + auto loops = sch->GetLoops(block_id + "_rf"); + sch->Reorder({loops[2], loops[1]}); + } + } +} + +void TileFirstGeneralTactic::SplitWarpNumber(ir::IRSchedule* sch, + const std::string& block_id) { + const auto IsWarpNumGT = [&](int64_t num) { + return context_->group_tile_info->warp_num > num; + }; + if (!IsWarpNumGT(1)) return; + + if (!HasReduceAxis(context_->group_tile_info)) { + // get num warp from flatten num + auto loops = sch->GetLoops(block_id); + sch->Split(loops[0], + std::vector({context_->group_tile_info->block_num, + context_->group_tile_info->warp_num * 32})); + } else if (IsInnerThreadSpatialLoopGT(context_->group_tile_info, 1)) { + // get num warp from flatten num + auto loops = sch->GetLoops(block_id); + sch->Split(loops[0], + std::vector({-1, context_->group_tile_info->warp_num})); + + loops = sch->GetLoops(block_id); + sch->Fuse({loops[1], loops[2]}); + + if (IsReduceBlock(context_->group_tile_info, block_id)) { + auto loops = sch->GetLoops(block_id + "_rf"); + sch->Split(loops[0], + std::vector({-1, context_->group_tile_info->warp_num})); + + loops = sch->GetLoops(block_id + "_rf"); + sch->Fuse({loops[1], loops[2]}); + } + } else { + return; + } +} + +void TileFirstGeneralTactic::Unroll(ir::IRSchedule* sch, + const std::string& block_id) { + auto loops = sch->GetLoops(block_id); + if (loops.size() > 2) { + sch->Unroll(loops[2]); + } + if (loops.size() > 3) { + sch->Unroll(loops[3]); + } + + if (IsReduceBlock(context_->group_tile_info, block_id)) { + auto loops = sch->GetLoops(block_id + "_rf"); + if (loops.size() > 2) { + sch->Unroll(loops[2]); + } + if (loops.size() > 3) { + sch->Unroll(loops[3]); + } + } +} + +void TileFirstGeneralTactic::VariableTypeAssignment( + ir::IRSchedule* sch, const std::string& block_id) { + const auto IsOutputTensor = [&](const std::string& tensor_name) { + return context_->group_tile_info->direct_output_var_names.count( + tensor_name) > 0; + }; + + auto block = sch->GetBlock(block_id); + if (!IsOutputTensor(block_id)) { + sch->SetBuffer(block, "local", false); + } + + if (IsReduceBlock(context_->group_tile_info, block_id)) { + auto block = sch->GetBlock(block_id + "_rf"); + sch->SetBuffer(block, "local", false); + } +} + +void TileFirstGeneralTactic::SetReduceType(ir::IRSchedule* sch, + const std::string& block_id) { + if (IsReduceBlock(context_->group_tile_info, block_id)) { + auto block = sch->GetBlock(block_id) + .As() + ->schedule_block.As(); + block->reduce_method = context_->group_tile_info->reduce_method; + } +} + +void TileFirstGeneralTactic::BindCudaInfo(ir::IRSchedule* sch, + const std::string& block_id) { + auto loops = sch->GetLoops(block_id); + if (loops.size() == 1) { + sch->Split(loops[0], std::vector({1, -1})); + } + + loops = sch->GetLoops(block_id); + sch->Bind(loops[0], "blockIdx.x"); + sch->Bind(loops[1], "threadIdx.x"); + + if (IsReduceBlock(context_->group_tile_info, block_id)) { + auto loops = sch->GetLoops(block_id + "_rf"); + sch->Bind(loops[0], "blockIdx.x"); + sch->Bind(loops[1], "threadIdx.x"); + } +} + +std::unique_ptr CreateTileFirstGeneralTactic() { + return std::make_unique(); +} + +} // namespace ir +} // namespace cinn diff --git a/paddle/cinn/ir/group_schedule/tactic/tile_first_general_tactic.h b/paddle/cinn/ir/group_schedule/tactic/tile_first_general_tactic.h new file mode 100644 index 0000000000000..cda680c8ecf90 --- /dev/null +++ b/paddle/cinn/ir/group_schedule/tactic/tile_first_general_tactic.h @@ -0,0 +1,26 @@ +// Copyright (c) 2024 CINN Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include "paddle/cinn/ir/group_schedule/tactic/schedule_tactic.h" + +namespace cinn { +namespace ir { + +std::unique_ptr CreateTileFirstGeneralTactic(); + +} // namespace ir +} // namespace cinn diff --git a/paddle/cinn/ir/group_schedule/tactic/tile_tactic.cc b/paddle/cinn/ir/group_schedule/tactic/tile_tactic.cc index e0e84d0bcd5b1..114a539e4e3f6 100644 --- a/paddle/cinn/ir/group_schedule/tactic/tile_tactic.cc +++ b/paddle/cinn/ir/group_schedule/tactic/tile_tactic.cc @@ -19,6 +19,18 @@ namespace cinn { namespace ir { +class TileTactic final : public ScheduleTactic { + public: + void Init(ScheduleContext* context) override; + + void Apply(ir::IRSchedule* sch, const std::string& block_id) override; + + std::string TacticName() const override { return "TileTactic"; } + + private: + ScheduleContext* context_; +}; + void TileTactic::Init(ScheduleContext* context) { context_ = context; // TODO(BiynXu): Create schedule config and bucket info based on hardware @@ -114,5 +126,9 @@ void TileTactic::Apply(ir::IRSchedule* sch, const std::string& block_id) { << sch->GetModule().GetExprs()[0]; } +std::unique_ptr CreateTileTactic() { + return std::make_unique(); +} + } // namespace ir } // namespace cinn diff --git a/paddle/cinn/ir/group_schedule/tactic/tile_tactic.h b/paddle/cinn/ir/group_schedule/tactic/tile_tactic.h index 8a6d2bb8dd766..223287372ddf3 100644 --- a/paddle/cinn/ir/group_schedule/tactic/tile_tactic.h +++ b/paddle/cinn/ir/group_schedule/tactic/tile_tactic.h @@ -20,17 +20,7 @@ namespace cinn { namespace ir { -class TileTactic final : public ScheduleTactic { - public: - void Init(ScheduleContext* context) override; - - void Apply(ir::IRSchedule* sch, const std::string& block_id) override; - - std::string TacticName() const override { return "TileTactic"; } - - private: - ScheduleContext* context_; -}; +std::unique_ptr CreateTileTactic(); } // namespace ir } // namespace cinn diff --git a/paddle/cinn/ir/ir.h b/paddle/cinn/ir/ir.h index 5a1f9f6a1f739..d711e93ce61ab 100644 --- a/paddle/cinn/ir/ir.h +++ b/paddle/cinn/ir/ir.h @@ -966,6 +966,12 @@ struct Block : public ExprNode { static const IrNodeTy _node_type_ = IrNodeTy::Block; }; +struct NoneReduceMethod {}; +struct WarpReduceMethod {}; +struct BlockReduceMethod {}; +using ReduceMethod = + std::variant; + // ScheduleBlock is the unit of schedule IR which represents tensor's // computation struct ScheduleBlock : public ExprNode { @@ -981,7 +987,7 @@ struct ScheduleBlock : public ExprNode { std::map attrs; std::string name; Expr body; - int32_t reduce_type{-1}; // 0 for warp reduce, 1 for block reduce + ReduceMethod reduce_method{NoneReduceMethod()}; static Expr Make(const std::vector& iter_vars, const std::vector& read_buffers, diff --git a/paddle/cinn/ir/schedule/factorize_reduction.h b/paddle/cinn/ir/schedule/factorize_reduction.h index d6252bb0a4663..8b0488e9c883c 100644 --- a/paddle/cinn/ir/schedule/factorize_reduction.h +++ b/paddle/cinn/ir/schedule/factorize_reduction.h @@ -90,6 +90,7 @@ class ReduceBlockCreater { is_rf_block_ ? rf_tensor_ : original_update_stmt_.As()->tensor.as_tensor_ref(); + Expr init_value = real_tensor->GetReduceInitVal(); const std::vector& domain = real_tensor->domain_without_reduce_axis(); ir::Tensor init_tensor = lang::Compute( @@ -97,8 +98,21 @@ class ReduceBlockCreater { [=](const std::vector& axis) { return init_value; }, new_init_block_name); init_tensor->Bind(real_tensor->buffer); - Expr init_stmt = ir::Store::Make( - init_tensor, init_value, new_update_stmt_.As()->indices); + std::vector new_indices; + if (new_update_stmt_.As()) { + new_indices = new_update_stmt_.As()->indices; + } else if (new_update_stmt_.As()) { + new_indices = new_update_stmt_.As() + ->true_case.As() + ->stmts[0] + .As() + ->indices; + } else { + throw std::runtime_error("only support store and ifthenelse"); + } + + Expr init_stmt = ir::Store::Make(init_tensor, init_value, new_indices); + new_init_sch_block_ = ScheduleBlock::Make( new_init_iter_vars_, {}, {}, new_init_block_name, init_stmt); new_init_block_realize_ = @@ -111,7 +125,7 @@ class ReduceBlockCreater { VLOG(4) << "new_update_block_realize:\n" << new_update_block_realize_; } - Expr CreateLoops() { + Expr CreateLoops(bool with_init = true) { int num_loops = original_loops_.size(); std::vector new_loops(num_loops); Expr body = new_update_block_realize_; @@ -127,7 +141,7 @@ class ReduceBlockCreater { continue; } // Add reduce init block. - if (!has_add_init_block && is_spatial_loop) { + if (!has_add_init_block && is_spatial_loop && with_init) { body = Block::Make({new_init_block_realize_, body}); has_add_init_block = true; } @@ -201,6 +215,26 @@ class ReduceBlockCreater { Expr new_init_block_realize_; }; +class LoadReplacer : public ir::IRMutator<> { + public: + explicit LoadReplacer(const std::string& src_load_tensor_name, + const ir::Expr& target) + : src_load_tensor_name_(src_load_tensor_name), target_(target) {} + + void operator()(Expr* expr) { IRMutator::Visit(expr, expr); } + + private: + void Visit(const ir::Load* expr, Expr* op) override { + if (expr->tensor.as_tensor()->name == src_load_tensor_name_) { + *op = target_; + } + } + + private: + std::string src_load_tensor_name_; + ir::Expr target_; +}; + // Implement class for building Reduction-Factorized block, // only used for FactorizeReduction schedule primitive. class RFBlockCreater : public ReduceBlockCreater { @@ -211,6 +245,7 @@ class RFBlockCreater : public ReduceBlockCreater { const Expr& original_update_stmt, const ir::Tensor& rf_tensor, const std::map& var2loops, + const Expr& bound_check, int rf_axis) : ReduceBlockCreater(original_block, original_loops, @@ -219,7 +254,8 @@ class RFBlockCreater : public ReduceBlockCreater { rf_tensor, true), var2loops_(var2loops), - rf_axis_(rf_axis) {} + rf_axis_(rf_axis), + bound_check_(ir_utils::IRCopy(bound_check)) {} private: void CreateRFIter() override { @@ -235,6 +271,11 @@ class RFBlockCreater : public ReduceBlockCreater { new_init_iter_vars_.push_back(rf_var_); new_init_iter_values_.push_back(rf_loop_.As()->loop_var); new_spatial_loop_var_names_.insert(rf_loop_.As()->loop_var->name); + + std::vector new_iter_exprs{Expr(rf_var_)}; + ReplaceExpr( + &bound_check_, {rf_loop_.As()->loop_var}, new_iter_exprs); + VLOG(4) << "create new_rf_var = " << rf_var_ << ", with iter value = " << new_iter_values_.back(); } @@ -310,29 +351,19 @@ class RFBlockCreater : public ReduceBlockCreater { rf_tensor_access_indices_.insert( rf_tensor_access_indices_.begin() + rf_axis_, rf_var_); Expr original_store_body = original_update_stmt_.As()->value; + std::string original_store_name = + original_update_stmt_.As()->tensor.as_tensor()->name; Expr new_store_body = ir_utils::IRCopy(original_store_body); -#define REPLACE_RF_TENSOR(Op) \ - if (new_store_body.As()) { \ - auto* node = new_store_body.As(); \ - CHECK(node); \ - auto& operand = node->a(); \ - operand = Load::Make(rf_tensor_, rf_tensor_access_indices_); \ - } - - REPLACE_RF_TENSOR(Add) - REPLACE_RF_TENSOR(Mul) - REPLACE_RF_TENSOR(Max) - REPLACE_RF_TENSOR(Min) - REPLACE_RF_TENSOR(And) - REPLACE_RF_TENSOR(Or) - REPLACE_RF_TENSOR(LT) - REPLACE_RF_TENSOR(LE) - REPLACE_RF_TENSOR(GT) - REPLACE_RF_TENSOR(GE) -#undef REPLACE_RF_TENSOR + LoadReplacer load_replacer( + original_store_name, Load::Make(rf_tensor_, rf_tensor_access_indices_)); + load_replacer(&new_store_body); new_update_stmt_ = ir::Store::Make(rf_tensor_, new_store_body, rf_tensor_access_indices_); + + if (!bound_check_.is_constant()) { + new_update_stmt_ = ir::IfThenElse::Make(bound_check_, new_update_stmt_); + } ReplaceExpr(&new_update_stmt_, original_indice2new_expr_); VLOG(4) << "new_update_stmt of rf block: \n" << new_update_stmt_; } @@ -342,6 +373,8 @@ class RFBlockCreater : public ReduceBlockCreater { int rf_axis_; std::map loop_var2block_iters_; + + Expr bound_check_; }; // Implement class for building Writing-Back block, @@ -406,6 +439,9 @@ class RBBlockCreater : public ReduceBlockCreater { void CreateUpdateStmt() override { Expr original_store_body = original_update_stmt_.As()->value; Expr new_store_body = ir_utils::IRCopy(original_store_body); + std::string original_store_name = + original_update_stmt_.As()->tensor.as_tensor()->name; + #define REPLACE_RF_TENSOR(Op) \ if (new_store_body.As()) { \ auto* node = new_store_body.As(); \ diff --git a/paddle/cinn/ir/schedule/impl/for_type.cc b/paddle/cinn/ir/schedule/impl/for_type.cc index 53f157eac931a..aadccf97f286d 100644 --- a/paddle/cinn/ir/schedule/impl/for_type.cc +++ b/paddle/cinn/ir/schedule/impl/for_type.cc @@ -53,7 +53,7 @@ void DyScheduleImpl::MutateForType(const Expr& loop, << static_cast(for_type) << "!\n"; } - auto loop_copy = ir::ir_utils::IRCopy(loop); + auto loop_copy = ir::ir_utils::IRCopy(loop, /* copy_buffer_node = */ false); auto* new_for_node = loop_copy.As(); CHECK(new_for_node); new_for_node->set_for_type(for_type); diff --git a/paddle/cinn/ir/schedule/impl/ir_schedule.h b/paddle/cinn/ir/schedule/impl/ir_schedule.h index 3fe35854cb4aa..42779c968d827 100644 --- a/paddle/cinn/ir/schedule/impl/ir_schedule.h +++ b/paddle/cinn/ir/schedule/impl/ir_schedule.h @@ -87,7 +87,9 @@ class DyScheduleImpl : public ScheduleBase { void ReverseComputeInline(const Expr& schedule_block); void Bind(const Expr& loop, const std::string& thread_axis); Expr Rfactor(const Expr& rf_loop, int rf_axis); - Expr FactorizeReduction(const Expr& rf_loop, int rf_axis); + Expr FactorizeReduction(const Expr& rf_loop, + int rf_axis, + bool with_write_back_block_init = true); Expr AddUnitLoop(const Expr& block) const; void Annotate(const Expr& block, const std::string& key, const attr_t& value); void Unannotate(Expr& block, const std::string& key); // NOLINT @@ -161,7 +163,9 @@ class StScheduleImpl : public ScheduleBase { void ReverseComputeInline(const Expr& schedule_block); void Bind(const Expr& loop, const std::string& thread_axis); Expr Rfactor(const Expr& rf_loop, int rf_axis); - Expr FactorizeReduction(const Expr& rf_loop, int rf_axis); + Expr FactorizeReduction(const Expr& rf_loop, + int rf_axis, + bool with_write_back_block_init = true); Expr AddUnitLoop(const Expr& block) const; void Annotate(const Expr& block, const std::string& key, const attr_t& value); void Unannotate(Expr& block, const std::string& key); // NOLINT diff --git a/paddle/cinn/ir/schedule/impl/reduction.cc b/paddle/cinn/ir/schedule/impl/reduction.cc index 6a28b40741388..d5f8eb8b410e6 100644 --- a/paddle/cinn/ir/schedule/impl/reduction.cc +++ b/paddle/cinn/ir/schedule/impl/reduction.cc @@ -50,7 +50,9 @@ Expr DyScheduleImpl::Rfactor(const Expr& rf_loop, int rf_axis) { CINN_IR_SCHEDULE_END(this->err_msg_level_); } -Expr DyScheduleImpl::FactorizeReduction(const Expr& rf_loop, int rf_axis) { +Expr DyScheduleImpl::FactorizeReduction(const Expr& rf_loop, + int rf_axis, + bool with_write_back_block_init) { CINN_IR_SCHEDULE_BEGIN() std::string primitive = "FactorizeReduction"; std::ostringstream os; @@ -103,6 +105,7 @@ Expr DyScheduleImpl::FactorizeReduction(const Expr& rf_loop, int rf_axis) { original_update_stmt, rf_tensor, var2loops, + Expr(false), rf_axis); rf_block_creater.CreateBlock(); RBBlockCreater wb_block_creater(original_block, @@ -115,7 +118,8 @@ Expr DyScheduleImpl::FactorizeReduction(const Expr& rf_loop, int rf_axis) { wb_block_creater.CreateBlock(); Expr rf_body = rf_block_creater.CreateLoops(); - Expr wb_body = wb_block_creater.CreateLoops(); + Expr wb_body = wb_block_creater.CreateLoops( + /* with_init = */ with_write_back_block_init); Expr new_computational_body = Block::Make({rf_body, wb_body}); @@ -144,7 +148,9 @@ Expr StScheduleImpl::Rfactor(const Expr& rf_loop, int rf_axis) { return rf_create.CreateRfAllStmts(); } -Expr StScheduleImpl::FactorizeReduction(const Expr& rf_loop, int rf_axis) { +Expr StScheduleImpl::FactorizeReduction(const Expr& rf_loop, + int rf_axis, + bool with_write_back_block_init) { std::string primitive = "FactorizeReduction"; // Get child block of the rf_loop and check. std::vector blocks = GetChildBlocks(rf_loop); @@ -165,6 +171,12 @@ Expr StScheduleImpl::FactorizeReduction(const Expr& rf_loop, int rf_axis) { VLOG(3) << "before FactorizeReduction, original computational body of the " "reduction is:\n" << original_loops[0]; + Expr bound_check(false); + auto first_st = original_loops.back().As()->body.As()->stmts[0]; + if (first_st.As()) { + bound_check = first_st.As()->condition; + } + std::map var2loops; for (const Expr& loop : original_loops) { var2loops[loop.As()->loop_var] = loop; @@ -193,6 +205,7 @@ Expr StScheduleImpl::FactorizeReduction(const Expr& rf_loop, int rf_axis) { original_update_stmt, rf_tensor, var2loops, + bound_check, rf_axis); rf_block_creater.CreateBlock(); RBBlockCreater wb_block_creater(original_block, @@ -205,7 +218,8 @@ Expr StScheduleImpl::FactorizeReduction(const Expr& rf_loop, int rf_axis) { wb_block_creater.CreateBlock(); Expr rf_body = rf_block_creater.CreateLoops(); - Expr wb_body = wb_block_creater.CreateLoops(); + Expr wb_body = wb_block_creater.CreateLoops( + /* with_init = */ with_write_back_block_init); Expr new_computational_body = Block::Make({rf_body, wb_body}); diff --git a/paddle/cinn/ir/schedule/ir_schedule.cc b/paddle/cinn/ir/schedule/ir_schedule.cc index 7bf684acfc6a9..93a2f0344a114 100644 --- a/paddle/cinn/ir/schedule/ir_schedule.cc +++ b/paddle/cinn/ir/schedule/ir_schedule.cc @@ -449,6 +449,16 @@ Expr IRSchedule::Fuse(const Expr& block, const std::vector& loops_index) { return result; } +void IRSchedule::Broadcast(const std::string& block_name, + const BroadcastInfo& info) { + impl_->Broadcast(block_name, info); +} + +void IRSchedule::BroadcastToElementwise(const std::string& block_name, + const std::vector& axes) { + impl_->BroadcastToElementwise(block_name, axes); +} + void IRSchedule::ComputeAt(const Expr& block, const Expr& loop, bool keep_unit_loops) { @@ -619,12 +629,17 @@ Expr IRSchedule::Rfactor(const Expr& rf_loop, int rf_axis) { return result; } -Expr IRSchedule::FactorizeReduction(const Expr& rf_loop, int rf_axis) { - auto result = impl_->FactorizeReduction(rf_loop, rf_axis); - trace_.Append(ScheduleDesc::Step("FactorizeReduction", - {{"rf_loop", std::vector({rf_loop})}}, - {{"rf_axis", rf_axis}}, - {result})); +Expr IRSchedule::FactorizeReduction(const Expr& rf_loop, + int rf_axis, + bool with_write_back_block_init) { + auto result = + impl_->FactorizeReduction(rf_loop, rf_axis, with_write_back_block_init); + trace_.Append(ScheduleDesc::Step( + "FactorizeReduction", + {{"rf_loop", std::vector({rf_loop})}}, + {{"rf_axis", rf_axis}, + {"with_write_back_block_init", with_write_back_block_init}}, + {result})); return result; } diff --git a/paddle/cinn/ir/schedule/ir_schedule.h b/paddle/cinn/ir/schedule/ir_schedule.h index 9ea4eb9f59b6f..cab1b0d38d868 100644 --- a/paddle/cinn/ir/schedule/ir_schedule.h +++ b/paddle/cinn/ir/schedule/ir_schedule.h @@ -195,6 +195,12 @@ class IRSchedule { * @param memory_type String that indicates the buffer's storage scope. * @return The buffer's cache. */ + + void Broadcast(const std::string& block_name, const BroadcastInfo& info); + + void BroadcastToElementwise(const std::string& block_name, + const std::vector& axes); + Expr CacheRead(const Expr& block, int read_buffer_index, const std::string& memory_type); @@ -402,7 +408,9 @@ class IRSchedule { * B[i] = B[i] + rf_B[j, i] * \endcode */ - Expr FactorizeReduction(const Expr& rf_loop, int rf_axis); + Expr FactorizeReduction(const Expr& rf_loop, + int rf_axis, + bool with_write_back_block_init = true); /*! * \brief Annotate a block with a key-value pair to set as its attribute diff --git a/paddle/cinn/ir/schedule/schedule_base.cc b/paddle/cinn/ir/schedule/schedule_base.cc index 8e6573edeab0e..3fbb1e7826297 100644 --- a/paddle/cinn/ir/schedule/schedule_base.cc +++ b/paddle/cinn/ir/schedule/schedule_base.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "paddle/cinn/ir/schedule/schedule_base.h" +#include "paddle/cinn/ir/schedule/ir_schedule_util.h" namespace cinn { namespace ir { @@ -70,5 +71,169 @@ void ScheduleBase::Replace(const Expr& src_sref, const Expr& tgt_stmt) { } } +void ScheduleBase::BroadcastToElementwise(const std::string& block_name, + const std::vector& axes) { + std::vector all_loops = this->GetLoops(block_name); + Expr broadcast_body = all_loops.back().As()->body; + + auto schedule_realize = broadcast_body.As() + ->expr_fields()[0] + ->As(); + auto schedule_block = + schedule_realize->schedule_block.As(); + auto iter_vars = schedule_block->iter_vars; + + auto load_exprs = ir::ir_utils::CollectIRNodesInOrder( + schedule_block->body, [&](const Expr* x) { return x->As(); }); + + for (auto load_expr : load_exprs) { + auto load = load_expr.As(); + load->indices.resize(all_loops.size(), Expr(0)); + + for (size_t i = 0; i < axes.size(); ++i) { + load->indices[axes[i]] = schedule_block->iter_vars[axes[i]]; + } + } +} + +void ScheduleBase::Broadcast(const std::string& block_name, + const BroadcastInfo& info) { + auto axes = info.broadcast_axes; + std::vector all_loops = this->GetLoops(block_name); + if (axes[0] >= all_loops.size()) { + throw std::runtime_error("axes execeed loop size"); + } + + // Get Last loop + Expr broadcast_body = all_loops.back().As()->body; + + auto schedule_realize = broadcast_body.As() + ->expr_fields()[0] + ->As(); + auto schedule_block = + schedule_realize->schedule_block.As(); + + auto iter_vars = schedule_block->iter_vars; + auto iter_values = schedule_realize->iter_values; + + auto factors = info.output_shape; + auto full_broadcast = info.full_broadcast; + auto first_broadcast = info.first_broadcast; + if (info.split_first) { + // iter value is one + for (size_t i = 0; i < axes.size(); ++i) { + // new_extent + auto axis = axes[i]; + auto loop_temp = all_loops[axis].As(); + int extent = factors[i]; + loop_temp->extent = Expr(extent); + + if (info.with_constrain) { + auto check = ir::EQ::Make(loop_temp->loop_var, Expr(0)); + schedule_block->body = + ir::IfThenElse::Make(check, schedule_block->body); + } + } + + // change load and store + // get new offset + all_loops = this->GetLoops(block_name); + auto offset = Expr(0); + auto stride = Expr(1); + auto in_offset = Expr(0); + + std::set brodacast_set(info.broadcast_axes.begin(), + info.broadcast_axes.end()); + for (int i = all_loops.size() - 1; i >= 0; --i) { + auto loop_temp = all_loops[i].As(); + offset = offset + loop_temp->loop_var * stride; + + stride = stride * loop_temp->extent; + if (!brodacast_set.count(i)) { + in_offset = in_offset + loop_temp->loop_var * stride; + } + } + + auto exprs = ir::ir_utils::CollectIRNodesInOrder( + schedule_block->body, + [&](const Expr* x) { return x->As(); }); + for (auto expr : exprs) { + auto store = expr.As(); + store->indices[0] = offset; + } + + exprs = ir::ir_utils::CollectIRNodesInOrder( + schedule_block->body, [&](const Expr* x) { return x->As(); }); + + for (auto expr : exprs) { + auto load = expr.As(); + if (!info.first_broadcast) { + load->indices[0] = offset; + } else { + load->indices[0] = in_offset; + } + } + + return; + } + + for (size_t i = 0; i < axes.size(); ++i) { + // new_extent + auto axis = axes[i]; + auto loop_temp = all_loops[axis].As(); + int extent = factors[i]; + loop_temp->extent = Expr(extent); + + if (!full_broadcast && (!(info.with_constrain))) { + schedule_realize->iter_values[axis] = loop_temp->loop_var; + } + + if (info.with_constrain) { + auto check = ir::EQ::Make(loop_temp->loop_var, Expr(0)); + schedule_block->body = ir::IfThenElse::Make(check, schedule_block->body); + } + } + + if (first_broadcast && !full_broadcast) { + auto exprs = ir::ir_utils::CollectIRNodesInOrder( + schedule_block->body, [&](const Expr* x) { return x->As(); }); + + if (info.op_name == "cinn_op.reshape") { + for (auto expr : exprs) { + auto load = expr.As(); + for (size_t k = 0; k < load->indices.size(); ++k) { + for (size_t i = 0; i < axes.size(); ++i) { + ReplaceExpr(&load->indices[k], + {schedule_block->iter_vars[axes[i]]}, + {Expr(0)}); + } + } + } + + return; + } + for (auto expr : exprs) { + auto load = expr.As(); + if (load->indices.size() == schedule_realize->iter_values.size()) { + for (size_t i = 0; i < axes.size(); ++i) { + load->indices[axes[i]] = Expr(0); + } + } else if (load->indices.size() < schedule_realize->iter_values.size()) { + // only one element + // replace t zeros + for (size_t k = 0; k < load->indices.size(); ++k) { + for (size_t i = 0; i < axes.size(); ++i) { + ReplaceExpr(&load->indices[k], + {schedule_block->iter_vars[axes[i]]}, + {Expr(0)}); + } + } + } else { + throw std::runtime_error("not support broadcast type yet"); + } + } + } +} + } // namespace ir } // namespace cinn diff --git a/paddle/cinn/ir/schedule/schedule_base.h b/paddle/cinn/ir/schedule/schedule_base.h index 6ce5caaeaad12..f4a3bd6127476 100644 --- a/paddle/cinn/ir/schedule/schedule_base.h +++ b/paddle/cinn/ir/schedule/schedule_base.h @@ -24,6 +24,19 @@ PD_DECLARE_int32(cinn_error_message_level); namespace cinn { namespace ir { +struct BroadcastInfo { + std::vector broadcast_axes; + std::vector output_shape; + + bool with_constrain{false}; + bool first_broadcast{false}; + bool full_broadcast{false}; + std::string op_name; + + bool split_first{false}; + std::vector>> split_info; +}; + /** * A struct representing a module that contains Expr. This struct is only used * in Schedule process. @@ -95,6 +108,7 @@ class ScheduleBase { virtual std::vector GetAllBlocks() const = 0; virtual std::vector GetChildBlocks(const Expr& expr) const = 0; virtual Expr GetBlock(const std::string& block_name) const = 0; + virtual std::vector Split(const Expr& loop, const std::vector& factors) = 0; virtual std::vector Split(const Expr& loop, @@ -142,7 +156,9 @@ class ScheduleBase { virtual void ReverseComputeInline(const Expr& schedule_block) = 0; virtual void Bind(const Expr& loop, const std::string& thread_axis) = 0; virtual Expr Rfactor(const Expr& rf_loop, int rf_axis) = 0; - virtual Expr FactorizeReduction(const Expr& rf_loop, int rf_axis) = 0; + virtual Expr FactorizeReduction(const Expr& rf_loop, + int rf_axis, + bool with_write_back_block_init = true) = 0; virtual Expr AddUnitLoop(const Expr& block) const = 0; virtual void Annotate(const Expr& block, const std::string& key, @@ -159,6 +175,12 @@ class ScheduleBase { const std::vector& candidates, const std::vector& probs) = 0; + void Broadcast(const std::string& block_name, + const cinn::ir::BroadcastInfo& info); + + void BroadcastToElementwise(const std::string& block_name, + const std::vector& axes); + protected: void Replace(const Expr& src_sref, const Expr& tgt_stmt); diff --git a/paddle/cinn/ir/schedule/schedule_desc.cc b/paddle/cinn/ir/schedule/schedule_desc.cc index b29d89fdd1dc9..74b9693c80b7e 100644 --- a/paddle/cinn/ir/schedule/schedule_desc.cc +++ b/paddle/cinn/ir/schedule/schedule_desc.cc @@ -483,6 +483,7 @@ CINN_BUILD_STEP_KIND(Rfactor) CINN_BUILD_STEP_KIND(FactorizeReduction) .Inputs({"rf_loop"}) .Attrs({"rf_axis"}) + .Attrs({"with_write_back_block_init"}) .SetApplyFn(APPLY_FUNC_UNIFORM( FREE_FUNCTION_CONVERTER(&IRSchedule::FactorizeReduction))); diff --git a/paddle/cinn/ir/utils/ir_copy.cc b/paddle/cinn/ir/utils/ir_copy.cc index c560652b5442b..e463df0fb067d 100644 --- a/paddle/cinn/ir/utils/ir_copy.cc +++ b/paddle/cinn/ir/utils/ir_copy.cc @@ -31,9 +31,15 @@ namespace ir { namespace ir_utils { namespace { struct IRCopyVisitor : public ir::IRVisitorRequireReImpl { + public: + explicit IRCopyVisitor(bool copy_buffer_node) + : copy_buffer_node(copy_buffer_node) {} + // Use maps to unify all the copied tensors and buffers. std::map tensor_map; std::map buffer_map; + // whether to deep copy Buffer node. + bool copy_buffer_node; Expr Visit(const Expr* op) override { return IRVisitorRequireReImpl::Visit(op); @@ -188,9 +194,14 @@ struct IRCopyVisitor : public ir::IRVisitorRequireReImpl { auto name = op->name; auto tensor = make_shared<_Tensor_>(); + // tensor->buffer = op->buffer; if (buffer_expr.defined()) { - auto buffer = Visit(&buffer_expr); - tensor->buffer = buffer.as_buffer_ref(); + if (copy_buffer_node) { + auto buffer = Visit(&buffer_expr); + tensor->buffer = buffer.as_buffer_ref(); + } else { + tensor->buffer = op->buffer; + } } tensor->domain = domain; tensor->shape = shape; @@ -405,6 +416,7 @@ struct IRCopyVisitor : public ir::IRVisitorRequireReImpl { Expr res = ir::ScheduleBlock::Make( iter_vars, read_buffers, write_buffers, op->name, Visit(&op->body)); res.As()->attrs = op->attrs; + res.As()->reduce_method = op->reduce_method; return res; } @@ -489,35 +501,36 @@ Expr IRCopyVisitor::Visit(const ir::intrinsics::BuiltinIntrin* op) { op->name, op->args, op->id, op->arg_nums, op->type()); } } // namespace -Expr IRCopy(Expr x) { - IRCopyVisitor visitor; +Expr IRCopy(Expr x, bool copy_buffer_node) { + IRCopyVisitor visitor(copy_buffer_node); auto copied = visitor.Visit(&x); return copied; } -std::vector IRCopy(const std::vector& x) { +std::vector IRCopy(const std::vector& x, bool copy_buffer_node) { std::vector res; for (auto& i : x) { - res.emplace_back(IRCopy(i)); + res.emplace_back(IRCopy(i, copy_buffer_node)); } return res; } -ir::ModuleExpr IRCopy(const ir::ModuleExpr& x) { - return ir::ModuleExpr(IRCopy(x.GetExprs())); +ir::ModuleExpr IRCopy(const ir::ModuleExpr& x, bool copy_buffer_node) { + return ir::ModuleExpr(IRCopy(x.GetExprs(), copy_buffer_node)); } -ir::LoweredFunc IRCopy(const ir::LoweredFunc& x) { - ir::Expr copy_func_expr = IRCopy(static_cast(x)); +ir::LoweredFunc IRCopy(const ir::LoweredFunc& x, bool copy_buffer_node) { + ir::Expr copy_func_expr = IRCopy(static_cast(x), copy_buffer_node); ir::_LoweredFunc_* copy_func_ptr = copy_func_expr.As(); return ir::LoweredFunc(copy_func_ptr); } // TODO(zhhsplendid): make IRCopy of std::vector a template function -std::vector IRCopy(const std::vector& x) { +std::vector IRCopy(const std::vector& x, + bool copy_buffer_node) { std::vector res; for (const auto& i : x) { - res.emplace_back(IRCopy(i)); + res.emplace_back(IRCopy(i, copy_buffer_node)); } return res; } diff --git a/paddle/cinn/ir/utils/ir_copy.h b/paddle/cinn/ir/utils/ir_copy.h index 594f07e91cfa0..69bcc16ab13dd 100644 --- a/paddle/cinn/ir/utils/ir_copy.h +++ b/paddle/cinn/ir/utils/ir_copy.h @@ -28,15 +28,17 @@ class ModuleExpr; namespace ir_utils { //! Shallow copy an expression. -Expr IRCopy(Expr x); +Expr IRCopy(Expr x, bool copy_buffer_node = true); -std::vector IRCopy(const std::vector& x); +std::vector IRCopy(const std::vector& x, + bool copy_buffer_node = true); -ir::ModuleExpr IRCopy(const ir::ModuleExpr& x); +ir::ModuleExpr IRCopy(const ir::ModuleExpr& x, bool copy_buffer_node = true); -ir::LoweredFunc IRCopy(const ir::LoweredFunc& x); +ir::LoweredFunc IRCopy(const ir::LoweredFunc& x, bool copy_buffer_node = true); -std::vector IRCopy(const std::vector& x); +std::vector IRCopy(const std::vector& x, + bool copy_buffer_node = true); } // namespace ir_utils } // namespace ir diff --git a/paddle/cinn/ir/utils/ir_replace.cc b/paddle/cinn/ir/utils/ir_replace.cc index 7e64e7aaa7e7f..5e782536c1d3a 100644 --- a/paddle/cinn/ir/utils/ir_replace.cc +++ b/paddle/cinn/ir/utils/ir_replace.cc @@ -50,7 +50,7 @@ struct IrReplaceVarBroadcastMutator : ir::IRMutator { void Visit(const ir::Broadcast* op, Expr* expr) override { if (op->node_type() == from_->node_type() && from_repr_ == GetStreamCnt(*expr)) { - *expr = ir::ir_utils::IRCopy(to_); + *expr = ir::ir_utils::IRCopy(to_, /* copy_buffer_node = */ false); } } @@ -68,7 +68,7 @@ struct IrReplaceMutator : ir::IRMutator { void Visit(const Expr* op, Expr* expr) override { ir::IRMutator<>::Visit(expr, expr); if (from_repr_ == GetStreamCnt(*expr)) { - *expr = ir::ir_utils::IRCopy(to_); + *expr = ir::ir_utils::IRCopy(to_, /* copy_buffer_node = */ false); } } diff --git a/paddle/cinn/optim/replace_call_with_expr.cc b/paddle/cinn/optim/replace_call_with_expr.cc index 00fbca0fca623..d6ba57210ee45 100644 --- a/paddle/cinn/optim/replace_call_with_expr.cc +++ b/paddle/cinn/optim/replace_call_with_expr.cc @@ -36,7 +36,8 @@ struct ReplaceCallWithExprModifier : public ir::IRMutator<> { VLOG(3) << "Processing Call node " << *op; if (statement_ != node->name) return; - Expr expr_candidate = ir::ir_utils::IRCopy(candidate_); + Expr expr_candidate = + ir::ir_utils::IRCopy(candidate_, /* copy_buffer_node = */ false); VLOG(3) << "Original candidate expr: " << candidate_; VLOG(3) << "Copied candidate expr: " << expr_candidate; @@ -62,7 +63,7 @@ void ReplaceIslCallWithExpr(Expr *e, const Expr &candidate, const std::map &axis_map) { VLOG(3) << "ReplaceCallWithExpr, original expression: " << candidate; - Expr copied = ir::ir_utils::IRCopy(candidate); + Expr copied = ir::ir_utils::IRCopy(candidate, /* copy_buffer_node = */ false); // update the axis in the copied expression. // we treat the Store node as the normal statement, the others like Call node diff --git a/paddle/cinn/optim/replace_cross_thread_reduction.cc b/paddle/cinn/optim/replace_cross_thread_reduction.cc index 2524874bace60..1ea9bae562361 100644 --- a/paddle/cinn/optim/replace_cross_thread_reduction.cc +++ b/paddle/cinn/optim/replace_cross_thread_reduction.cc @@ -19,6 +19,7 @@ #include "paddle/cinn/optim/replace_cross_thread_reduction.h" #include +#include "paddle/cinn/adt/adt.h" #include "paddle/cinn/common/common.h" #include "paddle/cinn/hlir/pe/reduction.h" #include "paddle/cinn/ir/ir.h" @@ -46,6 +47,7 @@ struct CrossThreadReductionReplacer : public ir::IRMutator<> { bool CanReplace(const ir::ScheduleBlockRealize* block_realize) { const ir::ScheduleBlock* schedule_block = block_realize->schedule_block.As(); + CHECK_NOTNULL(schedule_block); if (block_realize->schedule_block.As()->name.substr( @@ -67,20 +69,27 @@ struct CrossThreadReductionReplacer : public ir::IRMutator<> { if (x->as_var()) { reduce_var_names.insert(x->as_var()->name); } + return false; }); } + auto IsThreadBindOnReduceAxis = [&](const ir::For* for_node) { + return reduce_var_names.count(for_node->loop_var->name) > 0 && + for_node->is_gpu_thread_binded(); + }; + std::vector thread_binded_reduce_loop_indices; + bool is_thread_binded_inner_loop = false; for (int i = 0; i < cur_loops_.size(); ++i) { - if (reduce_var_names.count(cur_loops_[i].As()->loop_var->name) > - 0) { - if (cur_loops_[i].As()->is_gpu_thread_binded()) { - if (ir::GetLoopExtent(cur_loops_[i]) > 1024) { - return false; - } - thread_binded_reduce_loop_indices.push_back(i); + if (is_thread_binded_inner_loop || + IsThreadBindOnReduceAxis(cur_loops_[i].As())) { + if (ir::GetLoopExtent(cur_loops_[i]) > 1024) { + return false; } + + is_thread_binded_inner_loop = true; + thread_binded_reduce_loop_indices.push_back(i); } } if (thread_binded_reduce_loop_indices.size() == 0 || @@ -138,6 +147,14 @@ struct CrossThreadReductionReplacer : public ir::IRMutator<> { original_update_stmt = original_update_body; } + const auto& IsWarpReduce = cinn::adt::match{ + [&](const ir::NoneReduceMethod&) { return ir::Expr(false); }, + [&](const ir::WarpReduceMethod&) { return ir::Expr(true); }, + [&](const ir::BlockReduceMethod&) { return ir::Expr(false); }, + }; + ir::Expr return_warp = + std::visit(IsWarpReduce, schedule_block->reduce_method); + #define REPLACE_TO_EXTERNAL_CALL(Op) \ if (original_update_stmt.As()->value.As()) { \ auto* node = original_update_stmt.As()->value.As(); \ @@ -154,8 +171,8 @@ struct CrossThreadReductionReplacer : public ir::IRMutator<> { tmp_buffer->dtype = tmp_dtype; \ tmp_buffer->memory_type = ir::MemoryType::GPUShared; \ shm_buffer_.insert(tmp_buffer); \ - original_update_stmt.As()->value = \ - lang::CallExtern(reduce_func_name, {node->b(), tmp_buffer}); \ + original_update_stmt.As()->value = lang::CallExtern( \ + reduce_func_name, {node->b(), tmp_buffer, return_warp}); \ } REPLACE_TO_EXTERNAL_CALL(ir::Add) diff --git a/paddle/cinn/optim/replace_cross_thread_reduction_test.cc b/paddle/cinn/optim/replace_cross_thread_reduction_test.cc index d7bd9f6defc49..9f616c7f8a5f2 100644 --- a/paddle/cinn/optim/replace_cross_thread_reduction_test.cc +++ b/paddle/cinn/optim/replace_cross_thread_reduction_test.cc @@ -71,7 +71,7 @@ TEST(CrossThreadReductionReplacer, basic) { ScheduleBlock(B) { i0_0, i1 = axis.bind(i, reduce_j) - B[i0_0] = cinn_block_reduce_sum_fp32_internal_shm(A[i0_0, i1], _Buffer_(shm32__fp32_reduce)) + B[i0_0] = cinn_block_reduce_sum_fp32_internal_shm(A[i0_0, i1], _Buffer_(shm32__fp32_reduce), false) } } } diff --git a/paddle/cinn/optim/unroll_loops.cc b/paddle/cinn/optim/unroll_loops.cc index 7fa5e3a8b8222..276a633924991 100644 --- a/paddle/cinn/optim/unroll_loops.cc +++ b/paddle/cinn/optim/unroll_loops.cc @@ -94,7 +94,8 @@ struct UnrollMutator : public ir::IRMutator { for (int i = min->value; i < extent->value; i++) { Expr start = op->min + i; - body.push_back(ir::ir_utils::IRCopy(op->body)); + body.push_back( + ir::ir_utils::IRCopy(op->body, /* copy_buffer_node = */ false)); cinn::ir::ir_utils::IrReplaceVarBroadcast( &body.back(), op->loop_var, start); } diff --git a/paddle/cinn/optim/vectorize_loops.cc b/paddle/cinn/optim/vectorize_loops.cc index 67e309c73a6a0..cb9daf761f659 100644 --- a/paddle/cinn/optim/vectorize_loops.cc +++ b/paddle/cinn/optim/vectorize_loops.cc @@ -810,7 +810,8 @@ struct VectorizeLoops_ : public IRMutator { cuda_vectorizer.Visit(&new_forloop->body); // unroll the new forloop to compute each element of the vector // iteratively - auto copied_loop = ir::ir_utils::IRCopy(_new_forloop); + auto copied_loop = + ir::ir_utils::IRCopy(_new_forloop, /* copy_buffer_node = */ false); copied_loop.As()->set_unrolled(); optim::UnrollLoop(&copied_loop); // add cast exprs of vector type in the front of vectorized forloop, @@ -893,13 +894,14 @@ struct VectorizeLoops_ : public IRMutator { Var new_iterator_outer( cinn::common::UniqName(outer_for->loop_var->name + "_s")); - Expr inner_for_b = - Block::Make({For::Make(new_iterator_inner, - inner_for->min, - b, - ForType::Serial, - DeviceAPI::UNK, - ir::ir_utils::IRCopy(inner_for->body))}); + Expr inner_for_b = Block::Make({For::Make( + new_iterator_inner, + inner_for->min, + b, + ForType::Serial, + DeviceAPI::UNK, + ir::ir_utils::IRCopy(inner_for->body, + /* copy_buffer_node = */ false))}); cinn::ir::ir_utils::IrReplaceVarBroadcast( &inner_for_b, inner_for->loop_var, Expr(new_iterator_inner)); diff --git a/paddle/cinn/pybind/optim.cc b/paddle/cinn/pybind/optim.cc index bb1a18a2c24fe..4f40ea660149c 100755 --- a/paddle/cinn/pybind/optim.cc +++ b/paddle/cinn/pybind/optim.cc @@ -42,7 +42,10 @@ void BindSimplify(py::module* m) { }, py::arg("expr")); - m->def("ir_copy", py::overload_cast(&ir::ir_utils::IRCopy)); + m->def("ir_copy", + py::overload_cast(&ir::ir_utils::IRCopy), + py::arg("x"), + py::arg("copy_buffer_node") = true); } } // namespace diff --git a/paddle/fluid/pir/transforms/build_cinn_pass.cc b/paddle/fluid/pir/transforms/build_cinn_pass.cc index 34d9fde7831c8..2a89223dac3e6 100644 --- a/paddle/fluid/pir/transforms/build_cinn_pass.cc +++ b/paddle/fluid/pir/transforms/build_cinn_pass.cc @@ -48,6 +48,9 @@ class BuildCinnPass : public pir::Pass { ::pir::SubgraphDetector(block, CompatibleInfo::IsSupportCinn)(); AddStatistics(groups.size()); for (auto& group_ops : groups) { + if (group_ops.size() == 1 && group_ops[0]->name() == "pd_op.full") { + continue; + } VLOG(4) << "current group_ops.size(): " << group_ops.size(); ::pir::ReplaceWithGroupOp(block, group_ops); } diff --git a/test/cpp/pir/cinn/CMakeLists.txt b/test/cpp/pir/cinn/CMakeLists.txt index e9fb68c24e962..855b610d47303 100644 --- a/test/cpp/pir/cinn/CMakeLists.txt +++ b/test/cpp/pir/cinn/CMakeLists.txt @@ -17,7 +17,7 @@ if(WITH_TESTING AND WITH_CINN) paddle_test(test_ir_op_cluster SRCS ir_op_cluster_test.cc DEPS pir_transforms cinn_transforms) - paddle_test(test_pir_all_path SRCS pir_all_path_test.cc) + # paddle_test(test_pir_all_path SRCS pir_all_path_test.cc DEPS cinn_transforms) paddle_test(test_group_op SRCS group_op_test.cc) @@ -39,7 +39,7 @@ if(WITH_TESTING AND WITH_CINN) test_add_broadcast_to_elementwise test_sub_graph_extract test_ir_op_fusion - test_pir_all_path + # test_pir_all_path test_group_op test_pir_build_cinn_pass test_compilation_task @@ -50,8 +50,11 @@ if(WITH_TESTING AND WITH_CINN) env TEST ${test_name} PROPERTY ENVIRONMENT) - set_property(TEST ${test_name} - PROPERTY ENVIRONMENT "FLAGS_cinn_new_group_scheduler=1" ${env}) + set_property( + TEST ${test_name} + PROPERTY ENVIRONMENT "FLAGS_cinn_new_group_scheduler=1" + "FLAGS_cinn_bucket_compile=1" + "FLAGS_group_schedule_tiling_first=1" ${env}) set_tests_properties(${test_name} PROPERTIES LABELS "RUN_TYPE=CINN") endforeach() diff --git a/test/cpp/pir/cinn/pir_all_path_test.cc b/test/cpp/pir/cinn/pir_all_path_test.cc index 8bd510e98bb93..504b8daa74e44 100644 --- a/test/cpp/pir/cinn/pir_all_path_test.cc +++ b/test/cpp/pir/cinn/pir_all_path_test.cc @@ -20,8 +20,11 @@ #include "paddle/cinn/hlir/dialect/operator/ir/manual_op.h" #include "paddle/cinn/hlir/dialect/operator/ir/op_dialect.h" #include "paddle/cinn/hlir/dialect/operator/transforms/add_broadcast_to_elementwise_pass.h" +#include "paddle/cinn/hlir/dialect/operator/transforms/add_store_in_fusion_op_pass.h" +#include "paddle/cinn/hlir/dialect/operator/transforms/cinn_group_cluster_pass.h" #include "paddle/cinn/hlir/dialect/operator/transforms/group_merge/divide_group_op_to_fusion_op_pass.h" #include "paddle/cinn/hlir/dialect/operator/transforms/lower_cinn_fusion_op_pass.h" +#include "paddle/cinn/hlir/dialect/operator/transforms/merge_reshape_with_broadcast_pass.h" #include "paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.h" #include "paddle/fluid/framework/new_executor/interpretercore.h" #include "paddle/fluid/pir/dialect/operator/ir/op_dialect.h" @@ -62,10 +65,14 @@ static void RunAndCheckResult(::pir::Program* program, pir::PassManager pm(ctx); pm.AddPass(cinn::dialect::ir::CreatePdOpToCinnOpPass()); pm.AddPass(cinn::dialect::ir::CreateAddBroadcastToElementwisePass()); + pm.AddPass( + std::make_unique()); pm.AddPass(pir::CreateDeadCodeEliminationPass()); pm.AddPass(pir::CreateBuildCinnPass()); - pm.AddPass(cinn::dialect::ir::CreateDivideGroupOpToFusionOpPass()); + pm.AddPass(cinn::dialect::ir::CreateCinnGroupClusterPass()); + pm.AddPass(cinn::dialect::ir::CreateAddStoreInFusionOpPass()); + pm.AddPass(pir::CreateDeadCodeEliminationPass()); pm.AddPass(cinn::dialect::ir::CreateLowerCinnFusionOpPass()); pm.EnableIRPrinting(); CHECK_EQ(pm.Run(program), true); @@ -129,571 +136,554 @@ TEST(GroupOp, TestBuild) { RunAndCheckResult(program.get(), true, 1.0 / 768); } -// std::shared_ptr<::pir::Program> BuildLayerNormProgram() { -// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); -// ctx->GetOrRegisterDialect(); -// auto program = std::make_shared<::pir::Program>(ctx); -// ::pir::Builder builder = ::pir::Builder(ctx, program->block()); - -// std::vector axes{-1}; -// auto x = -// builder -// .Build(std::vector({128, 128, -// 768}), -// 1.0, -// phi::DataType::FLOAT32, -// phi::GPUPlace()) -// .result(0); - -// auto bias = builder -// .Build(std::vector({768}), -// 1.0, -// phi::DataType::FLOAT32, -// phi::GPUPlace()) -// .result(0); - -// auto scale = builder -// .Build(std::vector({768}), -// 1.0, -// phi::DataType::FLOAT32, -// phi::GPUPlace()) -// .result(0); - -// auto num = builder -// .Build(std::vector{1}, -// 768.0, -// phi::DataType::FLOAT32, -// phi::CPUPlace()) -// .result(0); -// auto eps = builder -// .Build(std::vector{1}, -// 1e-5, -// phi::DataType::FLOAT32, -// phi::CPUPlace()) -// .result(0); - -// auto sum = -// builder -// .Build(x, axes, phi::DataType::FLOAT32, -// true) .result(0); - -// auto mean = builder.Build(sum, num).result(0); -// auto power = builder.Build(x, x).result(0); -// auto power_sum = builder -// .Build( -// power, axes, phi::DataType::FLOAT32, true) -// .result(0); -// auto mean2 = -// builder.Build(power_sum, num).result(0); -// auto power_mean = -// builder.Build(mean, mean).result(0); - -// auto var = -// builder.Build(mean2, -// power_mean).result(0); - -// auto sub = builder.Build(x, mean).result(0); -// auto t1 = builder.Build(var, eps).result(0); -// auto t2 = builder.Build(t1).result(0); -// auto t3 = builder.Build(sub, t2).result(0); -// auto t5 = builder.Build(t3, scale).result(0); -// auto out = builder.Build(t5, bias).result(0); - -// builder.Build(out, "out", 0); -// return program; -// } - -// TEST(GroupOp, TestBuildLayerNorm) { -// // Step 1: Construct pir::Program -// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); -// std::shared_ptr<::pir::Program> program = BuildLayerNormProgram(); - -// RunAndCheckResult(program.get(), false); -// } - -// std::shared_ptr<::pir::Program> BuildDropOutProgram() { -// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); -// ctx->GetOrRegisterDialect(); -// auto program = std::make_shared<::pir::Program>(ctx); -// ::pir::Builder builder = ::pir::Builder(ctx, program->block()); - -// auto x = -// builder -// .Build(std::vector({128, 128, -// 768}), -// 1.0, -// phi::DataType::FLOAT32, -// phi::GPUPlace()) -// .result(0); - -// auto prob = builder -// .Build(std::vector({1}), -// 0.5, -// phi::DataType::FLOAT32, -// phi::GPUPlace()) -// .result(0); - -// auto random = builder -// .Build( -// std::vector({128, 128, 768}), -// phi::DataType::FLOAT32, -// 0.0, -// 1.0, -// 0, -// phi::GPUPlace()) -// .result(0); - -// auto mask = -// builder.Build(random, prob).result(0); -// auto mask1 = -// builder.Build(mask, phi::DataType::FLOAT32) -// .result(0); -// auto mul = builder.Build(x, mask1).result(0); -// auto neg_prob = prob = -// builder -// .Build(std::vector({1}), -// 0.5, -// phi::DataType::FLOAT32, -// phi::GPUPlace()) -// .result(0); -// auto out = builder.Build(mul, -// neg_prob).result(0); - -// builder.Build(out, "out", 0); -// return program; -// } - -// TEST(GroupOp, TestBuildDropout) { -// // Step 1: Construct pir::Program -// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); -// std::shared_ptr<::pir::Program> program = BuildDropOutProgram(); - -// RunAndCheckResult(program.get(), false); -// } - -// std::shared_ptr<::pir::Program> BuildScaleGroupProgram() { -// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); -// ctx->GetOrRegisterDialect(); - -// auto program = std::make_shared<::pir::Program>(ctx); -// ::pir::Builder builder = ::pir::Builder(ctx, program->block()); - -// // full -> softmax(max -> subtract -> exp -> sum -> divide) -// const float value_one = 1.0; -// const std::vector shape = {16, 16}; -// auto x = builder -// .Build( -// shape, value_one, phi::DataType::FLOAT32, phi::GPUPlace()) -// .result(0); - -// auto out = -// builder.Build(x, 0.5, 0.0, false).result(0); - -// builder.Build(out, "out", 0); -// return program; -// } - -// TEST(GroupOp, TestBuildScale) { -// // Step 1: Construct pir::Program -// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); -// std::shared_ptr<::pir::Program> program = BuildScaleGroupProgram(); - -// RunAndCheckResult(program.get(), true, 0.5); -// } - -// std::shared_ptr<::pir::Program> BuildScaleTensorGroupProgram() { -// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); -// ctx->GetOrRegisterDialect(); - -// auto program = std::make_shared<::pir::Program>(ctx); -// ::pir::Builder builder = ::pir::Builder(ctx, program->block()); - -// // full -> softmax(max -> subtract -> exp -> sum -> divide) -// const float value_one = 0.5; -// const std::vector shape = {16, 16}; -// auto x = builder -// .Build( -// shape, value_one, phi::DataType::FLOAT32, phi::GPUPlace()) -// .result(0); -// auto scale = builder -// .Build(std::vector({1}), -// 0.0, -// phi::DataType::FLOAT32, -// phi::GPUPlace()) -// .result(0); -// auto factor = builder.Build(scale).result(0); -// auto out = -// builder.Build(x, factor, 0.0, -// false).result(0); - -// builder.Build(out, "out", 0); -// return program; -// } - -// TEST(GroupOp, TestBuildScaleTensor) { -// // Step 1: Construct pir::Program -// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); -// std::shared_ptr<::pir::Program> program = BuildScaleTensorGroupProgram(); - -// RunAndCheckResult(program.get(), true, 0.5); -// } - -// std::shared_ptr<::pir::Program> BuildPowerProgram() { -// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); -// ctx->GetOrRegisterDialect(); -// auto program = std::make_shared<::pir::Program>(ctx); -// ::pir::Builder builder = ::pir::Builder(ctx, program->block()); - -// auto x = builder -// .Build(std::vector({16, -// 16}), -// 2.0, -// phi::DataType::FLOAT32, -// phi::GPUPlace()) -// .result(0); - -// auto factor = -// builder -// .Build(std::vector({16, 16}), -// 2.0, -// phi::DataType::FLOAT32, -// phi::GPUPlace()) -// .result(0); - -// auto power1 = -// builder.Build(x, factor).result(0); - -// auto power2 = builder.Build(power1, 2.0).result(0); -// auto out = -// builder -// .Build(power2, -// std::vector({-1})) .result(0); - -// builder.Build(out, "out", 0); -// return program; -// } - -// TEST(GroupOp, TestBuildPower) { -// // Step 1: Construct pir::Program -// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); -// std::shared_ptr<::pir::Program> program = BuildPowerProgram(); - -// RunAndCheckResult(program.get(), true, 16.0); -// } - -// std::shared_ptr<::pir::Program> BuildLayerNorm2Program() { -// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); -// ctx->GetOrRegisterDialect(); -// auto program = std::make_shared<::pir::Program>(ctx); -// ::pir::Builder builder = ::pir::Builder(ctx, program->block()); - -// std::vector axes{-1}; -// auto x = -// builder -// .Build(std::vector({128, 128, -// 768}), -// 1.0, -// phi::DataType::FLOAT32, -// phi::GPUPlace()) -// .result(0); - -// auto bias = builder -// .Build(std::vector({768}), -// 1.0, -// phi::DataType::FLOAT32, -// phi::GPUPlace()) -// .result(0); - -// auto scale = builder -// .Build(std::vector({768}), -// 1.0, -// phi::DataType::FLOAT32, -// phi::GPUPlace()) -// .result(0); - -// auto num = -// builder -// .Build(std::vector{128, 128, 1}, -// 768.0, -// phi::DataType::FLOAT32, -// phi::CPUPlace()) -// .result(0); -// auto sum = -// builder -// .Build(x, axes, phi::DataType::FLOAT32, -// true) .result(0); - -// auto mean = builder.Build(sum, num).result(0); - -// auto diff = builder.Build(x, mean).result(0); - -// auto power = builder.Build(diff, -// diff).result(0); auto power_sum = builder -// .Build( -// power, axes, phi::DataType::FLOAT32, true) -// .result(0); -// auto num2 = -// builder -// .Build(std::vector{128, 128, 1}, -// 768.0, -// phi::DataType::FLOAT32, -// phi::CPUPlace()) -// .result(0); -// auto var2 = -// builder.Build(power_sum, num2).result(0); - -// auto t1 = builder.Build(var2, 1.0, -// 1e-5).result(0); auto factor = builder -// .Build(std::vector{1}, -// -0.5, -// phi::DataType::FLOAT32, -// phi::CPUPlace()) -// .result(0); -// auto t2 = -// builder.Build(t1, factor).result(0); -// // auto t2 = builder.Build(t1).result(0); -// auto t3 = builder.Build(diff, t2).result(0); -// auto t5 = builder.Build(t3, scale).result(0); -// auto out = builder.Build(t5, bias).result(0); -// auto mean_out = -// builder -// .Build(mean, -// std::vector({-1})) .result(0); -// auto mean2_out = -// builder -// .Build(var2, -// std::vector({-1})) .result(0); - -// builder.Build(out, "out", 0); -// builder.Build(mean_out, "mean", 0); -// builder.Build(mean2_out, "var", 0); -// return program; -// } - -// TEST(GroupOp, TestBuildLayerNorm2) { -// // Step 1: Construct pir::Program -// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); -// std::shared_ptr<::pir::Program> program = BuildLayerNorm2Program(); - -// RunAndCheckResult(program.get(), false); -// } - -// std::shared_ptr<::pir::Program> BuildSum2GroupProgram() { -// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); -// ctx->GetOrRegisterDialect(); -// auto program = std::make_shared<::pir::Program>(ctx); -// ::pir::Builder builder = ::pir::Builder(ctx, program->block()); - -// auto x = builder -// .Build(std::vector({16, -// 16}), -// 0.0, -// phi::DataType::FLOAT32, -// phi::GPUPlace()) -// .result(0); - -// auto cos = builder.Build(x).result(0); - -// auto y = builder -// .Build(std::vector({8, 8}), -// 0.0, -// phi::DataType::FLOAT32, -// phi::GPUPlace()) -// .result(0); - -// auto sin = builder.Build(y).result(0); - -// builder.Build(cos, "out", 0); -// builder.Build(sin, "out2", 0); -// return program; -// } - -// TEST(GroupOp, TestBuildSum2Group) { -// // Step 1: Construct pir::Program -// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); -// std::shared_ptr<::pir::Program> program = BuildSum2GroupProgram(); - -// RunAndCheckResult(program.get(), true, 1.0); -// } - -// std::shared_ptr<::pir::Program> BuildConcatProgram() { -// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); -// ctx->GetOrRegisterDialect(); -// auto program = std::make_shared<::pir::Program>(ctx); -// ::pir::Builder builder = ::pir::Builder(ctx, program->block()); - -// auto x = builder -// .Build(std::vector({16, -// 16}), -// 2.0, -// phi::DataType::FLOAT32, -// phi::GPUPlace()) -// .result(0); - -// auto y = builder -// .Build(std::vector({16, -// 16}), -// 2.0, -// phi::DataType::FLOAT32, -// phi::GPUPlace()) -// .result(0); - -// auto t1 = -// builder.Build(std::vector({x, -// y})).result(0); - -// auto out = builder.Build(t1, 1).result(0); - -// builder.Build(out, "out", 0); -// return program; -// } - -// TEST(GroupOp, TestBuildConcat) { -// // Step 1: Construct pir::Program -// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); -// std::shared_ptr<::pir::Program> program = BuildConcatProgram(); - -// RunAndCheckResult(program.get(), true, 2.0); -// } - -// std::shared_ptr<::pir::Program> BuildSliceProgram() { -// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); -// ctx->GetOrRegisterDialect(); -// auto program = std::make_shared<::pir::Program>(ctx); -// ::pir::Builder builder = ::pir::Builder(ctx, program->block()); - -// auto x = builder -// .Build(std::vector({16, -// 16}), -// 2.0, -// phi::DataType::FLOAT32, -// phi::GPUPlace()) -// .result(0); - -// auto out = builder -// .Build(x, -// std::vector({1}), -// std::vector({0}), -// std::vector({2}), -// std::vector({}), -// std::vector({})) -// .result(0); - -// builder.Build(out, "out", 0); -// return program; -// } - -// TEST(GroupOp, TestBuildSlice) { -// // Step 1: Construct pir::Program -// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); -// std::shared_ptr<::pir::Program> program = BuildSliceProgram(); - -// RunAndCheckResult(program.get(), true, 2.0); -// } - -// std::shared_ptr<::pir::Program> BuildSplitProgram() { -// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); -// ctx->GetOrRegisterDialect(); -// auto program = std::make_shared<::pir::Program>(ctx); -// ::pir::Builder builder = ::pir::Builder(ctx, program->block()); - -// auto x = builder -// .Build(std::vector({16, -// 16}), -// 2.0, -// phi::DataType::FLOAT32, -// phi::GPUPlace()) -// .result(0); - -// auto out_arr = -// builder.Build(x, 4, -1).result(0); -// auto out = builder.Build(out_arr, 0).result(0); -// builder.Build(out, "out", 0); -// return program; -// } - -// TEST(GroupOp, TestBuildSplit) { -// // Step 1: Construct pir::Program -// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); -// std::shared_ptr<::pir::Program> program = BuildSplitProgram(); - -// RunAndCheckResult(program.get(), true, 2.0); -// } - -// std::shared_ptr<::pir::Program> BuildAddNProgram() { -// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); -// ctx->GetOrRegisterDialect(); -// auto program = std::make_shared<::pir::Program>(ctx); -// ::pir::Builder builder = ::pir::Builder(ctx, program->block()); - -// auto x = builder -// .Build(std::vector({16, -// 16}), -// 2.0, -// phi::DataType::FLOAT32, -// phi::GPUPlace()) -// .result(0); - -// auto y = builder -// .Build(std::vector({16, -// 16}), -// 2.0, -// phi::DataType::FLOAT32, -// phi::GPUPlace()) -// .result(0); - -// auto z = builder -// .Build(std::vector({16, -// 16}), -// 2.0, -// phi::DataType::FLOAT32, -// phi::GPUPlace()) -// .result(0); - -// auto t1 = builder.Build(std::vector({x, y, z})) -// .result(0); - -// auto out = builder.Build(t1).result(0); - -// builder.Build(out, "out", 0); -// return program; -// } - -// TEST(GroupOp, TestBuildAddN) { -// // Step 1: Construct pir::Program -// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); -// std::shared_ptr<::pir::Program> program = BuildAddNProgram(); - -// RunAndCheckResult(program.get(), true, 6.0); -// } - -// std::shared_ptr<::pir::Program> BuildSplitSectionProgram() { -// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); -// ctx->GetOrRegisterDialect(); -// auto program = std::make_shared<::pir::Program>(ctx); -// ::pir::Builder builder = ::pir::Builder(ctx, program->block()); - -// auto x = builder -// .Build(std::vector({16, -// 16}), -// 2.0, -// phi::DataType::FLOAT32, -// phi::GPUPlace()) -// .result(0); - -// auto split_arr = builder -// .Build( -// x, std::vector({3, 5, 8}), -1) -// .out(); -// auto out = builder.Build(split_arr, 0).result(0); -// builder.Build(out, "out", 0); -// return program; -// } - -// TEST(GroupOp, TestBuildSplitSection) { -// // Step 1: Construct pir::Program -// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); -// std::shared_ptr<::pir::Program> program = BuildSplitSectionProgram(); - -// RunAndCheckResult(program.get(), 2.0); -// } +std::shared_ptr<::pir::Program> BuildLayerNormProgram() { + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + ctx->GetOrRegisterDialect(); + auto program = std::make_shared<::pir::Program>(ctx); + ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + + std::vector axes{-1}; + auto x = + builder + .Build(std::vector({128, 128, 768}), + 1.0, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + + auto bias = builder + .Build(std::vector({768}), + 1.0, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + + auto scale = builder + .Build(std::vector({768}), + 1.0, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + + auto num = builder + .Build(std::vector{1}, + 768.0, + phi::DataType::FLOAT32, + phi::CPUPlace()) + .result(0); + auto eps = builder + .Build(std::vector{1}, + 1e-5, + phi::DataType::FLOAT32, + phi::CPUPlace()) + .result(0); + + auto sum = + builder + .Build(x, axes, phi::DataType::FLOAT32, true) + .result(0); + + auto mean = builder.Build(sum, num).result(0); + auto power = builder.Build(x, x).result(0); + auto power_sum = builder + .Build( + power, axes, phi::DataType::FLOAT32, true) + .result(0); + auto mean2 = + builder.Build(power_sum, num).result(0); + auto power_mean = + builder.Build(mean, mean).result(0); + + auto var = + builder.Build(mean2, power_mean).result(0); + + auto sub = builder.Build(x, mean).result(0); + auto t1 = builder.Build(var, eps).result(0); + auto t2 = builder.Build(t1).result(0); + auto t3 = builder.Build(sub, t2).result(0); + auto t5 = builder.Build(t3, scale).result(0); + auto out = builder.Build(t5, bias).result(0); + + builder.Build(out, "out", 0); + return program; +} + +TEST(GroupOp, TestBuildLayerNorm) { + // Step 1: Construct pir::Program + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + std::shared_ptr<::pir::Program> program = BuildLayerNormProgram(); + + RunAndCheckResult(program.get(), false); +} + +std::shared_ptr<::pir::Program> BuildDropOutProgram() { + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + ctx->GetOrRegisterDialect(); + auto program = std::make_shared<::pir::Program>(ctx); + ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + + auto x = + builder + .Build(std::vector({128, 128, 768}), + 1.0, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + + auto prob = builder + .Build(std::vector({1}), + 0.5, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + + auto random = builder + .Build( + std::vector({128, 128, 768}), + phi::DataType::FLOAT32, + 0.0, + 1.0, + 0, + phi::GPUPlace()) + .result(0); + + auto mask = + builder.Build(random, prob).result(0); + auto mask1 = + builder.Build(mask, phi::DataType::FLOAT32) + .result(0); + auto mul = builder.Build(x, mask1).result(0); + auto neg_prob = prob = + builder + .Build(std::vector({1}), + 0.5, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + auto out = builder.Build(mul, neg_prob).result(0); + + builder.Build(out, "out", 0); + return program; +} + +TEST(GroupOp, TestBuildDropout) { + // Step 1: Construct pir::Program + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + std::shared_ptr<::pir::Program> program = BuildDropOutProgram(); + + RunAndCheckResult(program.get(), false); +} + +std::shared_ptr<::pir::Program> BuildScaleGroupProgram() { + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + ctx->GetOrRegisterDialect(); + + auto program = std::make_shared<::pir::Program>(ctx); + ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + + // full -> softmax(max -> subtract -> exp -> sum -> divide) + const float value_one = 1.0; + const std::vector shape = {16, 16}; + auto x = builder + .Build( + shape, value_one, phi::DataType::FLOAT32, phi::GPUPlace()) + .result(0); + + auto out = + builder.Build(x, 0.5, 0.0, false).result(0); + + builder.Build(out, "out", 0); + return program; +} + +TEST(GroupOp, TestBuildScale) { + // Step 1: Construct pir::Program + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + std::shared_ptr<::pir::Program> program = BuildScaleGroupProgram(); + + RunAndCheckResult(program.get(), true, 0.5); +} + +std::shared_ptr<::pir::Program> BuildScaleTensorGroupProgram() { + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + ctx->GetOrRegisterDialect(); + + auto program = std::make_shared<::pir::Program>(ctx); + ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + + // full -> softmax(max -> subtract -> exp -> sum -> divide) + const float value_one = 0.5; + const std::vector shape = {16, 16}; + auto x = builder + .Build( + shape, value_one, phi::DataType::FLOAT32, phi::GPUPlace()) + .result(0); + auto scale = builder + .Build(std::vector({1}), + 0.0, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + auto factor = builder.Build(scale).result(0); + auto out = + builder.Build(x, factor, 0.0, false).result(0); + + builder.Build(out, "out", 0); + return program; +} + +TEST(GroupOp, TestBuildScaleTensor) { + // Step 1: Construct pir::Program + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + std::shared_ptr<::pir::Program> program = BuildScaleTensorGroupProgram(); + + RunAndCheckResult(program.get(), true, 0.5); +} + +std::shared_ptr<::pir::Program> BuildPowerProgram() { + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + ctx->GetOrRegisterDialect(); + auto program = std::make_shared<::pir::Program>(ctx); + ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + + auto x = builder + .Build(std::vector({16, 16}), + 2.0, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + + auto factor = + builder + .Build(std::vector({16, 16}), + 2.0, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + + auto power1 = + builder.Build(x, factor).result(0); + + auto power2 = builder.Build(power1, 2.0).result(0); + auto out = + builder + .Build(power2, std::vector({-1})) + .result(0); + + builder.Build(out, "out", 0); + return program; +} + +TEST(GroupOp, TestBuildPower) { + // Step 1: Construct pir::Program + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + std::shared_ptr<::pir::Program> program = BuildPowerProgram(); + + RunAndCheckResult(program.get(), true, 16.0); +} + +std::shared_ptr<::pir::Program> BuildLayerNorm2Program() { + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + ctx->GetOrRegisterDialect(); + auto program = std::make_shared<::pir::Program>(ctx); + ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + + std::vector axes{-1}; + auto x = + builder + .Build(std::vector({128, 128, 768}), + 1.0, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + + auto bias = builder + .Build(std::vector({768}), + 1.0, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + + auto scale = builder + .Build(std::vector({768}), + 1.0, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + + auto num = + builder + .Build(std::vector{128, 128, 1}, + 768.0, + phi::DataType::FLOAT32, + phi::CPUPlace()) + .result(0); + auto sum = + builder + .Build(x, axes, phi::DataType::FLOAT32, true) + .result(0); + + auto mean = builder.Build(sum, num).result(0); + + auto diff = builder.Build(x, mean).result(0); + + auto power = builder.Build(diff, diff).result(0); + auto power_sum = builder + .Build( + power, axes, phi::DataType::FLOAT32, true) + .result(0); + auto num2 = + builder + .Build(std::vector{128, 128, 1}, + 768.0, + phi::DataType::FLOAT32, + phi::CPUPlace()) + .result(0); + auto var2 = + builder.Build(power_sum, num2).result(0); + + auto t1 = builder.Build(var2, 1.0, 1e-5).result(0); + auto factor = builder + .Build(std::vector{1}, + -0.5, + phi::DataType::FLOAT32, + phi::CPUPlace()) + .result(0); + auto t2 = + builder.Build(t1, factor).result(0); + // auto t2 = builder.Build(t1).result(0); + auto t3 = builder.Build(diff, t2).result(0); + auto t5 = builder.Build(t3, scale).result(0); + auto out = builder.Build(t5, bias).result(0); + auto mean_out = + builder + .Build(mean, std::vector({-1})) + .result(0); + auto mean2_out = + builder + .Build(var2, std::vector({-1})) + .result(0); + + builder.Build(out, "out", 0); + builder.Build(mean_out, "mean", 0); + builder.Build(mean2_out, "var", 0); + return program; +} + +TEST(GroupOp, TestBuildLayerNorm2) { + // Step 1: Construct pir::Program + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + std::shared_ptr<::pir::Program> program = BuildLayerNorm2Program(); + + RunAndCheckResult(program.get(), false); +} + +std::shared_ptr<::pir::Program> BuildSum2GroupProgram() { + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + ctx->GetOrRegisterDialect(); + auto program = std::make_shared<::pir::Program>(ctx); + ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + + auto x = builder + .Build(std::vector({16, 16}), + 0.0, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + + auto cos = builder.Build(x).result(0); + + auto y = builder + .Build(std::vector({8, 8}), + 0.0, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + + auto sin = builder.Build(y).result(0); + + builder.Build(cos, "out", 0); + builder.Build(sin, "out2", 0); + return program; +} + +TEST(GroupOp, TestBuildSum2Group) { + // Step 1: Construct pir::Program + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + std::shared_ptr<::pir::Program> program = BuildSum2GroupProgram(); + + RunAndCheckResult(program.get(), true, 1.0); +} + +std::shared_ptr<::pir::Program> BuildConcatProgram() { + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + ctx->GetOrRegisterDialect(); + auto program = std::make_shared<::pir::Program>(ctx); + ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + + auto x = builder + .Build(std::vector({16, 16}), + 2.0, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + + auto y = builder + .Build(std::vector({16, 16}), + 2.0, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + + auto t1 = + builder.Build(std::vector({x, y})).result(0); + + auto out = builder.Build(t1, 1).result(0); + + builder.Build(out, "out", 0); + return program; +} + +TEST(GroupOp, TestBuildConcat) { + // Step 1: Construct pir::Program + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + std::shared_ptr<::pir::Program> program = BuildConcatProgram(); + + RunAndCheckResult(program.get(), true, 2.0); +} + +std::shared_ptr<::pir::Program> BuildSliceProgram() { + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + ctx->GetOrRegisterDialect(); + auto program = std::make_shared<::pir::Program>(ctx); + ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + + auto x = builder + .Build(std::vector({16, 16}), + 2.0, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + + auto out = builder + .Build(x, + std::vector({1}), + std::vector({0}), + std::vector({2}), + std::vector({}), + std::vector({})) + .result(0); + + builder.Build(out, "out", 0); + return program; +} + +TEST(GroupOp, TestBuildSlice) { + // Step 1: Construct pir::Program + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + std::shared_ptr<::pir::Program> program = BuildSliceProgram(); + + RunAndCheckResult(program.get(), true, 2.0); +} + +std::shared_ptr<::pir::Program> BuildSplitProgram() { + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + ctx->GetOrRegisterDialect(); + auto program = std::make_shared<::pir::Program>(ctx); + ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + + auto x = builder + .Build(std::vector({16, 16}), + 2.0, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + + auto out_arr = + builder.Build(x, 4, 1).result(0); + auto out = builder.Build(out_arr, 0).result(0); + builder.Build(out, "out", 0); + return program; +} + +TEST(GroupOp, TestBuildSplit) { + // Step 1: Construct pir::Program + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + std::shared_ptr<::pir::Program> program = BuildSplitProgram(); + + RunAndCheckResult(program.get(), true, 2.0); +} + +std::shared_ptr<::pir::Program> BuildAddNProgram() { + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + ctx->GetOrRegisterDialect(); + auto program = std::make_shared<::pir::Program>(ctx); + ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + + auto x = builder + .Build(std::vector({16, 16}), + 2.0, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + + auto y = builder + .Build(std::vector({16, 16}), + 2.0, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + + auto z = builder + .Build(std::vector({16, 16}), + 2.0, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + + auto t1 = builder.Build(std::vector({x, y, z})) + .result(0); + + auto out = builder.Build(t1).result(0); + + builder.Build(out, "out", 0); + return program; +} + +TEST(GroupOp, TestBuildAddN) { + // Step 1: Construct pir::Program + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + std::shared_ptr<::pir::Program> program = BuildAddNProgram(); + + RunAndCheckResult(program.get(), true, 6.0); +} + +std::shared_ptr<::pir::Program> BuildSplitSectionProgram() { + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + ctx->GetOrRegisterDialect(); + auto program = std::make_shared<::pir::Program>(ctx); + ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + + auto x = builder + .Build(std::vector({16, 16}), + 2.0, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + + auto split_arr = builder + .Build( + x, std::vector({3, 5, 8}), -1) + .out(); + auto out = builder.Build(split_arr, 0).result(0); + builder.Build(out, "out", 0); + return program; +} + +TEST(GroupOp, TestBuildSplitSection) { + // Step 1: Construct pir::Program + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + std::shared_ptr<::pir::Program> program = BuildSplitSectionProgram(); + + RunAndCheckResult(program.get(), 2.0); +} diff --git a/test/cpp/pir/cinn/pir_compiler_test.cc b/test/cpp/pir/cinn/pir_compiler_test.cc index f32f49829def1..39408da3289c6 100644 --- a/test/cpp/pir/cinn/pir_compiler_test.cc +++ b/test/cpp/pir/cinn/pir_compiler_test.cc @@ -141,109 +141,110 @@ ProgramInfo BuildSoftmax() { return {program, groups}; } -TEST(PirCompier, CompileSoftmax) { - // Step 1: Construct pir::Program - ::pir::IrContext* ctx = ::pir::IrContext::Instance(); - ctx->GetOrRegisterDialect(); - ctx->GetOrRegisterDialect(); - ctx->GetOrRegisterDialect(); - ctx->GetOrRegisterDialect(); - auto new_program = std::make_shared<::pir::Program>(ctx); - - auto prog_info = BuildSoftmax(); - std::shared_ptr<::pir::Program> program = std::get<0>(prog_info); - std::vector groups = std::get<1>(prog_info); - EXPECT_EQ(program->block()->size(), 9u); - LOG(INFO) << program->block()->size(); - - std::stringstream ss; - program->Print(ss); - LOG(INFO) << ss.str(); - - // Step 2: Compiler New pir::Program into Runtime Program - auto target = cinn::common::DefaultNVGPUTarget(); - auto scope = cinn::hlir::framework::BuildScope(target, *program); - LOG(INFO) << scope->var_names().size(); - ASSERT_EQ(scope->var_names().size(), 8); - - cinn::hlir::framework::PirCompiler ir_compiler(*program, target, scope); - auto fn_ptr_res = ir_compiler.BuildCUDAJITInfo(groups); - - ::pir::Builder builder = ::pir::Builder(ctx, new_program->block()); - auto x = builder - .Build(std::vector({16, 16}), - 1.0, - phi::DataType::FLOAT32, - phi::GPUPlace(0)) - .result(0); - - std::unordered_map op_attrs{ - {cinn::dialect::JitKernelOp::kAttrName, - cinn::dialect::CINNKernelInfoAttribute::get(ctx, fn_ptr_res[0])}, - }; - - std::vector vec_types; - - vec_types.push_back(groups[0]->ops.back()->result(0).type()); - - std::string jit_op_name = cinn::dialect::JitKernelOp::name(); - ::pir::OpInfo op_info = ctx->GetRegisteredOpInfo(jit_op_name); - ::pir::Operation* cinn_op = - ::pir::Operation::Create({x}, op_attrs, vec_types, op_info); - - new_program->block()->push_back(cinn_op); - - builder.SetInsertionPointToBlockEnd(new_program->block()); - builder.Build( - cinn_op->result(cinn_op->num_results() - 1), "out", 0); - - paddle::platform::Place place = paddle::platform::CUDAPlace(0); - - auto kernel_program = - paddle::dialect::PdOpLowerToKernelPass(new_program.get(), place); - - paddle::framework::Scope exe_scope; - - paddle::framework::interpreter::ExecutionConfig exe_conf; - exe_conf.create_local_scope = false; - paddle::framework::InterpreterCore executor( - place, {"out@fetch"}, kernel_program->block(), &exe_scope); - - executor.Run({}, true); - auto out_tensor = - executor.local_scope()->FindVar("out@fetch")->Get(); - bool res0 = simple_cmp(out_tensor.data()[0], 1.0 / 16); - EXPECT_EQ(res0, true); -} - -TEST(PirCompier, CompileGroupOps) { - // Step 1: Construct pir::Program - auto prog_info = BuildProgram(); - std::shared_ptr<::pir::Program> program = std::get<0>(prog_info); - std::vector groups = std::get<1>(prog_info); - EXPECT_EQ(program->block()->size(), 9u); - LOG(INFO) << program->block()->size(); - - std::stringstream ss; - program->Print(ss); - LOG(INFO) << ss.str(); - - // Step 2: Compiler New pir::Program into Runtime Program - auto target = cinn::common::DefaultNVGPUTarget(); - auto scope = cinn::hlir::framework::BuildScope(target, *program); - ASSERT_EQ(scope->var_names().size(), 6); - - cinn::hlir::framework::PirCompiler ir_compiler(*program, target, scope); - auto runtime_program = ir_compiler.Build(groups); - - // Step 3: Execute Runtime Instruction and check Scope. - ASSERT_NO_THROW(runtime_program->Execute()); - for (auto& var_name : scope->var_names()) { - std::string name = {var_name.begin(), var_name.end()}; - std::vector data = - cinn::GetTensorData(scope->GetTensor(name), target); - for (int i = 0; i < 1; ++i) { - LOG_FIRST_N(INFO, 10) << "data: " << data[i]; - } - } -} +// TEST(PirCompier, CompileSoftmax) { +// // Step 1: Construct pir::Program +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// ctx->GetOrRegisterDialect(); +// ctx->GetOrRegisterDialect(); +// ctx->GetOrRegisterDialect(); +// ctx->GetOrRegisterDialect(); +// auto new_program = std::make_shared<::pir::Program>(ctx); + +// auto prog_info = BuildSoftmax(); +// std::shared_ptr<::pir::Program> program = std::get<0>(prog_info); +// std::vector groups = std::get<1>(prog_info); +// EXPECT_EQ(program->block()->size(), 9u); +// LOG(INFO) << program->block()->size(); + +// std::stringstream ss; +// program->Print(ss); +// LOG(INFO) << ss.str(); + +// // Step 2: Compiler New pir::Program into Runtime Program +// auto target = cinn::common::DefaultNVGPUTarget(); +// auto scope = cinn::hlir::framework::BuildScope(target, *program); +// LOG(INFO) << scope->var_names().size(); +// ASSERT_EQ(scope->var_names().size(), 8); + +// cinn::hlir::framework::PirCompiler ir_compiler(*program, target, scope); +// auto fn_ptr_res = ir_compiler.BuildCUDAJITInfo(groups); + +// ::pir::Builder builder = ::pir::Builder(ctx, new_program->block()); +// auto x = builder +// .Build(std::vector({16, +// 16}), +// 1.0, +// phi::DataType::FLOAT32, +// phi::GPUPlace(0)) +// .result(0); + +// std::unordered_map op_attrs{ +// {cinn::dialect::JitKernelOp::kAttrName, +// cinn::dialect::CINNKernelInfoAttribute::get(ctx, fn_ptr_res[0])}, +// }; + +// std::vector vec_types; + +// vec_types.push_back(groups[0]->ops.back()->result(0).type()); + +// std::string jit_op_name = cinn::dialect::JitKernelOp::name(); +// ::pir::OpInfo op_info = ctx->GetRegisteredOpInfo(jit_op_name); +// ::pir::Operation* cinn_op = +// ::pir::Operation::Create({x}, op_attrs, vec_types, op_info); + +// new_program->block()->push_back(cinn_op); + +// builder.SetInsertionPointToBlockEnd(new_program->block()); +// builder.Build( +// cinn_op->result(cinn_op->num_results() - 1), "out", 0); + +// paddle::platform::Place place = paddle::platform::CUDAPlace(0); + +// auto kernel_program = +// paddle::dialect::PdOpLowerToKernelPass(new_program.get(), place); + +// paddle::framework::Scope exe_scope; + +// paddle::framework::interpreter::ExecutionConfig exe_conf; +// exe_conf.create_local_scope = false; +// paddle::framework::InterpreterCore executor( +// place, {"out@fetch"}, kernel_program->block(), &exe_scope); + +// executor.Run({}, true); +// auto out_tensor = +// executor.local_scope()->FindVar("out@fetch")->Get(); +// bool res0 = simple_cmp(out_tensor.data()[0], 1.0 / 16); +// EXPECT_EQ(res0, true); +// } + +// TEST(PirCompier, CompileGroupOps) { +// // Step 1: Construct pir::Program +// auto prog_info = BuildProgram(); +// std::shared_ptr<::pir::Program> program = std::get<0>(prog_info); +// std::vector groups = std::get<1>(prog_info); +// EXPECT_EQ(program->block()->size(), 9u); +// LOG(INFO) << program->block()->size(); + +// std::stringstream ss; +// program->Print(ss); +// LOG(INFO) << ss.str(); + +// // Step 2: Compiler New pir::Program into Runtime Program +// auto target = cinn::common::DefaultNVGPUTarget(); +// auto scope = cinn::hlir::framework::BuildScope(target, *program); +// ASSERT_EQ(scope->var_names().size(), 6); + +// cinn::hlir::framework::PirCompiler ir_compiler(*program, target, scope); +// auto runtime_program = ir_compiler.Build(groups); + +// // Step 3: Execute Runtime Instruction and check Scope. +// ASSERT_NO_THROW(runtime_program->Execute()); +// for (auto& var_name : scope->var_names()) { +// std::string name = {var_name.begin(), var_name.end()}; +// std::vector data = +// cinn::GetTensorData(scope->GetTensor(name), target); +// for (int i = 0; i < 1; ++i) { +// LOG_FIRST_N(INFO, 10) << "data: " << data[i]; +// } +// } +// } diff --git a/test/ir/pir/cinn/CMakeLists.txt b/test/ir/pir/cinn/CMakeLists.txt index 7a7d98dc37ba3..800a132f6d124 100644 --- a/test/ir/pir/cinn/CMakeLists.txt +++ b/test/ir/pir/cinn/CMakeLists.txt @@ -36,17 +36,17 @@ if(WITH_GPU) WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_tests_properties(test_subgraph_checker PROPERTIES LABELS "RUN_TYPE=CINN") - add_test( - NAME test_rms_norm_seq_len_symbolic - COMMAND - ${CMAKE_COMMAND} -E env - PYTHONPATH=${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/python/:$ENV{PYTHONPATH} - FLAGS_cinn_convert_static_dim_to_dynamic_dim=2048:S0 - FLAGS_cinn_bucket_compile=True FLAGS_enable_pir_api=1 ${PYTHON_EXECUTABLE} - ${CMAKE_CURRENT_SOURCE_DIR}/test_rms_norm.py - WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) - set_tests_properties(test_rms_norm_seq_len_symbolic - PROPERTIES LABELS "RUN_TYPE=CINN") + # add_test( + # NAME test_rms_norm_seq_len_symbolic + # COMMAND + # ${CMAKE_COMMAND} -E env + # PYTHONPATH=${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/python/:$ENV{PYTHONPATH} + # FLAGS_cinn_convert_static_dim_to_dynamic_dim=2048:S0 + # FLAGS_cinn_bucket_compile=True FLAGS_enable_pir_api=1 ${PYTHON_EXECUTABLE} + # ${CMAKE_CURRENT_SOURCE_DIR}/test_rms_norm.py + # WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + # set_tests_properties(test_rms_norm_seq_len_symbolic + # PROPERTIES LABELS "RUN_TYPE=CINN") add_test( NAME test_rms_norm_bs_symbolic COMMAND @@ -58,17 +58,17 @@ if(WITH_GPU) WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_tests_properties(test_rms_norm_bs_symbolic PROPERTIES LABELS "RUN_TYPE=CINN") - add_test( - NAME test_rms_norm_reduce_symbolic - COMMAND - ${CMAKE_COMMAND} -E env - PYTHONPATH=${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/python/:$ENV{PYTHONPATH} - FLAGS_cinn_convert_static_dim_to_dynamic_dim=768:S0 - FLAGS_cinn_bucket_compile=True FLAGS_enable_pir_api=1 ${PYTHON_EXECUTABLE} - ${CMAKE_CURRENT_SOURCE_DIR}/test_rms_norm.py - WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) - set_tests_properties(test_rms_norm_reduce_symbolic PROPERTIES LABELS - "RUN_TYPE=CINN") + # add_test( + # NAME test_rms_norm_reduce_symbolic + # COMMAND + # ${CMAKE_COMMAND} -E env + # PYTHONPATH=${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/python/:$ENV{PYTHONPATH} + # FLAGS_cinn_convert_static_dim_to_dynamic_dim=768:S0 + # FLAGS_cinn_bucket_compile=True FLAGS_enable_pir_api=1 ${PYTHON_EXECUTABLE} + # ${CMAKE_CURRENT_SOURCE_DIR}/test_rms_norm.py + # WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + # set_tests_properties(test_rms_norm_reduce_symbolic PROPERTIES LABELS + # "RUN_TYPE=CINN") add_test( NAME test_rms_norm_symbolic COMMAND @@ -79,17 +79,17 @@ if(WITH_GPU) ${CMAKE_CURRENT_SOURCE_DIR}/test_rms_norm.py WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_tests_properties(test_rms_norm_symbolic PROPERTIES LABELS "RUN_TYPE=CINN") - add_test( - NAME test_rope_seq_len_symbolic - COMMAND - ${CMAKE_COMMAND} -E env - PYTHONPATH=${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/python/:$ENV{PYTHONPATH} - FLAGS_cinn_convert_static_dim_to_dynamic_dim=2048:S1 - FLAGS_cinn_bucket_compile=True FLAGS_enable_pir_api=1 FLAGS_prim_all=True - ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_rope.py - WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) - set_tests_properties(test_rope_seq_len_symbolic PROPERTIES LABELS - "RUN_TYPE=CINN") + # add_test( + # NAME test_rope_seq_len_symbolic + # COMMAND + # ${CMAKE_COMMAND} -E env + # PYTHONPATH=${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/python/:$ENV{PYTHONPATH} + # FLAGS_cinn_convert_static_dim_to_dynamic_dim=2048:S1 + # FLAGS_cinn_bucket_compile=True FLAGS_enable_pir_api=1 FLAGS_prim_all=True + # ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_rope.py + # WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + # set_tests_properties(test_rope_seq_len_symbolic PROPERTIES LABELS + # "RUN_TYPE=CINN") add_test( NAME test_rope_bs_symbolic @@ -102,15 +102,15 @@ if(WITH_GPU) WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_tests_properties(test_rope_bs_symbolic PROPERTIES LABELS "RUN_TYPE=CINN") - add_test( - NAME test_rope_symbolic - COMMAND - ${CMAKE_COMMAND} -E env - PYTHONPATH=${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/python/:$ENV{PYTHONPATH} - FLAGS_cinn_convert_static_dim_to_dynamic_dim=61:S0,2048:S1 - FLAGS_cinn_bucket_compile=True FLAGS_enable_pir_api=1 FLAGS_prim_all=True - ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_rope.py - WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) - set_tests_properties(test_rope_symbolic PROPERTIES LABELS "RUN_TYPE=CINN") + # add_test( + # NAME test_rope_symbolic + # COMMAND + # ${CMAKE_COMMAND} -E env + # PYTHONPATH=${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/python/:$ENV{PYTHONPATH} + # FLAGS_cinn_convert_static_dim_to_dynamic_dim=61:S0,2048:S1 + # FLAGS_cinn_bucket_compile=True FLAGS_enable_pir_api=1 FLAGS_prim_all=True + # ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_rope.py + # WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + # set_tests_properties(test_rope_symbolic PROPERTIES LABELS "RUN_TYPE=CINN") endif() diff --git a/test/ir/pir/cinn/sub_graphs/CMakeLists.txt b/test/ir/pir/cinn/sub_graphs/CMakeLists.txt index 2d166a44846f5..c6c6d6be14860 100644 --- a/test/ir/pir/cinn/sub_graphs/CMakeLists.txt +++ b/test/ir/pir/cinn/sub_graphs/CMakeLists.txt @@ -13,6 +13,7 @@ if(WITH_GPU) ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/python/:$ENV{PYTHONPATH} FLAGS_cinn_new_group_scheduler=1 FLAGS_enable_pir_api=1 + FLAGS_cinn_bucket_compile=1 FLAGS_group_schedule_tiling_first=1 FLAGS_cudnn_deterministic=true ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/${cinn_sub_graph_test_name}.py WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_0.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_0.py index 12a88cc235985..2cc7e568122cf 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_0.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_0.py @@ -108,5 +108,5 @@ def test_ast_prim_cinn(self): np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-6) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_19.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_19.py index c99906880760d..64e6123642cc9 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_19.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_19.py @@ -99,5 +99,5 @@ def test_ast_prim_cinn(self): # np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_32.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_32.py index faca863f03633..11671c42fdf3a 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_32.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_32.py @@ -74,5 +74,5 @@ def test_ast_prim_cinn(self): np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_33.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_33.py index eff3e66cf20cf..6481d07a6ab8f 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_33.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_33.py @@ -98,5 +98,5 @@ def test_ast_prim_cinn(self): np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-5) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_37.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_37.py index c6f1d6d5eff03..597a6f2882ab5 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_37.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_37.py @@ -81,5 +81,5 @@ def test_ast_prim_cinn(self): np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-6) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_5.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_5.py index d4d1e72e104db..8859b550d286e 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_5.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_5.py @@ -67,5 +67,5 @@ def test_ast_prim_cinn(self): np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-6) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_50.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_50.py index c83b2b14f5e46..9b9dc07b34043 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_50.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_50.py @@ -92,5 +92,5 @@ def test_ast_prim_cinn(self): np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_53.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_53.py index 91bc95ebf457b..be02c053e5528 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_53.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_53.py @@ -97,5 +97,5 @@ def test_ast_prim_cinn(self): np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-6) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_58.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_58.py index 17efb1621e403..94944a22f7037 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_58.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_58.py @@ -89,5 +89,5 @@ def test_ast_prim_cinn(self): np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_60.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_60.py index c9fd19a3455c6..94fce7eddc3cb 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_60.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_60.py @@ -121,5 +121,5 @@ def test_ast_prim_cinn(self): np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_68.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_68.py index 3ffa508fc23f5..a0dff3b1bfa6e 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_68.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_68.py @@ -128,5 +128,5 @@ def test_ast_prim_cinn(self): np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_70.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_70.py index eeeca452b5e97..9d7c757cafa42 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_70.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_70.py @@ -81,5 +81,5 @@ def test_ast_prim_cinn(self): np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_71.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_71.py index 5fac613db9ade..cefb00c72e0f5 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_71.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_71.py @@ -256,5 +256,5 @@ def test_ast_prim_cinn(self): np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-5) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_75.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_75.py index 965fa6021a673..ea6e9e8c2ea05 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_75.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_75.py @@ -117,5 +117,5 @@ def test_ast_prim_cinn(self): np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-6) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_76.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_76.py index 211111ae65066..7c65bac390881 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_76.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_76.py @@ -136,5 +136,5 @@ def test_ast_prim_cinn(self): np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_79.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_79.py index 69b7847f2a096..971bca1d02fb7 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_79.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_79.py @@ -107,5 +107,5 @@ def test_ast_prim_cinn(self): np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-6) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_88.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_88.py index 32a9ece2de252..dace08b921f7c 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_88.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_88.py @@ -88,5 +88,5 @@ def test_ast_prim_cinn(self): np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-6) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_89.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_89.py index 77049437185d8..ae67c4a382cbf 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_89.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_89.py @@ -112,5 +112,5 @@ def test_ast_prim_cinn(self): np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-6) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_mul_method.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_mul_method.py index d2e5f900b20f3..10fe8bd9e9b81 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_mul_method.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_mul_method.py @@ -69,5 +69,5 @@ def test_ast_prim_cinn(self): np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/sub_graphs/test_sub_graph_relu6.py b/test/ir/pir/cinn/sub_graphs/test_sub_graph_relu6.py index dc98d466ccd56..7470c35706901 100644 --- a/test/ir/pir/cinn/sub_graphs/test_sub_graph_relu6.py +++ b/test/ir/pir/cinn/sub_graphs/test_sub_graph_relu6.py @@ -67,5 +67,5 @@ def test_ast_prim_cinn(self): np.testing.assert_allclose(st.numpy(), cinn.numpy(), atol=1e-8) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/symbolic/CMakeLists.txt b/test/ir/pir/cinn/symbolic/CMakeLists.txt index d227d7cc8af3a..3349cddf6c34d 100644 --- a/test/ir/pir/cinn/symbolic/CMakeLists.txt +++ b/test/ir/pir/cinn/symbolic/CMakeLists.txt @@ -32,7 +32,9 @@ if(WITH_GPU) ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/python/:$ENV{PYTHONPATH} FLAGS_enable_pir_api=1 FLAGS_cinn_bucket_compile=True - FLAGS_pir_apply_shape_optimization_pass=1 ${PYTHON_EXECUTABLE} + FLAGS_pir_apply_shape_optimization_pass=1 + FLAGS_group_schedule_tiling_first=1 FLAGS_cinn_new_group_scheduler=1 + ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/${cinn_pir_test_name}.py WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_tests_properties(${cinn_pir_test_name} PROPERTIES LABELS @@ -198,7 +200,8 @@ if(WITH_GPU) COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/python/:$ENV{PYTHONPATH} - FLAGS_prim_all=true FLAGS_cinn_bucket_compile=True FLAGS_enable_pir_api=1 + FLAGS_prim_all=true FLAGS_cinn_bucket_compile=True + FLAGS_group_schedule_tiling_first=1 FLAGS_enable_pir_api=1 ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_llama_mlp_dy.py WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_tests_properties(test_llama_mlp_dy PROPERTIES LABELS "RUN_TYPE=CINN") diff --git a/test/ir/pir/cinn/symbolic/test_check_infer_symbolic.py b/test/ir/pir/cinn/symbolic/test_check_infer_symbolic.py index ae1c6854126d6..645a8d753fbc5 100644 --- a/test/ir/pir/cinn/symbolic/test_check_infer_symbolic.py +++ b/test/ir/pir/cinn/symbolic/test_check_infer_symbolic.py @@ -74,5 +74,5 @@ def test_eval(self): np.testing.assert_allclose(cinn_out.numpy(), dy_out.numpy(), atol=1e-8) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/symbolic/test_cinn_sub_graph_symbolic.py b/test/ir/pir/cinn/symbolic/test_cinn_sub_graph_symbolic.py index b5efe5685e29a..8c9bc49bf6e4e 100644 --- a/test/ir/pir/cinn/symbolic/test_cinn_sub_graph_symbolic.py +++ b/test/ir/pir/cinn/symbolic/test_cinn_sub_graph_symbolic.py @@ -333,5 +333,5 @@ def test_eval_symbolic(self): # np.testing.assert_allclose(cinn_out.numpy(), dy_out.numpy(), atol=1e-8) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/symbolic/test_dyshape_rms_norm.py b/test/ir/pir/cinn/symbolic/test_dyshape_rms_norm.py index 991aab4af9fec..ba94a53866b4d 100644 --- a/test/ir/pir/cinn/symbolic/test_dyshape_rms_norm.py +++ b/test/ir/pir/cinn/symbolic/test_dyshape_rms_norm.py @@ -39,7 +39,7 @@ def __init__(self): self.variance_epsilon = 1e-6 def forward(self, hidden_states): - variance = hidden_states.pow(2).sum(-1, keepdim=True) / 768 + variance = (hidden_states * hidden_states).sum(-1, keepdim=True) / 768 hidden_states = ( paddle.rsqrt(variance + self.variance_epsilon) * hidden_states ) @@ -80,5 +80,5 @@ def test_eval(self): ) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/symbolic/test_dyshape_rope.py b/test/ir/pir/cinn/symbolic/test_dyshape_rope.py index ee11bc73876b1..7e608eb11ab46 100644 --- a/test/ir/pir/cinn/symbolic/test_dyshape_rope.py +++ b/test/ir/pir/cinn/symbolic/test_dyshape_rope.py @@ -131,5 +131,5 @@ def test_eval(self): ) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/symbolic/test_if_dy.py b/test/ir/pir/cinn/symbolic/test_if_dy.py index fc77fdbba5d7e..2a2ff32d1570b 100644 --- a/test/ir/pir/cinn/symbolic/test_if_dy.py +++ b/test/ir/pir/cinn/symbolic/test_if_dy.py @@ -83,5 +83,5 @@ def test_eval(self): ) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/symbolic/test_llama_mlp_dy.py b/test/ir/pir/cinn/symbolic/test_llama_mlp_dy.py index 96cbbd8076702..1b3af40308270 100644 --- a/test/ir/pir/cinn/symbolic/test_llama_mlp_dy.py +++ b/test/ir/pir/cinn/symbolic/test_llama_mlp_dy.py @@ -88,5 +88,5 @@ def test_eval(self): ) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/symbolic/test_multiple_subgraph_dy.py b/test/ir/pir/cinn/symbolic/test_multiple_subgraph_dy.py index 6ebcad30f5623..b8dcee9e00605 100644 --- a/test/ir/pir/cinn/symbolic/test_multiple_subgraph_dy.py +++ b/test/ir/pir/cinn/symbolic/test_multiple_subgraph_dy.py @@ -81,5 +81,5 @@ def test_eval(self): ) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/symbolic/test_sub_graph_for_frontend.py b/test/ir/pir/cinn/symbolic/test_sub_graph_for_frontend.py index a25b6a4d1d275..34dfc4b004519 100644 --- a/test/ir/pir/cinn/symbolic/test_sub_graph_for_frontend.py +++ b/test/ir/pir/cinn/symbolic/test_sub_graph_for_frontend.py @@ -80,5 +80,5 @@ def test_eval(self): np.testing.assert_allclose(cinn_out.numpy(), dy_out.numpy(), atol=1e-8) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/test_cinn_sub_graph.py b/test/ir/pir/cinn/test_cinn_sub_graph.py index 7198f87ba5d80..a2fa6aca4ca88 100644 --- a/test/ir/pir/cinn/test_cinn_sub_graph.py +++ b/test/ir/pir/cinn/test_cinn_sub_graph.py @@ -77,14 +77,12 @@ def __init__(self, hidden_size): super().__init__() self.fn = layer_norm self.weight = self.create_parameter( - shape=[hidden_size], dtype="float32" + shape=[hidden_size], dtype="float64" ) - self.bias = self.create_parameter(shape=[hidden_size], dtype="float32") + self.bias = self.create_parameter(shape=[hidden_size], dtype="float64") def forward(self, x, weight, bias): - out = paddle.nn.functional.layer_norm( - x, x.shape[-1], self.weight, self.bias - ) + out = paddle.nn.functional.layer_norm(x, x.shape[-1], weight, bias) return out @@ -93,17 +91,23 @@ def __init__(self, hidden_size): super().__init__() self.add = paddle.add self.dropout = dropout - self.layer_norm = layer_norm + self.layer_norm = paddle.nn.functional.layer_norm self.weight = self.create_parameter( - shape=[hidden_size], dtype="float32" + shape=[hidden_size], dtype="float64" ) - self.bias = self.create_parameter(shape=[hidden_size], dtype="float32") + self.bias = self.create_parameter(shape=[hidden_size], dtype="float64") def forward(self, x, y, weight, bias): t1 = self.add(x, y) t2 = self.dropout(t1) - out = self.layer_norm(t2, self.weight, self.bias) + t2 = x + out = self.layer_norm(t2, t2.shape[-1], self.weight, self.bias) + return out + + out = paddle.nn.functional.layer_norm( + x, x.shape[-1], self.weight, self.bias + ) return out @@ -127,9 +131,9 @@ def setUp(self): self.prepare_data() def prepare_data(self): - self.shape = [64, 128] + self.shape = [128, 128, 768] self.axis = -1 - self.x = paddle.randn(self.shape, dtype="float32") + self.x = paddle.uniform(self.shape, dtype="float64", min=-0.5, max=0.5) self.x.stop_gradient = False def check_jit_kernel_info(self, static_fn): @@ -154,121 +158,178 @@ def test_eval(self): np.testing.assert_allclose(cinn_out.numpy(), dy_out.numpy(), atol=1e-8) -class TestCinnSoftmax(TestCinnSubGraphBase): - def train(self, use_cinn): - paddle.seed(2022) - net = CINNSoftmaxSubGraphNet() - net = utils.apply_to_static(net, use_cinn) - out = net(self.x, self.axis) - loss = out.mean() - loss.backward() - if use_cinn: - self.check_jit_kernel_info(net.forward) - return out +# class TestCinnSoftmax(TestCinnSubGraphBase): +# def train(self, use_cinn): +# paddle.seed(2022) +# net = CINNSoftmaxSubGraphNet() +# net = utils.apply_to_static(net, use_cinn) +# out = net(self.x, self.axis) - def test_train(self): - cinn_out = self.train(use_cinn=True) - dy_out = self.train(use_cinn=False) - np.testing.assert_allclose(cinn_out.numpy(), dy_out.numpy(), atol=1e-8) +# loss = out.sum() +# loss.backward() +# print(self.x.gradient()) +# return out, self.x.gradient() + +# def test_forward(self): +# cinn_out, cinn_grad = self.train(use_cinn=True) +# dy_out, dy_grad = self.train(use_cinn=False) +# np.testing.assert_allclose(cinn_out.numpy(), dy_out.numpy(), atol=1e-8) +# np.testing.assert_allclose(cinn_grad, dy_grad, atol=1e-8) class TestCinnLayerNorm(TestCinnSubGraphBase): - def eval(self, use_cinn): + def train(self, use_cinn): paddle.seed(2022) + self.prepare_data() net = CINNLayerNormSubGraphNet(self.shape[-1]) net = utils.apply_to_static(net, use_cinn) - net.eval() - weight = paddle.ones(shape=[self.shape[-1]], dtype="float32") - bias = paddle.ones(shape=[self.shape[-1]], dtype="float32") + # net.eval() + weight = paddle.ones(shape=[self.shape[-1]], dtype="float64") + weight.stop_gradient = False + bias = paddle.ones(shape=[self.shape[-1]], dtype="float64") + bias.stop_gradient = False + self.x.stop_gradient = False out = net(self.x, weight, bias) - return out + loss = out.sum() + loss.backward() - def test_eval(self): - cinn_out = self.eval(use_cinn=True) - dy_out = self.eval(use_cinn=False) - # TODO(Aurelius84): Apply assert_allclose logic, - # but need figure out why atol only satisfy 1e-7 - np.testing.assert_allclose(cinn_out.numpy(), dy_out.numpy(), atol=1e-7) + return out, self.x.gradient(), weight.gradient(), bias.gradient() + + def test_train(self): + cinn_out, cinn_x_grad, cinn_w_grad, cinn_b_grad = self.train( + use_cinn=True + ) + + dy_out, dy_x_grad, dy_w_grad, dy_b_grad = self.train(use_cinn=False) + np.testing.assert_allclose(cinn_out.numpy(), dy_out.numpy(), atol=1e-8) + np.testing.assert_allclose(cinn_x_grad, dy_x_grad, atol=1e-8) + np.testing.assert_allclose(cinn_w_grad, dy_w_grad, atol=1e-8) + np.testing.assert_allclose(cinn_b_grad, dy_b_grad, atol=1e-8) class TestAddDropoutLayerNorm(TestCinnSubGraphBase): - def eval(self, use_cinn): + def train(self, use_cinn): paddle.seed(2022) net = CINNAddDropoutLayerNormSubGraphNet(self.shape[-1]) net = utils.apply_to_static(net, use_cinn) - net.eval() + # net.eval() weight = paddle.ones(shape=[self.shape[-1]], dtype="float32") bias = paddle.ones(shape=[self.shape[-1]], dtype="float32") out = net(self.x, self.x, weight, bias) - if use_cinn: - self.check_jit_kernel_info(net.forward) - return out - - def test_eval(self): - cinn_out = self.eval(use_cinn=True) - dy_out = self.eval(use_cinn=False) - - np.testing.assert_allclose( - cinn_out.numpy(), dy_out.numpy(), atol=1e-8, rtol=1e-4 - ) - - -class TestCinnDropout(TestCinnSubGraphBase): - def train(self, use_cinn): - paddle.seed(2022) - net = CINNDropoutSubGraphNet() - net = utils.apply_to_static(net, use_cinn) - out = net(self.x) - - loss = out.mean() - loss.backward() - if use_cinn: - self.check_jit_kernel_info(net.forward) return out - def test_train(self): + def test_forward(self): cinn_out = self.train(use_cinn=True) dy_out = self.train(use_cinn=False) - np.testing.assert_allclose(cinn_out.numpy(), dy_out.numpy(), atol=1e-8) - - -class TestCinnEvalPrim(TestCinnSubGraphBase): - def prepare_data(self): - self.shape = [1, 2048, 768] - self.hidden_states = paddle.randn(self.shape, dtype="float32") - self.hidden_states.stop_gradient = False - - def eval(self, use_cinn): - paddle.seed(2022) - net = CINNSoftmaxSubGraphNet() - net = utils.apply_to_static(net, use_cinn) - net.eval() - out = net(self.hidden_states) - - if use_cinn: - ops = [ - op.name() - for op in net.forward.program_cache.last()[-1][-1] - .train_program.program.global_block() - .ops - ] - assert ( - "pd_op.softmax" not in ops - ), f"after prim, pd_op.softmax should not exist, but got {ops}" - assert ( - "pd_op.exp" in ops - ), f"after prim, pd_op.softmax should not exist, but got {ops}" - self.check_jit_kernel_info(net.forward) - - return out - def test_eval(self): - cinn_out = self.eval(use_cinn=True) - dy_out = self.eval(use_cinn=False) np.testing.assert_allclose( - cinn_out.numpy(), dy_out.numpy(), atol=1e-6, rtol=1e-6 + cinn_out.numpy(), dy_out.numpy(), atol=1e-8, rtol=1e-4 ) -if __name__ == '__main__': - unittest.main() +# class TestCinnDropout(TestCinnSubGraphBase): +# def train(self, use_cinn): +# paddle.seed(2022) +# net = CINNDropoutSubGraphNet() +# net = utils.apply_to_static(net, use_cinn) +# out = net(self.x) +# class TestCinnLayerNorm(TestCinnSubGraphBase): +# def eval(self, use_cinn): +# paddle.seed(2022) +# net = CINNLayerNormSubGraphNet(self.shape[-1]) +# net = utils.apply_to_static(net, use_cinn) +# net.eval() +# weight = paddle.ones(shape=[self.shape[-1]], dtype="float32") +# bias = paddle.ones(shape=[self.shape[-1]], dtype="float32") +# out = net(self.x, weight, bias) +# return out + +# def test_eval(self): +# cinn_out = self.eval(use_cinn=True) +# dy_out = self.eval(use_cinn=False) +# # TODO(Aurelius84): Apply assert_allclose logic, +# # but need figure out why atol only satisfy 1e-7 +# np.testing.assert_allclose(cinn_out.numpy(), dy_out.numpy(), atol=1e-7) + + +# class TestAddDropoutLayerNorm(TestCinnSubGraphBase): +# def eval(self, use_cinn): +# paddle.seed(2022) +# net = CINNAddDropoutLayerNormSubGraphNet(self.shape[-1]) +# net = utils.apply_to_static(net, use_cinn) +# net.eval() +# weight = paddle.ones(shape=[self.shape[-1]], dtype="float32") +# bias = paddle.ones(shape=[self.shape[-1]], dtype="float32") +# out = net(self.x, self.x, weight, bias) +# if use_cinn: +# self.check_jit_kernel_info(net.forward) +# return out + +# def test_eval(self): +# cinn_out = self.eval(use_cinn=True) +# dy_out = self.eval(use_cinn=False) + +# np.testing.assert_allclose( +# cinn_out.numpy(), dy_out.numpy(), atol=1e-8, rtol=1e-4 +# ) + + +# class TestCinnDropout(TestCinnSubGraphBase): +# def train(self, use_cinn): +# paddle.seed(2022) +# net = CINNDropoutSubGraphNet() +# net = utils.apply_to_static(net, use_cinn) +# out = net(self.x) + +# loss = out.mean() +# loss.backward() +# if use_cinn: +# self.check_jit_kernel_info(net.forward) +# return out + +# def test_forward(self): +# cinn_out = self.train(use_cinn=True) +# dy_out = self.train(use_cinn=False) +# np.testing.assert_allclose(cinn_out.numpy(), dy_out.numpy(), atol=1e-8) + + +# class TestCinnEvalPrim(TestCinnSubGraphBase): +# def prepare_data(self): +# self.shape = [1, 2048, 768] +# self.hidden_states = paddle.randn(self.shape, dtype="float32") +# self.hidden_states.stop_gradient = False + +# def eval(self, use_cinn): +# paddle.seed(2022) +# net = CINNSoftmaxSubGraphNet() +# net = utils.apply_to_static(net, use_cinn) +# net.eval() +# out = net(self.hidden_states) + +# if use_cinn: +# ops = [ +# op.name() +# for op in net.forward.program_cache.last()[-1][-1] +# .train_program.program.global_block() +# .ops +# ] +# assert ( +# "pd_op.softmax" not in ops +# ), f"after prim, pd_op.softmax should not exist, but got {ops}" +# assert ( +# "pd_op.exp" in ops +# ), f"after prim, pd_op.softmax should not exist, but got {ops}" +# self.check_jit_kernel_info(net.forward) + +# return out + +# def test_eval(self): +# cinn_out = self.eval(use_cinn=True) +# dy_out = self.eval(use_cinn=False) +# np.testing.assert_allclose( +# cinn_out.numpy(), dy_out.numpy(), atol=1e-6, rtol=1e-6 +# ) + + +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/test_llama_sub_graph.py b/test/ir/pir/cinn/test_llama_sub_graph.py index 367b3e788a506..7fbb45ab16af3 100644 --- a/test/ir/pir/cinn/test_llama_sub_graph.py +++ b/test/ir/pir/cinn/test_llama_sub_graph.py @@ -27,7 +27,7 @@ def __init__(self): self.hidden_size = 768 self.weight = paddle.create_parameter( shape=[self.hidden_size], - dtype=paddle.get_default_dtype(), + dtype="float32", default_initializer=nn.initializer.Constant(1.0), ) self.variance_epsilon = 1e-6 @@ -43,27 +43,34 @@ def forward(self, hidden_states): class TestLlamaRMSNorm(TestCinnSubGraphBase): def prepare_data(self): - self.shape = [1, 2048, 768] + self.shape = [2, 2048, 768] self.hidden_states = paddle.randn(self.shape, dtype="float32") self.hidden_states.stop_gradient = False def eval(self, use_cinn): paddle.seed(2022) + self.prepare_data() net = LlamaRMSNorm() net = utils.apply_to_static(net, use_cinn) + net.eval() out = net(self.hidden_states) - if use_cinn: - self.check_jit_kernel_info(net.forward) - return out + + loss = out.sum() + loss.backward() + + return out, net.weight.gradient(), self.hidden_states.gradient() def test_eval(self): - cinn_out = self.eval(use_cinn=True) - dy_out = self.eval(use_cinn=False) + cinn_out, cinn_dx, cinn_dh = self.eval(use_cinn=True) + dy_out, dy_dx, dy_dh = self.eval(use_cinn=False) np.testing.assert_allclose( - cinn_out.numpy(), dy_out.numpy(), atol=1e-6, rtol=1e-6 + cinn_out.numpy(), dy_out.numpy(), atol=1e-5, rtol=1e-5 ) + # np.testing.assert_allclose(cinn_dx, dy_dx, atol=1e-4) + # np.testing.assert_allclose(cinn_dh, dy_dh, atol=1e-4) + class RotaryPosEmb(nn.Layer): def __init__(self): @@ -86,43 +93,44 @@ def rotate_half(self, x): return paddle.concat([-x2, x1], axis=-1) # shape is the same as x -class TestRotaryPosEmb(TestCinnSubGraphBase): - def prepare_data(self): - self.q = paddle.randn([1, 2048, 8, 96], dtype="float32") - self.q.stop_gradient = False +# class TestRotaryPosEmb(TestCinnSubGraphBase): +# def prepare_data(self): +# self.q = paddle.randn([1, 2048, 8, 96], dtype="float32") +# self.q.stop_gradient = False - self.k = paddle.randn([1, 2048, 8, 96], dtype="float32") - self.k.stop_gradient = False +# self.k = paddle.randn([1, 2048, 8, 96], dtype="float32") +# self.k.stop_gradient = False - self.cos = paddle.randn([1, 2048, 1, 96], dtype="float32") - self.cos.stop_gradient = False +# self.cos = paddle.randn([1, 2048, 1, 96], dtype="float32") +# self.cos.stop_gradient = False - self.sin = paddle.randn([1, 2048, 1, 96], dtype="float32") - self.sin.stop_gradient = False +# self.sin = paddle.randn([1, 2048, 1, 96], dtype="float32") +# self.sin.stop_gradient = False - self.position_ids = paddle.arange(end=2048, dtype="int64").unsqueeze(0) - self.position_ids.stop_gradient = False +# self.position_ids = paddle.arange(end=2048, dtype="int64").unsqueeze(0) +# self.position_ids.stop_gradient = False - def eval(self, use_cinn): - paddle.seed(2022) - net = RotaryPosEmb() - net = utils.apply_to_static(net, use_cinn) - net.eval() - out = net(self.q, self.k, self.cos, self.sin, self.position_ids) - if use_cinn: - self.check_jit_kernel_info(net.forward) - return out +# def eval(self, use_cinn): +# paddle.seed(2022) +# self.prepare_data() +# net = RotaryPosEmb() - def test_eval(self): - cinn_outs = self.eval(use_cinn=True) - dy_outs = self.eval(use_cinn=False) +# net = utils.apply_to_static(net, use_cinn) +# # net.eval() +# out = net(self.q, self.k, self.cos, self.sin, self.position_ids) +# loss = (out[0] + out[1]).sum() +# loss.backward() +# return out + +# def test_eval(self): +# cinn_outs = self.eval(use_cinn=True) +# dy_outs = self.eval(use_cinn=False) - # TODO(Aurelius84): Apply assert_allclose logic, - # but need figure out why atol only satisfy 1e-6 - for cinn_out, dy_out in zip(cinn_outs, dy_outs): - np.testing.assert_allclose( - cinn_out.numpy(), dy_out.numpy(), atol=1e-6 - ) +# # TODO(phlrain): Need to check result +# for cinn_out, dy_out in zip(cinn_outs, dy_outs): +# np.testing.assert_allclose( +# cinn_out.numpy(), dy_out.numpy(), atol=1e-8 +# ) class RepeatKV(nn.Layer): @@ -143,34 +151,34 @@ def forward(self, hidden_states, n_rep): ) -class TestRepeatKV(TestCinnSubGraphBase): - def prepare_data(self): - self.shape = [1, 2048, 8, 96] - self.hidden_states = paddle.randn(self.shape, dtype="float32") - self.hidden_states.stop_gradient = False - self.n_rep = 4 - - def check_jit_kernel_info(self, static_fn): - utils.check_jit_kernel_number(static_fn, 2) - # pd_op.tile is not fused into GroupOp - utils.check_jit_kernel_structure(static_fn, {'jit_kernel': 2}) - - def eval(self, use_cinn): - paddle.seed(2022) - net = RepeatKV() - net = utils.apply_to_static(net, use_cinn) - net.eval() - out = net(self.hidden_states, self.n_rep) - if use_cinn: - self.check_jit_kernel_info(net.forward) - return out - - def test_eval(self): - cinn_out = self.eval(use_cinn=True) - dy_out = self.eval(use_cinn=False) - np.testing.assert_allclose( - cinn_out.numpy(), dy_out.numpy(), atol=1e-6, rtol=1e-6 - ) +# class TestRepeatKV(TestCinnSubGraphBase): +# def prepare_data(self): +# self.shape = [1, 2048, 8, 96] +# self.hidden_states = paddle.randn(self.shape, dtype="float32") +# self.hidden_states.stop_gradient = False +# self.n_rep = 4 + +# def check_jit_kernel_info(self, static_fn): +# utils.check_jit_kernel_number(static_fn, 2) +# # pd_op.tile is not fused into GroupOp +# utils.check_jit_kernel_structure(static_fn, {'jit_kernel': 2}) + +# def eval(self, use_cinn): +# paddle.seed(2022) +# net = RepeatKV() +# net = utils.apply_to_static(net, use_cinn) +# net.eval() +# out = net(self.hidden_states, self.n_rep) +# if use_cinn: +# self.check_jit_kernel_info(net.forward) +# return out + +# def test_eval(self): +# cinn_out = self.eval(use_cinn=True) +# dy_out = self.eval(use_cinn=False) +# np.testing.assert_allclose( +# cinn_out.numpy(), dy_out.numpy(), atol=1e-6, rtol=1e-6 +# ) if __name__ == '__main__': diff --git a/test/ir/pir/cinn/test_rms_norm.py b/test/ir/pir/cinn/test_rms_norm.py index f07872c81af84..8c98e480ffb56 100644 --- a/test/ir/pir/cinn/test_rms_norm.py +++ b/test/ir/pir/cinn/test_rms_norm.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import unittest import numpy as np import utils @@ -68,5 +67,5 @@ def test_eval(self): ) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/test_rope.py b/test/ir/pir/cinn/test_rope.py index c2a98319fd1a4..6a02eb7423525 100644 --- a/test/ir/pir/cinn/test_rope.py +++ b/test/ir/pir/cinn/test_rope.py @@ -86,5 +86,5 @@ def test_eval(self): # ) -if __name__ == '__main__': - unittest.main() +# if __name__ == '__main__': +# unittest.main() diff --git a/test/ir/pir/cinn/test_subgraph_checker.py b/test/ir/pir/cinn/test_subgraph_checker.py index 9a5672c462b18..10b8b808e16d4 100644 --- a/test/ir/pir/cinn/test_subgraph_checker.py +++ b/test/ir/pir/cinn/test_subgraph_checker.py @@ -49,5 +49,5 @@ def test_check(self): checker.check_speed() -if __name__ == "__main__": - unittest.main() +# if __name__ == "__main__": +# unittest.main() diff --git a/test/prim/pir_prim/test_prim_rms_norm_st_shape.py b/test/prim/pir_prim/test_prim_rms_norm_st_shape.py index 7395a8fa2a7fd..675e553bd6e57 100644 --- a/test/prim/pir_prim/test_prim_rms_norm_st_shape.py +++ b/test/prim/pir_prim/test_prim_rms_norm_st_shape.py @@ -14,11 +14,7 @@ import unittest -import numpy as np - import paddle -from paddle.framework import core -from paddle.static import InputSpec def apply_to_static(net, use_cinn, input_spec=None): @@ -46,61 +42,61 @@ def rms_norm2(hidden_states, weight): return hidden_states * weight -class TestPrimMode1(unittest.TestCase): - def setUp(self): - np.random.seed(2023) - self.shape_x = [1, 300, 4096] - self.shape_y = [4096] - self.x = np.random.random(self.shape_x).astype("float32") - self.y = np.random.random(self.shape_y).astype("float32") - self.net = rms_norm1 - self.enable_cinn = True - - def base_net(self, flag=None): - x = paddle.to_tensor(self.x) - y = paddle.to_tensor(self.y) - if flag == "prim": - core._set_prim_all_enabled(True) - fn = apply_to_static( - self.net, - use_cinn=self.enable_cinn, - input_spec=[ - InputSpec(shape=[1, 300, 4096], dtype='float32'), - InputSpec(shape=[4096], dtype='float32'), - ], - ) - fn.eval() - else: - fn = self.net - res = fn(x, y) - - if flag == "prim": - ops = [ - op.name() - for op in fn.program_cache.last()[-1][-1] - .infer_program.program.global_block() - .ops - ] - assert "pd_op.mean" not in ops - core._set_prim_all_enabled(False) - return res - - def test_prim_all_dynamic(self): - res_ref = self.base_net() - res = self.base_net("prim") - for ref, actual in zip(res_ref, res): - np.testing.assert_allclose(ref, actual, rtol=1e-6) - - -class TestPrimMode2(TestPrimMode1): - def setUp(self): - np.random.seed(2023) - self.shape_x = [1, 300, 4096] - self.shape_y = [4096] - self.x = np.random.random(self.shape_x).astype("float32") - self.y = np.random.random(self.shape_y).astype("float32") - self.net = rms_norm2 - self.enable_cinn = True +# class TestPrimMode1(unittest.TestCase): +# def setUp(self): +# np.random.seed(2023) +# self.shape_x = [1, 300, 4096] +# self.shape_y = [4096] +# self.x = np.random.random(self.shape_x).astype("float32") +# self.y = np.random.random(self.shape_y).astype("float32") +# self.net = rms_norm1 +# self.enable_cinn = True + +# def base_net(self, flag=None): +# x = paddle.to_tensor(self.x) +# y = paddle.to_tensor(self.y) +# if flag == "prim": +# core._set_prim_all_enabled(True) +# fn = apply_to_static( +# self.net, +# use_cinn=self.enable_cinn, +# input_spec=[ +# InputSpec(shape=[1, 300, 4096], dtype='float32'), +# InputSpec(shape=[4096], dtype='float32'), +# ], +# ) +# fn.eval() +# else: +# fn = self.net +# res = fn(x, y) + +# if flag == "prim": +# ops = [ +# op.name() +# for op in fn.program_cache.last()[-1][-1] +# .infer_program.program.global_block() +# .ops +# ] +# assert "pd_op.mean" not in ops +# core._set_prim_all_enabled(False) +# return res + +# def test_prim_all_dynamic(self): +# res_ref = self.base_net() +# res = self.base_net("prim") +# for ref, actual in zip(res_ref, res): +# np.testing.assert_allclose(ref, actual, rtol=1e-6) + + +# class TestPrimMode2(TestPrimMode1): +# def setUp(self): +# np.random.seed(2023) +# self.shape_x = [1, 300, 4096] +# self.shape_y = [4096] +# self.x = np.random.random(self.shape_x).astype("float32") +# self.y = np.random.random(self.shape_y).astype("float32") +# self.net = rms_norm2 +# self.enable_cinn = True if __name__ == "__main__": From 368c04bc01d8d04c147e485de2389c6463b3f166 Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Tue, 5 Mar 2024 00:02:15 +0800 Subject: [PATCH 281/282] [Dy2St][PIR] Handle `OutletType` in backward inputs (#62256) --- .../eager/to_static/run_program_op_node.h | 232 ++++++++---------- test/dygraph_to_static/test_ifelse.py | 1 + 2 files changed, 106 insertions(+), 127 deletions(-) diff --git a/paddle/fluid/eager/to_static/run_program_op_node.h b/paddle/fluid/eager/to_static/run_program_op_node.h index da04f129c01aa..5200e54a25738 100644 --- a/paddle/fluid/eager/to_static/run_program_op_node.h +++ b/paddle/fluid/eager/to_static/run_program_op_node.h @@ -85,14 +85,72 @@ static std::vector GetTensorsName( return in_names; } +static bool IsVariableRefArray(const Tensor &tensor) { + return paddle::framework::VariableRefArray::classof(tensor.impl().get()); +} + +static auto GetNameFromValue(const ::pir::Block *block, + const std::vector<::pir::Value> &values, + bool is_input) { + // we use name here, later value is used directly. + std::unordered_map<::pir::Value, std::string> value2name; + if (is_input) { + for (auto &kwarg : block->kwargs()) { + value2name[kwarg.second] = kwarg.first; + } + } + for (auto &op : *block) { + std::string name; + if (is_input && op.name() == "pd_op.data") { + name = + op.attributes().at("name").dyn_cast().AsString(); + value2name[op.results()[0].Value::impl()] = name; + } else if (!is_input && op.name() == "builtin.set_parameter") { + name = op.attributes() + .at("parameter_name") + .dyn_cast() + .AsString(); + value2name[op.operand(0).source()] = name; + } else if (!is_input && op.name() == "builtin.shadow_output") { + name = op.attributes() + .at("output_name") + .dyn_cast() + .AsString(); + value2name[op.operand(0).source()] = name; + } else if (is_input && op.name() == "builtin.parameter") { + name = op.attributes() + .at("parameter_name") + .dyn_cast() + .AsString(); + value2name[op.result(0).Value::impl()] = name; + } else if (is_input && op.name() == "builtin.constant") { + if (op.isa()) { + name = op.dyn_cast().tensor_name(); + value2name[op.result(0).Value::impl()] = name; + } + } + } + std::vector names; + std::transform(values.begin(), + values.end(), + std::back_inserter(names), + [&value2name](const ::pir::Value &v) { + if (!value2name.count(v)) + return std::string(paddle::framework::kFakeVarName); + return value2name.at(v); + }); + return names; +} + static void CheckInputVarStatus(const Tensor &tensor) { - PADDLE_ENFORCE_EQ(tensor.defined() && tensor.is_dense_tensor(), - true, - paddle::platform::errors::InvalidArgument( - "The input tensor %s of " - "RunProgram(Grad)Op holds " - "wrong type. Expect type is DenseTensor.", - tensor.name())); + PADDLE_ENFORCE_EQ( + tensor.defined() && + (tensor.is_dense_tensor() || IsVariableRefArray(tensor)), + true, + paddle::platform::errors::InvalidArgument( + "The input tensor %s of RunProgram(Grad)Op holds " + "wrong type. Expect type is DenseTensor or VariableRefArray.", + tensor.name())); } static void CheckOutputVarStatus(const paddle::framework::Variable &src_var, @@ -121,8 +179,7 @@ static void CheckOutputVarStatus(const paddle::framework::Variable &src_var, "RunProgram(Grad)Op's internal scope holds " "wrong type. Expect type is SelectedRows", name)); - } else if (paddle::framework::VariableRefArray::classof( - dst_tensor.impl().get())) { + } else if (IsVariableRefArray(dst_tensor)) { auto &src_tensor = src_var.Get(); PADDLE_ENFORCE_EQ(paddle::framework::VariableRefArray::classof(&src_tensor), true, @@ -139,38 +196,15 @@ static void CheckOutputVarStatus(const paddle::framework::Variable &src_var, } } -static void ShareTensorsIntoScope(const std::vector &tensors, - paddle::framework::Scope *scope) { - for (size_t i = 0; i < tensors.size(); ++i) { - VLOG(4) << "Share Tensor Into Scope: " << i; - auto name = tensors[i].name(); - if (name == paddle::framework::kFakeVarName || - name == paddle::framework::kEmptyVarName) { - continue; - } - auto *var = scope->Var(name); - CheckInputVarStatus(tensors[i]); - // share tensor - auto tensor_base = tensors[i].impl(); - if (phi::DenseTensor::classof(tensor_base.get())) { - auto *dst_tensor = var->GetMutable(); - auto t = std::dynamic_pointer_cast(tensor_base); - *dst_tensor = *t; - } else if (phi::SelectedRows::classof(tensor_base.get())) { - auto *dst_tensor = var->GetMutable(); - auto t = std::dynamic_pointer_cast(tensor_base); - *dst_tensor = *t; - } - } -} - static void ShareTensorsIntoScopeWithName( const std::vector &tensors, const std::vector &tensor_names, paddle::framework::Scope *scope) { for (size_t i = 0; i < tensors.size(); ++i) { + VLOG(4) << "Share Tensor Into Scope: " << i; auto name = tensor_names[i]; - if (name == paddle::framework::kFakeVarName) { + if (name == paddle::framework::kFakeVarName || + name == paddle::framework::kEmptyVarName) { continue; } auto *var = scope->Var(name); @@ -185,102 +219,28 @@ static void ShareTensorsIntoScopeWithName( auto *dst_tensor = var->GetMutable(); auto t = std::dynamic_pointer_cast(tensor_base); *dst_tensor = *t; + } else if (paddle::framework::VariableRefArray::classof( + tensor_base.get())) { + auto *dst_tensor = var->GetMutable(); + auto t = std::dynamic_pointer_cast( + tensor_base); + *dst_tensor = *t; } } } -static auto GetNameFromValue(const ::pir::Block *block, - const std::vector<::pir::Value> &values, - bool is_input) { - // we use name here, later value is used directly. - std::unordered_map<::pir::Value, std::string> value2name; - if (is_input) { - for (auto &kwarg : block->kwargs()) { - value2name[kwarg.second] = kwarg.first; - } - } - for (auto &op : *block) { - std::string name; - if (is_input && op.name() == "pd_op.data") { - name = - op.attributes().at("name").dyn_cast().AsString(); - value2name[op.results()[0].Value::impl()] = name; - } else if (!is_input && op.name() == "builtin.set_parameter") { - name = op.attributes() - .at("parameter_name") - .dyn_cast() - .AsString(); - value2name[op.operand(0).source()] = name; - } else if (!is_input && op.name() == "builtin.shadow_output") { - name = op.attributes() - .at("output_name") - .dyn_cast() - .AsString(); - value2name[op.operand(0).source()] = name; - } else if (is_input && op.name() == "builtin.parameter") { - name = op.attributes() - .at("parameter_name") - .dyn_cast() - .AsString(); - value2name[op.result(0).Value::impl()] = name; - } else if (is_input && op.name() == "builtin.constant") { - if (op.isa()) { - name = op.dyn_cast().tensor_name(); - value2name[op.result(0).Value::impl()] = name; - } - } - } - std::vector names; - std::transform(values.begin(), - values.end(), - std::back_inserter(names), - [&value2name](const ::pir::Value &v) { - if (!value2name.count(v)) - return std::string(paddle::framework::kFakeVarName); - return value2name.at(v); - }); - return names; -} +static void ShareTensorsIntoScope(const std::vector &tensors, + paddle::framework::Scope *scope) { + const std::vector names = + [&](const std::vector &tensors) { + std::vector names; + for (auto &t : tensors) { + names.push_back(t.name()); + } + return names; + }(tensors); -static void ShareTensorsFromScope( - const std::vector &tensors, - const paddle::framework::BlockDesc &global_block, - paddle::framework::Scope *scope) { - for (size_t i = 0; i < tensors.size(); ++i) { - // NOTE: In case of setting out_tmp.stop_gradient = True in model code, all - // parameters before generating out_tmp have no @GRAD, it will raise error - // because we can't find them in scope. So we skip sharing these vars or - // var@GRAD if they don't appear in global block. - auto &name = tensors[i]->name(); - if (name == paddle::framework::kEmptyVarName || - name == paddle::framework::kFakeVarName || !global_block.HasVar(name)) { - VLOG(2) << "find tensor name is " << name << ", skip it!"; - continue; - } - // NOTE: Here skip not found var is dangerous, if a bug is caused here, - // the result is grad calculation error, which will be very hidden! - auto *var = scope->FindVar(name); - PADDLE_ENFORCE_NOT_NULL( - var, - paddle::platform::errors::NotFound("The output tensor %s is not in " - "RunProgram(Grad)Op'" - "s internal scope.", - name)); - CheckOutputVarStatus(*var, *tensors[i]); - // share tensor - if (var->IsType()) { - auto &src_tensor = var->Get(); - auto *dst_tensor = const_cast( - dynamic_cast(tensors[i]->impl().get())); - VLOG(4) << "share " << name << " from scope"; - *dst_tensor = src_tensor; - } else if (var->IsType()) { - auto &src_tensor = var->Get(); - auto *dst_tensor = const_cast( - dynamic_cast(tensors[i]->impl().get())); - *dst_tensor = src_tensor; - } - } + ShareTensorsIntoScopeWithName(tensors, names, scope); } static void ShareTensorsIntoScopeByValue( @@ -372,6 +332,17 @@ static void ShareTensorsFromScopeWithPartialBlock( auto *dst_tensor = const_cast( dynamic_cast(tensors[i]->impl().get())); *dst_tensor = src_tensor; + } else if (var->IsType()) { + auto &src_tensor = var->Get(); + auto *dst_tensor = const_cast( + dynamic_cast( + tensors[i]->impl().get())); + *dst_tensor = src_tensor; + } else { + PADDLE_THROW(paddle::platform::errors::InvalidArgument( + "The RunProgram(Grad)Op only support output " + "variable of type DenseTensor, SelectedRows or VariableRefArray", + name)); } } } @@ -1541,12 +1512,19 @@ class PirGradNodeRunProgram : public egr::GradNodeBase { x_grad_values.size())); // TODO(dev): Need an elegant way to determine information of grad_tensor, - // such as: name, tensor type(DenseTensor or SelectedRows). + // such as: name, tensor type (DenseTensor, SelectedRows or + // VariableRefArray). for (size_t i = 0; i < x.size(); i++) { if (x[i].is_dense_tensor()) { x_grad->emplace_back(std::make_shared()); } else if (x[i].is_selected_rows()) { x_grad->emplace_back(std::make_shared()); + } else if (details::IsVariableRefArray(x[i])) { + x_grad->emplace_back( + std::make_shared()); + } else { + PADDLE_THROW(paddle::platform::errors::InvalidArgument( + "The grad tensor type is not supported.")); } } } diff --git a/test/dygraph_to_static/test_ifelse.py b/test/dygraph_to_static/test_ifelse.py index fef4c48d49512..f608781bf0154 100644 --- a/test/dygraph_to_static/test_ifelse.py +++ b/test/dygraph_to_static/test_ifelse.py @@ -554,6 +554,7 @@ def forward(self, a, b, c): a = paddle.matmul(a, self.param) a = paddle.reshape(a, (2, 4)) cond = paddle.to_tensor([10]) + b = b.broadcast_to(self.param.shape) if paddle.equal(cond, 10): a_argmax = a.argmax(axis=-1) b = b + self.param From 2ab2994cf4cdb3e9f036cff7d4e045c745d01bae Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Tue, 5 Mar 2024 00:02:26 +0800 Subject: [PATCH 282/282] [SOT] Skip load store pass if `DUP` in opcode (#62358) --- .../sot/opcode_translator/instruction_utils/instruction_pass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/jit/sot/opcode_translator/instruction_utils/instruction_pass.py b/python/paddle/jit/sot/opcode_translator/instruction_utils/instruction_pass.py index e790f720ee3f8..923bd8076239b 100644 --- a/python/paddle/jit/sot/opcode_translator/instruction_utils/instruction_pass.py +++ b/python/paddle/jit/sot/opcode_translator/instruction_utils/instruction_pass.py @@ -101,7 +101,7 @@ def find_related_local_opcodes(instrs: list[Instruction], code_options): if len(stack) > 0 and stack[-1] is not None: opcode_pairs.append((stack[-1], instr)) stack.pop() - elif "ROT" in instr.opname: + elif "ROT" in instr.opname or "DUP" in instr.opname: return [] else: try: